From 112175aecc260dac07b348b3da689b7e5158486d Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 23 Jul 2026 15:14:12 -0400 Subject: [PATCH 01/11] chore: update workflow versions --- .github/actions/setup-go/action.yml | 10 +++++----- .github/workflows/benchmark.yml | 18 ++++++++--------- .github/workflows/build_external.yml | 20 +++++++++---------- .../cre-settings-schema-reminder.yml | 2 +- .github/workflows/dependabump.yml | 6 +++--- .github/workflows/go-mod-cache.yml | 2 +- .github/workflows/golangci_lint.yml | 6 +++--- .github/workflows/keystore.yml | 8 ++++---- .github/workflows/observability.yml | 4 ++-- .github/workflows/pkg.yml | 10 +++++----- .github/workflows/release.yml | 4 ++-- .github/workflows/sonar-scan.yml | 2 +- .github/workflows/validate-protos-version.yml | 2 +- 13 files changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index aabc070544..10a45de3c1 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -50,7 +50,7 @@ runs: echo "version=$version" | tee -a "$GITHUB_OUTPUT" - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v7 with: go-version: ${{ steps.go-version.outputs.version }} cache: false @@ -98,7 +98,7 @@ runs: # By default, restore the cache only. # Let the `go-mod-cache.yml` workflow handle the creation of new module caches. - - uses: actions/cache/restore@v4 + - uses: actions/cache/restore@v6 if: ${{ inputs.restore-module-cache-only == 'true' }} name: Cache Go Modules (Restore) id: cache-modules-restore @@ -111,7 +111,7 @@ runs: # If this is called, then it will create the cache entry upon a cache miss. # The cache is created after a cache miss, and after job completes successfully. - - uses: actions/cache@v4 + - uses: actions/cache@v6 if: ${{ inputs.restore-module-cache-only != 'true' }} id: cache-modules name: Cache Go Modules @@ -171,7 +171,7 @@ runs: # --- # We restore the build cache if it exists, but we do not create a new cache entry. # The restore key matches the prefix of a the primary key for the build cache from trunk. - - uses: actions/cache/restore@v4 + - uses: actions/cache/restore@v6 name: Cache Go Build Outputs (restore) id: build-cache-restore if: ${{ inputs.only-modules == 'false' && steps.build-cache-setup.outputs.create-build-cache != 'true' }} @@ -188,7 +188,7 @@ runs: # as we're only supplying the primary key. # - If the cache does not exist, then nothing will be restored and it will create a new cache entry once # the job completes successfully. - - uses: actions/cache@v4 + - uses: actions/cache@v6 if: ${{ inputs.only-modules == 'false' && steps.build-cache-setup.outputs.create-build-cache == 'true'}} id: build-cache name: Cache Go Build Outputs diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4b23917d24..af08b02976 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -10,18 +10,18 @@ on: jobs: benchmark: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout Code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Run Go Benchmark Action uses: patrickhuie19/benchmark-action@b6d5104868690adeac692f7c860a715fefe604d4 with: - benchmarks-pr: 'BenchmarkKeystore_Sign' # Use benchmarks specified in PR, or default benchmarks - benchmarks-merge: 'BenchmarkKeystore_Sign' # Default list of benchmarks to run on merges - trigger-branches: 'main' # Comma-separated list of branches for merging - use-gh-pages: 'true' # Enable or disable GitHub Pages - github-token: ${{ secrets.GITHUB_TOKEN }} # Use the built-in GitHub token - gh-pages-branch: 'gh-pages' # GitHub Pages branch (if use-gh-pages is true) - alert-comment-cc-users: '@patrickhuie19' + benchmarks-pr: "BenchmarkKeystore_Sign" # Use benchmarks specified in PR, or default benchmarks + benchmarks-merge: "BenchmarkKeystore_Sign" # Default list of benchmarks to run on merges + trigger-branches: "main" # Comma-separated list of branches for merging + use-gh-pages: "true" # Enable or disable GitHub Pages + github-token: ${{ secrets.GITHUB_TOKEN }} # Use the built-in GitHub token + gh-pages-branch: "gh-pages" # GitHub Pages branch (if use-gh-pages is true) + alert-comment-cc-users: "@patrickhuie19" diff --git a/.github/workflows/build_external.yml b/.github/workflows/build_external.yml index 8d13b53e52..2dd21c2c27 100644 --- a/.github/workflows/build_external.yml +++ b/.github/workflows/build_external.yml @@ -21,12 +21,12 @@ jobs: solana-ref: ${{ steps.set-git-refs.outputs.solana-ref }} starknet-ref: ${{ steps.set-git-refs.outputs.starknet-ref }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: persist-credentials: false - name: Get refs from PR body id: set-git-refs - uses: actions/github-script@v7 + uses: actions/github-script@v9 with: script: | const script = require('./.github/scripts/get-refs-from-pr-body.js') @@ -46,14 +46,14 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout the chainlink-common repo - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: repository: smartcontractkit/chainlink-common persist-credentials: false path: chainlink-common - name: Checkout the chainlink core repo - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: repository: smartcontractkit/chainlink ref: ${{ needs.init.outputs.core-ref }} @@ -90,13 +90,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the solana repo - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: repository: smartcontractkit/chainlink-solana ref: ${{ needs.init.outputs.solana-ref }} - name: Setup Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + uses: actions/setup-go@v7 with: go-version-file: "go.mod" @@ -131,13 +131,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the starknet repo - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: repository: smartcontractkit/chainlink-starknet ref: ${{ needs.init.outputs.starknet-ref }} - name: Setup Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + uses: actions/setup-go@v7 with: go-version-file: ./relayer/go.mod @@ -148,7 +148,7 @@ jobs: go mod tidy - name: Install Nix - uses: cachix/install-nix-action@8887e596b4ee1134dae06b98d573bd674693f47c # v26 + uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 # v31.11.0 with: nix_path: nixpkgs=channel:nixos-unstable @@ -167,7 +167,7 @@ jobs: # runs-on: ubuntu-latest # steps: # - name: Checkout the terra repo - # uses: actions/checkout@v4 + # uses: actions/checkout@v7 # with: # repository: smartcontractkit/chainlink-terra # - name: Setup Go diff --git a/.github/workflows/cre-settings-schema-reminder.yml b/.github/workflows/cre-settings-schema-reminder.yml index 2fbb100dba..cee30c1f22 100644 --- a/.github/workflows/cre-settings-schema-reminder.yml +++ b/.github/workflows/cre-settings-schema-reminder.yml @@ -11,7 +11,7 @@ jobs: reminder: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/dependabump.yml b/.github/workflows/dependabump.yml index db4204e345..e386a052e7 100644 --- a/.github/workflows/dependabump.yml +++ b/.github/workflows/dependabump.yml @@ -3,9 +3,9 @@ name: dependabump on: workflow_dispatch: schedule: - - cron: '0 0 * * 1-5' # every week-day at midnight + - cron: "0 0 * * 1-5" # every week-day at midnight -permissions: { } +permissions: {} jobs: dependabump: @@ -20,7 +20,7 @@ jobs: GH_TOKEN: ${{ github.token }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: ref: main diff --git a/.github/workflows/go-mod-cache.yml b/.github/workflows/go-mod-cache.yml index a0f4231e79..8450ebb874 100644 --- a/.github/workflows/go-mod-cache.yml +++ b/.github/workflows/go-mod-cache.yml @@ -33,7 +33,7 @@ jobs: pull-requests: read steps: - name: Checkout the repo - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: persist-credentials: false diff --git a/.github/workflows/golangci_lint.yml b/.github/workflows/golangci_lint.yml index c4b637c291..870189f1bb 100644 --- a/.github/workflows/golangci_lint.yml +++ b/.github/workflows/golangci_lint.yml @@ -4,7 +4,7 @@ on: pull_request: merge_group: schedule: - - cron: '0 0 * * 1-5' # every week-day at midnight + - cron: "0 0 * * 1-5" # every week-day at midnight jobs: detect-modules: @@ -16,7 +16,7 @@ jobs: modules: ${{ steps.changed-modules.outputs.modules-json }} steps: - name: Checkout the repo - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: fetch-depth: 0 persist-credentials: false @@ -53,7 +53,7 @@ jobs: pull-requests: read steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: persist-credentials: false diff --git a/.github/workflows/keystore.yml b/.github/workflows/keystore.yml index 06f01638ed..9079359c80 100644 --- a/.github/workflows/keystore.yml +++ b/.github/workflows/keystore.yml @@ -12,7 +12,7 @@ jobs: keystore-src: ${{ steps.keystore-changes.outputs.src }} steps: - name: Checkout the repo - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: persist-credentials: false - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 @@ -33,7 +33,7 @@ jobs: working-directory: keystore steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Go uses: ./.github/actions/setup-go @@ -72,7 +72,7 @@ jobs: working-directory: keystore steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Go uses: ./.github/actions/setup-go @@ -96,7 +96,7 @@ jobs: - name: Upload Go test results if: always() - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + uses: actions/upload-artifact@v7 with: name: go-race-results path: | diff --git a/.github/workflows/observability.yml b/.github/workflows/observability.yml index 87f8dbbd85..4e6f415507 100644 --- a/.github/workflows/observability.yml +++ b/.github/workflows/observability.yml @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + uses: actions/setup-go@v7 with: go-version-file: "go.mod" diff --git a/.github/workflows/pkg.yml b/.github/workflows/pkg.yml index 769f359377..fb6b1ade33 100644 --- a/.github/workflows/pkg.yml +++ b/.github/workflows/pkg.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Go uses: ./.github/actions/setup-go with: @@ -56,7 +56,7 @@ jobs: - name: Upload Go test coverage if: always() - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + uses: actions/upload-artifact@v7 with: name: go-test-coverage path: | @@ -66,7 +66,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Go uses: ./.github/actions/setup-go @@ -90,7 +90,7 @@ jobs: - name: Upload Go test results if: always() - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + uses: actions/upload-artifact@v7 with: name: go-race-results path: | @@ -100,7 +100,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Go uses: ./.github/actions/setup-go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b43a731231..530d41eba3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -118,7 +118,7 @@ jobs: dry-run: ${{ inputs.dry-run }} steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: fetch-depth: 1 @@ -208,7 +208,7 @@ jobs: contents: write steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/sonar-scan.yml b/.github/workflows/sonar-scan.yml index cc3d18070b..7884048c4c 100644 --- a/.github/workflows/sonar-scan.yml +++ b/.github/workflows/sonar-scan.yml @@ -9,7 +9,7 @@ jobs: if: always() steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} diff --git a/.github/workflows/validate-protos-version.yml b/.github/workflows/validate-protos-version.yml index 0457dd1831..2262017ff0 100644 --- a/.github/workflows/validate-protos-version.yml +++ b/.github/workflows/validate-protos-version.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Validate protos version uses: smartcontractkit/.github/actions/validate-protos-version@e5a68c3ae58d138e4e7d1d2bbc7918e2b51d7161 #v1.0.0 From 9627d9d3d9e75460d1018ea97fd8fe9065fe3084 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 23 Jul 2026 15:18:21 -0400 Subject: [PATCH 02/11] chore: update go version --- .tool-versions | 2 +- keystore/go.mod | 2 +- observability-lib/go.mod | 2 +- pkg/chipingress/go.mod | 2 +- pkg/monitoring/go.mod | 2 +- pkg/values/go.mod | 2 +- pkg/workflows/artifacts/utils.go | 2 +- pkg/workflows/sdk/v2/pb/go.mod | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.tool-versions b/.tool-versions index 4a45865e29..27eb26aa81 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,4 +1,4 @@ -golang 1.26.2 +golang 1.26.5 protoc 29.3 protoc-gen-go-grpc 1.3.0 golangci-lint 2.12.2 diff --git a/keystore/go.mod b/keystore/go.mod index 53e020c535..5fb11c9304 100644 --- a/keystore/go.mod +++ b/keystore/go.mod @@ -1,6 +1,6 @@ module github.com/smartcontractkit/chainlink-common/keystore -go 1.26.2 +go 1.26.5 require ( github.com/NethermindEth/juno v0.15.11 diff --git a/observability-lib/go.mod b/observability-lib/go.mod index d3a0e26fe6..76202f05be 100644 --- a/observability-lib/go.mod +++ b/observability-lib/go.mod @@ -1,6 +1,6 @@ module github.com/smartcontractkit/chainlink-common/observability-lib -go 1.26.2 +go 1.26.5 require ( github.com/go-resty/resty/v2 v2.17.2 diff --git a/pkg/chipingress/go.mod b/pkg/chipingress/go.mod index 33248607d5..bf31b51092 100644 --- a/pkg/chipingress/go.mod +++ b/pkg/chipingress/go.mod @@ -1,6 +1,6 @@ module github.com/smartcontractkit/chainlink-common/pkg/chipingress -go 1.26.2 +go 1.26.5 require ( github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.16.1 diff --git a/pkg/monitoring/go.mod b/pkg/monitoring/go.mod index 9b90e77b07..7bc56daf58 100644 --- a/pkg/monitoring/go.mod +++ b/pkg/monitoring/go.mod @@ -1,6 +1,6 @@ module github.com/smartcontractkit/chainlink-common/pkg/monitoring -go 1.26.2 +go 1.26.5 require ( github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 diff --git a/pkg/values/go.mod b/pkg/values/go.mod index cea7ae4e2e..7830981c56 100644 --- a/pkg/values/go.mod +++ b/pkg/values/go.mod @@ -1,3 +1,3 @@ module github.com/smartcontractkit/chainlink-common/pkg/values -go 1.26.2 +go 1.26.5 diff --git a/pkg/workflows/artifacts/utils.go b/pkg/workflows/artifacts/utils.go index cd7fe97f2a..63b8c1df62 100644 --- a/pkg/workflows/artifacts/utils.go +++ b/pkg/workflows/artifacts/utils.go @@ -94,7 +94,7 @@ func GetBuildCmd(inputFile string, outputFile string, rootFolder string) *exec.C return buildCmd } -// goToolchain returns a GOTOOLCHAIN value (e.g. "go1.26.2") to pin the build +// goToolchain returns a GOTOOLCHAIN value (e.g. "go1.26.5") to pin the build // to. It prefers the configured `go env GOTOOLCHAIN`; when that is unset (e.g. // "auto") it falls back to the go version declared in the module's go.mod. The // local Go version is not used as a fallback because it would not pin a diff --git a/pkg/workflows/sdk/v2/pb/go.mod b/pkg/workflows/sdk/v2/pb/go.mod index 49623e0fb4..ef1462d3b3 100644 --- a/pkg/workflows/sdk/v2/pb/go.mod +++ b/pkg/workflows/sdk/v2/pb/go.mod @@ -1,3 +1,3 @@ module github.com/smartcontractkit/chainlink-common/pkg/workflows/sdk/v2/pb -go 1.26.2 +go 1.26.5 From b5f41cb3dde98282dbc24702c4005db07c874617 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 23 Jul 2026 15:30:42 -0400 Subject: [PATCH 03/11] Add local cache for loopinstall binaries --- cache-plugins-plan.md | 107 ++++++++++ pkg/loop/cmd/loopinstall/cache.go | 258 +++++++++++++++++++++++++ pkg/loop/cmd/loopinstall/cache_test.go | 223 +++++++++++++++++++++ pkg/loop/cmd/loopinstall/install.go | 4 +- pkg/loop/cmd/loopinstall/main.go | 4 +- pkg/loop/cmd/loopinstall/main_test.go | 22 +-- pkg/loop/cmd/loopinstall/models.go | 1 + 7 files changed, 605 insertions(+), 14 deletions(-) create mode 100644 cache-plugins-plan.md create mode 100644 pkg/loop/cmd/loopinstall/cache.go create mode 100644 pkg/loop/cmd/loopinstall/cache_test.go diff --git a/cache-plugins-plan.md b/cache-plugins-plan.md new file mode 100644 index 0000000000..8f8b2ff4c2 --- /dev/null +++ b/cache-plugins-plan.md @@ -0,0 +1,107 @@ +# Implementation Plan: `loopinstall` Local Disk & Release Asset Binary Caching + +## Goal +Optimize Chainlink Docker build times by enhancing [`loopinstall`](https://github.com/smartcontractkit/chainlink-common/tree/main/pkg/loop/cmd/loopinstall) in `chainlink-common` to cache and reuse pre-compiled plugin binaries locally on disk, with optional GitHub Release asset downloads. + +This prevents single-plugin `gitRef` updates in `plugins.public.yaml` from forcing full recompilation of all 22 remote plugins, dropping stage build time from **117s to ~10s** without relying on 3rd-party storage services. + +--- + +## User Review Required + +> [!IMPORTANT] +> - **No 3rd-Party Dependencies**: Relies strictly on local disk caching (`CL_LOOPINSTALL_CACHE_DIR`) and standard GitHub Releases (authenticated via existing `GIT_AUTH_TOKEN` / GATI). +> - **Zero Cache Invalidation Risk**: Cache keys include SHA256 of `(pluginType, moduleURI, gitRef, GOOS, GOARCH, goflags)`. Any commit/tag/flag change automatically creates a unique key and bypasses stale binaries. + +--- + +## Open Questions + +> [!NOTE] +> 1. Should GitHub Release asset downloading be enabled by default when `gitRef` is a release tag (e.g. `v1.3.1`), or strictly controlled via a flag `-enable-release-download` / `CL_LOOPINSTALL_DOWNLOAD_RELEASES=true`? + +Ans: It should be auto-allowed if any release/tag with a binary for that repo exists. + +--- + +## Proposed Changes + +### Component 1: `chainlink-common` (`pkg/loop/cmd/loopinstall`) + +#### [MODIFY] `models.go` +Add fields for cache directory and cache configuration: + +```go +type PluginDef struct { + Enabled *bool `yaml:"enabled,omitempty"` + ModuleURI string `yaml:"moduleURI"` + GitRef string `yaml:"gitRef"` + InstallPath string `yaml:"installPath"` + Libs []string `yaml:"libs"` + Flags string `yaml:"flags,omitempty"` + EnvVars []string `yaml:"envvars,omitempty"` + BinaryURL string `yaml:"binaryURL,omitempty"` // Optional explicit release asset URL override +} +``` + +#### [MODIFY] `main.go` +Add `-cache-dir` / `-c` CLI flag and `CL_LOOPINSTALL_CACHE_DIR` env var handling: + +```go +var cacheDir string +flag.StringVar(&cacheDir, "cache-dir", "", "Directory path for caching compiled plugin binaries") + +if cacheDir == "" { + cacheDir = os.Getenv("CL_LOOPINSTALL_CACHE_DIR") +} +``` + +#### [MODIFY] `install.go` +Implement 3-tier resolution logic in `downloadAndInstallPlugin`: +1. **Tier 1 (Local Disk Cache Hit)**: Check `${cacheDir}/${cacheKey}`. If present, copy directly to `$GOBIN/${binaryName}`. (Duration: **< 100ms**). +2. **Tier 2 (GitHub Release Asset Fallback)**: If `-enable-release-download` is set and `gitRef` is a tag, attempt HTTP GET using `GIT_AUTH_TOKEN`. If HTTP 200, write binary to cache directory and `$GOBIN`. (Duration: **~1s**). +3. **Tier 3 (Source Build Fallback)**: Execute existing `go mod download` + `go build` logic. Upon successful compilation, write binary to `${cacheDir}/${cacheKey}` for future runs. (Duration: **~8s**). + +```go +func computeCacheKey(pluginType string, plugin PluginDef, defaults DefaultsConfig, goos, goarch string) string { + h := sha256.New() + h.Write([]byte(fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s", + pluginType, plugin.ModuleURI, plugin.GitRef, plugin.Flags, defaults.GoFlags, goos, goarch))) + return fmt.Sprintf("%s-%x", filepath.Base(plugin.ModuleURI), h.Sum(nil)[:12]) +} +``` + +--- + +### Component 2: `chainlink` Repository (`core/chainlink.Dockerfile` & `plugins/chainlink.Dockerfile`) + +#### [MODIFY] [core/chainlink.Dockerfile](file:///Users/adamhamrick/Projects/chainlink/core/chainlink.Dockerfile) & [plugins/chainlink.Dockerfile](file:///Users/adamhamrick/Projects/chainlink/plugins/chainlink.Dockerfile) +Mount `/tmp/loopinstall-cache` into BuildKit during `build-remote-plugins` stage: + +```dockerfile +ENV CL_LOOPINSTALL_OUTPUT_DIR=/tmp/loopinstall-output \ + CL_LOOPINSTALL_CACHE_DIR=/tmp/loopinstall-cache \ + GIT_CONFIG_GLOBAL=/tmp/gitconfig-github-token + +RUN --mount=type=cache,target=/tmp/loopinstall-cache,id=loopinstall-binary-cache \ + --mount=type=secret,id=GIT_AUTH_TOKEN \ + set -e && \ + ... +``` + +--- + +## Verification Plan + +### Automated Tests +1. **Unit Tests in `chainlink-common`**: + - Run `go test ./pkg/loop/cmd/loopinstall/...` + - Test cache hit returns binary without invoking `execCommand("go", "build")`. + - Test cache miss compiles and writes output binary to `cacheDir`. + - Test invalidation when `gitRef` or `flags` change. + +### Manual Verification +1. Run `loopinstall` on `plugins.public.yaml` with `-cache-dir /tmp/test-cache`. +2. Verify all 22 binaries are built on run 1. +3. Update 1 plugin `gitRef` in `plugins.public.yaml` and run `loopinstall` again on run 2. +4. Verify 21 plugins hit cache (<1s) and only 1 plugin builds from source. diff --git a/pkg/loop/cmd/loopinstall/cache.go b/pkg/loop/cmd/loopinstall/cache.go new file mode 100644 index 0000000000..d0158a3204 --- /dev/null +++ b/pkg/loop/cmd/loopinstall/cache.go @@ -0,0 +1,258 @@ +package main + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "time" +) + +// resolveCacheDir resolves the cache directory location. +// Priority: +// 1) cliCacheDir parameter +// 2) CL_LOOPINSTALL_CACHE_DIR environment variable +// 3) User cache directory ($HOME/Library/Caches on macOS, $XDG_CACHE_HOME on Linux) or temp directory fallback +func resolveCacheDir(cliCacheDir string) (string, error) { + if cliCacheDir != "" { + return cliCacheDir, nil + } + if envCache := os.Getenv("CL_LOOPINSTALL_CACHE_DIR"); envCache != "" { + return envCache, nil + } + userCache, err := os.UserCacheDir() + if err != nil || userCache == "" { + return filepath.Join(os.TempDir(), "loopinstall-cache"), nil + } + return filepath.Join(userCache, "loopinstall-cache"), nil +} + +// computeCacheKey calculates a deterministic cache key for a plugin. +func computeCacheKey(pluginType string, plugin PluginDef, defaults DefaultsConfig, goos, goarch string) string { + h := sha256.New() + _, _ = fmt.Fprintf(h, "%s|%s|%s|%s|%s|%s|%s", + pluginType, plugin.ModuleURI, plugin.GitRef, plugin.Flags, defaults.GoFlags, goos, goarch) + base := filepath.Base(filepath.Clean(plugin.ModuleURI)) + return fmt.Sprintf("%s-%x", base, h.Sum(nil)[:12]) +} + +// getAuthToken checks GIT_AUTH_TOKEN first, falling back to GITHUB_TOKEN. +func getAuthToken() string { + if token := os.Getenv("GIT_AUTH_TOKEN"); token != "" { + return token + } + return os.Getenv("GITHUB_TOKEN") +} + +// githubReleaseAsset represents GitHub release JSON structure. +type githubReleaseAsset struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` +} + +type githubReleaseResponse struct { + Assets []githubReleaseAsset `json:"assets"` +} + +// tryDownloadGitHubRelease attempts to download a pre-compiled binary from GitHub Releases. +func tryDownloadGitHubRelease(plugin PluginDef, binaryName, goos, goarch, githubAPIBaseURL string) ([]byte, error) { + if githubAPIBaseURL == "" { + githubAPIBaseURL = "https://api.github.com" + } + token := getAuthToken() + + // Extract owner and repo from moduleURI (e.g. github.com/owner/repo) + parts := strings.Split(strings.TrimPrefix(plugin.ModuleURI, "github.com/"), "/") + if len(parts) < 2 { + return nil, fmt.Errorf("not a github repository: %s", plugin.ModuleURI) + } + owner, repo := parts[0], parts[1] + + ref := plugin.GitRef + if ref == "" { + return nil, fmt.Errorf("empty gitRef, release download skipped") + } + + url := fmt.Sprintf("%s/repos/%s/%s/releases/tags/%s", githubAPIBaseURL, owner, repo, ref) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + + client := &http.Client{Timeout: 15 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("release lookup HTTP %d", resp.StatusCode) + } + + var release githubReleaseResponse + if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + return nil, fmt.Errorf("failed to decode release response: %w", err) + } + + // Candidates for asset naming + expectedNames := map[string]bool{ + fmt.Sprintf("%s_%s_%s", binaryName, goos, goarch): true, + fmt.Sprintf("%s-%s-%s", binaryName, goos, goarch): true, + binaryName: true, + } + + var downloadURL string + for _, asset := range release.Assets { + if expectedNames[asset.Name] { + downloadURL = asset.BrowserDownloadURL + break + } + } + + if downloadURL == "" { + return nil, fmt.Errorf("no matching release asset found for binary %s (%s/%s)", binaryName, goos, goarch) + } + + // Download asset + dlReq, err := http.NewRequest("GET", downloadURL, nil) + if err != nil { + return nil, err + } + if token != "" && !strings.Contains(downloadURL, "githubusercontent.com") && !strings.Contains(downloadURL, "amazonaws.com") { + dlReq.Header.Set("Authorization", "Bearer "+token) + } + + dlResp, err := client.Do(dlReq) + if err != nil { + return nil, err + } + defer func() { _ = dlResp.Body.Close() }() + + if dlResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("asset download HTTP %d", dlResp.StatusCode) + } + + return io.ReadAll(dlResp.Body) +} + +// copyFile helper copies content from src file to dst file. +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return err + } + defer func() { _ = out.Close() }() + + _, err = io.Copy(out, in) + return err +} + +// downloadAndInstallPluginWithCache handles binary resolution across 3 tiers: +// Tier 1: Local Disk Cache +// Tier 2: GitHub Release Asset +// Tier 3: Source compilation fallback +func downloadAndInstallPluginWithCache(pluginType string, pluginIdx int, plugin PluginDef, defaults DefaultsConfig, cacheDir string, githubAPIBaseURL string) error { + pluginKey := fmt.Sprintf("%s[%d]", pluginType, pluginIdx) + + resolvedCacheDir, err := resolveCacheDir(cacheDir) + if err != nil { + return fmt.Errorf("%s - failed to resolve cache dir: %w", pluginKey, err) + } + + goos := os.Getenv("GOOS") + if goos == "" { + goos = runtime.GOOS + } + goarch := os.Getenv("GOARCH") + if goarch == "" { + goarch = runtime.GOARCH + } + + cacheKey := computeCacheKey(pluginType, plugin, defaults, goos, goarch) + cachedBinaryPath := filepath.Join(resolvedCacheDir, cacheKey) + + outputDir := os.Getenv("GOBIN") + if outputDir == "" { + gopath := os.Getenv("GOPATH") + if gopath == "" { + gopath = filepath.Join(os.Getenv("HOME"), "go") + } + outputDir = filepath.Join(gopath, "bin") + } + + // Compute binary name + isLocal := filepath.IsAbs(plugin.ModuleURI) || strings.HasPrefix(plugin.ModuleURI, "."+string(filepath.Separator)) + installArg := determineInstallArg(plugin.InstallPath, plugin.ModuleURI, isLocal) + binaryName := filepath.Base(installArg) + if binaryName == "." { + binaryName = filepath.Base(filepath.Clean(plugin.ModuleURI)) + } + outputPath := filepath.Join(outputDir, binaryName) + + // Tier 1: Check Local Disk Cache Hit + if _, err := os.Stat(cachedBinaryPath); err == nil { + log.Printf("%s - local disk cache hit (%s)", pluginKey, cacheKey) + if err := copyFile(cachedBinaryPath, outputPath); err != nil { + return fmt.Errorf("%s - failed to copy cached binary to target: %w", pluginKey, err) + } + return nil + } + + // Tier 2: Try GitHub Release Asset Fallback + if strings.HasPrefix(plugin.ModuleURI, "github.com/") && plugin.GitRef != "" { + log.Printf("%s - cache miss, attempting release asset download for %s@%s", pluginKey, plugin.ModuleURI, plugin.GitRef) + assetData, relErr := tryDownloadGitHubRelease(plugin, binaryName, goos, goarch, githubAPIBaseURL) + if relErr == nil { + log.Printf("%s - release asset downloaded successfully", pluginKey) + if err := os.MkdirAll(resolvedCacheDir, 0755); err != nil { + return fmt.Errorf("%s - failed to create cache dir: %w", pluginKey, err) + } + if err := os.WriteFile(cachedBinaryPath, assetData, 0755); err != nil { + return fmt.Errorf("%s - failed to write binary to cache: %w", pluginKey, err) + } + if err := copyFile(cachedBinaryPath, outputPath); err != nil { + return fmt.Errorf("%s - failed to copy binary to output: %w", pluginKey, err) + } + return nil + } + log.Printf("%s - release asset download not available (%v), falling back to source build", pluginKey, relErr) + } + + // Tier 3: Source Build Fallback + if err := downloadAndInstallPlugin(pluginType, pluginIdx, plugin, defaults); err != nil { + return err + } + + // Write compiled output to cache dir for future runs + if _, err := os.Stat(outputPath); err == nil { + if err := os.MkdirAll(resolvedCacheDir, 0755); err == nil { + if err := copyFile(outputPath, cachedBinaryPath); err != nil { + log.Printf("%s - warning: failed to write compiled binary to cache: %v", pluginKey, err) + } + } + } + + return nil +} diff --git a/pkg/loop/cmd/loopinstall/cache_test.go b/pkg/loop/cmd/loopinstall/cache_test.go new file mode 100644 index 0000000000..815993cdd0 --- /dev/null +++ b/pkg/loop/cmd/loopinstall/cache_test.go @@ -0,0 +1,223 @@ +package main + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestComputeCacheKey(t *testing.T) { + defaults := DefaultsConfig{GoFlags: "-tags=foo"} + plugin1 := PluginDef{ + ModuleURI: "github.com/smartcontractkit/chainlink-cosmos", + GitRef: "v1.0.0", + Flags: "-s -w", + } + plugin2 := PluginDef{ + ModuleURI: "github.com/smartcontractkit/chainlink-cosmos", + GitRef: "v1.0.1", // different gitRef + Flags: "-s -w", + } + + key1 := computeCacheKey("cosmos", plugin1, defaults, "linux", "amd64") + key2 := computeCacheKey("cosmos", plugin2, defaults, "linux", "amd64") + + assert.NotEqual(t, key1, key2, "expected different cache keys for different gitRef") + + // Verify key format: baseName-hash + expectedPrefix := "chainlink-cosmos-" + assert.True(t, strings.HasPrefix(key1, expectedPrefix), "expected key %q to start with prefix %q", key1, expectedPrefix) + + // Hash component check + h := sha256.New() + h.Write([]byte("cosmos|github.com/smartcontractkit/chainlink-cosmos|v1.0.0|-s -w|-tags=foo|linux|amd64")) + expectedKey := fmt.Sprintf("chainlink-cosmos-%x", h.Sum(nil)[:12]) + assert.Equal(t, expectedKey, key1) +} + +func TestResolveCacheDir(t *testing.T) { + tests := []struct { + name string + cliFlag string + envVar string + wantPrefix string + }{ + { + name: "cli_flag_takes_precedence", + cliFlag: "/tmp/cli-cache", + envVar: "/tmp/env-cache", + wantPrefix: "/tmp/cli-cache", + }, + { + name: "env_var_fallback", + cliFlag: "", + envVar: "/tmp/env-cache", + wantPrefix: "/tmp/env-cache", + }, + { + name: "default_system_cache", + cliFlag: "", + envVar: "", + wantPrefix: "loopinstall-cache", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("CL_LOOPINSTALL_CACHE_DIR", tc.envVar) + got, err := resolveCacheDir(tc.cliFlag) + require.NoError(t, err) + require.NotEmpty(t, got) + + if tc.cliFlag != "" { + assert.Equal(t, tc.cliFlag, got) + } else if tc.envVar != "" { + assert.Equal(t, tc.envVar, got) + } else { + assert.Contains(t, got, tc.wantPrefix) + } + }) + } +} + +func TestDownloadAndInstallPlugin_Tier1_LocalCacheHit(t *testing.T) { + tempCacheDir := t.TempDir() + tempBinDir := t.TempDir() + t.Setenv("GOBIN", tempBinDir) + + plugin := PluginDef{ + ModuleURI: "github.com/smartcontractkit/chainlink-solana", + GitRef: "v1.0.0", + InstallPath: ".", + } + defaults := DefaultsConfig{} + + cacheKey := computeCacheKey("solana", plugin, defaults, runtime.GOOS, runtime.GOARCH) + cachedBinaryPath := filepath.Join(tempCacheDir, cacheKey) + expectedContent := []byte("cached-binary-content") + err := os.WriteFile(cachedBinaryPath, expectedContent, 0755) + require.NoError(t, err) + + // Mock execCommand to fail if go build is called (should NOT be called on Tier 1 hit) + withMockExec(t, func(cmd *exec.Cmd) error { + require.Failf(t, "execCommand invoked", "execCommand should not be called on cache hit, command was: %v", cmd.Args) + return fmt.Errorf("should not be called") + }, func() { + err := downloadAndInstallPluginWithCache("solana", 0, plugin, defaults, tempCacheDir, "") + require.NoError(t, err) + + installedPath := filepath.Join(tempBinDir, "chainlink-solana") + gotContent, err := os.ReadFile(installedPath) + require.NoError(t, err) + assert.Equal(t, string(expectedContent), string(gotContent)) + }) +} + +func TestDownloadAndInstallPlugin_Tier2_GitHubReleaseAsset(t *testing.T) { + tempCacheDir := t.TempDir() + tempBinDir := t.TempDir() + t.Setenv("GOBIN", tempBinDir) + t.Setenv("GITHUB_TOKEN", "mock-token") + + // Set up mock HTTP server for GitHub release lookup & asset download + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "Bearer mock-token", r.Header.Get("Authorization")) + + if r.URL.Path == "/repos/smartcontractkit/chainlink-starknet/releases/tags/v1.2.0" { + assetName := fmt.Sprintf("chainlink-starknet_%s_%s", runtime.GOOS, runtime.GOARCH) + jsonResp := fmt.Sprintf(`{ + "assets": [ + { + "name": "%s", + "browser_download_url": "http://%s/download/%s" + } + ] + }`, assetName, r.Host, assetName) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(jsonResp)) + return + } + if filepath.Base(r.URL.Path) == fmt.Sprintf("chainlink-starknet_%s_%s", runtime.GOOS, runtime.GOARCH) { + _, _ = w.Write([]byte("release-asset-binary")) + return + } + http.NotFound(w, r) + })) + defer mockServer.Close() + + plugin := PluginDef{ + ModuleURI: "github.com/smartcontractkit/chainlink-starknet", + GitRef: "v1.2.0", + InstallPath: ".", + } + defaults := DefaultsConfig{} + + withMockExec(t, func(cmd *exec.Cmd) error { + require.Failf(t, "execCommand invoked", "execCommand should not be called on GitHub Release hit, command was: %v", cmd.Args) + return fmt.Errorf("should not be called") + }, func() { + err := downloadAndInstallPluginWithCache("starknet", 0, plugin, defaults, tempCacheDir, "http://"+mockServer.Listener.Addr().String()) + require.NoError(t, err) + + installedPath := filepath.Join(tempBinDir, "chainlink-starknet") + gotContent, err := os.ReadFile(installedPath) + require.NoError(t, err) + assert.Equal(t, "release-asset-binary", string(gotContent)) + + // Also check binary was saved to cache + cacheKey := computeCacheKey("starknet", plugin, defaults, runtime.GOOS, runtime.GOARCH) + cachedPath := filepath.Join(tempCacheDir, cacheKey) + cachedContent, err := os.ReadFile(cachedPath) + require.NoError(t, err) + assert.Equal(t, "release-asset-binary", string(cachedContent)) + }) +} + +func TestDownloadAndInstallPlugin_Tier3_SourceBuildAndCacheWrite(t *testing.T) { + tempCacheDir := t.TempDir() + tempBinDir := t.TempDir() + t.Setenv("GOBIN", tempBinDir) + + plugin := PluginDef{ + ModuleURI: "github.com/smartcontractkit/chainlink-common", + GitRef: "v2.0.0", + InstallPath: ".", + } + defaults := DefaultsConfig{} + + cacheKey := computeCacheKey("common", plugin, defaults, runtime.GOOS, runtime.GOARCH) + targetBinPath := filepath.Join(tempBinDir, "chainlink-common") + + withMockExec(t, func(cmd *exec.Cmd) error { + // Mock go mod download and go build + if len(cmd.Args) >= 3 && cmd.Args[1] == "mod" && cmd.Args[2] == "download" { + type dl struct{ Dir string } + _ = json.NewEncoder(cmd.Stdout).Encode(dl{Dir: t.TempDir()}) + return nil + } + if len(cmd.Args) >= 2 && cmd.Args[0] == "go" && cmd.Args[1] == "build" { + return os.WriteFile(targetBinPath, []byte("compiled-source-binary"), 0755) + } + return fmt.Errorf("unexpected command: %v", cmd.Args) + }, func() { + err := downloadAndInstallPluginWithCache("common", 0, plugin, defaults, tempCacheDir, "http://invalid-host") + require.NoError(t, err) + + // Verify binary copied to cache dir after build + cachedPath := filepath.Join(tempCacheDir, cacheKey) + cachedContent, err := os.ReadFile(cachedPath) + require.NoError(t, err) + assert.Equal(t, "compiled-source-binary", string(cachedContent)) + }) +} diff --git a/pkg/loop/cmd/loopinstall/install.go b/pkg/loop/cmd/loopinstall/install.go index 90999c23ed..4901daea29 100644 --- a/pkg/loop/cmd/loopinstall/install.go +++ b/pkg/loop/cmd/loopinstall/install.go @@ -365,7 +365,7 @@ func writeBuildManifest(tasks []PluginInstallTask, outputFile string) error { } // installPlugins installs plugins concurrently using a worker pool. -func installPlugins(tasks []PluginInstallTask, concurrency int, verbose bool, outputFile string) error { +func installPlugins(tasks []PluginInstallTask, concurrency int, verbose bool, outputFile string, cacheDir string) error { if len(tasks) == 0 { log.Println("No enabled plugins found to install") return nil @@ -395,7 +395,7 @@ func installPlugins(tasks []PluginInstallTask, concurrency int, verbose bool, ou start := time.Now() - err := downloadAndInstallPlugin(task.PluginType, 0, task.Plugin, task.Defaults) + err := downloadAndInstallPluginWithCache(task.PluginType, 0, task.Plugin, task.Defaults, cacheDir, "") duration := time.Since(start) if err != nil { diff --git a/pkg/loop/cmd/loopinstall/main.go b/pkg/loop/cmd/loopinstall/main.go index 4166fe6925..252d44727b 100644 --- a/pkg/loop/cmd/loopinstall/main.go +++ b/pkg/loop/cmd/loopinstall/main.go @@ -13,6 +13,7 @@ func main() { var concurrency int var sequential bool var outputFile string + var cacheDir string // Define flags flag.BoolVar(&showHelp, "help", false, "Show help") @@ -25,6 +26,7 @@ func main() { flag.BoolVar(&sequential, "s", false, "Install plugins sequentially (shorthand)") flag.StringVar(&outputFile, "output-installation-artifacts", "", "Path for installation artifacts JSON file (optional)") flag.StringVar(&outputFile, "o", "", "Path for installation artifacts JSON file (optional, shorthand)") + flag.StringVar(&cacheDir, "cache-dir", "", "Directory path for caching compiled plugin binaries (optional)") // Parse flags flag.Parse() @@ -85,7 +87,7 @@ func main() { } // Install all plugins - if err := installPlugins(allTasks, concurrency, verbose, outputFile); err != nil { + if err := installPlugins(allTasks, concurrency, verbose, outputFile, cacheDir); err != nil { os.Exit(1) } } diff --git a/pkg/loop/cmd/loopinstall/main_test.go b/pkg/loop/cmd/loopinstall/main_test.go index 61fca86736..e61b9b616e 100644 --- a/pkg/loop/cmd/loopinstall/main_test.go +++ b/pkg/loop/cmd/loopinstall/main_test.go @@ -96,7 +96,7 @@ func TestWriteBuildManifest(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(dir) + defer func() { _ = os.RemoveAll(dir) }() // Create tasks with and without libs tasks := []PluginInstallTask{ @@ -186,7 +186,7 @@ func TestWriteBuildManifest(t *testing.T) { } // For a plugin with empty libs, the libs field should be omitted or empty in the JSON - if testNoLibs.Libs != nil && len(testNoLibs.Libs) > 0 { + if len(testNoLibs.Libs) > 0 { t.Errorf("Expected empty libs, got %v", testNoLibs.Libs) } } @@ -200,7 +200,7 @@ func TestFileSpecificDefaults(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tempDir) + defer func() { _ = os.RemoveAll(tempDir) }() // Create first config file with complex ldflags file1Content := ` @@ -337,7 +337,7 @@ plugins: // Skip actual module download and installation since we're mocking // by not executing the real commands, but still call installPlugins // to test the file-specific defaults behavior - if err := installPlugins(allTasks, 1, true, outputFile); err != nil { + if err := installPlugins(allTasks, 1, true, outputFile, ""); err != nil { t.Fatalf("Failed to install plugins: %v", err) } @@ -528,7 +528,7 @@ func TestProcessConfigFile(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tempDir) + defer func() { _ = os.RemoveAll(tempDir) }() // Config with duplicate plugin definition duplicateContent := ` @@ -585,7 +585,7 @@ func TestDownloadAndInstallPlugin(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tempDir) + defer func() { _ = os.RemoveAll(tempDir) }() // Test cases tests := []struct { @@ -788,7 +788,7 @@ func TestFlags(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tempDir) + defer func() { _ = os.RemoveAll(tempDir) }() mockDownload := func(cmd *exec.Cmd) error { if stdout, ok := cmd.Stdout.(*bytes.Buffer); ok { @@ -862,7 +862,7 @@ func TestEnvVars(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tempDir) + defer func() { _ = os.RemoveAll(tempDir) }() mockDownload := func(cmd *exec.Cmd) error { if stdout, ok := cmd.Stdout.(*bytes.Buffer); ok { @@ -945,7 +945,7 @@ func TestEnvVars(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tempDir) + defer func() { _ = os.RemoveAll(tempDir) }() mockDownload := func(cmd *exec.Cmd) error { if stdout, ok := cmd.Stdout.(*bytes.Buffer); ok { @@ -1023,7 +1023,7 @@ func TestEnvVars(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tempDir) + defer func() { _ = os.RemoveAll(tempDir) }() mockDownload := func(cmd *exec.Cmd) error { if stdout, ok := cmd.Stdout.(*bytes.Buffer); ok { @@ -1108,7 +1108,7 @@ func TestEnvVars(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tempDir) + defer func() { _ = os.RemoveAll(tempDir) }() mockDownload := func(cmd *exec.Cmd) error { if stdout, ok := cmd.Stdout.(*bytes.Buffer); ok { diff --git a/pkg/loop/cmd/loopinstall/models.go b/pkg/loop/cmd/loopinstall/models.go index 18ac4a34f1..968b494fbb 100644 --- a/pkg/loop/cmd/loopinstall/models.go +++ b/pkg/loop/cmd/loopinstall/models.go @@ -21,6 +21,7 @@ type PluginDef struct { Libs []string `yaml:"libs"` Flags string `yaml:"flags,omitempty"` EnvVars []string `yaml:"envvars,omitempty"` + BinaryURL string `yaml:"binaryURL,omitempty"` // Optional explicit release asset URL override } // ModDownloadResult represents the JSON directory (dir) output from 'go mod download -json' From 50fa82309523e2ae7ca6b456da2b329241ff5004 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 23 Jul 2026 15:32:40 -0400 Subject: [PATCH 04/11] Cleanup --- cache-plugins-plan.md | 107 ------------------------------------------ 1 file changed, 107 deletions(-) delete mode 100644 cache-plugins-plan.md diff --git a/cache-plugins-plan.md b/cache-plugins-plan.md deleted file mode 100644 index 8f8b2ff4c2..0000000000 --- a/cache-plugins-plan.md +++ /dev/null @@ -1,107 +0,0 @@ -# Implementation Plan: `loopinstall` Local Disk & Release Asset Binary Caching - -## Goal -Optimize Chainlink Docker build times by enhancing [`loopinstall`](https://github.com/smartcontractkit/chainlink-common/tree/main/pkg/loop/cmd/loopinstall) in `chainlink-common` to cache and reuse pre-compiled plugin binaries locally on disk, with optional GitHub Release asset downloads. - -This prevents single-plugin `gitRef` updates in `plugins.public.yaml` from forcing full recompilation of all 22 remote plugins, dropping stage build time from **117s to ~10s** without relying on 3rd-party storage services. - ---- - -## User Review Required - -> [!IMPORTANT] -> - **No 3rd-Party Dependencies**: Relies strictly on local disk caching (`CL_LOOPINSTALL_CACHE_DIR`) and standard GitHub Releases (authenticated via existing `GIT_AUTH_TOKEN` / GATI). -> - **Zero Cache Invalidation Risk**: Cache keys include SHA256 of `(pluginType, moduleURI, gitRef, GOOS, GOARCH, goflags)`. Any commit/tag/flag change automatically creates a unique key and bypasses stale binaries. - ---- - -## Open Questions - -> [!NOTE] -> 1. Should GitHub Release asset downloading be enabled by default when `gitRef` is a release tag (e.g. `v1.3.1`), or strictly controlled via a flag `-enable-release-download` / `CL_LOOPINSTALL_DOWNLOAD_RELEASES=true`? - -Ans: It should be auto-allowed if any release/tag with a binary for that repo exists. - ---- - -## Proposed Changes - -### Component 1: `chainlink-common` (`pkg/loop/cmd/loopinstall`) - -#### [MODIFY] `models.go` -Add fields for cache directory and cache configuration: - -```go -type PluginDef struct { - Enabled *bool `yaml:"enabled,omitempty"` - ModuleURI string `yaml:"moduleURI"` - GitRef string `yaml:"gitRef"` - InstallPath string `yaml:"installPath"` - Libs []string `yaml:"libs"` - Flags string `yaml:"flags,omitempty"` - EnvVars []string `yaml:"envvars,omitempty"` - BinaryURL string `yaml:"binaryURL,omitempty"` // Optional explicit release asset URL override -} -``` - -#### [MODIFY] `main.go` -Add `-cache-dir` / `-c` CLI flag and `CL_LOOPINSTALL_CACHE_DIR` env var handling: - -```go -var cacheDir string -flag.StringVar(&cacheDir, "cache-dir", "", "Directory path for caching compiled plugin binaries") - -if cacheDir == "" { - cacheDir = os.Getenv("CL_LOOPINSTALL_CACHE_DIR") -} -``` - -#### [MODIFY] `install.go` -Implement 3-tier resolution logic in `downloadAndInstallPlugin`: -1. **Tier 1 (Local Disk Cache Hit)**: Check `${cacheDir}/${cacheKey}`. If present, copy directly to `$GOBIN/${binaryName}`. (Duration: **< 100ms**). -2. **Tier 2 (GitHub Release Asset Fallback)**: If `-enable-release-download` is set and `gitRef` is a tag, attempt HTTP GET using `GIT_AUTH_TOKEN`. If HTTP 200, write binary to cache directory and `$GOBIN`. (Duration: **~1s**). -3. **Tier 3 (Source Build Fallback)**: Execute existing `go mod download` + `go build` logic. Upon successful compilation, write binary to `${cacheDir}/${cacheKey}` for future runs. (Duration: **~8s**). - -```go -func computeCacheKey(pluginType string, plugin PluginDef, defaults DefaultsConfig, goos, goarch string) string { - h := sha256.New() - h.Write([]byte(fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s", - pluginType, plugin.ModuleURI, plugin.GitRef, plugin.Flags, defaults.GoFlags, goos, goarch))) - return fmt.Sprintf("%s-%x", filepath.Base(plugin.ModuleURI), h.Sum(nil)[:12]) -} -``` - ---- - -### Component 2: `chainlink` Repository (`core/chainlink.Dockerfile` & `plugins/chainlink.Dockerfile`) - -#### [MODIFY] [core/chainlink.Dockerfile](file:///Users/adamhamrick/Projects/chainlink/core/chainlink.Dockerfile) & [plugins/chainlink.Dockerfile](file:///Users/adamhamrick/Projects/chainlink/plugins/chainlink.Dockerfile) -Mount `/tmp/loopinstall-cache` into BuildKit during `build-remote-plugins` stage: - -```dockerfile -ENV CL_LOOPINSTALL_OUTPUT_DIR=/tmp/loopinstall-output \ - CL_LOOPINSTALL_CACHE_DIR=/tmp/loopinstall-cache \ - GIT_CONFIG_GLOBAL=/tmp/gitconfig-github-token - -RUN --mount=type=cache,target=/tmp/loopinstall-cache,id=loopinstall-binary-cache \ - --mount=type=secret,id=GIT_AUTH_TOKEN \ - set -e && \ - ... -``` - ---- - -## Verification Plan - -### Automated Tests -1. **Unit Tests in `chainlink-common`**: - - Run `go test ./pkg/loop/cmd/loopinstall/...` - - Test cache hit returns binary without invoking `execCommand("go", "build")`. - - Test cache miss compiles and writes output binary to `cacheDir`. - - Test invalidation when `gitRef` or `flags` change. - -### Manual Verification -1. Run `loopinstall` on `plugins.public.yaml` with `-cache-dir /tmp/test-cache`. -2. Verify all 22 binaries are built on run 1. -3. Update 1 plugin `gitRef` in `plugins.public.yaml` and run `loopinstall` again on run 2. -4. Verify 21 plugins hit cache (<1s) and only 1 plugin builds from source. From 77fd63d076dca1b75dcec92105a37fd964eee8a6 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 23 Jul 2026 15:40:54 -0400 Subject: [PATCH 05/11] Final go mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 40479c3317..3af714c950 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/smartcontractkit/chainlink-common -go 1.26.2 +go 1.26.5 require ( github.com/Masterminds/semver/v3 v3.4.0 From d59288af466448cbc4a30df2d978fb9fa658dd8d Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 23 Jul 2026 15:48:21 -0400 Subject: [PATCH 06/11] Comments --- pkg/loop/cmd/loopinstall/cache.go | 48 ++++++- pkg/loop/cmd/loopinstall/cache_test.go | 172 +++++++++++++++++++++++-- 2 files changed, 203 insertions(+), 17 deletions(-) diff --git a/pkg/loop/cmd/loopinstall/cache.go b/pkg/loop/cmd/loopinstall/cache.go index d0158a3204..6d0685b8b2 100644 --- a/pkg/loop/cmd/loopinstall/cache.go +++ b/pkg/loop/cmd/loopinstall/cache.go @@ -36,8 +36,8 @@ func resolveCacheDir(cliCacheDir string) (string, error) { // computeCacheKey calculates a deterministic cache key for a plugin. func computeCacheKey(pluginType string, plugin PluginDef, defaults DefaultsConfig, goos, goarch string) string { h := sha256.New() - _, _ = fmt.Fprintf(h, "%s|%s|%s|%s|%s|%s|%s", - pluginType, plugin.ModuleURI, plugin.GitRef, plugin.Flags, defaults.GoFlags, goos, goarch) + _, _ = fmt.Fprintf(h, "%s|%s|%s|%s|%s|%s|%s|%s|%s|%v|%v", + pluginType, plugin.ModuleURI, plugin.GitRef, plugin.InstallPath, plugin.Flags, defaults.GoFlags, plugin.BinaryURL, goos, goarch, plugin.EnvVars, defaults.EnvVars) base := filepath.Base(filepath.Clean(plugin.ModuleURI)) return fmt.Sprintf("%s-%x", base, h.Sum(nil)[:12]) } @@ -171,11 +171,19 @@ func copyFile(src, dst string) error { // downloadAndInstallPluginWithCache handles binary resolution across 3 tiers: // Tier 1: Local Disk Cache -// Tier 2: GitHub Release Asset +// Tier 2: GitHub Release Asset / BinaryURL // Tier 3: Source compilation fallback func downloadAndInstallPluginWithCache(pluginType string, pluginIdx int, plugin PluginDef, defaults DefaultsConfig, cacheDir string, githubAPIBaseURL string) error { pluginKey := fmt.Sprintf("%s[%d]", pluginType, pluginIdx) + if !isPluginEnabled(plugin) { + log.Printf("%s - skipping disabled plugin", pluginKey) + return nil + } + if err := plugin.Validate(); err != nil { + return fmt.Errorf("%s - plugin input validation failed: %w", pluginKey, err) + } + resolvedCacheDir, err := resolveCacheDir(cacheDir) if err != nil { return fmt.Errorf("%s - failed to resolve cache dir: %w", pluginKey, err) @@ -220,7 +228,39 @@ func downloadAndInstallPluginWithCache(pluginType string, pluginIdx int, plugin return nil } - // Tier 2: Try GitHub Release Asset Fallback + // Tier 2: Try explicit binary URL override, then GitHub Release Asset fallback + if plugin.BinaryURL != "" { + log.Printf("%s - cache miss, attempting binaryURL download (%s)", pluginKey, plugin.BinaryURL) + + client := &http.Client{Timeout: 15 * time.Second} + req, err := http.NewRequest("GET", plugin.BinaryURL, nil) + if err == nil { + if token := getAuthToken(); token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := client.Do(req) + if err == nil { + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusOK { + assetData, err := io.ReadAll(resp.Body) + if err == nil { + log.Printf("%s - binaryURL downloaded successfully", pluginKey) + if err := os.MkdirAll(resolvedCacheDir, 0755); err != nil { + return fmt.Errorf("%s - failed to create cache dir: %w", pluginKey, err) + } + if err := os.WriteFile(cachedBinaryPath, assetData, 0755); err != nil { + return fmt.Errorf("%s - failed to write binary to cache: %w", pluginKey, err) + } + if err := copyFile(cachedBinaryPath, outputPath); err != nil { + return fmt.Errorf("%s - failed to copy binary to output: %w", pluginKey, err) + } + return nil + } + } + } + } + } + if strings.HasPrefix(plugin.ModuleURI, "github.com/") && plugin.GitRef != "" { log.Printf("%s - cache miss, attempting release asset download for %s@%s", pluginKey, plugin.ModuleURI, plugin.GitRef) assetData, relErr := tryDownloadGitHubRelease(plugin, binaryName, goos, goarch, githubAPIBaseURL) diff --git a/pkg/loop/cmd/loopinstall/cache_test.go b/pkg/loop/cmd/loopinstall/cache_test.go index 815993cdd0..7ba6510d7d 100644 --- a/pkg/loop/cmd/loopinstall/cache_test.go +++ b/pkg/loop/cmd/loopinstall/cache_test.go @@ -1,7 +1,6 @@ package main import ( - "crypto/sha256" "encoding/json" "fmt" "net/http" @@ -18,16 +17,20 @@ import ( ) func TestComputeCacheKey(t *testing.T) { + t.Parallel() + defaults := DefaultsConfig{GoFlags: "-tags=foo"} plugin1 := PluginDef{ - ModuleURI: "github.com/smartcontractkit/chainlink-cosmos", - GitRef: "v1.0.0", - Flags: "-s -w", + ModuleURI: "github.com/smartcontractkit/chainlink-cosmos", + GitRef: "v1.0.0", + InstallPath: "cmd/cosmos", + Flags: "-s -w", } plugin2 := PluginDef{ - ModuleURI: "github.com/smartcontractkit/chainlink-cosmos", - GitRef: "v1.0.1", // different gitRef - Flags: "-s -w", + ModuleURI: "github.com/smartcontractkit/chainlink-cosmos", + GitRef: "v1.0.1", // different gitRef + InstallPath: "cmd/cosmos", + Flags: "-s -w", } key1 := computeCacheKey("cosmos", plugin1, defaults, "linux", "amd64") @@ -35,15 +38,158 @@ func TestComputeCacheKey(t *testing.T) { assert.NotEqual(t, key1, key2, "expected different cache keys for different gitRef") - // Verify key format: baseName-hash expectedPrefix := "chainlink-cosmos-" assert.True(t, strings.HasPrefix(key1, expectedPrefix), "expected key %q to start with prefix %q", key1, expectedPrefix) +} + +func TestComputeCacheKey_Variations(t *testing.T) { + t.Parallel() + + basePlugin := PluginDef{ + ModuleURI: "github.com/smartcontractkit/chainlink-cosmos", + GitRef: "v1.0.0", + InstallPath: ".", + Flags: "-s -w", + EnvVars: []string{"CGO_ENABLED=0"}, + BinaryURL: "https://example.com/binary", + } + baseDefaults := DefaultsConfig{GoFlags: "-tags=foo", EnvVars: []string{"FOO=bar"}} + baseKey := computeCacheKey("cosmos", basePlugin, baseDefaults, "linux", "amd64") + + tests := []struct { + name string + modify func(p *PluginDef, d *DefaultsConfig) + different bool + }{ + { + name: "different_install_path", + modify: func(p *PluginDef, d *DefaultsConfig) { + p.InstallPath = "cmd/plugin" + }, + different: true, + }, + { + name: "different_binary_url", + modify: func(p *PluginDef, d *DefaultsConfig) { + p.BinaryURL = "https://example.com/other-binary" + }, + different: true, + }, + { + name: "different_plugin_envvars", + modify: func(p *PluginDef, d *DefaultsConfig) { + p.EnvVars = []string{"CGO_ENABLED=1"} + }, + different: true, + }, + { + name: "different_defaults_envvars", + modify: func(p *PluginDef, d *DefaultsConfig) { + d.EnvVars = []string{"FOO=baz"} + }, + different: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + p := basePlugin + d := baseDefaults + tc.modify(&p, &d) + key := computeCacheKey("cosmos", p, d, "linux", "amd64") + if tc.different { + assert.NotEqual(t, baseKey, key) + } else { + assert.Equal(t, baseKey, key) + } + }) + } +} + +func TestDownloadAndInstallPlugin_DisabledAndValidation(t *testing.T) { + tempCacheDir := t.TempDir() + tempBinDir := t.TempDir() + t.Setenv("GOBIN", tempBinDir) - // Hash component check - h := sha256.New() - h.Write([]byte("cosmos|github.com/smartcontractkit/chainlink-cosmos|v1.0.0|-s -w|-tags=foo|linux|amd64")) - expectedKey := fmt.Sprintf("chainlink-cosmos-%x", h.Sum(nil)[:12]) - assert.Equal(t, expectedKey, key1) + falseVal := false + tests := []struct { + name string + plugin PluginDef + wantErr bool + errContains string + wasSkipped bool + }{ + { + name: "disabled_plugin_skipped", + plugin: PluginDef{ + ModuleURI: "github.com/smartcontractkit/chainlink-cosmos", + Enabled: &falseVal, + }, + wantErr: false, + wasSkipped: true, + }, + { + name: "invalid_plugin_validation_failed", + plugin: PluginDef{ + ModuleURI: "", // empty module URI fails validation + }, + wantErr: true, + errContains: "plugin input validation failed", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + withMockExec(t, func(cmd *exec.Cmd) error { + require.Failf(t, "execCommand invoked", "execCommand should not be called for disabled or invalid plugin") + return fmt.Errorf("should not be called") + }, func() { + err := downloadAndInstallPluginWithCache("cosmos", 0, tc.plugin, DefaultsConfig{}, tempCacheDir, "") + if tc.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errContains) + } else { + require.NoError(t, err) + } + }) + }) + } +} + +func TestDownloadAndInstallPlugin_Tier2_BinaryURL(t *testing.T) { + tempCacheDir := t.TempDir() + tempBinDir := t.TempDir() + t.Setenv("GOBIN", tempBinDir) + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/custom-binary" { + _, _ = w.Write([]byte("custom-url-binary")) + return + } + http.NotFound(w, r) + })) + defer mockServer.Close() + + plugin := PluginDef{ + ModuleURI: "github.com/smartcontractkit/chainlink-custom", + GitRef: "v1.0.0", + InstallPath: ".", + BinaryURL: mockServer.URL + "/custom-binary", + } + defaults := DefaultsConfig{} + + withMockExec(t, func(cmd *exec.Cmd) error { + require.Failf(t, "execCommand invoked", "execCommand should not be called when BinaryURL succeeds") + return fmt.Errorf("should not be called") + }, func() { + err := downloadAndInstallPluginWithCache("custom", 0, plugin, defaults, tempCacheDir, "") + require.NoError(t, err) + + installedPath := filepath.Join(tempBinDir, "chainlink-custom") + gotContent, err := os.ReadFile(installedPath) + require.NoError(t, err) + assert.Equal(t, "custom-url-binary", string(gotContent)) + }) } func TestResolveCacheDir(t *testing.T) { From d9877d2f8d742a8c0333163b0213fd421823328d Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 23 Jul 2026 15:53:58 -0400 Subject: [PATCH 07/11] Windows --- pkg/loop/cmd/loopinstall/cache.go | 11 +++++- pkg/loop/cmd/loopinstall/cache_test.go | 49 ++++++++++++++++++++++++++ pkg/loop/cmd/loopinstall/main.go | 2 +- 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/pkg/loop/cmd/loopinstall/cache.go b/pkg/loop/cmd/loopinstall/cache.go index 6d0685b8b2..b003048251 100644 --- a/pkg/loop/cmd/loopinstall/cache.go +++ b/pkg/loop/cmd/loopinstall/cache.go @@ -112,6 +112,11 @@ func tryDownloadGitHubRelease(plugin PluginDef, binaryName, goos, goarch, github fmt.Sprintf("%s-%s-%s", binaryName, goos, goarch): true, binaryName: true, } + if goos == "windows" && !strings.HasSuffix(binaryName, ".exe") { + expectedNames[binaryName+".exe"] = true + expectedNames[fmt.Sprintf("%s_%s_%s.exe", binaryName, goos, goarch)] = true + expectedNames[fmt.Sprintf("%s-%s-%s.exe", binaryName, goos, goarch)] = true + } var downloadURL string for _, asset := range release.Assets { @@ -217,7 +222,11 @@ func downloadAndInstallPluginWithCache(pluginType string, pluginIdx int, plugin if binaryName == "." { binaryName = filepath.Base(filepath.Clean(plugin.ModuleURI)) } - outputPath := filepath.Join(outputDir, binaryName) + outputBinaryName := binaryName + if goos == "windows" && !strings.HasSuffix(outputBinaryName, ".exe") { + outputBinaryName += ".exe" + } + outputPath := filepath.Join(outputDir, outputBinaryName) // Tier 1: Check Local Disk Cache Hit if _, err := os.Stat(cachedBinaryPath); err == nil { diff --git a/pkg/loop/cmd/loopinstall/cache_test.go b/pkg/loop/cmd/loopinstall/cache_test.go index 7ba6510d7d..874711941d 100644 --- a/pkg/loop/cmd/loopinstall/cache_test.go +++ b/pkg/loop/cmd/loopinstall/cache_test.go @@ -367,3 +367,52 @@ func TestDownloadAndInstallPlugin_Tier3_SourceBuildAndCacheWrite(t *testing.T) { assert.Equal(t, "compiled-source-binary", string(cachedContent)) }) } + +func TestDownloadAndInstallPlugin_Tier2_WindowsExe(t *testing.T) { + tempCacheDir := t.TempDir() + tempBinDir := t.TempDir() + t.Setenv("GOBIN", tempBinDir) + t.Setenv("GOOS", "windows") + t.Setenv("GOARCH", "amd64") + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/repos/smartcontractkit/chainlink-win/releases/tags/v1.0.0" { + jsonResp := `{ + "assets": [ + { + "name": "chainlink-win_windows_amd64.exe", + "browser_download_url": "http://` + r.Host + `/download/chainlink-win.exe" + } + ] + }` + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(jsonResp)) + return + } + if r.URL.Path == "/download/chainlink-win.exe" { + _, _ = w.Write([]byte("windows-exe-binary")) + return + } + http.NotFound(w, r) + })) + defer mockServer.Close() + + plugin := PluginDef{ + ModuleURI: "github.com/smartcontractkit/chainlink-win", + GitRef: "v1.0.0", + InstallPath: ".", + } + + withMockExec(t, func(cmd *exec.Cmd) error { + require.Failf(t, "execCommand invoked", "execCommand should not be called on GitHub Release hit") + return fmt.Errorf("should not be called") + }, func() { + err := downloadAndInstallPluginWithCache("win", 0, plugin, DefaultsConfig{}, tempCacheDir, "http://"+mockServer.Listener.Addr().String()) + require.NoError(t, err) + + installedPath := filepath.Join(tempBinDir, "chainlink-win.exe") + gotContent, err := os.ReadFile(installedPath) + require.NoError(t, err) + assert.Equal(t, "windows-exe-binary", string(gotContent)) + }) +} diff --git a/pkg/loop/cmd/loopinstall/main.go b/pkg/loop/cmd/loopinstall/main.go index 252d44727b..1c4f3b80fc 100644 --- a/pkg/loop/cmd/loopinstall/main.go +++ b/pkg/loop/cmd/loopinstall/main.go @@ -26,7 +26,7 @@ func main() { flag.BoolVar(&sequential, "s", false, "Install plugins sequentially (shorthand)") flag.StringVar(&outputFile, "output-installation-artifacts", "", "Path for installation artifacts JSON file (optional)") flag.StringVar(&outputFile, "o", "", "Path for installation artifacts JSON file (optional, shorthand)") - flag.StringVar(&cacheDir, "cache-dir", "", "Directory path for caching compiled plugin binaries (optional)") + flag.StringVar(&cacheDir, "cache-dir", "", "Directory path for caching compiled plugin binaries (optional; defaults to $CL_LOOPINSTALL_CACHE_DIR, else os.UserCacheDir())") // Parse flags flag.Parse() From 156cc896468bcccbe5b82493843ecd7f7096d6d8 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 23 Jul 2026 15:56:05 -0400 Subject: [PATCH 08/11] Security --- pkg/loop/cmd/loopinstall/cache.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/loop/cmd/loopinstall/cache.go b/pkg/loop/cmd/loopinstall/cache.go index b003048251..30011b13c1 100644 --- a/pkg/loop/cmd/loopinstall/cache.go +++ b/pkg/loop/cmd/loopinstall/cache.go @@ -88,6 +88,7 @@ func tryDownloadGitHubRelease(plugin PluginDef, binaryName, goos, goarch, github if token != "" { req.Header.Set("Authorization", "Bearer "+token) } + req.Header.Set("User-Agent", "chainlink-common-loopinstall") req.Header.Set("Accept", "application/vnd.github.v3+json") client := &http.Client{Timeout: 15 * time.Second} @@ -245,7 +246,9 @@ func downloadAndInstallPluginWithCache(pluginType string, pluginIdx int, plugin req, err := http.NewRequest("GET", plugin.BinaryURL, nil) if err == nil { if token := getAuthToken(); token != "" { - req.Header.Set("Authorization", "Bearer "+token) + if strings.Contains(plugin.BinaryURL, "github.com") || strings.Contains(plugin.BinaryURL, "githubusercontent.com") || strings.Contains(plugin.BinaryURL, "amazonaws.com") { + req.Header.Set("Authorization", "Bearer "+token) + } } resp, err := client.Do(req) if err == nil { From b934e342f0aaba9e3983d4f9c48a48fa9a7f16ae Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 23 Jul 2026 16:16:21 -0400 Subject: [PATCH 09/11] Fix tests --- pkg/workflows/artifacts/artifacts_test.go | 10 +++++----- pkg/workflows/wasm/host/module.go | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/workflows/artifacts/artifacts_test.go b/pkg/workflows/artifacts/artifacts_test.go index 6c14d6b71c..76f8576f4b 100644 --- a/pkg/workflows/artifacts/artifacts_test.go +++ b/pkg/workflows/artifacts/artifacts_test.go @@ -48,7 +48,7 @@ func (s *ArtifactsTestSuite) TestArtifacts() { // Compare the keccak256 hash of the binary data against a value produced by // the pinned Go toolchain (see GetBuildCmd in utils.go). Because Compile sets // GOTOOLCHAIN from the nearest go.mod, this hash is stable across machines. - expKeccak256Hash, err := hex.DecodeString("a057a58ff8212122016515b2922b7c3893525f7f5afe95c8442e0cd629d68420") + expKeccak256Hash, err := hex.DecodeString("a3f5ea60cfd4527c2ec076f3a923ba9c4f577558d65df49467e259c9e5420f24") s.NoError(err, "failed to decode expected keccak256 hash") keccak256FromSha3Lib := sha3.NewLegacyKeccak256() keccak256FromSha3Lib.Write(b64EncodedBinaryData) @@ -59,15 +59,15 @@ func (s *ArtifactsTestSuite) TestArtifacts() { s.NoError(err, "failed to prepare artifacts") base64EncodedBinaryData := artifacts.GetBinaryData() - s.Len(base64EncodedBinaryData, 636684, "binary data size should match the pinned toolchain output") - s.Equal("m1upG3s6AJQvOA8AAK295+EaARsHAADf/YcBgFURwPQAANDq5wFQVVVVVVVVVVVV3ZMQEI7ZtgMAAKqq", + s.Len(base64EncodedBinaryData, 636840, "binary data size should match the pinned toolchain output") + s.Equal("m+qtG5syABJYnQAAWHW/95AQATYOAAC+fg9vTQEBTA8AANV+HgBVVVVVVVVVVVVV3ZQQEI7ZtgMAgKqq", string(base64EncodedBinaryData[0:80])) - s.Equal("gUEoFNoVRfyHGTsZmdg7wCJbGVibOhmYmsDAytgg92FTTmiddpI/x8SYzdANBkPGhtLoj/Hn7jvK26YE", + s.Equal("hIAEjIDBYIsW8d9MXI1NnZwh4W0M7cxdDc1NISCl7eFN4ZQ3JNuf5C+mJsxGnhAo0vZkxrYmrFZAykNM", string(base64EncodedBinaryData[len(base64EncodedBinaryData)-80:])) s.Equal("myContract: 0x44DD9D24349965E5e20E3D6118F560BCd64828E9\nchainID: 11155111", string(artifacts.GetConfigData())) - s.Equal("004789eee3f5eee474ef64fe8b9251086083ad14af7b9135c9f33b661a128b3e", artifacts.GetWorkflowID()) + s.Equal("00313a2a7d640a071ff0bd92e4fdc42cdf7c5437662b13d66c081949f959a5f2", artifacts.GetWorkflowID()) } func (s *ArtifactsTestSuite) TestArtifactsSadPaths() { diff --git a/pkg/workflows/wasm/host/module.go b/pkg/workflows/wasm/host/module.go index 0cd4798a42..da80ba6708 100644 --- a/pkg/workflows/wasm/host/module.go +++ b/pkg/workflows/wasm/host/module.go @@ -706,6 +706,9 @@ func runWasm[I, O proto.Message]( instance, err := linkWasm(m, store, exec) if err != nil { + if ctx.Err() != nil || strings.Contains(err.Error(), "wasm trap: interrupt") { + return o, context.DeadlineExceeded + } return o, fmt.Errorf("error linking wasm: %w", err) } @@ -744,7 +747,7 @@ func runWasm[I, O proto.Message]( // Note - there is no other reliable signal on the error that can be used to infer it is due to epoch deadline // being reached, so if an error is returned after the deadline it is assumed it is due to that and return // context.DeadlineExceeded. - if err != nil && ((executionDuration >= maxTimeout-m.cfg.TickInterval) || ctx.Err() != nil) { // As start could be called just before epoch update 1 tick interval is deducted to account for this + if err != nil && ((executionDuration >= maxTimeout-m.cfg.TickInterval) || ctx.Err() != nil || strings.Contains(err.Error(), "wasm trap: interrupt")) { // As start could be called just before epoch update 1 tick interval is deducted to account for this m.cfg.Logger.Errorw("start function returned error after deadline reached, returning deadline exceeded error", "errFromStartFunction", err) return o, context.DeadlineExceeded } From 9fdba7645e2ef6472540bf1cab1fc43d3e7d682c Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 23 Jul 2026 16:24:05 -0400 Subject: [PATCH 10/11] Comments --- pkg/loop/cmd/loopinstall/cache.go | 32 ++++++++++++++++++-------- pkg/loop/cmd/loopinstall/cache_test.go | 8 +++++-- pkg/loop/cmd/loopinstall/main.go | 7 ++++-- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/pkg/loop/cmd/loopinstall/cache.go b/pkg/loop/cmd/loopinstall/cache.go index 30011b13c1..0a491d8be7 100644 --- a/pkg/loop/cmd/loopinstall/cache.go +++ b/pkg/loop/cmd/loopinstall/cache.go @@ -52,6 +52,7 @@ func getAuthToken() string { // githubReleaseAsset represents GitHub release JSON structure. type githubReleaseAsset struct { + ID int64 `json:"id"` Name string `json:"name"` BrowserDownloadURL string `json:"browser_download_url"` } @@ -119,26 +120,29 @@ func tryDownloadGitHubRelease(plugin PluginDef, binaryName, goos, goarch, github expectedNames[fmt.Sprintf("%s-%s-%s.exe", binaryName, goos, goarch)] = true } - var downloadURL string + var assetID int64 for _, asset := range release.Assets { if expectedNames[asset.Name] { - downloadURL = asset.BrowserDownloadURL + assetID = asset.ID break } } - if downloadURL == "" { + if assetID == 0 { return nil, fmt.Errorf("no matching release asset found for binary %s (%s/%s)", binaryName, goos, goarch) } - // Download asset - dlReq, err := http.NewRequest("GET", downloadURL, nil) + // Download asset via GitHub API assets endpoint to preserve auth on redirects + dlURL := fmt.Sprintf("%s/repos/%s/%s/releases/assets/%d", githubAPIBaseURL, owner, repo, assetID) + dlReq, err := http.NewRequest("GET", dlURL, nil) if err != nil { return nil, err } - if token != "" && !strings.Contains(downloadURL, "githubusercontent.com") && !strings.Contains(downloadURL, "amazonaws.com") { + if token != "" { dlReq.Header.Set("Authorization", "Bearer "+token) } + dlReq.Header.Set("User-Agent", "chainlink-common-loopinstall") + dlReq.Header.Set("Accept", "application/octet-stream") dlResp, err := client.Do(dlReq) if err != nil { @@ -244,18 +248,26 @@ func downloadAndInstallPluginWithCache(pluginType string, pluginIdx int, plugin client := &http.Client{Timeout: 15 * time.Second} req, err := http.NewRequest("GET", plugin.BinaryURL, nil) - if err == nil { + if err != nil { + log.Printf("%s - binaryURL request creation failed (%v)", pluginKey, err) + } else { if token := getAuthToken(); token != "" { if strings.Contains(plugin.BinaryURL, "github.com") || strings.Contains(plugin.BinaryURL, "githubusercontent.com") || strings.Contains(plugin.BinaryURL, "amazonaws.com") { req.Header.Set("Authorization", "Bearer "+token) } } resp, err := client.Do(req) - if err == nil { + if err != nil { + log.Printf("%s - binaryURL request failed (%v)", pluginKey, err) + } else { defer func() { _ = resp.Body.Close() }() - if resp.StatusCode == http.StatusOK { + if resp.StatusCode != http.StatusOK { + log.Printf("%s - binaryURL returned HTTP %d", pluginKey, resp.StatusCode) + } else { assetData, err := io.ReadAll(resp.Body) - if err == nil { + if err != nil { + log.Printf("%s - failed to read binaryURL response body (%v)", pluginKey, err) + } else { log.Printf("%s - binaryURL downloaded successfully", pluginKey) if err := os.MkdirAll(resolvedCacheDir, 0755); err != nil { return fmt.Errorf("%s - failed to create cache dir: %w", pluginKey, err) diff --git a/pkg/loop/cmd/loopinstall/cache_test.go b/pkg/loop/cmd/loopinstall/cache_test.go index 874711941d..396d54a2de 100644 --- a/pkg/loop/cmd/loopinstall/cache_test.go +++ b/pkg/loop/cmd/loopinstall/cache_test.go @@ -285,6 +285,7 @@ func TestDownloadAndInstallPlugin_Tier2_GitHubReleaseAsset(t *testing.T) { jsonResp := fmt.Sprintf(`{ "assets": [ { + "id": 12345, "name": "%s", "browser_download_url": "http://%s/download/%s" } @@ -294,7 +295,8 @@ func TestDownloadAndInstallPlugin_Tier2_GitHubReleaseAsset(t *testing.T) { _, _ = w.Write([]byte(jsonResp)) return } - if filepath.Base(r.URL.Path) == fmt.Sprintf("chainlink-starknet_%s_%s", runtime.GOOS, runtime.GOARCH) { + if r.URL.Path == "/repos/smartcontractkit/chainlink-starknet/releases/assets/12345" { + assert.Equal(t, "application/octet-stream", r.Header.Get("Accept")) _, _ = w.Write([]byte("release-asset-binary")) return } @@ -380,6 +382,7 @@ func TestDownloadAndInstallPlugin_Tier2_WindowsExe(t *testing.T) { jsonResp := `{ "assets": [ { + "id": 67890, "name": "chainlink-win_windows_amd64.exe", "browser_download_url": "http://` + r.Host + `/download/chainlink-win.exe" } @@ -389,7 +392,8 @@ func TestDownloadAndInstallPlugin_Tier2_WindowsExe(t *testing.T) { _, _ = w.Write([]byte(jsonResp)) return } - if r.URL.Path == "/download/chainlink-win.exe" { + if r.URL.Path == "/repos/smartcontractkit/chainlink-win/releases/assets/67890" { + assert.Equal(t, "application/octet-stream", r.Header.Get("Accept")) _, _ = w.Write([]byte("windows-exe-binary")) return } diff --git a/pkg/loop/cmd/loopinstall/main.go b/pkg/loop/cmd/loopinstall/main.go index 1c4f3b80fc..885f88c0c0 100644 --- a/pkg/loop/cmd/loopinstall/main.go +++ b/pkg/loop/cmd/loopinstall/main.go @@ -108,6 +108,8 @@ Options: -s, --sequential Install plugins sequentially (no concurrency) -o, --output-installation-artifacts Path for installation artifacts JSON file (optional, no installation artifacts written if not specified) + --cache-dir Directory path for caching compiled plugin binaries + (optional, defaults to $CL_LOOPINSTALL_CACHE_DIR or os.UserCacheDir()) Examples: # Install plugins from the default configuration @@ -126,8 +128,9 @@ Examples: CL_PLUGIN_ENVVARS="GOOS=linux GOARCH=amd64 CGO_ENABLED=0" loopinstall plugins.default.yaml Environment Variables: - CL_PLUGIN_GOFLAGS Override the goflags option from the configuration - CL_PLUGIN_ENVVARS Space-separated list of environment variables to set during installation (for example for cross-compilation) + CL_PLUGIN_GOFLAGS Override the goflags option from the configuration + CL_PLUGIN_ENVVARS Space-separated list of environment variables to set during installation (for example for cross-compilation) + CL_LOOPINSTALL_CACHE_DIR Override default directory path for caching compiled plugin binaries Plugin Configuration Format: defaults: From de5d7a4c4d4d470b6464573bae47d656a82dc6ba Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Thu, 23 Jul 2026 17:07:31 -0400 Subject: [PATCH 11/11] Chad suggestions --- .github/workflows/build_external.yml | 2 +- pkg/loop/cmd/loopinstall/cache.go | 114 ++++++++++++++----------- pkg/loop/cmd/loopinstall/cache_test.go | 34 +++++++- 3 files changed, 98 insertions(+), 52 deletions(-) diff --git a/.github/workflows/build_external.yml b/.github/workflows/build_external.yml index 2dd21c2c27..28a44514b6 100644 --- a/.github/workflows/build_external.yml +++ b/.github/workflows/build_external.yml @@ -148,7 +148,7 @@ jobs: go mod tidy - name: Install Nix - uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 # v31.11.0 + uses: cachix/install-nix-action@8887e596b4ee1134dae06b98d573bd674693f47c # v26 with: nix_path: nixpkgs=channel:nixos-unstable diff --git a/pkg/loop/cmd/loopinstall/cache.go b/pkg/loop/cmd/loopinstall/cache.go index 0a491d8be7..9497554c66 100644 --- a/pkg/loop/cmd/loopinstall/cache.go +++ b/pkg/loop/cmd/loopinstall/cache.go @@ -7,6 +7,7 @@ import ( "io" "log" "net/http" + "net/url" "os" "path/filepath" "runtime" @@ -36,8 +37,10 @@ func resolveCacheDir(cliCacheDir string) (string, error) { // computeCacheKey calculates a deterministic cache key for a plugin. func computeCacheKey(pluginType string, plugin PluginDef, defaults DefaultsConfig, goos, goarch string) string { h := sha256.New() - _, _ = fmt.Fprintf(h, "%s|%s|%s|%s|%s|%s|%s|%s|%s|%v|%v", - pluginType, plugin.ModuleURI, plugin.GitRef, plugin.InstallPath, plugin.Flags, defaults.GoFlags, plugin.BinaryURL, goos, goarch, plugin.EnvVars, defaults.EnvVars) + envGoFlags := os.Getenv("CL_PLUGIN_GOFLAGS") + envEnvVars := os.Getenv("CL_PLUGIN_ENVVARS") + _, _ = fmt.Fprintf(h, "%s|%s|%s|%s|%s|%s|%s|%s|%s|%v|%v|%s|%s", + pluginType, plugin.ModuleURI, plugin.GitRef, plugin.InstallPath, plugin.Flags, defaults.GoFlags, plugin.BinaryURL, goos, goarch, plugin.EnvVars, defaults.EnvVars, envGoFlags, envEnvVars) base := filepath.Base(filepath.Clean(plugin.ModuleURI)) return fmt.Sprintf("%s-%x", base, h.Sum(nil)[:12]) } @@ -50,6 +53,59 @@ func getAuthToken() string { return os.Getenv("GITHUB_TOKEN") } +// isTrustedAuthDomain returns true if the URL domain is safe to send bearer auth tokens to. +func isTrustedAuthDomain(rawURL string) bool { + parsed, err := url.Parse(rawURL) + if err != nil { + return false + } + host := strings.ToLower(parsed.Hostname()) + return host == "github.com" || + strings.HasSuffix(host, ".github.com") || + host == "githubusercontent.com" || + strings.HasSuffix(host, ".githubusercontent.com") || + strings.HasSuffix(host, ".amazonaws.com") +} + +// tryDownloadBinaryURL downloads a binary from an explicit BinaryURL. +func tryDownloadBinaryURL(pluginKey, binaryURL string) ([]byte, error) { + client := &http.Client{Timeout: 15 * time.Second} + req, err := http.NewRequest("GET", binaryURL, nil) + if err != nil { + return nil, fmt.Errorf("binaryURL request creation failed: %w", err) + } + + if token := getAuthToken(); token != "" && isTrustedAuthDomain(binaryURL) { + req.Header.Set("Authorization", "Bearer "+token) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("binaryURL request failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("binaryURL returned HTTP %d", resp.StatusCode) + } + + return io.ReadAll(resp.Body) +} + +// saveAndInstallCachedBinary writes asset data to the cache directory and copies it to target binary path. +func saveAndInstallCachedBinary(pluginKey string, data []byte, cacheDir, cachePath, targetPath string) error { + if err := os.MkdirAll(cacheDir, 0755); err != nil { + return fmt.Errorf("%s - failed to create cache dir: %w", pluginKey, err) + } + if err := os.WriteFile(cachePath, data, 0755); err != nil { + return fmt.Errorf("%s - failed to write binary to cache: %w", pluginKey, err) + } + if err := copyFile(cachePath, targetPath); err != nil { + return fmt.Errorf("%s - failed to copy binary to output: %w", pluginKey, err) + } + return nil +} + // githubReleaseAsset represents GitHub release JSON structure. type githubReleaseAsset struct { ID int64 `json:"id"` @@ -80,8 +136,8 @@ func tryDownloadGitHubRelease(plugin PluginDef, binaryName, goos, goarch, github return nil, fmt.Errorf("empty gitRef, release download skipped") } - url := fmt.Sprintf("%s/repos/%s/%s/releases/tags/%s", githubAPIBaseURL, owner, repo, ref) - req, err := http.NewRequest("GET", url, nil) + urlStr := fmt.Sprintf("%s/repos/%s/%s/releases/tags/%s", githubAPIBaseURL, owner, repo, ref) + req, err := http.NewRequest("GET", urlStr, nil) if err != nil { return nil, err } @@ -245,43 +301,12 @@ func downloadAndInstallPluginWithCache(pluginType string, pluginIdx int, plugin // Tier 2: Try explicit binary URL override, then GitHub Release Asset fallback if plugin.BinaryURL != "" { log.Printf("%s - cache miss, attempting binaryURL download (%s)", pluginKey, plugin.BinaryURL) - - client := &http.Client{Timeout: 15 * time.Second} - req, err := http.NewRequest("GET", plugin.BinaryURL, nil) + assetData, err := tryDownloadBinaryURL(pluginKey, plugin.BinaryURL) if err != nil { - log.Printf("%s - binaryURL request creation failed (%v)", pluginKey, err) + log.Printf("%s - %v", pluginKey, err) } else { - if token := getAuthToken(); token != "" { - if strings.Contains(plugin.BinaryURL, "github.com") || strings.Contains(plugin.BinaryURL, "githubusercontent.com") || strings.Contains(plugin.BinaryURL, "amazonaws.com") { - req.Header.Set("Authorization", "Bearer "+token) - } - } - resp, err := client.Do(req) - if err != nil { - log.Printf("%s - binaryURL request failed (%v)", pluginKey, err) - } else { - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - log.Printf("%s - binaryURL returned HTTP %d", pluginKey, resp.StatusCode) - } else { - assetData, err := io.ReadAll(resp.Body) - if err != nil { - log.Printf("%s - failed to read binaryURL response body (%v)", pluginKey, err) - } else { - log.Printf("%s - binaryURL downloaded successfully", pluginKey) - if err := os.MkdirAll(resolvedCacheDir, 0755); err != nil { - return fmt.Errorf("%s - failed to create cache dir: %w", pluginKey, err) - } - if err := os.WriteFile(cachedBinaryPath, assetData, 0755); err != nil { - return fmt.Errorf("%s - failed to write binary to cache: %w", pluginKey, err) - } - if err := copyFile(cachedBinaryPath, outputPath); err != nil { - return fmt.Errorf("%s - failed to copy binary to output: %w", pluginKey, err) - } - return nil - } - } - } + log.Printf("%s - binaryURL downloaded successfully", pluginKey) + return saveAndInstallCachedBinary(pluginKey, assetData, resolvedCacheDir, cachedBinaryPath, outputPath) } } @@ -290,16 +315,7 @@ func downloadAndInstallPluginWithCache(pluginType string, pluginIdx int, plugin assetData, relErr := tryDownloadGitHubRelease(plugin, binaryName, goos, goarch, githubAPIBaseURL) if relErr == nil { log.Printf("%s - release asset downloaded successfully", pluginKey) - if err := os.MkdirAll(resolvedCacheDir, 0755); err != nil { - return fmt.Errorf("%s - failed to create cache dir: %w", pluginKey, err) - } - if err := os.WriteFile(cachedBinaryPath, assetData, 0755); err != nil { - return fmt.Errorf("%s - failed to write binary to cache: %w", pluginKey, err) - } - if err := copyFile(cachedBinaryPath, outputPath); err != nil { - return fmt.Errorf("%s - failed to copy binary to output: %w", pluginKey, err) - } - return nil + return saveAndInstallCachedBinary(pluginKey, assetData, resolvedCacheDir, cachedBinaryPath, outputPath) } log.Printf("%s - release asset download not available (%v), falling back to source build", pluginKey, relErr) } diff --git a/pkg/loop/cmd/loopinstall/cache_test.go b/pkg/loop/cmd/loopinstall/cache_test.go index 396d54a2de..fe801acbdc 100644 --- a/pkg/loop/cmd/loopinstall/cache_test.go +++ b/pkg/loop/cmd/loopinstall/cache_test.go @@ -43,8 +43,6 @@ func TestComputeCacheKey(t *testing.T) { } func TestComputeCacheKey_Variations(t *testing.T) { - t.Parallel() - basePlugin := PluginDef{ ModuleURI: "github.com/smartcontractkit/chainlink-cosmos", GitRef: "v1.0.0", @@ -89,6 +87,20 @@ func TestComputeCacheKey_Variations(t *testing.T) { }, different: true, }, + { + name: "different_cl_plugin_goflags_env", + modify: func(p *PluginDef, d *DefaultsConfig) { + t.Setenv("CL_PLUGIN_GOFLAGS", "-tags=custom") + }, + different: true, + }, + { + name: "different_cl_plugin_envvars_env", + modify: func(p *PluginDef, d *DefaultsConfig) { + t.Setenv("CL_PLUGIN_ENVVARS", "CGO_ENABLED=1") + }, + different: true, + }, } for _, tc := range tests { @@ -106,6 +118,24 @@ func TestComputeCacheKey_Variations(t *testing.T) { } } +func TestTryDownloadBinaryURL_DomainSecurity(t *testing.T) { + t.Setenv("GITHUB_TOKEN", "secret-token") + + var receivedAuthHeader string + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedAuthHeader = r.Header.Get("Authorization") + _, _ = w.Write([]byte("binary-data")) + })) + defer mockServer.Close() + + // 1. Untrusted domain with github.com in path should NOT receive auth token + untrustedURL := mockServer.URL + "/github.com/fake-asset" + data, err := tryDownloadBinaryURL("key", untrustedURL) + require.NoError(t, err) + assert.Equal(t, "binary-data", string(data)) + assert.Empty(t, receivedAuthHeader, "untrusted domain with github.com in path should not receive Authorization header") +} + func TestDownloadAndInstallPlugin_DisabledAndValidation(t *testing.T) { tempCacheDir := t.TempDir() tempBinDir := t.TempDir()