From 7e35079241f460640626adb0ca320fb29aabd8a9 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 24 Jul 2026 10:11:28 -0400 Subject: [PATCH 01/10] Add docker build cache dance --- actions/build-push-docker/action.yml | 41 ++++++++++++ .../scripts/test-cache-dance-config.sh | 54 +++++++++++++++ actions/ctf-build-image/action.yml | 32 +++++++-- .../scripts/test-cache-event-overrides.sh | 66 +++++++++++++++++++ 4 files changed, 187 insertions(+), 6 deletions(-) create mode 100755 actions/build-push-docker/scripts/test-cache-dance-config.sh create mode 100755 actions/ctf-build-image/scripts/test-cache-event-overrides.sh diff --git a/actions/build-push-docker/action.yml b/actions/build-push-docker/action.yml index 6a7465f73..8d3edf8ea 100644 --- a/actions/build-push-docker/action.yml +++ b/actions/build-push-docker/action.yml @@ -123,6 +123,21 @@ inputs: Defaults to `true` for backwards compatibility. Ignored when inputs.docker-push is false. required: false default: "true" + cache-dance: + description: | + Whether to enable buildkit-cache-dance to persist RUN --mount=type=cache buildkit cache mounts. + See: https://github.com/reproducible-containers/buildkit-cache-dance + See: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts + required: false + default: "false" + cache-dance-cache-map: + description: | + JSON string mapping cache mount paths/names for buildkit-cache-dance (e.g. '{"go-mod-cache": "/go/pkg/mod"}'). + Required if cache-dance is set to true. + See: https://github.com/reproducible-containers/buildkit-cache-dance + See: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts + required: false + default: "" outputs: docker-repository-name: @@ -172,6 +187,23 @@ runs: exit 1 fi + - name: Validate cache-dance configuration + shell: bash + env: + CACHE_DANCE: ${{ inputs.cache-dance }} + CACHE_MAP: ${{ inputs.cache-dance-cache-map }} + run: | + if [[ "${CACHE_DANCE}" == "true" ]]; then + if [[ -z "${CACHE_MAP}" ]]; then + echo "::error::cache-dance-cache-map input is required when cache-dance is true." + exit 1 + fi + if ! echo "${CACHE_MAP}" | jq . >/dev/null 2>&1; then + echo "::error::cache-dance-cache-map must be valid JSON." + exit 1 + fi + fi + - name: Check runner arch and platform compatibility shell: bash env: @@ -361,6 +393,15 @@ runs: echo "no-cache=false" | tee -a "${GITHUB_OUTPUT}" echo "cache-from=${CACHE_FROM}" | tee -a "${GITHUB_OUTPUT}" + # Persist BuildKit RUN --mount=type=cache directories across GHA runs + # See: https://github.com/reproducible-containers/buildkit-cache-dance + # See: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts + - name: Cache Dance + if: ${{ inputs.cache-dance == 'true' }} + uses: reproducible-containers/buildkit-cache-dance@5422eac04292c961a382e0f584ea0f03ad9da723 # v3.4.0 + with: + cache-map: ${{ inputs.cache-dance-cache-map }} + - name: Build & push image id: build-image uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 diff --git a/actions/build-push-docker/scripts/test-cache-dance-config.sh b/actions/build-push-docker/scripts/test-cache-dance-config.sh new file mode 100755 index 000000000..7a793e69f --- /dev/null +++ b/actions/build-push-docker/scripts/test-cache-dance-config.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -e + +echo "===== Testing build-push-docker cache dance configuration =====" + +# Helper script/function to test cache dance inputs validation +validate_cache_dance_config() { + local cache_dance="$1" + local cache_map="$2" + + if [ "$cache_dance" = "true" ]; then + if [ -z "$cache_map" ]; then + echo "ERROR: cache-dance-cache-map is required when cache-dance is true." + return 1 + fi + # Check if cache_map is valid JSON or non-empty string + if ! echo "$cache_map" | jq . >/dev/null 2>&1; then + echo "ERROR: cache-dance-cache-map must be valid JSON." + return 1 + fi + echo "VALID: cache-dance enabled with valid cache-map" + return 0 + else + echo "VALID: cache-dance disabled" + return 0 + fi +} + +echo "Test 1: Cache dance disabled (default)" +validate_cache_dance_config "false" "" +echo "Test 1 passed." + +echo "Test 2: Cache dance enabled with missing cache-map (should fail)" +if validate_cache_dance_config "true" ""; then + echo "FAIL: Expected failure when cache-map is missing" + exit 1 +else + echo "Test 2 passed (failed as expected)." +fi + +echo "Test 3: Cache dance enabled with invalid JSON cache-map (should fail)" +if validate_cache_dance_config "true" "invalid-json"; then + echo "FAIL: Expected failure when cache-map is invalid JSON" + exit 1 +else + echo "Test 3 passed (failed as expected)." +fi + +echo "Test 4: Cache dance enabled with valid JSON cache-map" +VALID_MAP='{"go-mod-cache": "/go/pkg/mod", "go-build-cache": "/root/.cache/go-build"}' +validate_cache_dance_config "true" "$VALID_MAP" +echo "Test 4 passed." + +echo "All cache dance configuration tests completed successfully." diff --git a/actions/ctf-build-image/action.yml b/actions/ctf-build-image/action.yml index 48e049868..453844796 100644 --- a/actions/ctf-build-image/action.yml +++ b/actions/ctf-build-image/action.yml @@ -123,6 +123,22 @@ inputs: on runners with sufficient disk (e.g. RunsOn with 100GB+) to save ~30-60s. default: "true" + docker-restore-cache: + required: false + description: | + Optionally override docker restore cache behavior ("true" or "false"). + If unset, cache is restored for non-schedule and non-push events. + See: https://docs.docker.com/build/ci/github-actions/cache/ + default: "" + + docker-save-cache: + required: false + description: | + Optionally override docker save cache behavior ("true" or "false"). + If unset, cache is saved only on schedule and push events. + See: https://docs.docker.com/build/ci/github-actions/cache/ + default: "" + outputs: docker-image-sha-digest-amd64: description: "Docker image SHA digest for platform: amd64" @@ -197,13 +213,17 @@ runs: docker-attestations: "false" docker-registry-url: ${{ inputs.docker-registry-url }} docker-repository-name: ${{ inputs.docker-repository-name }} - # only save on events which are expected to be from the default branch - docker-save-cache: - ${{ github.event_name == 'schedule' || github.event_name == 'push' }} - # dont use cache on events which are expected to be from the default branch + # only save on events which are expected to be from the default branch (unless overridden) + docker-save-cache: >- + ${{ inputs.docker-save-cache != '' && inputs.docker-save-cache || + (inputs.docker-save-cache == '' && (github.event_name == 'schedule' || + github.event_name == 'push')) }} + # dont use cache on events which are expected to be from the default branch (unless overridden) # this is to create a fresh cache/snapshot unpolluted by previous cache entries - docker-restore-cache: - ${{ github.event_name != 'schedule' && github.event_name != 'push' }} + docker-restore-cache: >- + ${{ inputs.docker-restore-cache != '' && inputs.docker-restore-cache + || (inputs.docker-restore-cache == '' && github.event_name != + 'schedule' && github.event_name != 'push') }} docker-build-cache-to: "type=gha,timeout=10m,mode=max,ignore-error=true,compression=zstd,compression-level=3,scope=ctf-build-image-${{ inputs.cache-scope || format('{0}-{1}', runner.os, runner.arch) }}" diff --git a/actions/ctf-build-image/scripts/test-cache-event-overrides.sh b/actions/ctf-build-image/scripts/test-cache-event-overrides.sh new file mode 100755 index 000000000..da38af292 --- /dev/null +++ b/actions/ctf-build-image/scripts/test-cache-event-overrides.sh @@ -0,0 +1,66 @@ +#!/bin/bash +set -e + +echo "===== Testing ctf-build-image cache event overrides =====" + +# Helper function to evaluate cache save decision logic +resolve_save_cache() { + local input_val="$1" + local event_name="$2" + + if [ -n "$input_val" ]; then + echo "$input_val" + else + if [ "$event_name" = "schedule" ] || [ "$event_name" = "push" ]; then + echo "true" + else + echo "false" + fi + fi +} + +# Helper function to evaluate cache restore decision logic +resolve_restore_cache() { + local input_val="$1" + local event_name="$2" + + if [ -n "$input_val" ]; then + echo "$input_val" + else + if [ "$event_name" != "schedule" ] && [ "$event_name" != "push" ]; then + echo "true" + else + echo "false" + fi + fi +} + +echo "Test 1: Default behavior on push event (save=true, restore=false)" +RES_SAVE=$(resolve_save_cache "" "push") +RES_RESTORE=$(resolve_restore_cache "" "push") +[ "$RES_SAVE" = "true" ] || (echo "FAIL: expected save=true, got $RES_SAVE" && exit 1) +[ "$RES_RESTORE" = "false" ] || (echo "FAIL: expected restore=false, got $RES_RESTORE" && exit 1) +echo "Test 1 passed." + +echo "Test 2: Default behavior on pull_request event (save=false, restore=true)" +RES_SAVE=$(resolve_save_cache "" "pull_request") +RES_RESTORE=$(resolve_restore_cache "" "pull_request") +[ "$RES_SAVE" = "false" ] || (echo "FAIL: expected save=false, got $RES_SAVE" && exit 1) +[ "$RES_RESTORE" = "true" ] || (echo "FAIL: expected restore=true, got $RES_RESTORE" && exit 1) +echo "Test 2 passed." + +echo "Test 3: Override save cache on pull_request event (save=true)" +RES_SAVE=$(resolve_save_cache "true" "pull_request") +RES_RESTORE=$(resolve_restore_cache "" "pull_request") +[ "$RES_SAVE" = "true" ] || (echo "FAIL: expected save=true, got $RES_SAVE" && exit 1) +[ "$RES_RESTORE" = "true" ] || (echo "FAIL: expected restore=true, got $RES_RESTORE" && exit 1) +echo "Test 3 passed." + +echo "Test 4: Override restore cache on push event (restore=true)" +RES_SAVE=$(resolve_save_cache "" "push") +RES_RESTORE=$(resolve_restore_cache "true" "push") +[ "$RES_SAVE" = "true" ] || (echo "FAIL: expected save=true, got $RES_SAVE" && exit 1) +[ "$RES_RESTORE" = "true" ] || (echo "FAIL: expected restore=true, got $RES_RESTORE" && exit 1) +echo "Test 4 passed." + +echo "All cache event override tests completed successfully." From 75f7437d42630b8f771b1a7879079ed9743165dc Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 24 Jul 2026 10:26:19 -0400 Subject: [PATCH 02/10] Enable cache dance through ctf-build-image --- actions/ctf-build-image/action.yml | 20 ++++++++++++++++++- .../scripts/test-cache-event-overrides.sh | 12 ++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/actions/ctf-build-image/action.yml b/actions/ctf-build-image/action.yml index 453844796..ac2dda8ff 100644 --- a/actions/ctf-build-image/action.yml +++ b/actions/ctf-build-image/action.yml @@ -139,6 +139,22 @@ inputs: See: https://docs.docker.com/build/ci/github-actions/cache/ default: "" + cache-dance: + required: false + description: | + Whether to enable buildkit-cache-dance to persist RUN --mount=type=cache buildkit cache mounts. + See: https://github.com/reproducible-containers/buildkit-cache-dance + See: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts + default: "false" + + cache-dance-cache-map: + required: false + description: | + JSON string mapping cache mount paths/names for buildkit-cache-dance (e.g. '{"go-mod-cache": "/go/pkg/mod"}'). + See: https://github.com/reproducible-containers/buildkit-cache-dance + See: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts + default: "" + outputs: docker-image-sha-digest-amd64: description: "Docker image SHA digest for platform: amd64" @@ -200,7 +216,7 @@ runs: - name: Build push docker image id: build-image - uses: smartcontractkit/.github/actions/build-push-docker@build-push-docker/v1 + uses: smartcontractkit/.github/actions/build-push-docker@7e35079241f460640626adb0ca320fb29aabd8a9 # DEBUG: Using cache dance version with: context: . platform: ${{ inputs.platform }} @@ -230,6 +246,8 @@ runs: docker-build-cache-from: "type=gha,timeout=10m,scope=ctf-build-image-${{ inputs.cache-scope || format('{0}-{1}', runner.os, runner.arch) }}" + cache-dance: ${{ inputs.cache-dance }} + cache-dance-cache-map: ${{ inputs.cache-dance-cache-map }} tags: type=raw,value=${{ inputs.image-tag }} aws-account-number: ${{ inputs.aws-account-number }} diff --git a/actions/ctf-build-image/scripts/test-cache-event-overrides.sh b/actions/ctf-build-image/scripts/test-cache-event-overrides.sh index da38af292..a04f15a05 100755 --- a/actions/ctf-build-image/scripts/test-cache-event-overrides.sh +++ b/actions/ctf-build-image/scripts/test-cache-event-overrides.sh @@ -63,4 +63,14 @@ RES_RESTORE=$(resolve_restore_cache "true" "push") [ "$RES_RESTORE" = "true" ] || (echo "FAIL: expected restore=true, got $RES_RESTORE" && exit 1) echo "Test 4 passed." -echo "All cache event override tests completed successfully." +echo "Test 5: Forward cache-dance inputs" +resolve_cache_dance_forwarding() { + local dance="$1" + local map="$2" + echo "dance=$dance,map=$map" +} +FORWARD=$(resolve_cache_dance_forwarding "true" '{"go-mod-cache": "/go/pkg/mod"}') +[ "$FORWARD" = 'dance=true,map={"go-mod-cache": "/go/pkg/mod"}' ] || (echo "FAIL: expected dance forwarding match" && exit 1) +echo "Test 5 passed." + +echo "All cache event override and dance forwarding tests completed successfully." From 9468293a3c669ba719a2a3fdfee1722e8a2cbc5b Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 24 Jul 2026 12:07:00 -0400 Subject: [PATCH 03/10] Fix test scripts + action versins --- actions/build-push-docker/action.yml | 12 +--- .../scripts/test-cache-dance-config.sh | 34 ++------- .../scripts/validate-cache-dance-config.sh | 19 +++++ actions/ctf-build-image/action.yml | 22 +++--- .../scripts/resolve-cache-settings.sh | 32 +++++++++ .../scripts/test-cache-event-overrides.sh | 69 ++++++------------- 6 files changed, 92 insertions(+), 96 deletions(-) create mode 100755 actions/build-push-docker/scripts/validate-cache-dance-config.sh create mode 100755 actions/ctf-build-image/scripts/resolve-cache-settings.sh diff --git a/actions/build-push-docker/action.yml b/actions/build-push-docker/action.yml index 8d3edf8ea..52b79ef87 100644 --- a/actions/build-push-docker/action.yml +++ b/actions/build-push-docker/action.yml @@ -192,17 +192,7 @@ runs: env: CACHE_DANCE: ${{ inputs.cache-dance }} CACHE_MAP: ${{ inputs.cache-dance-cache-map }} - run: | - if [[ "${CACHE_DANCE}" == "true" ]]; then - if [[ -z "${CACHE_MAP}" ]]; then - echo "::error::cache-dance-cache-map input is required when cache-dance is true." - exit 1 - fi - if ! echo "${CACHE_MAP}" | jq . >/dev/null 2>&1; then - echo "::error::cache-dance-cache-map must be valid JSON." - exit 1 - fi - fi + run: ${{ github.action_path }}/scripts/validate-cache-dance-config.sh - name: Check runner arch and platform compatibility shell: bash diff --git a/actions/build-push-docker/scripts/test-cache-dance-config.sh b/actions/build-push-docker/scripts/test-cache-dance-config.sh index 7a793e69f..ce6fc4a8a 100755 --- a/actions/build-push-docker/scripts/test-cache-dance-config.sh +++ b/actions/build-push-docker/scripts/test-cache-dance-config.sh @@ -1,37 +1,17 @@ #!/bin/bash set -e -echo "===== Testing build-push-docker cache dance configuration =====" - -# Helper script/function to test cache dance inputs validation -validate_cache_dance_config() { - local cache_dance="$1" - local cache_map="$2" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VALIDATE_SCRIPT="${SCRIPT_DIR}/validate-cache-dance-config.sh" - if [ "$cache_dance" = "true" ]; then - if [ -z "$cache_map" ]; then - echo "ERROR: cache-dance-cache-map is required when cache-dance is true." - return 1 - fi - # Check if cache_map is valid JSON or non-empty string - if ! echo "$cache_map" | jq . >/dev/null 2>&1; then - echo "ERROR: cache-dance-cache-map must be valid JSON." - return 1 - fi - echo "VALID: cache-dance enabled with valid cache-map" - return 0 - else - echo "VALID: cache-dance disabled" - return 0 - fi -} +echo "===== Testing build-push-docker cache dance configuration =====" echo "Test 1: Cache dance disabled (default)" -validate_cache_dance_config "false" "" +"$VALIDATE_SCRIPT" "false" "" echo "Test 1 passed." echo "Test 2: Cache dance enabled with missing cache-map (should fail)" -if validate_cache_dance_config "true" ""; then +if "$VALIDATE_SCRIPT" "true" ""; then echo "FAIL: Expected failure when cache-map is missing" exit 1 else @@ -39,7 +19,7 @@ else fi echo "Test 3: Cache dance enabled with invalid JSON cache-map (should fail)" -if validate_cache_dance_config "true" "invalid-json"; then +if "$VALIDATE_SCRIPT" "true" "invalid-json"; then echo "FAIL: Expected failure when cache-map is invalid JSON" exit 1 else @@ -48,7 +28,7 @@ fi echo "Test 4: Cache dance enabled with valid JSON cache-map" VALID_MAP='{"go-mod-cache": "/go/pkg/mod", "go-build-cache": "/root/.cache/go-build"}' -validate_cache_dance_config "true" "$VALID_MAP" +"$VALIDATE_SCRIPT" "true" "$VALID_MAP" echo "Test 4 passed." echo "All cache dance configuration tests completed successfully." diff --git a/actions/build-push-docker/scripts/validate-cache-dance-config.sh b/actions/build-push-docker/scripts/validate-cache-dance-config.sh new file mode 100755 index 000000000..db80b564a --- /dev/null +++ b/actions/build-push-docker/scripts/validate-cache-dance-config.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +# Validates cache-dance configuration inputs. +# Accepts arguments ($1: cache_dance, $2: cache_map) or environment variables (CACHE_DANCE, CACHE_MAP). + +CACHE_DANCE="${1:-${CACHE_DANCE}}" +CACHE_MAP="${2:-${CACHE_MAP}}" + +if [ "${CACHE_DANCE}" = "true" ]; then + if [ -z "${CACHE_MAP}" ]; then + echo "::error::cache-dance-cache-map input is required when cache-dance is true." + exit 1 + fi + if ! echo "${CACHE_MAP}" | jq . >/dev/null 2>&1; then + echo "::error::cache-dance-cache-map must be valid JSON." + exit 1 + fi +fi diff --git a/actions/ctf-build-image/action.yml b/actions/ctf-build-image/action.yml index ac2dda8ff..44d1c9f46 100644 --- a/actions/ctf-build-image/action.yml +++ b/actions/ctf-build-image/action.yml @@ -214,9 +214,19 @@ runs: if: inputs.free-disk-space == 'true' uses: smartcontractkit/.github/actions/free-disk-space@free-disk-space/v1 + - name: Resolve cache settings + shell: bash + id: cache-settings + env: + DOCKER_SAVE_CACHE_INPUT: ${{ inputs.docker-save-cache }} + DOCKER_RESTORE_CACHE_INPUT: ${{ inputs.docker-restore-cache }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + run: | + ${{ github.action_path }}/scripts/resolve-cache-settings.sh >> "$GITHUB_OUTPUT" + - name: Build push docker image id: build-image - uses: smartcontractkit/.github/actions/build-push-docker@7e35079241f460640626adb0ca320fb29aabd8a9 # DEBUG: Using cache dance version + uses: smartcontractkit/.github/actions/build-push-docker@build-push-docker/v1 with: context: . platform: ${{ inputs.platform }} @@ -230,16 +240,10 @@ runs: docker-registry-url: ${{ inputs.docker-registry-url }} docker-repository-name: ${{ inputs.docker-repository-name }} # only save on events which are expected to be from the default branch (unless overridden) - docker-save-cache: >- - ${{ inputs.docker-save-cache != '' && inputs.docker-save-cache || - (inputs.docker-save-cache == '' && (github.event_name == 'schedule' || - github.event_name == 'push')) }} + docker-save-cache: ${{ steps.cache-settings.outputs.save-cache }} # dont use cache on events which are expected to be from the default branch (unless overridden) # this is to create a fresh cache/snapshot unpolluted by previous cache entries - docker-restore-cache: >- - ${{ inputs.docker-restore-cache != '' && inputs.docker-restore-cache - || (inputs.docker-restore-cache == '' && github.event_name != - 'schedule' && github.event_name != 'push') }} + docker-restore-cache: ${{ steps.cache-settings.outputs.restore-cache }} docker-build-cache-to: "type=gha,timeout=10m,mode=max,ignore-error=true,compression=zstd,compression-level=3,scope=ctf-build-image-${{ inputs.cache-scope || format('{0}-{1}', runner.os, runner.arch) }}" diff --git a/actions/ctf-build-image/scripts/resolve-cache-settings.sh b/actions/ctf-build-image/scripts/resolve-cache-settings.sh new file mode 100755 index 000000000..f4f51edfd --- /dev/null +++ b/actions/ctf-build-image/scripts/resolve-cache-settings.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -e + +# Resolves docker save/restore cache settings given input overrides and GitHub event name. +# Usage: resolve-cache-settings.sh + +SAVE_INPUT="${1:-${DOCKER_SAVE_CACHE_INPUT}}" +RESTORE_INPUT="${2:-${DOCKER_RESTORE_CACHE_INPUT}}" +EVENT_NAME="${3:-${GITHUB_EVENT_NAME}}" + +if [ -n "$SAVE_INPUT" ]; then + SAVE_CACHE="$SAVE_INPUT" +else + if [ "$EVENT_NAME" = "schedule" ] || [ "$EVENT_NAME" = "push" ]; then + SAVE_CACHE="true" + else + SAVE_CACHE="false" + fi +fi + +if [ -n "$RESTORE_INPUT" ]; then + RESTORE_CACHE="$RESTORE_INPUT" +else + if [ "$EVENT_NAME" != "schedule" ] && [ "$EVENT_NAME" != "push" ]; then + RESTORE_CACHE="true" + else + RESTORE_CACHE="false" + fi +fi + +echo "save-cache=${SAVE_CACHE}" +echo "restore-cache=${RESTORE_CACHE}" diff --git a/actions/ctf-build-image/scripts/test-cache-event-overrides.sh b/actions/ctf-build-image/scripts/test-cache-event-overrides.sh index a04f15a05..44afbf5c4 100755 --- a/actions/ctf-build-image/scripts/test-cache-event-overrides.sh +++ b/actions/ctf-build-image/scripts/test-cache-event-overrides.sh @@ -1,76 +1,47 @@ #!/bin/bash set -e -echo "===== Testing ctf-build-image cache event overrides =====" - -# Helper function to evaluate cache save decision logic -resolve_save_cache() { - local input_val="$1" - local event_name="$2" - - if [ -n "$input_val" ]; then - echo "$input_val" - else - if [ "$event_name" = "schedule" ] || [ "$event_name" = "push" ]; then - echo "true" - else - echo "false" - fi - fi -} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RESOLVE_SCRIPT="${SCRIPT_DIR}/resolve-cache-settings.sh" -# Helper function to evaluate cache restore decision logic -resolve_restore_cache() { - local input_val="$1" - local event_name="$2" +echo "===== Testing ctf-build-image cache event overrides =====" - if [ -n "$input_val" ]; then - echo "$input_val" - else - if [ "$event_name" != "schedule" ] && [ "$event_name" != "push" ]; then - echo "true" - else - echo "false" - fi - fi +parse_val() { + local output="$1" + local key="$2" + echo "$output" | grep "^${key}=" | cut -d'=' -f2 } echo "Test 1: Default behavior on push event (save=true, restore=false)" -RES_SAVE=$(resolve_save_cache "" "push") -RES_RESTORE=$(resolve_restore_cache "" "push") +RES=$("$RESOLVE_SCRIPT" "" "" "push") +RES_SAVE=$(parse_val "$RES" "save-cache") +RES_RESTORE=$(parse_val "$RES" "restore-cache") [ "$RES_SAVE" = "true" ] || (echo "FAIL: expected save=true, got $RES_SAVE" && exit 1) [ "$RES_RESTORE" = "false" ] || (echo "FAIL: expected restore=false, got $RES_RESTORE" && exit 1) echo "Test 1 passed." echo "Test 2: Default behavior on pull_request event (save=false, restore=true)" -RES_SAVE=$(resolve_save_cache "" "pull_request") -RES_RESTORE=$(resolve_restore_cache "" "pull_request") +RES=$("$RESOLVE_SCRIPT" "" "" "pull_request") +RES_SAVE=$(parse_val "$RES" "save-cache") +RES_RESTORE=$(parse_val "$RES" "restore-cache") [ "$RES_SAVE" = "false" ] || (echo "FAIL: expected save=false, got $RES_SAVE" && exit 1) [ "$RES_RESTORE" = "true" ] || (echo "FAIL: expected restore=true, got $RES_RESTORE" && exit 1) echo "Test 2 passed." echo "Test 3: Override save cache on pull_request event (save=true)" -RES_SAVE=$(resolve_save_cache "true" "pull_request") -RES_RESTORE=$(resolve_restore_cache "" "pull_request") +RES=$("$RESOLVE_SCRIPT" "true" "" "pull_request") +RES_SAVE=$(parse_val "$RES" "save-cache") +RES_RESTORE=$(parse_val "$RES" "restore-cache") [ "$RES_SAVE" = "true" ] || (echo "FAIL: expected save=true, got $RES_SAVE" && exit 1) [ "$RES_RESTORE" = "true" ] || (echo "FAIL: expected restore=true, got $RES_RESTORE" && exit 1) echo "Test 3 passed." echo "Test 4: Override restore cache on push event (restore=true)" -RES_SAVE=$(resolve_save_cache "" "push") -RES_RESTORE=$(resolve_restore_cache "true" "push") +RES=$("$RESOLVE_SCRIPT" "" "true" "push") +RES_SAVE=$(parse_val "$RES" "save-cache") +RES_RESTORE=$(parse_val "$RES" "restore-cache") [ "$RES_SAVE" = "true" ] || (echo "FAIL: expected save=true, got $RES_SAVE" && exit 1) [ "$RES_RESTORE" = "true" ] || (echo "FAIL: expected restore=true, got $RES_RESTORE" && exit 1) echo "Test 4 passed." -echo "Test 5: Forward cache-dance inputs" -resolve_cache_dance_forwarding() { - local dance="$1" - local map="$2" - echo "dance=$dance,map=$map" -} -FORWARD=$(resolve_cache_dance_forwarding "true" '{"go-mod-cache": "/go/pkg/mod"}') -[ "$FORWARD" = 'dance=true,map={"go-mod-cache": "/go/pkg/mod"}' ] || (echo "FAIL: expected dance forwarding match" && exit 1) -echo "Test 5 passed." - -echo "All cache event override and dance forwarding tests completed successfully." +echo "All cache event override tests completed successfully." From 28d979ede840a3c716cd467e380be3a0e1e03f64 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 24 Jul 2026 12:38:42 -0400 Subject: [PATCH 04/10] Consolidate cache inputs --- actions/build-push-docker/action.yml | 33 ++++--- .../scripts/test-cache-dance-config.sh | 54 ++++++++--- .../scripts/validate-cache-dance-config.sh | 32 +++++-- actions/ctf-build-image/action.yml | 91 ++++++++++++------- .../scripts/resolve-cache-settings.sh | 56 +++++++++--- .../scripts/test-cache-event-overrides.sh | 36 ++++++-- 6 files changed, 209 insertions(+), 93 deletions(-) diff --git a/actions/build-push-docker/action.yml b/actions/build-push-docker/action.yml index 52b79ef87..a7fa33bdd 100644 --- a/actions/build-push-docker/action.yml +++ b/actions/build-push-docker/action.yml @@ -123,21 +123,29 @@ inputs: Defaults to `true` for backwards compatibility. Ignored when inputs.docker-push is false. required: false default: "true" - cache-dance: + cache-mode: description: | - Whether to enable buildkit-cache-dance to persist RUN --mount=type=cache buildkit cache mounts. - See: https://github.com/reproducible-containers/buildkit-cache-dance - See: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts + Caching mode preset: "auto" (default), "read-write", "read-only", "write-only", or "off". required: false - default: "false" - cache-dance-cache-map: + default: "auto" + cache-map: description: | JSON string mapping cache mount paths/names for buildkit-cache-dance (e.g. '{"go-mod-cache": "/go/pkg/mod"}'). - Required if cache-dance is set to true. + Passing a non-empty cache-map automatically enables buildkit-cache-dance. See: https://github.com/reproducible-containers/buildkit-cache-dance See: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts required: false default: "" + cache-dance: + description: | + Deprecated: Explicitly enable/disable buildkit-cache-dance. If unset, cache-dance is automatically enabled when cache-map is provided. + required: false + default: "" + cache-dance-cache-map: + description: | + Deprecated alias for cache-map. + required: false + default: "" outputs: docker-repository-name: @@ -188,11 +196,14 @@ runs: fi - name: Validate cache-dance configuration + id: validate-cache-dance shell: bash env: CACHE_DANCE: ${{ inputs.cache-dance }} - CACHE_MAP: ${{ inputs.cache-dance-cache-map }} - run: ${{ github.action_path }}/scripts/validate-cache-dance-config.sh + CACHE_MAP: ${{ inputs.cache-map || inputs.cache-dance-cache-map }} + run: + ${{ github.action_path }}/scripts/validate-cache-dance-config.sh >> + "$GITHUB_OUTPUT" - name: Check runner arch and platform compatibility shell: bash @@ -387,10 +398,10 @@ runs: # See: https://github.com/reproducible-containers/buildkit-cache-dance # See: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts - name: Cache Dance - if: ${{ inputs.cache-dance == 'true' }} + if: ${{ steps.validate-cache-dance.outputs.cache-dance == 'true' }} uses: reproducible-containers/buildkit-cache-dance@5422eac04292c961a382e0f584ea0f03ad9da723 # v3.4.0 with: - cache-map: ${{ inputs.cache-dance-cache-map }} + cache-map: ${{ steps.validate-cache-dance.outputs.cache-map }} - name: Build & push image id: build-image diff --git a/actions/build-push-docker/scripts/test-cache-dance-config.sh b/actions/build-push-docker/scripts/test-cache-dance-config.sh index ce6fc4a8a..2ae0784c0 100755 --- a/actions/build-push-docker/scripts/test-cache-dance-config.sh +++ b/actions/build-push-docker/scripts/test-cache-dance-config.sh @@ -4,31 +4,55 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" VALIDATE_SCRIPT="${SCRIPT_DIR}/validate-cache-dance-config.sh" -echo "===== Testing build-push-docker cache dance configuration =====" +echo "===== Testing build-push-docker cache dance configuration (Option 1) =====" -echo "Test 1: Cache dance disabled (default)" -"$VALIDATE_SCRIPT" "false" "" +parse_val() { + local output="$1" + local key="$2" + echo "$output" | grep "^${key}=" | cut -d'=' -f2 +} + +echo "Test 1: Default - no cache-dance and no cache-map" +RES=$("$VALIDATE_SCRIPT" "" "") +DANCE=$(parse_val "$RES" "cache-dance") +[ "$DANCE" = "false" ] || (echo "FAIL: expected cache-dance=false, got $DANCE" && exit 1) echo "Test 1 passed." -echo "Test 2: Cache dance enabled with missing cache-map (should fail)" -if "$VALIDATE_SCRIPT" "true" ""; then - echo "FAIL: Expected failure when cache-map is missing" - exit 1 -else - echo "Test 2 passed (failed as expected)." -fi +echo "Test 2: Implicit enable - cache-map provided without explicit cache-dance flag" +VALID_MAP='{"go-mod-cache": "/go/pkg/mod"}' +RES=$("$VALIDATE_SCRIPT" "" "$VALID_MAP") +DANCE=$(parse_val "$RES" "cache-dance") +MAP=$(parse_val "$RES" "cache-map") +[ "$DANCE" = "true" ] || (echo "FAIL: expected cache-dance=true, got $DANCE" && exit 1) +[ "$MAP" = "$VALID_MAP" ] || (echo "FAIL: expected cache-map match" && exit 1) +echo "Test 2 passed." -echo "Test 3: Cache dance enabled with invalid JSON cache-map (should fail)" -if "$VALIDATE_SCRIPT" "true" "invalid-json"; then +echo "Test 3: Invalid JSON in cache-map (should fail)" +if "$VALIDATE_SCRIPT" "" "invalid-json"; then echo "FAIL: Expected failure when cache-map is invalid JSON" exit 1 else echo "Test 3 passed (failed as expected)." fi -echo "Test 4: Cache dance enabled with valid JSON cache-map" -VALID_MAP='{"go-mod-cache": "/go/pkg/mod", "go-build-cache": "/root/.cache/go-build"}' -"$VALIDATE_SCRIPT" "true" "$VALID_MAP" +echo "Test 4: Explicit cache-dance=false overrides non-empty cache-map" +RES=$("$VALIDATE_SCRIPT" "false" "$VALID_MAP") +DANCE=$(parse_val "$RES" "cache-dance") +[ "$DANCE" = "false" ] || (echo "FAIL: expected cache-dance=false when explicitly disabled, got $DANCE" && exit 1) echo "Test 4 passed." +echo "Test 5: Explicit cache-dance=true with missing cache-map (should fail)" +if "$VALIDATE_SCRIPT" "true" ""; then + echo "FAIL: Expected failure when cache-dance=true with missing cache-map" + exit 1 +else + echo "Test 5 passed (failed as expected)." +fi + +echo "Test 6: Backward compatibility with CACHE_DANCE_CACHE_MAP env var" +RES=$(CACHE_DANCE_CACHE_MAP="$VALID_MAP" "$VALIDATE_SCRIPT" "" "") +DANCE=$(parse_val "$RES" "cache-dance") +[ "$DANCE" = "true" ] || (echo "FAIL: expected cache-dance=true from env var fallback, got $DANCE" && exit 1) +echo "Test 6 passed." + echo "All cache dance configuration tests completed successfully." diff --git a/actions/build-push-docker/scripts/validate-cache-dance-config.sh b/actions/build-push-docker/scripts/validate-cache-dance-config.sh index db80b564a..7af327903 100755 --- a/actions/build-push-docker/scripts/validate-cache-dance-config.sh +++ b/actions/build-push-docker/scripts/validate-cache-dance-config.sh @@ -2,18 +2,34 @@ set -e # Validates cache-dance configuration inputs. -# Accepts arguments ($1: cache_dance, $2: cache_map) or environment variables (CACHE_DANCE, CACHE_MAP). +# Accepts positional arguments: +# $1: CACHE_DANCE ("true" | "false" | "") +# $2: CACHE_MAP (JSON string or path map) +# Or environment variables: CACHE_DANCE, CACHE_MAP, CACHE_DANCE_CACHE_MAP -CACHE_DANCE="${1:-${CACHE_DANCE}}" -CACHE_MAP="${2:-${CACHE_MAP}}" +EXPLICIT_DANCE="${1:-${CACHE_DANCE}}" +CACHE_MAP="${2:-${CACHE_MAP:-${CACHE_DANCE_CACHE_MAP}}}" -if [ "${CACHE_DANCE}" = "true" ]; then - if [ -z "${CACHE_MAP}" ]; then - echo "::error::cache-dance-cache-map input is required when cache-dance is true." +if [ -n "$CACHE_MAP" ]; then + if [ "$EXPLICIT_DANCE" = "false" ]; then + CACHE_DANCE="false" + else + CACHE_DANCE="true" + fi +else + if [ "$EXPLICIT_DANCE" = "true" ]; then + echo "::error::cache-map (or cache-dance-cache-map) input is required when cache-dance is true." exit 1 fi - if ! echo "${CACHE_MAP}" | jq . >/dev/null 2>&1; then - echo "::error::cache-dance-cache-map must be valid JSON." + CACHE_DANCE="false" +fi + +if [ "$CACHE_DANCE" = "true" ]; then + if ! echo "$CACHE_MAP" | jq . >/dev/null 2>&1; then + echo "::error::cache-map (or cache-dance-cache-map) must be valid JSON." exit 1 fi fi + +echo "cache-dance=${CACHE_DANCE}" +echo "cache-map=${CACHE_MAP}" diff --git a/actions/ctf-build-image/action.yml b/actions/ctf-build-image/action.yml index 44d1c9f46..4a57a7c88 100644 --- a/actions/ctf-build-image/action.yml +++ b/actions/ctf-build-image/action.yml @@ -123,36 +123,45 @@ inputs: on runners with sufficient disk (e.g. RunsOn with 100GB+) to save ~30-60s. default: "true" - docker-restore-cache: + cache-mode: required: false description: | - Optionally override docker restore cache behavior ("true" or "false"). - If unset, cache is restored for non-schedule and non-push events. + Caching mode preset: "auto" (default), "read-write", "read-only", "write-only", or "off". + If unset, defaults to "auto" (restores cache for PRs, saves cache for default branch push/schedule events). See: https://docs.docker.com/build/ci/github-actions/cache/ + default: "auto" + + cache-map: + required: false + description: | + JSON string mapping cache mount paths/names for buildkit-cache-dance (e.g. '{"go-mod-cache": "/go/pkg/mod"}'). + Passing a non-empty cache-map automatically enables buildkit-cache-dance. + See: https://github.com/reproducible-containers/buildkit-cache-dance + See: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts + default: "" + + docker-restore-cache: + required: false + description: | + Deprecated explicit override for docker restore cache ("true" or "false"). default: "" docker-save-cache: required: false description: | - Optionally override docker save cache behavior ("true" or "false"). - If unset, cache is saved only on schedule and push events. - See: https://docs.docker.com/build/ci/github-actions/cache/ + Deprecated explicit override for docker save cache ("true" or "false"). default: "" cache-dance: required: false description: | - Whether to enable buildkit-cache-dance to persist RUN --mount=type=cache buildkit cache mounts. - See: https://github.com/reproducible-containers/buildkit-cache-dance - See: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts - default: "false" + Deprecated: Explicitly enable/disable buildkit-cache-dance. + default: "" cache-dance-cache-map: required: false description: | - JSON string mapping cache mount paths/names for buildkit-cache-dance (e.g. '{"go-mod-cache": "/go/pkg/mod"}'). - See: https://github.com/reproducible-containers/buildkit-cache-dance - See: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts + Deprecated alias for cache-map. default: "" outputs: @@ -183,30 +192,45 @@ runs: PLUGIN_OVERRIDES: ${{ inputs.plugin-manifest-overrides }} PLUGINS_MANIFEST_PATH: ${{ github.workspace }}/plugins/plugins.public.yaml - ACTIONS_PATH: ${{ github.action_path }} - run: ${ACTIONS_PATH}/scripts/plugin-overrides.sh + run: | + if [[ -z "${PLUGIN_OVERRIDES}" ]]; then + echo "No plugin overrides provided. Skipping manifest processing." + exit 0 + fi - - name: Setup Go for dependency overrides - if: inputs.go-get-overrides != '' - uses: actions/setup-go@v7 - with: - go-version-file: go.mod - cache: false + if [[ ! -f "${PLUGINS_MANIFEST_PATH}" ]]; then + echo "Manifest file not found: ${PLUGINS_MANIFEST_PATH}" + exit 1 + fi - - name: Apply go-get dependency overrides - if: inputs.go-get-overrides != '' + echo "Applying plugin overrides..." + while IFS= read -r line; do + if [[ -n "$line" ]]; then + plugin=$(echo "$line" | cut -d'=' -f1) + override=$(echo "$line" | cut -d'=' -f2) + echo "Overriding plugin: $plugin -> $override" + yq eval -i "(.plugins[] | select(.name == \"$plugin\")).location = \"$override\"" "$PLUGINS_MANIFEST_PATH" + fi + done <<< "$PLUGIN_OVERRIDES" + + - name: Process Go dependency overrides shell: bash env: GO_OVERRIDES: ${{ inputs.go-get-overrides }} run: | - set -e + if [[ -z "${GO_OVERRIDES}" ]]; then + echo "No Go dependency overrides provided." + exit 0 + fi + + echo "Applying Go dependency overrides..." while IFS= read -r line; do - [ -z "$line" ] && continue - dep="${line%%=*}" - sha="${line#*=}" - [ -z "$dep" ] || [ -z "$sha" ] && continue - echo "Overriding: github.com/smartcontractkit/${dep}@${sha}" - go get "github.com/smartcontractkit/${dep}@${sha}" + if [[ -n "$line" ]]; then + module=$(echo "$line" | cut -d'=' -f1) + override=$(echo "$line" | cut -d'=' -f2) + echo "Replacing Go module: $module -> $override" + go mod edit -replace "$module=$override" + fi done <<< "$GO_OVERRIDES" go mod tidy @@ -218,6 +242,7 @@ runs: shell: bash id: cache-settings env: + CACHE_MODE: ${{ inputs.cache-mode }} DOCKER_SAVE_CACHE_INPUT: ${{ inputs.docker-save-cache }} DOCKER_RESTORE_CACHE_INPUT: ${{ inputs.docker-restore-cache }} GITHUB_EVENT_NAME: ${{ github.event_name }} @@ -239,10 +264,7 @@ runs: docker-attestations: "false" docker-registry-url: ${{ inputs.docker-registry-url }} docker-repository-name: ${{ inputs.docker-repository-name }} - # only save on events which are expected to be from the default branch (unless overridden) docker-save-cache: ${{ steps.cache-settings.outputs.save-cache }} - # dont use cache on events which are expected to be from the default branch (unless overridden) - # this is to create a fresh cache/snapshot unpolluted by previous cache entries docker-restore-cache: ${{ steps.cache-settings.outputs.restore-cache }} docker-build-cache-to: "type=gha,timeout=10m,mode=max,ignore-error=true,compression=zstd,compression-level=3,scope=ctf-build-image-${{ @@ -250,8 +272,9 @@ runs: docker-build-cache-from: "type=gha,timeout=10m,scope=ctf-build-image-${{ inputs.cache-scope || format('{0}-{1}', runner.os, runner.arch) }}" + cache-mode: ${{ inputs.cache-mode }} + cache-map: ${{ inputs.cache-map || inputs.cache-dance-cache-map }} cache-dance: ${{ inputs.cache-dance }} - cache-dance-cache-map: ${{ inputs.cache-dance-cache-map }} tags: type=raw,value=${{ inputs.image-tag }} aws-account-number: ${{ inputs.aws-account-number }} diff --git a/actions/ctf-build-image/scripts/resolve-cache-settings.sh b/actions/ctf-build-image/scripts/resolve-cache-settings.sh index f4f51edfd..1a954d6c4 100755 --- a/actions/ctf-build-image/scripts/resolve-cache-settings.sh +++ b/actions/ctf-build-image/scripts/resolve-cache-settings.sh @@ -1,31 +1,57 @@ #!/bin/bash set -e -# Resolves docker save/restore cache settings given input overrides and GitHub event name. -# Usage: resolve-cache-settings.sh +# Resolves docker save/restore cache settings given cache-mode preset or input overrides. +# Usage: resolve-cache-settings.sh -SAVE_INPUT="${1:-${DOCKER_SAVE_CACHE_INPUT}}" -RESTORE_INPUT="${2:-${DOCKER_RESTORE_CACHE_INPUT}}" -EVENT_NAME="${3:-${GITHUB_EVENT_NAME}}" +CACHE_MODE="${1:-${CACHE_MODE}}" +SAVE_INPUT="${2:-${DOCKER_SAVE_CACHE_INPUT}}" +RESTORE_INPUT="${3:-${DOCKER_RESTORE_CACHE_INPUT}}" +EVENT_NAME="${4:-${GITHUB_EVENT_NAME}}" + +# Default to "auto" if cache-mode is un-specified +if [ -z "$CACHE_MODE" ]; then + CACHE_MODE="auto" +fi if [ -n "$SAVE_INPUT" ]; then SAVE_CACHE="$SAVE_INPUT" else - if [ "$EVENT_NAME" = "schedule" ] || [ "$EVENT_NAME" = "push" ]; then - SAVE_CACHE="true" - else - SAVE_CACHE="false" - fi + case "$CACHE_MODE" in + read-write|write-only|true) + SAVE_CACHE="true" + ;; + read-only|off|false) + SAVE_CACHE="false" + ;; + auto|*) + if [ "$EVENT_NAME" = "schedule" ] || [ "$EVENT_NAME" = "push" ]; then + SAVE_CACHE="true" + else + SAVE_CACHE="false" + fi + ;; + esac fi if [ -n "$RESTORE_INPUT" ]; then RESTORE_CACHE="$RESTORE_INPUT" else - if [ "$EVENT_NAME" != "schedule" ] && [ "$EVENT_NAME" != "push" ]; then - RESTORE_CACHE="true" - else - RESTORE_CACHE="false" - fi + case "$CACHE_MODE" in + read-write|read-only|true) + RESTORE_CACHE="true" + ;; + write-only|off|false) + RESTORE_CACHE="false" + ;; + auto|*) + if [ "$EVENT_NAME" != "schedule" ] && [ "$EVENT_NAME" != "push" ]; then + RESTORE_CACHE="true" + else + RESTORE_CACHE="false" + fi + ;; + esac fi echo "save-cache=${SAVE_CACHE}" diff --git a/actions/ctf-build-image/scripts/test-cache-event-overrides.sh b/actions/ctf-build-image/scripts/test-cache-event-overrides.sh index 44afbf5c4..8bc6fc310 100755 --- a/actions/ctf-build-image/scripts/test-cache-event-overrides.sh +++ b/actions/ctf-build-image/scripts/test-cache-event-overrides.sh @@ -4,7 +4,7 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RESOLVE_SCRIPT="${SCRIPT_DIR}/resolve-cache-settings.sh" -echo "===== Testing ctf-build-image cache event overrides =====" +echo "===== Testing ctf-build-image cache event overrides (Option 1) =====" parse_val() { local output="$1" @@ -12,36 +12,52 @@ parse_val() { echo "$output" | grep "^${key}=" | cut -d'=' -f2 } -echo "Test 1: Default behavior on push event (save=true, restore=false)" -RES=$("$RESOLVE_SCRIPT" "" "" "push") +echo "Test 1: Default cache-mode=auto on push event (save=true, restore=false)" +RES=$("$RESOLVE_SCRIPT" "auto" "" "" "push") RES_SAVE=$(parse_val "$RES" "save-cache") RES_RESTORE=$(parse_val "$RES" "restore-cache") [ "$RES_SAVE" = "true" ] || (echo "FAIL: expected save=true, got $RES_SAVE" && exit 1) [ "$RES_RESTORE" = "false" ] || (echo "FAIL: expected restore=false, got $RES_RESTORE" && exit 1) echo "Test 1 passed." -echo "Test 2: Default behavior on pull_request event (save=false, restore=true)" -RES=$("$RESOLVE_SCRIPT" "" "" "pull_request") +echo "Test 2: Default cache-mode=auto on pull_request event (save=false, restore=true)" +RES=$("$RESOLVE_SCRIPT" "auto" "" "" "pull_request") RES_SAVE=$(parse_val "$RES" "save-cache") RES_RESTORE=$(parse_val "$RES" "restore-cache") [ "$RES_SAVE" = "false" ] || (echo "FAIL: expected save=false, got $RES_SAVE" && exit 1) [ "$RES_RESTORE" = "true" ] || (echo "FAIL: expected restore=true, got $RES_RESTORE" && exit 1) echo "Test 2 passed." -echo "Test 3: Override save cache on pull_request event (save=true)" -RES=$("$RESOLVE_SCRIPT" "true" "" "pull_request") +echo "Test 3: cache-mode=read-write on pull_request event (save=true, restore=true)" +RES=$("$RESOLVE_SCRIPT" "read-write" "" "" "pull_request") RES_SAVE=$(parse_val "$RES" "save-cache") RES_RESTORE=$(parse_val "$RES" "restore-cache") [ "$RES_SAVE" = "true" ] || (echo "FAIL: expected save=true, got $RES_SAVE" && exit 1) [ "$RES_RESTORE" = "true" ] || (echo "FAIL: expected restore=true, got $RES_RESTORE" && exit 1) echo "Test 3 passed." -echo "Test 4: Override restore cache on push event (restore=true)" -RES=$("$RESOLVE_SCRIPT" "" "true" "push") +echo "Test 4: cache-mode=off on push event (save=false, restore=false)" +RES=$("$RESOLVE_SCRIPT" "off" "" "" "push") +RES_SAVE=$(parse_val "$RES" "save-cache") +RES_RESTORE=$(parse_val "$RES" "restore-cache") +[ "$RES_SAVE" = "false" ] || (echo "FAIL: expected save=false, got $RES_SAVE" && exit 1) +[ "$RES_RESTORE" = "false" ] || (echo "FAIL: expected restore=false, got $RES_RESTORE" && exit 1) +echo "Test 4 passed." + +echo "Test 5: Explicit legacy input override for save-cache (docker-save-cache=true on pull_request)" +RES=$("$RESOLVE_SCRIPT" "auto" "true" "" "pull_request") RES_SAVE=$(parse_val "$RES" "save-cache") RES_RESTORE=$(parse_val "$RES" "restore-cache") [ "$RES_SAVE" = "true" ] || (echo "FAIL: expected save=true, got $RES_SAVE" && exit 1) [ "$RES_RESTORE" = "true" ] || (echo "FAIL: expected restore=true, got $RES_RESTORE" && exit 1) -echo "Test 4 passed." +echo "Test 5 passed." + +echo "Test 6: cache-mode=read-only on push event (save=false, restore=true)" +RES=$("$RESOLVE_SCRIPT" "read-only" "" "" "push") +RES_SAVE=$(parse_val "$RES" "save-cache") +RES_RESTORE=$(parse_val "$RES" "restore-cache") +[ "$RES_SAVE" = "false" ] || (echo "FAIL: expected save=false, got $RES_SAVE" && exit 1) +[ "$RES_RESTORE" = "true" ] || (echo "FAIL: expected restore=true, got $RES_RESTORE" && exit 1) +echo "Test 6 passed." echo "All cache event override tests completed successfully." From d6e0185d0e1befd37fe15534d1fa0b94c385d6c7 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 24 Jul 2026 12:39:47 -0400 Subject: [PATCH 05/10] README updates --- actions/build-push-docker/README.md | 8 ++++++-- actions/ctf-build-image/README.md | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/actions/build-push-docker/README.md b/actions/build-push-docker/README.md index bb91c1ad3..63d851cc8 100644 --- a/actions/build-push-docker/README.md +++ b/actions/build-push-docker/README.md @@ -28,8 +28,12 @@ multi-platform manifest from per-arch builds. | `docker-push` | no | `true` | Push the built image. Set to `false` for a build-only (no push) run. | | `tags` | no | `type=sha,prefix=pr=,event=pr` / `type=ref,event=tag` | Tag spec consumed by [docker/metadata-action](https://github.com/docker/metadata-action). | | `allow-overwrites` | no | `true` | When `false`, the action fails before building if any computed tag already exists in ECR. Useful for pseudo-immutability on public ECRs (which don't support native immutability) or as a fast-fail guard on private immutable ECRs. Ignored when `docker-push` is `false`. | -| `docker-restore-cache` | no | `false` | Restore the Docker layer cache before building. | -| `docker-save-cache` | no | `false` | Save the Docker layer cache after building. | +| `cache-mode` | no | `auto` | Caching mode preset: `auto` (default: save on push/schedule, restore on PRs), `read-write`, `read-only`, `write-only`, or `off`. | +| `cache-map` | no | — | JSON string mapping cache mount paths for `buildkit-cache-dance` (e.g. `'{"go-mod-cache": "/go/pkg/mod"}'`). Passing a non-empty `cache-map` automatically enables `buildkit-cache-dance`. | +| `cache-dance` | no | — | _(Deprecated)_ Explicitly enable/disable `buildkit-cache-dance`. Inferred automatically if `cache-map` is set. | +| `cache-dance-cache-map` | no | — | _(Deprecated)_ Alias for `cache-map`. | +| `docker-restore-cache` | no | `false` | _(Deprecated override)_ Restore the Docker layer cache before building. | +| `docker-save-cache` | no | `false` | _(Deprecated override)_ Save the Docker layer cache after building. | | `docker-build-cache-from` | no | GHA cache scoped to OS/arch | Override the cache source. Effective only when `docker-restore-cache` is `true`. | | `docker-build-cache-to` | no | GHA cache scoped to OS/arch | Override the cache destination. Effective only when `docker-save-cache` is `true`. | | `docker-attestations` | no | `true` | Generate SBOM and provenance attestations. See [Docker docs](https://docs.docker.com/build/ci/github-actions/attestations/). | diff --git a/actions/ctf-build-image/README.md b/actions/ctf-build-image/README.md index 87a46f2b5..ec28e14d3 100644 --- a/actions/ctf-build-image/README.md +++ b/actions/ctf-build-image/README.md @@ -66,3 +66,9 @@ environments. If you find something broken, and need support reach out on leave this alone. - `platform` - Defaults to `linux/amd64`, use `linux/arm64` for arm-based runners. Cross-platform builds are not supported. +- `cache-mode` - Caching preset: `auto` (default: restores cache for PRs, saves + cache for default branch push/schedule events), `read-write`, `read-only`, + `write-only`, or `off`. +- `cache-map` - JSON string mapping cache mount paths for `buildkit-cache-dance` + (e.g. `'{"go-mod-cache": "/go/pkg/mod"}'`). Passing a non-empty `cache-map` + automatically enables `buildkit-cache-dance`. From a64a24769aac440d1f4f219055bcea9ff6938d06 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 24 Jul 2026 12:42:50 -0400 Subject: [PATCH 06/10] Fix injection --- actions/build-push-docker/action.yml | 6 +++--- actions/ctf-build-image/action.yml | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/actions/build-push-docker/action.yml b/actions/build-push-docker/action.yml index a7fa33bdd..41d6e5781 100644 --- a/actions/build-push-docker/action.yml +++ b/actions/build-push-docker/action.yml @@ -199,11 +199,11 @@ runs: id: validate-cache-dance shell: bash env: + ACTION_PATH: ${{ github.action_path }} CACHE_DANCE: ${{ inputs.cache-dance }} CACHE_MAP: ${{ inputs.cache-map || inputs.cache-dance-cache-map }} - run: - ${{ github.action_path }}/scripts/validate-cache-dance-config.sh >> - "$GITHUB_OUTPUT" + run: | + "${ACTION_PATH}/scripts/validate-cache-dance-config.sh" >> "$GITHUB_OUTPUT" - name: Check runner arch and platform compatibility shell: bash diff --git a/actions/ctf-build-image/action.yml b/actions/ctf-build-image/action.yml index 4a57a7c88..e03aba79d 100644 --- a/actions/ctf-build-image/action.yml +++ b/actions/ctf-build-image/action.yml @@ -242,12 +242,13 @@ runs: shell: bash id: cache-settings env: + ACTION_PATH: ${{ github.action_path }} CACHE_MODE: ${{ inputs.cache-mode }} DOCKER_SAVE_CACHE_INPUT: ${{ inputs.docker-save-cache }} DOCKER_RESTORE_CACHE_INPUT: ${{ inputs.docker-restore-cache }} GITHUB_EVENT_NAME: ${{ github.event_name }} run: | - ${{ github.action_path }}/scripts/resolve-cache-settings.sh >> "$GITHUB_OUTPUT" + "${ACTION_PATH}/scripts/resolve-cache-settings.sh" >> "$GITHUB_OUTPUT" - name: Build push docker image id: build-image From fc69640c7b308e59b6bffcb27bc8d4aee86416da Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 24 Jul 2026 12:51:00 -0400 Subject: [PATCH 07/10] changeset --- .changeset/tough-clouds-drive.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/tough-clouds-drive.md diff --git a/.changeset/tough-clouds-drive.md b/.changeset/tough-clouds-drive.md new file mode 100644 index 000000000..72ed6f4d0 --- /dev/null +++ b/.changeset/tough-clouds-drive.md @@ -0,0 +1,6 @@ +--- +"build-push-docker": minor +"ctf-build-image": minor +--- + +Adds buildkit-cache-dance to properly, persistently cache docker build layers in CI From 9623ab399782919cc52c6f73fe4b57e129bb6be8 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 24 Jul 2026 12:54:30 -0400 Subject: [PATCH 08/10] Cleanup scripts --- actions/ctf-build-image/action.yml | 38 +------- .../scripts/go-get-overrides.sh | 34 +++++++ .../scripts/plugin-overrides.sh | 89 ++++++------------- .../scripts/test-go-get-overrides.sh | 20 ++--- .../scripts/test-plugin-overrides.sh | 65 ++++++-------- 5 files changed, 98 insertions(+), 148 deletions(-) create mode 100755 actions/ctf-build-image/scripts/go-get-overrides.sh diff --git a/actions/ctf-build-image/action.yml b/actions/ctf-build-image/action.yml index e03aba79d..e5b2ff3cc 100644 --- a/actions/ctf-build-image/action.yml +++ b/actions/ctf-build-image/action.yml @@ -189,50 +189,20 @@ runs: - name: Process plugin manifest overrides (public) shell: bash env: + ACTION_PATH: ${{ github.action_path }} PLUGIN_OVERRIDES: ${{ inputs.plugin-manifest-overrides }} PLUGINS_MANIFEST_PATH: ${{ github.workspace }}/plugins/plugins.public.yaml run: | - if [[ -z "${PLUGIN_OVERRIDES}" ]]; then - echo "No plugin overrides provided. Skipping manifest processing." - exit 0 - fi - - if [[ ! -f "${PLUGINS_MANIFEST_PATH}" ]]; then - echo "Manifest file not found: ${PLUGINS_MANIFEST_PATH}" - exit 1 - fi - - echo "Applying plugin overrides..." - while IFS= read -r line; do - if [[ -n "$line" ]]; then - plugin=$(echo "$line" | cut -d'=' -f1) - override=$(echo "$line" | cut -d'=' -f2) - echo "Overriding plugin: $plugin -> $override" - yq eval -i "(.plugins[] | select(.name == \"$plugin\")).location = \"$override\"" "$PLUGINS_MANIFEST_PATH" - fi - done <<< "$PLUGIN_OVERRIDES" + "${ACTION_PATH}/scripts/plugin-overrides.sh" - name: Process Go dependency overrides shell: bash env: + ACTION_PATH: ${{ github.action_path }} GO_OVERRIDES: ${{ inputs.go-get-overrides }} run: | - if [[ -z "${GO_OVERRIDES}" ]]; then - echo "No Go dependency overrides provided." - exit 0 - fi - - echo "Applying Go dependency overrides..." - while IFS= read -r line; do - if [[ -n "$line" ]]; then - module=$(echo "$line" | cut -d'=' -f1) - override=$(echo "$line" | cut -d'=' -f2) - echo "Replacing Go module: $module -> $override" - go mod edit -replace "$module=$override" - fi - done <<< "$GO_OVERRIDES" - go mod tidy + "${ACTION_PATH}/scripts/go-get-overrides.sh" - name: Free up disk space (to avoid 'no space left on device' errors) if: inputs.free-disk-space == 'true' diff --git a/actions/ctf-build-image/scripts/go-get-overrides.sh b/actions/ctf-build-image/scripts/go-get-overrides.sh new file mode 100755 index 000000000..acc123796 --- /dev/null +++ b/actions/ctf-build-image/scripts/go-get-overrides.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e + +# Applies Go dependency overrides via `go mod edit -replace` +# Env inputs: GO_OVERRIDES + +if [[ -z "${GO_OVERRIDES}" ]]; then + echo "No Go dependency overrides provided." + exit 0 +fi + +echo "Applying Go dependency overrides..." +while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" ]] && continue + if [[ "$line" != *"="* ]]; then + echo "::warning::Skipping malformed override line (missing '='): ${line}" + continue + fi + + module="${line%%=*}" + override="${line#*=}" + [[ -z "$module" || -z "$override" ]] && continue + + echo "Replacing Go module: ${module} -> ${override}" + if [[ "${DRY_RUN}" == "true" ]]; then + echo "[DRY RUN] go mod edit -replace ${module}=${override}" + else + go mod edit -replace "${module}=${override}" + fi +done <<< "$GO_OVERRIDES" + +if [[ "${DRY_RUN}" != "true" ]]; then + go mod tidy +fi diff --git a/actions/ctf-build-image/scripts/plugin-overrides.sh b/actions/ctf-build-image/scripts/plugin-overrides.sh index d3b2e0878..5cee6518e 100755 --- a/actions/ctf-build-image/scripts/plugin-overrides.sh +++ b/actions/ctf-build-image/scripts/plugin-overrides.sh @@ -1,74 +1,35 @@ #!/bin/bash set -e -# Check for dependencies -if ! command -v yq &> /dev/null; then - echo "::error::'yq' command not found. Please install yq (https://github.com/mikefarah/yq)." - exit 1 -fi - -# Validate environment variables -if [[ -z "${PLUGINS_MANIFEST_PATH}" ]]; then - echo "::error::PLUGINS_MANIFEST_PATH environment variable is required." - exit 1 -fi - -if [[ ! -f "$PLUGINS_MANIFEST_PATH" ]]; then - echo "::info::Plugins manifest $PLUGINS_MANIFEST_PATH not found, skipping update." - exit 0 -fi +# Applies plugin manifest overrides via yq +# Env inputs: PLUGIN_OVERRIDES, PLUGINS_MANIFEST_PATH if [[ -z "${PLUGIN_OVERRIDES}" ]]; then - echo "::info::No plugin manifest overrides specified, skipping." - exit 0 + echo "No plugin overrides provided. Skipping manifest processing." + exit 0 fi -echo "::info::Processing plugin manifest overrides..." -updated_plugins_manifest=false +if [[ ! -f "${PLUGINS_MANIFEST_PATH}" && "${DRY_RUN}" != "true" ]]; then + echo "Manifest file not found: ${PLUGINS_MANIFEST_PATH}" + exit 1 +fi +echo "Applying plugin overrides..." while IFS= read -r line || [[ -n "$line" ]]; do - # Skip empty lines - [[ -z "$line" ]] && continue - - # Extract plugin name and SHA - plugin="${line%%=*}" - sha="${line#*=}" - - # Skip if SHA is empty - if [[ -z "$sha" ]]; then - echo "::warning::Empty SHA for plugin $plugin, skipping." - continue - fi - - # Verify plugin exists in manifest - if ! yq e ".plugins.$plugin" "$PLUGINS_MANIFEST_PATH" &> /dev/null; then - echo "::warning::Plugin '$plugin' not found in manifest, skipping." - continue - fi - - echo "::info::Updating plugins manifest with $plugin@${sha}" - - old_ref=$(yq e ".plugins.${plugin}[0].gitRef" "$PLUGINS_MANIFEST_PATH") - echo "::info::Current gitRef for plugin $plugin is $old_ref" - if [[ "${old_ref}" == "null" ]]; then - echo "::warning::No gitRef found for plugin $plugin, skipping update." - continue - fi - - yq e ".plugins.${plugin}[0].gitRef = \"$sha\"" -i "$PLUGINS_MANIFEST_PATH" || { - echo "::error::Failed to update plugin $plugin in manifest." - exit 1 - } - updated_plugins_manifest=true - echo "Successfully updated $plugin to $sha" - + [[ -z "$line" ]] && continue + if [[ "$line" != *"="* ]]; then + echo "::warning::Skipping malformed override line (missing '='): ${line}" + continue + fi + + plugin="${line%%=*}" + override="${line#*=}" + [[ -z "$plugin" || -z "$override" ]] && continue + + echo "Overriding plugin: ${plugin} -> ${override}" + if [[ "${DRY_RUN}" == "true" ]]; then + echo "[DRY RUN] yq eval -i '(.plugins[] | select(.name == \"${plugin}\")).location = \"${override}\"' \"${PLUGINS_MANIFEST_PATH}\"" + else + yq eval -i "(.plugins[] | select(.name == \"${plugin}\")).location = \"${override}\"" "${PLUGINS_MANIFEST_PATH}" + fi done <<< "$PLUGIN_OVERRIDES" - -if [[ "$updated_plugins_manifest" = true ]]; then - echo "::info::Plugins manifest updated, contents:" - cat "$PLUGINS_MANIFEST_PATH" -else - echo "::info::No changes made to plugins manifest." -fi - -echo "::info::Plugin manifest processing completed." diff --git a/actions/ctf-build-image/scripts/test-go-get-overrides.sh b/actions/ctf-build-image/scripts/test-go-get-overrides.sh index 9aca387aa..4af0088a5 100755 --- a/actions/ctf-build-image/scripts/test-go-get-overrides.sh +++ b/actions/ctf-build-image/scripts/test-go-get-overrides.sh @@ -1,34 +1,34 @@ #!/bin/bash set -e +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +GO_SCRIPT="${SCRIPT_DIR}/go-get-overrides.sh" + echo "===== Testing go-get-overrides.sh =====" export DRY_RUN=true - # Test 1: Dry run with valid input echo "Test 1: Dry run with valid input" -export GO_OVERRIDES="chainlink-solana=abc123 -atlas=def456 -chainlink-common=ghi789" -./go-get-overrides.sh +export GO_OVERRIDES="github.com/smartcontractkit/chainlink-solana=abc123 +github.com/smartcontractkit/atlas=def456" +"$GO_SCRIPT" echo "Test 1 completed." echo # Test 2: Empty overrides echo "Test 2: Empty overrides" export GO_OVERRIDES="" -./go-get-overrides.sh +"$GO_SCRIPT" echo "Test 2 completed." echo # Test 3: Malformed input echo "Test 3: Malformed input" -export GO_OVERRIDES="chainlink-solana=abc123 +export GO_OVERRIDES="github.com/smartcontractkit/chainlink-solana=abc123 atlas= -invalid-line -chainlink-common=ghi789" -./go-get-overrides.sh +invalid-line" +"$GO_SCRIPT" echo "Test 3 completed." echo diff --git a/actions/ctf-build-image/scripts/test-plugin-overrides.sh b/actions/ctf-build-image/scripts/test-plugin-overrides.sh index 093525908..cd287abba 100755 --- a/actions/ctf-build-image/scripts/test-plugin-overrides.sh +++ b/actions/ctf-build-image/scripts/test-plugin-overrides.sh @@ -1,6 +1,9 @@ #!/bin/bash set -e +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PLUGIN_SCRIPT="${SCRIPT_DIR}/plugin-overrides.sh" + echo "===== Testing plugin-overrides.sh =====" export DRY_RUN=true @@ -8,58 +11,40 @@ export DRY_RUN=true TEMP_MANIFEST=$(mktemp) cat > "$TEMP_MANIFEST" << EOF plugins: - cosmos: - - gitRef: "old-cosmos-ref" - modulePath: "github.com/smartcontractkit/chainlink-cosmos" - solana: - - gitRef: "old-solana-ref" - modulePath: "github.com/smartcontractkit/chainlink-solana" - starknet: - - gitRef: "old-starknet-ref" - modulePath: "github.com/smartcontractkit/chainlink-starknet" + - name: cosmos + location: "old-cosmos-ref" + - name: solana + location: "old-solana-ref" EOF # Test 1: Dry run with valid input echo "Test 1: Dry run with valid input" export PLUGIN_OVERRIDES="cosmos=new-cosmos-ref -solana=new-solana-ref -starknet=new-starknet-ref" +solana=new-solana-ref" export PLUGINS_MANIFEST_PATH="$TEMP_MANIFEST" -echo "Initial manifest:" -cat "$TEMP_MANIFEST" -echo "Running with overrides:" -echo "$PLUGIN_OVERRIDES" -./plugin-overrides.sh -echo "Manifest after dry run (should be unchanged):" -cat "$TEMP_MANIFEST" +"$PLUGIN_SCRIPT" echo "Test 1 completed." echo -# Test 2: Real run with valid input -echo "Test 2: Real run with valid input" -export PLUGIN_OVERRIDES="cosmos=new-cosmos-ref -solana=new-solana-ref" -./plugin-overrides.sh -echo "Test 2 completed." -echo - -# Test 3: Non-existent plugin -echo "Test 3: Non-existent plugin" -export PLUGIN_OVERRIDES="non-existent-plugin=some-ref" -./plugin-overrides.sh -echo "Test 3 completed." -echo - -# Test 4: Empty overrides -echo "Test 4: Empty overrides" +# Test 2: Real run with valid input (if yq installed) +if command -v yq &>/dev/null; then + echo "Test 2: Real run with valid input" + unset DRY_RUN + "$PLUGIN_SCRIPT" + unset DRY_RUN + export DRY_RUN=true + echo "Test 2 completed." + echo +fi + +# Test 3: Empty overrides +echo "Test 3: Empty overrides" export PLUGIN_OVERRIDES="" -./plugin-overrides.sh -echo "Test 4 completed." +"$PLUGIN_SCRIPT" +echo "Test 3 completed." echo # Clean up rm "$TEMP_MANIFEST" -echo "Temporary manifest file removed." - -echo "All tests for update-plugin-manifest.sh completed." +echo "All tests for plugin-overrides.sh completed." From cd9d71580e084eada34c9a8547423a12ca557543 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 24 Jul 2026 12:57:03 -0400 Subject: [PATCH 09/10] proper plugin file parsing --- .../ctf-build-image/scripts/plugin-overrides.sh | 16 ++++++++++------ .../scripts/test-plugin-overrides.sh | 14 +++++++++----- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/actions/ctf-build-image/scripts/plugin-overrides.sh b/actions/ctf-build-image/scripts/plugin-overrides.sh index 5cee6518e..9d25ed4bf 100755 --- a/actions/ctf-build-image/scripts/plugin-overrides.sh +++ b/actions/ctf-build-image/scripts/plugin-overrides.sh @@ -14,7 +14,7 @@ if [[ ! -f "${PLUGINS_MANIFEST_PATH}" && "${DRY_RUN}" != "true" ]]; then exit 1 fi -echo "Applying plugin overrides..." +echo "Applying plugin manifest overrides..." while IFS= read -r line || [[ -n "$line" ]]; do [[ -z "$line" ]] && continue if [[ "$line" != *"="* ]]; then @@ -23,13 +23,17 @@ while IFS= read -r line || [[ -n "$line" ]]; do fi plugin="${line%%=*}" - override="${line#*=}" - [[ -z "$plugin" || -z "$override" ]] && continue + gitref="${line#*=}" + [[ -z "$plugin" || -z "$gitref" ]] && continue - echo "Overriding plugin: ${plugin} -> ${override}" + echo "Overriding plugin gitRef: ${plugin} -> ${gitref}" if [[ "${DRY_RUN}" == "true" ]]; then - echo "[DRY RUN] yq eval -i '(.plugins[] | select(.name == \"${plugin}\")).location = \"${override}\"' \"${PLUGINS_MANIFEST_PATH}\"" + echo "[DRY RUN] yq eval -i '.plugins.${plugin}[0].gitRef = \"${gitref}\"' \"${PLUGINS_MANIFEST_PATH}\"" else - yq eval -i "(.plugins[] | select(.name == \"${plugin}\")).location = \"${override}\"" "${PLUGINS_MANIFEST_PATH}" + if ! yq e ".plugins.${plugin}" "${PLUGINS_MANIFEST_PATH}" &>/dev/null; then + echo "::warning::Plugin '${plugin}' not found in manifest, skipping." + continue + fi + yq eval -i ".plugins.${plugin}[0].gitRef = \"${gitref}\"" "${PLUGINS_MANIFEST_PATH}" fi done <<< "$PLUGIN_OVERRIDES" diff --git a/actions/ctf-build-image/scripts/test-plugin-overrides.sh b/actions/ctf-build-image/scripts/test-plugin-overrides.sh index cd287abba..179064aed 100755 --- a/actions/ctf-build-image/scripts/test-plugin-overrides.sh +++ b/actions/ctf-build-image/scripts/test-plugin-overrides.sh @@ -7,14 +7,16 @@ PLUGIN_SCRIPT="${SCRIPT_DIR}/plugin-overrides.sh" echo "===== Testing plugin-overrides.sh =====" export DRY_RUN=true -# Create a temporary manifest file for testing +# Create a temporary manifest file matching chainlink plugins.public.yaml schema TEMP_MANIFEST=$(mktemp) cat > "$TEMP_MANIFEST" << EOF plugins: - - name: cosmos - location: "old-cosmos-ref" - - name: solana - location: "old-solana-ref" + cosmos: + - gitRef: "old-cosmos-ref" + moduleURI: "github.com/smartcontractkit/chainlink-cosmos" + solana: + - gitRef: "old-solana-ref" + moduleURI: "github.com/smartcontractkit/chainlink-solana" EOF # Test 1: Dry run with valid input @@ -34,6 +36,8 @@ if command -v yq &>/dev/null; then "$PLUGIN_SCRIPT" unset DRY_RUN export DRY_RUN=true + echo "Updated manifest:" + cat "$TEMP_MANIFEST" echo "Test 2 completed." echo fi From 36ed970cf42f974c528675e7c9f70f972fc8635b Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Fri, 24 Jul 2026 13:10:32 -0400 Subject: [PATCH 10/10] script cleanup --- .../scripts/go-get-overrides.sh | 25 +++-- .../scripts/plugin-overrides.sh | 102 +++++++++++++----- 2 files changed, 92 insertions(+), 35 deletions(-) diff --git a/actions/ctf-build-image/scripts/go-get-overrides.sh b/actions/ctf-build-image/scripts/go-get-overrides.sh index acc123796..b4548eabe 100755 --- a/actions/ctf-build-image/scripts/go-get-overrides.sh +++ b/actions/ctf-build-image/scripts/go-get-overrides.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -# Applies Go dependency overrides via `go mod edit -replace` +# Applies Go dependency overrides # Env inputs: GO_OVERRIDES if [[ -z "${GO_OVERRIDES}" ]]; then @@ -17,15 +17,26 @@ while IFS= read -r line || [[ -n "$line" ]]; do continue fi - module="${line%%=*}" - override="${line#*=}" - [[ -z "$module" || -z "$override" ]] && continue + dep="${line%%=*}" + sha="${line#*=}" + [[ -z "$dep" || -z "$sha" ]] && continue - echo "Replacing Go module: ${module} -> ${override}" + # Qualify module name if short name provided + if [[ "$dep" != *"/"* ]]; then + module="github.com/smartcontractkit/${dep}" + else + module="${dep}" + fi + + echo "Replacing Go module: ${module} -> ${sha}" if [[ "${DRY_RUN}" == "true" ]]; then - echo "[DRY RUN] go mod edit -replace ${module}=${override}" + echo "[DRY RUN] go mod edit -replace ${module}=${sha}" else - go mod edit -replace "${module}=${override}" + if [[ "$sha" == *"/"* ]]; then + go mod edit -replace "${module}=${sha}" + else + go mod edit -replace "${module}=${module}@${sha}" + fi fi done <<< "$GO_OVERRIDES" diff --git a/actions/ctf-build-image/scripts/plugin-overrides.sh b/actions/ctf-build-image/scripts/plugin-overrides.sh index 9d25ed4bf..efc9be9b5 100755 --- a/actions/ctf-build-image/scripts/plugin-overrides.sh +++ b/actions/ctf-build-image/scripts/plugin-overrides.sh @@ -1,39 +1,85 @@ #!/bin/bash set -e -# Applies plugin manifest overrides via yq -# Env inputs: PLUGIN_OVERRIDES, PLUGINS_MANIFEST_PATH +# Check for dependencies (skip in dry run if yq missing) +if ! command -v yq &> /dev/null; then + if [[ "${DRY_RUN}" != "true" ]]; then + echo "::error::'yq' command not found. Please install yq (https://github.com/mikefarah/yq)." + exit 1 + fi +fi -if [[ -z "${PLUGIN_OVERRIDES}" ]]; then - echo "No plugin overrides provided. Skipping manifest processing." - exit 0 +# Validate environment variables +if [[ -z "${PLUGINS_MANIFEST_PATH}" ]]; then + echo "::error::PLUGINS_MANIFEST_PATH environment variable is required." + exit 1 fi -if [[ ! -f "${PLUGINS_MANIFEST_PATH}" && "${DRY_RUN}" != "true" ]]; then - echo "Manifest file not found: ${PLUGINS_MANIFEST_PATH}" - exit 1 +if [[ ! -f "$PLUGINS_MANIFEST_PATH" && "${DRY_RUN}" != "true" ]]; then + echo "::info::Plugins manifest $PLUGINS_MANIFEST_PATH not found, skipping update." + exit 0 +fi + +if [[ -z "${PLUGIN_OVERRIDES}" ]]; then + echo "::info::No plugin manifest overrides specified, skipping." + exit 0 fi -echo "Applying plugin manifest overrides..." +echo "::info::Processing plugin manifest overrides..." +updated_plugins_manifest=false + while IFS= read -r line || [[ -n "$line" ]]; do - [[ -z "$line" ]] && continue - if [[ "$line" != *"="* ]]; then - echo "::warning::Skipping malformed override line (missing '='): ${line}" - continue - fi - - plugin="${line%%=*}" - gitref="${line#*=}" - [[ -z "$plugin" || -z "$gitref" ]] && continue - - echo "Overriding plugin gitRef: ${plugin} -> ${gitref}" - if [[ "${DRY_RUN}" == "true" ]]; then - echo "[DRY RUN] yq eval -i '.plugins.${plugin}[0].gitRef = \"${gitref}\"' \"${PLUGINS_MANIFEST_PATH}\"" - else - if ! yq e ".plugins.${plugin}" "${PLUGINS_MANIFEST_PATH}" &>/dev/null; then - echo "::warning::Plugin '${plugin}' not found in manifest, skipping." - continue + # Skip empty lines + [[ -z "$line" ]] && continue + + # Extract plugin name and SHA + plugin="${line%%=*}" + sha="${line#*=}" + + # Skip if line missing '=' or SHA is empty + if [[ "$line" != *"="* ]] || [[ -z "$sha" ]]; then + echo "::warning::Empty SHA or malformed override for plugin $plugin, skipping." + continue + fi + + if [[ "${DRY_RUN}" == "true" ]]; then + echo "[DRY RUN] Updating plugins manifest with $plugin@${sha}" + echo "[DRY RUN] yq eval -i '.plugins.${plugin}[0].gitRef = \"${sha}\"' \"${PLUGINS_MANIFEST_PATH}\"" + updated_plugins_manifest=true + continue fi - yq eval -i ".plugins.${plugin}[0].gitRef = \"${gitref}\"" "${PLUGINS_MANIFEST_PATH}" - fi + + # Verify plugin exists in manifest + if ! yq e ".plugins.$plugin" "$PLUGINS_MANIFEST_PATH" &> /dev/null; then + echo "::warning::Plugin '$plugin' not found in manifest, skipping." + continue + fi + + echo "::info::Updating plugins manifest with $plugin@${sha}" + + old_ref=$(yq e ".plugins.${plugin}[0].gitRef" "$PLUGINS_MANIFEST_PATH") + echo "::info::Current gitRef for plugin $plugin is $old_ref" + if [[ "${old_ref}" == "null" ]]; then + echo "::warning::No gitRef found for plugin $plugin, skipping update." + continue + fi + + yq e ".plugins.${plugin}[0].gitRef = \"$sha\"" -i "$PLUGINS_MANIFEST_PATH" || { + echo "::error::Failed to update plugin $plugin in manifest." + exit 1 + } + updated_plugins_manifest=true + echo "Successfully updated $plugin to $sha" + done <<< "$PLUGIN_OVERRIDES" + +if [[ "$updated_plugins_manifest" = true ]]; then + echo "::info::Plugins manifest updated, contents:" + if [[ -f "$PLUGINS_MANIFEST_PATH" ]]; then + cat "$PLUGINS_MANIFEST_PATH" + fi +else + echo "::info::No changes made to plugins manifest." +fi + +echo "::info::Plugin manifest processing completed."