[DX-4045] Better Docker Caching

.dockerignore

@@ -2,7 +2,21 @@
 .git
 .DS_Store
 .envrc
+.dbenv
+.npmrc
+.gitattributes
+.gitignore
+.dockerignore
+.golangci.yml
+.mockery.yaml
+.nancy-ignore
+.tarignore
+.tool-versions
+nix.conf
+nix-darwin-shell-hook.sh
+LICENSE
 *.log
+**/*_test.go
 node_modules/
 **/node_modules/
 vendor/
@@ -13,9 +27,20 @@ examples/
 
 .changeset/
 .github/
+.vscode/
+.claude/
+.cursor/
 docs/
 fuzz/
 
+*.md
+sonar-project.properties
+pnpm-lock.yaml
+flake.lock
+flake.nix
+shell.nix
+
+
 devenv/
 deployment/
 integration/
@@ -45,3 +70,8 @@ gcr_creds.env
 
 go.work
 go.work.sum
+
+# buildkit-cache-dance host dirs (cache-dir / scratch-dir). These hold the injected
+# Go module + build caches and must never enter the build context.
+cache-mount/
+scratch/

.github/workflows/integration-tests.yml

@@ -348,14 +348,19 @@ jobs:
           aws-region: ${{ secrets.QA_AWS_REGION }}
           aws-role-arn: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           free-disk-space: "false"
-          docker-additional-build-args: |
-            CL_IS_PROD_BUILD=false
           # go-get-overrides is empty when inputs.evm-ref is unset so ctf-build-image skips an unnecessary setup-go / go mod tidy.
           go-get-overrides: >-
             ${{ inputs.evm-ref && format('chainlink-evm={0}', inputs.evm-ref) || '' }}
           gati-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }}
           gati-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
+
           cache-scope: ${{ matrix.image.cache-scope }}
+          # Persist only the Go build cache (GOCACHE), not the module cache.
+          # Module cache is large, and doesn't save us much time,
+          # at least compared to just using the build cache
+          # It also has mysterious failures when trying to upload
+          cache-map: |
+            {"cache-mount/go-build-cache": {"target": "/var/cache-target", "id": "go-build-cache"}}
 
   run-core-cre-e2e-tests-setup:
     name: Run Core CRE E2E Tests Setup

.gitignore

@@ -138,4 +138,4 @@ core/scripts/cre/environment/binaries/*
 .claude/
 .cursor/
 .gemini/
-opencode.json
+opencode.json

core/chainlink.Dockerfile

@@ -7,35 +7,45 @@
 # here so that source-only changes never invalidate their layer cache.
 FROM golang:1.26.4-bookworm AS deps-base
 RUN go version
-RUN apt-get update && apt-get install -y jq && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y --no-install-recommends jq=1.6-2.1+deb12u2 && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /chainlink
 
-ADD go.mod go.sum ./
+COPY go.mod go.sum ./
 COPY plugins/scripts/setup_git_auth.sh ./plugins/scripts/
 
 # CL_GOPRIVATE: set to "github.com/smartcontractkit/*" when building images
 # that depend on private Go modules (e.g. chainlink-internal-solana). When
 # empty (the default), go mod download uses the public module proxy as usual.
 ARG CL_GOPRIVATE=""
-ENV GOPRIVATE="${CL_GOPRIVATE}"
+ENV GOPRIVATE="${CL_GOPRIVATE}" \
+    GOCACHE=/root/.cache/go-build \
+    GOMODCACHE=/go/pkg/mod
 RUN --mount=type=secret,id=GIT_AUTH_TOKEN \
-    set -e && \
-    export GIT_CONFIG_GLOBAL=/tmp/gitconfig-go-mod-download && \
-    trap 'rm -f "$GIT_CONFIG_GLOBAL"' EXIT && \
-    ./plugins/scripts/setup_git_auth.sh && \
-    go mod download
+    --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \
+    --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \
+    set -e \
+    && export GIT_CONFIG_GLOBAL=/tmp/gitconfig-go-mod-download \
+    && trap 'rm -f "$GIT_CONFIG_GLOBAL"' EXIT \
+    && ./plugins/scripts/setup_git_auth.sh \
+    && go mod download
 
 COPY GNUmakefile package.json ./
 COPY tools/bin/ldflags ./tools/bin/
 
-# Stage: deps — full source tree for stages that compile chainlink code.
+
+# Stage: deps — Go source tree for stages that compile chainlink code.
+# `COPY . .` relies on .dockerignore excluding everything not needed for the
+# build (docs, CI config, test infra, non-Go tooling) so unrelated workspace
+# churn doesn't invalidate the layer.
 FROM deps-base AS deps
 COPY . .
 
 # Stage: Delve debugger (no source needed, branches from deps-base)
 FROM deps-base AS build-delve
-RUN go install github.com/go-delve/delve/cmd/dlv@v1.24.2
+RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \
+    --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \
+    go install github.com/go-delve/delve/cmd/dlv@v1.24.2
 
 # Stage: Remote plugins — only manifest YAMLs, no source tree.
 # Cached as long as go.mod/go.sum and plugin manifests are unchanged,
@@ -55,36 +65,39 @@
 ENV CL_LOOPINSTALL_OUTPUT_DIR=/tmp/loopinstall-output \
     GIT_CONFIG_GLOBAL=/tmp/gitconfig-github-token
 RUN --mount=type=secret,id=GIT_AUTH_TOKEN \
+    --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \
+    --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \
     set -e && \
     trap 'rm -f "$GIT_CONFIG_GLOBAL"' EXIT && \
     ./plugins/scripts/setup_git_auth.sh && \
     mkdir -p /gobins "${CL_LOOPINSTALL_OUTPUT_DIR}" && \
     GOBIN=/gobins CL_LOOPINSTALL_OUTPUT_DIR=${CL_LOOPINSTALL_OUTPUT_DIR} make install-plugins-public && \
     if [ "${CL_INSTALL_PRIVATE_PLUGINS}" = "true" ]; then \
-        GOBIN=/gobins CL_LOOPINSTALL_OUTPUT_DIR=${CL_LOOPINSTALL_OUTPUT_DIR} make install-plugins-private; \
+      GOBIN=/gobins CL_LOOPINSTALL_OUTPUT_DIR=${CL_LOOPINSTALL_OUTPUT_DIR} make install-plugins-private; \
     fi && \
     if [ "${CL_INSTALL_TESTING_PLUGINS}" = "true" ]; then \
-        GOBIN=/gobins CL_LOOPINSTALL_OUTPUT_DIR=${CL_LOOPINSTALL_OUTPUT_DIR} make install-plugins-testing; \
-    fi
-
-RUN mkdir -p /tmp/lib && \
+      GOBIN=/gobins CL_LOOPINSTALL_OUTPUT_DIR=${CL_LOOPINSTALL_OUTPUT_DIR} make install-plugins-testing; \
+    fi && \
+    mkdir -p /tmp/lib && \
     ./plugins/scripts/copy_loopinstall_libs.sh \
     "$CL_LOOPINSTALL_OUTPUT_DIR" \
     /tmp/lib
 
 # Stage: Local plugins (needs source tree for ./plugins/cmd/...)
 FROM deps AS build-local-plugins
-RUN --mount=type=cache,target=/root/.cache/go-build,id=go-build-local-plugins \
-    mkdir -p /gobins && \
-    GOBIN=/gobins make install-plugins-local
+RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \
+    --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \
+    mkdir -p /gobins \
+    && GOBIN=/gobins make install-plugins-local
 
 # Stage: Chainlink binary (needs source tree)
 FROM deps AS build-chainlink
 ARG COMMIT_SHA
 ARG VERSION_TAG
 ARG CL_IS_PROD_BUILD=true
 
-RUN --mount=type=cache,target=/root/.cache/go-build,id=go-build-chainlink \
+RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \
+    --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \
     mkdir -p /gobins && \
     if [ "$CL_IS_PROD_BUILD" = "false" ]; then \
           GOBIN=/gobins make install-chainlink-dev; \
@@ -96,21 +109,27 @@
 # Final Image
 ##
 FROM ubuntu:24.04 AS final
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 ARG CHAINLINK_USER=root
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y ca-certificates gnupg lsb-release curl && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates=20260601~24.04.1 \
+    gnupg=2.4.4-2ubuntu17.4 \
+    lsb-release=12.0-2 \
+    curl=8.5.0-2ubuntu10.11 \
+    && rm -rf /var/lib/apt/lists/*
 
 # Install Postgres for CLI tools, needed specifically for DB backups
 RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \
-      | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \
+    | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \
     && gpg --no-default-keyring \
-           --keyring /usr/share/keyrings/postgresql-archive-keyring.gpg \
-           --fingerprint B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 \
+        --keyring /usr/share/keyrings/postgresql-archive-keyring.gpg \
+        --fingerprint B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 \
     && echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] \
-       https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \
-       > /etc/apt/sources.list.d/pgdg.list \
-    && apt-get update && apt-get install -y postgresql-client-17 \
+    https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \
+        >/etc/apt/sources.list.d/pgdg.list \
+    && apt-get update && apt-get install -y --no-install-recommends postgresql-client-17=17.10-1.pgdg24.04+1 \
     && rm -rf /var/lib/apt/lists/*
 
 # Prod images (CHAINLINK_USER=chainlink) run as UID:GID 14933:14933 for deterministic
@@ -163,4 +182,4 @@
 
 FROM final AS debug
 
-COPY --from=build-delve /go/bin/dlv /usr/local/bin/
+COPY --from=build-delve /go/bin/dlv /usr/local/bin/

plugins/chainlink.Dockerfile

@@ -9,32 +9,41 @@
 # here so that source-only changes never invalidate their layer cache.
 FROM golang:1.26.4-bookworm AS deps-base
 RUN go version
-RUN apt-get update && apt-get install -y jq && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y --no-install-recommends jq=1.6-2.1+deb12u2 && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /chainlink
 
-ADD go.mod go.sum ./
+COPY go.mod go.sum ./
 COPY plugins/scripts/setup_git_auth.sh ./plugins/scripts/
 
 ARG CL_GOPRIVATE=""
-ENV GOPRIVATE="${CL_GOPRIVATE}"
+ENV GOPRIVATE="${CL_GOPRIVATE}" \
+    GOCACHE=/root/.cache/go-build \
+    GOMODCACHE=/go/pkg/mod
 RUN --mount=type=secret,id=GIT_AUTH_TOKEN \
-    set -e && \
-    export GIT_CONFIG_GLOBAL=/tmp/gitconfig-go-mod-download && \
-    trap 'rm -f "$GIT_CONFIG_GLOBAL"' EXIT && \
-    ./plugins/scripts/setup_git_auth.sh && \
-    go mod download
+    --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \
+    --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \
+    set -e \
+    && export GIT_CONFIG_GLOBAL=/tmp/gitconfig-go-mod-download \
+    && trap 'rm -f "$GIT_CONFIG_GLOBAL"' EXIT \
+    && ./plugins/scripts/setup_git_auth.sh \
+    && go mod download
 
 COPY GNUmakefile package.json ./
 COPY tools/bin/ldflags ./tools/bin/
 
-# Stage: deps — full source tree for stages that compile chainlink code.
+# Stage: deps — Go source tree for stages that compile chainlink code.
+# `COPY . .` relies on .dockerignore excluding everything not needed for the
+# build (docs, CI config, test infra, non-Go tooling) so unrelated workspace
+# churn doesn't invalidate the layer.
 FROM deps-base AS deps
 COPY . .
 
 # Stage: Delve debugger (no source needed, branches from deps-base)
 FROM deps-base AS build-delve
-RUN go install github.com/go-delve/delve/cmd/dlv@v1.24.2
+RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \
+    --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \
+    go install github.com/go-delve/delve/cmd/dlv@v1.24.2
 
 # Stage: Remote plugins — only manifest YAMLs, no source tree.
 # Cached as long as go.mod/go.sum and plugin manifests are unchanged,
@@ -54,6 +63,8 @@
 ENV CL_LOOPINSTALL_OUTPUT_DIR=/tmp/loopinstall-output \
     GIT_CONFIG_GLOBAL=/tmp/gitconfig-github-token
 RUN --mount=type=secret,id=GIT_AUTH_TOKEN \
+    --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \
+    --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \
     set -e && \
     trap 'rm -f "$GIT_CONFIG_GLOBAL"' EXIT && \
     ./plugins/scripts/setup_git_auth.sh && \
@@ -64,18 +75,18 @@
     fi && \
     if [ "${CL_INSTALL_TESTING_PLUGINS}" = "true" ]; then \
         GOBIN=/gobins CL_LOOPINSTALL_OUTPUT_DIR=${CL_LOOPINSTALL_OUTPUT_DIR} make install-plugins-testing; \
-    fi
-
-RUN mkdir -p /tmp/lib && \
+    fi && \
+    mkdir -p /tmp/lib && \
     ./plugins/scripts/copy_loopinstall_libs.sh \
     "$CL_LOOPINSTALL_OUTPUT_DIR" \
     /tmp/lib
 
 # Stage: Local plugins (needs source tree for ./plugins/cmd/...)
 FROM deps AS build-local-plugins
-RUN --mount=type=cache,target=/root/.cache/go-build,id=go-build-local-plugins \
-    mkdir -p /gobins && \
-    GOBIN=/gobins make install-plugins-local
+RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \
+    --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \
+    mkdir -p /gobins \
+    && GOBIN=/gobins make install-plugins-local
 
 # Stage: Chainlink binary (needs source tree)
 FROM deps AS build-chainlink
@@ -84,33 +95,40 @@
 ARG COMMIT_SHA
 ARG VERSION_TAG
 
-RUN --mount=type=cache,target=/root/.cache/go-build,id=go-build-chainlink \
+RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \
+    --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \
     mkdir -p /gobins && \
     if [ "$CL_IS_PROD_BUILD" = "false" ]; then \
           GOBIN=/gobins make install-chainlink-dev; \
-    else \
+      else \
           GOBIN=/gobins make install-chainlink; \
-    fi
+      fi
 
 ##
 # Final Image
 ##
 FROM ubuntu:24.04 AS final
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 ARG CHAINLINK_USER=root
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y ca-certificates gnupg lsb-release curl && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates=20260601~24.04.1 \
+    gnupg=2.4.4-2ubuntu17.4 \
+    lsb-release=12.0-2 \
+    curl=8.5.0-2ubuntu10.11 \
+    && rm -rf /var/lib/apt/lists/*
 
 # Install Postgres for CLI tools, needed specifically for DB backups
 RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \
-      | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \
+    | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \
     && gpg --no-default-keyring \
-           --keyring /usr/share/keyrings/postgresql-archive-keyring.gpg \
-           --fingerprint B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 \
+        --keyring /usr/share/keyrings/postgresql-archive-keyring.gpg \
+        --fingerprint B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 \
     && echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] \
        https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \
-       > /etc/apt/sources.list.d/pgdg.list \
-    && apt-get update && apt-get install -y postgresql-client-17 \
+        >/etc/apt/sources.list.d/pgdg.list \
+    && apt-get update && apt-get install -y --no-install-recommends postgresql-client-17=17.10-1.pgdg24.04+1 \
     && rm -rf /var/lib/apt/lists/*
 
 # Prod images (CHAINLINK_USER=chainlink) run as UID:GID 14933:14933 for deterministic ownership.
@@ -157,4 +175,4 @@
 
 FROM final AS debug
 
-COPY --from=build-delve /go/bin/dlv /usr/local/bin/dlv
+COPY --from=build-delve /go/bin/dlv /usr/local/bin/dlv