diff --git a/.dockerignore b/.dockerignore index dfa28275ce4..dd3eddee92b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,21 @@ .git .DS_Store .envrc +.dbenv +.npmrc +.gitattributes +.gitignore +.dockerignore +.golangci.yml +.mockery.yaml +.nancy-ignore +.tarignore +.tool-versions +nix.conf +nix-darwin-shell-hook.sh +LICENSE *.log +**/*_test.go node_modules/ **/node_modules/ vendor/ @@ -13,9 +27,20 @@ examples/ .changeset/ .github/ +.vscode/ +.claude/ +.cursor/ docs/ fuzz/ +*.md +sonar-project.properties +pnpm-lock.yaml +flake.lock +flake.nix +shell.nix + + devenv/ deployment/ integration/ @@ -45,3 +70,8 @@ gcr_creds.env go.work go.work.sum + +# buildkit-cache-dance host dirs (cache-dir / scratch-dir). These hold the injected +# Go module + build caches and must never enter the build context. +cache-mount/ +scratch/ diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 9e369c9dac1..60d14555040 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -348,14 +348,19 @@ jobs: aws-region: ${{ secrets.QA_AWS_REGION }} aws-role-arn: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} free-disk-space: "false" - docker-additional-build-args: | - CL_IS_PROD_BUILD=false # go-get-overrides is empty when inputs.evm-ref is unset so ctf-build-image skips an unnecessary setup-go / go mod tidy. go-get-overrides: >- ${{ inputs.evm-ref && format('chainlink-evm={0}', inputs.evm-ref) || '' }} gati-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }} gati-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }} + cache-scope: ${{ matrix.image.cache-scope }} + # Persist only the Go build cache (GOCACHE), not the module cache. + # Module cache is large, and doesn't save us much time, + # at least compared to just using the build cache + # It also has mysterious failures when trying to upload + cache-map: | + {"cache-mount/go-build-cache": {"target": "/var/cache-target", "id": "go-build-cache"}} run-core-cre-e2e-tests-setup: name: Run Core CRE E2E Tests Setup diff --git a/.gitignore b/.gitignore index bdf9df52c4e..f55405bd535 100644 --- a/.gitignore +++ b/.gitignore @@ -138,4 +138,4 @@ core/scripts/cre/environment/binaries/* .claude/ .cursor/ .gemini/ -opencode.json \ No newline at end of file +opencode.json diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile index c68ca264de9..b63e28ecf09 100644 --- a/core/chainlink.Dockerfile +++ b/core/chainlink.Dockerfile @@ -7,35 +7,45 @@ # here so that source-only changes never invalidate their layer cache. FROM golang:1.26.4-bookworm AS deps-base RUN go version -RUN apt-get update && apt-get install -y jq && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends jq=1.6-2.1+deb12u2 && rm -rf /var/lib/apt/lists/* WORKDIR /chainlink -ADD go.mod go.sum ./ +COPY go.mod go.sum ./ COPY plugins/scripts/setup_git_auth.sh ./plugins/scripts/ # CL_GOPRIVATE: set to "github.com/smartcontractkit/*" when building images # that depend on private Go modules (e.g. chainlink-internal-solana). When # empty (the default), go mod download uses the public module proxy as usual. ARG CL_GOPRIVATE="" -ENV GOPRIVATE="${CL_GOPRIVATE}" +ENV GOPRIVATE="${CL_GOPRIVATE}" \ + GOCACHE=/root/.cache/go-build \ + GOMODCACHE=/go/pkg/mod RUN --mount=type=secret,id=GIT_AUTH_TOKEN \ - set -e && \ - export GIT_CONFIG_GLOBAL=/tmp/gitconfig-go-mod-download && \ - trap 'rm -f "$GIT_CONFIG_GLOBAL"' EXIT && \ - ./plugins/scripts/setup_git_auth.sh && \ - go mod download + --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ + --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ + set -e \ + && export GIT_CONFIG_GLOBAL=/tmp/gitconfig-go-mod-download \ + && trap 'rm -f "$GIT_CONFIG_GLOBAL"' EXIT \ + && ./plugins/scripts/setup_git_auth.sh \ + && go mod download COPY GNUmakefile package.json ./ COPY tools/bin/ldflags ./tools/bin/ -# Stage: deps — full source tree for stages that compile chainlink code. + +# Stage: deps — Go source tree for stages that compile chainlink code. +# `COPY . .` relies on .dockerignore excluding everything not needed for the +# build (docs, CI config, test infra, non-Go tooling) so unrelated workspace +# churn doesn't invalidate the layer. FROM deps-base AS deps COPY . . # Stage: Delve debugger (no source needed, branches from deps-base) FROM deps-base AS build-delve -RUN go install github.com/go-delve/delve/cmd/dlv@v1.24.2 +RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ + --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ + go install github.com/go-delve/delve/cmd/dlv@v1.24.2 # Stage: Remote plugins — only manifest YAMLs, no source tree. # Cached as long as go.mod/go.sum and plugin manifests are unchanged, @@ -55,28 +65,30 @@ COPY plugins/scripts/ ./plugins/scripts/ ENV CL_LOOPINSTALL_OUTPUT_DIR=/tmp/loopinstall-output \ GIT_CONFIG_GLOBAL=/tmp/gitconfig-github-token RUN --mount=type=secret,id=GIT_AUTH_TOKEN \ + --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ + --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ set -e && \ trap 'rm -f "$GIT_CONFIG_GLOBAL"' EXIT && \ ./plugins/scripts/setup_git_auth.sh && \ mkdir -p /gobins "${CL_LOOPINSTALL_OUTPUT_DIR}" && \ GOBIN=/gobins CL_LOOPINSTALL_OUTPUT_DIR=${CL_LOOPINSTALL_OUTPUT_DIR} make install-plugins-public && \ if [ "${CL_INSTALL_PRIVATE_PLUGINS}" = "true" ]; then \ - GOBIN=/gobins CL_LOOPINSTALL_OUTPUT_DIR=${CL_LOOPINSTALL_OUTPUT_DIR} make install-plugins-private; \ + GOBIN=/gobins CL_LOOPINSTALL_OUTPUT_DIR=${CL_LOOPINSTALL_OUTPUT_DIR} make install-plugins-private; \ fi && \ if [ "${CL_INSTALL_TESTING_PLUGINS}" = "true" ]; then \ - GOBIN=/gobins CL_LOOPINSTALL_OUTPUT_DIR=${CL_LOOPINSTALL_OUTPUT_DIR} make install-plugins-testing; \ - fi - -RUN mkdir -p /tmp/lib && \ + GOBIN=/gobins CL_LOOPINSTALL_OUTPUT_DIR=${CL_LOOPINSTALL_OUTPUT_DIR} make install-plugins-testing; \ + fi && \ + mkdir -p /tmp/lib && \ ./plugins/scripts/copy_loopinstall_libs.sh \ "$CL_LOOPINSTALL_OUTPUT_DIR" \ /tmp/lib # Stage: Local plugins (needs source tree for ./plugins/cmd/...) FROM deps AS build-local-plugins -RUN --mount=type=cache,target=/root/.cache/go-build,id=go-build-local-plugins \ - mkdir -p /gobins && \ - GOBIN=/gobins make install-plugins-local +RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ + --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ + mkdir -p /gobins \ + && GOBIN=/gobins make install-plugins-local # Stage: Chainlink binary (needs source tree) FROM deps AS build-chainlink @@ -84,7 +96,8 @@ ARG COMMIT_SHA ARG VERSION_TAG ARG CL_IS_PROD_BUILD=true -RUN --mount=type=cache,target=/root/.cache/go-build,id=go-build-chainlink \ +RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ + --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ mkdir -p /gobins && \ if [ "$CL_IS_PROD_BUILD" = "false" ]; then \ GOBIN=/gobins make install-chainlink-dev; \ @@ -96,21 +109,27 @@ RUN --mount=type=cache,target=/root/.cache/go-build,id=go-build-chainlink \ # Final Image ## FROM ubuntu:24.04 AS final +SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG CHAINLINK_USER=root ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y ca-certificates gnupg lsb-release curl && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates=20260601~24.04.1 \ + gnupg=2.4.4-2ubuntu17.4 \ + lsb-release=12.0-2 \ + curl=8.5.0-2ubuntu10.11 \ + && rm -rf /var/lib/apt/lists/* # Install Postgres for CLI tools, needed specifically for DB backups RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ - | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \ + | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \ && gpg --no-default-keyring \ - --keyring /usr/share/keyrings/postgresql-archive-keyring.gpg \ - --fingerprint B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 \ + --keyring /usr/share/keyrings/postgresql-archive-keyring.gpg \ + --fingerprint B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 \ && echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] \ - https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ - > /etc/apt/sources.list.d/pgdg.list \ - && apt-get update && apt-get install -y postgresql-client-17 \ + https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ + >/etc/apt/sources.list.d/pgdg.list \ + && apt-get update && apt-get install -y --no-install-recommends postgresql-client-17=17.10-1.pgdg24.04+1 \ && rm -rf /var/lib/apt/lists/* # Prod images (CHAINLINK_USER=chainlink) run as UID:GID 14933:14933 for deterministic @@ -163,4 +182,4 @@ CMD ["local", "node"] FROM final AS debug -COPY --from=build-delve /go/bin/dlv /usr/local/bin/ +COPY --from=build-delve /go/bin/dlv /usr/local/bin/ \ No newline at end of file diff --git a/plugins/chainlink.Dockerfile b/plugins/chainlink.Dockerfile index d7c5636c781..d98c88b1162 100644 --- a/plugins/chainlink.Dockerfile +++ b/plugins/chainlink.Dockerfile @@ -9,32 +9,41 @@ # here so that source-only changes never invalidate their layer cache. FROM golang:1.26.4-bookworm AS deps-base RUN go version -RUN apt-get update && apt-get install -y jq && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends jq=1.6-2.1+deb12u2 && rm -rf /var/lib/apt/lists/* WORKDIR /chainlink -ADD go.mod go.sum ./ +COPY go.mod go.sum ./ COPY plugins/scripts/setup_git_auth.sh ./plugins/scripts/ ARG CL_GOPRIVATE="" -ENV GOPRIVATE="${CL_GOPRIVATE}" +ENV GOPRIVATE="${CL_GOPRIVATE}" \ + GOCACHE=/root/.cache/go-build \ + GOMODCACHE=/go/pkg/mod RUN --mount=type=secret,id=GIT_AUTH_TOKEN \ - set -e && \ - export GIT_CONFIG_GLOBAL=/tmp/gitconfig-go-mod-download && \ - trap 'rm -f "$GIT_CONFIG_GLOBAL"' EXIT && \ - ./plugins/scripts/setup_git_auth.sh && \ - go mod download + --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ + --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ + set -e \ + && export GIT_CONFIG_GLOBAL=/tmp/gitconfig-go-mod-download \ + && trap 'rm -f "$GIT_CONFIG_GLOBAL"' EXIT \ + && ./plugins/scripts/setup_git_auth.sh \ + && go mod download COPY GNUmakefile package.json ./ COPY tools/bin/ldflags ./tools/bin/ -# Stage: deps — full source tree for stages that compile chainlink code. +# Stage: deps — Go source tree for stages that compile chainlink code. +# `COPY . .` relies on .dockerignore excluding everything not needed for the +# build (docs, CI config, test infra, non-Go tooling) so unrelated workspace +# churn doesn't invalidate the layer. FROM deps-base AS deps COPY . . # Stage: Delve debugger (no source needed, branches from deps-base) FROM deps-base AS build-delve -RUN go install github.com/go-delve/delve/cmd/dlv@v1.24.2 +RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ + --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ + go install github.com/go-delve/delve/cmd/dlv@v1.24.2 # Stage: Remote plugins — only manifest YAMLs, no source tree. # Cached as long as go.mod/go.sum and plugin manifests are unchanged, @@ -54,6 +63,8 @@ COPY plugins/scripts/ ./plugins/scripts/ ENV CL_LOOPINSTALL_OUTPUT_DIR=/tmp/loopinstall-output \ GIT_CONFIG_GLOBAL=/tmp/gitconfig-github-token RUN --mount=type=secret,id=GIT_AUTH_TOKEN \ + --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ + --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ set -e && \ trap 'rm -f "$GIT_CONFIG_GLOBAL"' EXIT && \ ./plugins/scripts/setup_git_auth.sh && \ @@ -64,18 +75,18 @@ RUN --mount=type=secret,id=GIT_AUTH_TOKEN \ fi && \ if [ "${CL_INSTALL_TESTING_PLUGINS}" = "true" ]; then \ GOBIN=/gobins CL_LOOPINSTALL_OUTPUT_DIR=${CL_LOOPINSTALL_OUTPUT_DIR} make install-plugins-testing; \ - fi - -RUN mkdir -p /tmp/lib && \ + fi && \ + mkdir -p /tmp/lib && \ ./plugins/scripts/copy_loopinstall_libs.sh \ "$CL_LOOPINSTALL_OUTPUT_DIR" \ /tmp/lib # Stage: Local plugins (needs source tree for ./plugins/cmd/...) FROM deps AS build-local-plugins -RUN --mount=type=cache,target=/root/.cache/go-build,id=go-build-local-plugins \ - mkdir -p /gobins && \ - GOBIN=/gobins make install-plugins-local +RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ + --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ + mkdir -p /gobins \ + && GOBIN=/gobins make install-plugins-local # Stage: Chainlink binary (needs source tree) FROM deps AS build-chainlink @@ -84,33 +95,40 @@ ARG GO_GCFLAGS ARG COMMIT_SHA ARG VERSION_TAG -RUN --mount=type=cache,target=/root/.cache/go-build,id=go-build-chainlink \ +RUN --mount=type=cache,id=go-mod-cache,target=/go/pkg/mod \ + --mount=type=cache,id=go-build-cache,target=/root/.cache/go-build \ mkdir -p /gobins && \ if [ "$CL_IS_PROD_BUILD" = "false" ]; then \ GOBIN=/gobins make install-chainlink-dev; \ - else \ + else \ GOBIN=/gobins make install-chainlink; \ - fi + fi ## # Final Image ## FROM ubuntu:24.04 AS final +SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG CHAINLINK_USER=root ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y ca-certificates gnupg lsb-release curl && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates=20260601~24.04.1 \ + gnupg=2.4.4-2ubuntu17.4 \ + lsb-release=12.0-2 \ + curl=8.5.0-2ubuntu10.11 \ + && rm -rf /var/lib/apt/lists/* # Install Postgres for CLI tools, needed specifically for DB backups RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ - | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \ + | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \ && gpg --no-default-keyring \ - --keyring /usr/share/keyrings/postgresql-archive-keyring.gpg \ - --fingerprint B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 \ + --keyring /usr/share/keyrings/postgresql-archive-keyring.gpg \ + --fingerprint B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 \ && echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] \ https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ - > /etc/apt/sources.list.d/pgdg.list \ - && apt-get update && apt-get install -y postgresql-client-17 \ + >/etc/apt/sources.list.d/pgdg.list \ + && apt-get update && apt-get install -y --no-install-recommends postgresql-client-17=17.10-1.pgdg24.04+1 \ && rm -rf /var/lib/apt/lists/* # Prod images (CHAINLINK_USER=chainlink) run as UID:GID 14933:14933 for deterministic ownership. @@ -157,4 +175,4 @@ CMD ["local", "node"] FROM final AS debug -COPY --from=build-delve /go/bin/dlv /usr/local/bin/dlv +COPY --from=build-delve /go/bin/dlv /usr/local/bin/dlv \ No newline at end of file