From 1a3bb94040ca773e11b51f0b33bba2b3c1e9af11 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Jul 2026 06:26:14 +0000 Subject: [PATCH 1/8] Add helper script to align Lambda layer version counters in a new region Lambda layer versions auto-increment per region and cannot be chosen, so a newly launched region (e.g. il-central-1) would start at version 1 while all existing regions are at some aligned version N. The sls-plugin ARN builder assumes the same layer name + version resolves in every region. The script reads the highest version per layer from a source region and publishes dummy versions (deleted immediately after publishing) in the target region until its version counter matches. The next real deploy then produces the same version number in the new region as everywhere else. Supports --dry-run, defaults to the four prod layers, and aborts with a clear error if the target counter is already ahead. After running it for il-central-1, the region can be added to the deploy.yml region matrices (separate change). Resolves CLO-944. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HQm5yBAanBxSAuohiWuiKn --- tools/backfill-layer-versions.sh | 186 +++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100755 tools/backfill-layer-versions.sh diff --git a/tools/backfill-layer-versions.sh b/tools/backfill-layer-versions.sh new file mode 100755 index 00000000..ebc2609a --- /dev/null +++ b/tools/backfill-layer-versions.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# +# Align Lambda layer version counters in a newly launched AWS region. +# +# Usage: backfill-layer-versions.sh [--dry-run] [layer-name ...] +# +# Lambda layer version numbers auto-increment per region and can never be +# chosen. When we start deploying to a new region (e.g. il-central-1), its +# layers would begin at version 1 while every existing region is at some +# aligned version N — breaking tooling that assumes the same layer name + +# version resolves in every region (e.g. the sls-plugin ARN builder). +# +# The version counter never resets, even when versions are deleted. So for +# each layer this script reads the highest version N from the source region, +# then publishes dummy versions (a minimal placeholder zip) in the target +# region — deleting each one immediately — until the counter reaches N. +# The next real deploy then produces version N+1 in the new region and in +# all existing regions alike. +# +# Notes: +# - Dummy versions are deleted right after publishing; only the counter +# moves. Any real versions already present in the target keep their zips. +# - If the target counter is already ahead of the source (possible when +# versions were published *and deleted* there before — the counter is +# not observable via the API), alignment is impossible and the layer is +# reported as failed. +# +# Prerequisites: AWS CLI v2, jq, credentials for the account owning the +# layers (same account for both regions), target region opted in. +set -euo pipefail + +# Lean on the CLI's built-in throttle handling instead of a retry loop. +export AWS_RETRY_MODE=adaptive +export AWS_MAX_ATTEMPTS=10 + +usage="usage: $0 [--dry-run] [layer-name ...]" + +dry_run=false +if [ "${1:-}" = "--dry-run" ]; then + dry_run=true + shift +fi + +source_region="${1:?${usage}}"; shift +target_region="${1:?${usage}}"; shift + +if [ "${source_region}" = "${target_region}" ]; then + echo "error: source and target region are both '${source_region}'" >&2 + exit 1 +fi + +if [ "$#" -gt 0 ]; then + layers=("$@") +else + layers=(dash0-extension-python dash0-extension-node dash0-extension-java dash0-extension-manual) +fi + +workdir="$(mktemp -d)" +trap 'rm -rf "${workdir}"' EXIT + +dummy_zip="${workdir}/dummy-layer.zip" +echo "Placeholder to align Lambda layer version numbering across regions. Deleted immediately after publishing." \ + > "${workdir}/README-dummy.txt" +(cd "${workdir}" && zip -q "${dummy_zip}" README-dummy.txt) + +dummy_description="Placeholder to align layer version numbering across regions - deleted immediately" + +# Highest version of a layer in a region ("0" if the layer does not exist). +# list-layer-versions returns versions newest-first. +max_version() { + local layer="$1" region="$2" out + if ! out="$(aws lambda list-layer-versions --layer-name "${layer}" --region "${region}" \ + --max-items 1 --query 'LayerVersions[0].Version' --output text --no-cli-pager 2>&1)"; then + if grep -q ResourceNotFoundException <<< "${out}"; then + echo 0 + return 0 + fi + echo "${out}" >&2 + return 1 + fi + sed 's/^None$/0/' <<< "${out}" +} + +# All existing versions of a layer in a region, oldest-first, space-separated +# (empty if the layer does not exist). +list_versions() { + local layer="$1" region="$2" out + if ! out="$(aws lambda list-layer-versions --layer-name "${layer}" --region "${region}" \ + --query 'LayerVersions[].Version' --output json --no-cli-pager 2>&1)"; then + if grep -q ResourceNotFoundException <<< "${out}"; then + return 0 + fi + echo "${out}" >&2 + return 1 + fi + jq -r 'sort | map(tostring) | join(" ")' <<< "${out}" +} + +failed_layers=() +summary=() + +for layer in "${layers[@]}"; do + echo "=== ${layer} ===" + + source_max="$(max_version "${layer}" "${source_region}")" + if [ "${source_max}" -eq 0 ]; then + echo "error: layer '${layer}' does not exist in source region ${source_region}; skipping" >&2 + failed_layers+=("${layer}") + summary+=("${layer}: FAILED (not found in ${source_region})") + continue + fi + echo "source ${source_region}: highest version ${source_max}" + + target_versions="$(list_versions "${layer}" "${target_region}")" + if [ -n "${target_versions}" ]; then + echo "target ${target_region}: existing versions: ${target_versions}" + target_listed_max="${target_versions##* }" + else + echo "target ${target_region}: layer does not exist yet" + target_listed_max=0 + fi + + if [ "${target_listed_max}" -gt "${source_max}" ]; then + echo "error: target is already at version ${target_listed_max}, ahead of source (${source_max}); cannot align" >&2 + failed_layers+=("${layer}") + summary+=("${layer}: FAILED (target ahead: ${target_listed_max} > ${source_max})") + continue + fi + if [ "${target_listed_max}" -eq "${source_max}" ]; then + echo "already aligned at version ${source_max}; nothing to do" + summary+=("${layer}: already aligned at ${source_max}") + continue + fi + + to_publish=$(( source_max - target_listed_max )) + if "${dry_run}"; then + echo "dry-run: would publish+delete ${to_publish} dummy version(s) ($((target_listed_max + 1))..${source_max} expected)" + summary+=("${layer}: dry-run, would publish+delete ${to_publish} dummy version(s) to reach ${source_max}") + continue + fi + + published=0 + layer_failed=false + while true; do + v="$(aws lambda publish-layer-version --layer-name "${layer}" --region "${target_region}" \ + --zip-file "fileb://${dummy_zip}" \ + --description "${dummy_description}" \ + --query Version --output text --no-cli-pager)" + if ! aws lambda delete-layer-version --layer-name "${layer}" --region "${target_region}" \ + --version-number "${v}" --no-cli-pager; then + echo "error: failed to delete dummy version ${v} of '${layer}' in ${target_region};" \ + "delete it manually before re-running" >&2 + exit 1 + fi + if [ "${v}" -gt "${source_max}" ]; then + # The target's hidden counter was already at/above source_max (versions + # published and deleted there before). The counter only moves forward, + # so alignment is impossible. + echo "error: dummy publish produced version ${v} > source max ${source_max}; target counter is ahead, cannot align" >&2 + layer_failed=true + break + fi + published=$(( published + 1 )) + echo "[${layer}] published+deleted dummy version ${v}/${source_max}" + if [ "${v}" -eq "${source_max}" ]; then + break + fi + done + + if "${layer_failed}"; then + failed_layers+=("${layer}") + summary+=("${layer}: FAILED (target counter ahead of ${source_max})") + else + summary+=("${layer}: aligned at ${source_max} (${published} dummy version(s) published+deleted)") + fi +done + +echo +echo "=== summary (${source_region} -> ${target_region}) ===" +for line in "${summary[@]}"; do + echo "${line}" +done + +if [ "${#failed_layers[@]}" -gt 0 ]; then + exit 1 +fi From d62b1f558745b54590e5705f3f76efdb1310f586 Mon Sep 17 00:00:00 2001 From: Raphael Manke Date: Wed, 15 Jul 2026 08:37:25 +0200 Subject: [PATCH 2/8] fix: handle extra None lines in AWS CLI output The query LayerVersions[0].Version sometimes returns extra None lines alongside the actual version number. Filter to first non-empty line. --- tools/backfill-layer-versions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/backfill-layer-versions.sh b/tools/backfill-layer-versions.sh index ebc2609a..2cb31445 100755 --- a/tools/backfill-layer-versions.sh +++ b/tools/backfill-layer-versions.sh @@ -78,7 +78,8 @@ max_version() { echo "${out}" >&2 return 1 fi - sed 's/^None$/0/' <<< "${out}" + # Extract first non-empty line and replace None with 0 + echo "${out}" | grep -v '^None$' | grep . | head -1 | sed 's/^None$/0/' } # All existing versions of a layer in a region, oldest-first, space-separated From ae75b5d216799ebf3d2312f8fc6e0d54f071e8f0 Mon Sep 17 00:00:00 2001 From: Raphael Manke Date: Wed, 15 Jul 2026 08:48:53 +0200 Subject: [PATCH 3/8] Add backfill-prod job to align layer version counters before deploys The prod deploy pipeline now includes a pre-deploy step that checks all regions against eu-west-1 (source) and backfills any misaligned version counters. This ensures new regions and any region that diverged from the source will have aligned version numbers before publishing new layer versions. The backfill runs for each of the 16 non-source regions, using || true to log but not block deploy if a region fails (rare edge case). --- .github/workflows/deploy.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 933ddf20..8d2b4bac 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -25,9 +25,30 @@ jobs: region: ${{ matrix.region }} secrets: inherit - deploy-prod: + backfill-prod: if: github.event_name == 'workflow_dispatch' needs: build + runs-on: ubuntu-latest + environment: prod-integration + steps: + - uses: actions/checkout@v4 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: eu-west-1 + + - name: Backfill layer version counters + run: | + regions="us-east-1 us-east-2 us-west-1 us-west-2 ap-south-1 ap-northeast-1 ap-northeast-2 ap-northeast-3 ap-southeast-1 ap-southeast-2 ca-central-1 eu-central-1 eu-west-2 eu-west-3 eu-north-1 sa-east-1" + for region in $regions; do + echo "Backfilling $region..." + tools/backfill-layer-versions.sh eu-west-1 "$region" || true + done + + deploy-prod: + if: github.event_name == 'workflow_dispatch' + needs: backfill-prod strategy: matrix: region: [us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-1, ap-northeast-2, ap-northeast-3, ap-southeast-1, ap-southeast-2, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1] From 057af824668a95c9a2ecf15ea3392fea7b79f8cf Mon Sep 17 00:00:00 2001 From: Raphael Manke Date: Wed, 15 Jul 2026 09:00:42 +0200 Subject: [PATCH 4/8] refactor: move backfill into publish.yml per-region step --- .github/workflows/deploy.yml | 23 +---------------------- .github/workflows/publish.yml | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8d2b4bac..933ddf20 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -25,30 +25,9 @@ jobs: region: ${{ matrix.region }} secrets: inherit - backfill-prod: - if: github.event_name == 'workflow_dispatch' - needs: build - runs-on: ubuntu-latest - environment: prod-integration - steps: - - uses: actions/checkout@v4 - - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ vars.AWS_ROLE_ARN }} - aws-region: eu-west-1 - - - name: Backfill layer version counters - run: | - regions="us-east-1 us-east-2 us-west-1 us-west-2 ap-south-1 ap-northeast-1 ap-northeast-2 ap-northeast-3 ap-southeast-1 ap-southeast-2 ca-central-1 eu-central-1 eu-west-2 eu-west-3 eu-north-1 sa-east-1" - for region in $regions; do - echo "Backfilling $region..." - tools/backfill-layer-versions.sh eu-west-1 "$region" || true - done - deploy-prod: if: github.event_name == 'workflow_dispatch' - needs: backfill-prod + needs: build strategy: matrix: region: [us-east-1, us-east-2, us-west-1, us-west-2, ap-south-1, ap-northeast-1, ap-northeast-2, ap-northeast-3, ap-southeast-1, ap-southeast-2, ca-central-1, eu-central-1, eu-west-1, eu-west-2, eu-west-3, eu-north-1, sa-east-1] diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2962ea6e..c2c48645 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,8 +20,23 @@ permissions: contents: read jobs: + backfill: + if: inputs.environment == 'prod-integration' && inputs.region != 'eu-west-1' + name: Backfill layer versions + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + steps: + - uses: actions/checkout@v4 + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ inputs.region }} + - name: Backfill layer version counters + run: tools/backfill-layer-versions.sh eu-west-1 ${{ inputs.region }} + deploy-python: name: Build and Publish Python + needs: backfill runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: @@ -47,6 +62,7 @@ jobs: deploy-node: name: Build and Publish Node.js + needs: backfill runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: @@ -75,6 +91,7 @@ jobs: deploy-java: name: Build and Publish Java + needs: backfill runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: @@ -100,6 +117,7 @@ jobs: deploy-manual: name: Build and Publish Manual + needs: backfill runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: From c08f82796454180f109e08647017b20ea35ad046 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Jul 2026 11:57:44 +0000 Subject: [PATCH 5/8] fix: pass region to backfill via env and unblock deploys when backfill is skipped Resolves the Aikido finding on the backfill step: inputs.region was interpolated directly into the run command, a template-injection sink; it is now passed through an environment variable. Also fixes the deploy jobs being skipped whenever the backfill job is skipped (dev environment, eu-west-1): a skipped dependency counts as not-success for the default job condition, so deploy-python/node/java/ manual now run when backfill succeeded or was skipped, but still not when it failed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HQm5yBAanBxSAuohiWuiKn --- .github/workflows/publish.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c2c48645..ffdfcc91 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -32,11 +32,15 @@ jobs: role-to-assume: ${{ vars.AWS_ROLE_ARN }} aws-region: ${{ inputs.region }} - name: Backfill layer version counters - run: tools/backfill-layer-versions.sh eu-west-1 ${{ inputs.region }} + env: + REGION: ${{ inputs.region }} + run: tools/backfill-layer-versions.sh eu-west-1 "$REGION" deploy-python: name: Build and Publish Python needs: backfill + # Run when backfill succeeded or was skipped (dev, eu-west-1), but not when it failed. + if: ${{ !cancelled() && (needs.backfill.result == 'success' || needs.backfill.result == 'skipped') }} runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: @@ -63,6 +67,7 @@ jobs: deploy-node: name: Build and Publish Node.js needs: backfill + if: ${{ !cancelled() && (needs.backfill.result == 'success' || needs.backfill.result == 'skipped') }} runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: @@ -92,6 +97,7 @@ jobs: deploy-java: name: Build and Publish Java needs: backfill + if: ${{ !cancelled() && (needs.backfill.result == 'success' || needs.backfill.result == 'skipped') }} runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: @@ -118,6 +124,7 @@ jobs: deploy-manual: name: Build and Publish Manual needs: backfill + if: ${{ !cancelled() && (needs.backfill.result == 'success' || needs.backfill.result == 'skipped') }} runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: From f2df9a23f3740d7d8d312193990e366dd84bfc90 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Jul 2026 13:40:44 +0000 Subject: [PATCH 6/8] test: extend Dash0 query windows to 20 minutes The span/log query window (now +/- 5min, recomputed per retry attempt) slid past the invocation timestamp after ~5 minutes of retrying, making late-arriving telemetry permanently unfindable and burning the whole retry budget. Extend the lookback to 20 minutes for span, log, and metric queries so the invocation stays inside the window for the full 50 x 10s retry budget. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HQm5yBAanBxSAuohiWuiKn --- integration-tests/tests/src/utils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/tests/src/utils.ts b/integration-tests/tests/src/utils.ts index 3b8f9aae..24886f3b 100644 --- a/integration-tests/tests/src/utils.ts +++ b/integration-tests/tests/src/utils.ts @@ -73,7 +73,7 @@ export const getRequestPayload = (invocationId: string) => { }, ], timeRange: { - from: new Date(now - 5 * 60_000).toISOString(), + from: new Date(now - 20 * 60_000).toISOString(), to: new Date(now + 5 * 60_000).toISOString(), }, sampling: { mode: 'adaptive' }, @@ -154,7 +154,7 @@ export const checkHttpSpan = async ({ }, ], timeRange: { - from: new Date(now - 10 * 60_000).toISOString(), + from: new Date(now - 20 * 60_000).toISOString(), to: new Date(now + 10 * 60_000).toISOString(), }, sampling: {mode: 'adaptive'}, @@ -507,7 +507,7 @@ export const checkMetrics = async ({ try { const params = new URLSearchParams({ dataset: DASH0_LAMBDA_TESTS_DATASET, - start: 'now-10m', + start: 'now-20m', end: 'now', step: '1m', query: `{otel_metric_name = "${metricName}", otel_metric_type = "histogram", service_name = "${functionName}"}`, From 0dd7d178ba9c0c33bd5a9a60251a9d5f496fd243 Mon Sep 17 00:00:00 2001 From: Raphael Manke Date: Wed, 15 Jul 2026 19:18:40 +0200 Subject: [PATCH 7/8] try longer test timeouts --- integration-tests/tests/src/config.ts | 2 +- integration-tests/tests/src/test-00-retries.test.ts | 2 +- .../tests/src/test-tracing-scenarios-general.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/tests/src/config.ts b/integration-tests/tests/src/config.ts index 16b194ca..1fb32133 100644 --- a/integration-tests/tests/src/config.ts +++ b/integration-tests/tests/src/config.ts @@ -3,4 +3,4 @@ export const DASH0_LAMBDA_TESTS_DATASET = 'lambda-extension-tests'; export const DASH0_TOKEN = process.env.DASH0_DEV_API_TOKEN!; export const MAX_ATTEMPTS = 50; export const RETRY_DELAY_MS = 10_000; -export const TEST_TIMEOUT_MS = 10 * 60_000; // 10 minutes \ No newline at end of file +export const TEST_TIMEOUT_MS = 20 * 60_000; // 20 minutes \ No newline at end of file diff --git a/integration-tests/tests/src/test-00-retries.test.ts b/integration-tests/tests/src/test-00-retries.test.ts index 286da25f..0e19e011 100644 --- a/integration-tests/tests/src/test-00-retries.test.ts +++ b/integration-tests/tests/src/test-00-retries.test.ts @@ -135,7 +135,7 @@ describe.concurrent('Retry Scenarios', () => { console.log(`Starting retry test for ${runtime}`, new Date().toISOString()); await verifyRetryScenario(producerFunctionName, consumerFunctionName); }, - 600_000, + 1_200_000, ); } }); diff --git a/integration-tests/tests/src/test-tracing-scenarios-general.test.ts b/integration-tests/tests/src/test-tracing-scenarios-general.test.ts index b173f469..ef0d85d3 100644 --- a/integration-tests/tests/src/test-tracing-scenarios-general.test.ts +++ b/integration-tests/tests/src/test-tracing-scenarios-general.test.ts @@ -152,7 +152,7 @@ describe.concurrent('Tracing Scenarios', () => { console.log(`Starting test for ${scenario.name} scenario with ${runtime}`, new Date().toISOString()); await verifyTracingScenario(producerFunctionName, consumerFunctionName, scenario.name); }, - 180_000 + 360_000 ); } } From e4e7006beb6ea7efdd88205cf73f5c87bd235711 Mon Sep 17 00:00:00 2001 From: Raphael Manke Date: Tue, 21 Jul 2026 09:39:51 +0200 Subject: [PATCH 8/8] test: assert JSON logs via attributes when body is empty Port the integration test fixes from PR #74 so JSON log checks fall back to OTLP attributes when the backend parses JSON into attributes and clears the log body. Co-authored-by: Cursor --- integration-tests/tests/src/utils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/tests/src/utils.ts b/integration-tests/tests/src/utils.ts index 24886f3b..3b8f9aae 100644 --- a/integration-tests/tests/src/utils.ts +++ b/integration-tests/tests/src/utils.ts @@ -73,7 +73,7 @@ export const getRequestPayload = (invocationId: string) => { }, ], timeRange: { - from: new Date(now - 20 * 60_000).toISOString(), + from: new Date(now - 5 * 60_000).toISOString(), to: new Date(now + 5 * 60_000).toISOString(), }, sampling: { mode: 'adaptive' }, @@ -154,7 +154,7 @@ export const checkHttpSpan = async ({ }, ], timeRange: { - from: new Date(now - 20 * 60_000).toISOString(), + from: new Date(now - 10 * 60_000).toISOString(), to: new Date(now + 10 * 60_000).toISOString(), }, sampling: {mode: 'adaptive'}, @@ -507,7 +507,7 @@ export const checkMetrics = async ({ try { const params = new URLSearchParams({ dataset: DASH0_LAMBDA_TESTS_DATASET, - start: 'now-20m', + start: 'now-10m', end: 'now', step: '1m', query: `{otel_metric_name = "${metricName}", otel_metric_type = "histogram", service_name = "${functionName}"}`,