Backfill Lambda layer versions for multi-region consistency

.github/workflows/publish.yml

@@ -20,8 +20,27 @@ permissions:
   contents: read
 
 jobs:
+  backfill:
+    if: inputs.environment == 'prod-integration' && inputs.region != 'eu-west-1'
+    name: Backfill layer versions
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+          aws-region: ${{ inputs.region }}
+      - name: Backfill layer version counters
+        env:
+          REGION: ${{ inputs.region }}
+        run: tools/backfill-layer-versions.sh eu-west-1 "$REGION"
+
   deploy-python:
     name: Build and Publish Python
+    needs: backfill
+    # Run when backfill succeeded or was skipped (dev, eu-west-1), but not when it failed.
+    if: ${{ !cancelled() && (needs.backfill.result == 'success' || needs.backfill.result == 'skipped') }}
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
     steps:
@@ -47,6 +66,8 @@ jobs:
 
   deploy-node:
     name: Build and Publish Node.js
+    needs: backfill
+    if: ${{ !cancelled() && (needs.backfill.result == 'success' || needs.backfill.result == 'skipped') }}
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
     steps:
@@ -75,6 +96,8 @@ jobs:
 
   deploy-java:
     name: Build and Publish Java
+    needs: backfill
+    if: ${{ !cancelled() && (needs.backfill.result == 'success' || needs.backfill.result == 'skipped') }}
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
     steps:
@@ -100,6 +123,8 @@ jobs:
 
   deploy-manual:
     name: Build and Publish Manual
+    needs: backfill
+    if: ${{ !cancelled() && (needs.backfill.result == 'success' || needs.backfill.result == 'skipped') }}
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
     steps:

integration-tests/tests/src/config.ts

@@ -3,4 +3,4 @@ export const DASH0_LAMBDA_TESTS_DATASET = 'lambda-extension-tests';
 export const DASH0_TOKEN = process.env.DASH0_DEV_API_TOKEN!;
 export const MAX_ATTEMPTS = 50;
 export const RETRY_DELAY_MS = 10_000;
-export const TEST_TIMEOUT_MS = 10 * 60_000; // 10 minutes
+export const TEST_TIMEOUT_MS = 20 * 60_000; // 20 minutes

integration-tests/tests/src/test-00-retries.test.ts

@@ -135,7 +135,7 @@ describe.concurrent('Retry Scenarios', () => {
                 console.log(`Starting retry test for ${runtime}`, new Date().toISOString());
                 await verifyRetryScenario(producerFunctionName, consumerFunctionName);
             },
-            600_000,
+            1_200_000,
         );
     }
 });

integration-tests/tests/src/test-tracing-scenarios-general.test.ts

@@ -152,7 +152,7 @@ describe.concurrent('Tracing Scenarios', () => {
                     console.log(`Starting test for ${scenario.name} scenario with ${runtime}`, new Date().toISOString());
                     await verifyTracingScenario(producerFunctionName, consumerFunctionName, scenario.name);
                 },
-                180_000
+                360_000
             );
         }
     }

integration-tests/tests/src/utils.ts

@@ -50,7 +50,7 @@
             },
         ],
         timeRange: {
-            from: new Date(now - 5 * 60_000).toISOString(),
+            from: new Date(now - 20 * 60_000).toISOString(),
             to: new Date(now + 5 * 60_000).toISOString(),
         },
         sampling: { mode: 'adaptive' },
@@ -63,7 +63,7 @@
 ) : Promise<string> => {
     const payload = eventPayload ? eventPayload : JSON.stringify({ parameter1: 'right' });
 
     const response = await lambdaClient.send(
         new InvokeCommand({
             FunctionName: functionName,
             InvocationType: 'RequestResponse',
@@ -131,7 +131,7 @@
                         },
                     ],
                     timeRange: {
-                        from: new Date(now - 10 * 60_000).toISOString(),
+                        from: new Date(now - 20 * 60_000).toISOString(),
                         to: new Date(now + 10 * 60_000).toISOString(),
                     },
                     sampling: {mode: 'adaptive'},
@@ -478,7 +478,7 @@
             try {
                 const params = new URLSearchParams({
                     dataset: DASH0_LAMBDA_TESTS_DATASET,
-                    start: 'now-10m',
+                    start: 'now-20m',
                     end: 'now',
                     step: '1m',
                     query: `{otel_metric_name = "${metricName}", otel_metric_type = "histogram", service_name = "${functionName}"}`,

tools/backfill-layer-versions.sh

@@ -0,0 +1,187 @@
+#!/usr/bin/env bash
+#
+# Align Lambda layer version counters in a newly launched AWS region.
+#
+# Usage: backfill-layer-versions.sh [--dry-run] <source-region> <target-region> [layer-name ...]
+#
+# Lambda layer version numbers auto-increment per region and can never be
+# chosen. When we start deploying to a new region (e.g. il-central-1), its
+# layers would begin at version 1 while every existing region is at some
+# aligned version N — breaking tooling that assumes the same layer name +
+# version resolves in every region (e.g. the sls-plugin ARN builder).
+#
+# The version counter never resets, even when versions are deleted. So for
+# each layer this script reads the highest version N from the source region,
+# then publishes dummy versions (a minimal placeholder zip) in the target
+# region — deleting each one immediately — until the counter reaches N.
+# The next real deploy then produces version N+1 in the new region and in
+# all existing regions alike.
+#
+# Notes:
+#   - Dummy versions are deleted right after publishing; only the counter
+#     moves. Any real versions already present in the target keep their zips.
+#   - If the target counter is already ahead of the source (possible when
+#     versions were published *and deleted* there before — the counter is
+#     not observable via the API), alignment is impossible and the layer is
+#     reported as failed.
+#
+# Prerequisites: AWS CLI v2, jq, credentials for the account owning the
+# layers (same account for both regions), target region opted in.
+set -euo pipefail
+
+# Lean on the CLI's built-in throttle handling instead of a retry loop.
+export AWS_RETRY_MODE=adaptive
+export AWS_MAX_ATTEMPTS=10
+
+usage="usage: $0 [--dry-run] <source-region> <target-region> [layer-name ...]"
+
+dry_run=false
+if [ "${1:-}" = "--dry-run" ]; then
+  dry_run=true
+  shift
+fi
+
+source_region="${1:?${usage}}"; shift
+target_region="${1:?${usage}}"; shift
+
+if [ "${source_region}" = "${target_region}" ]; then
+  echo "error: source and target region are both '${source_region}'" >&2
+  exit 1
+fi
+
+if [ "$#" -gt 0 ]; then
+  layers=("$@")
+else
+  layers=(dash0-extension-python dash0-extension-node dash0-extension-java dash0-extension-manual)
+fi
+
+workdir="$(mktemp -d)"
+trap 'rm -rf "${workdir}"' EXIT
+
+dummy_zip="${workdir}/dummy-layer.zip"
+echo "Placeholder to align Lambda layer version numbering across regions. Deleted immediately after publishing." \
+  > "${workdir}/README-dummy.txt"
+(cd "${workdir}" && zip -q "${dummy_zip}" README-dummy.txt)
+
+dummy_description="Placeholder to align layer version numbering across regions - deleted immediately"
+
+# Highest version of a layer in a region ("0" if the layer does not exist).
+# list-layer-versions returns versions newest-first.
+max_version() {
+  local layer="$1" region="$2" out
+  if ! out="$(aws lambda list-layer-versions --layer-name "${layer}" --region "${region}" \
+      --max-items 1 --query 'LayerVersions[0].Version' --output text --no-cli-pager 2>&1)"; then
+    if grep -q ResourceNotFoundException <<< "${out}"; then
+      echo 0
+      return 0
+    fi
+    echo "${out}" >&2
+    return 1
+  fi
+  # Extract first non-empty line and replace None with 0
+  echo "${out}" | grep -v '^None$' | grep . | head -1 | sed 's/^None$/0/'
+}
+
+# All existing versions of a layer in a region, oldest-first, space-separated
+# (empty if the layer does not exist).
+list_versions() {
+  local layer="$1" region="$2" out
+  if ! out="$(aws lambda list-layer-versions --layer-name "${layer}" --region "${region}" \
+      --query 'LayerVersions[].Version' --output json --no-cli-pager 2>&1)"; then
+    if grep -q ResourceNotFoundException <<< "${out}"; then
+      return 0
+    fi
+    echo "${out}" >&2
+    return 1
+  fi
+  jq -r 'sort | map(tostring) | join(" ")' <<< "${out}"
+}
+
+failed_layers=()
+summary=()
+
+for layer in "${layers[@]}"; do
+  echo "=== ${layer} ==="
+
+  source_max="$(max_version "${layer}" "${source_region}")"
+  if [ "${source_max}" -eq 0 ]; then
+    echo "error: layer '${layer}' does not exist in source region ${source_region}; skipping" >&2
+    failed_layers+=("${layer}")
+    summary+=("${layer}: FAILED (not found in ${source_region})")
+    continue
+  fi
+  echo "source ${source_region}: highest version ${source_max}"
+
+  target_versions="$(list_versions "${layer}" "${target_region}")"
+  if [ -n "${target_versions}" ]; then
+    echo "target ${target_region}: existing versions: ${target_versions}"
+    target_listed_max="${target_versions##* }"
+  else
+    echo "target ${target_region}: layer does not exist yet"
+    target_listed_max=0
+  fi
+
+  if [ "${target_listed_max}" -gt "${source_max}" ]; then
+    echo "error: target is already at version ${target_listed_max}, ahead of source (${source_max}); cannot align" >&2
+    failed_layers+=("${layer}")
+    summary+=("${layer}: FAILED (target ahead: ${target_listed_max} > ${source_max})")
+    continue
+  fi
+  if [ "${target_listed_max}" -eq "${source_max}" ]; then
+    echo "already aligned at version ${source_max}; nothing to do"
+    summary+=("${layer}: already aligned at ${source_max}")
+    continue
+  fi
+
+  to_publish=$(( source_max - target_listed_max ))
+  if "${dry_run}"; then
+    echo "dry-run: would publish+delete ${to_publish} dummy version(s) ($((target_listed_max + 1))..${source_max} expected)"
+    summary+=("${layer}: dry-run, would publish+delete ${to_publish} dummy version(s) to reach ${source_max}")
+    continue
+  fi
+
+  published=0
+  layer_failed=false
+  while true; do
+    v="$(aws lambda publish-layer-version --layer-name "${layer}" --region "${target_region}" \
+      --zip-file "fileb://${dummy_zip}" \
+      --description "${dummy_description}" \
+      --query Version --output text --no-cli-pager)"
+    if ! aws lambda delete-layer-version --layer-name "${layer}" --region "${target_region}" \
+        --version-number "${v}" --no-cli-pager; then
+      echo "error: failed to delete dummy version ${v} of '${layer}' in ${target_region};" \
+        "delete it manually before re-running" >&2
+      exit 1
+    fi
+    if [ "${v}" -gt "${source_max}" ]; then
+      # The target's hidden counter was already at/above source_max (versions
+      # published and deleted there before). The counter only moves forward,
+      # so alignment is impossible.
+      echo "error: dummy publish produced version ${v} > source max ${source_max}; target counter is ahead, cannot align" >&2
+      layer_failed=true
+      break
+    fi
+    published=$(( published + 1 ))
+    echo "[${layer}] published+deleted dummy version ${v}/${source_max}"
+    if [ "${v}" -eq "${source_max}" ]; then
+      break
+    fi
+  done
+
+  if "${layer_failed}"; then
+    failed_layers+=("${layer}")
+    summary+=("${layer}: FAILED (target counter ahead of ${source_max})")
+  else
+    summary+=("${layer}: aligned at ${source_max} (${published} dummy version(s) published+deleted)")
+  fi
+done
+
+echo
+echo "=== summary (${source_region} -> ${target_region}) ==="
+for line in "${summary[@]}"; do
+  echo "${line}"
+done
+
+if [ "${#failed_layers[@]}" -gt 0 ]; then
+  exit 1
+fi