diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2962ea6e..ffdfcc91 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,8 +20,27 @@ permissions: contents: read jobs: + backfill: + if: inputs.environment == 'prod-integration' && inputs.region != 'eu-west-1' + name: Backfill layer versions + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + steps: + - uses: actions/checkout@v4 + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ inputs.region }} + - name: Backfill layer version counters + env: + REGION: ${{ inputs.region }} + run: tools/backfill-layer-versions.sh eu-west-1 "$REGION" + deploy-python: name: Build and Publish Python + needs: backfill + # Run when backfill succeeded or was skipped (dev, eu-west-1), but not when it failed. + if: ${{ !cancelled() && (needs.backfill.result == 'success' || needs.backfill.result == 'skipped') }} runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: @@ -47,6 +66,8 @@ jobs: deploy-node: name: Build and Publish Node.js + needs: backfill + if: ${{ !cancelled() && (needs.backfill.result == 'success' || needs.backfill.result == 'skipped') }} runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: @@ -75,6 +96,8 @@ jobs: deploy-java: name: Build and Publish Java + needs: backfill + if: ${{ !cancelled() && (needs.backfill.result == 'success' || needs.backfill.result == 'skipped') }} runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: @@ -100,6 +123,8 @@ jobs: deploy-manual: name: Build and Publish Manual + needs: backfill + if: ${{ !cancelled() && (needs.backfill.result == 'success' || needs.backfill.result == 'skipped') }} runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: diff --git a/integration-tests/tests/src/config.ts b/integration-tests/tests/src/config.ts index 16b194ca..1fb32133 100644 --- a/integration-tests/tests/src/config.ts +++ b/integration-tests/tests/src/config.ts @@ -3,4 +3,4 @@ export const DASH0_LAMBDA_TESTS_DATASET = 'lambda-extension-tests'; export const DASH0_TOKEN = process.env.DASH0_DEV_API_TOKEN!; export const MAX_ATTEMPTS = 50; export const RETRY_DELAY_MS = 10_000; -export const TEST_TIMEOUT_MS = 10 * 60_000; // 10 minutes \ No newline at end of file +export const TEST_TIMEOUT_MS = 20 * 60_000; // 20 minutes \ No newline at end of file diff --git a/integration-tests/tests/src/test-00-retries.test.ts b/integration-tests/tests/src/test-00-retries.test.ts index 286da25f..0e19e011 100644 --- a/integration-tests/tests/src/test-00-retries.test.ts +++ b/integration-tests/tests/src/test-00-retries.test.ts @@ -135,7 +135,7 @@ describe.concurrent('Retry Scenarios', () => { console.log(`Starting retry test for ${runtime}`, new Date().toISOString()); await verifyRetryScenario(producerFunctionName, consumerFunctionName); }, - 600_000, + 1_200_000, ); } }); diff --git a/integration-tests/tests/src/test-tracing-scenarios-general.test.ts b/integration-tests/tests/src/test-tracing-scenarios-general.test.ts index b173f469..ef0d85d3 100644 --- a/integration-tests/tests/src/test-tracing-scenarios-general.test.ts +++ b/integration-tests/tests/src/test-tracing-scenarios-general.test.ts @@ -152,7 +152,7 @@ describe.concurrent('Tracing Scenarios', () => { console.log(`Starting test for ${scenario.name} scenario with ${runtime}`, new Date().toISOString()); await verifyTracingScenario(producerFunctionName, consumerFunctionName, scenario.name); }, - 180_000 + 360_000 ); } } diff --git a/integration-tests/tests/src/utils.ts b/integration-tests/tests/src/utils.ts index 39d074ba..465142a7 100644 --- a/integration-tests/tests/src/utils.ts +++ b/integration-tests/tests/src/utils.ts @@ -50,7 +50,7 @@ export const getRequestPayload = (invocationId: string) => { }, ], timeRange: { - from: new Date(now - 5 * 60_000).toISOString(), + from: new Date(now - 20 * 60_000).toISOString(), to: new Date(now + 5 * 60_000).toISOString(), }, sampling: { mode: 'adaptive' }, @@ -131,7 +131,7 @@ export const checkHttpSpan = async ({ }, ], timeRange: { - from: new Date(now - 10 * 60_000).toISOString(), + from: new Date(now - 20 * 60_000).toISOString(), to: new Date(now + 10 * 60_000).toISOString(), }, sampling: {mode: 'adaptive'}, @@ -478,7 +478,7 @@ export const checkMetrics = async ({ try { const params = new URLSearchParams({ dataset: DASH0_LAMBDA_TESTS_DATASET, - start: 'now-10m', + start: 'now-20m', end: 'now', step: '1m', query: `{otel_metric_name = "${metricName}", otel_metric_type = "histogram", service_name = "${functionName}"}`, diff --git a/tools/backfill-layer-versions.sh b/tools/backfill-layer-versions.sh new file mode 100755 index 00000000..2cb31445 --- /dev/null +++ b/tools/backfill-layer-versions.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# +# Align Lambda layer version counters in a newly launched AWS region. +# +# Usage: backfill-layer-versions.sh [--dry-run] [layer-name ...] +# +# Lambda layer version numbers auto-increment per region and can never be +# chosen. When we start deploying to a new region (e.g. il-central-1), its +# layers would begin at version 1 while every existing region is at some +# aligned version N — breaking tooling that assumes the same layer name + +# version resolves in every region (e.g. the sls-plugin ARN builder). +# +# The version counter never resets, even when versions are deleted. So for +# each layer this script reads the highest version N from the source region, +# then publishes dummy versions (a minimal placeholder zip) in the target +# region — deleting each one immediately — until the counter reaches N. +# The next real deploy then produces version N+1 in the new region and in +# all existing regions alike. +# +# Notes: +# - Dummy versions are deleted right after publishing; only the counter +# moves. Any real versions already present in the target keep their zips. +# - If the target counter is already ahead of the source (possible when +# versions were published *and deleted* there before — the counter is +# not observable via the API), alignment is impossible and the layer is +# reported as failed. +# +# Prerequisites: AWS CLI v2, jq, credentials for the account owning the +# layers (same account for both regions), target region opted in. +set -euo pipefail + +# Lean on the CLI's built-in throttle handling instead of a retry loop. +export AWS_RETRY_MODE=adaptive +export AWS_MAX_ATTEMPTS=10 + +usage="usage: $0 [--dry-run] [layer-name ...]" + +dry_run=false +if [ "${1:-}" = "--dry-run" ]; then + dry_run=true + shift +fi + +source_region="${1:?${usage}}"; shift +target_region="${1:?${usage}}"; shift + +if [ "${source_region}" = "${target_region}" ]; then + echo "error: source and target region are both '${source_region}'" >&2 + exit 1 +fi + +if [ "$#" -gt 0 ]; then + layers=("$@") +else + layers=(dash0-extension-python dash0-extension-node dash0-extension-java dash0-extension-manual) +fi + +workdir="$(mktemp -d)" +trap 'rm -rf "${workdir}"' EXIT + +dummy_zip="${workdir}/dummy-layer.zip" +echo "Placeholder to align Lambda layer version numbering across regions. Deleted immediately after publishing." \ + > "${workdir}/README-dummy.txt" +(cd "${workdir}" && zip -q "${dummy_zip}" README-dummy.txt) + +dummy_description="Placeholder to align layer version numbering across regions - deleted immediately" + +# Highest version of a layer in a region ("0" if the layer does not exist). +# list-layer-versions returns versions newest-first. +max_version() { + local layer="$1" region="$2" out + if ! out="$(aws lambda list-layer-versions --layer-name "${layer}" --region "${region}" \ + --max-items 1 --query 'LayerVersions[0].Version' --output text --no-cli-pager 2>&1)"; then + if grep -q ResourceNotFoundException <<< "${out}"; then + echo 0 + return 0 + fi + echo "${out}" >&2 + return 1 + fi + # Extract first non-empty line and replace None with 0 + echo "${out}" | grep -v '^None$' | grep . | head -1 | sed 's/^None$/0/' +} + +# All existing versions of a layer in a region, oldest-first, space-separated +# (empty if the layer does not exist). +list_versions() { + local layer="$1" region="$2" out + if ! out="$(aws lambda list-layer-versions --layer-name "${layer}" --region "${region}" \ + --query 'LayerVersions[].Version' --output json --no-cli-pager 2>&1)"; then + if grep -q ResourceNotFoundException <<< "${out}"; then + return 0 + fi + echo "${out}" >&2 + return 1 + fi + jq -r 'sort | map(tostring) | join(" ")' <<< "${out}" +} + +failed_layers=() +summary=() + +for layer in "${layers[@]}"; do + echo "=== ${layer} ===" + + source_max="$(max_version "${layer}" "${source_region}")" + if [ "${source_max}" -eq 0 ]; then + echo "error: layer '${layer}' does not exist in source region ${source_region}; skipping" >&2 + failed_layers+=("${layer}") + summary+=("${layer}: FAILED (not found in ${source_region})") + continue + fi + echo "source ${source_region}: highest version ${source_max}" + + target_versions="$(list_versions "${layer}" "${target_region}")" + if [ -n "${target_versions}" ]; then + echo "target ${target_region}: existing versions: ${target_versions}" + target_listed_max="${target_versions##* }" + else + echo "target ${target_region}: layer does not exist yet" + target_listed_max=0 + fi + + if [ "${target_listed_max}" -gt "${source_max}" ]; then + echo "error: target is already at version ${target_listed_max}, ahead of source (${source_max}); cannot align" >&2 + failed_layers+=("${layer}") + summary+=("${layer}: FAILED (target ahead: ${target_listed_max} > ${source_max})") + continue + fi + if [ "${target_listed_max}" -eq "${source_max}" ]; then + echo "already aligned at version ${source_max}; nothing to do" + summary+=("${layer}: already aligned at ${source_max}") + continue + fi + + to_publish=$(( source_max - target_listed_max )) + if "${dry_run}"; then + echo "dry-run: would publish+delete ${to_publish} dummy version(s) ($((target_listed_max + 1))..${source_max} expected)" + summary+=("${layer}: dry-run, would publish+delete ${to_publish} dummy version(s) to reach ${source_max}") + continue + fi + + published=0 + layer_failed=false + while true; do + v="$(aws lambda publish-layer-version --layer-name "${layer}" --region "${target_region}" \ + --zip-file "fileb://${dummy_zip}" \ + --description "${dummy_description}" \ + --query Version --output text --no-cli-pager)" + if ! aws lambda delete-layer-version --layer-name "${layer}" --region "${target_region}" \ + --version-number "${v}" --no-cli-pager; then + echo "error: failed to delete dummy version ${v} of '${layer}' in ${target_region};" \ + "delete it manually before re-running" >&2 + exit 1 + fi + if [ "${v}" -gt "${source_max}" ]; then + # The target's hidden counter was already at/above source_max (versions + # published and deleted there before). The counter only moves forward, + # so alignment is impossible. + echo "error: dummy publish produced version ${v} > source max ${source_max}; target counter is ahead, cannot align" >&2 + layer_failed=true + break + fi + published=$(( published + 1 )) + echo "[${layer}] published+deleted dummy version ${v}/${source_max}" + if [ "${v}" -eq "${source_max}" ]; then + break + fi + done + + if "${layer_failed}"; then + failed_layers+=("${layer}") + summary+=("${layer}: FAILED (target counter ahead of ${source_max})") + else + summary+=("${layer}: aligned at ${source_max} (${published} dummy version(s) published+deleted)") + fi +done + +echo +echo "=== summary (${source_region} -> ${target_region}) ===" +for line in "${summary[@]}"; do + echo "${line}" +done + +if [ "${#failed_layers[@]}" -gt 0 ]; then + exit 1 +fi