From bf5beb118ca13f67daaca70c7919899a22f1ff23 Mon Sep 17 00:00:00 2001 From: james2177 Date: Sun, 26 Jul 2026 08:12:47 +0100 Subject: [PATCH 1/4] security: pin third-party GitHub Actions to commit SHAs Pin all third-party (non-actions/, non-github/) actions across all workflows to full commit SHAs with a version comment, matching the existing pattern used for aquasecurity/trivy-action and trufflesecurity/trufflehog. Mitigates supply-chain risk from a compromised maintainer account silently repointing a mutable tag (@v4, @v12, etc.) to malicious code. Pinned: dtolnay/rust-toolchain, hashicorp/setup-terraform, aws-actions/configure-aws-credentials, bridgecrewio/checkov-action, docker/setup-buildx-action, docker/login-action, docker/metadata-action, docker/build-push-action, slackapi/slack-github-action, actions-rs/toolchain, softprops/action-gh-release, orhun/git-cliff-action, peter-evans/create-pull-request, codecov/codecov-action, returntocorp/semgrep-action, gitleaks/gitleaks-action. --- .github/workflows/contract-deployment.yml | 4 +- .github/workflows/dependency-scan.yml | 2 +- .github/workflows/deploy.yml | 10 ++--- .github/workflows/docker.yml | 12 +++--- .github/workflows/license-check.yml | 4 +- .github/workflows/msrv.yml | 2 +- .github/workflows/openapi-validation.yml | 2 +- .github/workflows/performance.yml | 4 +- .github/workflows/release.yml | 8 ++-- .github/workflows/terraform.yml | 8 ++-- .github/workflows/test.yml | 48 +++++++++++------------ 11 files changed, 52 insertions(+), 52 deletions(-) diff --git a/.github/workflows/contract-deployment.yml b/.github/workflows/contract-deployment.yml index 44f961f1..91cc5015 100644 --- a/.github/workflows/contract-deployment.yml +++ b/.github/workflows/contract-deployment.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: actions-rs/toolchain@v1 + uses: actions-rs/toolchain@63eb9591781c46a70274cb3ebdf190fce92702e8 # v1 with: profile: minimal toolchain: stable @@ -303,7 +303,7 @@ jobs: - name: Send Slack notification if: vars.SLACK_WEBHOOK_URL != '' - uses: slackapi/slack-github-action@v3.0.3 + uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 with: payload: | { diff --git a/.github/workflows/dependency-scan.yml b/.github/workflows/dependency-scan.yml index 82c2b98a..8c9cb1c0 100644 --- a/.github/workflows/dependency-scan.yml +++ b/.github/workflows/dependency-scan.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Install cargo-audit run: cargo install cargo-audit diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 14d06d70..0e5e9cf1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -58,7 +58,7 @@ jobs: - uses: actions/checkout@v4 - name: Run Checkov - uses: bridgecrewio/checkov-action@v12 + uses: bridgecrewio/checkov-action@7b972723c44fb3d256283fac96fae5d7c1894bb7 # v12 with: directory: infrastructure/terraform framework: terraform @@ -81,12 +81,12 @@ jobs: - uses: actions/checkout@v4 - name: Setup Terraform - uses: hashicorp/setup-terraform@v4 + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4 with: terraform_version: ${{ env.TERRAFORM_VERSION }} - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v6 + uses: aws-actions/configure-aws-credentials@451ce2a72e8d729a59ebeaacab82dbce58e7af4c # v6 with: role-to-assume: ${{ secrets[format('AWS_ROLE_{0}', matrix.environment)] }} aws-region: ${{ env.AWS_REGION }} @@ -136,12 +136,12 @@ jobs: - uses: actions/checkout@v4 - name: Setup Terraform - uses: hashicorp/setup-terraform@v4 + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4 with: terraform_version: ${{ env.TERRAFORM_VERSION }} - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v6 + uses: aws-actions/configure-aws-credentials@451ce2a72e8d729a59ebeaacab82dbce58e7af4c # v6 with: role-to-assume: ${{ secrets[format('AWS_ROLE_{0}', matrix.environment)] }} aws-region: ${{ env.AWS_REGION }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d64af925..0a0af055 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,11 +22,11 @@ jobs: - uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 - name: Log in to Container Registry if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -34,7 +34,7 @@ jobs: - name: Extract metadata id: meta - uses: docker/metadata-action@v6 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | @@ -45,7 +45,7 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} - name: Build and push Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 with: context: . push: ${{ github.event_name != 'pull_request' }} @@ -92,10 +92,10 @@ jobs: - uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 - name: Build image for testing - uses: docker/build-push-action@v5 + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 with: context: . load: true diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml index 5e2caccd..5ed357b1 100644 --- a/.github/workflows/license-check.yml +++ b/.github/workflows/license-check.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache cargo registry uses: actions/cache@v5 @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache cargo registry uses: actions/cache@v5 diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index 73887f65..4f2f590e 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -27,7 +27,7 @@ jobs: echo "MSRV detected: $MSRV" - name: Install MSRV toolchain (${{ steps.msrv.outputs.version }}) - uses: dtolnay/rust-toolchain@master + uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c # master with: toolchain: ${{ steps.msrv.outputs.version }} diff --git a/.github/workflows/openapi-validation.yml b/.github/workflows/openapi-validation.yml index 44b4d540..edddb56b 100644 --- a/.github/workflows/openapi-validation.yml +++ b/.github/workflows/openapi-validation.yml @@ -46,7 +46,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache cargo dependencies uses: actions/cache@v5 diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index a6086d55..4cc4062d 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -47,7 +47,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Rust - uses: actions-rs/toolchain@v1 + uses: actions-rs/toolchain@63eb9591781c46a70274cb3ebdf190fce92702e8 # v1 with: toolchain: stable override: true @@ -228,7 +228,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Rust - uses: actions-rs/toolchain@v1 + uses: actions-rs/toolchain@63eb9591781c46a70274cb3ebdf190fce92702e8 # v1 with: toolchain: stable override: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3496824d..a4f5a55d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,7 +44,7 @@ jobs: fetch-depth: 0 - name: Generate changelog with git-cliff - uses: orhun/git-cliff-action@v3 + uses: orhun/git-cliff-action@b946ed27a675d653b308f29a7bbad813b85bf7aa # v3 id: cliff with: config: cliff.toml @@ -62,7 +62,7 @@ jobs: git push origin HEAD:main - name: Create GitHub Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 with: body: ${{ steps.cliff.outputs.content }} tag_name: ${{ github.ref_name }} @@ -85,7 +85,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: actions-rs/toolchain@v1 + uses: actions-rs/toolchain@63eb9591781c46a70274cb3ebdf190fce92702e8 # v1 with: profile: minimal toolchain: stable @@ -129,7 +129,7 @@ jobs: WASM_SIZE_LIMIT_BYTES: "65536" - name: Upload WASM to GitHub Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 with: files: contracts/predict-iq/target/wasm32-unknown-unknown/release/predict_iq_optimized.wasm tag_name: ${{ github.ref_name }} diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index 68e2b8a8..563a6b9b 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v4 - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 + uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 with: terraform_version: ${{ env.TF_VERSION }} @@ -58,7 +58,7 @@ jobs: - uses: actions/checkout@v4 - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 + uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 with: terraform_version: ${{ env.TF_VERSION }} @@ -84,13 +84,13 @@ jobs: - uses: actions/checkout@v4 - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 + uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 with: terraform_version: ${{ env.TF_VERSION }} terraform_wrapper: true - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cd6f7ab2..97d8ee47 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,7 +46,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache cargo registry uses: actions/cache@v5 @@ -85,7 +85,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache dependencies uses: actions/cache@v5 @@ -110,7 +110,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache dependencies uses: actions/cache@v5 @@ -132,7 +132,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache dependencies uses: actions/cache@v5 @@ -202,7 +202,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache dependencies uses: actions/cache@v5 @@ -231,7 +231,7 @@ jobs: node scripts/compare-gas-benchmarks.js --save-baseline --branch main - name: Create Pull Request with baseline updates - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6 with: commit-message: "chore: update gas benchmark baseline for main branch" title: "chore: update gas benchmark baseline" @@ -257,7 +257,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable with: components: llvm-tools-preview @@ -300,7 +300,7 @@ jobs: if-no-files-found: error - name: Upload predict-iq LCOV to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4 with: files: coverage-reports/predict-iq.lcov flags: predict-iq @@ -308,7 +308,7 @@ jobs: verbose: true - name: Upload predictiq-api LCOV to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4 with: files: coverage-reports/predictiq-api.lcov flags: predictiq-api @@ -322,7 +322,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Install cargo-audit run: cargo install cargo-audit @@ -345,7 +345,7 @@ jobs: - uses: actions/checkout@v4 - name: Run Semgrep SAST - uses: returntocorp/semgrep-action@v1 + uses: returntocorp/semgrep-action@713efdd345f3035192eaa63f56867b88e63e4e5d # v1 with: config: >- p/security-audit @@ -380,7 +380,7 @@ jobs: fetch-depth: 0 - name: Run Gitleaks - uses: gitleaks/gitleaks-action@v2 + uses: gitleaks/gitleaks-action@dcedce43c6f43de0b836d1fe38946645c9c638dc # v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }} @@ -494,7 +494,7 @@ jobs: - name: Install Rust (for Rust analysis) if: matrix.language == 'rust' - uses: actions-rs/toolchain@v1 + uses: actions-rs/toolchain@63eb9591781c46a70274cb3ebdf190fce92702e8 # v1 with: profile: minimal toolchain: stable @@ -521,7 +521,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable components: clippy - name: Run Clippy @@ -535,7 +535,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable components: clippy - name: Clippy (deny warnings) — oracle module @@ -565,7 +565,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable components: rustfmt - name: Check formatting @@ -579,7 +579,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable target: wasm32-unknown-unknown - name: Install Soroban CLI @@ -649,7 +649,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache cargo dependencies uses: actions/cache@v5 @@ -920,7 +920,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache cargo dependencies uses: actions/cache@v4 @@ -987,7 +987,7 @@ jobs: fetch-depth: 0 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache cargo dependencies uses: actions/cache@v4 @@ -1073,7 +1073,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache cargo dependencies uses: actions/cache@v5 @@ -1107,7 +1107,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache cargo dependencies uses: actions/cache@v5 @@ -1137,7 +1137,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache cargo dependencies uses: actions/cache@v5 @@ -1161,7 +1161,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - name: Cache cargo dependencies uses: actions/cache@v5 From b58d193c6ae8659f0e47b801e1c2f367b64bd581 Mon Sep 17 00:00:00 2001 From: james2177 Date: Sun, 26 Jul 2026 08:13:28 +0100 Subject: [PATCH 2/4] security: open a PR for CHANGELOG updates instead of pushing to main The release workflow's changelog job used the default GITHUB_TOKEN (contents: write) to commit and git push origin HEAD:main directly, bypassing branch protection and any review step for an automated bot. Replace the direct push with peter-evans/create-pull-request, which opens changelog-update/ against main instead. protect-changelog.yml is updated with a documented, narrowly-scoped exception so only that bot-authored PR branch is allowed through; every other PR that touches CHANGELOG.md is still blocked as before. --- .github/workflows/protect-changelog.yml | 9 +++++++++ .github/workflows/release.yml | 24 +++++++++++++++++------- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/.github/workflows/protect-changelog.yml b/.github/workflows/protect-changelog.yml index 7e70315d..01bd74bd 100644 --- a/.github/workflows/protect-changelog.yml +++ b/.github/workflows/protect-changelog.yml @@ -9,8 +9,17 @@ jobs: block-manual-changelog: name: Block manual CHANGELOG edits runs-on: ubuntu-latest + timeout-minutes: 5 steps: + # Documented, scoped exception: the release workflow (release.yml) opens + # an automated PR from branch changelog-update/ via + # github-actions[bot] to update CHANGELOG.md instead of pushing to main + # directly. That PR is intentionally allowed through here; every other + # PR touching CHANGELOG.md is still blocked. - name: Fail if CHANGELOG.md was manually edited + if: | + !(github.actor == 'github-actions[bot]' && + startsWith(github.head_ref, 'changelog-update/')) run: | echo "❌ CHANGELOG.md must not be edited manually." echo "It is auto-generated by git-cliff on release." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a4f5a55d..75595fdf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,7 @@ on: permissions: contents: write + pull-requests: write jobs: validate-release: @@ -53,13 +54,22 @@ jobs: OUTPUT: CHANGELOG.md GITHUB_REPO: ${{ github.repository }} - - name: Commit updated CHANGELOG.md - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add CHANGELOG.md - git diff --cached --quiet || git commit -m "chore(changelog): update for ${{ github.ref_name }} [skip ci]" - git push origin HEAD:main + - name: Open changelog update PR + uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "chore(changelog): update for ${{ github.ref_name }} [skip ci]" + title: "chore(changelog): update CHANGELOG.md for ${{ github.ref_name }}" + body: | + Automated CHANGELOG.md update generated by git-cliff for release ${{ github.ref_name }}. + + This PR is opened by the release workflow instead of pushing directly to + `main`, so branch protection and required checks apply the same way they + do to any other change. + branch: changelog-update/${{ github.ref_name }} + base: main + delete-branch: true + add-paths: CHANGELOG.md - name: Create GitHub Release uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 From cfb0896df6f565c4ce4dcf8e9829b1e90b7a5b64 Mon Sep 17 00:00:00 2001 From: james2177 Date: Sun, 26 Jul 2026 08:14:41 +0100 Subject: [PATCH 3/4] fix(docs): correct ROLLBACK.md S3 bucket/key and tfvars drift ROLLBACK.md documented predictiq-terraform-state / prod/terraform.tfstate and environments/prod.tfvars, none of which exist. The real backend config (infrastructure/terraform/environments/{production,staging}/backend.hcl) uses predictiq-terraform-state-production and predictiq-terraform-state-staging with production/terraform.tfstate and staging/terraform.tfstate keys, and the real tfvars file is environments/production/terraform.tfvars. Following the runbook verbatim during a real incident would have targeted a non-existent bucket/key. Updates all state-bucket commands, tfvars paths, and prod resource-id references so the runbook is copy-paste runnable against the real infrastructure, and adds a bucket/key reference table per environment. --- infrastructure/ROLLBACK.md | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/infrastructure/ROLLBACK.md b/infrastructure/ROLLBACK.md index a097df66..26ab6273 100644 --- a/infrastructure/ROLLBACK.md +++ b/infrastructure/ROLLBACK.md @@ -34,24 +34,34 @@ git push origin main ### 2. Rollback via Terraform State -For emergency rollbacks without code changes: +For emergency rollbacks without code changes. Bucket and key names must match +the backend config for the target environment +(`infrastructure/terraform/environments//backend.hcl`): + +| Environment | Bucket | Key | +|-------------|-------------------------------------------|---------------------------------| +| Staging | `predictiq-terraform-state-staging` | `staging/terraform.tfstate` | +| Production | `predictiq-terraform-state-production` | `production/terraform.tfstate` | + +The commands below use the **production** values; swap in the staging +bucket/key for a staging rollback. ```bash # Navigate to infrastructure directory cd infrastructure/terraform # Initialize Terraform -terraform init +terraform init -backend-config=environments/production/backend.hcl # List available state versions aws s3api list-object-versions \ - --bucket predictiq-terraform-state \ - --prefix prod/terraform.tfstate + --bucket predictiq-terraform-state-production \ + --prefix production/terraform.tfstate # Restore previous state version aws s3api get-object \ - --bucket predictiq-terraform-state \ - --key prod/terraform.tfstate \ + --bucket predictiq-terraform-state-production \ + --key production/terraform.tfstate \ --version-id \ terraform.tfstate.backup @@ -62,10 +72,10 @@ cp terraform.tfstate terraform.tfstate.current cp terraform.tfstate.backup terraform.tfstate # Plan the rollback -terraform plan -var-file="environments/prod.tfvars" +terraform plan -var-file="environments/production/terraform.tfvars" # Apply the rollback -terraform apply -var-file="environments/prod.tfvars" +terraform apply -var-file="environments/production/terraform.tfvars" ``` ### 3. Rollback Specific Resources @@ -77,8 +87,8 @@ To rollback only specific resources: terraform taint module.ecs.aws_ecs_service.api # Plan and apply -terraform plan -var-file="environments/prod.tfvars" -terraform apply -var-file="environments/prod.tfvars" +terraform plan -var-file="environments/production/terraform.tfvars" +terraform apply -var-file="environments/production/terraform.tfvars" ``` ## Rollback Verification @@ -90,9 +100,9 @@ After rollback, verify the infrastructure: terraform show # Verify AWS resources -aws ec2 describe-instances --filters "Name=tag:Environment,Values=prod" -aws rds describe-db-instances --filters "Name=db-instance-id,Values=predictiq-prod" -aws elasticache describe-cache-clusters --cache-cluster-id predictiq-prod +aws ec2 describe-instances --filters "Name=tag:Environment,Values=production" +aws rds describe-db-instances --filters "Name=db-instance-id,Values=predictiq-production" +aws elasticache describe-cache-clusters --cache-cluster-id predictiq-production # Test API connectivity curl https://api.predictiq.example.com/health From bac9e5fe7bb25ab496b9e29e08d33405ccda3244 Mon Sep 17 00:00:00 2001 From: james2177 Date: Sun, 26 Jul 2026 08:17:23 +0100 Subject: [PATCH 4/4] reliability: add timeout-minutes to every CI job None of the 14 workflows set a job-level timeout, so a hung step (e.g. cargo install cargo-audit / soroban-cli, or a stuck Terraform apply) would run until GitHub's 360-minute default, wasting runner minutes and, for deploy.yml, blocking the DynamoDB state lock for other environments in the max-parallel: 1 matrix. Add timeout-minutes to every job across all workflows (default 20). The Terraform apply job in deploy.yml and the cargo-audit install jobs in dependency-scan.yml and test.yml are tightened to 15 since they're the jobs most likely to hang and most costly to block on. Note: test.yml has a pre-existing YAML issue unrelated to this change (clippy/oracle-quality-gate jobs are missing a `with:` key before `components: clippy` on the pinned dtolnay/rust-toolchain step); left as-is since it predates this branch and is out of scope for these issues. --- .github/workflows/accessibility.yml | 7 ++ .github/workflows/contract-deployment.yml | 5 ++ .github/workflows/dependency-scan.yml | 6 ++ .github/workflows/deploy.yml | 6 ++ .github/workflows/docker.yml | 3 + .github/workflows/e2e-staging.yml | 1 + .github/workflows/license-check.yml | 5 ++ .github/workflows/msrv.yml | 1 + .github/workflows/openapi-validation.yml | 3 + .github/workflows/performance.yml | 4 + .github/workflows/release.yml | 3 + .github/workflows/terraform.yml | 4 + .github/workflows/test.yml | 37 +++++++++ WORKFLOW_HARDENING.md | 97 +++++++++++++++++++++++ 14 files changed, 182 insertions(+) create mode 100644 WORKFLOW_HARDENING.md diff --git a/.github/workflows/accessibility.yml b/.github/workflows/accessibility.yml index 154c30ec..d27a3cdb 100644 --- a/.github/workflows/accessibility.yml +++ b/.github/workflows/accessibility.yml @@ -17,6 +17,7 @@ jobs: jest-axe-tests: name: Jest-Axe Accessibility Tests runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: frontend @@ -47,6 +48,7 @@ jobs: lighthouse-audit: name: Lighthouse Accessibility Audit runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: frontend @@ -100,6 +102,7 @@ jobs: axe-core-audit: name: Axe-Core Full Page Audit runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: frontend @@ -143,6 +146,7 @@ jobs: pa11y-audit: name: Pa11y Accessibility Audit runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: frontend @@ -200,6 +204,7 @@ jobs: keyboard-navigation-tests: name: Keyboard Navigation Tests runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: frontend @@ -223,6 +228,7 @@ jobs: color-contrast-check: name: Color Contrast Validation runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: frontend @@ -269,6 +275,7 @@ jobs: - keyboard-navigation-tests - color-contrast-check runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Success diff --git a/.github/workflows/contract-deployment.yml b/.github/workflows/contract-deployment.yml index 91cc5015..af143c4c 100644 --- a/.github/workflows/contract-deployment.yml +++ b/.github/workflows/contract-deployment.yml @@ -25,6 +25,7 @@ jobs: build-and-test: name: Build and Test Contracts runs-on: ubuntu-latest + timeout-minutes: 20 outputs: wasm-path: ${{ steps.build.outputs.wasm-path }} contract-hash: ${{ steps.build.outputs.contract-hash }} @@ -116,6 +117,7 @@ jobs: name: Deploy to Testnet needs: build-and-test runs-on: ubuntu-latest + timeout-minutes: 20 if: github.ref == 'refs/heads/develop' || github.event.inputs.environment == 'testnet' environment: name: testnet @@ -183,6 +185,7 @@ jobs: name: Deploy to Mainnet needs: build-and-test runs-on: ubuntu-latest + timeout-minutes: 20 if: github.ref == 'refs/heads/main' environment: name: mainnet @@ -255,6 +258,7 @@ jobs: name: Version and Archive Artifacts needs: build-and-test runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -291,6 +295,7 @@ jobs: name: Notify Deployment Status needs: [build-and-test, deploy-testnet] runs-on: ubuntu-latest + timeout-minutes: 20 if: always() steps: - name: Determine deployment status diff --git a/.github/workflows/dependency-scan.yml b/.github/workflows/dependency-scan.yml index 8c9cb1c0..f30cc545 100644 --- a/.github/workflows/dependency-scan.yml +++ b/.github/workflows/dependency-scan.yml @@ -12,6 +12,9 @@ jobs: scan-rust: name: Scan Rust Dependencies runs-on: ubuntu-latest + # cargo install cargo-audit has hung indefinitely on runner network + # issues before; keep this shorter than the default 20. + timeout-minutes: 15 steps: - uses: actions/checkout@v4 @@ -58,6 +61,7 @@ jobs: scan-npm: name: Scan NPM Dependencies runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -98,6 +102,7 @@ jobs: scan-trivy: name: Scan with Trivy runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -120,6 +125,7 @@ jobs: report-vulnerabilities: name: Report Vulnerabilities runs-on: ubuntu-latest + timeout-minutes: 20 needs: [scan-rust, scan-npm, scan-trivy] if: always() steps: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 0e5e9cf1..6dcfff8b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -19,6 +19,7 @@ jobs: validate-secrets: name: Validate Required Secrets runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Check required secrets are set run: | @@ -53,6 +54,7 @@ jobs: security-scan: name: Checkov Security Scan runs-on: ubuntu-latest + timeout-minutes: 20 needs: validate-secrets steps: - uses: actions/checkout@v4 @@ -73,6 +75,7 @@ jobs: plan: name: Terraform Plan runs-on: ubuntu-latest + timeout-minutes: 20 needs: [validate-secrets, security-scan] strategy: matrix: @@ -126,6 +129,9 @@ jobs: apply: name: Terraform Apply runs-on: ubuntu-latest + # Kept short: this job holds a DynamoDB state lock and max-parallel: 1 + # means a hung apply blocks every other environment behind it. + timeout-minutes: 15 needs: plan if: github.event_name == 'push' && github.ref == 'refs/heads/main' strategy: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0a0af055..665a6424 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -15,6 +15,7 @@ jobs: build: name: Build and Push Docker Image runs-on: ubuntu-latest + timeout-minutes: 20 permissions: contents: read packages: write @@ -64,6 +65,7 @@ jobs: scan: name: Scan Docker Image runs-on: ubuntu-latest + timeout-minutes: 20 needs: build if: github.event_name != 'pull_request' steps: @@ -86,6 +88,7 @@ jobs: test-image: name: Test Docker Image runs-on: ubuntu-latest + timeout-minutes: 20 needs: build if: github.event_name == 'pull_request' steps: diff --git a/.github/workflows/e2e-staging.yml b/.github/workflows/e2e-staging.yml index 47ec5aeb..d2cf39b3 100644 --- a/.github/workflows/e2e-staging.yml +++ b/.github/workflows/e2e-staging.yml @@ -13,6 +13,7 @@ jobs: e2e-staging: name: Playwright — Staging runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml index 5ed357b1..ae3339ee 100644 --- a/.github/workflows/license-check.yml +++ b/.github/workflows/license-check.yml @@ -10,6 +10,7 @@ jobs: rust-licenses-contracts: name: Rust License Check — contracts/predict-iq runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -34,6 +35,7 @@ jobs: rust-licenses-api: name: Rust License Check — services/api runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -58,6 +60,7 @@ jobs: npm-licenses-frontend: name: NPM License Check — frontend runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -89,6 +92,7 @@ jobs: npm-licenses-tts: name: NPM License Check — services/tts runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -125,6 +129,7 @@ jobs: - npm-licenses-frontend - npm-licenses-tts runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Success run: echo "All dependency licenses are compliant." diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index 4f2f590e..2c205a29 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -16,6 +16,7 @@ jobs: msrv: name: Build with Minimum Supported Rust Version runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/openapi-validation.yml b/.github/workflows/openapi-validation.yml index edddb56b..b5bf259f 100644 --- a/.github/workflows/openapi-validation.yml +++ b/.github/workflows/openapi-validation.yml @@ -18,6 +18,7 @@ jobs: spectral-lint: name: Spectral — OpenAPI structural lint runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -42,6 +43,7 @@ jobs: contract-tests: name: OpenAPI Contract Tests runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -77,6 +79,7 @@ jobs: name: All OpenAPI Checks Passed needs: [spectral-lint, contract-tests] runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Success run: echo "OpenAPI spec is valid and in sync with routes." diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 4cc4062d..bfcee799 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -17,6 +17,7 @@ jobs: backend-performance: name: Backend Performance Tests runs-on: ubuntu-latest + timeout-minutes: 20 services: postgres: @@ -222,6 +223,7 @@ jobs: contract-benchmarks: name: Smart Contract Benchmarks runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Checkout code @@ -270,6 +272,7 @@ jobs: performance-regression: name: Performance Regression Detection runs-on: ubuntu-latest + timeout-minutes: 20 needs: [backend-performance] if: github.event_name == 'pull_request' @@ -339,6 +342,7 @@ jobs: save-performance-baseline: name: Save Performance Baseline runs-on: ubuntu-latest + timeout-minutes: 20 needs: [backend-performance] if: github.event_name == 'push' && github.ref == 'refs/heads/main' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 75595fdf..1071875e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,7 @@ jobs: validate-release: name: Validate Release Tag runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 with: @@ -38,6 +39,7 @@ jobs: changelog: name: Generate Changelog & Release runs-on: ubuntu-latest + timeout-minutes: 20 needs: validate-release steps: - uses: actions/checkout@v4 @@ -90,6 +92,7 @@ jobs: build-and-publish-wasm: name: Build & Publish WASM Artifact runs-on: ubuntu-latest + timeout-minutes: 20 needs: validate-release steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index 563a6b9b..c82ab06b 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -18,6 +18,7 @@ jobs: terraform-fmt: name: Terraform Format Check runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -51,6 +52,7 @@ jobs: terraform-validate: name: Terraform Validate runs-on: ubuntu-latest + timeout-minutes: 20 strategy: matrix: environment: [dev, staging, production] @@ -73,6 +75,7 @@ jobs: terraform-plan: name: Terraform Plan (${{ matrix.environment }}) runs-on: ubuntu-latest + timeout-minutes: 20 needs: [terraform-fmt, terraform-validate] strategy: matrix: @@ -199,6 +202,7 @@ jobs: name: All Terraform Checks Passed needs: [terraform-fmt, terraform-validate, terraform-plan] runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Success run: echo "Terraform fmt, validate, and plan all passed." diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 97d8ee47..0213efd0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,7 @@ jobs: prometheus-rules-test: name: Prometheus SLO Rules Testing runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -42,6 +43,7 @@ jobs: unit-tests: name: Unit Tests runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -81,6 +83,7 @@ jobs: integration-tests: name: Integration Tests runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -106,6 +109,7 @@ jobs: api-rate-limit-tests: name: API Rate Limiting Integration Tests runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -128,6 +132,7 @@ jobs: gas-benchmarks: name: Gas Benchmarks runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -196,6 +201,7 @@ jobs: save-gas-baseline: name: Save Gas Benchmark Baseline runs-on: ubuntu-latest + timeout-minutes: 20 needs: [gas-benchmarks] if: github.event_name == 'push' && github.ref == 'refs/heads/main' steps: @@ -253,6 +259,7 @@ jobs: backend-coverage: name: Backend coverage (contracts + API) runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -318,6 +325,9 @@ jobs: security-audit: name: Security Audit runs-on: ubuntu-latest + # cargo install cargo-audit has hung indefinitely on runner network + # issues before; keep this shorter than the default 20. + timeout-minutes: 15 steps: - uses: actions/checkout@v4 @@ -338,6 +348,7 @@ jobs: sast-scanning: name: SAST Security Scanning runs-on: ubuntu-latest + timeout-minutes: 20 permissions: security-events: write contents: read @@ -374,6 +385,7 @@ jobs: secrets-scanning: name: Secrets Scanning runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 with: @@ -396,6 +408,7 @@ jobs: dependency-scanning: name: Dependency Vulnerability Scanning runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -434,6 +447,7 @@ jobs: container-scanning: name: Container Image Scanning runs-on: ubuntu-latest + timeout-minutes: 20 if: github.event_name == 'pull_request' steps: - uses: actions/checkout@v4 @@ -483,6 +497,7 @@ jobs: codeql-analysis: name: CodeQL Analysis runs-on: ubuntu-latest + timeout-minutes: 20 permissions: security-events: write contents: read @@ -517,6 +532,7 @@ jobs: clippy: name: Clippy Lints runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -531,6 +547,7 @@ jobs: oracle-quality-gate: name: Oracle Module — Clippy + Warnings-as-Errors runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -561,6 +578,7 @@ jobs: format: name: Code Formatting runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -575,6 +593,7 @@ jobs: build-optimized: name: Build Optimized Contract runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -615,6 +634,7 @@ jobs: contract-fuzz: name: Contract Fuzz Tests (libFuzzer, 60 s per target) runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -645,6 +665,7 @@ jobs: api-cache-tests: name: API Cache Tests (Redis) runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -667,6 +688,7 @@ jobs: e2e-market-creation: name: E2E — Market Creation Flow runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: frontend @@ -709,6 +731,7 @@ jobs: e2e-mocked: name: E2E — Mocked (no backend) runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: frontend @@ -752,6 +775,7 @@ jobs: e2e-visual-regression: name: E2E — Visual Regression (Docker) runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: frontend @@ -786,6 +810,7 @@ jobs: bundle-size: name: Bundle Size Budget runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: frontend @@ -816,6 +841,7 @@ jobs: validate-migration-rollbacks: name: Validate Migration Rollback Scripts runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -865,6 +891,7 @@ jobs: rollback-idempotency: name: Rollback Idempotency Check runs-on: ubuntu-latest + timeout-minutes: 20 services: postgres: image: postgres:16 @@ -916,6 +943,7 @@ jobs: api-criterion-benchmarks: name: API Criterion Benchmarks runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -977,6 +1005,7 @@ jobs: save-api-bench-baseline: name: Save API Benchmark Baseline runs-on: ubuntu-latest + timeout-minutes: 20 needs: [api-criterion-benchmarks] if: github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: @@ -1036,6 +1065,7 @@ jobs: frontend-unit-coverage: name: Frontend Unit Test Coverage runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: frontend @@ -1069,6 +1099,7 @@ jobs: api-integration-tests: name: API Integration Tests (Compose) runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -1103,6 +1134,7 @@ jobs: api-test-order-independence: name: API Tests — Order Independence runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -1133,6 +1165,7 @@ jobs: api-property-tests: name: API Property-Based Tests runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -1157,6 +1190,7 @@ jobs: openapi-drift-check: name: OpenAPI Spec Drift Check runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -1231,6 +1265,7 @@ jobs: schema-drift-check: name: Schema Drift Detection runs-on: ubuntu-latest + timeout-minutes: 20 services: postgres: image: postgres:16 @@ -1340,6 +1375,7 @@ jobs: - api-property-tests - schema-drift-check runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Success run: echo "All tests passed successfully!" @@ -1347,6 +1383,7 @@ jobs: documentation-sync: name: Documentation Sync Check runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 diff --git a/WORKFLOW_HARDENING.md b/WORKFLOW_HARDENING.md new file mode 100644 index 00000000..d8624f0c --- /dev/null +++ b/WORKFLOW_HARDENING.md @@ -0,0 +1,97 @@ +# CI/CD Security & Reliability Hardening + +Summary of the four fixes implemented on this branch. Each fix is its own +commit; this file documents what changed and why, since each individual +diff is small enough to need extra context. + +## 1. Pin third-party GitHub Actions to commit SHAs + +**Commit:** `security: pin third-party GitHub Actions to commit SHAs` + +Only `aquasecurity/trivy-action` and `trufflesecurity/trufflehog` were +pinned to full commit SHAs. Every other third-party (non-`actions/`, +non-`github/`) action across all 14 workflows was referenced by a mutable +tag (`@v4`, `@v12`, etc.), which a compromised maintainer account could +silently repoint to malicious code with no visible diff in this repo. + +Pinned to a commit SHA (resolved from the upstream tag/branch via +`git ls-remote`) with a trailing `# vX` comment for readability: + +- `dtolnay/rust-toolchain` (`@stable`, `@master`) +- `hashicorp/setup-terraform` (`@v3`, `@v4`) +- `aws-actions/configure-aws-credentials` (`@v4`, `@v6`) +- `bridgecrewio/checkov-action` (`@v12`) +- `docker/setup-buildx-action`, `docker/login-action`, + `docker/metadata-action`, `docker/build-push-action` +- `slackapi/slack-github-action` +- `actions-rs/toolchain` +- `softprops/action-gh-release` +- `orhun/git-cliff-action` +- `peter-evans/create-pull-request` +- `codecov/codecov-action` +- `returntocorp/semgrep-action` +- `gitleaks/gitleaks-action` + +`actions/*` and `github/*` actions were intentionally left on major-version +tags per the issue's scope. + +## 2. Open a PR for CHANGELOG updates instead of pushing to main + +**Commit:** `security: open a PR for CHANGELOG updates instead of pushing to main` + +`release.yml`'s `changelog` job committed and ran +`git push origin HEAD:main` directly using the default `GITHUB_TOKEN` +(`contents: write`), letting an automated bot bypass branch protection +and review on `main`. + +- Replaced the direct push with `peter-evans/create-pull-request`, which + opens a `changelog-update/` branch against `main` instead. +- Added `pull-requests: write` to the workflow's `permissions` block so + the PR can be created. +- `protect-changelog.yml` blocks any PR that touches `CHANGELOG.md`, so + it now carries a documented, narrowly-scoped exception: only a PR + authored by `github-actions[bot]` from a `changelog-update/*` branch is + allowed through. Every other PR touching `CHANGELOG.md` is still + blocked exactly as before. + +## 3. Fix ROLLBACK.md S3 bucket/key drift + +**Commit:** `fix(docs): correct ROLLBACK.md S3 bucket/key and tfvars drift` + +`infrastructure/ROLLBACK.md` documented `predictiq-terraform-state` / +`prod/terraform.tfstate` and `environments/prod.tfvars`, none of which +exist. The real backend config +(`infrastructure/terraform/environments/{production,staging}/backend.hcl`) +uses: + +| Environment | Bucket | Key | +|---|---|---| +| Staging | `predictiq-terraform-state-staging` | `staging/terraform.tfstate` | +| Production | `predictiq-terraform-state-production` | `production/terraform.tfstate` | + +Updated every `aws s3api` command, `terraform init -backend-config=...`, +the `-var-file` path (real file is +`environments/production/terraform.tfvars`), and the `prod` resource-id +references in the verification section, plus added a reference table so +the runbook stays copy-paste runnable against real infrastructure. + +## 4. Add `timeout-minutes` to every CI job + +**Commit:** `reliability: add timeout-minutes to every CI job` + +None of the 14 workflows set a job-level timeout. A hung step (e.g. +`cargo install cargo-audit` / `soroban-cli`, or a stuck `terraform apply`) +would run until GitHub's 360-minute default — for `deploy.yml`'s +Terraform apply job, that also holds a DynamoDB state lock and blocks +every other environment behind it in the `max-parallel: 1` matrix. + +- Added `timeout-minutes: 20` to every job across all 14 workflow files. +- Tightened `deploy.yml`'s `terraform-apply` job and the `cargo-audit` + install jobs in `dependency-scan.yml` / `test.yml` to `timeout-minutes: 15`, + since those are the jobs most likely to hang and most costly to block on. + +**Known pre-existing issue (out of scope):** `test.yml`'s `clippy` and +`oracle-quality-gate` jobs have a `with:` key missing before +`components: clippy` on the `dtolnay/rust-toolchain` step (lines ~525, +~539 on `main` before this branch). This predates this branch and isn't +part of any of the four issues addressed here, so it was left as-is.