diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 19ebfde2..4fad504a 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -23,5 +23,12 @@ jobs: - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y bats jq + - name: Install gomplate + run: | + sudo curl -fsSL -o /usr/local/bin/gomplate \ + https://github.com/hairyhenderson/gomplate/releases/download/v4.3.3/gomplate_linux-amd64 + sudo chmod +x /usr/local/bin/gomplate + gomplate --version + - name: Run unit tests run: make test-unit diff --git a/.gitignore b/.gitignore index 10d7b0f7..9b70a76f 100644 --- a/.gitignore +++ b/.gitignore @@ -138,7 +138,7 @@ np-agent-manifest.yaml # Git worktrees .worktrees/ -DS_Store +.DS_Store # Integration test runtime data frontend/deployment/tests/integration/volume/ @@ -153,4 +153,7 @@ testing/docker/certs/ .claude/ # Visual Studio Code -.vscode/ \ No newline at end of file +.vscode/ + +# Superpowers specs and implementation plans (local-only) +docs/superpowers/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 39b50a80..525b83f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- Add support to get AWS credentials via assume role +- Add support to auto-create ALBs on scope create +- Fix race condition on IAM role creation when multiple scopes are created concurrently +- Add nodeSelector support for scheduled task scopes ## [1.12.0] - 2026-06-08 - Fix: do not inject file parameter as env vars diff --git a/k8s/README.md b/k8s/README.md index 59d19980..b67915f9 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -134,6 +134,42 @@ AWS IAM configuration for Kubernetes service accounts. | **IAM_POLICIES** | List of IAM policies to attach to the role | `security.iam_policies` | | **IAM_BOUNDARY_ARN** | ARN of the permissions boundary policy | `security.iam_boundary_arn` | +#### Assuming an IAM role for AWS operations + +By default the scope's AWS CLI calls (IAM, ELBv2, Route53, S3, CloudWatch) use +the agent's own credentials. To run them under a dedicated IAM role per account, +configure the nullplatform **AWS IAM provider** (`aws-iam-configuration`) with an +`iam_role_arns.arns` entry whose `selector` is `containers`: + +```hcl +attributes = { + iam_role_arns = { + arns = [ + { selector = "containers", arn = "arn:aws:iam:::role/" } + ] + } +} +``` + +Resolution precedence (first non-empty wins): + +1. `CONTAINERS_ASSUME_ROLE_ARN` environment variable (explicit override). +2. AWS IAM provider entry matching the selector (`CONTAINERS_ASSUME_ROLE_SELECTOR`, default `containers`). +3. `CONTAINERS_ASSUME_ROLE_ARN_DEFAULT` environment variable. +4. None configured → the agent's credentials are used (no role assumed). + +The IAM provider is resolved **for the scope's dimensions** by the platform: it +is read from `CONTEXT.providers["identity-access-control"]`, which the engine +populates with the most-specific provider config whose `dimensions` are a subset +of the scope's dimensions (the empty-dimension config is the default). The +selector is then matched within that already-resolved config. For this to work, +`identity-access-control` must be listed under `provider_categories` in +`values.yaml`. This lets different dimensions map to different assumable roles +using the same matching the rest of the platform uses. + +The target role's trust policy must allow the agent's role to call +`sts:AssumeRole`. + #### Vault HashiCorp Vault configuration for secrets management. diff --git a/k8s/deployment/templates/deployment.yaml.tpl b/k8s/deployment/templates/deployment.yaml.tpl index cce128bc..21881443 100644 --- a/k8s/deployment/templates/deployment.yaml.tpl +++ b/k8s/deployment/templates/deployment.yaml.tpl @@ -412,4 +412,4 @@ spec: maxUnavailable: 25% maxSurge: 25% revisionHistoryLimit: 10 - progressDeadlineSeconds: 600 \ No newline at end of file + progressDeadlineSeconds: 600 diff --git a/k8s/deployment/tests/build_deployment.bats b/k8s/deployment/tests/build_deployment.bats index 41adfe2a..be88f49d 100644 --- a/k8s/deployment/tests/build_deployment.bats +++ b/k8s/deployment/tests/build_deployment.bats @@ -213,6 +213,8 @@ _render_context() { "capabilities": { "cpu_millicores": 100, "ram_memory": 128, + "cpu_millicores_limit": 200, + "ram_memory_limit": 256, "additional_ports": [], "scaling_type": "fixed", "autoscaling": { diff --git a/k8s/deployment/workflows/delete.yaml b/k8s/deployment/workflows/delete.yaml index 36e0cf1a..538679a5 100644 --- a/k8s/deployment/workflows/delete.yaml +++ b/k8s/deployment/workflows/delete.yaml @@ -10,6 +10,16 @@ steps: parameters: level: string message: string + - name: assume role + type: script + file: "$SERVICE_PATH/utils/assume_role_step" + output: + - name: AWS_ACCESS_KEY_ID + type: environment + - name: AWS_SECRET_ACCESS_KEY + type: environment + - name: AWS_SESSION_TOKEN + type: environment - name: build context type: script file: "$SERVICE_PATH/deployment/build_context" diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml index c0b827c9..3f06694f 100644 --- a/k8s/deployment/workflows/finalize.yaml +++ b/k8s/deployment/workflows/finalize.yaml @@ -12,6 +12,16 @@ steps: parameters: level: string message: string + - name: assume role + type: script + file: "$SERVICE_PATH/utils/assume_role_step" + output: + - name: AWS_ACCESS_KEY_ID + type: environment + - name: AWS_SECRET_ACCESS_KEY + type: environment + - name: AWS_SESSION_TOKEN + type: environment - name: build context type: script file: "$SERVICE_PATH/deployment/build_context" diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml index b7bc8134..92a0bb40 100644 --- a/k8s/deployment/workflows/initial.yaml +++ b/k8s/deployment/workflows/initial.yaml @@ -12,6 +12,16 @@ steps: parameters: level: string message: string + - name: assume role + type: script + file: "$SERVICE_PATH/utils/assume_role_step" + output: + - name: AWS_ACCESS_KEY_ID + type: environment + - name: AWS_SECRET_ACCESS_KEY + type: environment + - name: AWS_SESSION_TOKEN + type: environment - name: build context type: script file: "$SERVICE_PATH/deployment/build_context" diff --git a/k8s/deployment/workflows/rollback.yaml b/k8s/deployment/workflows/rollback.yaml index 729d06f0..96c2b1c4 100644 --- a/k8s/deployment/workflows/rollback.yaml +++ b/k8s/deployment/workflows/rollback.yaml @@ -12,6 +12,16 @@ steps: parameters: level: string message: string + - name: assume role + type: script + file: "$SERVICE_PATH/utils/assume_role_step" + output: + - name: AWS_ACCESS_KEY_ID + type: environment + - name: AWS_SECRET_ACCESS_KEY + type: environment + - name: AWS_SESSION_TOKEN + type: environment - name: build context type: script file: "$SERVICE_PATH/deployment/build_context" diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml index 54f90bdf..ce9a9a67 100644 --- a/k8s/deployment/workflows/switch_traffic.yaml +++ b/k8s/deployment/workflows/switch_traffic.yaml @@ -12,6 +12,16 @@ steps: parameters: level: string message: string + - name: assume role + type: script + file: "$SERVICE_PATH/utils/assume_role_step" + output: + - name: AWS_ACCESS_KEY_ID + type: environment + - name: AWS_SECRET_ACCESS_KEY + type: environment + - name: AWS_SESSION_TOKEN + type: environment - name: build context type: script file: "$SERVICE_PATH/deployment/build_context" diff --git a/k8s/diagnose/tests/scope/health_probe_endpoints.bats b/k8s/diagnose/tests/scope/health_probe_endpoints.bats index 3621480d..92471cb4 100644 --- a/k8s/diagnose/tests/scope/health_probe_endpoints.bats +++ b/k8s/diagnose/tests/scope/health_probe_endpoints.bats @@ -180,7 +180,10 @@ EOF stripped=$(strip_ansi "$output") assert_contains "$stripped" "Readiness Probe on HTTP://8080/health:" assert_contains "$stripped" "HTTP 404 - Health check endpoint not found" - assert_contains "$stripped" "Update probe path or implement the endpoint in application" + # Remediation guidance lives in the structured evidence (suggested_actions), + # not in stdout - the check publishes it for the AI summarizer, not operators. + actions=$(jq -r '.evidence.suggested_actions[]' "$SCRIPT_OUTPUT_FILE") + assert_contains "$actions" "Update probe path or implement health endpoint in application" } @test "scope/health_probe_endpoints: updates status to failed on 404" { @@ -261,7 +264,10 @@ EOF stripped=$(strip_ansi "$output") assert_contains "$stripped" "Readiness Probe on HTTP://8080/health:" assert_contains "$stripped" "HTTP 500 - Application error" - assert_contains "$stripped" "Check application logs and fix internal errors or dependencies" + # Remediation guidance lives in the structured evidence (suggested_actions), + # not in stdout - the check publishes it for the AI summarizer, not operators. + actions=$(jq -r '.evidence.suggested_actions[]' "$SCRIPT_OUTPUT_FILE") + assert_contains "$actions" "Check application logs for internal errors" } @test "scope/health_probe_endpoints: updates status to warning on 500" { diff --git a/k8s/docs/autocreate-alb.md b/k8s/docs/autocreate-alb.md new file mode 100644 index 00000000..a800d2a0 --- /dev/null +++ b/k8s/docs/autocreate-alb.md @@ -0,0 +1,76 @@ +# ALB Autocreation + +The k8s scope can provision new Application Load Balancers (ALBs) on demand when the declared pool of ALBs is exhausted. The behavior is opt-in and only triggers during scope creation; existing scopes are never moved to autocreated ALBs automatically. + +## When the autocreate path runs + +The flow only triggers when **all** of the following are true: + +- `ALB_AUTOCREATE_ENABLED=true` in `values.yaml` or in the `container-orchestration` provider. +- `DNS_TYPE=route53` (autocreation requires the same load-balancing path used by Route53 scopes). +- Every candidate ALB in the pool (base + additional balancers declared in the provider) reports a rule count `>= ALB_MAX_CAPACITY`. +- The scope being created does not already have a Route53 record (a scope being recreated reuses its existing ALB and does not trigger autocreation). + +If any candidate is below the threshold, the scope creation uses that candidate and the autocreate path is not taken. + +## Configuration + +| Key | Default | Description | +|---|---|---| +| `ALB_AUTOCREATE_ENABLED` | `false` | Master switch. When `false`, behavior is identical to previous releases. | +| `ALB_AUTOCREATE_NAME_PREFIX` | `nullplatform-auto-` | Prefix for autocreated ALB names. Final name format: `-<6 hex chars>`. Must match `^[a-z0-9-]+$` and be ≤18 chars so the rendered name stays under the AWS 32-char ALB name limit. | +| `ALB_AUTOCREATE_TIMEOUT_SECONDS` | `300` | How long `wait_for_alb` polls AWS for the new ALB to reach `state=active` before failing the scope creation. The AWS Load Balancer Controller usually takes 2–4 minutes. Must be a positive integer. | + +All three keys are also readable from `providers.container-orchestration.balancer.{autocreate_enabled, autocreate_name_prefix, autocreate_timeout_seconds}`. + +## How it works + +1. `resolve_balancer` evaluates the candidate pool — the base ALB plus the `additional_public_names` / `additional_private_names` list declared in the `container-orchestration` provider — and picks the least-loaded one. +2. If that candidate's rule count is at or above `ALB_MAX_CAPACITY` and `ALB_AUTOCREATE_ENABLED=true`, `resolve_balancer` sources `autocreate_alb`. +3. `autocreate_alb` generates a unique ALB name (`-<6 hex>`) and **patches the container-orchestration provider via `np provider patch`** to append the new name to `additional_public_names` or `additional_private_names` (visibility-dependent). The provider is the authoritative registry of ALBs the platform uses. +4. `autocreate_alb` renders `scope/templates/ingress-dummy.yaml.tpl` into `$OUTPUT_DIR/ingress-dummy-.yaml`. The dummy Ingress carries `alb.ingress.kubernetes.io/group.name=` and `alb.ingress.kubernetes.io/load-balancer-name=`, which is what makes the AWS Load Balancer Controller materialize the ALB once the file is applied. +5. The workflow step `apply autocreated ingress` (in `k8s/scope/workflows/create.yaml`) applies whatever templates are in `$OUTPUT_DIR` via the standard `apply_templates` script. Its `post: wait for alb` runs `wait_for_alb`, which polls `aws elbv2 describe-load-balancers` every 10 seconds until the ALB reports `State.Code=active` (or `failed`/timeout, in which case the scope creation fails). An info-level heartbeat is emitted every ~30s so the operator can see progress. +6. Once active, `wait_for_alb` tags the ALB with `nullplatform:managed-by=autocreate`, `nullplatform:visibility=internet-facing|internal`, and `nullplatform:created-by-scope-id=`. **These tags are audit metadata only**, surfacing the lineage of which scope provisioned which ALB. Discovery does NOT depend on these tags. +7. The rest of the scope creation proceeds with `ALB_NAME` set to the new ALB. + +## How concurrent scope creations behave + +When scope A triggers autocreate, the provider is patched **before** the ALB is active. Scope B that starts during this window reads the provider list, sees the new ALB name, and treats it as a normal candidate. AWS will return `LoadBalancerNotFound` for the in-flight ALB during the few seconds before it shows up in the API; `resolve_balancer` interprets that error specifically as "0 rules" so the in-flight ALB wins least-loaded selection in scope B and no second autocreate fires. Scope B then waits on the same ALB via its own `wait_for_alb` step. + +## Required permissions + +In addition to the permissions already required for capacity validation, the autocreate path needs: + +**Nullplatform API credentials.** The script calls `np provider list` and `np provider patch`, so the workflow environment must provide either `NP_TOKEN` or `NULLPLATFORM_API_KEY` with write access to the container-orchestration provider for the relevant NRN. Without these, the patch step fails with `❌ Failed to patch container-orchestration provider with new ALB`. + +**AWS IAM (agent role).** + +- `elasticloadbalancing:AddTags` — for the audit tags `wait_for_alb` applies once the ALB is active. Failure here is non-fatal (logged as a warning, the scope creation proceeds). + +No new Kubernetes permissions are needed beyond those the agent already has for scope resources. + +## Operational notes + +- Scope creations that trigger autocreation are slower (typically 2–4 minutes extra). This is the expected behavior, not a regression. The platform logs `🔧 Best candidate ALB '...' is at or above capacity (X/Y); triggering autocreate` when it happens, followed by `⏳ Still waiting for ALB '...' to become active (provisioning, ~30s elapsed)` heartbeats while the controller provisions. +- The dummy Ingress (`nullplatform-autocreate-`) is created in the scope's namespace. It exposes no real traffic — the rule points to a fixed `404` response via the standard `alb.ingress.kubernetes.io/actions.response-404` annotation — and exists only to keep the ALB alive in the eyes of the AWS Load Balancer Controller. Deleting the dummy Ingress will cause the controller to delete the ALB. +- The ALB is registered in the nullplatform provider (not in the customer's IaC). Two consequences: + 1. The provider becomes the source of truth for the ALB pool; subsequent scope creations read it directly. + 2. The cloud's IaC (Terraform, OpenTofu, CloudFormation) is **not** updated automatically. If your IaC is the source of truth for ALB inventory, you should reconcile autocreated ALBs into it through your own process. + +## Failure modes + +| Failure | Outcome | +|---|---| +| `ALB_AUTOCREATE_NAME_PREFIX` invalid (bad charset or >18 chars) | Scope creation exits 1 with the validation error before any AWS or provider call. | +| `np provider list` cannot find a container-orchestration provider for the NRN | Scope creation exits 1 with `❌ No container-orchestration provider found for NRN ''`. | +| `np provider patch` fails (no API token / no write access) | Scope creation exits 1 with `❌ Failed to patch container-orchestration provider with new ALB` + hint about `NP_TOKEN` / `NULLPLATFORM_API_KEY`. | +| `gomplate` render of the dummy Ingress fails | Scope creation exits 1 with `❌ Failed to render ingress-dummy template`. | +| ALB never reaches `active` within `ALB_AUTOCREATE_TIMEOUT_SECONDS` | Scope creation exits 1; check controller logs and AWS quota for ALBs in the region. | +| AWS reports the ALB state as `failed` | Scope creation exits 1 immediately. | +| `AddTags` call fails (no IAM permission) | Logged as `⚠️ Could not tag ALB '' (audit only — provider registration already succeeded)`. The scope creation continues; the tags are documentation only. | + +## What is out of scope + +- Migration of existing scopes to autocreated ALBs. Use the `Recreate scope` action if needed. +- Automatic cleanup of unused autocreated ALBs (no scopes referencing them). +- Updating the cloud IaC (Terraform / OpenTofu / CloudFormation) with the new ALB. diff --git a/k8s/scope/build_context b/k8s/scope/build_context index 8328eab6..0650f897 100755 --- a/k8s/scope/build_context +++ b/k8s/scope/build_context @@ -208,6 +208,13 @@ K8S_MODIFIERS=$(get_config_value \ ) K8S_MODIFIERS=$(echo "$K8S_MODIFIERS" | jq .) +OUTPUT_DIR="$SERVICE_PATH/output/$SCOPE_ID" +if [ -n "${NP_OUTPUT_DIR:-}" ]; then + OUTPUT_DIR="$NP_OUTPUT_DIR/output/$SCOPE_ID" +fi +mkdir -p "$OUTPUT_DIR" +export OUTPUT_DIR + source "$SCRIPT_DIR/networking/resolve_balancer" NAMESPACE_SLUG=$(echo "$CONTEXT" | jq -r .namespace.slug) @@ -224,18 +231,12 @@ CONTEXT=$(echo "$CONTEXT" | jq \ --arg gateway_name "$GATEWAY_NAME" \ --arg alb_name "$ALB_NAME" \ --arg component "$COMPONENT" \ + --arg base_domain "$DOMAIN" \ --argjson modifiers "$K8S_MODIFIERS" \ - '. + {ingress_visibility: $ingress_visibility, k8s_namespace: $k8s_namespace, gateway_name: $gateway_name, region: $region, k8s_modifiers: $modifiers, alb_name: $alb_name, component: $component}') + '. + {ingress_visibility: $ingress_visibility, k8s_namespace: $k8s_namespace, gateway_name: $gateway_name, region: $region, k8s_modifiers: $modifiers, alb_name: $alb_name, component: $component, base_domain: $base_domain}') -OUTPUT_DIR="$SERVICE_PATH/output/$SCOPE_ID" -if [ -n "${NP_OUTPUT_DIR:-}" ]; then - OUTPUT_DIR="$NP_OUTPUT_DIR/output/$SCOPE_ID" -fi - -export OUTPUT_DIR export CONTEXT export REGION - -mkdir -p "$OUTPUT_DIR" +export DOMAIN log info "✅ Scope context built successfully" diff --git a/k8s/scope/iam/create_role b/k8s/scope/iam/create_role index cfe3342f..2307c703 100644 --- a/k8s/scope/iam/create_role +++ b/k8s/scope/iam/create_role @@ -161,7 +161,7 @@ for ((i=0; i<$POLICIES_COUNT; i++)); do POLICY_NAME="inline-policy-$((i+1))" log debug "📝 Attaching inline policy: $POLICY_NAME" - TEMP_POLICY_FILE="/tmp/inline-policy-$i.json" + TEMP_POLICY_FILE="$OUTPUT_DIR/inline-policy-$i.json" echo "$POLICY_VALUE" > "$TEMP_POLICY_FILE" aws iam put-role-policy \ diff --git a/k8s/scope/networking/autocreate_alb b/k8s/scope/networking/autocreate_alb new file mode 100644 index 00000000..935904dc --- /dev/null +++ b/k8s/scope/networking/autocreate_alb @@ -0,0 +1,217 @@ +#!/bin/bash + +# Provisions a new ALB on demand when the existing pool is exhausted. +# +# Workflow contract: +# 1. Generates a unique ALB name. +# 2. Patches the container-orchestration provider via `np provider patch` so +# the new ALB is recorded as an additional balancer. Subsequent scope +# creations that read the provider see the new ALB and re-use it instead +# of triggering another autocreate. +# 3. Renders the dummy-ingress template to $OUTPUT_DIR. The next workflow +# step (apply_templates) applies it; that is what triggers the AWS Load +# Balancer Controller to actually provision the ALB. +# 4. Exports ALB_NAME (the new name) and ALB_AUTOCREATED=true so the wait +# step downstream knows to poll for active state and tag the ALB. +# +# Inputs (env vars): +# CONTEXT - Scope CONTEXT JSON +# INGRESS_VISIBILITY - "internet-facing" or "internal" +# K8S_NAMESPACE - Namespace for the dummy Ingress +# OUTPUT_DIR - Where the rendered YAML is written +# ALB_AUTOCREATE_NAME_PREFIX - Optional prefix for the new ALB +# +# Outputs (env vars): +# ALB_NAME - Replaced with the new ALB name +# ALB_AUTOCREATED - Set to "true" + +_AUTOCREATE_ALB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +generate_alb_name() { + local prefix="$1" + local visibility="$2" + + local vis_short + if [ "$visibility" = "internet-facing" ]; then + vis_short="public" + else + vis_short="private" + fi + + local suffix + suffix=$(LC_ALL=C tr -dc 'a-f0-9' < /dev/urandom 2>/dev/null | head -c 6) || { + suffix=$(printf '%06x' $((RANDOM * RANDOM % 16777215))) + } + + echo "${prefix}${vis_short}-${suffix}" +} + +# Reads the container-orchestration provider id and current additional balancer +# list for the requested visibility, then patches the provider to append the new +# ALB name. Surfaces the existing balancer.* attributes so the patch is a +# merge of the full balancer object, not a destructive replacement. +register_alb_in_provider() { + local new_alb_name="$1" + local visibility="$2" + local nrn + nrn=$(echo "$CONTEXT" | jq -r '.scope.nrn // empty') + if [ -z "$nrn" ]; then + log error "❌ Could not read scope NRN from CONTEXT — cannot patch provider" + exit 1 + fi + + local provider_json + provider_json=$(np provider list --categories container-orchestration --nrn "$nrn" --format json 2>/dev/null) || { + log error "❌ Failed to list container-orchestration provider for NRN '$nrn'" + exit 1 + } + + local provider_id + provider_id=$(echo "$provider_json" | jq -r '.results[0].id // empty') + if [ -z "$provider_id" ]; then + log error "❌ No container-orchestration provider found for NRN '$nrn'" + exit 1 + fi + + local field + if [ "$visibility" = "internet-facing" ]; then + field="additional_public_names" + else + field="additional_private_names" + fi + + # Merge: current balancer.* + appended ALB in the right field. + local patch_body + patch_body=$(echo "$provider_json" | jq -c \ + --arg field "$field" \ + --arg new_alb "$new_alb_name" \ + '{ + attributes: { + balancer: ( + (.results[0].attributes.balancer // {}) as $bal | + $bal + { ($field): (($bal[$field] // []) + [$new_alb] | unique) } + ) + } + }') + + log info "📝 Registering ALB '$new_alb_name' in container-orchestration provider ($field)" + if ! np provider patch --id "$provider_id" --body "$patch_body" --no-output 2>/dev/null; then + log error "❌ Failed to patch container-orchestration provider with new ALB" + log error "💡 Possible causes: agent lacks write permission on the provider, or NP_TOKEN/NULLPLATFORM_API_KEY is missing" + exit 1 + fi +} + +# Builds the dummy ingress host via the same domain-generate binary the +# platform uses for scope domains. Substituting scopeSlug with the ALB name +# keeps the host inside whatever wildcard cert/DNS pattern the platform +# already maintains. +generate_dummy_host() { + local alb_name="$1" + + local account_slug namespace_slug application_slug + account_slug=$(echo "$CONTEXT" | jq -r '.account.slug') + namespace_slug=$(echo "$CONTEXT" | jq -r '.namespace.slug') + application_slug=$(echo "$CONTEXT" | jq -r '.application.slug') + + local host + host=$("$SERVICE_PATH/scope/networking/dns/domain/domain-generate" \ + --accountSlug="$account_slug" \ + --namespaceSlug="$namespace_slug" \ + --applicationSlug="$application_slug" \ + --scopeSlug="$alb_name" \ + --domain="$DOMAIN" \ + --useAccountSlug="${USE_ACCOUNT_SLUG:-false}") || { + log error "❌ Failed to generate dummy ingress host via domain-generate" + log error "💡 Possible causes:" + log error " The domain-generate binary returned an error" + log error "🔧 How to fix:" + log error " • Check the domain-generate binary exists: ls -la $SERVICE_PATH/scope/networking/dns/domain/domain-generate" + log error " • Verify the input slugs are valid" + exit 1 + } + + echo "$host" +} + +render_dummy_ingress() { + local alb_name="$1" + local visibility="$2" + local namespace="$3" + + if [ -z "${OUTPUT_DIR:-}" ]; then + log error "❌ OUTPUT_DIR is not set — autocreate_alb must run after OUTPUT_DIR is exported" + exit 1 + fi + mkdir -p "$OUTPUT_DIR" + + local dummy_host + dummy_host=$(generate_dummy_host "$alb_name") + log debug "📋 Dummy ingress host: $dummy_host" + + # The context file MUST have a .json extension. gomplate uses the extension + # to pick the parser; a plain mktemp path is treated as an opaque string + # and the template fails with "can't evaluate field X in type string". + local context_path + context_path="$OUTPUT_DIR/ingress-dummy-${alb_name}-context.json" + # Use double quotes so $context_path is baked into the trap string now. + # Single quotes would defer expansion until the trap fires on RETURN, by + # which point the local variable is out of scope and `set -u` would trip + # with "context_path: unbound variable". + trap "rm -f '$context_path'" RETURN + + echo "$CONTEXT" | jq \ + --arg alb_name "$alb_name" \ + --arg ingress_visibility "$visibility" \ + --arg k8s_namespace "$namespace" \ + --arg dummy_host "$dummy_host" \ + '. + {alb_name: $alb_name, ingress_visibility: $ingress_visibility, k8s_namespace: $k8s_namespace, dummy_host: $dummy_host}' \ + > "$context_path" + + local template_path="${INGRESS_DUMMY_TEMPLATE:-$SERVICE_PATH/scope/templates/ingress-dummy.yaml.tpl}" + local out_path="$OUTPUT_DIR/ingress-dummy-${alb_name}.yaml" + + if ! gomplate -c .="$context_path" --file "$template_path" --out "$out_path"; then + log error "❌ Failed to render ingress-dummy template" + log error "📋 Template: $template_path" + exit 1 + fi + + log debug "📝 Rendered dummy ingress to $out_path" +} + +# ============================================================================= +# Main +# ============================================================================= + +NAME_PREFIX=$(get_config_value \ + --env ALB_AUTOCREATE_NAME_PREFIX \ + --provider '.providers["container-orchestration"].balancer.autocreate_name_prefix' \ + --default "nullplatform-auto-" +) + +# Final ALB name is "-<6 hex>". AWS rejects names that +# exceed 32 chars or contain anything outside [a-zA-Z0-9-]. If an invalid name +# slips through, the AWS Load Balancer Controller silently refuses to create +# the ALB and wait_for_alb hangs to timeout with an opaque error. Catch it +# here with a clear message instead. +if ! [[ "$NAME_PREFIX" =~ ^[a-z0-9-]+$ ]]; then + log error "❌ ALB_AUTOCREATE_NAME_PREFIX must match ^[a-z0-9-]+$, got: '$NAME_PREFIX'" + exit 1 +fi + +# 14 = len("private-") + 6 hex chars; total must stay ≤32 (AWS ALB name limit) +if [ "${#NAME_PREFIX}" -gt 18 ]; then + log error "❌ ALB_AUTOCREATE_NAME_PREFIX must be ≤18 chars (AWS caps ALB names at 32, the visibility+hex suffix uses 14); got ${#NAME_PREFIX}" + exit 1 +fi + +NEW_ALB_NAME=$(generate_alb_name "$NAME_PREFIX" "$INGRESS_VISIBILITY") +log info "🔧 Autocreating ALB '$NEW_ALB_NAME' (visibility=$INGRESS_VISIBILITY)" + +register_alb_in_provider "$NEW_ALB_NAME" "$INGRESS_VISIBILITY" + +render_dummy_ingress "$NEW_ALB_NAME" "$INGRESS_VISIBILITY" "$K8S_NAMESPACE" + +export ALB_NAME="$NEW_ALB_NAME" +export ALB_AUTOCREATED="true" diff --git a/k8s/scope/networking/resolve_balancer b/k8s/scope/networking/resolve_balancer index fd07a68f..88e46bb4 100755 --- a/k8s/scope/networking/resolve_balancer +++ b/k8s/scope/networking/resolve_balancer @@ -23,27 +23,47 @@ _RESOLVE_BALANCER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -if ! type -t log >/dev/null 2>&1; then - source "$_RESOLVE_BALANCER_DIR/../../logging" -fi - -if ! type -t get_config_value >/dev/null 2>&1; then - source "$_RESOLVE_BALANCER_DIR/../../utils/get_config_value" -fi - # Queries AWS ELBv2 to count listener rules on an ALB's HTTPS (443) listener. # The default rule is excluded since it always exists. +# +# Special case for autocreate race: an ALB just registered in the provider by +# a concurrent scope creation may not yet be visible to AWS APIs while the +# Load Balancer Controller is still provisioning it. Treat +# LoadBalancerNotFound as "0 rules", so the in-flight ALB is selected as the +# least-loaded candidate and no second autocreate is triggered. +# # Usage: get_alb_rule_count # Returns: integer rule count on stdout, non-zero exit on failure get_alb_rule_count() { local alb_name="$1" - local alb_arn + # Single AWS call: capture stdout+stderr together. On success stdout has the + # ARN; on failure stderr has the error (we case-match LoadBalancerNotFound + # for the autocreate race). Avoids two `describe-load-balancers` calls per + # candidate per scope creation. + local alb_arn aws_exit alb_arn=$(aws elbv2 describe-load-balancers \ --names "$alb_name" \ --region "$REGION" \ --query 'LoadBalancers[0].LoadBalancerArn' \ - --output text 2>/dev/null) || return 1 + --output text 2>&1) + aws_exit=$? + + if [ "$aws_exit" -ne 0 ]; then + case "$alb_arn" in + *LoadBalancerNotFound*) + # Concurrent autocreate race: the ALB was just registered in the + # provider by another scope but is not yet visible to AWS APIs. Return + # 0 rules on stdout (same contract as the success path below) so the + # in-flight ALB wins least-loaded selection and we don't trigger a + # second autocreate. + log debug "📋 ALB '$alb_name' not yet visible in AWS (likely being provisioned); treating as 0 rules" >&2 + echo 0 + return 0 + ;; + esac + return 1 + fi if [ -z "$alb_arn" ] || [ "$alb_arn" = "None" ]; then return 1 @@ -128,6 +148,8 @@ get_alb_from_route53() { # Main logic # ============================================================================= +log debug "🔍 Resolving ALB for visibility=$INGRESS_VISIBILITY (DNS_TYPE=$DNS_TYPE)" + # Resolve the base ALB name from configuration ALB_NAME="k8s-nullplatform-$INGRESS_VISIBILITY" @@ -145,68 +167,122 @@ else ) fi -if [[ "$DNS_TYPE" == "route53" ]]; then - # Priority 1: Check Route53 for an existing DNS record - SCOPE_DOMAIN_VAL=$(echo "$CONTEXT" | jq -r '.scope.domain // empty') - EXISTING_ALB="" +log debug "📋 Base ALB from configuration: '$ALB_NAME'" - if [ -n "$SCOPE_DOMAIN_VAL" ]; then - EXISTING_ALB=$(get_alb_from_route53 "$SCOPE_DOMAIN_VAL" "$REGION" 2>/dev/null) || true - fi +if [[ "$DNS_TYPE" != "route53" ]]; then + log debug "📋 DNS type is '$DNS_TYPE', skipping Route53 lookup and load balancing" + export ALB_NAME + return 0 2>/dev/null || true +fi + +# Priority 1: Check Route53 for an existing DNS record (DNS/ingress consistency) +SCOPE_DOMAIN_VAL=$(echo "$CONTEXT" | jq -r '.scope.domain // empty') +EXISTING_ALB="" + +if [ -n "$SCOPE_DOMAIN_VAL" ]; then + log debug "📋 Looking up Route53 alias for domain '$SCOPE_DOMAIN_VAL'..." + EXISTING_ALB=$(get_alb_from_route53 "$SCOPE_DOMAIN_VAL" "$REGION" 2>/dev/null) || true +fi + +if [ -n "$EXISTING_ALB" ]; then + log info "📝 Using ALB '$EXISTING_ALB' from Route53 record for $SCOPE_DOMAIN_VAL" + ALB_NAME="$EXISTING_ALB" + export ALB_NAME + return 0 2>/dev/null || true +fi - if [ -n "$EXISTING_ALB" ]; then - log info "📝 Using ALB '$EXISTING_ALB' from Route53 record for $SCOPE_DOMAIN_VAL" - ALB_NAME="$EXISTING_ALB" - else - # Priority 2: If additional balancers configured, pick the least-loaded one - ADDITIONAL_BALANCERS="" - if [ "$INGRESS_VISIBILITY" = "internet-facing" ]; then - ADDITIONAL_BALANCERS=$(get_config_value \ - --provider '.providers["scope-configurations"].networking.additional_public_balancers' \ - --provider '.providers["container-orchestration"].balancer.additional_public_names' \ - --default "" - ) - else - ADDITIONAL_BALANCERS=$(get_config_value \ - --provider '.providers["scope-configurations"].networking.additional_private_balancers' \ - --provider '.providers["container-orchestration"].balancer.additional_private_names' \ - --default "" - ) - fi - - if [ -n "$ADDITIONAL_BALANCERS" ] && [ "$ADDITIONAL_BALANCERS" != "null" ] && [ "$ADDITIONAL_BALANCERS" != "[]" ]; then - log debug "🔍 Additional balancers configured, resolving least-loaded ALB..." - - CANDIDATES=$(echo "$ADDITIONAL_BALANCERS" | jq -r --arg base "$ALB_NAME" '[$base] + . | .[]') - - log debug "📋 Candidate balancers: $(echo "$CANDIDATES" | paste -sd ',' - | sed 's/,/, /g')" - - MIN_RULES=-1 - BEST_ALB="$ALB_NAME" - - for CANDIDATE in $CANDIDATES; do - RULE_COUNT=$(get_alb_rule_count "$CANDIDATE" 2>/dev/null) || { - log warn "⚠️ Could not query rules for ALB '$CANDIDATE', skipping" - continue - } - - log debug "📋 ALB '$CANDIDATE': $RULE_COUNT rules" - - if [ "$MIN_RULES" -eq -1 ] || [ "$RULE_COUNT" -lt "$MIN_RULES" ]; then - MIN_RULES=$RULE_COUNT - BEST_ALB="$CANDIDATE" - fi - done - - if [ "$BEST_ALB" != "$ALB_NAME" ]; then - log info "📝 Selected ALB '$BEST_ALB' ($MIN_RULES rules) over default '$ALB_NAME'" - fi - - ALB_NAME="$BEST_ALB" - fi +log debug "📋 No Route53 record found; evaluating candidate pool" + +# Priority 2: build candidate pool (base + any additional balancers from provider) +ADDITIONAL_BALANCERS="" +if [ "$INGRESS_VISIBILITY" = "internet-facing" ]; then + ADDITIONAL_BALANCERS=$(get_config_value \ + --provider '.providers["scope-configurations"].networking.additional_public_balancers' \ + --provider '.providers["container-orchestration"].balancer.additional_public_names' \ + --default "" + ) +else + ADDITIONAL_BALANCERS=$(get_config_value \ + --provider '.providers["scope-configurations"].networking.additional_private_balancers' \ + --provider '.providers["container-orchestration"].balancer.additional_private_names' \ + --default "" + ) +fi + +if [ -n "$ADDITIONAL_BALANCERS" ] && [ "$ADDITIONAL_BALANCERS" != "null" ] && [ "$ADDITIONAL_BALANCERS" != "[]" ]; then + CANDIDATES=$(echo "$ADDITIONAL_BALANCERS" | jq -r --arg base "$ALB_NAME" '[$base] + . | .[]') + log debug "📋 Candidate balancers (base + additional): $(echo "$CANDIDATES" | paste -sd ',' - | sed 's/,/, /g')" +else + CANDIDATES="$ALB_NAME" + log debug "📋 No additional balancers configured; candidate pool is just the base ALB '$ALB_NAME'" +fi + +# Pick least-loaded candidate. We evaluate every candidate (including the base +# alone) so capacity + autocreate decisions can fire on single-ALB setups too. +MIN_RULES=-1 +BEST_ALB="$ALB_NAME" + +for CANDIDATE in $CANDIDATES; do + # No outer stderr redirect: get_alb_rule_count locally suppresses raw aws + # CLI stderr and emits only intentional `log` messages there, which we want + # visible in the operator output. + RULE_COUNT=$(get_alb_rule_count "$CANDIDATE") || { + log warn "⚠️ Could not query rules for ALB '$CANDIDATE', skipping" + continue + } + + log debug "📋 ALB '$CANDIDATE': $RULE_COUNT rules" + + if [ "$MIN_RULES" -eq -1 ] || [ "$RULE_COUNT" -lt "$MIN_RULES" ]; then + MIN_RULES=$RULE_COUNT + BEST_ALB="$CANDIDATE" fi +done + +if [ "$BEST_ALB" != "$ALB_NAME" ]; then + log info "📝 Selected ALB '$BEST_ALB' ($MIN_RULES rules) over default '$ALB_NAME'" else - log debug "📋 DNS type is '$DNS_TYPE', skipping Route53 lookup and load balancing" + log debug "📋 Sticking with base ALB '$BEST_ALB' ($MIN_RULES rules)" +fi + +ALB_NAME="$BEST_ALB" + +# Autocreate evaluation: when the chosen ALB is at or above the capacity +# threshold and autocreate is enabled, source autocreate_alb to provision a +# new ALB and replace ALB_NAME with it. +AUTOCREATE_ENABLED=$(get_config_value \ + --env ALB_AUTOCREATE_ENABLED \ + --provider '.providers["container-orchestration"].balancer.autocreate_enabled' \ + --default "false" +) + +MAX_CAPACITY=$(get_config_value \ + --env ALB_MAX_CAPACITY \ + --provider '.providers["scope-configurations"].networking.alb_max_capacity' \ + --provider '.providers["container-orchestration"].balancer.alb_capacity_threshold' \ + --default "75" +) + +log debug "📋 Autocreate enabled: $AUTOCREATE_ENABLED | Capacity threshold: $MAX_CAPACITY" + +# Without this guard a non-numeric MAX_CAPACITY would silently disable the +# autocreate trigger (the `-ge` comparison errors out and evaluates false). +if ! [[ "$MAX_CAPACITY" =~ ^[0-9]+$ ]]; then + log warn "⚠️ ALB_MAX_CAPACITY must be numeric, got: '$MAX_CAPACITY' — skipping autocreate evaluation" + export ALB_NAME + return 0 2>/dev/null || true +fi + +if [ "$MIN_RULES" -lt 0 ]; then + log debug "📋 Could not determine rule counts for any candidate (AWS query failures); skipping autocreate evaluation" +elif [ "$MIN_RULES" -lt "$MAX_CAPACITY" ]; then + log debug "📋 Best candidate ALB '$BEST_ALB' is within capacity ($MIN_RULES/$MAX_CAPACITY); autocreate not needed" +elif [ "$AUTOCREATE_ENABLED" != "true" ]; then + log debug "📋 Best candidate ALB '$BEST_ALB' is over capacity ($MIN_RULES/$MAX_CAPACITY) but autocreate is disabled; validate_alb_capacity will reject the deployment" +else + log info "🔧 Best candidate ALB '$BEST_ALB' is at or above capacity ($MIN_RULES/$MAX_CAPACITY); triggering autocreate" + # autocreate_alb exports ALB_NAME with the new name (or exits on failure) + source "$_RESOLVE_BALANCER_DIR/autocreate_alb" fi export ALB_NAME diff --git a/k8s/scope/networking/wait_for_alb b/k8s/scope/networking/wait_for_alb new file mode 100644 index 00000000..0909f39a --- /dev/null +++ b/k8s/scope/networking/wait_for_alb @@ -0,0 +1,118 @@ +#!/bin/bash + +# Waits for the resolved ALB to reach `active` state in AWS. Runs as the post +# step of the apply_templates that lands the dummy Ingress (the trigger for +# the AWS Load Balancer Controller to provision the ALB). +# +# When ALB_AUTOCREATED is set, also tags the ALB so the cloud has a record of +# what is managed by the platform. The tag is documentation only — the +# authoritative registration is in the container-orchestration provider, done +# by autocreate_alb. +# +# Inputs (env vars): +# ALB_NAME - Resolved ALB name +# REGION - AWS region +# INGRESS_VISIBILITY - For the visibility tag value +# ALB_AUTOCREATED - "true" when this scope triggered autocreate +# ALB_AUTOCREATE_TIMEOUT_SECONDS - Max seconds to wait (default 300) +# CONTEXT - For scope-id in the tag value + +# Step gating: only block on ALB readiness for the path that actually relies +# on it — route53 scopes that manage ALBs via the AWS Load Balancer Controller. +# Other DNS types (azure, external_dns) provision a different networking stack +# and would never see the ALB transition to `active`, so the wait would always +# time out and fail the scope creation. +if [ "${DNS_TYPE:-}" != "route53" ]; then + log debug "📋 DNS type is '${DNS_TYPE:-unset}', skipping ALB active-state wait" + return 0 2>/dev/null || exit 0 +fi + +TIMEOUT_SECONDS=$(get_config_value \ + --env ALB_AUTOCREATE_TIMEOUT_SECONDS \ + --provider '.providers["container-orchestration"].balancer.autocreate_timeout_seconds' \ + --default "300" +) + +if ! [[ "$TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then + log error "❌ ALB_AUTOCREATE_TIMEOUT_SECONDS must be a positive integer, got: '$TIMEOUT_SECONDS'" + exit 1 +fi + +deadline=$(($(date +%s) + TIMEOUT_SECONDS)) +poll_interval=10 +polls_per_heartbeat=3 +# Counter-based heartbeat: every Nth poll we emit a progress log. Using a +# counter (instead of wall-clock diff) keeps the displayed elapsed time on +# clean 30s boundaries; otherwise the latency of each AWS call drifts the +# wall-clock heartbeat to odd numbers like 33s or 63s. +polls_since_heartbeat=0 +heartbeats_emitted=0 + +log info "⏳ Waiting up to ${TIMEOUT_SECONDS}s for ALB '$ALB_NAME' to become active..." + +state="" +alb_arn="" +while [ "$(date +%s)" -lt "$deadline" ]; do + lb_json=$(aws elbv2 describe-load-balancers \ + --names "$ALB_NAME" \ + --region "$REGION" \ + --output json 2>/dev/null) || lb_json="" + + alb_arn=$(echo "$lb_json" | jq -r '.LoadBalancers[0].LoadBalancerArn // empty' 2>/dev/null) || alb_arn="" + state=$(echo "$lb_json" | jq -r '.LoadBalancers[0].State.Code // empty' 2>/dev/null) || state="" + + if [ -n "$alb_arn" ]; then + log debug "📋 ALB '$ALB_NAME' state: ${state:-pending}" + + if [ "$state" = "active" ]; then + log info "✅ ALB '$ALB_NAME' is active" + break + fi + + if [ "$state" = "failed" ]; then + log error "❌ ALB '$ALB_NAME' reached state 'failed'" + exit 1 + fi + fi + + polls_since_heartbeat=$((polls_since_heartbeat + 1)) + if [ "$polls_since_heartbeat" -ge "$polls_per_heartbeat" ]; then + heartbeats_emitted=$((heartbeats_emitted + 1)) + elapsed=$((heartbeats_emitted * polls_per_heartbeat * poll_interval)) + log info "⏳ Still waiting for ALB '$ALB_NAME' to become active (${state:-pending}, ~${elapsed}s elapsed)" + polls_since_heartbeat=0 + fi + + sleep "$poll_interval" +done + +if [ "$state" != "active" ]; then + log error "❌ Timed out after ${TIMEOUT_SECONDS}s waiting for ALB '$ALB_NAME' to become active" + log error "💡 Possible causes:" + log error " The AWS Load Balancer Controller may be slow, mis-configured, or the AWS account may be hitting an ALB quota" + log error "🔧 How to fix:" + log error " • Check controller logs: kubectl -n kube-system logs deploy/aws-load-balancer-controller" + log error " • Verify ALB quota: aws service-quotas get-service-quota --service-code elasticloadbalancing --quota-code L-53DA6B97" + exit 1 +fi + +# Audit tags — only on the scope that triggered the autocreate, so the cloud +# carries the lineage of which scope created which ALB. Failure is non-fatal: +# the provider registration (the authoritative source) already succeeded; the +# tags are documentation only. +if [ "${ALB_AUTOCREATED:-false}" = "true" ]; then + scope_id=$(echo "$CONTEXT" | jq -r '.scope.id // "unknown"') + if ! aws elbv2 add-tags \ + --resource-arns "$alb_arn" \ + --region "$REGION" \ + --tags \ + "Key=nullplatform:managed-by,Value=autocreate" \ + "Key=nullplatform:visibility,Value=$INGRESS_VISIBILITY" \ + "Key=nullplatform:created-by-scope-id,Value=$scope_id" \ + >/dev/null 2>&1; then + log warn "⚠️ Could not tag ALB '$ALB_NAME' (audit only — provider registration already succeeded, continuing)" + log warn "💡 The agent role needs the IAM permission 'elasticloadbalancing:AddTags' on Application Load Balancers in region '$REGION' to write the audit tags (nullplatform:managed-by, nullplatform:visibility, nullplatform:created-by-scope-id)" + else + log debug "📋 Tagged ALB '$ALB_NAME' with managed-by=autocreate" + fi +fi diff --git a/k8s/scope/templates/ingress-dummy.yaml.tpl b/k8s/scope/templates/ingress-dummy.yaml.tpl new file mode 100644 index 00000000..cff2e154 --- /dev/null +++ b/k8s/scope/templates/ingress-dummy.yaml.tpl @@ -0,0 +1,30 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: nullplatform-autocreate-{{ .alb_name }} + namespace: {{ .k8s_namespace }} + labels: + nullplatform: "true" + nullplatform-autocreate: "true" + alb_name: {{ .alb_name }} + annotations: + alb.ingress.kubernetes.io/actions.response-404: '{"type":"fixed-response","fixedResponseConfig":{"contentType":"text/plain","statusCode":"404","messageBody":"404 scope not found or has not been deployed yet"}}' + alb.ingress.kubernetes.io/group.name: {{ .alb_name }} + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]' + alb.ingress.kubernetes.io/load-balancer-name: {{ .alb_name }} + alb.ingress.kubernetes.io/scheme: {{ .ingress_visibility }} + alb.ingress.kubernetes.io/ssl-redirect: '443' + alb.ingress.kubernetes.io/target-type: ip +spec: + ingressClassName: alb + rules: + - host: {{ .dummy_host }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: response-404 + port: + name: use-annotation diff --git a/k8s/scope/tests/build_context.bats b/k8s/scope/tests/build_context.bats index bd86e56b..61b8563c 100644 --- a/k8s/scope/tests/build_context.bats +++ b/k8s/scope/tests/build_context.bats @@ -154,6 +154,7 @@ teardown() { "gateway_name": "co-gateway-public", "alb_name": "co-balancer-public", "component": "test-namespace-test-app", + "base_domain": "cloud-domain.io", "k8s_modifiers": {} }' @@ -213,6 +214,7 @@ teardown() { "gateway_name": "co-gateway-private", "alb_name": "co-balancer-private", "component": "test-namespace-test-app", + "base_domain": "cloud-domain.io", "k8s_modifiers": {} }' diff --git a/k8s/scope/tests/iam/create_role.bats b/k8s/scope/tests/iam/create_role.bats index ef624dbe..bbfd9051 100644 --- a/k8s/scope/tests/iam/create_role.bats +++ b/k8s/scope/tests/iam/create_role.bats @@ -263,6 +263,58 @@ teardown() { assert_contains "$output" "✅ Successfully attached inline policy: inline-policy-1" } +# ============================================================================= +# Test: Inline policy document is written under OUTPUT_DIR, not /tmp +# (regression: shared /tmp path collided across concurrent create_role runs) +# ============================================================================= +@test "create_role: inline policy document is written under OUTPUT_DIR" { + export IAM='{ + "ENABLED": "true", + "PREFIX": "test-prefix", + "ROLE": { + "BOUNDARY_ARN": null, + "POLICIES": [ + {"TYPE": "inline", "VALUE": "{\"Version\":\"2012-10-17\",\"Statement\":[]}"} + ] + } + }' + + aws() { + case "$*" in + *"eks describe-cluster"*) + echo "https://oidc.eks.us-east-1.amazonaws.com/id/ABCDEF1234567890" + ;; + *"sts get-caller-identity"*) + echo "123456789012" + ;; + *"iam create-role"*) + echo '{"Role": {"Arn": "arn:aws:iam::123456789012:role/test-prefix-test-scope-123"}}' + ;; + *"iam put-role-policy"*) + # Emit the args so the test can assert on the --policy-document path + echo "PUT_ROLE_POLICY_ARGS: $*" + ;; + *) + return 0 + ;; + esac + } + export -f aws + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "0" + assert_contains "$output" "--policy-document file://$OUTPUT_DIR/inline-policy-0.json" + + # Guard against the shared /tmp path regression + if echo "$output" | grep -q "file:///tmp/inline-policy"; then + uses_tmp="true" + else + uses_tmp="false" + fi + assert_false "$uses_tmp" "inline policy document under /tmp" +} + # ============================================================================= # Test: Unknown policy type shows warning # ============================================================================= diff --git a/k8s/scope/tests/networking/autocreate_alb.bats b/k8s/scope/tests/networking/autocreate_alb.bats new file mode 100644 index 00000000..2f0f7d06 --- /dev/null +++ b/k8s/scope/tests/networking/autocreate_alb.bats @@ -0,0 +1,308 @@ +#!/usr/bin/env bats +# ============================================================================= +# Unit tests for networking/autocreate_alb +# ============================================================================= + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + + log() { if [ "$1" = "error" ]; then echo "$2" >&2; else echo "$2"; fi; } + export -f log + + source "$PROJECT_ROOT/k8s/utils/get_config_value" + export -f get_config_value + + export SCRIPT="$PROJECT_ROOT/k8s/scope/networking/autocreate_alb" + export REGION="us-east-1" + export INGRESS_VISIBILITY="internet-facing" + export K8S_NAMESPACE="test-ns" + export SERVICE_PATH="$PROJECT_ROOT/k8s" + export DOMAIN="nullapps.io" + export OUTPUT_DIR="$(mktemp -d)" + export PATCH_BODY_FILE="$OUTPUT_DIR/_patch_body" + + export CONTEXT='{ + "scope": { + "id": "scope-1", + "slug": "scope-1", + "nrn": "organization=1:account=2:namespace=3:application=4:scope=5" + }, + "namespace": { "id": "ns-1", "slug": "ns-1" }, + "application": { "id": "app-1", "slug": "app-1" }, + "account": { "id": "acc-1", "slug": "acc-1" }, + "deployment": { "id": "dep-1" }, + "providers": { + "container-orchestration": {} + } + }' + + # Default mocks — each test overrides as needed. + gomplate() { + local prev="" + for arg in "$@"; do + if [ "$prev" = "--out" ]; then echo "rendered" > "$arg"; fi + prev="$arg" + done + return 0 + } + export -f gomplate + + np() { + if [ "$1" = "provider" ] && [ "$2" = "list" ]; then + echo '{"results":[{"id":"prov-1","attributes":{"balancer":{}}}]}' + return 0 + fi + if [ "$1" = "provider" ] && [ "$2" = "patch" ]; then + local prev="" + for arg in "$@"; do + if [ "$prev" = "--body" ]; then echo "$arg" > "$PATCH_BODY_FILE"; fi + prev="$arg" + done + return 0 + fi + return 1 + } + export -f np + export -f gomplate +} + +teardown() { + unset -f log gomplate np get_config_value + rm -rf "$OUTPUT_DIR" + unset ALB_NAME ALB_AUTOCREATED +} + +# ============================================================================= +# Happy path — full log sequence (info logs only; debug needs LOG_LEVEL=debug) +# ============================================================================= +@test "autocreate_alb: full happy-path log sequence (default prefix, public visibility)" { + run bash -c 'export LOG_LEVEL=debug; source "$SCRIPT"; echo "ALB_NAME=$ALB_NAME ALB_AUTOCREATED=$ALB_AUTOCREATED"' + + assert_equal "$status" "0" + # First log: name generated + visibility echoed + [[ "$output" =~ "🔧 Autocreating ALB 'nullplatform-auto-public-"[a-f0-9]{6}"' (visibility=internet-facing)" ]] + # Provider patch log (field name appears explicitly) + [[ "$output" =~ "📝 Registering ALB 'nullplatform-auto-public-"[a-f0-9]{6}"' in container-orchestration provider (additional_public_names)" ]] + # Render confirmation (debug) + assert_contains "$output" "📝 Rendered dummy ingress to $OUTPUT_DIR/ingress-dummy-" + # Exports + [[ "$output" =~ "ALB_NAME=nullplatform-auto-public-"[a-f0-9]{6}" ALB_AUTOCREATED=true" ]] +} + +@test "autocreate_alb: internal visibility selects additional_private_names field in registration log" { + export INGRESS_VISIBILITY="internal" + + run bash -c 'source "$SCRIPT"; echo "ALB_NAME=$ALB_NAME"' + + assert_equal "$status" "0" + [[ "$output" =~ "🔧 Autocreating ALB 'nullplatform-auto-private-"[a-f0-9]{6}"' (visibility=internal)" ]] + [[ "$output" =~ "📝 Registering ALB 'nullplatform-auto-private-"[a-f0-9]{6}"' in container-orchestration provider (additional_private_names)" ]] +} + +@test "autocreate_alb: custom prefix flows into both autocreate and registration logs" { + export ALB_AUTOCREATE_NAME_PREFIX="custom-" + + run bash -c 'source "$SCRIPT"; echo "ALB_NAME=$ALB_NAME"' + + assert_equal "$status" "0" + [[ "$output" =~ "🔧 Autocreating ALB 'custom-public-"[a-f0-9]{6}"' (visibility=internet-facing)" ]] + [[ "$output" =~ "ALB_NAME=custom-public-"[a-f0-9]{6} ]] +} + +# ============================================================================= +# Provider patch shape +# ============================================================================= +@test "autocreate_alb: patches additional_public_names preserving existing entries" { + np() { + if [ "$1" = "provider" ] && [ "$2" = "list" ]; then + echo '{"results":[{"id":"prov-1","attributes":{"balancer":{"additional_public_names":["existing-1"]}}}]}' + return 0 + fi + if [ "$1" = "provider" ] && [ "$2" = "patch" ]; then + local prev="" + for arg in "$@"; do + if [ "$prev" = "--body" ]; then echo "$arg" > "$PATCH_BODY_FILE"; fi + prev="$arg" + done + return 0 + fi + return 1 + } + export -f np + + source "$SCRIPT" + + local body + body=$(cat "$PATCH_BODY_FILE") + echo "$body" | jq -e '.attributes.balancer.additional_public_names | length == 2' + echo "$body" | jq -e '.attributes.balancer.additional_public_names[0] == "existing-1"' + echo "$body" | jq -e ".attributes.balancer.additional_public_names[1] == \"$ALB_NAME\"" +} + +@test "autocreate_alb: patches additional_private_names for internal visibility" { + export INGRESS_VISIBILITY="internal" + + source "$SCRIPT" + + cat "$PATCH_BODY_FILE" | jq -e '.attributes.balancer.additional_private_names | length == 1' +} + +@test "autocreate_alb: deduplicates name in the patched list" { + np() { + if [ "$1" = "provider" ] && [ "$2" = "list" ]; then + echo '{"results":[{"id":"prov-1","attributes":{"balancer":{"additional_public_names":["a","b"]}}}]}' + return 0 + fi + if [ "$1" = "provider" ] && [ "$2" = "patch" ]; then + local prev="" + for arg in "$@"; do + if [ "$prev" = "--body" ]; then echo "$arg" > "$PATCH_BODY_FILE"; fi + prev="$arg" + done + return 0 + fi + return 1 + } + export -f np + + source "$SCRIPT" + + cat "$PATCH_BODY_FILE" | jq -e '.attributes.balancer.additional_public_names | length == 3' +} + +# ============================================================================= +# Error paths — full failure log +# ============================================================================= +@test "autocreate_alb: exits with full log when provider list returns no results" { + np() { + if [ "$1" = "provider" ] && [ "$2" = "list" ]; then echo '{"results":[]}'; return 0; fi + return 1 + } + export -f np + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "1" + [[ "$output" =~ "🔧 Autocreating ALB 'nullplatform-auto-public-"[a-f0-9]{6}"' (visibility=internet-facing)" ]] + assert_contains "$output" "❌ No container-orchestration provider found for NRN 'organization=1:account=2:namespace=3:application=4:scope=5'" +} + +@test "autocreate_alb: exits with full log when np provider list fails" { + np() { + if [ "$1" = "provider" ] && [ "$2" = "list" ]; then return 2; fi + return 1 + } + export -f np + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "1" + assert_contains "$output" "❌ Failed to list container-orchestration provider for NRN 'organization=1:account=2:namespace=3:application=4:scope=5'" +} + +@test "autocreate_alb: exits with full log when np provider patch fails" { + np() { + if [ "$1" = "provider" ] && [ "$2" = "list" ]; then + echo '{"results":[{"id":"prov-1","attributes":{"balancer":{}}}]}' + return 0 + fi + if [ "$1" = "provider" ] && [ "$2" = "patch" ]; then return 5; fi + return 1 + } + export -f np + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "1" + [[ "$output" =~ "📝 Registering ALB 'nullplatform-auto-public-"[a-f0-9]{6}"' in container-orchestration provider (additional_public_names)" ]] + assert_contains "$output" "❌ Failed to patch container-orchestration provider with new ALB" + assert_contains "$output" "💡 Possible causes: agent lacks write permission on the provider, or NP_TOKEN/NULLPLATFORM_API_KEY is missing" +} + +@test "autocreate_alb: exits with full log when CONTEXT has no scope.nrn" { + export CONTEXT=$(echo "$CONTEXT" | jq 'del(.scope.nrn)') + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "1" + assert_contains "$output" "❌ Could not read scope NRN from CONTEXT — cannot patch provider" +} + +@test "autocreate_alb: exits with full log when gomplate fails to render" { + gomplate() { return 1; } + export -f gomplate + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "1" + assert_contains "$output" "❌ Failed to render ingress-dummy template" + assert_contains "$output" "📋 Template: $SERVICE_PATH/scope/templates/ingress-dummy.yaml.tpl" +} + +@test "autocreate_alb: exits with full log when OUTPUT_DIR is not set" { + unset OUTPUT_DIR + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "1" + assert_contains "$output" "❌ OUTPUT_DIR is not set — autocreate_alb must run after OUTPUT_DIR is exported" +} + +# ============================================================================= +# Side effects — rendered YAML file +# ============================================================================= +@test "autocreate_alb: renders the dummy ingress yaml file inside OUTPUT_DIR" { + source "$SCRIPT" + + [ -f "$OUTPUT_DIR/ingress-dummy-${ALB_NAME}.yaml" ] +} + +# ============================================================================= +# Dummy host generation +# ============================================================================= +@test "autocreate_alb: rejects ALB_AUTOCREATE_NAME_PREFIX with invalid chars (uppercase)" { + export ALB_AUTOCREATE_NAME_PREFIX="Bad-Prefix-" + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "1" + assert_contains "$output" "❌ ALB_AUTOCREATE_NAME_PREFIX must match ^[a-z0-9-]+\$, got: 'Bad-Prefix-'" +} + +@test "autocreate_alb: rejects ALB_AUTOCREATE_NAME_PREFIX longer than 18 chars" { + export ALB_AUTOCREATE_NAME_PREFIX="this-prefix-is-far-too-long-" + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "1" + assert_contains "$output" "❌ ALB_AUTOCREATE_NAME_PREFIX must be ≤18 chars" +} + +@test "autocreate_alb: derives dummy host via domain-generate using the ALB name as scopeSlug" { + # Run with the real domain-generate binary (no mock) — it's deterministic. + # Use a custom gomplate that writes the rendered output through so we can + # inspect the final host. + gomplate() { + local prev="" + for arg in "$@"; do + if [ "$prev" = "--out" ]; then OUT_PATH="$arg"; fi + if [ "$prev" = "-c" ]; then CTX_ARG="$arg"; fi + prev="$arg" + done + # Copy the context json next to the rendered file for assertion access + local ctx_file="${CTX_ARG#.=}" + cp "$ctx_file" "$OUTPUT_DIR/_rendered_context.json" + echo "rendered" > "$OUT_PATH" + return 0 + } + export -f gomplate + + source "$SCRIPT" + + # domain-generate produces "ns-app--.nullapps.io" with the + # slugs taken from CONTEXT.namespace / .application and scopeSlug=$ALB_NAME. + local dummy_host + dummy_host=$(jq -r '.dummy_host' "$OUTPUT_DIR/_rendered_context.json") + [[ "$dummy_host" =~ ^ns-1-app-1-${ALB_NAME}-[a-z0-9]+\.nullapps\.io$ ]] +} diff --git a/k8s/scope/tests/networking/resolve_balancer.bats b/k8s/scope/tests/networking/resolve_balancer.bats index 676d8c67..36243c4c 100644 --- a/k8s/scope/tests/networking/resolve_balancer.bats +++ b/k8s/scope/tests/networking/resolve_balancer.bats @@ -9,6 +9,7 @@ setup() { log() { if [ "$1" = "error" ]; then echo "$2" >&2; else echo "$2"; fi; } export -f log source "$PROJECT_ROOT/k8s/utils/get_config_value" + export -f get_config_value export SCRIPT="$PROJECT_ROOT/k8s/scope/networking/resolve_balancer" export REGION="us-east-1" @@ -84,75 +85,79 @@ mock_route53_alb() { export -f aws" } +# Handles the three elbv2 describe-* calls used by get_alb_rule_count. +# Reads rule counts from MOCK_RULES_FILE ("alb_name count" lines). +# Returns non-zero when an ALB name is not found in MOCK_RULES_FILE. +_mock_aws_elbv2_rule_count() { + case "$*" in + *describe-load-balancers*--names*) + local name="" prev="" + for arg in "$@"; do + if [ "$prev" = "--names" ]; then name="$arg"; fi + prev="$arg" + done + if ! grep -q "^${name} " "$MOCK_RULES_FILE" 2>/dev/null; then + return 1 + fi + echo "arn:aws:elasticloadbalancing:us-east-1:123:loadbalancer/app/${name}/abc" + ;; + *describe-listeners*) + local lb_arn="" prev="" + for arg in "$@"; do + if [ "$prev" = "--load-balancer-arn" ]; then lb_arn="$arg"; fi + prev="$arg" + done + local alb_name + alb_name=$(echo "$lb_arn" | sed 's|.*/app/||;s|/.*||') + echo "arn:aws:elasticloadbalancing:us-east-1:123:listener/app/${alb_name}/abc/def" + ;; + *describe-rules*) + local listener_arn="" prev="" + for arg in "$@"; do + if [ "$prev" = "--listener-arn" ]; then listener_arn="$arg"; fi + prev="$arg" + done + local alb_name + alb_name=$(echo "$listener_arn" | sed 's|.*/app/||;s|/.*||') + local count + count=$(grep "^${alb_name} " "$MOCK_RULES_FILE" | awk '{print $2}') + if [ -z "$count" ]; then + return 1 + fi + local rules='{"Rules": [{"IsDefault": true}' + local i=0 + while [ "$i" -lt "$count" ]; do + rules="${rules}, {\"IsDefault\": false}" + i=$((i + 1)) + done + echo "${rules}]}" + ;; + *) + return 1 + ;; + esac +} +export -f _mock_aws_elbv2_rule_count + # Sets up aws mock with no Route53 record but with rule counts for ALBs. -# Write rule counts to MOCK_RULES_FILE as "alb_name count" lines. -# The mock returns the ALB ARN with the name embedded so describe-rules -# can look up the correct rule count. +# Write rule counts to MOCK_RULES_FILE as "alb_name count" lines before calling. +# The mock returns ARNs with the name embedded so describe-rules can look up counts. mock_alb_rules() { > "$MOCK_RULES_FILE" for pair in "$@"; do echo "$pair" >> "$MOCK_RULES_FILE" done - local rules_file="$MOCK_RULES_FILE" - eval "aws() { - case \"\$*\" in - *list-resource-record-sets*) - echo 'None' - return 0 - ;; - *describe-load-balancers*--names*) - local name='' - local prev='' - for arg in \"\$@\"; do - if [ \"\$prev\" = '--names' ]; then name=\"\$arg\"; fi - prev=\"\$arg\" - done - if ! grep -q \"^\${name} \" '${rules_file}' 2>/dev/null; then - return 1 - fi - echo \"arn:aws:elasticloadbalancing:us-east-1:123:loadbalancer/app/\${name}/abc\" - return 0 - ;; - *describe-listeners*) - local lb_arn='' - local prev='' - for arg in \"\$@\"; do - if [ \"\$prev\" = '--load-balancer-arn' ]; then lb_arn=\"\$arg\"; fi - prev=\"\$arg\" - done - local alb_name=\$(echo \"\$lb_arn\" | sed 's|.*/app/||;s|/.*||') - echo \"arn:aws:elasticloadbalancing:us-east-1:123:listener/app/\${alb_name}/abc/def\" - return 0 - ;; - *describe-rules*) - local listener_arn='' - local prev='' - for arg in \"\$@\"; do - if [ \"\$prev\" = '--listener-arn' ]; then listener_arn=\"\$arg\"; fi - prev=\"\$arg\" - done - local alb_name=\$(echo \"\$listener_arn\" | sed 's|.*/app/||;s|/.*||') - local count=\$(grep \"^\${alb_name} \" '${rules_file}' | awk '{print \$2}') - if [ -z \"\$count\" ]; then - return 1 - fi - local rules='{\"Rules\": [{\"IsDefault\": true}' - local i=0 - while [ \$i -lt \$count ]; do - rules=\"\${rules}, {\\\"IsDefault\\\": false}\" - i=\$((i + 1)) - done - rules=\"\${rules}]}\" - echo \"\$rules\" - return 0 - ;; - *) - return 1 + aws() { + case "$*" in + *list-resource-record-sets*) echo "None" ;; + *describe-load-balancers*--names* | *describe-listeners* | *describe-rules*) + _mock_aws_elbv2_rule_count "$@" ;; + *) return 1 ;; esac } - export -f aws" + export -f aws } # ============================================================================= @@ -368,8 +373,8 @@ mock_alb_rules() { run bash -c 'export LOG_LEVEL=debug; source "$SCRIPT"' - assert_contains "$output" "🔍 Additional balancers configured, resolving least-loaded ALB..." - assert_contains "$output" "📋 Candidate balancers: co-balancer-public, alb-extra-1, alb-extra-2" + assert_contains "$output" "🔍 Resolving ALB for visibility=internet-facing (DNS_TYPE=route53)" + assert_contains "$output" "📋 Candidate balancers (base + additional): co-balancer-public, alb-extra-1, alb-extra-2" } # ============================================================================= @@ -497,3 +502,176 @@ mock_alb_rules() { assert_contains "$output" "DNS type is 'external_dns', skipping Route53 lookup and load balancing" } + +# ============================================================================= +# LoadBalancerNotFound treated as 0 rules (concurrent autocreate race case) +# ============================================================================= +@test "resolve_balancer: LoadBalancerNotFound is treated as 0 rules and the in-flight ALB wins selection" { + export INGRESS_VISIBILITY="internet-facing" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["in-flight-alb"] + ') + + aws() { + case "$*" in + *list-resource-record-sets*) echo "None"; return 0 ;; + *describe-load-balancers*--names*in-flight-alb*) + echo "An error occurred (LoadBalancerNotFound) when calling the DescribeLoadBalancers operation" >&2 + return 254 + ;; + *describe-load-balancers*--names*) + echo "arn:aws:elasticloadbalancing:us-east-1:123:loadbalancer/app/co-balancer-public/abc" + return 0 + ;; + *describe-listeners*) + echo "arn:aws:elasticloadbalancing:us-east-1:123:listener/app/co-balancer-public/abc/def" + return 0 + ;; + *describe-rules*) + local rules='{"Rules": [{"IsDefault": true}' + local i=0 + while [ $i -lt 50 ]; do + rules="${rules}, {\"IsDefault\": false}" + i=$((i + 1)) + done + rules="${rules}]}" + echo "$rules" + return 0 + ;; + *) return 1 ;; + esac + } + export -f aws + + run bash -c 'export LOG_LEVEL=debug; source "$SCRIPT"; echo "ALB_NAME=$ALB_NAME"' + + assert_equal "$status" "0" + assert_contains "$output" "🔍 Resolving ALB for visibility=internet-facing (DNS_TYPE=route53)" + assert_contains "$output" "📋 Candidate balancers (base + additional): co-balancer-public, in-flight-alb" + assert_contains "$output" "📋 ALB 'co-balancer-public': 50 rules" + assert_contains "$output" "📋 ALB 'in-flight-alb' not yet visible in AWS (likely being provisioned); treating as 0 rules" + assert_contains "$output" "📋 ALB 'in-flight-alb': 0 rules" + assert_contains "$output" "📝 Selected ALB 'in-flight-alb' (0 rules) over default 'co-balancer-public'" + assert_contains "$output" "ALB_NAME=in-flight-alb" +} + +# ============================================================================= +# Autocreate trigger +# ============================================================================= +@test "resolve_balancer: sources autocreate_alb and logs full trigger sequence when all candidates over threshold" { + export INGRESS_VISIBILITY="internet-facing" + export ALB_AUTOCREATE_ENABLED="true" + export ALB_MAX_CAPACITY="50" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + mock_alb_rules "co-balancer-public 60" "alb-extra-1 55" + + # Stub autocreate_alb so the trigger path runs end-to-end without requiring + # gomplate / np. The stub exports the new ALB name as the real script would. + cat > "$BATS_TEST_TMPDIR/autocreate_alb_stub" <<'STUB' +export ALB_NAME="auto-public-stubbed" +export ALB_AUTOCREATED="true" +STUB + PATCHED_SCRIPT="$BATS_TEST_TMPDIR/resolve_balancer_patched" + AUTOCREATE_STUB="$BATS_TEST_TMPDIR/autocreate_alb_stub" + sed "s|\\\$_RESOLVE_BALANCER_DIR/autocreate_alb|$AUTOCREATE_STUB|" "$SCRIPT" > "$PATCHED_SCRIPT" + + run bash -c "export LOG_LEVEL=debug; source '$PATCHED_SCRIPT'; echo \"ALB_NAME=\$ALB_NAME ALB_AUTOCREATED=\$ALB_AUTOCREATED\"" + + assert_equal "$status" "0" + assert_contains "$output" "📋 ALB 'co-balancer-public': 60 rules" + assert_contains "$output" "📋 ALB 'alb-extra-1': 55 rules" + assert_contains "$output" "🔧 Best candidate ALB 'alb-extra-1' is at or above capacity (55/50); triggering autocreate" + assert_contains "$output" "ALB_NAME=auto-public-stubbed ALB_AUTOCREATED=true" +} + +@test "resolve_balancer: does not autocreate (no trigger log) when feature disabled even if candidates full" { + export INGRESS_VISIBILITY="internet-facing" + export ALB_AUTOCREATE_ENABLED="false" + export ALB_MAX_CAPACITY="50" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + mock_alb_rules "co-balancer-public 60" "alb-extra-1 55" + + run bash -c 'source "$SCRIPT"; echo "ALB_NAME=$ALB_NAME"' + + assert_equal "$status" "0" + assert_contains "$output" "📝 Selected ALB 'alb-extra-1' (55 rules) over default 'co-balancer-public'" + assert_contains "$output" "ALB_NAME=alb-extra-1" + # No autocreate trigger log + [[ "$output" != *"triggering autocreate"* ]] +} + +@test "resolve_balancer: does not autocreate when at least one candidate below threshold" { + export INGRESS_VISIBILITY="internet-facing" + export ALB_AUTOCREATE_ENABLED="true" + export ALB_MAX_CAPACITY="50" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + mock_alb_rules "co-balancer-public 60" "alb-extra-1 10" + + run bash -c 'source "$SCRIPT"; echo "ALB_NAME=$ALB_NAME"' + + assert_equal "$status" "0" + assert_contains "$output" "📝 Selected ALB 'alb-extra-1' (10 rules) over default 'co-balancer-public'" + assert_contains "$output" "ALB_NAME=alb-extra-1" + [[ "$output" != *"triggering autocreate"* ]] +} + +@test "resolve_balancer: triggers autocreate on single-ALB setup (no additional balancers) when over capacity" { + export INGRESS_VISIBILITY="internet-facing" + export ALB_AUTOCREATE_ENABLED="true" + export ALB_MAX_CAPACITY="10" + # No additional_public_balancers — pool is just the base ALB + mock_alb_rules "co-balancer-public 16" + + cat > "$BATS_TEST_TMPDIR/autocreate_alb_stub" <<'STUB' +export ALB_NAME="auto-public-stubbed" +export ALB_AUTOCREATED="true" +STUB + PATCHED_SCRIPT="$BATS_TEST_TMPDIR/resolve_balancer_patched" + AUTOCREATE_STUB="$BATS_TEST_TMPDIR/autocreate_alb_stub" + sed "s|\\\$_RESOLVE_BALANCER_DIR/autocreate_alb|$AUTOCREATE_STUB|" "$SCRIPT" > "$PATCHED_SCRIPT" + + run bash -c "export LOG_LEVEL=debug; source '$PATCHED_SCRIPT'; echo \"ALB_NAME=\$ALB_NAME ALB_AUTOCREATED=\$ALB_AUTOCREATED\"" + + assert_equal "$status" "0" + assert_contains "$output" "📋 No additional balancers configured; candidate pool is just the base ALB 'co-balancer-public'" + assert_contains "$output" "📋 ALB 'co-balancer-public': 16 rules" + assert_contains "$output" "🔧 Best candidate ALB 'co-balancer-public' is at or above capacity (16/10); triggering autocreate" + assert_contains "$output" "ALB_NAME=auto-public-stubbed ALB_AUTOCREATED=true" +} + +@test "resolve_balancer: single-ALB setup over capacity but autocreate disabled logs reason and lets validate reject" { + export INGRESS_VISIBILITY="internet-facing" + export ALB_AUTOCREATE_ENABLED="false" + export ALB_MAX_CAPACITY="10" + mock_alb_rules "co-balancer-public 16" + + run bash -c 'export LOG_LEVEL=debug; source "$SCRIPT"; echo "ALB_NAME=$ALB_NAME"' + + assert_equal "$status" "0" + assert_contains "$output" "📋 ALB 'co-balancer-public': 16 rules" + assert_contains "$output" "📋 Best candidate ALB 'co-balancer-public' is over capacity (16/10) but autocreate is disabled; validate_alb_capacity will reject the deployment" + assert_contains "$output" "ALB_NAME=co-balancer-public" +} + +@test "resolve_balancer: emits full warn when ALB_MAX_CAPACITY is non-numeric and skips autocreate" { + export INGRESS_VISIBILITY="internet-facing" + export ALB_AUTOCREATE_ENABLED="true" + export ALB_MAX_CAPACITY="not-a-number" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + mock_alb_rules "co-balancer-public 60" "alb-extra-1 55" + + run bash -c 'source "$SCRIPT"; echo "ALB_NAME=$ALB_NAME"' + + assert_equal "$status" "0" + assert_contains "$output" "⚠️ ALB_MAX_CAPACITY must be numeric, got: 'not-a-number' — skipping autocreate evaluation" + assert_contains "$output" "ALB_NAME=alb-extra-1" + [[ "$output" != *"triggering autocreate"* ]] +} diff --git a/k8s/scope/tests/networking/wait_for_alb.bats b/k8s/scope/tests/networking/wait_for_alb.bats new file mode 100644 index 00000000..db2492c8 --- /dev/null +++ b/k8s/scope/tests/networking/wait_for_alb.bats @@ -0,0 +1,219 @@ +#!/usr/bin/env bats +# ============================================================================= +# Unit tests for networking/wait_for_alb +# ============================================================================= + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + + log() { if [ "$1" = "error" ]; then echo "$2" >&2; else echo "$2"; fi; } + export -f log + + source "$PROJECT_ROOT/k8s/utils/get_config_value" + export -f get_config_value + + export SCRIPT="$PROJECT_ROOT/k8s/scope/networking/wait_for_alb" + export REGION="us-east-1" + export ALB_NAME="test-alb" + export INGRESS_VISIBILITY="internet-facing" + export DNS_TYPE="route53" + export ALB_AUTOCREATE_TIMEOUT_SECONDS="2" + + export CONTEXT='{ + "scope": { "id": "scope-1" }, + "providers": { "container-orchestration": {} } + }' + + aws() { return 1; } + export -f aws +} + +teardown() { + unset -f log aws get_config_value + unset ALB_AUTOCREATED +} + +# Builds an aws() mock that returns the given state in a single +# describe-load-balancers --output json response. add-tags returns 0. +mock_aws_state() { + local state="$1" + local arn="arn:aws:elasticloadbalancing:us-east-1:123:loadbalancer/app/$ALB_NAME/abc" + eval "aws() { + case \"\$*\" in + *describe-load-balancers*) + echo '{\"LoadBalancers\":[{\"LoadBalancerArn\":\"${arn}\",\"State\":{\"Code\":\"${state}\"}}]}' + return 0 + ;; + *add-tags*) return 0 ;; + esac + return 1 + } + export -f aws" +} + +# ============================================================================= +# Active state +# ============================================================================= +@test "wait_for_alb: success path logs full sequence when ALB is already active" { + mock_aws_state "active" + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "0" + assert_contains "$output" "⏳ Waiting up to 2s for ALB 'test-alb' to become active..." + assert_contains "$output" "📋 ALB 'test-alb' state: active" + assert_contains "$output" "✅ ALB 'test-alb' is active" +} + +@test "wait_for_alb: honors timeout value in the initial wait log" { + export ALB_AUTOCREATE_TIMEOUT_SECONDS="120" + mock_aws_state "active" + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "0" + assert_contains "$output" "⏳ Waiting up to 120s for ALB 'test-alb' to become active..." +} + +# ============================================================================= +# Failed state +# ============================================================================= +@test "wait_for_alb: exits with full failure log when ALB reaches state=failed" { + mock_aws_state "failed" + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "1" + assert_contains "$output" "⏳ Waiting up to 2s for ALB 'test-alb' to become active..." + assert_contains "$output" "📋 ALB 'test-alb' state: failed" + assert_contains "$output" "❌ ALB 'test-alb' reached state 'failed'" +} + +# ============================================================================= +# Timeout with full diagnostic log +# ============================================================================= +@test "wait_for_alb: timeout emits diagnostic causes and fix hints" { + export ALB_AUTOCREATE_TIMEOUT_SECONDS="1" + mock_aws_state "provisioning" + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "1" + assert_contains "$output" "⏳ Waiting up to 1s for ALB 'test-alb' to become active..." + assert_contains "$output" "📋 ALB 'test-alb' state: provisioning" + assert_contains "$output" "❌ Timed out after 1s waiting for ALB 'test-alb' to become active" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" " The AWS Load Balancer Controller may be slow, mis-configured, or the AWS account may be hitting an ALB quota" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" " • Check controller logs: kubectl -n kube-system logs deploy/aws-load-balancer-controller" + assert_contains "$output" " • Verify ALB quota: aws service-quotas get-service-quota --service-code elasticloadbalancing --quota-code L-53DA6B97" +} + +# ============================================================================= +# Heartbeat +# ============================================================================= +@test "wait_for_alb: emits heartbeat info log on clean elapsed boundaries" { + # Shrink the poll interval and the polls-per-heartbeat so the test runs + # in seconds instead of full 30s intervals. With poll_interval=1 and + # polls_per_heartbeat=2, the first heartbeat fires after 2 polls and + # the displayed elapsed is 2*1=2s. + PATCHED_SCRIPT="$BATS_TEST_TMPDIR/wait_for_alb_patched" + sed -e 's/^poll_interval=10$/poll_interval=1/' \ + -e 's/^polls_per_heartbeat=3$/polls_per_heartbeat=2/' \ + "$SCRIPT" > "$PATCHED_SCRIPT" + + export ALB_AUTOCREATE_TIMEOUT_SECONDS="5" + mock_aws_state "provisioning" + + run bash -c "source '$PATCHED_SCRIPT'" + + # Times out as expected. We should see at least one heartbeat with a clean + # elapsed multiple (2s, 4s) — never odd numbers like 3s or 5s. + assert_equal "$status" "1" + assert_contains "$output" "⏳ Still waiting for ALB 'test-alb' to become active (provisioning, ~2s elapsed)" +} + +# ============================================================================= +# Tagging on autocreate +# ============================================================================= +@test "wait_for_alb: tags ALB and logs full tag-success message when ALB_AUTOCREATED=true" { + export ALB_AUTOCREATED="true" + mock_aws_state "active" + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "0" + assert_contains "$output" "⏳ Waiting up to 2s for ALB 'test-alb' to become active..." + assert_contains "$output" "📋 ALB 'test-alb' state: active" + assert_contains "$output" "✅ ALB 'test-alb' is active" + assert_contains "$output" "📋 Tagged ALB 'test-alb' with managed-by=autocreate" +} + +@test "wait_for_alb: does not tag (no tag log) when ALB_AUTOCREATED is unset" { + mock_aws_state "active" + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "0" + assert_contains "$output" "✅ ALB 'test-alb' is active" + # No tagging log should appear + [[ "$output" != *"Tagged ALB"* ]] + [[ "$output" != *"Could not tag ALB"* ]] +} + +@test "wait_for_alb: early returns for DNS_TYPE != route53 without polling" { + export DNS_TYPE="external_dns" + # If the polling code ran, the default `aws()` returning 1 would loop until + # timeout (2s) and exit 1. The early-return must skip that entirely. + aws() { echo "AWS SHOULD NOT BE CALLED" >&2; return 1; } + export -f aws + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "0" + assert_contains "$output" "📋 DNS type is 'external_dns', skipping ALB active-state wait" + [[ "$output" != *"AWS SHOULD NOT BE CALLED"* ]] + [[ "$output" != *"Waiting up to"* ]] +} + +@test "wait_for_alb: exits with validation error when ALB_AUTOCREATE_TIMEOUT_SECONDS is non-numeric" { + export ALB_AUTOCREATE_TIMEOUT_SECONDS="not-a-number" + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "1" + assert_contains "$output" "❌ ALB_AUTOCREATE_TIMEOUT_SECONDS must be a positive integer, got: 'not-a-number'" +} + +@test "wait_for_alb: exits with validation error when ALB_AUTOCREATE_TIMEOUT_SECONDS is zero" { + export ALB_AUTOCREATE_TIMEOUT_SECONDS="0" + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "1" + assert_contains "$output" "❌ ALB_AUTOCREATE_TIMEOUT_SECONDS must be a positive integer, got: '0'" +} + +@test "wait_for_alb: tag failure logs warn with required IAM permission but exits 0" { + export ALB_AUTOCREATED="true" + local arn="arn:aws:elasticloadbalancing:us-east-1:123:loadbalancer/app/$ALB_NAME/abc" + eval "aws() { + case \"\$*\" in + *describe-load-balancers*) + echo '{\"LoadBalancers\":[{\"LoadBalancerArn\":\"${arn}\",\"State\":{\"Code\":\"active\"}}]}' + return 0 + ;; + *add-tags*) return 1 ;; + esac + return 1 + } + export -f aws" + + run bash -c 'source "$SCRIPT"' + + assert_equal "$status" "0" + assert_contains "$output" "✅ ALB 'test-alb' is active" + assert_contains "$output" "⚠️ Could not tag ALB 'test-alb' (audit only — provider registration already succeeded, continuing)" + assert_contains "$output" "💡 The agent role needs the IAM permission 'elasticloadbalancing:AddTags' on Application Load Balancers in region 'us-east-1' to write the audit tags (nullplatform:managed-by, nullplatform:visibility, nullplatform:created-by-scope-id)" +} diff --git a/k8s/scope/tests/validate_alb_capacity.bats b/k8s/scope/tests/validate_alb_capacity.bats index af08defd..0878e4fe 100644 --- a/k8s/scope/tests/validate_alb_capacity.bats +++ b/k8s/scope/tests/validate_alb_capacity.bats @@ -127,6 +127,7 @@ teardown() { assert_contains "$output" "Increase ALB_MAX_CAPACITY in values.yaml or container-orchestration provider (AWS limit is 100 per listener)" assert_contains "$output" "Request an AWS service quota increase for rules per ALB listener" assert_contains "$output" "Consider using a separate ALB for additional scopes" + assert_contains "$output" "Enable ALB autocreation so the platform provisions a new ALB automatically when the pool is exhausted: set ALB_AUTOCREATE_ENABLED=true in values.yaml or in the container-orchestration provider" } @test "validate_alb_capacity: fails when over capacity" { diff --git a/k8s/scope/tests/wait_on_balancer.bats b/k8s/scope/tests/wait_on_balancer.bats index 83d384d4..96f4406c 100644 --- a/k8s/scope/tests/wait_on_balancer.bats +++ b/k8s/scope/tests/wait_on_balancer.bats @@ -72,9 +72,15 @@ teardown() { # external_dns: Success after retries # ============================================================================= @test "wait_on_balancer: external_dns success after retries" { - local call_count=0 + # The script reads kubectl output via command substitution ($(kubectl ...)), + # which runs the mock in a subshell — an in-memory counter would reset every + # call. Persist the attempt count in a file so it survives across subshells. + export CALL_COUNT_FILE="$BATS_TEST_TMPDIR/dnsendpoint_calls" + echo 0 > "$CALL_COUNT_FILE" kubectl() { - call_count=$((call_count + 1)) + local call_count + call_count=$(($(cat "$CALL_COUNT_FILE") + 1)) + echo "$call_count" > "$CALL_COUNT_FILE" case "$*" in "get dnsendpoint k8s-my-app-my-scope-scope-123-dns -n default-namespace -o jsonpath={.status.observedGeneration}") if [ "$call_count" -ge 2 ]; then diff --git a/k8s/scope/validate_alb_capacity b/k8s/scope/validate_alb_capacity index fc5eb50e..e3c81f0e 100755 --- a/k8s/scope/validate_alb_capacity +++ b/k8s/scope/validate_alb_capacity @@ -135,6 +135,7 @@ if [[ "$TOTAL_RULES" -ge "$ALB_MAX_CAPACITY" ]]; then log error " • Increase ALB_MAX_CAPACITY in values.yaml or container-orchestration provider (AWS limit is 100 per listener)" log error " • Request an AWS service quota increase for rules per ALB listener" log error " • Consider using a separate ALB for additional scopes" + log error " • Enable ALB autocreation so the platform provisions a new ALB automatically when the pool is exhausted: set ALB_AUTOCREATE_ENABLED=true in values.yaml or in the container-orchestration provider" log error "" exit 1 fi diff --git a/k8s/scope/workflows/create.yaml b/k8s/scope/workflows/create.yaml index 9c0f3006..69afe2c8 100644 --- a/k8s/scope/workflows/create.yaml +++ b/k8s/scope/workflows/create.yaml @@ -10,6 +10,16 @@ steps: parameters: level: string message: string + - name: assume role + type: script + file: "$SERVICE_PATH/utils/assume_role_step" + output: + - name: AWS_ACCESS_KEY_ID + type: environment + - name: AWS_SECRET_ACCESS_KEY + type: environment + - name: AWS_SESSION_TOKEN + type: environment - name: build context type: script file: "$SERVICE_PATH/scope/build_context" @@ -22,6 +32,20 @@ steps: type: environment - name: OUTPUT_DIR type: environment + - name: ALB_NAME + type: environment + - name: ALB_AUTOCREATED + type: environment + - name: apply autocreated ingress + type: script + file: "$SERVICE_PATH/apply_templates" + configuration: + ACTION: apply + DRY_RUN: false + post: + name: wait for alb + type: script + file: "$SERVICE_PATH/scope/networking/wait_for_alb" - name: validate alb capacity type: script file: "$SERVICE_PATH/scope/validate_alb_capacity" diff --git a/k8s/scope/workflows/delete.yaml b/k8s/scope/workflows/delete.yaml index cf02790d..e411bedb 100644 --- a/k8s/scope/workflows/delete.yaml +++ b/k8s/scope/workflows/delete.yaml @@ -10,6 +10,16 @@ steps: parameters: level: string message: string + - name: assume role + type: script + file: "$SERVICE_PATH/utils/assume_role_step" + output: + - name: AWS_ACCESS_KEY_ID + type: environment + - name: AWS_SECRET_ACCESS_KEY + type: environment + - name: AWS_SESSION_TOKEN + type: environment - name: build context type: script file: "$SERVICE_PATH/scope/build_context" diff --git a/k8s/specs/requirements/aws/data.tf b/k8s/specs/requirements/aws/data.tf new file mode 100644 index 00000000..038d1e22 --- /dev/null +++ b/k8s/specs/requirements/aws/data.tf @@ -0,0 +1,2 @@ +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} diff --git a/k8s/specs/requirements/aws/locals.tf b/k8s/specs/requirements/aws/locals.tf new file mode 100644 index 00000000..b47445c9 --- /dev/null +++ b/k8s/specs/requirements/aws/locals.tf @@ -0,0 +1,21 @@ +locals { + # Module identifier + iam_module_name = "iam" + + # Whether resources are created + iam_create = var.iam_create_role + + # Derived names (overridable via variables) + permissions_role_name = var.permissions_role_name != "" ? var.permissions_role_name : "nullplatform_${var.cluster_name}_k8s_role" + policies_name_prefix = var.policies_name_prefix != "" ? var.policies_name_prefix : "nullplatform_${var.cluster_name}" + + # Primary agent role trusted by the permissions role. Defaults to the + # conventional agent role name for the cluster when not provided explicitly. + agent_role_arn = var.agent_role_arn != "" ? var.agent_role_arn : "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/nullplatform-${var.cluster_name}-agent-role" + + # Default tags applied to every IAM resource + iam_default_tags = merge(var.iam_resource_tags_json, { + ManagedBy = "nullplatform-custom-scope-role" + Module = local.iam_module_name + }) +} diff --git a/k8s/specs/requirements/aws/main.tf b/k8s/specs/requirements/aws/main.tf new file mode 100644 index 00000000..9b4d63c6 --- /dev/null +++ b/k8s/specs/requirements/aws/main.tf @@ -0,0 +1,166 @@ +################################################################################ +# IAM permissions role assumed by the nullplatform agent role +# +# Holds the actual workload policies (Route53, EKS, ELB). Its trust policy +# trusts only the agent IRSA role (plus any additional roles), so an agent's IRSA +# token cannot exercise these permissions without first assuming it (sts:AssumeRole). +# +# This is the "permissions role" half of the reference module +# tofu-modules/infrastructure/aws/iam/agent. The IRSA agent role itself is +# created once at cluster setup and is out of scope for this module. +################################################################################ + +resource "aws_iam_role" "nullplatform_agent_permissions" { + count = local.iam_create ? 1 : 0 + + name = local.permissions_role_name + description = "Permissions role assumed by the nullplatform agent role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { AWS = concat([local.agent_role_arn], var.additional_agent_role_arns) } + Action = "sts:AssumeRole" + }] + }) + + tags = local.iam_default_tags +} + +################################################################################ +# Policy attachments +################################################################################ + +resource "aws_iam_role_policy_attachment" "permissions_route53" { + count = local.iam_create ? 1 : 0 + + role = aws_iam_role.nullplatform_agent_permissions[0].name + policy_arn = aws_iam_policy.nullplatform_route53_policy[0].arn +} + +resource "aws_iam_role_policy_attachment" "permissions_eks" { + count = local.iam_create ? 1 : 0 + + role = aws_iam_role.nullplatform_agent_permissions[0].name + policy_arn = aws_iam_policy.nullplatform_eks_policy[0].arn +} + +resource "aws_iam_role_policy_attachment" "permissions_elb" { + count = local.iam_create ? 1 : 0 + + role = aws_iam_role.nullplatform_agent_permissions[0].name + policy_arn = aws_iam_policy.nullplatform_elb_policy[0].arn +} + +################################################################################ +# Route 53 IAM policy +# Manage Route 53 DNS records for service discovery. +################################################################################ + +resource "aws_iam_policy" "nullplatform_route53_policy" { + count = local.iam_create ? 1 : 0 + + name = "${local.policies_name_prefix}_route53_policy" + description = "Policy for managing Route 53 DNS records" + tags = local.iam_default_tags + policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone", + "route53:ListHostedZones", + "route53:ListHostedZonesByName" + ], + "Resource" : [ + "arn:aws:route53:::hostedzone/*" + ], + } + ] + }) +} + +################################################################################ +# Elastic Load Balancing (ELB) IAM policy +# Describe and monitor load balancers and target groups. +################################################################################ + +resource "aws_iam_policy" "nullplatform_elb_policy" { + count = local.iam_create ? 1 : 0 + + name = "${local.policies_name_prefix}_elb_policy" + description = "Policy for managing Elastic Load Balancing resources" + tags = local.iam_default_tags + policy = jsonencode( + { + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeTargetGroups" + ], + "Resource" : "*", + "Condition" : { + "StringEquals" : { + "aws:RequestedRegion" : [ + data.aws_region.current.region + ] + } + } + }, + { + "Effect" : "Allow", + "Action" : [ + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeRules" + ], + "Resource" : [ + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/k8s-nullplatform-*", + "arn:aws:elasticloadbalancing:*:*:targetgroup/k8s-nullplatform-*" + ], + } + ] + } + ) +} + +################################################################################ +# EKS IAM policy +# Describe and list EKS cluster resources. +################################################################################ + +resource "aws_iam_policy" "nullplatform_eks_policy" { + count = local.iam_create ? 1 : 0 + + name = "${local.policies_name_prefix}_eks_policy" + description = "Policy for managing EKS cluster resources" + tags = local.iam_default_tags + policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "eks:DescribeCluster", + "eks:ListClusters", + "eks:DescribeNodegroup", + "eks:ListNodegroups", + "eks:DescribeAddon", + "eks:ListAddons" + ], + "Resource" : [ + "arn:aws:eks:*:*:cluster/*", + "arn:aws:eks:*:*:nodegroup/*", + "arn:aws:eks:*:*:addon/*" + ], + } + ] + }) +} diff --git a/k8s/specs/requirements/aws/outputs.tf b/k8s/specs/requirements/aws/outputs.tf new file mode 100644 index 00000000..2e750313 --- /dev/null +++ b/k8s/specs/requirements/aws/outputs.tf @@ -0,0 +1,14 @@ +output "permissions_role_arn" { + description = "ARN of the permissions role assumed by the nullplatform agent role" + value = local.iam_create ? aws_iam_role.nullplatform_agent_permissions[0].arn : "" +} + +output "permissions_role_name" { + description = "Name of the permissions role" + value = local.iam_create ? aws_iam_role.nullplatform_agent_permissions[0].name : "" +} + +output "permissions_role_id" { + description = "ID of the permissions role" + value = local.iam_create ? aws_iam_role.nullplatform_agent_permissions[0].id : "" +} diff --git a/k8s/specs/requirements/aws/variables.tf b/k8s/specs/requirements/aws/variables.tf new file mode 100644 index 00000000..ec1b7df1 --- /dev/null +++ b/k8s/specs/requirements/aws/variables.tf @@ -0,0 +1,50 @@ +variable "agent_role_arn" { + description = "ARN of the primary nullplatform agent IRSA role allowed to assume this permissions role via sts:AssumeRole, and always a trusted principal of the role's trust policy. Defaults (when empty) to the conventional agent role for the cluster: arn:aws:iam:::role/nullplatform--agent-role." + type = string + default = "" + + validation { + condition = var.agent_role_arn == "" || can(regex("^arn:aws:iam::[0-9]{12}:role/.+", var.agent_role_arn)) + error_message = "agent_role_arn must be empty (to use the derived default) or match arn:aws:iam:::role/" + } +} + +variable "additional_agent_role_arns" { + description = "Extra IAM role ARNs allowed to assume this permissions role, appended to agent_role_arn in the trust policy. Defaults to none." + type = list(string) + default = [] + + validation { + condition = alltrue([for arn in var.additional_agent_role_arns : can(regex("^arn:aws:iam::[0-9]{12}:role/.+", arn))]) + error_message = "each additional_agent_role_arns entry must match arn:aws:iam:::role/" + } +} + +variable "cluster_name" { + description = "Name of the cluster where the agent runs. Used to derive default resource names." + type = string +} + +variable "permissions_role_name" { + description = "Override for the permissions IAM role name. Defaults to nullplatform_{cluster_name}_k8s_role." + type = string + default = "" +} + +variable "policies_name_prefix" { + description = "Override for the IAM policy name prefix. Defaults to nullplatform_{cluster_name}." + type = string + default = "" +} + +variable "iam_create_role" { + description = "Whether to create the permissions role and its policies. When false, the module produces no resources." + type = bool + default = true +} + +variable "iam_resource_tags_json" { + description = "Tags to apply to IAM resources created by this module." + type = map(string) + default = {} +} diff --git a/k8s/specs/requirements/aws/versions.tf b/k8s/specs/requirements/aws/versions.tf new file mode 100644 index 00000000..e54c7789 --- /dev/null +++ b/k8s/specs/requirements/aws/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + # v6+ required: the ELB policy reads data.aws_region.current.region, + # which replaced the v5 `.name` attribute. + version = ">= 6.0" + } + } +} diff --git a/k8s/utils/assume_role b/k8s/utils/assume_role new file mode 100644 index 00000000..a2c1d180 --- /dev/null +++ b/k8s/utils/assume_role @@ -0,0 +1,43 @@ +#!/bin/bash +# Sourceable helper — do NOT execute directly. +# Reads CONTAINERS_ASSUME_ROLE_ARN from the environment. If set, calls +# sts:AssumeRole and exports temporary credentials so all subsequent AWS calls +# use that role. If empty, does nothing — the agent's credentials (pod IRSA) +# handle auth. +# +# Requires: aws CLI, jq, and the `log` function (loaded by the workflow). +# Expects: CONTAINERS_ASSUME_ROLE_ARN (set by utils/assume_role_step), +# SCOPE_ID (optional, used for the session name). + +if [ -n "${CONTAINERS_ASSUME_ROLE_ARN:-}" ]; then + log info " 🔑 Assuming role: $CONTAINERS_ASSUME_ROLE_ARN" + + _ar_sts_error=$(mktemp) + if ! ASSUMED_CREDS=$(aws sts assume-role \ + --role-arn "$CONTAINERS_ASSUME_ROLE_ARN" \ + --role-session-name "np-containers-${SCOPE_ID:-workflow}" \ + --output json 2>"$_ar_sts_error"); then + log error " ❌ sts:AssumeRole failed for $CONTAINERS_ASSUME_ROLE_ARN" + log error "$(cat "$_ar_sts_error")" + rm -f "$_ar_sts_error" + return 1 + fi + rm -f "$_ar_sts_error" + + _ar_access_key=$(echo "$ASSUMED_CREDS" | jq -r '.Credentials.AccessKeyId // ""') + _ar_secret_key=$(echo "$ASSUMED_CREDS" | jq -r '.Credentials.SecretAccessKey // ""') + _ar_session_token=$(echo "$ASSUMED_CREDS" | jq -r '.Credentials.SessionToken // ""') + + if [ -z "$_ar_access_key" ] || [ -z "$_ar_secret_key" ] || [ -z "$_ar_session_token" ]; then + log error " ❌ sts:AssumeRole returned incomplete credentials for $CONTAINERS_ASSUME_ROLE_ARN" + return 1 + fi + + export AWS_ACCESS_KEY_ID="$_ar_access_key" + export AWS_SECRET_ACCESS_KEY="$_ar_secret_key" + export AWS_SESSION_TOKEN="$_ar_session_token" + + log info " ✅ Role assumed successfully" +else + log debug " ✅ assume_role=skipped (using agent credentials)" +fi diff --git a/k8s/utils/assume_role_lib b/k8s/utils/assume_role_lib new file mode 100644 index 00000000..ca008e13 --- /dev/null +++ b/k8s/utils/assume_role_lib @@ -0,0 +1,49 @@ +#!/bin/bash +# Sourceable library of PURE helpers for assume-role resolution. +# +# Input is the AWS IAM provider exactly as it appears in +# CONTEXT.providers["identity-access-control"] — the platform already resolved it +# for the scope's dimensions (most-specific config whose dimensions are a subset +# of the scope's wins). These helpers only pick the selector, so they make NO +# np/aws calls and have no side effects on source — fully unit-testable. +# +# Requires (at call time): jq. + +# arn_for_selector +# Given CONTEXT.providers["identity-access-control"], echoes the ARN whose entry +# in .iam_role_arns.arns[] matches , or "" if none. First match wins. +# Returns "" on empty/malformed input (never crashes). +arn_for_selector() { + local json="$1" selector="$2" + [ -n "$json" ] || return 0 + [ -n "$selector" ] || return 0 + # jq exits non-zero (status 5) on malformed JSON; swallow it so the helper + # honors its "never crashes, returns empty" contract instead of propagating + # jq's failure as the function's return code. + printf '%s' "$json" | jq -r --arg sel "$selector" ' + [ .iam_role_arns.arns[]? + | select(.selector == $sel) + | .arn ] + | first // ""' 2>/dev/null || true +} + +# resolve_assume_role_arn +# Echoes the ARN to assume ("" = use agent credentials): +# 1. $CONTAINERS_ASSUME_ROLE_ARN env var (explicit override) +# 2. AWS IAM provider entry matching (already dimension-resolved +# by the platform via CONTEXT.providers["identity-access-control"]) +# 3. $CONTAINERS_ASSUME_ROLE_ARN_DEFAULT env var (per-account agent default) +# Note: CONTAINERS_ASSUME_ROLE_ARN="" (explicitly empty) is treated the same as +# unset — the chain continues to the next source. +resolve_assume_role_arn() { + local iam_json="$1" selector="$2" arn="" + + arn="${CONTAINERS_ASSUME_ROLE_ARN:-}" + + if [ -z "$arn" ] && [ -n "$iam_json" ] && [ -n "$selector" ]; then + arn=$(arn_for_selector "$iam_json" "$selector") + fi + + arn="${arn:-${CONTAINERS_ASSUME_ROLE_ARN_DEFAULT:-}}" + printf '%s' "$arn" +} diff --git a/k8s/utils/assume_role_step b/k8s/utils/assume_role_step new file mode 100755 index 00000000..e857d718 --- /dev/null +++ b/k8s/utils/assume_role_step @@ -0,0 +1,44 @@ +#!/bin/bash +# Dedicated workflow step: resolve the target IAM role and assume it, exporting +# temporary credentials so every subsequent step inherits them. +# +# Runs FIRST in each AWS-touching workflow (right after `load logging`, which +# provides the `log` function). The workflow YAML must declare AWS_ACCESS_KEY_ID, +# AWS_SECRET_ACCESS_KEY and AWS_SESSION_TOKEN as output:environment so the engine +# propagates them. +# +# The AWS IAM provider (category "identity-access-control") is read from +# CONTEXT.providers[...], where the platform has ALREADY resolved it for the +# scope's dimensions. Requires "identity-access-control" to be listed in +# values.yaml provider_categories. +# +# Resolution precedence (see resolve_assume_role_arn in assume_role_lib): +# $CONTAINERS_ASSUME_ROLE_ARN -> IAM provider by selector +# -> $CONTAINERS_ASSUME_ROLE_ARN_DEFAULT -> agent credentials +# +# Requires: aws CLI, jq. Expects: CONTEXT (engine-injected), SCOPE_ID (optional). + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/assume_role_lib" + +CONTAINERS_ASSUME_ROLE_SELECTOR="${CONTAINERS_ASSUME_ROLE_SELECTOR:-containers}" + +# IAM provider as resolved for the scope's dimensions by the platform. +IAM_PROVIDER=$(echo "${CONTEXT:-}" | jq -c '.providers["identity-access-control"] // {}' 2>/dev/null) + +CONTAINERS_ASSUME_ROLE_ARN=$(resolve_assume_role_arn "$IAM_PROVIDER" "$CONTAINERS_ASSUME_ROLE_SELECTOR") +export CONTAINERS_ASSUME_ROLE_ARN + +# utils/assume_role performs sts:AssumeRole and exports AWS_* when an ARN is set, +# or no-ops (leaving agent credentials in place) when empty. Non-zero only when +# sts:AssumeRole itself fails. +if ! source "$SCRIPT_DIR/assume_role"; then + log error " ❌ assume_role step failed: could not assume $CONTAINERS_ASSUME_ROLE_ARN" + log error "" + log error "💡 Possible causes:" + log error " • The agent's role is not allowed to sts:AssumeRole the target role" + log error " • The target role does not exist or does not trust the agent role" + log error " • There is no role ARN configured for selector=$CONTAINERS_ASSUME_ROLE_SELECTOR" + log error "" + exit 1 +fi diff --git a/k8s/utils/tests/assume_role.bats b/k8s/utils/tests/assume_role.bats new file mode 100644 index 00000000..3b5c3fb8 --- /dev/null +++ b/k8s/utils/tests/assume_role.bats @@ -0,0 +1,92 @@ +#!/usr/bin/env bats +# ============================================================================= +# Unit tests for utils/assume_role - performs sts:AssumeRole, exports AWS_*. +# +# Success paths source the helper IN-PROCESS and assert the exported env vars +# directly (closest to how the workflow consumes them); log output is captured +# to a file so every message is asserted in full. Failure paths use `run` to +# capture the non-zero status and the error message. +# ============================================================================= + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + export HELPER="$BATS_TEST_DIRNAME/../assume_role" + + log() { if [ "$1" = "error" ]; then echo "$2" >&2; else echo "$2"; fi; } + export -f log + + export SCOPE_ID="scope-123" + unset CONTAINERS_ASSUME_ROLE_ARN AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + + # Mock aws sts assume-role - success by default + aws() { + case "$*" in + *"sts assume-role"*) + echo '{"Credentials":{"AccessKeyId":"AKIAEXAMPLE","SecretAccessKey":"secret123","SessionToken":"token123"}}' + return 0 + ;; + *) return 0 ;; + esac + } + export -f aws +} + +teardown() { + unset CONTAINERS_ASSUME_ROLE_ARN AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN +} + +@test "assume_role: no-op when CONTAINERS_ASSUME_ROLE_ARN is empty (uses agent creds)" { + logf=$(mktemp) + source "$HELPER" >"$logf" 2>&1 + # No credentials exported — the agent's own credentials stay in effect. + [ -z "${AWS_ACCESS_KEY_ID:-}" ] + [ -z "${AWS_SECRET_ACCESS_KEY:-}" ] + [ -z "${AWS_SESSION_TOKEN:-}" ] + assert_contains "$(cat "$logf")" " ✅ assume_role=skipped (using agent credentials)" +} + +@test "assume_role: exports AWS_* and logs all messages when ARN is set" { + export CONTAINERS_ASSUME_ROLE_ARN="arn:aws:iam::111:role/containers-role" + logf=$(mktemp) + source "$HELPER" >"$logf" 2>&1 + # Exported env vars asserted directly (no echo). + [ "$AWS_ACCESS_KEY_ID" = "AKIAEXAMPLE" ] + [ "$AWS_SECRET_ACCESS_KEY" = "secret123" ] + [ "$AWS_SESSION_TOKEN" = "token123" ] + # Every user-facing message asserted in full, including emojis. + assert_contains "$(cat "$logf")" " 🔑 Assuming role: arn:aws:iam::111:role/containers-role" + assert_contains "$(cat "$logf")" " ✅ Role assumed successfully" +} + +@test "assume_role: returns non-zero and logs error when sts:AssumeRole fails" { + export CONTAINERS_ASSUME_ROLE_ARN="arn:aws:iam::111:role/containers-role" + aws() { + case "$*" in + *"sts assume-role"*) echo "AccessDenied" >&2; return 1 ;; + *) return 0 ;; + esac + } + export -f aws + run bash -c "source '$HELPER'" + [ "$status" -ne 0 ] + assert_contains "$output" " ❌ sts:AssumeRole failed for arn:aws:iam::111:role/containers-role" +} + +@test "assume_role: returns non-zero and logs error on malformed STS JSON" { + export CONTAINERS_ASSUME_ROLE_ARN="arn:aws:iam::111:role/containers-role" + aws() { echo "not-json"; return 0; } + export -f aws + run bash -c "source '$HELPER'" + [ "$status" -ne 0 ] + assert_contains "$output" " ❌ sts:AssumeRole returned incomplete credentials for arn:aws:iam::111:role/containers-role" +} + +@test "assume_role: returns non-zero when sts JSON lacks Credentials" { + export CONTAINERS_ASSUME_ROLE_ARN="arn:aws:iam::111:role/containers-role" + aws() { echo '{}'; return 0; } + export -f aws + run bash -c "source '$HELPER'" + [ "$status" -ne 0 ] + assert_contains "$output" " ❌ sts:AssumeRole returned incomplete credentials for arn:aws:iam::111:role/containers-role" +} diff --git a/k8s/utils/tests/assume_role_lib.bats b/k8s/utils/tests/assume_role_lib.bats new file mode 100644 index 00000000..f0814534 --- /dev/null +++ b/k8s/utils/tests/assume_role_lib.bats @@ -0,0 +1,80 @@ +#!/usr/bin/env bats +# ============================================================================= +# Unit tests for utils/assume_role_lib - pure assume-role ARN resolution. +# +# The library reads the AWS IAM provider as it appears in +# CONTEXT.providers["identity-access-control"] — already resolved for the scope's +# dimensions by the platform. The functions are pure JSON processors (jq only), +# so no np/aws mocking is needed. +# ============================================================================= + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + source "$BATS_TEST_DIRNAME/../assume_role_lib" + unset CONTAINERS_ASSUME_ROLE_ARN CONTAINERS_ASSUME_ROLE_ARN_DEFAULT +} + +teardown() { + unset CONTAINERS_ASSUME_ROLE_ARN CONTAINERS_ASSUME_ROLE_ARN_DEFAULT +} + +# --- arn_for_selector --------------------------------------------------------- + +@test "arn_for_selector: returns the arn matching the selector" { + json='{"iam_role_arns":{"arns":[{"selector":"lambda","arn":"arn:aws:iam::111:role/lambda-role"},{"selector":"containers","arn":"arn:aws:iam::111:role/containers-role"}]}}' + run arn_for_selector "$json" "containers" + [ "$status" -eq 0 ] + assert_equal "$output" "arn:aws:iam::111:role/containers-role" +} + +@test "arn_for_selector: first match wins when selector is duplicated" { + json='{"iam_role_arns":{"arns":[{"selector":"containers","arn":"arn:aws:iam::111:role/first"},{"selector":"containers","arn":"arn:aws:iam::111:role/second"}]}}' + run arn_for_selector "$json" "containers" + [ "$status" -eq 0 ] + assert_equal "$output" "arn:aws:iam::111:role/first" +} + +@test "arn_for_selector: empty when no selector matches" { + json='{"iam_role_arns":{"arns":[{"selector":"lambda","arn":"arn:aws:iam::111:role/lambda-role"}]}}' + run arn_for_selector "$json" "containers" + [ "$status" -eq 0 ] + assert_equal "$output" "" +} + +@test "arn_for_selector: empty on empty/malformed json (no crash)" { + run arn_for_selector "{}" "containers" + [ "$status" -eq 0 ] + assert_equal "$output" "" + run arn_for_selector "not-json" "containers" + [ "$status" -eq 0 ] + assert_equal "$output" "" +} + +# --- resolve_assume_role_arn (precedence) ------------------------------------ + +@test "resolve_assume_role_arn: CONTAINERS_ASSUME_ROLE_ARN env wins (override)" { + export CONTAINERS_ASSUME_ROLE_ARN="arn:aws:iam::111:role/override" + run resolve_assume_role_arn '{"iam_role_arns":{"arns":[{"selector":"containers","arn":"arn:aws:iam::111:role/containers-role"}]}}' "containers" + [ "$status" -eq 0 ] + assert_equal "$output" "arn:aws:iam::111:role/override" +} + +@test "resolve_assume_role_arn: resolves from the IAM provider by selector" { + run resolve_assume_role_arn '{"iam_role_arns":{"arns":[{"selector":"containers","arn":"arn:aws:iam::111:role/containers-role"}]}}' "containers" + [ "$status" -eq 0 ] + assert_equal "$output" "arn:aws:iam::111:role/containers-role" +} + +@test "resolve_assume_role_arn: falls back to CONTAINERS_ASSUME_ROLE_ARN_DEFAULT" { + export CONTAINERS_ASSUME_ROLE_ARN_DEFAULT="arn:aws:iam::111:role/agent-default" + run resolve_assume_role_arn '{}' "containers" + [ "$status" -eq 0 ] + assert_equal "$output" "arn:aws:iam::111:role/agent-default" +} + +@test "resolve_assume_role_arn: empty when nothing is configured" { + run resolve_assume_role_arn '{}' "containers" + [ "$status" -eq 0 ] + assert_equal "$output" "" +} diff --git a/k8s/utils/tests/assume_role_step.bats b/k8s/utils/tests/assume_role_step.bats new file mode 100644 index 00000000..4a1aad99 --- /dev/null +++ b/k8s/utils/tests/assume_role_step.bats @@ -0,0 +1,101 @@ +#!/usr/bin/env bats +# ============================================================================= +# Unit tests for utils/assume_role_step - resolves the role and assumes it. +# +# The IAM provider is read from CONTEXT.providers["identity-access-control"] +# (already dimension-resolved by the platform), so the tests only mock `aws` +# (sts). Success paths source the step IN-PROCESS and assert exported env vars +# directly; failure paths use `run` and assert the full hint output. +# ============================================================================= + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + export STEP="$BATS_TEST_DIRNAME/../assume_role_step" + + log() { if [ "$1" = "error" ]; then echo "$2" >&2; else echo "$2"; fi; } + export -f log + + export SCOPE_ID="scope-123" + # CONTEXT with the IAM provider (resolved) carrying a containers role. + export CONTEXT='{"providers":{"identity-access-control":{"iam_role_arns":{"arns":[{"selector":"containers","arn":"arn:aws:iam::111:role/containers-role"}]}}}}' + unset CONTAINERS_ASSUME_ROLE_ARN CONTAINERS_ASSUME_ROLE_ARN_DEFAULT CONTAINERS_ASSUME_ROLE_SELECTOR \ + AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + + aws() { + case "$*" in + *"sts assume-role"*) echo '{"Credentials":{"AccessKeyId":"AKIA1","SecretAccessKey":"sec1","SessionToken":"tok1"}}'; return 0 ;; + *) return 0 ;; + esac + } + export -f aws +} + +teardown() { + unset CONTAINERS_ASSUME_ROLE_ARN CONTAINERS_ASSUME_ROLE_ARN_DEFAULT CONTAINERS_ASSUME_ROLE_SELECTOR \ + AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN +} + +@test "assume_role_step: resolves the containers role from CONTEXT and exports creds" { + logf=$(mktemp) + source "$STEP" >"$logf" 2>&1 + [ "$CONTAINERS_ASSUME_ROLE_ARN" = "arn:aws:iam::111:role/containers-role" ] + [ "$AWS_ACCESS_KEY_ID" = "AKIA1" ] + [ "$AWS_SECRET_ACCESS_KEY" = "sec1" ] + [ "$AWS_SESSION_TOKEN" = "tok1" ] + assert_contains "$(cat "$logf")" " 🔑 Assuming role: arn:aws:iam::111:role/containers-role" + assert_contains "$(cat "$logf")" " ✅ Role assumed successfully" +} + +@test "assume_role_step: no IAM provider in CONTEXT is not an error (agent creds remain)" { + export CONTEXT='{"providers":{}}' + logf=$(mktemp) + source "$STEP" >"$logf" 2>&1 + [ -z "${AWS_ACCESS_KEY_ID:-}" ] + [ -z "$CONTAINERS_ASSUME_ROLE_ARN" ] + assert_contains "$(cat "$logf")" " ✅ assume_role=skipped (using agent credentials)" +} + +@test "assume_role_step: uses the dimension-resolved provider already in CONTEXT" { + # The platform injects whichever IAM config matched the scope's dimensions; + # the step just reads it. Here the resolved config carries a region-specific role. + export CONTEXT='{"providers":{"identity-access-control":{"iam_role_arns":{"arns":[{"selector":"containers","arn":"arn:aws:iam::111:role/us-east-1-role"}]}}}}' + logf=$(mktemp) + source "$STEP" >"$logf" 2>&1 + [ "$CONTAINERS_ASSUME_ROLE_ARN" = "arn:aws:iam::111:role/us-east-1-role" ] + assert_contains "$(cat "$logf")" " 🔑 Assuming role: arn:aws:iam::111:role/us-east-1-role" + assert_contains "$(cat "$logf")" " ✅ Role assumed successfully" +} + +@test "assume_role_step: honors CONTAINERS_ASSUME_ROLE_SELECTOR override" { + export CONTAINERS_ASSUME_ROLE_SELECTOR="custom" + export CONTEXT='{"providers":{"identity-access-control":{"iam_role_arns":{"arns":[{"selector":"custom","arn":"arn:aws:iam::111:role/custom-role"}]}}}}' + logf=$(mktemp) + source "$STEP" >"$logf" 2>&1 + [ "$CONTAINERS_ASSUME_ROLE_ARN" = "arn:aws:iam::111:role/custom-role" ] +} + +@test "assume_role_step: pre-set CONTAINERS_ASSUME_ROLE_ARN overrides provider resolution" { + export CONTAINERS_ASSUME_ROLE_ARN="arn:aws:iam::111:role/explicit-override" + logf=$(mktemp) + source "$STEP" >"$logf" 2>&1 + [ "$CONTAINERS_ASSUME_ROLE_ARN" = "arn:aws:iam::111:role/explicit-override" ] + [ "$AWS_ACCESS_KEY_ID" = "AKIA1" ] +} + +@test "assume_role_step: exits non-zero with full hints when sts:AssumeRole fails" { + aws() { + case "$*" in + *"sts assume-role"*) echo "AccessDenied" >&2; return 1 ;; + *) return 0 ;; + esac + } + export -f aws + run bash -c "source '$STEP'" + [ "$status" -ne 0 ] + assert_contains "$output" " ❌ assume_role step failed: could not assume arn:aws:iam::111:role/containers-role" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" " • The agent's role is not allowed to sts:AssumeRole the target role" + assert_contains "$output" " • The target role does not exist or does not trust the agent role" + assert_contains "$output" " • There is no role ARN configured for selector=containers" +} diff --git a/k8s/values.yaml b/k8s/values.yaml index 97c68ce6..8a93225f 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -2,6 +2,7 @@ provider_categories: - container-orchestration - cloud-providers - scope-configurations + - identity-access-control configuration: K8S_NAMESPACE: nullplatform CREATE_K8S_NAMESPACE_IF_NOT_EXIST: true @@ -19,6 +20,13 @@ configuration: ALB_MAX_LISTENERS: 48 ALB_METRICS_PUBLISH_ENABLED: false # ALB_METRICS_PUBLISH_TARGET: cloudwatch # Available values: cloudwatch | datadog + ALB_AUTOCREATE_ENABLED: false +# When true and every candidate ALB is at or above ALB_MAX_CAPACITY, the scope +# creation provisions a new ALB via a dummy Ingress, patches the +# container-orchestration provider to register the new ALB in the +# additional_*_names list, and waits for it to become active. + ALB_AUTOCREATE_NAME_PREFIX: nullplatform-auto- + ALB_AUTOCREATE_TIMEOUT_SECONDS: 300 DEPLOYMENT_MAX_WAIT_IN_SECONDS: 600 DEPLOYMENT_TEMPLATE: "$SERVICE_PATH/deployment/templates/deployment.yaml.tpl" SECRET_TEMPLATE: "$SERVICE_PATH/deployment/templates/secret.yaml.tpl" diff --git a/scheduled_task/deployment/templates/deployment.yaml.tpl b/scheduled_task/deployment/templates/deployment.yaml.tpl index a1d9f1f1..c556acfd 100644 --- a/scheduled_task/deployment/templates/deployment.yaml.tpl +++ b/scheduled_task/deployment/templates/deployment.yaml.tpl @@ -124,6 +124,11 @@ spec: tolerations: {{ data.ToYAML $tolerations | indent 10 }} {{- end }} + {{- $nodeSelector := index $deployment "nodeselector" }} + {{- if $nodeSelector }} + nodeSelector: +{{ data.ToYAML $nodeSelector | indent 12 }} + {{- end }} {{- end }} {{- if .pull_secrets.ENABLED }} imagePullSecrets: diff --git a/testing/run_bats_tests.sh b/testing/run_bats_tests.sh index d17384e6..91c7edc7 100755 --- a/testing/run_bats_tests.sh +++ b/testing/run_bats_tests.sh @@ -82,11 +82,24 @@ run_tests_in_dir() { local exit_code=0 ( cd "$test_dir" - # Use script to force TTY for colored output - # Exclude integration directory - those tests are run by run_integration_tests.sh - # --print-output-on-failure: only show test output when a test fails - script -q /dev/null bats --formatter pretty --print-output-on-failure $(find . -name "*.bats" -not -path "*/integration/*" | sort) - ) 2>&1 | tee "$temp_output" || exit_code=$? + # Force a TTY so the bats "pretty" formatter renders; --print-output-on-failure + # only dumps test output when a test fails. Exclude integration tests (run + # separately). `script`'s command syntax differs by platform: BSD/macOS takes a + # positional command (script ), while util-linux (Linux CI) needs + # -c "" . Using the wrong one silently runs zero tests. + local test_files + test_files=$(find . -name "*.bats" -not -path "*/integration/*" | sort | tr '\n' ' ') + local bats_cmd="bats --formatter pretty --print-output-on-failure $test_files" + # The pretty formatter shells out to `tput`, which needs $TERM. CI runners and + # containers often leave it unset - default it so tput can find a terminfo entry. + export TERM="${TERM:-xterm}" + if [ "$(uname)" = "Darwin" ]; then + script -q /dev/null $bats_cmd + else + script -qec "$bats_cmd" /dev/null + fi + ) 2>&1 | tee "$temp_output" + exit_code=${PIPESTATUS[0]} # Extract failed tests from output # Strip all ANSI escape codes (colors, cursor movements, etc.) @@ -191,4 +204,12 @@ if [ ${#FAILED_TESTS[@]} -gt 0 ]; then exit 1 fi +# A non-zero runner exit with no parsed "✗" lines means bats itself failed to run +# (e.g. a malformed .bats file or a broken `script` invocation) - never report success. +if [ "$HAS_FAILURES" -ne 0 ]; then + echo -e "${RED}BATS runner exited with errors but no individual test failures were parsed.${NC}" + echo -e "${RED}This usually means the test harness failed to execute - check the output above.${NC}" + exit 1 +fi + echo -e "${GREEN}All BATS tests passed!${NC}"