diff --git a/.github/workflows/e2e-os-service.yml b/.github/workflows/e2e-os-service.yml new file mode 100644 index 000000000..8b210db5d --- /dev/null +++ b/.github/workflows/e2e-os-service.yml @@ -0,0 +1,259 @@ +name: E2E OS-Service Self-Update + +# The os-service self-update flow end-to-end: a real alien-launcher +# supervising a real alien-operator against an in-process manager — +# promote, rollback, backoff, crash recovery, die-with-parent, and the +# min-launcher gate. Linux now; macOS and Windows jobs join with their +# platform phases. + +on: + pull_request: + paths: + - "crates/alien-launcher/**" + - "crates/alien-operator/**" + - "crates/alien-deploy-cli/**" + - "crates/alien-core/src/sync.rs" + - "crates/alien-core/src/self_update.rs" + - "crates/alien-manager/src/release_manifest.rs" + - "crates/alien-test/src/os_service.rs" + - "crates/alien-test/tests/e2e_os_service.rs" + - ".github/workflows/e2e-os-service.yml" + schedule: + # Nightly on main, catching drift from changes outside the path filter. + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +env: + CARGO_INCREMENTAL: "0" + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + +jobs: + linux-os-service: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork != true + runs-on: depot-ubuntu-24.04-arm-16 + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - name: Install protoc (opentelemetry-proto build dependency) + run: sudo apt-get update -qq && sudo apt-get install -y -qq protobuf-compiler + + - name: Build the launcher and the test-hooks operator + run: | + cargo build -p alien-launcher + cargo build -p alien-operator --features test-hooks + + - name: Launcher unit + platform-guard tests + run: cargo test -p alien-launcher + + - name: Install zig + cargo-zigbuild (the orphan-guard test builds a real workload OCI) + run: | + set -euo pipefail + curl -fsSL https://ziglang.org/download/0.13.0/zig-linux-aarch64-0.13.0.tar.xz -o /tmp/zig.tar.xz + sudo mkdir -p /opt/zig-0.13.0 + sudo tar xJf /tmp/zig.tar.xz -C /opt/zig-0.13.0 --strip-components=1 + sudo ln -sf /opt/zig-0.13.0/zig /usr/local/bin/zig + zig version + rustup target add aarch64-unknown-linux-musl + cargo install cargo-zigbuild --locked + + - name: E2E scenarios + env: + ALIEN_LAUNCHER_BINARY: ${{ github.workspace }}/target/debug/alien-launcher + ALIEN_OPERATOR_BINARY: ${{ github.workspace }}/target/debug/alien-operator + RUST_LOG: info + run: | + cargo test -p alien-test --features e2e-os-service --test e2e_os_service -- --test-threads=4 + + - name: systemd smoke — unit template under a real init + run: | + set -euo pipefail + # Prove the Type=notify + watchdog wiring under real systemd: the + # launcher must reach "active" (which requires it to send READY=1) + # and stop cleanly. The operator here is a stub HTTP server that + # serves /livez -> 200 (the launcher gates its startup READY on + # liveness); the real sync/update flow is covered by the E2E above. + DATA_DIR=$(mktemp -d) + sudo mkdir -p "$DATA_DIR"/{versions/0.0.1,launcher,state,state-snapshots,failed,download} + # Stub operator: bind ALIEN_HEALTH_ADDR (the launcher passes it), + # answer every GET with 200, exit on SIGTERM. + sudo tee "$DATA_DIR/versions/0.0.1/alien-operator" >/dev/null <<'OP' + #!/usr/bin/env python3 + import http.server, os + host, port = os.environ["ALIEN_HEALTH_ADDR"].rsplit(":", 1) + class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200); self.end_headers(); self.wfile.write(b"ok") + def log_message(self, *a): pass + http.server.HTTPServer((host, int(port)), H).serve_forever() + OP + sudo chmod 755 "$DATA_DIR/versions/0.0.1/alien-operator" + sudo ln -s versions/0.0.1 "$DATA_DIR/current" + sudo ln -s versions/0.0.1 "$DATA_DIR/last-stable" + sudo cp target/debug/alien-launcher "$DATA_DIR/launcher/alien-launcher" + + sudo tee /etc/systemd/system/alien-e2e-smoke.service >/dev/null </dev/null || true + sudo rm /etc/systemd/system/alien-e2e-smoke.service + sudo systemctl daemon-reload + + # Windows: the platform unit + state-machine tests (the real proof of the + # junction VersionStore, the Job-Object ChildSupervisor, and the SCM control + # mapping) plus an SCM smoke. The full seven-scenario E2E rig is Unix-only for + # now (symlinks, /bin/sh version wrappers, pgrep/ps, a musl workload build) — + # porting `os_service.rs` to Windows is tracked separately (doc 17 T3.7). + windows-os-service: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork != true + runs-on: depot-windows-2025-16 + timeout-minutes: 60 + env: + CARGO_INCREMENTAL: "0" + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + # ring / aws-lc-sys build their C with the bundled cmake builder + NASM. + AWS_LC_SYS_CMAKE_BUILDER: "1" + + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@nightly + + - name: Install build tools (protoc, nasm, cmake) + shell: bash + run: choco install protoc nasm cmake -y + + - name: Build the launcher + shell: bash + run: | + export PATH="/c/Program Files/NASM:$PATH" + cargo build -p alien-launcher + + # The real proof of T3.1–T3.4 on native Windows: the junction VersionStore's + # flip + crash-residue reconciliation + locked-file gc, the full Phase-0 + # state-machine suite over that store, the Job-Object kill-on-close + # ChildSupervisor (spawn/stop/exit-code + drop-kills-child), and the SCM + # control-signal mapping — all run here as real processes. + - name: Launcher + operator unit + platform tests + shell: bash + run: | + export PATH="/c/Program Files/NASM:$PATH" + cargo test -p alien-launcher -p alien-operator + + # SCM smoke (the runner is admin): register the launcher as a real service, + # prove it starts (a tiny stub operator answers /livez so the launcher + # reports RUNNING), the doc-12 recovery config lands, and a stop propagates + # cleanly through the launcher to the operator, then delete. + - name: SCM smoke — sc create/start/qfailure/stop/delete + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + $svc = "alien-e2e-scm" + $data = "C:\alien-scm" # fixed, space-free: keeps sc binPath simple + $ver = Join-Path $data "versions\0.0.1" + + # Clean any residue from a previous run on this (reused) runner. + sc.exe stop $svc *> $null + sc.exe delete $svc *> $null + if (Test-Path $data) { Remove-Item -Recurse -Force $data } + foreach ($d in "versions\0.0.1","launcher","state","state-snapshots","failed","download") { + New-Item -ItemType Directory -Force -Path (Join-Path $data $d) | Out-Null + } + + # A tiny stub "operator": bind ALIEN_HEALTH_ADDR (the launcher passes it) + # and answer every GET 200 — the launcher gates its SCM 'Running' on the + # operator's /livez. Compiled to a real .exe so the launcher can + # CreateProcess it (the spawn name is alien-operator.exe). + $stub = @' + use std::io::{Read, Write}; + use std::net::TcpListener; + fn main() { + let addr = std::env::var("ALIEN_HEALTH_ADDR").expect("ALIEN_HEALTH_ADDR"); + let listener = TcpListener::bind(&addr).expect("bind health addr"); + for stream in listener.incoming() { + let mut stream = match stream { Ok(s) => s, Err(_) => continue }; + let mut buf = [0u8; 1024]; + let _ = stream.read(&mut buf); + let _ = stream.write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", + ); + } + } + '@ + $stubRs = Join-Path $env:RUNNER_TEMP "stub_operator.rs" + Set-Content -Path $stubRs -Value $stub -Encoding ascii + & rustc $stubRs -O -o (Join-Path $ver "alien-operator.exe") + if ($LASTEXITCODE -ne 0) { throw "rustc stub operator failed" } + + Copy-Item "target\debug\alien-launcher.exe" (Join-Path $data "launcher\alien-launcher.exe") + + # Junction pointers, exactly as the installer seeds them. + New-Item -ItemType Junction -Path (Join-Path $data "current") -Target $ver | Out-Null + New-Item -ItemType Junction -Path (Join-Path $data "last-stable") -Target $ver | Out-Null + + $launcher = Join-Path $data "launcher\alien-launcher.exe" + $bin = "$launcher --data-dir $data --probation-secs 10 --health-port 7799" + + Write-Host "== sc create ==" + sc.exe create $svc binPath= $bin start= demand + if ($LASTEXITCODE -ne 0) { throw "sc create failed" } + + Write-Host "== sc failure (doc-12 recovery config) ==" + sc.exe failure $svc reset= 86400 actions= restart/5000/restart/5000/restart/5000 + if ($LASTEXITCODE -ne 0) { throw "sc failure failed" } + $qf = (sc.exe qfailure $svc | Out-String) + Write-Host $qf + if ($qf -notmatch "86400") { throw "recovery reset period (86400) not applied" } + if ($qf -notmatch "(?i)restart") { throw "restart actions not applied" } + + Write-Host "== sc start ==" + sc.exe start $svc | Out-Null + $running = $false + foreach ($i in 1..30) { + Start-Sleep -Seconds 1 + if ((sc.exe query $svc | Out-String) -match "RUNNING") { $running = $true; break } + } + if (-not $running) { + sc.exe query $svc + sc.exe delete $svc *> $null + throw "service never reached RUNNING (launcher did not gate on the operator /livez)" + } + Write-Host "service RUNNING" + + Write-Host "== sc stop (must propagate to the operator) ==" + sc.exe stop $svc | Out-Null + $stopped = $false + foreach ($i in 1..30) { + Start-Sleep -Seconds 1 + if ((sc.exe query $svc | Out-String) -match "STOPPED") { $stopped = $true; break } + } + sc.exe delete $svc *> $null + if (-not $stopped) { throw "service did not stop cleanly" } + Write-Host "service STOPPED cleanly — stop propagated through to the operator" diff --git a/Cargo.lock b/Cargo.lock index 4fb0b308d..9aa06a750 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,7 +85,7 @@ dependencies = [ [[package]] name = "alien-aws-clients" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-aws-clients", "alien-client-core", @@ -127,7 +127,7 @@ dependencies = [ [[package]] name = "alien-azure-clients" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-client-core", "alien-core", @@ -170,7 +170,7 @@ dependencies = [ [[package]] name = "alien-bindings" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -220,7 +220,7 @@ dependencies = [ [[package]] name = "alien-build" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-build", "alien-core", @@ -254,7 +254,7 @@ dependencies = [ [[package]] name = "alien-cli" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-bindings", "alien-build", @@ -324,7 +324,7 @@ dependencies = [ [[package]] name = "alien-cli-common" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-core", "alien-deployment", @@ -338,7 +338,7 @@ dependencies = [ [[package]] name = "alien-client-config" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -356,7 +356,7 @@ dependencies = [ [[package]] name = "alien-client-core" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-error", "anyhow", @@ -375,7 +375,7 @@ dependencies = [ [[package]] name = "alien-cloudformation" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-core", "alien-error", @@ -390,7 +390,7 @@ dependencies = [ [[package]] name = "alien-commands" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -424,7 +424,7 @@ dependencies = [ [[package]] name = "alien-commands-client" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-core", "base64 0.22.1", @@ -439,7 +439,7 @@ dependencies = [ [[package]] name = "alien-core" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-error", "alien-macros", @@ -458,8 +458,10 @@ dependencies = [ "reqwest 0.12.28", "rstest", "schemars 0.8.22", + "semver", "serde", "serde_json", + "tempfile", "thiserror 2.0.18", "tokio", "tracing", @@ -470,7 +472,7 @@ dependencies = [ [[package]] name = "alien-deploy-cli" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-cli-common", "alien-core", @@ -493,6 +495,7 @@ dependencies = [ "getrandom 0.2.17", "hex", "hostname 0.4.2", + "junction", "libc", "reqwest 0.12.28", "serde", @@ -513,7 +516,7 @@ dependencies = [ [[package]] name = "alien-deployment" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -544,7 +547,7 @@ dependencies = [ [[package]] name = "alien-error" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-error-derive", "anyhow", @@ -557,7 +560,7 @@ dependencies = [ [[package]] name = "alien-error-derive" -version = "1.13.0" +version = "1.13.2" dependencies = [ "proc-macro2", "quote", @@ -567,7 +570,7 @@ dependencies = [ [[package]] name = "alien-gcp-clients" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-client-core", "alien-core", @@ -600,10 +603,11 @@ dependencies = [ [[package]] name = "alien-helm" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-core", "alien-error", + "alien-helm", "indexmap 2.14.0", "insta", "serde", @@ -614,7 +618,7 @@ dependencies = [ [[package]] name = "alien-infra" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -674,7 +678,7 @@ dependencies = [ [[package]] name = "alien-k8s-clients" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-client-core", "alien-core", @@ -701,9 +705,33 @@ dependencies = [ "workspace_root", ] +[[package]] +name = "alien-launcher" +version = "1.13.2" +dependencies = [ + "alien-core", + "alien-error", + "chrono", + "command-group", + "junction", + "nix 0.29.0", + "sd-notify", + "semver", + "serde", + "serde_json", + "sha2 0.10.9", + "signal-hook", + "tempfile", + "tracing", + "tracing-subscriber", + "ureq 3.3.0", + "windows-service", + "windows-sys 0.59.0", +] + [[package]] name = "alien-local" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-bindings", "alien-build", @@ -744,7 +772,7 @@ dependencies = [ [[package]] name = "alien-macros" -version = "1.13.0" +version = "1.13.2" dependencies = [ "proc-macro2", "quote", @@ -753,7 +781,7 @@ dependencies = [ [[package]] name = "alien-manager" -version = "1.13.0" +version = "1.13.2" dependencies = [ "aegis", "alien-bindings", @@ -792,6 +820,7 @@ dependencies = [ "rustls 0.23.41", "sea-query", "sec", + "semver", "serde", "serde_json", "sha2 0.10.9", @@ -812,7 +841,7 @@ dependencies = [ [[package]] name = "alien-manager-api" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-error", "chrono", @@ -831,7 +860,7 @@ dependencies = [ [[package]] name = "alien-observer" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -851,7 +880,7 @@ dependencies = [ [[package]] name = "alien-operator" -version = "1.13.0" +version = "1.13.2" dependencies = [ "aegis", "alien-client-config", @@ -860,6 +889,7 @@ dependencies = [ "alien-deployment", "alien-error", "alien-infra", + "alien-k8s-clients", "alien-local", "alien-manager-api", "async-trait", @@ -871,9 +901,11 @@ dependencies = [ "bon", "chrono", "clap", + "ed25519-dalek", "fs2", "futures", "hostname 0.4.2", + "k8s-openapi", "libc", "opentelemetry-proto", "prost 0.13.5", @@ -881,6 +913,7 @@ dependencies = [ "rustls 0.23.41", "serde", "serde_json", + "sha2 0.10.9", "tempfile", "tokio", "tokio-tungstenite", @@ -895,7 +928,7 @@ dependencies = [ [[package]] name = "alien-permissions" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-core", "alien-error", @@ -914,7 +947,7 @@ dependencies = [ [[package]] name = "alien-platform-api" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-error", "chrono", @@ -932,7 +965,7 @@ dependencies = [ [[package]] name = "alien-preflights" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-core", "alien-error", @@ -949,7 +982,7 @@ dependencies = [ [[package]] name = "alien-runtime" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-bindings", "alien-commands", @@ -966,6 +999,7 @@ dependencies = [ "chrono", "clap", "cloudevents-sdk", + "command-group", "config", "dotenvy", "futures-util", @@ -1007,14 +1041,14 @@ dependencies = [ [[package]] name = "alien-sdk" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-bindings", ] [[package]] name = "alien-terraform" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-core", "alien-error", @@ -1029,7 +1063,7 @@ dependencies = [ [[package]] name = "alien-test" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -1075,7 +1109,7 @@ dependencies = [ [[package]] name = "alien-test-app" -version = "1.13.0" +version = "1.13.2" dependencies = [ "alien-bindings", "alien-error", @@ -2741,6 +2775,18 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "command-group" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68fa787550392a9d58f44c21a3022cfb3ea3e2458b7f85d3b399d0ceeccf409" +dependencies = [ + "async-trait", + "nix 0.27.1", + "tokio", + "winapi", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -3156,6 +3202,33 @@ dependencies = [ "cmov", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "darling" version = "0.20.11" @@ -3608,6 +3681,16 @@ dependencies = [ "spki 0.7.3", ] +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8 0.10.2", + "signature 2.2.0", +] + [[package]] name = "ed25519-compact" version = "2.3.1" @@ -3618,6 +3701,20 @@ dependencies = [ "getrandom 0.4.3", ] +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.16.0" @@ -3850,6 +3947,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "filetime" version = "0.2.29" @@ -5267,6 +5370,16 @@ dependencies = [ "serde", ] +[[package]] +name = "junction" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfc352a66ba903c23239ef51e809508b6fc2b0f90e3476ac7a9ff47e863ae95" +dependencies = [ + "scopeguard", + "windows-sys 0.61.2", +] + [[package]] name = "jwt" version = "0.16.0" @@ -6001,6 +6114,29 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "libc", +] + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -6289,7 +6425,7 @@ dependencies = [ "sha2 0.10.9", "tar", "toml 0.8.23", - "ureq", + "ureq 2.12.1", "url", "urlencoding", "uuid", @@ -8286,6 +8422,15 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sd-notify" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b943eadf71d8b69e661330cb0e2656e31040acf21ee7708e2c238a0ec6af2bf4" +dependencies = [ + "libc", +] + [[package]] name = "sea-query" version = "0.32.7" @@ -10377,6 +10522,31 @@ dependencies = [ "webpki-roots 0.26.11", ] +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64 0.22.1", + "log", + "percent-encoding", + "ureq-proto", + "utf8-zero", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64 0.22.1", + "http 1.4.2", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -10402,6 +10572,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" diff --git a/Cargo.toml b/Cargo.toml index 39836c1f1..44001f851 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ members = [ "crates/alien-helm", "crates/alien-manager", "crates/alien-operator", + "crates/alien-launcher", "crates/alien-deploy-cli", "client-sdks/manager/rust", "client-sdks/platform/rust", @@ -150,6 +151,31 @@ rand = "0.9" rand_distr = "0.5" regex = "1" reqwest = { version = "0.12.2", default-features = false } +# Launcher platform shims: process groups + Job Objects behind one API. +command-group = "5.0" +# systemd notify/watchdog protocol without linking libsystemd (launcher, Linux). +sd-notify = "0.4" +# Safe Unix signal registration for the launcher's poll-based control channel. +signal-hook = "0.3" +# Unix syscall surface for the launcher shims (pdeathsig, killpg, statvfs). +nix = { version = "0.29", features = ["signal", "process", "fs"] } +# Windows launcher shims: SCM service host + Win32 (console Ctrl-C, junctions). +windows-service = "0.7" +windows-sys = "0.59" +# Directory junctions for the Windows version store's `current`/`last-stable`. +junction = "1.2" +# Blocking HTTP client for the launcher's /readyz probe — deliberately tiny so +# the must-never-break supervisor binary stays free of an async runtime. No +# default features: the probe only ever speaks plain HTTP to 127.0.0.1, so +# TLS (rustls/ring) would be dead weight in the safety-net binary. +ureq = { version = "3.1", default-features = false } +# Spec-correct SemVer parsing + precedence ordering (the same crate cargo uses). +# One parser for validation AND comparison everywhere versions are ordered — +# hand-rolled ordering gets prerelease precedence wrong (1.4.0-rc.1 < 1.4.0). +semver = "1.0" +# ed25519 signature verification for self-update artifacts (feature-gated stub +# until the signing workstream lands - see the enforce-signature features). +ed25519-dalek = "2.1" rkyv = "0.7.43" rstest = "0.25.0" schemars = { version = "0.8", features = ["indexmap2"] } diff --git a/client-sdks/manager/openapi.json b/client-sdks/manager/openapi.json index 3b2ffcadb..29a1f693c 100644 --- a/client-sdks/manager/openapi.json +++ b/client-sdks/manager/openapi.json @@ -886,6 +886,53 @@ } } }, + "/v1/deployments/{id}/target-operator-version": { + "put": { + "tags": [ + "deployments" + ], + "summary": "Admin-only knob behind the dashboard's \"Set target version\" control\n(and the equivalent flow on the SaaS API). Writes\n`target_operator_version` on the deployment row; the sync handler reads\nit on each /v1/sync and emits `operator_target` whenever it differs from\nthe operator's reported version, until they match.", + "operationId": "setDeploymentTargetOperatorVersion", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Deployment ID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetTargetOperatorVersionRequest" + } + } + }, + "required": true + }, + "responses": { + "202": { + "description": "Target operator version updated" + }, + "400": { + "description": "Invalid pin (not semver, build metadata, or a downgrade)" + }, + "401": { + "description": "Unauthorized" + }, + "403": { + "description": "Forbidden" + }, + "404": { + "description": "Not found" + } + } + } + }, "/v1/initialize": { "post": { "tags": [ @@ -932,6 +979,46 @@ ] } }, + "/v1/rejoin": { + "post": { + "tags": [ + "sync" + ], + "summary": "`POST /v1/rejoin` — re-acquire a deployment-scoped sync token for an\nexisting deployment by name. The OSS handler is intentionally lean:\nthe dg-token caller specifies the name, the store looks up the row,\nand a fresh `Deployment` token is minted and returned. Returns\n`404 Deployment not found` when no row matches — agents should fall\nback to `/v1/initialize` in that case.", + "description": "Distinct from `/v1/initialize`'s idempotency branch because the agent\nwants an explicit, distinguishable code path for \"I lost local state,\nplease re-attach me\" vs \"I'm a brand-new pod claiming a name\". The\nplatform-mode override forwards this to the SaaS rejoin endpoint\nwhere audit / event semantics differ.", + "operationId": "rejoin", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RejoinRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Existing deployment rejoined; fresh token returned", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RejoinResponse" + } + } + } + }, + "404": { + "description": "No deployment with that name in caller's deployment group" + } + }, + "security": [ + { + "bearer": [] + } + ] + } + }, "/v1/releases": { "get": { "tags": [ @@ -1418,18 +1505,56 @@ "deploymentId": { "type": "string" }, + "launcherVersion": { + "type": [ + "string", + "null" + ], + "description": "Version of the alien-launcher supervising this operator (os-service\nonly; reported, never driven). Gates binary targets against\nmin_launcher_version." + }, "observedInventoryBatches": { "type": "array", "items": { "$ref": "#/components/schemas/ObservedInventoryBatch" } }, + "operatorArch": { + "type": [ + "string", + "null" + ], + "description": "Operator host arch — `x86_64` / `aarch64`." + }, + "operatorImageRepository": { + "type": [ + "string", + "null" + ], + "description": "Image repository the operator was pulled from (no tag), chart-injected.\nSurfaced in the dashboard pin-version UI. Optional and Kubernetes-only." + }, + "operatorOs": { + "type": [ + "string", + "null" + ], + "description": "Operator host OS — `linux` / `macos` / `windows`." + }, + "operatorUpdate": { + "description": "Outcome of the operator's in-flight self-update (structured\n`OperatorUpdateReport`), forwarded verbatim to the platform reconcile so\nthe dashboard can show a truthful failed/in-progress state; opaque to the\nstandalone manager. Optional for back-compat." + }, "operatorVersion": { "type": [ "string", "null" ] }, + "packaging": { + "type": [ + "string", + "null" + ], + "description": "Supervisor packaging — `os-service` / `kubernetes`." + }, "resourceHeartbeats": { "type": "array", "items": { @@ -1452,6 +1577,9 @@ "currentState": { "description": "Authoritative deployment state from the manager.\n\nReturned when a pull deployment attaches with an empty local state while\nthe manager already has imported or previously reconciled state." }, + "operatorTarget": { + "description": "Desired agent self-update target. The payload carries either `binary`\n(OS-service flow) or `helm` (Kubernetes flow); the agent picks the\none matching its packaging." + }, "target": {} } }, @@ -6283,9 +6411,46 @@ } ] }, + "launcherVersion": { + "type": [ + "string", + "null" + ], + "description": "Version of the frozen alien-launcher supervising the operator\n(os-service only). Drives the \"redeploy required\" surface when it is\nbelow a target's minLauncherVersion." + }, "name": { "type": "string" }, + "operatorArch": { + "type": [ + "string", + "null" + ] + }, + "operatorImageRepository": { + "type": [ + "string", + "null" + ] + }, + "operatorOs": { + "type": [ + "string", + "null" + ] + }, + "operatorVersion": { + "type": [ + "string", + "null" + ] + }, + "packaging": { + "type": [ + "string", + "null" + ] + }, "platform": { "$ref": "#/components/schemas/Platform" }, @@ -6302,6 +6467,12 @@ "status": { "type": "string" }, + "targetOperatorVersion": { + "type": [ + "string", + "null" + ] + }, "updatedAt": { "type": [ "string", @@ -12128,6 +12299,33 @@ } } }, + "RejoinRequest": { + "type": "object", + "description": "`POST /v1/rejoin` — re-acquire a deployment-scoped sync token for an\nagent whose persistent state was wiped (e.g. emptyDir on pod restart).\n\nThe agent calls this when `/v1/initialize` returns a name-conflict\nerror: the deployment row already exists, so creating it would 409,\nbut the agent legitimately needs a new sync token to keep operating\nagainst it. Auth is the same dg bearer the chart originally mounted.\n\n`name` is required — without it the server can't disambiguate which\nrow to reattach to.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + } + } + }, + "RejoinResponse": { + "type": "object", + "required": [ + "deploymentId", + "token" + ], + "properties": { + "deploymentId": { + "type": "string" + }, + "token": { + "type": "string" + } + } + }, "ReleaseRequest": { "type": "object", "required": [ @@ -13050,6 +13248,18 @@ } } }, + "SetTargetOperatorVersionRequest": { + "type": "object", + "properties": { + "targetOperatorVersion": { + "type": [ + "string", + "null" + ], + "description": "Target operator version (semver, no `+build` metadata).\n`None`/omitted clears the target." + } + } + }, "StackByPlatform": { "type": "object", "description": "The release API accepts stacks keyed by platform.\nOnly one platform stack needs to be present.", diff --git a/client-sdks/manager/rust/openapi-3.0.json b/client-sdks/manager/rust/openapi-3.0.json index 05ef2fb06..3ca583f7c 100644 --- a/client-sdks/manager/rust/openapi-3.0.json +++ b/client-sdks/manager/rust/openapi-3.0.json @@ -886,6 +886,53 @@ } } }, + "/v1/deployments/{id}/target-operator-version": { + "put": { + "tags": [ + "deployments" + ], + "summary": "Admin-only knob behind the dashboard's \"Set target version\" control\n(and the equivalent flow on the SaaS API). Writes\n`target_operator_version` on the deployment row; the sync handler reads\nit on each /v1/sync and emits `operator_target` whenever it differs from\nthe operator's reported version, until they match.", + "operationId": "setDeploymentTargetOperatorVersion", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Deployment ID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetTargetOperatorVersionRequest" + } + } + }, + "required": true + }, + "responses": { + "202": { + "description": "Target operator version updated" + }, + "400": { + "description": "Invalid pin (not semver, build metadata, or a downgrade)" + }, + "401": { + "description": "Unauthorized" + }, + "403": { + "description": "Forbidden" + }, + "404": { + "description": "Not found" + } + } + } + }, "/v1/initialize": { "post": { "tags": [ @@ -932,6 +979,46 @@ ] } }, + "/v1/rejoin": { + "post": { + "tags": [ + "sync" + ], + "summary": "`POST /v1/rejoin` — re-acquire a deployment-scoped sync token for an\nexisting deployment by name. The OSS handler is intentionally lean:\nthe dg-token caller specifies the name, the store looks up the row,\nand a fresh `Deployment` token is minted and returned. Returns\n`404 Deployment not found` when no row matches — agents should fall\nback to `/v1/initialize` in that case.", + "description": "Distinct from `/v1/initialize`'s idempotency branch because the agent\nwants an explicit, distinguishable code path for \"I lost local state,\nplease re-attach me\" vs \"I'm a brand-new pod claiming a name\". The\nplatform-mode override forwards this to the SaaS rejoin endpoint\nwhere audit / event semantics differ.", + "operationId": "rejoin", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RejoinRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Existing deployment rejoined; fresh token returned", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RejoinResponse" + } + } + } + }, + "404": { + "description": "No deployment with that name in caller's deployment group" + } + }, + "security": [ + { + "bearer": [] + } + ] + } + }, "/v1/releases": { "get": { "tags": [ @@ -1408,16 +1495,44 @@ "deploymentId": { "type": "string" }, + "launcherVersion": { + "type": "string", + "description": "Version of the alien-launcher supervising this operator (os-service\nonly; reported, never driven). Gates binary targets against\nmin_launcher_version.", + "nullable": true + }, "observedInventoryBatches": { "type": "array", "items": { "$ref": "#/components/schemas/ObservedInventoryBatch" } }, + "operatorArch": { + "type": "string", + "description": "Operator host arch — `x86_64` / `aarch64`.", + "nullable": true + }, + "operatorImageRepository": { + "type": "string", + "description": "Image repository the operator was pulled from (no tag), chart-injected.\nSurfaced in the dashboard pin-version UI. Optional and Kubernetes-only.", + "nullable": true + }, + "operatorOs": { + "type": "string", + "description": "Operator host OS — `linux` / `macos` / `windows`.", + "nullable": true + }, + "operatorUpdate": { + "description": "Outcome of the operator's in-flight self-update (structured\n`OperatorUpdateReport`), forwarded verbatim to the platform reconcile so\nthe dashboard can show a truthful failed/in-progress state; opaque to the\nstandalone manager. Optional for back-compat." + }, "operatorVersion": { "type": "string", "nullable": true }, + "packaging": { + "type": "string", + "description": "Supervisor packaging — `os-service` / `kubernetes`.", + "nullable": true + }, "resourceHeartbeats": { "type": "array", "items": { @@ -1438,6 +1553,9 @@ "currentState": { "description": "Authoritative deployment state from the manager.\n\nReturned when a pull deployment attaches with an empty local state while\nthe manager already has imported or previously reconciled state." }, + "operatorTarget": { + "description": "Desired agent self-update target. The payload carries either `binary`\n(OS-service flow) or `helm` (Kubernetes flow); the agent picks the\none matching its packaging." + }, "target": {} } }, @@ -5474,9 +5592,34 @@ "$ref": "#/components/schemas/ImportSourceKind", "nullable": true }, + "launcherVersion": { + "type": "string", + "description": "Version of the frozen alien-launcher supervising the operator\n(os-service only). Drives the \"redeploy required\" surface when it is\nbelow a target's minLauncherVersion.", + "nullable": true + }, "name": { "type": "string" }, + "operatorArch": { + "type": "string", + "nullable": true + }, + "operatorImageRepository": { + "type": "string", + "nullable": true + }, + "operatorOs": { + "type": "string", + "nullable": true + }, + "operatorVersion": { + "type": "string", + "nullable": true + }, + "packaging": { + "type": "string", + "nullable": true + }, "platform": { "$ref": "#/components/schemas/Platform" }, @@ -5493,6 +5636,10 @@ "status": { "type": "string" }, + "targetOperatorVersion": { + "type": "string", + "nullable": true + }, "updatedAt": { "type": "string", "nullable": true @@ -10371,6 +10518,33 @@ } } }, + "RejoinRequest": { + "type": "object", + "description": "`POST /v1/rejoin` — re-acquire a deployment-scoped sync token for an\nagent whose persistent state was wiped (e.g. emptyDir on pod restart).\n\nThe agent calls this when `/v1/initialize` returns a name-conflict\nerror: the deployment row already exists, so creating it would 409,\nbut the agent legitimately needs a new sync token to keep operating\nagainst it. Auth is the same dg bearer the chart originally mounted.\n\n`name` is required — without it the server can't disambiguate which\nrow to reattach to.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + } + } + }, + "RejoinResponse": { + "type": "object", + "required": [ + "deploymentId", + "token" + ], + "properties": { + "deploymentId": { + "type": "string" + }, + "token": { + "type": "string" + } + } + }, "ReleaseRequest": { "type": "object", "required": [ @@ -11262,6 +11436,16 @@ } } }, + "SetTargetOperatorVersionRequest": { + "type": "object", + "properties": { + "targetOperatorVersion": { + "type": "string", + "description": "Target operator version (semver, no `+build` metadata).\n`None`/omitted clears the target.", + "nullable": true + } + } + }, "StackByPlatform": { "type": "object", "description": "The release API accepts stacks keyed by platform.\nOnly one platform stack needs to be present.", diff --git a/client-sdks/manager/typescript/.speakeasy/gen.yaml b/client-sdks/manager/typescript/.speakeasy/gen.yaml index 3d847851e..453d2aa02 100644 --- a/client-sdks/manager/typescript/.speakeasy/gen.yaml +++ b/client-sdks/manager/typescript/.speakeasy/gen.yaml @@ -16,6 +16,8 @@ generation: requestResponseComponentNamesFeb2024: true securityFeb2025: true sharedErrorComponentsApr2025: true + sharedNestedComponentsJan2026: false + nameOverrideFeb2026: false auth: oAuth2ClientCredentialsEnabled: false oAuth2PasswordEnabled: false @@ -25,6 +27,7 @@ generation: schemas: allOfMergeStrategy: shallowMerge requestBodyFieldName: "" + versioningStrategy: automatic persistentEdits: {} tests: generateTests: false @@ -57,6 +60,8 @@ typescript: enumFormat: union envVarPrefix: ALIEN_MANAGER exportZodModelNamespace: false + fixEnumNameSanitization: false + flatAdditionalProperties: false flattenGlobalSecurity: true flatteningOrder: parameters-first formStringArrayEncodeMode: encoded-string @@ -75,6 +80,7 @@ typescript: inputModelSuffix: input jsonpath: rfc9535 laxMode: strict + legacyFileNaming: true maxMethodParams: 0 methodArguments: infer-optional-args modelPropertyCasing: camel @@ -83,10 +89,13 @@ typescript: outputModelSuffix: output packageName: '@alienplatform/manager-api' preApplyUnionDiscriminators: true + preserveModelFieldNames: false responseFormat: flat sseFlatResponse: true templateVersion: v2 unionStrategy: left-to-right usageSDKInitImports: [] useIndexModules: true + useOxlint: false + useTsgo: false zodVersion: v4 diff --git a/client-sdks/platform/typescript/docs/models/operations/createdeploymentresponse.md b/client-sdks/platform/typescript/docs/models/operations/createdeploymentresponse.md new file mode 100644 index 000000000..aa92c232f --- /dev/null +++ b/client-sdks/platform/typescript/docs/models/operations/createdeploymentresponse.md @@ -0,0 +1,55 @@ +# CreateDeploymentResponse + + +## Supported Types + +### `operations.CreateDeploymentResponseBody` + +```typescript +const value: operations.CreateDeploymentResponseBody = { + deployment: { + id: "dep_0c29fq4a2yjb7kx3smwdgxlc", + name: "acme-prod", + status: "deleting", + projectId: "prj_mcytp6z3j91f7tn5ryqsfwtr", + platform: "aws", + deploymentProtocolVersion: 108843, + deploymentGroupId: "dg_r27ict8c7vcgsumpj90ackf7b", + stackSettings: {}, + currentReleaseId: "rel_WbhQgksrawSKIpEN0NAssHX9", + desiredReleaseId: "rel_WbhQgksrawSKIpEN0NAssHX9", + pinnedReleaseId: "rel_WbhQgksrawSKIpEN0NAssHX9", + retryRequested: true, + createdAt: new Date("2024-05-26T23:54:19.383Z"), + updatedAt: new Date("2026-10-15T18:20:54.194Z"), + managerId: "mgr_enxscjrqiiu2lrc672hwwuc5", + workspaceId: "ws_It13CUaGEhLLAB87simX0", + }, +}; +``` + +### `models.CreateDeploymentResponse` + +```typescript +const value: models.CreateDeploymentResponse = { + deployment: { + id: "dep_0c29fq4a2yjb7kx3smwdgxlc", + name: "acme-prod", + status: "deleting", + projectId: "prj_mcytp6z3j91f7tn5ryqsfwtr", + platform: "aws", + deploymentProtocolVersion: 108843, + deploymentGroupId: "dg_r27ict8c7vcgsumpj90ackf7b", + stackSettings: {}, + currentReleaseId: "rel_WbhQgksrawSKIpEN0NAssHX9", + desiredReleaseId: "rel_WbhQgksrawSKIpEN0NAssHX9", + pinnedReleaseId: "rel_WbhQgksrawSKIpEN0NAssHX9", + retryRequested: true, + createdAt: new Date("2024-05-26T23:54:19.383Z"), + updatedAt: new Date("2026-10-15T18:20:54.194Z"), + managerId: "mgr_enxscjrqiiu2lrc672hwwuc5", + workspaceId: "ws_It13CUaGEhLLAB87simX0", + }, +}; +``` + diff --git a/client-sdks/platform/typescript/docs/models/operations/createdeploymentresponsebody.md b/client-sdks/platform/typescript/docs/models/operations/createdeploymentresponsebody.md new file mode 100644 index 000000000..875628ea2 --- /dev/null +++ b/client-sdks/platform/typescript/docs/models/operations/createdeploymentresponsebody.md @@ -0,0 +1,37 @@ +# CreateDeploymentResponseBody + +Existing deployment returned for idempotent deployment-group registration. + +## Example Usage + +```typescript +import { CreateDeploymentResponseBody } from "@alienplatform/platform-api/models/operations"; + +let value: CreateDeploymentResponseBody = { + deployment: { + id: "dep_0c29fq4a2yjb7kx3smwdgxlc", + name: "acme-prod", + status: "deleting", + projectId: "prj_mcytp6z3j91f7tn5ryqsfwtr", + platform: "aws", + deploymentProtocolVersion: 108843, + deploymentGroupId: "dg_r27ict8c7vcgsumpj90ackf7b", + stackSettings: {}, + currentReleaseId: "rel_WbhQgksrawSKIpEN0NAssHX9", + desiredReleaseId: "rel_WbhQgksrawSKIpEN0NAssHX9", + pinnedReleaseId: "rel_WbhQgksrawSKIpEN0NAssHX9", + retryRequested: true, + createdAt: new Date("2024-05-26T23:54:19.383Z"), + updatedAt: new Date("2026-10-15T18:20:54.194Z"), + managerId: "mgr_enxscjrqiiu2lrc672hwwuc5", + workspaceId: "ws_It13CUaGEhLLAB87simX0", + }, +}; +``` + +## Fields + +| Field | Type | Required | Description | +| ------------------------------------------------------------------ | ------------------------------------------------------------------ | ------------------------------------------------------------------ | ------------------------------------------------------------------ | +| `deployment` | [models.Deployment](../../models/deployment.md) | :heavy_check_mark: | N/A | +| `token` | *string* | :heavy_minus_sign: | Deployment token (only returned when using deployment group token) | \ No newline at end of file diff --git a/crates/alien-core/Cargo.toml b/crates/alien-core/Cargo.toml index 1e0733e00..fcd7a6d4b 100644 --- a/crates/alien-core/Cargo.toml +++ b/crates/alien-core/Cargo.toml @@ -18,6 +18,7 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } bon = { workspace = true } chrono = { workspace = true, features = ["serde"] } +semver = { workspace = true, features = ["serde"] } indexmap = { workspace = true, features = ["serde"] } clap = { workspace = true, features = ["derive"], optional = true } utoipa = { workspace = true, features = ["indexmap", "chrono"], optional = true } @@ -37,6 +38,7 @@ tracing = { workspace = true } url = { workspace = true, features = ["serde"] } [dev-dependencies] +tempfile = { workspace = true } insta = { workspace = true, features = ["yaml", "json"] } rstest = { workspace = true } axum = { workspace = true, features = ["tokio", "http2"] } diff --git a/crates/alien-core/src/lib.rs b/crates/alien-core/src/lib.rs index 9d0bd646c..6bc3859c9 100644 --- a/crates/alien-core/src/lib.rs +++ b/crates/alien-core/src/lib.rs @@ -77,6 +77,7 @@ pub mod presigned; pub use presigned::*; pub mod embedded_config; +pub mod self_update; pub mod sync; pub mod commands_types; diff --git a/crates/alien-core/src/runtime_environment.rs b/crates/alien-core/src/runtime_environment.rs index e89294ca3..e2e363ddf 100644 --- a/crates/alien-core/src/runtime_environment.rs +++ b/crates/alien-core/src/runtime_environment.rs @@ -268,7 +268,7 @@ pub fn worker_transport_runtime_environment_plan( }], Platform::Kubernetes | Platform::Machines => vec![RuntimeEnvironmentEntry { name: ENV_ALIEN_TRANSPORT, - value: RuntimeEnvironmentValue::Literal("http"), + value: RuntimeEnvironmentValue::Literal("local"), }], Platform::Local | Platform::Test => vec![RuntimeEnvironmentEntry { name: ENV_ALIEN_TRANSPORT, @@ -543,12 +543,12 @@ mod tests { } #[test] - fn kubernetes_worker_environment_uses_http_proxy_transport() { + fn kubernetes_worker_environment_uses_local_transport() { let entries = worker_transport_runtime_environment_plan(Platform::Kubernetes); assert!(entries.iter().any(|entry| { entry.name == ENV_ALIEN_TRANSPORT - && entry.value == RuntimeEnvironmentValue::Literal("http") + && entry.value == RuntimeEnvironmentValue::Literal("local") })); } } diff --git a/crates/alien-core/src/self_update.rs b/crates/alien-core/src/self_update.rs new file mode 100644 index 000000000..df24d112c --- /dev/null +++ b/crates/alien-core/src/self_update.rs @@ -0,0 +1,352 @@ +//! The on-disk handoff protocol between `alien-launcher` and `alien-operator` +//! for the os-service self-update flow. +//! +//! The two binaries coordinate exclusively through the operator's data dir +//! and an exit code: the operator downloads + verifies + stages a new binary, +//! writes `pending.json`, and exits with [`EXIT_CODE_UPDATE_HANDOFF`]; the +//! launcher validates the stage, performs the health-gated swap, and — on a +//! failed probation — rolls back and records the failure in +//! `failed/.json`, which the operator translates into +//! `SyncRequest.operator_update` on its next sync (the launcher has no +//! network path to the manager). +//! +//! These types ARE the protocol: both sides must serialize identically, so +//! they live here in the shared crate rather than in either binary. + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; + +use crate::sync::OperatorUpdatePhase; + +// --------------------------------------------------------------------------- +// Environment + exit-code contract +// --------------------------------------------------------------------------- + +/// Set to `1` by the launcher on spawn; enables the operator's os-service +/// self-update actuator. Kubernetes detection (`KUBERNETES_SERVICE_HOST`) +/// takes precedence even if this is somehow present. +pub const ENV_SELF_UPDATE: &str = "ALIEN_SELF_UPDATE"; +/// The launcher's version, set by the launcher on spawn and reported by the +/// operator on sync (the `min_launcher_version` gate input). The launcher is +/// frozen — reported, never driven. +pub const ENV_LAUNCHER_VERSION: &str = "ALIEN_LAUNCHER_VERSION"; +/// The `127.0.0.1:` address the operator must bind its `/readyz` + +/// `/livez` endpoints to; the launcher probes it during probation. +pub const ENV_HEALTH_ADDR: &str = "ALIEN_HEALTH_ADDR"; + +/// Exit code by which the operator requests an update handoff: it has staged +/// a new version and written `pending.json`; the launcher validates and +/// swaps. `0` = clean stop; anything else = crash (launcher respawns with +/// backoff). +pub const EXIT_CODE_UPDATE_HANDOFF: i32 = 10; + +// --------------------------------------------------------------------------- +// Version +// --------------------------------------------------------------------------- + +/// An operator version — a validated SemVer value. +/// +/// Newtype over `semver::Version` so every comparison in the update flow uses +/// spec-correct SemVer *precedence* (`1.10.0 > 1.9.0`, `1.4.0-rc.1 < 1.4.0`, +/// prerelease segments compared numerically-then-lexically) instead of string +/// ordering. Serializes as the plain version string, which is also the +/// on-disk directory name under `versions/`. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Version(semver::Version); + +impl Version { + /// Parse a SemVer string (e.g. `"1.4.0"`, `"1.5.0-rc.1"`). + pub fn parse(s: &str) -> Result { + semver::Version::parse(s).map(Self) + } + + /// The canonical string form — used on the wire and as the `versions/` + /// directory name. + pub fn as_str(&self) -> String { + self.0.to_string() + } +} + +impl FromStr for Version { + type Err = semver::Error; + + fn from_str(s: &str) -> Result { + Self::parse(s) + } +} + +impl std::fmt::Display for Version { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +// --------------------------------------------------------------------------- +// Marker files +// --------------------------------------------------------------------------- + +/// `pending.json` — written by the OPERATOR after staging a new binary, +/// immediately before exiting with [`EXIT_CODE_UPDATE_HANDOFF`]. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PendingMarker { + /// The staged version (present under `versions//`). + pub version: Version, + /// SHA-256 (lowercase hex) of the staged binary; the launcher re-hashes + /// and validates before swapping. + pub sha256: String, + /// When staging completed. + pub staged_at: DateTime, +} + +/// `failed/.json` — written by the LAUNCHER on a health-gate +/// rollback (or a pre-swap failure). Doubles as the report handoff: the +/// OPERATOR translates the newest record into `SyncRequest.operator_update` +/// on every sync, and applies exponential backoff before re-acting on a +/// matching target. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FailureRecord { + /// The version whose update failed. + pub version: Version, + /// SHA-256 of the artifact that failed — a target with a different digest + /// ignores this record (new artifact = fresh start). + pub sha256: String, + /// Which stage failed, in the sync-wire vocabulary. + pub phase: OperatorUpdatePhase, + /// Human-readable detail (probe timeout, crash exit status, …). + pub message: String, + /// How many attempts for this version have failed so far (1-based). + pub attempts: u32, + /// Completion time of the most recent failed attempt — the backoff clock. + pub last_failed_at: DateTime, +} + +/// Exponential backoff between failed update attempts, keyed on the record's +/// `attempts`: `30 s · 2^(attempts−1)`, capped at 5 min. Mirrors the +/// Kubernetes actuator's Job backoff so both packagings converge at the same +/// pace. +pub fn backoff_delay(attempts: u32) -> Duration { + const BASE_SECS: u64 = 30; + const CAP_SECS: u64 = 300; + let factor = 2u64.saturating_pow(attempts.saturating_sub(1)); + Duration::from_secs(BASE_SECS.saturating_mul(factor).min(CAP_SECS)) +} + +// --------------------------------------------------------------------------- +// Atomic marker I/O (normative: write temp → fsync → rename) +// --------------------------------------------------------------------------- + +/// Atomically write a JSON marker: serialize → write `.tmp` → fsync → +/// rename over `path`. The rename is the commit point; a crash before it +/// leaves only a `.tmp` file, which readers ignore — the protocol then sees +/// the marker as absent, which is always an older, classifiable state. +pub fn write_json_atomic(path: &Path, value: &T) -> std::io::Result<()> { + let tmp = tmp_path(path); + let json = serde_json::to_vec_pretty(value).map_err(std::io::Error::other)?; + + let mut file = std::fs::File::create(&tmp)?; + file.write_all(&json)?; + file.sync_all()?; + drop(file); + + std::fs::rename(&tmp, path) +} + +/// Read a JSON marker. Absent file → `Ok(None)`. A file that exists but does +/// not parse is genuine corruption (atomic writes rule out torn markers) and +/// surfaces as `InvalidData` — never silently treated as absent. +pub fn read_json(path: &Path) -> std::io::Result> { + let bytes = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e), + }; + serde_json::from_slice(&bytes) + .map(Some) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) +} + +/// Remove a marker. Idempotent — an absent marker is success. +pub fn remove_json(path: &Path) -> std::io::Result<()> { + match std::fs::remove_file(path) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +fn tmp_path(path: &Path) -> PathBuf { + let mut name = path + .file_name() + .expect("marker paths always have a file name") + .to_os_string(); + name.push(".tmp"); + path.with_file_name(name) +} + +// --------------------------------------------------------------------------- +// Store paths (shared layout knowledge) +// --------------------------------------------------------------------------- + +/// `pending.json` inside a data dir. +pub fn pending_path(data_dir: &Path) -> PathBuf { + data_dir.join("pending.json") +} + +/// `failed/.json` inside a data dir. +pub fn failure_path(data_dir: &Path, version: &Version) -> PathBuf { + data_dir.join("failed").join(format!("{version}.json")) +} + +/// `failed/` directory inside a data dir. +pub fn failed_dir(data_dir: &Path) -> PathBuf { + data_dir.join("failed") +} + +/// `versions//` inside a data dir. +pub fn version_dir(data_dir: &Path, version: &Version) -> PathBuf { + data_dir.join("versions").join(version.as_str()) +} + +/// `download/` staging area inside a data dir. +pub fn download_dir(data_dir: &Path) -> PathBuf { + data_dir.join("download") +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn version(s: &str) -> Version { + Version::parse(s).expect("test version should parse") + } + + /// `Version` must order by SemVer precedence, not string order. + #[test] + fn version_orders_by_semver_precedence() { + assert!(version("1.10.0") > version("1.9.0"), "numeric, not lexical"); + assert!(version("1.4.0-rc.1") < version("1.4.0"), "prerelease < release"); + assert!(version("1.4.0-rc.2") < version("1.4.0-rc.10"), "numeric prerelease segments"); + assert_eq!(version("1.4.0"), version("1.4.0")); + assert!(Version::parse("not-a-version").is_err()); + assert!(Version::parse("01.2.3").is_err(), "leading zeros rejected"); + } + + /// `Version` serializes as the plain string (wire form + dir name). + #[test] + fn version_serializes_transparent() { + let v = version("1.5.0-rc.1"); + assert_eq!(serde_json::to_value(&v).unwrap(), "1.5.0-rc.1"); + let back: Version = serde_json::from_value(serde_json::json!("1.5.0-rc.1")).unwrap(); + assert_eq!(back, v); + assert_eq!(v.to_string(), "1.5.0-rc.1"); + } + + /// `pending.json` round-trips with exact camelCase keys. + #[test] + fn pending_marker_roundtrip_exact_keys() { + let marker = PendingMarker { + version: version("1.4.0"), + sha256: "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3f9c71a1e4a6f2e0e6d5c4b3a" + .to_string(), + staged_at: "2026-07-08T12:00:00Z".parse().unwrap(), + }; + let json = serde_json::to_value(&marker).unwrap(); + assert_eq!(json["version"], "1.4.0"); + assert_eq!(json["stagedAt"], "2026-07-08T12:00:00Z"); + let mut keys: Vec<&str> = json.as_object().unwrap().keys().map(String::as_str).collect(); + keys.sort_unstable(); + assert_eq!(keys, ["sha256", "stagedAt", "version"], "exact key set"); + let back: PendingMarker = serde_json::from_value(json).unwrap(); + assert_eq!(back, marker); + } + + /// `failed/.json` round-trips with exact camelCase keys and the + /// sync-wire kebab-case phase, so the operator can translate the record + /// into `SyncRequest.operator_update` without re-mapping. + #[test] + fn failure_record_roundtrip_exact_keys() { + let record = FailureRecord { + version: version("1.4.0"), + sha256: "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3f9c71a1e4a6f2e0e6d5c4b3a" + .to_string(), + phase: OperatorUpdatePhase::Apply, + message: "readyz never returned 200 within the probation window".to_string(), + attempts: 3, + last_failed_at: "2026-07-08T12:05:00Z".parse().unwrap(), + }; + let json = serde_json::to_value(&record).unwrap(); + assert_eq!(json["version"], "1.4.0"); + assert_eq!(json["phase"], "apply", "kebab-case wire vocabulary"); + assert_eq!(json["attempts"], 3); + assert_eq!(json["lastFailedAt"], "2026-07-08T12:05:00Z"); + let back: FailureRecord = serde_json::from_value(json).unwrap(); + assert_eq!(back, record); + } + + /// Backoff: 30s · 2^(n−1), capped at 5 min, mirroring the K8s actuator. + #[test] + fn backoff_delay_doubles_and_caps() { + assert_eq!(backoff_delay(1), Duration::from_secs(30)); + assert_eq!(backoff_delay(2), Duration::from_secs(60)); + assert_eq!(backoff_delay(3), Duration::from_secs(120)); + assert_eq!(backoff_delay(4), Duration::from_secs(240)); + assert_eq!(backoff_delay(5), Duration::from_secs(300), "capped"); + assert_eq!(backoff_delay(50), Duration::from_secs(300), "no overflow"); + assert_eq!(backoff_delay(0), Duration::from_secs(30), "0 treated as first"); + } + + /// Atomic write leaves no temp residue; absent reads None; corruption is + /// loud; removal is idempotent. + #[test] + fn marker_io_contract() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + + let marker = PendingMarker { + version: version("1.4.0"), + sha256: "aa".repeat(32), + staged_at: Utc::now(), + }; + write_json_atomic(&path, &marker).unwrap(); + let back: Option = read_json(&path).unwrap(); + assert_eq!(back, Some(marker)); + assert!( + !dir.path().join("pending.json.tmp").exists(), + "no temp residue after commit" + ); + + std::fs::write(&path, b"{ not json").unwrap(); + let err = read_json::(&path).expect_err("corrupt marker must error"); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + + remove_json(&path).unwrap(); + remove_json(&path).expect("removing an absent marker is Ok"); + assert_eq!(read_json::(&path).unwrap(), None); + } + + /// The shared path layout matches the version-store layout. + #[test] + fn store_paths_layout() { + let data = Path::new("/var/lib/alien-operator"); + assert_eq!(pending_path(data), data.join("pending.json")); + assert_eq!( + failure_path(data, &version("1.4.0")), + data.join("failed/1.4.0.json") + ); + assert_eq!(version_dir(data, &version("1.4.0")), data.join("versions/1.4.0")); + assert_eq!(download_dir(data), data.join("download")); + } +} diff --git a/crates/alien-core/src/sync.rs b/crates/alien-core/src/sync.rs index 9a24ca14f..123a919d6 100644 --- a/crates/alien-core/src/sync.rs +++ b/crates/alien-core/src/sync.rs @@ -1,7 +1,8 @@ -//! Sync protocol types for agent ↔ manager communication. +//! Sync protocol types for operator ↔ manager communication. //! -//! The agent periodically calls `POST /v1/sync` with a `SyncRequest` and -//! receives a `SyncResponse` containing the target deployment state. +//! The operator periodically calls `POST /v1/sync` with a `SyncRequest` and +//! receives a `SyncResponse` containing the target deployment state (and, when a +//! self-update is pinned, an `operator_target`). use serde::{Deserialize, Serialize}; @@ -36,13 +37,13 @@ pub struct OperatorCapabilityReport { pub detail: Option, } -/// Request sent by the agent to the manager during periodic sync. +/// Request sent by the operator to the manager during periodic sync. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SyncRequest { - /// The deployment ID this agent is managing. + /// The deployment ID this operator is managing. pub deployment_id: String, - /// Current deployment state as seen by the agent. + /// Current deployment state as seen by the operator. #[serde(skip_serializing_if = "Option::is_none")] pub current_state: Option, /// Managed Alien resource status samples emitted by the Operator's deployment step. @@ -62,34 +63,300 @@ pub struct SyncRequest { /// Report-only capabilities observed by the Operator. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub capabilities: Vec, - /// Version of the Operator binary reporting this sync. + /// Version of the Operator binary reporting this sync (from + /// `env!("CARGO_PKG_VERSION")` at build time). Lets the manager build + /// fleet-wide version inventory and decide whether to send an + /// `operator_target`. Optional for back-compat with older operators. #[serde(default, skip_serializing_if = "Option::is_none")] pub operator_version: Option, + /// Version of the `alien-launcher` supervising this operator (os-service + /// packaging only; sourced from `ALIEN_LAUNCHER_VERSION`, which the launcher + /// sets on spawn). Reported, never driven — the launcher is frozen and only + /// changes via a state-preserving redeploy. The manager compares it against + /// `OperatorBinaryTarget::min_launcher_version` and withholds targets the + /// installed launcher is too old to actuate. None on Kubernetes and for + /// operators not run under a launcher. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub launcher_version: Option, + /// Host OS the operator runs on. From `std::env::consts::OS`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_os: Option, + /// Host CPU architecture the operator runs on. From `std::env::consts::ARCH`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_arch: Option, + /// How the operator is supervised — `os-service` (launcher) or `kubernetes` + /// (Helm). Detected at runtime from `KUBERNETES_SERVICE_HOST`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub packaging: Option, + /// Container image repository the operator was pulled from (without the + /// tag), e.g. `ghcr.io/alien-dev/alien-operator`. The chart injects this via + /// `ALIEN_OPERATOR_IMAGE_REPOSITORY` (= `.Values.runtime.image.repository`), + /// so admins can see the supply-chain link before pinning a new tag. + /// Optional and Kubernetes-only — the os-service packaging fills the same role + /// with its launcher manifest URL. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_image_repository: Option, + /// Outcome of the operator's in-flight self-update for the currently-pinned + /// target, if any. Absent when no update is in flight. Lets the manager + /// distinguish "still converging" from "the last attempt failed" instead of + /// inferring failure from a stalled version. Optional for back-compat. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_update: Option, +} + +/// Supervisor packaging for an operator. Drives which `operator_target` payload +/// (`binary` vs `helm`) the manager sends and how the operator actuates the +/// upgrade. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum OperatorPackaging { + /// Native OS service — the launcher swaps the binary on disk. + OsService, + /// Kubernetes pod — operator creates a Helm-runner Job that runs `helm upgrade --atomic`. + Kubernetes, +} + +/// Host operating system an operator runs on. +/// +/// Serialized values match `std::env::consts::OS`, so the manager can line the +/// operator up against the binaries it builds. Unsupported OSes are reported as +/// `None` (the operator can't self-update to a binary that doesn't exist) rather +/// than as a string the manager can't act on. Prefer this over +/// `instance_catalog::Architecture` / `build_targets::BinaryTarget`: those model +/// buildable *targets* (and spell arm as `arm64`), not the OS the operator +/// happens to run on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum OperatorOs { + Linux, + Macos, + Windows, +} + +impl OperatorOs { + /// Detect the current host OS. `None` for OSes we don't ship binaries for. + pub fn detect() -> Option { + Self::from_consts(std::env::consts::OS) + } + + /// Map a `std::env::consts::OS` value to a supported OS. + pub fn from_consts(os: &str) -> Option { + match os { + "linux" => Some(Self::Linux), + "macos" => Some(Self::Macos), + "windows" => Some(Self::Windows), + _ => None, + } + } + + /// Wire/string form (matches `std::env::consts::OS`). + pub fn as_str(self) -> &'static str { + match self { + Self::Linux => "linux", + Self::Macos => "macos", + Self::Windows => "windows", + } + } +} + +/// Host CPU architecture an operator runs on. +/// +/// Serialized values match `std::env::consts::ARCH` (`x86_64` / `aarch64`) — +/// deliberately not the `arm64` spelling `instance_catalog::Architecture` uses, +/// so it round-trips exactly what the operator reports. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum OperatorArch { + #[serde(rename = "x86_64")] + X86_64, + #[serde(rename = "aarch64")] + Aarch64, +} + +impl OperatorArch { + /// Detect the current host arch. `None` for arches we don't ship binaries for. + pub fn detect() -> Option { + Self::from_consts(std::env::consts::ARCH) + } + + /// Map a `std::env::consts::ARCH` value to a supported architecture. + pub fn from_consts(arch: &str) -> Option { + match arch { + "x86_64" => Some(Self::X86_64), + "aarch64" => Some(Self::Aarch64), + _ => None, + } + } + + /// Wire/string form (matches `std::env::consts::ARCH`). + pub fn as_str(self) -> &'static str { + match self { + Self::X86_64 => "x86_64", + Self::Aarch64 => "aarch64", + } + } } -/// Response from the manager to the agent sync request. +/// Operator-reported state of the current self-update attempt. +/// +/// Success is deliberately not a variant — convergence +/// (`operator_version == target_operator_version`) is the success signal. This +/// report only distinguishes "an attempt is running" from "the last attempt +/// failed", so the manager can surface a truthful failure instead of inferring +/// one from a stalled version. Internally tagged by `state`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "camelCase")] +pub enum OperatorUpdateReport { + /// A Job for `target_version` is currently running (attempt `attempt`, + /// 1-based). Reported so the dashboard can show "running" from the operator's + /// own view rather than only "pinned". + #[serde(rename_all = "camelCase")] + InProgress { + target_version: String, + attempt: u32, + }, + /// The most recent attempt for `target_version` failed; the operator is still + /// on its prior version (rolled back / never swapped) and will back off and + /// retry. Whether the *episode* is terminal is decided manager-side. + #[serde(rename_all = "camelCase")] + Failed { + target_version: String, + /// Which stage failed — see `OperatorUpdatePhase`. + phase: OperatorUpdatePhase, + /// Human-readable detail: helm error, image-pull reason, k8s API error. + message: String, + attempt: u32, + }, +} + +/// Which stage of a self-update attempt failed. Maps to operator triage: +/// `Pull` → the image tag/registry; `Apply` → the chart/values/cluster. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum OperatorUpdatePhase { + /// Creating the Helm-runner Job / staging the binary failed — the operator + /// knows directly (the k8s API rejected the create, or a download failed). + Spawn, + /// The new pod's image could not be pulled (ImagePullBackOff / ErrImagePull + /// / not-found). Read from the Job pod's container `waiting.reason`. + Pull, + /// The image pulled, but `helm upgrade` failed or `--atomic` rolled back + /// (or, on os-service, the launcher's health-gated swap rolled back). + Apply, +} + +/// Response from the manager to the operator sync request. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SyncResponse { /// Authoritative deployment state from the manager. /// - /// Pull agents use this to hydrate local state when attaching to an - /// already-imported deployment. Absent means the agent's local state is + /// Pull operators use this to hydrate local state when attaching to an + /// already-imported deployment. Absent means the operator's local state is /// already authoritative or no state has been established yet. #[serde(default, skip_serializing_if = "Option::is_none")] pub current_state: Option, - /// Target deployment the agent should converge toward. + /// Target deployment the operator should converge toward. /// None means no changes needed or this is an observe-only deployment. #[serde(skip_serializing_if = "Option::is_none")] pub target: Option, /// Public URL for the commands API (e.g. `https://manager.example.com/v1`). /// Cloud-deployed workers use this to poll for pending commands. - /// When absent, the agent falls back to its sync URL. + /// When absent, the operator falls back to its sync URL. #[serde(default, skip_serializing_if = "Option::is_none")] pub commands_url: Option, + /// Desired operator self-update target. The operator acts on whichever + /// payload matches its packaging: `binary` for `os-service`, `helm` for + /// `kubernetes`. None means no upgrade pending. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_target: Option, +} + +/// Desired operator upgrade payload, sent by the manager on the sync exchange. +/// +/// The operator reads `binary` or `helm` depending on its packaging. The manager is +/// the single source of truth for the target version per deployment per channel, +/// and on Kubernetes also for the full desired values. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OperatorTarget { + /// Target operator version (e.g. "1.4.0"). + pub version: String, + /// The operator should refuse the upgrade if its own version is older than + /// this — used by the manager to enforce a floor on incremental migrations. + pub min_supported_version: String, + /// OS-service actuation payload. Present iff the deployment's packaging is `os-service`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub binary: Option, + /// Kubernetes actuation payload. Present iff the deployment's packaging is `kubernetes`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub helm: Option, +} + +/// OS-service binary upgrade payload — the operator downloads, verifies the +/// SHA-256, stages the binary, and exits; the launcher performs the +/// health-gated swap. +/// +/// The manager resolves the artifact for THIS host's `(os, arch)` — both are +/// known from the `SyncRequest` — and sends exactly one url + sha256 + +/// signature. (An earlier draft carried an artifacts map with a single sha256; +/// that was unsound — one digest cannot cover N different binaries — and +/// unnecessary, since the manager already knows the host.) +/// +/// The `signature` field is **future work**: it rides along on the wire so +/// newer operators can enforce it once the signing infrastructure lands, but +/// the current launcher trusts SHA-256 + HTTPS for the download. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OperatorBinaryTarget { + /// Download URL for this host's `(os, arch)` — resolved by the manager + /// from the host's reported `operator_os` / `operator_arch`. + pub url: String, + /// SHA-256 digest of exactly that artifact, lowercase hex. + pub sha256: String, + /// ed25519 detached signature over that artifact, base64-encoded. + /// **Future:** verified against the launcher's pinned public key before + /// the binary is exec'd. Not enforced in the current iteration. + pub signature: String, + /// The installed (frozen) launcher must be >= this version, or the manager + /// withholds the target and surfaces "redeploy required" instead. The + /// launcher never self-updates; it is only replaced by a state-preserving + /// redeploy. + pub min_launcher_version: String, +} + +/// Kubernetes upgrade payload — the operator writes the full values to a +/// ConfigMap (sensitive values to a Secret) and creates a Helm-runner Job that +/// runs `helm upgrade --atomic`. Helm's revision-scoped rollback covers both the +/// image and the values. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OperatorHelmTarget { + /// OCI registry reference for the chart (e.g. `oci://ghcr.io/alienplatform/alien-operator`). + pub chart_repo: String, + /// Chart version (e.g. `"1.4.0"`). + pub chart_version: String, + /// Full desired values document (manager-owned). Stored verbatim by Helm in + /// the release Secret, so MUST NOT contain raw secrets — those go via + /// `sensitive_values` as references. + pub values: serde_json::Value, + /// Sensitive values, expressed as references to existing Kubernetes Secrets + /// in the namespace. Keyed by JSON-Pointer path into `values`. The operator + /// materializes these into a Secret the chart mounts via + /// `valueFrom: secretKeyRef:`, so the material never reaches the release + /// history Secret base64-encoded. + #[serde(default)] + pub sensitive_values: std::collections::BTreeMap, } -/// Target deployment state for the agent to converge toward. +/// Reference to a Kubernetes Secret key holding sensitive data. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SecretRef { + pub name: String, + pub key: String, +} + +/// Target deployment state for the operator to converge toward. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TargetDeployment { @@ -103,20 +370,37 @@ pub struct TargetDeployment { mod tests { use super::*; - #[test] - fn test_sync_request_serialization() { - let req = SyncRequest { + fn empty_sync_request() -> SyncRequest { + SyncRequest { deployment_id: "dep_abc123".to_string(), current_state: None, heartbeats: Vec::new(), observed_inventory_batches: Vec::new(), capabilities: Vec::new(), operator_version: None, - }; + launcher_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + operator_update: None, + } + } + + fn empty_sync_response() -> SyncResponse { + SyncResponse { + current_state: None, + target: None, + commands_url: None, + operator_target: None, + } + } + #[test] + fn test_sync_request_serialization() { + let req = empty_sync_request(); let json = serde_json::to_value(&req).unwrap(); assert_eq!(json["deploymentId"], "dep_abc123"); - // current_state is None → should be omitted assert!(json.get("currentState").is_none()); assert!(json.get("resourceHeartbeats").is_none()); assert!(json.get("capabilities").is_none()); @@ -137,24 +421,15 @@ mod tests { #[test] fn test_sync_response_empty() { - let resp = SyncResponse { - current_state: None, - target: None, - commands_url: None, - }; + let resp = empty_sync_response(); let json = serde_json::to_value(&resp).unwrap(); - // target is None → should be omitted assert!(json.get("target").is_none()); assert!(json.get("currentState").is_none()); } #[test] fn test_sync_response_roundtrip_no_target() { - let resp = SyncResponse { - current_state: None, - target: None, - commands_url: None, - }; + let resp = empty_sync_response(); let serialized = serde_json::to_string(&resp).unwrap(); let deserialized: SyncResponse = serde_json::from_str(&serialized).unwrap(); assert!(deserialized.target.is_none()); @@ -163,7 +438,6 @@ mod tests { #[test] fn test_sync_request_with_camel_case() { - // Verify camelCase renaming works correctly let json = r#"{"deploymentId": "dep_1", "currentState": null}"#; let req: SyncRequest = serde_json::from_str(json).unwrap(); assert_eq!(req.deployment_id, "dep_1"); @@ -176,83 +450,212 @@ mod tests { assert!(serde_json::from_str::(json).is_err()); } + // --- operator self-update wire format tests -------------------------- + + /// An operator that fills in the self-update fields produces JSON the + /// manager can deserialize, with the expected camelCase + kebab-case values. #[test] - fn test_sync_request_heartbeats_roundtrip() { - let json = serde_json::json!({ - "deploymentId": "dep_1", - "resourceHeartbeats": [{ - "deploymentId": "dep_1", - "resourceId": "api", - "resourceType": "container", - "controllerPlatform": "kubernetes", - "backend": "kubernetes", - "observedAt": "2026-01-01T00:00:00Z", - "data": { - "resourceType": "container", - "data": { - "backend": "kubernetes", - "status": { - "health": "healthy", - "lifecycle": "running", - "message": null, - "stale": false, - "partial": false, - "collectionIssues": [] - }, - "namespace": "default", - "name": "api", - "workloadKind": "deployment", - "replicas": { "desired": 1, "current": 1, "ready": 1, "available": 1, "updated": null, "misscheduled": null }, - "restarts": 0, - "cpu": null, - "memory": null, - "workload": null, - "pods": [], - "instances": [], - "events": [] - } - }, - "raw": [] - }] - }); - - let req: SyncRequest = serde_json::from_value(json).unwrap(); - assert_eq!(req.heartbeats.len(), 1); - assert_eq!(req.heartbeats[0].resource_id, "api"); - assert!(req.capabilities.is_empty()); + fn test_sync_request_with_self_update_fields_roundtrip() { + let mut req = empty_sync_request(); + req.operator_version = Some("1.3.5".to_string()); + req.launcher_version = Some("0.1.0".to_string()); + req.operator_os = Some(OperatorOs::Linux); + req.operator_arch = Some(OperatorArch::Aarch64); + req.packaging = Some(OperatorPackaging::Kubernetes); + req.operator_image_repository = Some("ghcr.io/alien-dev/alien-operator".to_string()); + let json = serde_json::to_value(&req).unwrap(); + assert_eq!(json["operatorVersion"], "1.3.5"); + assert_eq!(json["launcherVersion"], "0.1.0"); + assert_eq!(json["operatorOs"], "linux"); + assert_eq!(json["operatorArch"], "aarch64"); + assert_eq!(json["packaging"], "kubernetes"); // kebab-case enum + assert_eq!( + json["operatorImageRepository"], + "ghcr.io/alien-dev/alien-operator" + ); + let back: SyncRequest = serde_json::from_value(json).unwrap(); + assert_eq!(back.operator_version.as_deref(), Some("1.3.5")); + assert_eq!(back.launcher_version.as_deref(), Some("0.1.0")); + assert_eq!(back.packaging, Some(OperatorPackaging::Kubernetes)); + assert_eq!(back.operator_os, Some(OperatorOs::Linux)); + assert_eq!(back.operator_arch, Some(OperatorArch::Aarch64)); + } + + /// The os/arch enums must serialize to the exact `std::env::consts` spellings + /// the operator reports — in particular `aarch64` (not the `arm64` that + /// `instance_catalog::Architecture` uses), which is the reason they're + /// dedicated enums rather than a reuse of the catalog one. + #[test] + fn test_operator_os_arch_wire_values() { + assert_eq!(serde_json::to_value(OperatorOs::Linux).unwrap(), "linux"); + assert_eq!(serde_json::to_value(OperatorOs::Macos).unwrap(), "macos"); + assert_eq!(serde_json::to_value(OperatorOs::Windows).unwrap(), "windows"); + assert_eq!(serde_json::to_value(OperatorArch::X86_64).unwrap(), "x86_64"); + assert_eq!(serde_json::to_value(OperatorArch::Aarch64).unwrap(), "aarch64"); - let serialized = serde_json::to_value(&req).unwrap(); - assert_eq!(serialized["resourceHeartbeats"][0]["resourceId"], "api"); + // from_consts round-trips std::env::consts values; unknowns are dropped. + assert_eq!(OperatorOs::from_consts("macos"), Some(OperatorOs::Macos)); + assert_eq!(OperatorOs::from_consts("freebsd"), None); + assert_eq!(OperatorArch::from_consts("aarch64"), Some(OperatorArch::Aarch64)); + assert_eq!(OperatorArch::from_consts("arm"), None); + assert_eq!(OperatorOs::Macos.as_str(), "macos"); + assert_eq!(OperatorArch::Aarch64.as_str(), "aarch64"); } + /// Backward compat: an old operator omits the self-update fields; the + /// manager deserializes them as None/empty. #[test] - fn test_sync_response_observe_only_state_roundtrip() { - let state = DeploymentState { - status: crate::DeploymentStatus::Running, - platform: crate::Platform::Kubernetes, - current_release: None, - target_release: None, - stack_state: None, - error: None, - environment_info: None, - runtime_metadata: None, - retry_requested: false, - protocol_version: crate::DEPLOYMENT_PROTOCOL_VERSION, + fn test_sync_request_old_operator_no_self_update_fields() { + let json = r#"{"deploymentId": "dep_old", "currentState": null}"#; + let req: SyncRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.deployment_id, "dep_old"); + assert!(req.operator_version.is_none()); + assert!(req.launcher_version.is_none()); + assert!(req.operator_os.is_none()); + assert!(req.operator_arch.is_none()); + assert!(req.packaging.is_none()); + assert!(req.operator_update.is_none()); + } + + /// `launcherVersion` is omitted from the JSON when unset (Kubernetes and + /// launcher-less operators), so old managers see no unknown-field noise. + #[test] + fn test_sync_request_launcher_version_omitted_when_none() { + let req = empty_sync_request(); + let json = serde_json::to_value(&req).unwrap(); + assert!(json.get("launcherVersion").is_none()); + } + + /// The SyncResponse's `operatorTarget` is omitted when None so an old + /// operator still gets a clean response. + #[test] + fn test_sync_response_old_client_no_operator_target() { + let resp = empty_sync_response(); + let json = serde_json::to_value(&resp).unwrap(); + assert!(json.get("operatorTarget").is_none()); + } + + #[test] + fn test_operator_packaging_kebab_case() { + assert_eq!( + serde_json::to_value(OperatorPackaging::OsService).unwrap(), + serde_json::json!("os-service") + ); + assert_eq!( + serde_json::to_value(OperatorPackaging::Kubernetes).unwrap(), + serde_json::json!("kubernetes") + ); + let r: OperatorPackaging = serde_json::from_str("\"os-service\"").unwrap(); + assert_eq!(r, OperatorPackaging::OsService); + } + + /// `OperatorUpdateReport` round-trips: internally tagged by `state` + /// (camelCase variant), camelCase fields, kebab-case phase. + #[test] + fn test_operator_update_report_roundtrip() { + let in_progress = OperatorUpdateReport::InProgress { + target_version: "1.4.0".to_string(), + attempt: 1, }; - assert!(!state.has_desired()); + let json = serde_json::to_value(&in_progress).unwrap(); + assert_eq!(json["state"], "inProgress"); + assert_eq!(json["targetVersion"], "1.4.0"); + assert_eq!(json["attempt"], 1); + assert_eq!( + serde_json::from_value::(json).unwrap(), + in_progress + ); - let resp = SyncResponse { - current_state: Some(state), - target: None, - commands_url: None, + let failed = OperatorUpdateReport::Failed { + target_version: "1.4.0".to_string(), + phase: OperatorUpdatePhase::Pull, + message: "image :1.4.0 not found".to_string(), + attempt: 3, }; + let json = serde_json::to_value(&failed).unwrap(); + assert_eq!(json["state"], "failed"); + assert_eq!(json["phase"], "pull"); // kebab-case + assert_eq!(json["targetVersion"], "1.4.0"); + assert_eq!(json["message"], "image :1.4.0 not found"); + assert_eq!(json["attempt"], 3); + assert_eq!( + serde_json::from_value::(json).unwrap(), + failed + ); + } - let serialized = serde_json::to_string(&resp).unwrap(); - let deserialized: SyncResponse = serde_json::from_str(&serialized).unwrap(); - let current_state = deserialized.current_state.unwrap(); + /// `OperatorBinaryTarget` is per-host resolved: exactly one url + sha256 + + /// signature + minLauncherVersion, all camelCase. (The earlier artifacts-map + /// shape with a single sha256 was unsound — one digest cannot cover N + /// binaries — and must not reappear.) + #[test] + fn test_operator_binary_target_roundtrip() { + let binary = OperatorBinaryTarget { + url: "https://example.com/releases/v1.4.0/alien-operator-1.4.0-linux-x86_64" + .to_string(), + sha256: "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3f9c71a1e4a6f2e0e6d5c4b3a" + .to_string(), + signature: "c2lnbmF0dXJlLXBsYWNlaG9sZGVy".to_string(), + min_launcher_version: "0.1.0".to_string(), + }; + let target = OperatorTarget { + version: "1.4.0".to_string(), + min_supported_version: "1.0.0".to_string(), + binary: Some(binary), + helm: None, + }; + let json = serde_json::to_value(&target).unwrap(); + assert!(json.get("helm").is_none(), "helm must be omitted"); + let b = &json["binary"]; + assert_eq!( + b["url"], + "https://example.com/releases/v1.4.0/alien-operator-1.4.0-linux-x86_64" + ); + assert_eq!( + b["sha256"], + "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3f9c71a1e4a6f2e0e6d5c4b3a" + ); + assert_eq!(b["signature"], "c2lnbmF0dXJlLXBsYWNlaG9sZGVy"); + assert_eq!(b["minLauncherVersion"], "0.1.0"); + assert!( + b.get("artifacts").is_none(), + "the artifacts map shape must not reappear on the wire" + ); + let back: OperatorTarget = serde_json::from_value(json).unwrap(); + let back_binary = back.binary.expect("binary should round-trip"); + assert_eq!(back_binary.min_launcher_version, "0.1.0"); + assert_eq!(back_binary.sha256.len(), 64); + assert!(back.helm.is_none()); + } - assert_eq!(current_state.status, crate::DeploymentStatus::Running); - assert!(!current_state.has_desired()); - assert!(deserialized.target.is_none()); + #[test] + fn test_operator_target_helm_roundtrip() { + let mut sensitive = std::collections::BTreeMap::new(); + sensitive.insert( + "/management/token".to_string(), + SecretRef { + name: "alien-operator".to_string(), + key: "sync-token".to_string(), + }, + ); + let helm = OperatorHelmTarget { + chart_repo: "oci://ghcr.io/alienplatform/alien-operator".to_string(), + chart_version: "1.4.0".to_string(), + values: serde_json::json!({"runtime": {"image": {"tag": "1.4.0"}}}), + sensitive_values: sensitive, + }; + let target = OperatorTarget { + version: "1.4.0".to_string(), + min_supported_version: "1.3.0".to_string(), + binary: None, + helm: Some(helm), + }; + let json = serde_json::to_value(&target).unwrap(); + assert_eq!(json["minSupportedVersion"], "1.3.0"); + assert!(json.get("binary").is_none(), "binary must be omitted"); + assert_eq!(json["helm"]["chartVersion"], "1.4.0"); + let back: OperatorTarget = serde_json::from_value(json).unwrap(); + assert!(back.binary.is_none()); + assert_eq!(back.helm.unwrap().chart_version, "1.4.0"); } } diff --git a/crates/alien-deploy-cli/Cargo.toml b/crates/alien-deploy-cli/Cargo.toml index 6399ae078..40e92e96d 100644 --- a/crates/alien-deploy-cli/Cargo.toml +++ b/crates/alien-deploy-cli/Cargo.toml @@ -67,3 +67,5 @@ libc = "0.2" [target.'cfg(windows)'.dependencies] fs2 = "0.4" +# Directory-junction pointers for the launcher layout's current/last-stable. +junction = { workspace = true } diff --git a/crates/alien-deploy-cli/src/commands/install_launcher.rs b/crates/alien-deploy-cli/src/commands/install_launcher.rs new file mode 100644 index 000000000..39ab806cf --- /dev/null +++ b/crates/alien-deploy-cli/src/commands/install_launcher.rs @@ -0,0 +1,740 @@ +//! Launcher-based install layout for the operator OS service. +//! +//! The service the init system runs is `alien-launcher`; the launcher spawns +//! the operator from the version store and performs health-gated binary +//! swaps with last-stable rollback. The launcher itself is FROZEN — it never +//! rewrites its own binary — so re-running this installer is the one and only +//! way it changes ("redeploy"), and the install must therefore be +//! **idempotent and state-preserving**: `state/` and the secret files are +//! never recreated or overwritten (wiping them would re-initialize the +//! deployment identity and orphan the deployment on the manager), while the +//! binaries and the unit file are always refreshed. +//! +//! On-disk layout (under the data dir): +//! +//! ```text +//! launcher/alien-launcher # the supervisor; replaced on redeploy only +//! versions//alien-operator # installed operator version(s) +//! current -> versions/ # desired version (symlink) +//! last-stable -> versions/ # proven-good fallback (symlink) +//! state/ # encrypted DB — NEVER touched here +//! sync-token, encryption-key # secrets — written once, then reused +//! ``` +//! +//! The operator's configuration flows through the unit's `Environment=` +//! lines: every operator flag has an env alias (SYNC_URL, SYNC_TOKEN_FILE, +//! DATA_DIR, …), and the launcher's child inherits the launcher's +//! environment, so no argv plumbing is needed through the supervisor. + +#[cfg(unix)] +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::process::Command; + +#[cfg(windows)] +use std::ffi::OsString; + +use alien_error::{AlienError, Context, IntoAlienError}; +#[cfg(windows)] +use service_manager::{ + RestartPolicy, ServiceInstallCtx, ServiceLabel, ServiceManager, ServiceStartCtx, ServiceStopCtx, +}; + +use crate::error::{ErrorData, Result}; +use crate::output; + +/// Shorthand for the operator-service error variant this module uses +/// throughout. +fn service_error(message: String) -> ErrorData { + ErrorData::OperatorServiceError { message } +} + +/// The launcher layout is the default on Linux and Windows (the OSes whose +/// service shim is wired); macOS keeps the legacy direct-operator service until +/// its launcher phase lands. `--no-launcher` forces legacy anywhere. +pub fn use_launcher_layout(no_launcher: bool) -> bool { + use_launcher_layout_for(no_launcher, std::env::consts::OS) +} + +fn use_launcher_layout_for(no_launcher: bool, os: &str) -> bool { + !no_launcher && matches!(os, "linux" | "windows") +} + +/// The operator binary filename inside `versions//` — `.exe` on Windows. +fn operator_binary_name() -> String { + format!("alien-operator{}", std::env::consts::EXE_SUFFIX) +} + +/// The launcher binary filename inside `launcher/` — `.exe` on Windows. +fn launcher_binary_name() -> String { + format!("alien-launcher{}", std::env::consts::EXE_SUFFIX) +} + +/// Locate the launcher binary: explicit flag → `ALIEN_LAUNCHER_BINARY` env → +/// next to the operator binary → on PATH. +pub fn which_launcher_binary( + explicit: Option, + operator_binary: &Path, +) -> Result { + if let Some(path) = explicit { + if path.is_file() { + return Ok(path); + } + return Err(AlienError::new(service_error(format!( + "--launcher-binary '{}' does not exist", + path.display() + )))); + } + if let Ok(env_path) = std::env::var("ALIEN_LAUNCHER_BINARY") { + let path = PathBuf::from(&env_path); + if path.is_file() { + return Ok(path); + } + return Err(AlienError::new(service_error(format!( + "ALIEN_LAUNCHER_BINARY is set to '{env_path}' but the file does not exist" + )))); + } + if let Some(sibling) = operator_binary + .parent() + .map(|dir| dir.join(launcher_binary_name())) + { + if sibling.is_file() { + return Ok(sibling); + } + } + which::which("alien-launcher").into_alien_error().context( + service_error( + "alien-launcher binary not found. Pass --launcher-binary, set \ + ALIEN_LAUNCHER_BINARY, or place it next to the operator binary" + .to_string(), + ), + ) +} + +/// Ask a binary for its version (` --version` → last token). +pub fn binary_version(binary: &Path) -> Result { + let output = Command::new(binary) + .arg("--version") + .output() + .into_alien_error() + .context(service_error(format!( + "failed to run '{} --version'", + binary.display() + )))?; + let stdout = String::from_utf8_lossy(&output.stdout); + parse_version_output(&stdout).ok_or_else(|| { + AlienError::new(service_error(format!( + "could not parse a version from '{} --version' output: '{}'", + binary.display(), + stdout.trim() + ))) + }) +} + +fn parse_version_output(stdout: &str) -> Option { + let token = stdout.split_whitespace().last()?; + // Sanity: semver-ish (the store directory name must parse launcher-side). + if token.split('.').count() >= 3 || token.split('.').count() == 3 { + Some(token.to_string()) + } else { + None + } +} + +/// Write (or refresh) the launcher layout under `data_dir`. +/// +/// Normative idempotency rules: +/// - `state/` is created if missing and otherwise NEVER touched; +/// - binaries are ALWAYS refreshed (copy to a temp name + rename, so a +/// still-running old binary keeps its inode — no ETXTBSY, no torn write); +/// - the `current`/`last-stable` pointers are created only when absent: on a +/// redeploy over a live store they are the launcher's truth, not ours. +pub fn write_layout( + data_dir: &Path, + operator_binary: &Path, + operator_version: &str, + launcher_binary: &Path, +) -> Result<()> { + for dir in ["versions", "state", "state-snapshots", "failed", "download", "launcher"] { + std::fs::create_dir_all(data_dir.join(dir)) + .into_alien_error() + .context(service_error(format!( + "failed to create '{}'", + data_dir.join(dir).display() + )))?; + } + + let version_dir = data_dir.join("versions").join(operator_version); + std::fs::create_dir_all(&version_dir) + .into_alien_error() + .context(service_error(format!( + "failed to create '{}'", + version_dir.display() + )))?; + install_binary(operator_binary, &version_dir.join(operator_binary_name()))?; + install_binary( + launcher_binary, + &data_dir.join("launcher").join(launcher_binary_name()), + )?; + + let target = Path::new("versions").join(operator_version); + ensure_pointer(data_dir, "current", &target)?; + ensure_pointer(data_dir, "last-stable", &target)?; + Ok(()) +} + +/// Copy a binary into place via temp + rename; always refreshes; preserves a +/// running old inode. +fn install_binary(from: &Path, to: &Path) -> Result<()> { + let tmp = to.with_extension("tmp"); + std::fs::copy(from, &tmp) + .into_alien_error() + .context(service_error(format!( + "failed to copy '{}' to '{}'", + from.display(), + tmp.display() + )))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o755)) + .into_alien_error() + .context(service_error(format!( + "failed to mark '{}' executable", + tmp.display() + )))?; + } + std::fs::rename(&tmp, to) + .into_alien_error() + .context(service_error(format!( + "failed to move '{}' into place", + to.display() + ))) +} + +/// Create a pointer only if it does not exist yet — a symlink on Unix, a +/// directory junction on Windows (which needs an absolute target, so the +/// relative `target` is rooted at `data_dir`). Matches the launcher's store, +/// which reads the version from the pointer target's final component. +fn ensure_pointer(data_dir: &Path, name: &str, target: &Path) -> Result<()> { + let path = data_dir.join(name); + if path.symlink_metadata().is_ok() { + return Ok(()); + } + #[cfg(unix)] + { + std::os::unix::fs::symlink(target, &path) + .into_alien_error() + .context(service_error(format!( + "failed to create the '{name}' pointer" + )))?; + } + #[cfg(windows)] + { + junction::create(data_dir.join(target), &path) + .into_alien_error() + .context(service_error(format!( + "failed to create the '{name}' junction" + )))?; + } + #[cfg(not(any(unix, windows)))] + { + let _ = target; + return Err(AlienError::new(service_error( + "the launcher layout is only supported on Unix and Windows".to_string(), + ))); + } + Ok(()) +} + +/// Render the systemd unit for the launcher. `Type=notify` + `WatchdogSec` +/// let systemd supervise the launcher's liveness while the launcher owns the +/// operator's version health (two-level supervision). +pub fn render_unit( + launcher_path: &Path, + data_dir: &Path, + service_user: Option<&str>, + environment: &[(String, String)], +) -> String { + let mut env_lines = String::new(); + for (key, value) in environment { + env_lines.push_str(&format!("Environment=\"{key}={value}\"\n")); + } + let user_line = service_user + .map(|user| format!("User={user}\n")) + .unwrap_or_default(); + // StateDirectory only applies to the canonical /var/lib path; custom data + // dirs rely on ReadWritePaths alone. + let state_directory_line = if data_dir == Path::new("/var/lib/alien-operator") { + "StateDirectory=alien-operator\n" + } else { + "" + }; + + format!( + "\ +[Unit] +Description=Alien Operator Launcher +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +NotifyAccess=main +WatchdogSec=60 +ExecStart={launcher} --data-dir {data_dir} +Restart=always +RestartSec=2 +{user_line}{state_directory_line}NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +ReadWritePaths={data_dir} +{env_lines} +[Install] +WantedBy=multi-user.target +", + launcher = launcher_path.display(), + data_dir = data_dir.display(), + ) +} + +/// Install (or redeploy) the launcher-based service: stop it, refresh the +/// layout, then register + start it — systemd on Linux, the SCM on Windows. +pub fn install_launcher_service( + service_label: &str, + data_dir: &Path, + operator_binary: &Path, + launcher_binary: &Path, + service_user: Option<&str>, + environment: &[(String, String)], +) -> Result<()> { + let operator_version = binary_version(operator_binary)?; + + output::step(1, 4, "Stopping the service (if running)"); + stop_service_best_effort(service_label); + + output::step( + 2, + 4, + &format!("Writing the version store (operator {operator_version})"), + ); + write_layout(data_dir, operator_binary, &operator_version, launcher_binary)?; + + register_and_start(service_label, data_dir, service_user, environment) +} + +/// Stop the service if running (best-effort — a first install has nothing to +/// stop; the restart is what adopts the refreshed layout). +#[cfg(unix)] +fn stop_service_best_effort(service_label: &str) { + let _ = systemctl(&["stop", &format!("{service_label}.service")]); +} + +#[cfg(windows)] +fn stop_service_best_effort(service_label: &str) { + if let (Ok(label), Ok(manager)) = ( + service_label.parse::(), + ::native(), + ) { + let _ = manager.stop(ServiceStopCtx { label }); + } +} + +#[cfg(not(any(unix, windows)))] +fn stop_service_best_effort(_service_label: &str) {} + +/// Linux: write the systemd unit, daemon-reload, enable + start. +#[cfg(unix)] +fn register_and_start( + service_label: &str, + data_dir: &Path, + service_user: Option<&str>, + environment: &[(String, String)], +) -> Result<()> { + output::step(3, 4, "Installing the systemd unit"); + let launcher_path = data_dir.join("launcher").join(launcher_binary_name()); + let unit = render_unit(&launcher_path, data_dir, service_user, environment); + let unit_path = PathBuf::from(format!("/etc/systemd/system/{service_label}.service")); + write_file(&unit_path, unit.as_bytes())?; + + output::step(4, 4, "Enabling + starting the service"); + systemctl(&["daemon-reload"])?; + systemctl(&["enable", "--now", &format!("{service_label}.service")])?; + Ok(()) +} + +/// Windows: register `alien-launcher` as the SCM service, apply the doc-12 +/// recovery config (`sc.exe` can't set restart policy via `create`), and start +/// it. The launcher passes its environment through to the operator child, so +/// the operator's config flows via the service `environment` — the mirror of +/// the systemd `Environment=` lines. `service_user` is Linux-only. +#[cfg(windows)] +fn register_and_start( + service_label: &str, + data_dir: &Path, + _service_user: Option<&str>, + environment: &[(String, String)], +) -> Result<()> { + output::step(3, 4, "Registering the launcher service"); + let label: ServiceLabel = service_label.parse().map_err(|_| { + AlienError::new(service_error(format!( + "invalid service label '{service_label}'" + ))) + })?; + let manager = ::native() + .into_alien_error() + .context(service_error("no supported service manager found".to_string()))?; + let launcher_path = data_dir.join("launcher").join(launcher_binary_name()); + manager + .install(ServiceInstallCtx { + label: label.clone(), + program: launcher_path, + args: vec![OsString::from("--data-dir"), OsString::from(data_dir)], + contents: None, + username: None, + working_directory: None, + environment: Some(environment.to_vec()), + autostart: true, + // `sc create` can't set restart policy; we apply the doc-12 recovery + // config right after via `sc failure`. `Never` here avoids + // service-manager's misleading "will not restart" warning. + restart_policy: RestartPolicy::Never, + }) + .into_alien_error() + .context(service_error( + "failed to register the launcher service".to_string(), + ))?; + + configure_failure_actions(&label.to_qualified_name())?; + + output::step(4, 4, "Starting the service"); + manager + .start(ServiceStartCtx { label }) + .into_alien_error() + .context(service_error( + "failed to start the launcher service".to_string(), + ))?; + Ok(()) +} + +/// Apply the SCM recovery config via `sc.exe failure` (doc 12). +#[cfg(windows)] +fn configure_failure_actions(service_name: &str) -> Result<()> { + let status = Command::new("sc") + .args(failure_action_args(service_name)) + .status() + .into_alien_error() + .context(service_error("failed to run 'sc failure'".to_string()))?; + if !status.success() { + return Err(AlienError::new(service_error(format!( + "'sc failure' exited with {status}" + )))); + } + Ok(()) +} + +/// The `sc failure` argument vector (doc 12): reset the failure counter after +/// 24 h and restart three times at 5 s each. OS-agnostic so it is unit-tested on +/// any host. +#[cfg(any(windows, test))] +fn failure_action_args(service_name: &str) -> Vec { + vec![ + "failure".to_string(), + service_name.to_string(), + "reset=".to_string(), + "86400".to_string(), + "actions=".to_string(), + "restart/5000/restart/5000/restart/5000".to_string(), + ] +} + +#[cfg(unix)] +fn write_file(path: &Path, contents: &[u8]) -> Result<()> { + let mut file = std::fs::File::create(path) + .into_alien_error() + .context(service_error(format!( + "failed to create '{}' (are you root?)", + path.display() + )))?; + file.write_all(contents) + .into_alien_error() + .context(service_error(format!( + "failed to write '{}'", + path.display() + ))) +} + +#[cfg(unix)] +fn systemctl(args: &[&str]) -> Result<()> { + let status = Command::new("systemctl") + .args(args) + .status() + .into_alien_error() + .context(service_error(format!( + "failed to run systemctl {args:?}" + )))?; + if !status.success() { + return Err(AlienError::new(service_error(format!( + "systemctl {args:?} exited with {status}" + )))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn layout_decision_matrix() { + assert!(use_launcher_layout_for(false, "linux")); + assert!(use_launcher_layout_for(false, "windows")); + assert!(!use_launcher_layout_for(true, "linux"), "--no-launcher wins"); + assert!( + !use_launcher_layout_for(true, "windows"), + "--no-launcher wins on Windows too" + ); + assert!( + !use_launcher_layout_for(false, "macos"), + "macOS keeps legacy until its launcher phase" + ); + } + + #[test] + fn failure_actions_match_doc_12() { + assert_eq!( + failure_action_args("dev.alien.operator"), + vec![ + "failure", + "dev.alien.operator", + "reset=", + "86400", + "actions=", + "restart/5000/restart/5000/restart/5000", + ] + ); + } + + #[test] + fn version_output_parses() { + assert_eq!(parse_version_output("operator 1.11.2\n"), Some("1.11.2".to_string())); + assert_eq!( + parse_version_output("alien-launcher 1.11.2"), + Some("1.11.2".to_string()) + ); + assert_eq!(parse_version_output(""), None); + assert_eq!(parse_version_output("no version here"), None); + } + + #[cfg(unix)] + #[test] + fn layout_is_exact_and_idempotency_preserves_state_and_secrets() { + let root = tempfile::tempdir().unwrap(); + let data_dir = root.path().join("data"); + let operator = root.path().join("alien-operator-artifact"); + let launcher = root.path().join("alien-launcher-artifact"); + std::fs::write(&operator, b"operator-v1-bytes").unwrap(); + std::fs::write(&launcher, b"launcher-v1-bytes").unwrap(); + + write_layout(&data_dir, &operator, "1.11.2", &launcher).expect("fresh install"); + + // Exact tree (sorted relative paths of files + symlinks). + let mut entries: Vec = walk(&data_dir); + entries.sort(); + assert_eq!( + entries, + vec![ + "current".to_string(), + "last-stable".to_string(), + "launcher/alien-launcher".to_string(), + "versions/1.11.2/alien-operator".to_string(), + ], + "empty dirs (state/, download/, …) plus exactly these entries" + ); + for dir in ["state", "state-snapshots", "failed", "download"] { + assert!(data_dir.join(dir).is_dir(), "{dir}/ must exist"); + } + assert_eq!( + std::fs::read_link(data_dir.join("current")).unwrap(), + Path::new("versions/1.11.2") + ); + // Executable bits set. + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(data_dir.join("versions/1.11.2/alien-operator")) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o111, 0o111); + + // Simulate a live system: state written, secrets present, current + // moved forward by a self-update to a newer version. + std::fs::write(data_dir.join("state/db"), b"live-state").unwrap(); + std::fs::write(data_dir.join("sync-token"), b"secret-token").unwrap(); + std::fs::create_dir_all(data_dir.join("versions/2.0.0")).unwrap(); + std::fs::remove_file(data_dir.join("current")).unwrap(); + std::os::unix::fs::symlink("versions/2.0.0", data_dir.join("current")).unwrap(); + + // Redeploy with refreshed artifacts. + std::fs::write(&operator, b"operator-v2-bytes").unwrap(); + std::fs::write(&launcher, b"launcher-v2-bytes").unwrap(); + write_layout(&data_dir, &operator, "1.11.2", &launcher).expect("redeploy"); + + assert_eq!( + std::fs::read(data_dir.join("state/db")).unwrap(), + b"live-state", + "state/ must survive a redeploy byte-identical" + ); + assert_eq!( + std::fs::read(data_dir.join("sync-token")).unwrap(), + b"secret-token", + "secrets must survive a redeploy" + ); + assert_eq!( + std::fs::read(data_dir.join("launcher/alien-launcher")).unwrap(), + b"launcher-v2-bytes", + "the launcher binary is always refreshed — this IS the redeploy" + ); + assert_eq!( + std::fs::read(data_dir.join("versions/1.11.2/alien-operator")).unwrap(), + b"operator-v2-bytes" + ); + assert_eq!( + std::fs::read_link(data_dir.join("current")).unwrap(), + Path::new("versions/2.0.0"), + "a live store's pointers are the launcher's truth — never clobbered" + ); + } + + /// Windows analogue: junction pointers + `.exe` binary names, same + /// state/secret-preserving redeploy guarantees. Pointer reads resolve the + /// junction target's final component (the version), like the launcher store. + #[cfg(windows)] + #[test] + fn layout_is_exact_and_idempotency_preserves_state_and_secrets_windows() { + let root = tempfile::tempdir().unwrap(); + let data_dir = root.path().join("data"); + let operator = root.path().join("alien-operator-artifact"); + let launcher = root.path().join("alien-launcher-artifact"); + std::fs::write(&operator, b"operator-v1-bytes").unwrap(); + std::fs::write(&launcher, b"launcher-v1-bytes").unwrap(); + + write_layout(&data_dir, &operator, "1.11.2", &launcher).expect("fresh install"); + + let mut entries: Vec = walk(&data_dir); + entries.sort(); + assert_eq!( + entries, + vec![ + "current".to_string(), + "last-stable".to_string(), + "launcher/alien-launcher.exe".to_string(), + "versions/1.11.2/alien-operator.exe".to_string(), + ] + ); + for dir in ["state", "state-snapshots", "failed", "download"] { + assert!(data_dir.join(dir).is_dir(), "{dir}/ must exist"); + } + let current = junction::get_target(data_dir.join("current")).unwrap(); + assert_eq!(current.file_name().unwrap().to_str().unwrap(), "1.11.2"); + + // Live system: state + secrets present, current advanced by a self-update. + std::fs::write(data_dir.join("state/db"), b"live-state").unwrap(); + std::fs::write(data_dir.join("sync-token"), b"secret-token").unwrap(); + std::fs::create_dir_all(data_dir.join("versions/2.0.0")).unwrap(); + std::fs::remove_dir(data_dir.join("current")).unwrap(); + junction::create(data_dir.join("versions/2.0.0"), data_dir.join("current")).unwrap(); + + std::fs::write(&operator, b"operator-v2-bytes").unwrap(); + std::fs::write(&launcher, b"launcher-v2-bytes").unwrap(); + write_layout(&data_dir, &operator, "1.11.2", &launcher).expect("redeploy"); + + assert_eq!(std::fs::read(data_dir.join("state/db")).unwrap(), b"live-state"); + assert_eq!( + std::fs::read(data_dir.join("sync-token")).unwrap(), + b"secret-token" + ); + assert_eq!( + std::fs::read(data_dir.join("launcher/alien-launcher.exe")).unwrap(), + b"launcher-v2-bytes", + "the launcher binary is always refreshed — this IS the redeploy" + ); + let current = junction::get_target(data_dir.join("current")).unwrap(); + assert_eq!( + current.file_name().unwrap().to_str().unwrap(), + "2.0.0", + "a live store's pointers are the launcher's truth — never clobbered" + ); + } + + fn walk(root: &Path) -> Vec { + let mut out = Vec::new(); + fn rec(root: &Path, dir: &Path, out: &mut Vec) { + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + let meta = path.symlink_metadata().unwrap(); + if meta.is_dir() { + rec(root, &path, out); + } else { + out.push(path.strip_prefix(root).unwrap().to_string_lossy().into_owned()); + } + } + } + rec(root, root, &mut out); + out + } + + #[test] + fn unit_file_golden() { + let unit = render_unit( + Path::new("/var/lib/alien-operator/launcher/alien-launcher"), + Path::new("/var/lib/alien-operator"), + None, + &[ + ("PLATFORM".to_string(), "aws".to_string()), + ("SYNC_URL".to_string(), "https://manager.example.com".to_string()), + ( + "SYNC_TOKEN_FILE".to_string(), + "/var/lib/alien-operator/sync-token".to_string(), + ), + ], + ); + let expected = "\ +[Unit] +Description=Alien Operator Launcher +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +NotifyAccess=main +WatchdogSec=60 +ExecStart=/var/lib/alien-operator/launcher/alien-launcher --data-dir /var/lib/alien-operator +Restart=always +RestartSec=2 +StateDirectory=alien-operator +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +ReadWritePaths=/var/lib/alien-operator +Environment=\"PLATFORM=aws\" +Environment=\"SYNC_URL=https://manager.example.com\" +Environment=\"SYNC_TOKEN_FILE=/var/lib/alien-operator/sync-token\" + +[Install] +WantedBy=multi-user.target +"; + assert_eq!(unit, expected); + + // Custom data dir: no StateDirectory, ReadWritePaths follows; user set. + let unit = render_unit( + Path::new("/opt/x/launcher/alien-launcher"), + Path::new("/opt/x"), + Some("alien"), + &[], + ); + assert!(unit.contains("User=alien\n")); + assert!(!unit.contains("StateDirectory=")); + assert!(unit.contains("ReadWritePaths=/opt/x\n")); + assert!(unit.contains("ExecStart=/opt/x/launcher/alien-launcher --data-dir /opt/x\n")); + } +} diff --git a/crates/alien-deploy-cli/src/commands/mod.rs b/crates/alien-deploy-cli/src/commands/mod.rs index 858e94283..5e3866a80 100644 --- a/crates/alien-deploy-cli/src/commands/mod.rs +++ b/crates/alien-deploy-cli/src/commands/mod.rs @@ -1,4 +1,5 @@ pub mod down; +pub mod install_launcher; pub mod join; pub mod list; pub mod operator; diff --git a/crates/alien-deploy-cli/src/commands/operator.rs b/crates/alien-deploy-cli/src/commands/operator.rs index 1cc7bbf29..18690383c 100644 --- a/crates/alien-deploy-cli/src/commands/operator.rs +++ b/crates/alien-deploy-cli/src/commands/operator.rs @@ -30,7 +30,20 @@ pub enum OperatorCommand { /// Show the operator service status Status, /// Uninstall the alien-operator service - Uninstall, + Uninstall(UninstallArgs), +} + +#[derive(Args)] +pub struct UninstallArgs { + /// Also delete the data directory (state DB, secrets, version store). + /// Default keeps data so a reinstall resumes the same deployment + /// identity — wiping it orphans the deployment on the manager. + #[arg(long)] + pub purge: bool, + + /// Data directory to purge (with --purge). Defaults to the service default. + #[arg(long)] + pub data_dir: Option, } #[derive(Args)] @@ -78,6 +91,20 @@ pub struct InstallArgs { /// Override the shell command used by Local runtime debug shells. #[arg(long)] pub local_debug_shell_command: Option, + + /// Path to the alien-launcher binary (Linux launcher layout). If omitted, + /// checks ALIEN_LAUNCHER_BINARY, then next to the operator binary, then PATH. + #[arg(long)] + pub launcher_binary: Option, + + /// Install the legacy direct-operator service instead of the + /// launcher-supervised layout (no self-update support). + #[arg(long)] + pub no_launcher: bool, + + /// Unix user the service runs as (Linux launcher layout). Defaults to root. + #[arg(long)] + pub service_user: Option, } pub async fn operator_command(args: OperatorArgs) -> Result<()> { @@ -86,7 +113,7 @@ pub async fn operator_command(args: OperatorArgs) -> Result<()> { OperatorCommand::Start => start(), OperatorCommand::Stop => stop(), OperatorCommand::Status => status(), - OperatorCommand::Uninstall => uninstall(), + OperatorCommand::Uninstall(uninstall_args) => uninstall(uninstall_args.purge, uninstall_args.data_dir), } } @@ -137,7 +164,7 @@ pub fn uninstall_service_if_installed() -> Result<()> { .context(ErrorData::OperatorServiceError { message: "Failed to check service status".to_string(), })? { - ServiceStatus::Running | ServiceStatus::Stopped(_) => uninstall(), + ServiceStatus::Running | ServiceStatus::Stopped(_) => uninstall(false, None), ServiceStatus::NotInstalled => { output::info("alien-operator service is not installed"); Ok(()) @@ -231,6 +258,68 @@ fn install(args: InstallArgs) -> Result<()> { } } + // Linux default: the launcher-supervised layout (health-gated binary + // swaps with rollback). The operator's config flows through the unit's + // Environment= lines — every operator flag has an env alias — and the + // launcher's child inherits them. `--no-launcher` keeps the legacy + // direct-operator service (no self-update). + if crate::commands::install_launcher::use_launcher_layout(args.no_launcher) { + let launcher_binary = crate::commands::install_launcher::which_launcher_binary( + args.launcher_binary.clone(), + &binary_path, + )?; + let mut environment: Vec<(String, String)> = vec![ + ("PLATFORM".to_string(), args.platform.clone()), + ("SYNC_URL".to_string(), args.sync_url.clone()), + ( + "SYNC_TOKEN_FILE".to_string(), + sync_token_path.display().to_string(), + ), + ("DATA_DIR".to_string(), data_dir.clone()), + ( + "OPERATOR_ENCRYPTION_KEY_FILE".to_string(), + encryption_key_path.display().to_string(), + ), + ]; + if let Some(deployment_id) = &args.deployment_id { + environment.push(("DEPLOYMENT_ID".to_string(), deployment_id.clone())); + } + if let Some(operator_name) = &args.operator_name { + environment.push(("OPERATOR_NAME".to_string(), operator_name.clone())); + } + if args.public_endpoints.is_some() { + environment.push(( + "PUBLIC_ENDPOINTS_FILE".to_string(), + public_endpoints_path.display().to_string(), + )); + } + if args.enable_local_debug { + environment.push(("ALIEN_ENABLE_LOCAL_DEBUG".to_string(), "true".to_string())); + } + if let Some(shell_command) = &args.local_debug_shell_command { + environment.push(( + "ALIEN_LOCAL_DEBUG_SHELL_COMMAND".to_string(), + shell_command.clone(), + )); + } + + crate::commands::install_launcher::install_launcher_service( + SERVICE_LABEL, + std::path::Path::new(&data_dir), + &binary_path, + &launcher_binary, + args.service_user.as_deref(), + &environment, + )?; + + output::success("alien-operator service installed under the launcher"); + output::info(&format!(" Operator: {}", binary_path.display())); + output::info(&format!(" Launcher: {}", launcher_binary.display())); + output::info(&format!(" Data dir: {}", data_dir)); + output::info(&format!(" Sync URL: {}", args.sync_url)); + return Ok(()); + } + let mut service_args = vec![ OsString::from("--service"), OsString::from("--platform"), @@ -409,7 +498,7 @@ fn status() -> Result<()> { Ok(()) } -fn uninstall() -> Result<()> { +fn uninstall(purge: bool, data_dir: Option) -> Result<()> { let manager = get_manager()?; // Stop first (ignore errors if not running) @@ -423,6 +512,20 @@ fn uninstall() -> Result<()> { })?; output::success("alien-operator service uninstalled"); + + let data_dir = data_dir.unwrap_or_else(default_service_data_dir); + if purge { + std::fs::remove_dir_all(&data_dir) + .into_alien_error() + .context(ErrorData::OperatorServiceError { + message: format!("Failed to purge data directory {data_dir}"), + })?; + output::info(&format!(" Purged data dir: {data_dir}")); + } else { + output::info(&format!( + " Data dir kept: {data_dir} (state + deployment identity survive a reinstall; pass --purge to delete)" + )); + } Ok(()) } diff --git a/crates/alien-deploy-cli/src/commands/up.rs b/crates/alien-deploy-cli/src/commands/up.rs index 5c73ef492..11564018b 100644 --- a/crates/alien-deploy-cli/src/commands/up.rs +++ b/crates/alien-deploy-cli/src/commands/up.rs @@ -123,6 +123,17 @@ pub struct UpArgs { #[arg(long)] pub local_debug_shell_command: Option, + /// Install the legacy direct-operator service instead of the launcher-supervised + /// layout (no self-update). Only affects Linux — macOS/Windows use the direct + /// layout until their launcher installers land. + #[arg(long)] + pub no_launcher: bool, + + /// Path to the alien-launcher binary for the launcher-supervised layout (Linux). + /// Defaults to ALIEN_LAUNCHER_BINARY, then next to the operator binary, then PATH. + #[arg(long, env = "ALIEN_LAUNCHER_BINARY")] + pub launcher_binary: Option, + /// Kubernetes namespace for Helm installs. #[arg(long, env = "ALIEN_KUBERNETES_NAMESPACE")] pub namespace: Option, @@ -2662,6 +2673,13 @@ async fn run_local_pull_model( public_endpoints: public_endpoints.cloned(), enable_local_debug: args.enable_local_debug, local_debug_shell_command: args.local_debug_shell_command.clone(), + // Default to the launcher-supervised layout so deployments can self-update, + // matching `operator install`. `use_launcher_layout_for` gates this to Linux; + // macOS/Windows fall back to the direct operator until their installers land. + // `--no-launcher` keeps the legacy direct-operator service (no self-update). + launcher_binary: args.launcher_binary.clone(), + no_launcher: args.no_launcher, + service_user: None, }; super::operator::install_service(install_args)?; diff --git a/crates/alien-deployment/src/helpers.rs b/crates/alien-deployment/src/helpers.rs index 97709305d..88a05cad3 100644 --- a/crates/alien-deployment/src/helpers.rs +++ b/crates/alien-deployment/src/helpers.rs @@ -37,7 +37,11 @@ pub async fn collect_environment_info( Platform::Aws => collect_aws_env_info(client_config).await, Platform::Gcp => collect_gcp_env_info(client_config).await, Platform::Azure => collect_azure_env_info(client_config).await, - Platform::Local => collect_local_env_info(client_config).await, + // For pure Kubernetes (no base_platform), there is no cloud account/region + // to report. Treat it like Local — return hostname/os/arch runtime metadata. + // k8s-on-cloud deployments take a different path via environment_collection_context + // in pending.rs, which substitutes the base cloud platform here. + Platform::Local | Platform::Kubernetes => collect_local_env_info(client_config).await, Platform::Test => collect_test_env_info().await, _ => Err(AlienError::new(ErrorData::MissingConfiguration { message: format!( diff --git a/crates/alien-deployment/src/pending.rs b/crates/alien-deployment/src/pending.rs index 402ee7e36..74ef2c430 100644 --- a/crates/alien-deployment/src/pending.rs +++ b/crates/alien-deployment/src/pending.rs @@ -22,11 +22,20 @@ pub async fn handle_pending( info!("Handling Pending status"); // Step 1: Initialize stack state. Direct platform deployments may carry a - // user-selected resource prefix in their initial stack state. - let stack_state = current - .stack_state - .clone() - .unwrap_or_else(|| StackState::new(current.platform)); + // user-selected resource prefix in their initial stack state. For pull-model + // agents with ephemeral storage (e.g. Kubernetes via Helm), the random prefix + // from `StackState::new` is regenerated on every restart, which breaks the + // alignment with Helm-created ServiceAccount names and vault-secret names. + // `ALIEN_RESOURCE_PREFIX` lets the chart pin a stable prefix that matches + // `serviceAccountPrefix` so SA + secret names stay valid across restarts. + let stack_state = current.stack_state.clone().unwrap_or_else(|| { + match std::env::var("ALIEN_RESOURCE_PREFIX") { + Ok(prefix) if !prefix.trim().is_empty() => { + StackState::with_resource_prefix(current.platform, prefix.trim().to_string()) + } + _ => StackState::new(current.platform), + } + }); info!( "Initialized stack state for platform {:?}", current.platform diff --git a/crates/alien-helm/Cargo.toml b/crates/alien-helm/Cargo.toml index edd204caf..0995b37bd 100644 --- a/crates/alien-helm/Cargo.toml +++ b/crates/alien-helm/Cargo.toml @@ -22,3 +22,6 @@ tempfile = { workspace = true, optional = true } [dev-dependencies] insta = { workspace = true } tempfile = { workspace = true } +# Activate the `test-utils` feature so external integration tests can reach +# `alien_helm::test_utils::*` (the module is gated on `test-utils`). +alien-helm = { path = ".", features = ["test-utils"] } diff --git a/crates/alien-helm/src/generator.rs b/crates/alien-helm/src/generator.rs index 4cf31de7f..054403c2a 100644 --- a/crates/alien-helm/src/generator.rs +++ b/crates/alien-helm/src/generator.rs @@ -1352,12 +1352,31 @@ runtime: podAnnotations: {} automountServiceAccountToken: true encryption: - # Set this explicitly, or reference an existing Secret below. - key: "replace-me-with-a-stable-64-character-encryption-secret" + # Leave empty to let the chart generate a stable random 64-hex-char key + # on first install (preserved across upgrades via `lookup`). To pin the + # key explicitly, set it here (must be 64 hex chars = 256-bit AES). To + # source it from an external secret store, set `existingSecret.name`. + key: "" existingSecret: name: "" key: encryption-key + # Agent self-update inputs the agent passes to the Helm-runner Job it + # spawns on `operator_target.helm`. Set chartRef + chartVersion to the OCI + # ref + version used at install time — the agent re-uses them in + # `helm upgrade --reuse-values`. Leave blank if you don't want to enable + # in-cluster agent upgrades for this install. + upgrade: + chartRef: "" + chartVersion: "" + helmRunnerImage: "alpine/helm:3.18.4" + # Extra flags appended to the `helm upgrade` command the agent's + # helm-runner Job runs (e.g. `--plain-http` for local-dev OCI + # registries served over HTTP). Production should leave empty. + extraArgs: "" replicas: 1 + # Helm's --atomic --wait gives up after this many seconds if /readyz + # hasn't returned 200 — the revision is then rolled back automatically. + progressDeadlineSeconds: 120 resources: requests: cpu: 100m @@ -1371,16 +1390,20 @@ runtime: service: type: ClusterIP probes: + # /livez is process liveness; /readyz turns 200 only after the agent + # completes at least one /v1/sync round-trip with the manager — the + # gate Helm's --atomic --wait relies on so a freshly-rolled agent + # is not considered ready until it has actually reached the manager. liveness: enabled: true - path: /health + path: /livez initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 2 failureThreshold: 3 readiness: enabled: true - path: /health + path: /readyz initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 2 @@ -1405,7 +1428,16 @@ runtime: data: mountPath: /var/lib/deployment-operator persistence: - enabled: false + # Enabled by default: the agent's `data_dir` holds its persistent + # deployment_id + sync-token. Without a PVC, any pod restart (e.g. + # the rolling restart triggered by self-update / `operator_target.helm`) + # wipes that state, the new pod re-runs `/v1/initialize`, hits a + # name-conflict 409, crashloops, and helm `--atomic` rolls back. + # Operators on clusters without a default StorageClass must either + # set `storageClassName`, point at an `existingClaim`, or + # explicitly disable this and accept that self-update will not + # survive a pod roll. + enabled: true existingClaim: "" storageClassName: "" accessModes: @@ -1502,7 +1534,7 @@ clusterBootstrap: append_service_accounts(&mut yaml, analysis); append_stack_settings(&mut yaml, stack_settings)?; - yaml.push_str("\ninfrastructure: null\n\nbasePlatform: null\nbasePlatformConfig:\n gcp:\n projectId: \"\"\n region: \"\"\n aws:\n region: \"\"\n azure:\n location: \"\"\n subscriptionId: \"\"\n tenantId: \"\"\nserviceAccountPrefix: \"\"\nmanagerServiceAccount:\n annotations: {}\n labels: {}\n"); + yaml.push_str("\ninfrastructure: null\n\nbasePlatform: null\nbasePlatformConfig:\n gcp:\n projectId: \"\"\n region: \"\"\n aws:\n region: \"\"\n azure:\n location: \"\"\n subscriptionId: \"\"\n tenantId: \"\"\nserviceAccountPrefix: \"\"\nmanagerServiceAccount:\n annotations: {}\n labels: {}\n\n# Operator self-update. When the operator receives operator_target.helm on\n# /v1/sync it creates a short-lived Helm-runner Job that runs `helm upgrade\n# --atomic`. The Job runs under the release-derived upgrader SA; keep it\n# optional so charts that don't want self-update can disable it.\nupgrader:\n enabled: true\n"); append_services(&mut yaml, analysis); yaml.push_str("\npublicEndpoints: {}\n"); @@ -2051,7 +2083,18 @@ fn values_schema_json() -> String { } } }, + "upgrade": { + "type": "object", + "additionalProperties": false, + "properties": { + "chartRef": { "type": "string" }, + "chartVersion": { "type": "string" }, + "helmRunnerImage": { "type": "string" }, + "extraArgs": { "type": "string" } + } + }, "replicas": { "type": "integer", "minimum": 1 }, + "progressDeadlineSeconds": { "type": "integer", "minimum": 1 }, "resources": { "type": "object" }, "api": { "type": "object", @@ -2408,6 +2451,13 @@ fn values_schema_json() -> String { } }, "serviceAccountPrefix": { "type": "string" }, + "upgrader": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" } + } + }, "services": { "type": "object", "additionalProperties": { @@ -2504,6 +2554,18 @@ helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} {{- regexReplaceAll "[^a-z0-9-]" $raw "-" | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* + ServiceAccount used by the Helm-runner Job the agent creates when it + acts on operator_target.helm. Held as a least-privilege boundary; bound + to the existing Role so the Job can mutate the Deployment + release + Secrets. +*/}} +{{- define "deployment.upgraderServiceAccountName" -}} +{{- $prefix := default (include "deployment.fullname" .) .Values.serviceAccountPrefix -}} +{{- $raw := printf "%s-upgrader-sa" $prefix | lower -}} +{{- regexReplaceAll "[^a-z0-9-]" $raw "-" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{- define "deployment.serviceAccountName" -}} {{- $prefix := default (include "deployment.fullname" .root) .root.Values.serviceAccountPrefix -}} {{- $raw := printf "%s-%s-sa" $prefix .name | lower -}} @@ -2534,6 +2596,14 @@ helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} {{- define "deployment.heartbeatNodeClusterRoleName" -}} {{- printf "%s-heartbeat-nodes" (include "deployment.fullname" .) | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{- /* Name of the ClusterRole that grants the agent self-update Job permission + to manage the chart-owned cluster-scoped resources (currently just the + heartbeat ClusterRole+Binding). Only created when both `upgrader.enabled` + and the heartbeat node-collection feature are on. */ -}} +{{- define "deployment.upgraderClusterRoleName" -}} +{{- printf "%s-upgrader" (include "deployment.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} "# .to_string() } @@ -2569,6 +2639,22 @@ metadata: {{- end }} --- {{- end }} +{{- if .Values.upgrader.enabled }} +# The upgrader ServiceAccount (release-derived, see +# `deployment.upgraderServiceAccountName`) is used by the Helm-runner Job the +# operator creates when it acts on operator_target.helm. It exists as a +# least-privilege boundary for the Job — the operator pod itself uses its own +# manager SA and only needs to create Jobs + stage ConfigMaps/Secrets. The +# protection against bad helm upgrades is the chart's `required` values, not +# RBAC. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "deployment.upgraderServiceAccountName" . }} + labels: + {{- include "deployment.labels" . | nindent 4 }} +--- +{{- end }} "# .to_string() } @@ -2587,6 +2673,17 @@ rules: - apiGroups: [""] resources: ["configmaps", "secrets", "services", "pods", "pods/log", "persistentvolumeclaims"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + # ServiceAccounts + RBAC objects need to live here for `helm upgrade + # --reuse-values` to inspect (and patch) the release's existing + # resources during agent self-update. Without this, the upgrader SA + # — which is bound to this Role — can't `get` the SAs the chart + # already created and helm 4xx's out. + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch"] @@ -2632,6 +2729,18 @@ metadata: subjects: - kind: ServiceAccount name: {{ include "deployment.managerServiceAccountName" . }} + {{- /* Execution ServiceAccounts (workers/daemons/containers) need RBAC to + read their own vault secrets (e.g. the injected ALIEN_COMMANDS_TOKEN). + Without this binding, the worker pod gets 403 reading any secret + provisioned by the KubernetesVaultController. */}} + {{- range $name, $account := .Values.serviceAccounts }} + - kind: ServiceAccount + name: {{ include "deployment.serviceAccountName" (dict "root" $ "name" $name) }} + {{- end }} + {{- if .Values.upgrader.enabled }} + - kind: ServiceAccount + name: {{ include "deployment.upgraderServiceAccountName" . }} + {{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -2686,6 +2795,40 @@ rules: - apiGroups: ["metrics.k8s.io"] resources: ["nodes"] verbs: ["get", "list", "watch"] +{{- if .Values.upgrader.enabled }} +--- +# Narrow cluster-scoped RBAC for the agent self-update helm-runner Job. +# The chart creates exactly one cluster-scoped resource type pair — +# the heartbeat ClusterRole + ClusterRoleBinding above — and the +# upgrader SA needs to be able to `get/update/patch/delete` them +# during `helm upgrade --reuse-values`. `resourceNames` scopes this +# to ONLY the chart's own cluster objects; no enumeration of other +# tenants' cluster resources. Verbs are deliberately minimal (no +# `list`/`watch`, no `create` — chart install is what creates them +# the first time, run by the customer's helm operator). If a future +# chart version introduces new cluster-scoped resources, add their +# names to `resourceNames` (and a `create` verb on a separate rule +# without `resourceNames` if the upgrader needs to add new ones). +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "deployment.upgraderClusterRoleName" . }} + labels: + {{- include "deployment.labels" . | nindent 4 }} +rules: + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles", "clusterrolebindings"] + resourceNames: + # The heartbeat-nodes cluster pair — the existing reason the + # upgrader needs cluster-scope access at all. + - {{ include "deployment.heartbeatNodeClusterRoleName" . | quote }} + # And the upgrader cluster pair itself — helm now tracks these as + # chart-owned, so every `helm upgrade --reuse-values` does a `get` + # on them to compute the diff. Without self-reference, the first + # upgrade trips on `clusterroles "-upgrader" is forbidden`. + - {{ include "deployment.upgraderClusterRoleName" . | quote }} + verbs: ["get", "update", "patch", "delete"] +{{- end }} {{- end }} "# .to_string() @@ -2708,14 +2851,63 @@ roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: {{ include "deployment.heartbeatNodeClusterRoleName" . }} +{{- if .Values.upgrader.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "deployment.upgraderClusterRoleName" . }} + labels: + {{- include "deployment.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "deployment.upgraderServiceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "deployment.upgraderClusterRoleName" . }} +{{- end }} {{- end }} "# .to_string() } fn secret_tpl() -> String { + // Encryption key resolution order: + // 1. user-provided `runtime.encryption.key` + // 2. existing in-cluster Secret's encryption-key (preserves the key + // across `helm upgrade` so previously-encrypted data stays readable) + // 3. freshly generated via `randBytes 32` — crypto/rand-backed in + // sprig 3.2+; if your Helm bundles an older sprig, set the key + // explicitly via `runtime.encryption.key` or + // `runtime.encryption.existingSecret.name`. + // + // `lookup` returns nil during `helm template` (no cluster access), so + // a `helm template | kubectl apply -f -` workflow would generate a + // fresh key on each render — install via `helm install/upgrade` to + // keep the key stable, or always set `runtime.encryption.key`. r#"{{- $createManagementSecret := not .Values.management.existingSecret.name -}} {{- $createEncryptionSecret := not .Values.runtime.encryption.existingSecret.name -}} +{{- $encryptionKey := "" -}} +{{- if $createEncryptionSecret -}} + {{- $providedKey := .Values.runtime.encryption.key | default "" | trim -}} + {{- $existingKey := "" -}} + {{- $existingSecret := lookup "v1" "Secret" .Release.Namespace (include "deployment.fullname" .) -}} + {{- if and $existingSecret $existingSecret.data -}} + {{- with index $existingSecret.data "encryption-key" -}} + {{- $existingKey = b64dec . -}} + {{- end -}} + {{- end -}} + {{- if $providedKey -}} + {{- $encryptionKey = $providedKey -}} + {{- else if $existingKey -}} + {{- $encryptionKey = $existingKey -}} + {{- else -}} + {{- /* sprig randBytes returns base64; b64dec to raw bytes then hex */ -}} + {{- $encryptionKey = printf "%x" (b64dec (randBytes 32)) -}} + {{- end -}} +{{- end -}} {{- if or $createManagementSecret $createEncryptionSecret .Values.infrastructure .Values.logCollector.enabled }} apiVersion: v1 kind: Secret @@ -2726,10 +2918,10 @@ metadata: type: Opaque stringData: {{- if $createManagementSecret }} - sync-token: {{ .Values.management.token | quote }} + sync-token: {{ required "management.token or management.existingSecret.name is required — pass the full values document" .Values.management.token | quote }} {{- end }} {{- if $createEncryptionSecret }} - encryption-key: {{ required "runtime.encryption.key or runtime.encryption.existingSecret.name is required" .Values.runtime.encryption.key | quote }} + encryption-key: {{ $encryptionKey | quote }} {{- end }} {{- if .Values.infrastructure }} external-bindings.json: {{ toJson .Values.infrastructure | quote }} @@ -2817,6 +3009,11 @@ metadata: {{- include "deployment.labels" . | nindent 4 }} spec: replicas: {{ .Values.runtime.replicas }} + # Recreate guarantees exactly one agent runs at any time, so the + # InstanceLock is never contended — even during a self-update. + strategy: + type: Recreate + progressDeadlineSeconds: {{ .Values.runtime.progressDeadlineSeconds | default 120 }} selector: matchLabels: app.kubernetes.io/name: {{ include "deployment.name" . }} @@ -2902,16 +3099,62 @@ spec: - name: AZURE_REGION value: {{ .Values.basePlatformConfig.azure.location | quote }} {{- end }} + # `required` chart guardrail: any helm upgrade that does not + # carry the full values document fails to render (Helm aborts + # before touching the release). This is the protection against + # bare `helm upgrade` silently resetting the agent's manager + # config — manager-triggered, operator-triggered, or otherwise. - name: SYNC_URL - value: {{ .Values.management.url | quote }} + value: {{ required "management.url is required — pass the full values document" .Values.management.url | quote }} - name: OPERATOR_NAME - value: {{ .Values.management.name | quote }} + value: {{ required "management.name is required — pass the full values document" .Values.management.name | quote }} {{- if .Values.management.deploymentId }} - name: DEPLOYMENT_ID value: {{ .Values.management.deploymentId | quote }} {{- end }} - name: KUBERNETES_NAMESPACE value: {{ .Release.Namespace | quote }} + - name: KUBERNETES_HELM_RELEASE + value: {{ .Release.Name | quote }} + - name: ALIEN_OPERATOR_UPGRADER_SA + value: {{ include "deployment.upgraderServiceAccountName" . | quote }} + # Reported back on /v1/sync so the dashboard can surface the + # registry an admin will pull a new tag from when pinning a + # target operator version. + - name: ALIEN_OPERATOR_IMAGE_REPOSITORY + value: {{ .Values.runtime.image.repository | quote }} + {{- if .Values.runtime.upgrade.chartRef }} + # Used by the operator when it receives `operator_target.helm` and + # spawns a Helm-runner Job to apply the new version. The Job + # runs `helm upgrade --reuse-values` against this chart so only + # the manager-supplied `values` override (e.g. image.tag) flips. + - name: ALIEN_OPERATOR_CHART_REF + value: {{ .Values.runtime.upgrade.chartRef | quote }} + {{- end }} + {{- if .Values.runtime.upgrade.chartVersion }} + - name: ALIEN_OPERATOR_CHART_VERSION + value: {{ .Values.runtime.upgrade.chartVersion | quote }} + {{- end }} + {{- if .Values.runtime.upgrade.helmRunnerImage }} + - name: ALIEN_OPERATOR_HELM_RUNNER_IMAGE + value: {{ .Values.runtime.upgrade.helmRunnerImage | quote }} + {{- end }} + {{- if .Values.runtime.upgrade.extraArgs }} + # Extra flags spliced into the `helm upgrade` command the + # operator's helm-runner Job runs. Use sparingly — exists for + # local-dev/insecure OCI registries (`--plain-http`) and + # similar one-off escape hatches; production should leave empty. + - name: ALIEN_OPERATOR_HELM_EXTRA_ARGS + value: {{ .Values.runtime.upgrade.extraArgs | quote }} + {{- end }} + {{- if .Values.serviceAccountPrefix }} + # Pin the deployment's resource_prefix to the same value used for + # ServiceAccount naming, so Helm-created SAs and vault secret names + # stay aligned across operator restarts (pull-model storage is ephemeral + # and the operator would otherwise regenerate a random prefix each time). + - name: ALIEN_RESOURCE_PREFIX + value: {{ .Values.serviceAccountPrefix | quote }} + {{- end }} - name: OPERATOR_SETUP_METHOD value: "helm" - name: DATA_DIR @@ -4360,8 +4603,18 @@ mod tests { let files = chart.files.clone(); crate::test_utils::helm_lint(&files).assert_ok("helm chart"); - crate::test_utils::helm_template_and_validate(&files, None) - .assert_ok("helm template registered setup"); + // Manager-fetch path: a token + deploymentId are required (the chart + // refuses to install without them — guards against half-configured + // values). + let manager_fetch_values = r#" +management: + url: "https://manager.example.com" + name: "test-manager" + token: "test-sync-token" + deploymentId: "test-deployment-id" +"#; + crate::test_utils::helm_template_and_validate(&files, Some(manager_fetch_values)) + .assert_ok("helm template manager-fetch path / registered setup"); crate::test_utils::helm_template_and_validate(&files, Some(&files["examples/onprem.yaml"])) .assert_ok("helm template external-bindings initialize path"); } @@ -4379,7 +4632,15 @@ mod tests { ) .expect("chart should render"); + // The secret.yaml guardrail requires `management.token`, so render with + // the manager-fetch overlay like the sibling render tests (see f6db6046); + // this test was missed in that sweep. let values = r#" +management: + url: "https://manager.example.com" + name: "test-manager" + token: "test-sync-token" + deploymentId: "test-deployment-id" logCollector: enabled: true token: test-collector-token diff --git a/crates/alien-helm/tests/generator/boot_paths_tests.rs b/crates/alien-helm/tests/generator/boot_paths_tests.rs index d1beb2de0..028fb05b9 100644 --- a/crates/alien-helm/tests/generator/boot_paths_tests.rs +++ b/crates/alien-helm/tests/generator/boot_paths_tests.rs @@ -18,7 +18,18 @@ fn schema_accepts_registered_setup_default_values() { .build(); let chart = render(&stack, StackSettings::default()); let files = chart.files; - test_utils::helm_template_and_validate(&files, None).assert_ok("registered setup"); + // The manager-fetch path requires `management.{url,name,token,deploymentId}` + // — the chart `required` guardrails reject installs missing them, so the + // test must pass a minimal values overlay. + let manager_fetch_values = r#" +management: + url: "https://manager.example.com" + name: "test-manager" + token: "test-sync-token" + deploymentId: "test-deployment-id" +"#; + test_utils::helm_template_and_validate(&files, Some(manager_fetch_values)) + .assert_ok("manager-fetch path / registered setup"); } #[test] diff --git a/crates/alien-helm/tests/generator/helpers.rs b/crates/alien-helm/tests/generator/helpers.rs index d6ca4bfff..1720b2347 100644 --- a/crates/alien-helm/tests/generator/helpers.rs +++ b/crates/alien-helm/tests/generator/helpers.rs @@ -37,13 +37,33 @@ pub fn snapshot_chart(name: &str, chart: &HelmChart) { insta::assert_snapshot!(name, buf); } +/// Minimal `management` block that satisfies the chart's `required` +/// guardrails on the manager-fetch path. Templates abort without these. +pub const MANAGER_FETCH_VALUES: &str = r#" +management: + url: "https://manager.example.com" + name: "test-manager" + token: "test-sync-token" + deploymentId: "test-deployment-id" +"#; + +/// Patch the chart's default `values.yaml` so the manager-fetch +/// `required` guardrails are satisfied. Used by tests that drive the +/// chart with values.yaml content as the starting point. +pub fn patch_values_with_manager_fetch(values_yaml: &str) -> String { + values_yaml + .replace(" token: \"\"", " token: \"test-sync-token\"") + .replace(" name: \"\"", " name: \"test-manager\"") + .replace(" url: \"\"", " url: \"https://manager.example.com\"") +} + /// Run `helm lint` + `helm template` + `kubeconform` against the chart /// for the default values and every generated example values file. pub fn assert_helm_valid(chart: &HelmChart, context: &str) { let files = linter_files(chart); test_utils::helm_lint(&files).assert_ok(format!("{context} helm lint")); - test_utils::helm_template_and_validate(&files, None) - .assert_ok(format!("{context} helm template default values")); + test_utils::helm_template_and_validate(&files, Some(MANAGER_FETCH_VALUES)) + .assert_ok(format!("{context} helm template manager-fetch default values")); for (path, values) in files .iter() diff --git a/crates/alien-helm/tests/generator/manager_only_tests.rs b/crates/alien-helm/tests/generator/manager_only_tests.rs index b43dbe349..bdad76182 100644 --- a/crates/alien-helm/tests/generator/manager_only_tests.rs +++ b/crates/alien-helm/tests/generator/manager_only_tests.rs @@ -3,7 +3,10 @@ //! IRSA / Workload Identity / Federated Identity. use super::{ - helpers::{assert_helm_valid, render, snapshot_chart}, + helpers::{ + assert_helm_valid, patch_values_with_manager_fetch, render, snapshot_chart, + MANAGER_FETCH_VALUES, + }, test_utils::{self, LinterStatus}, }; use alien_core::{ @@ -166,7 +169,7 @@ fn chart_role_rbac_allows_every_cleanup_route_resource() { assert!(role.contains("resources: [\"healthcheckpolicy\"]")); assert!(role.contains("eq $routeApi \"gateway\"")); - let rendered = test_utils::helm_template(&chart.files, None); + let rendered = test_utils::helm_template(&chart.files, Some(MANAGER_FETCH_VALUES)); match &rendered.status { LinterStatus::Passed => { assert!(rendered @@ -239,7 +242,7 @@ fn workload_vault_permissions_render_runtime_secret_rbac() { ); } - let rendered = test_utils::helm_template(&chart.files, None); + let rendered = test_utils::helm_template(&chart.files, Some(MANAGER_FETCH_VALUES)); match &rendered.status { LinterStatus::Passed => { assert!(rendered.stdout.contains("kind: RoleBinding")); @@ -289,10 +292,12 @@ fn gcp_base_platform_config_renders_operator_environment() { let chart = render(&stack, StackSettings::default()); let files = chart.files.clone(); let values = files.get("values.yaml").expect("values.yaml"); - let gcp_values = values - .replace("basePlatform: null", "basePlatform: gcp") - .replace("projectId: \"\"", "projectId: alien-test-target") - .replace("region: \"\"", "region: us-east4"); + let gcp_values = patch_values_with_manager_fetch( + &values + .replace("basePlatform: null", "basePlatform: gcp") + .replace("projectId: \"\"", "projectId: alien-test-target") + .replace("region: \"\"", "region: us-east4"), + ); let rendered = test_utils::helm_template(&files, Some(&gcp_values)); match &rendered.status { @@ -317,9 +322,11 @@ fn aws_base_platform_config_renders_operator_environment() { let chart = render(&stack, StackSettings::default()); let files = chart.files.clone(); let values = files.get("values.yaml").expect("values.yaml"); - let aws_values = values - .replace("basePlatform: null", "basePlatform: aws") - .replace("region: \"\"", "region: us-east-1"); + let aws_values = patch_values_with_manager_fetch( + &values + .replace("basePlatform: null", "basePlatform: aws") + .replace("region: \"\"", "region: us-east-1"), + ); let rendered = test_utils::helm_template(&files, Some(&aws_values)); match &rendered.status { @@ -348,7 +355,7 @@ fn cluster_bootstrap_renders_only_when_enabled() { assert!(values.contains("eksAutoMode:")); assert!(values.contains("arm64NodePool:")); - let default_rendered = test_utils::helm_template(&files, None); + let default_rendered = test_utils::helm_template(&files, Some(MANAGER_FETCH_VALUES)); match &default_rendered.status { LinterStatus::Passed => { assert!(!default_rendered.stdout.contains("kind: StorageClass")); @@ -362,12 +369,14 @@ fn cluster_bootstrap_renders_only_when_enabled() { } } - let enabled_values = values - .replace( - "clusterBootstrap:\n metricsServer:\n enabled: false\n image: registry.k8s.io/metrics-server/metrics-server:v0.8.1\n storageClass:\n default:\n enabled: false\n name: \"\"\n provisioner: \"\"\n parameters: {}\n ingress:\n eksAutoMode:\n enabled: false", - "clusterBootstrap:\n metricsServer:\n enabled: true\n image: registry.k8s.io/metrics-server/metrics-server:v0.8.1\n storageClass:\n default:\n enabled: true\n name: gp3\n provisioner: ebs.csi.eks.amazonaws.com\n parameters:\n type: gp3\n fsType: ext4\n encrypted: \"true\"\n ingress:\n eksAutoMode:\n enabled: true", - ) - .replace("arm64NodePool:\n enabled: false", "arm64NodePool:\n enabled: true"); + let enabled_values = patch_values_with_manager_fetch( + &values + .replace( + "clusterBootstrap:\n metricsServer:\n enabled: false\n image: registry.k8s.io/metrics-server/metrics-server:v0.8.1\n storageClass:\n default:\n enabled: false\n name: \"\"\n provisioner: \"\"\n parameters: {}\n ingress:\n eksAutoMode:\n enabled: false", + "clusterBootstrap:\n metricsServer:\n enabled: true\n image: registry.k8s.io/metrics-server/metrics-server:v0.8.1\n storageClass:\n default:\n enabled: true\n name: gp3\n provisioner: ebs.csi.eks.amazonaws.com\n parameters:\n type: gp3\n fsType: ext4\n encrypted: \"true\"\n ingress:\n eksAutoMode:\n enabled: true", + ) + .replace("arm64NodePool:\n enabled: false", "arm64NodePool:\n enabled: true"), + ); let enabled_rendered = test_utils::helm_template(&files, Some(&enabled_values)); match &enabled_rendered.status { LinterStatus::Passed => { @@ -408,7 +417,7 @@ fn uninstall_cleanup_hook_preserves_pvcs_by_default() { let chart = render(&stack, StackSettings::default()); let files = chart.files.clone(); - let rendered = test_utils::helm_template(&files, None); + let rendered = test_utils::helm_template(&files, Some(MANAGER_FETCH_VALUES)); match &rendered.status { LinterStatus::Passed => { assert!(rendered.stdout.contains("kind: Job")); @@ -449,13 +458,14 @@ fn heartbeat_collection_rbac_is_namespace_scoped_with_optional_node_reads() { assert!(cluster_role.contains(r#"resources: ["nodes"]"#)); assert!(cluster_role.contains(r#"apiGroups: ["metrics.k8s.io"]"#)); - test_utils::helm_template_and_validate(&files, None).assert_ok("heartbeat RBAC default values"); + test_utils::helm_template_and_validate(&files, Some(MANAGER_FETCH_VALUES)) + .assert_ok("heartbeat RBAC default values"); let values = files.get("values.yaml").expect("values.yaml"); - let disabled_values = values.replace( + let disabled_values = patch_values_with_manager_fetch(&values.replace( "heartbeat:\n collection:\n nodes:\n enabled: true", "heartbeat:\n collection:\n nodes:\n enabled: false", - ); + )); test_utils::helm_template_and_validate(&files, Some(&disabled_values)) .assert_ok("heartbeat RBAC node collection disabled"); diff --git a/crates/alien-helm/tests/generator/snapshots/generator__generator__helpers__data_layer.snap b/crates/alien-helm/tests/generator/snapshots/generator__generator__helpers__data_layer.snap index 843c3fbc6..fa8cc2402 100644 --- a/crates/alien-helm/tests/generator/snapshots/generator__generator__helpers__data_layer.snap +++ b/crates/alien-helm/tests/generator/snapshots/generator__generator__helpers__data_layer.snap @@ -33,12 +33,31 @@ runtime: podAnnotations: {} automountServiceAccountToken: true encryption: - # Set this explicitly, or reference an existing Secret below. - key: "replace-me-with-a-stable-64-character-encryption-secret" + # Leave empty to let the chart generate a stable random 64-hex-char key + # on first install (preserved across upgrades via `lookup`). To pin the + # key explicitly, set it here (must be 64 hex chars = 256-bit AES). To + # source it from an external secret store, set `existingSecret.name`. + key: "" existingSecret: name: "" key: encryption-key + # Agent self-update inputs the agent passes to the Helm-runner Job it + # spawns on `operator_target.helm`. Set chartRef + chartVersion to the OCI + # ref + version used at install time — the agent re-uses them in + # `helm upgrade --reuse-values`. Leave blank if you don't want to enable + # in-cluster agent upgrades for this install. + upgrade: + chartRef: "" + chartVersion: "" + helmRunnerImage: "alpine/helm:3.18.4" + # Extra flags appended to the `helm upgrade` command the agent's + # helm-runner Job runs (e.g. `--plain-http` for local-dev OCI + # registries served over HTTP). Production should leave empty. + extraArgs: "" replicas: 1 + # Helm's --atomic --wait gives up after this many seconds if /readyz + # hasn't returned 200 — the revision is then rolled back automatically. + progressDeadlineSeconds: 120 resources: requests: cpu: 100m @@ -52,16 +71,20 @@ runtime: service: type: ClusterIP probes: + # /livez is process liveness; /readyz turns 200 only after the agent + # completes at least one /v1/sync round-trip with the manager — the + # gate Helm's --atomic --wait relies on so a freshly-rolled agent + # is not considered ready until it has actually reached the manager. liveness: enabled: true - path: /health + path: /livez initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 2 failureThreshold: 3 readiness: enabled: true - path: /health + path: /readyz initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 2 @@ -86,7 +109,16 @@ runtime: data: mountPath: /var/lib/deployment-operator persistence: - enabled: false + # Enabled by default: the agent's `data_dir` holds its persistent + # deployment_id + sync-token. Without a PVC, any pod restart (e.g. + # the rolling restart triggered by self-update / `operator_target.helm`) + # wipes that state, the new pod re-runs `/v1/initialize`, hits a + # name-conflict 409, crashloops, and helm `--atomic` rolls back. + # Operators on clusters without a default StorageClass must either + # set `storageClassName`, point at an `existingClaim`, or + # explicitly disable this and accept that self-update will not + # survive a pod roll. + enabled: true existingClaim: "" storageClassName: "" accessModes: @@ -201,6 +233,13 @@ managerServiceAccount: annotations: {} labels: {} +# Operator self-update. When the operator receives operator_target.helm on +# /v1/sync it creates a short-lived Helm-runner Job that runs `helm upgrade +# --atomic`. The Job runs under the release-derived upgrader SA; keep it +# optional so charts that don't want self-update can disable it. +upgrader: + enabled: true + services: {} @@ -282,7 +321,18 @@ ephemeralStorage: } } }, + "upgrade": { + "type": "object", + "additionalProperties": false, + "properties": { + "chartRef": { "type": "string" }, + "chartVersion": { "type": "string" }, + "helmRunnerImage": { "type": "string" }, + "extraArgs": { "type": "string" } + } + }, "replicas": { "type": "integer", "minimum": 1 }, + "progressDeadlineSeconds": { "type": "integer", "minimum": 1 }, "resources": { "type": "object" }, "api": { "type": "object", @@ -639,6 +689,13 @@ ephemeralStorage: } }, "serviceAccountPrefix": { "type": "string" }, + "upgrader": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" } + } + }, "services": { "type": "object", "additionalProperties": { @@ -732,6 +789,18 @@ helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} {{- regexReplaceAll "[^a-z0-9-]" $raw "-" | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* + ServiceAccount used by the Helm-runner Job the agent creates when it + acts on operator_target.helm. Held as a least-privilege boundary; bound + to the existing Role so the Job can mutate the Deployment + release + Secrets. +*/}} +{{- define "deployment.upgraderServiceAccountName" -}} +{{- $prefix := default (include "deployment.fullname" .) .Values.serviceAccountPrefix -}} +{{- $raw := printf "%s-upgrader-sa" $prefix | lower -}} +{{- regexReplaceAll "[^a-z0-9-]" $raw "-" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{- define "deployment.serviceAccountName" -}} {{- $prefix := default (include "deployment.fullname" .root) .root.Values.serviceAccountPrefix -}} {{- $raw := printf "%s-%s-sa" $prefix .name | lower -}} @@ -763,6 +832,14 @@ helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} {{- printf "%s-heartbeat-nodes" (include "deployment.fullname" .) | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{- /* Name of the ClusterRole that grants the agent self-update Job permission + to manage the chart-owned cluster-scoped resources (currently just the + heartbeat ClusterRole+Binding). Only created when both `upgrader.enabled` + and the heartbeat node-collection feature are on. */ -}} +{{- define "deployment.upgraderClusterRoleName" -}} +{{- printf "%s-upgrader" (include "deployment.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + === templates/serviceaccount.yaml === apiVersion: v1 kind: ServiceAccount @@ -794,6 +871,22 @@ metadata: {{- end }} --- {{- end }} +{{- if .Values.upgrader.enabled }} +# The upgrader ServiceAccount (release-derived, see +# `deployment.upgraderServiceAccountName`) is used by the Helm-runner Job the +# operator creates when it acts on operator_target.helm. It exists as a +# least-privilege boundary for the Job — the operator pod itself uses its own +# manager SA and only needs to create Jobs + stage ConfigMaps/Secrets. The +# protection against bad helm upgrades is the chart's `required` values, not +# RBAC. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "deployment.upgraderServiceAccountName" . }} + labels: + {{- include "deployment.labels" . | nindent 4 }} +--- +{{- end }} === templates/role.yaml === {{- $stackSettings := default dict .Values.stackSettings -}} @@ -809,6 +902,17 @@ rules: - apiGroups: [""] resources: ["configmaps", "secrets", "services", "pods", "pods/log", "persistentvolumeclaims"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + # ServiceAccounts + RBAC objects need to live here for `helm upgrade + # --reuse-values` to inspect (and patch) the release's existing + # resources during agent self-update. Without this, the upgrader SA + # — which is bound to this Role — can't `get` the SAs the chart + # already created and helm 4xx's out. + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch"] @@ -851,6 +955,18 @@ metadata: subjects: - kind: ServiceAccount name: {{ include "deployment.managerServiceAccountName" . }} + {{- /* Execution ServiceAccounts (workers/daemons/containers) need RBAC to + read their own vault secrets (e.g. the injected ALIEN_COMMANDS_TOKEN). + Without this binding, the worker pod gets 403 reading any secret + provisioned by the KubernetesVaultController. */}} + {{- range $name, $account := .Values.serviceAccounts }} + - kind: ServiceAccount + name: {{ include "deployment.serviceAccountName" (dict "root" $ "name" $name) }} + {{- end }} + {{- if .Values.upgrader.enabled }} + - kind: ServiceAccount + name: {{ include "deployment.upgraderServiceAccountName" . }} + {{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -902,6 +1018,40 @@ rules: - apiGroups: ["metrics.k8s.io"] resources: ["nodes"] verbs: ["get", "list", "watch"] +{{- if .Values.upgrader.enabled }} +--- +# Narrow cluster-scoped RBAC for the agent self-update helm-runner Job. +# The chart creates exactly one cluster-scoped resource type pair — +# the heartbeat ClusterRole + ClusterRoleBinding above — and the +# upgrader SA needs to be able to `get/update/patch/delete` them +# during `helm upgrade --reuse-values`. `resourceNames` scopes this +# to ONLY the chart's own cluster objects; no enumeration of other +# tenants' cluster resources. Verbs are deliberately minimal (no +# `list`/`watch`, no `create` — chart install is what creates them +# the first time, run by the customer's helm operator). If a future +# chart version introduces new cluster-scoped resources, add their +# names to `resourceNames` (and a `create` verb on a separate rule +# without `resourceNames` if the upgrader needs to add new ones). +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "deployment.upgraderClusterRoleName" . }} + labels: + {{- include "deployment.labels" . | nindent 4 }} +rules: + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles", "clusterrolebindings"] + resourceNames: + # The heartbeat-nodes cluster pair — the existing reason the + # upgrader needs cluster-scope access at all. + - {{ include "deployment.heartbeatNodeClusterRoleName" . | quote }} + # And the upgrader cluster pair itself — helm now tracks these as + # chart-owned, so every `helm upgrade --reuse-values` does a `get` + # on them to compute the diff. Without self-reference, the first + # upgrade trips on `clusterroles "-upgrader" is forbidden`. + - {{ include "deployment.upgraderClusterRoleName" . | quote }} + verbs: ["get", "update", "patch", "delete"] +{{- end }} {{- end }} === templates/clusterrolebinding.yaml === @@ -921,11 +1071,47 @@ roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: {{ include "deployment.heartbeatNodeClusterRoleName" . }} +{{- if .Values.upgrader.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "deployment.upgraderClusterRoleName" . }} + labels: + {{- include "deployment.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "deployment.upgraderServiceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "deployment.upgraderClusterRoleName" . }} +{{- end }} {{- end }} === templates/secret.yaml === {{- $createManagementSecret := not .Values.management.existingSecret.name -}} {{- $createEncryptionSecret := not .Values.runtime.encryption.existingSecret.name -}} +{{- $encryptionKey := "" -}} +{{- if $createEncryptionSecret -}} + {{- $providedKey := .Values.runtime.encryption.key | default "" | trim -}} + {{- $existingKey := "" -}} + {{- $existingSecret := lookup "v1" "Secret" .Release.Namespace (include "deployment.fullname" .) -}} + {{- if and $existingSecret $existingSecret.data -}} + {{- with index $existingSecret.data "encryption-key" -}} + {{- $existingKey = b64dec . -}} + {{- end -}} + {{- end -}} + {{- if $providedKey -}} + {{- $encryptionKey = $providedKey -}} + {{- else if $existingKey -}} + {{- $encryptionKey = $existingKey -}} + {{- else -}} + {{- /* sprig randBytes returns base64; b64dec to raw bytes then hex */ -}} + {{- $encryptionKey = printf "%x" (b64dec (randBytes 32)) -}} + {{- end -}} +{{- end -}} {{- if or $createManagementSecret $createEncryptionSecret .Values.infrastructure .Values.logCollector.enabled }} apiVersion: v1 kind: Secret @@ -936,10 +1122,10 @@ metadata: type: Opaque stringData: {{- if $createManagementSecret }} - sync-token: {{ .Values.management.token | quote }} + sync-token: {{ required "management.token or management.existingSecret.name is required — pass the full values document" .Values.management.token | quote }} {{- end }} {{- if $createEncryptionSecret }} - encryption-key: {{ required "runtime.encryption.key or runtime.encryption.existingSecret.name is required" .Values.runtime.encryption.key | quote }} + encryption-key: {{ $encryptionKey | quote }} {{- end }} {{- if .Values.infrastructure }} external-bindings.json: {{ toJson .Values.infrastructure | quote }} @@ -973,6 +1159,11 @@ metadata: {{- include "deployment.labels" . | nindent 4 }} spec: replicas: {{ .Values.runtime.replicas }} + # Recreate guarantees exactly one agent runs at any time, so the + # InstanceLock is never contended — even during a self-update. + strategy: + type: Recreate + progressDeadlineSeconds: {{ .Values.runtime.progressDeadlineSeconds | default 120 }} selector: matchLabels: app.kubernetes.io/name: {{ include "deployment.name" . }} @@ -1058,16 +1249,62 @@ spec: - name: AZURE_REGION value: {{ .Values.basePlatformConfig.azure.location | quote }} {{- end }} + # `required` chart guardrail: any helm upgrade that does not + # carry the full values document fails to render (Helm aborts + # before touching the release). This is the protection against + # bare `helm upgrade` silently resetting the agent's manager + # config — manager-triggered, operator-triggered, or otherwise. - name: SYNC_URL - value: {{ .Values.management.url | quote }} + value: {{ required "management.url is required — pass the full values document" .Values.management.url | quote }} - name: OPERATOR_NAME - value: {{ .Values.management.name | quote }} + value: {{ required "management.name is required — pass the full values document" .Values.management.name | quote }} {{- if .Values.management.deploymentId }} - name: DEPLOYMENT_ID value: {{ .Values.management.deploymentId | quote }} {{- end }} - name: KUBERNETES_NAMESPACE value: {{ .Release.Namespace | quote }} + - name: KUBERNETES_HELM_RELEASE + value: {{ .Release.Name | quote }} + - name: ALIEN_OPERATOR_UPGRADER_SA + value: {{ include "deployment.upgraderServiceAccountName" . | quote }} + # Reported back on /v1/sync so the dashboard can surface the + # registry an admin will pull a new tag from when pinning a + # target operator version. + - name: ALIEN_OPERATOR_IMAGE_REPOSITORY + value: {{ .Values.runtime.image.repository | quote }} + {{- if .Values.runtime.upgrade.chartRef }} + # Used by the operator when it receives `operator_target.helm` and + # spawns a Helm-runner Job to apply the new version. The Job + # runs `helm upgrade --reuse-values` against this chart so only + # the manager-supplied `values` override (e.g. image.tag) flips. + - name: ALIEN_OPERATOR_CHART_REF + value: {{ .Values.runtime.upgrade.chartRef | quote }} + {{- end }} + {{- if .Values.runtime.upgrade.chartVersion }} + - name: ALIEN_OPERATOR_CHART_VERSION + value: {{ .Values.runtime.upgrade.chartVersion | quote }} + {{- end }} + {{- if .Values.runtime.upgrade.helmRunnerImage }} + - name: ALIEN_OPERATOR_HELM_RUNNER_IMAGE + value: {{ .Values.runtime.upgrade.helmRunnerImage | quote }} + {{- end }} + {{- if .Values.runtime.upgrade.extraArgs }} + # Extra flags spliced into the `helm upgrade` command the + # operator's helm-runner Job runs. Use sparingly — exists for + # local-dev/insecure OCI registries (`--plain-http`) and + # similar one-off escape hatches; production should leave empty. + - name: ALIEN_OPERATOR_HELM_EXTRA_ARGS + value: {{ .Values.runtime.upgrade.extraArgs | quote }} + {{- end }} + {{- if .Values.serviceAccountPrefix }} + # Pin the deployment's resource_prefix to the same value used for + # ServiceAccount naming, so Helm-created SAs and vault secret names + # stay aligned across operator restarts (pull-model storage is ephemeral + # and the operator would otherwise regenerate a random prefix each time). + - name: ALIEN_RESOURCE_PREFIX + value: {{ .Values.serviceAccountPrefix | quote }} + {{- end }} - name: OPERATOR_SETUP_METHOD value: "helm" - name: DATA_DIR diff --git a/crates/alien-helm/tests/generator/snapshots/generator__generator__helpers__manager_only_pure_worker.snap b/crates/alien-helm/tests/generator/snapshots/generator__generator__helpers__manager_only_pure_worker.snap index a9057c6c3..312fcbd61 100644 --- a/crates/alien-helm/tests/generator/snapshots/generator__generator__helpers__manager_only_pure_worker.snap +++ b/crates/alien-helm/tests/generator/snapshots/generator__generator__helpers__manager_only_pure_worker.snap @@ -33,12 +33,31 @@ runtime: podAnnotations: {} automountServiceAccountToken: true encryption: - # Set this explicitly, or reference an existing Secret below. - key: "replace-me-with-a-stable-64-character-encryption-secret" + # Leave empty to let the chart generate a stable random 64-hex-char key + # on first install (preserved across upgrades via `lookup`). To pin the + # key explicitly, set it here (must be 64 hex chars = 256-bit AES). To + # source it from an external secret store, set `existingSecret.name`. + key: "" existingSecret: name: "" key: encryption-key + # Agent self-update inputs the agent passes to the Helm-runner Job it + # spawns on `operator_target.helm`. Set chartRef + chartVersion to the OCI + # ref + version used at install time — the agent re-uses them in + # `helm upgrade --reuse-values`. Leave blank if you don't want to enable + # in-cluster agent upgrades for this install. + upgrade: + chartRef: "" + chartVersion: "" + helmRunnerImage: "alpine/helm:3.18.4" + # Extra flags appended to the `helm upgrade` command the agent's + # helm-runner Job runs (e.g. `--plain-http` for local-dev OCI + # registries served over HTTP). Production should leave empty. + extraArgs: "" replicas: 1 + # Helm's --atomic --wait gives up after this many seconds if /readyz + # hasn't returned 200 — the revision is then rolled back automatically. + progressDeadlineSeconds: 120 resources: requests: cpu: 100m @@ -52,16 +71,20 @@ runtime: service: type: ClusterIP probes: + # /livez is process liveness; /readyz turns 200 only after the agent + # completes at least one /v1/sync round-trip with the manager — the + # gate Helm's --atomic --wait relies on so a freshly-rolled agent + # is not considered ready until it has actually reached the manager. liveness: enabled: true - path: /health + path: /livez initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 2 failureThreshold: 3 readiness: enabled: true - path: /health + path: /readyz initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 2 @@ -86,7 +109,16 @@ runtime: data: mountPath: /var/lib/deployment-operator persistence: - enabled: false + # Enabled by default: the agent's `data_dir` holds its persistent + # deployment_id + sync-token. Without a PVC, any pod restart (e.g. + # the rolling restart triggered by self-update / `operator_target.helm`) + # wipes that state, the new pod re-runs `/v1/initialize`, hits a + # name-conflict 409, crashloops, and helm `--atomic` rolls back. + # Operators on clusters without a default StorageClass must either + # set `storageClassName`, point at an `existingClaim`, or + # explicitly disable this and accept that self-update will not + # survive a pod roll. + enabled: true existingClaim: "" storageClassName: "" accessModes: @@ -203,6 +235,13 @@ managerServiceAccount: annotations: {} labels: {} +# Operator self-update. When the operator receives operator_target.helm on +# /v1/sync it creates a short-lived Helm-runner Job that runs `helm upgrade +# --atomic`. The Job runs under the release-derived upgrader SA; keep it +# optional so charts that don't want self-update can disable it. +upgrader: + enabled: true + services: api: type: clusterIp @@ -288,7 +327,18 @@ ephemeralStorage: } } }, + "upgrade": { + "type": "object", + "additionalProperties": false, + "properties": { + "chartRef": { "type": "string" }, + "chartVersion": { "type": "string" }, + "helmRunnerImage": { "type": "string" }, + "extraArgs": { "type": "string" } + } + }, "replicas": { "type": "integer", "minimum": 1 }, + "progressDeadlineSeconds": { "type": "integer", "minimum": 1 }, "resources": { "type": "object" }, "api": { "type": "object", @@ -645,6 +695,13 @@ ephemeralStorage: } }, "serviceAccountPrefix": { "type": "string" }, + "upgrader": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" } + } + }, "services": { "type": "object", "additionalProperties": { @@ -738,6 +795,18 @@ helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} {{- regexReplaceAll "[^a-z0-9-]" $raw "-" | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* + ServiceAccount used by the Helm-runner Job the agent creates when it + acts on operator_target.helm. Held as a least-privilege boundary; bound + to the existing Role so the Job can mutate the Deployment + release + Secrets. +*/}} +{{- define "deployment.upgraderServiceAccountName" -}} +{{- $prefix := default (include "deployment.fullname" .) .Values.serviceAccountPrefix -}} +{{- $raw := printf "%s-upgrader-sa" $prefix | lower -}} +{{- regexReplaceAll "[^a-z0-9-]" $raw "-" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{- define "deployment.serviceAccountName" -}} {{- $prefix := default (include "deployment.fullname" .root) .root.Values.serviceAccountPrefix -}} {{- $raw := printf "%s-%s-sa" $prefix .name | lower -}} @@ -769,6 +838,14 @@ helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} {{- printf "%s-heartbeat-nodes" (include "deployment.fullname" .) | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{- /* Name of the ClusterRole that grants the agent self-update Job permission + to manage the chart-owned cluster-scoped resources (currently just the + heartbeat ClusterRole+Binding). Only created when both `upgrader.enabled` + and the heartbeat node-collection feature are on. */ -}} +{{- define "deployment.upgraderClusterRoleName" -}} +{{- printf "%s-upgrader" (include "deployment.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + === templates/serviceaccount.yaml === apiVersion: v1 kind: ServiceAccount @@ -800,6 +877,22 @@ metadata: {{- end }} --- {{- end }} +{{- if .Values.upgrader.enabled }} +# The upgrader ServiceAccount (release-derived, see +# `deployment.upgraderServiceAccountName`) is used by the Helm-runner Job the +# operator creates when it acts on operator_target.helm. It exists as a +# least-privilege boundary for the Job — the operator pod itself uses its own +# manager SA and only needs to create Jobs + stage ConfigMaps/Secrets. The +# protection against bad helm upgrades is the chart's `required` values, not +# RBAC. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "deployment.upgraderServiceAccountName" . }} + labels: + {{- include "deployment.labels" . | nindent 4 }} +--- +{{- end }} === templates/role.yaml === {{- $stackSettings := default dict .Values.stackSettings -}} @@ -815,6 +908,17 @@ rules: - apiGroups: [""] resources: ["configmaps", "secrets", "services", "pods", "pods/log", "persistentvolumeclaims"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + # ServiceAccounts + RBAC objects need to live here for `helm upgrade + # --reuse-values` to inspect (and patch) the release's existing + # resources during agent self-update. Without this, the upgrader SA + # — which is bound to this Role — can't `get` the SAs the chart + # already created and helm 4xx's out. + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch"] @@ -857,6 +961,18 @@ metadata: subjects: - kind: ServiceAccount name: {{ include "deployment.managerServiceAccountName" . }} + {{- /* Execution ServiceAccounts (workers/daemons/containers) need RBAC to + read their own vault secrets (e.g. the injected ALIEN_COMMANDS_TOKEN). + Without this binding, the worker pod gets 403 reading any secret + provisioned by the KubernetesVaultController. */}} + {{- range $name, $account := .Values.serviceAccounts }} + - kind: ServiceAccount + name: {{ include "deployment.serviceAccountName" (dict "root" $ "name" $name) }} + {{- end }} + {{- if .Values.upgrader.enabled }} + - kind: ServiceAccount + name: {{ include "deployment.upgraderServiceAccountName" . }} + {{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -908,6 +1024,40 @@ rules: - apiGroups: ["metrics.k8s.io"] resources: ["nodes"] verbs: ["get", "list", "watch"] +{{- if .Values.upgrader.enabled }} +--- +# Narrow cluster-scoped RBAC for the agent self-update helm-runner Job. +# The chart creates exactly one cluster-scoped resource type pair — +# the heartbeat ClusterRole + ClusterRoleBinding above — and the +# upgrader SA needs to be able to `get/update/patch/delete` them +# during `helm upgrade --reuse-values`. `resourceNames` scopes this +# to ONLY the chart's own cluster objects; no enumeration of other +# tenants' cluster resources. Verbs are deliberately minimal (no +# `list`/`watch`, no `create` — chart install is what creates them +# the first time, run by the customer's helm operator). If a future +# chart version introduces new cluster-scoped resources, add their +# names to `resourceNames` (and a `create` verb on a separate rule +# without `resourceNames` if the upgrader needs to add new ones). +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "deployment.upgraderClusterRoleName" . }} + labels: + {{- include "deployment.labels" . | nindent 4 }} +rules: + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles", "clusterrolebindings"] + resourceNames: + # The heartbeat-nodes cluster pair — the existing reason the + # upgrader needs cluster-scope access at all. + - {{ include "deployment.heartbeatNodeClusterRoleName" . | quote }} + # And the upgrader cluster pair itself — helm now tracks these as + # chart-owned, so every `helm upgrade --reuse-values` does a `get` + # on them to compute the diff. Without self-reference, the first + # upgrade trips on `clusterroles "-upgrader" is forbidden`. + - {{ include "deployment.upgraderClusterRoleName" . | quote }} + verbs: ["get", "update", "patch", "delete"] +{{- end }} {{- end }} === templates/clusterrolebinding.yaml === @@ -927,11 +1077,47 @@ roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: {{ include "deployment.heartbeatNodeClusterRoleName" . }} +{{- if .Values.upgrader.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "deployment.upgraderClusterRoleName" . }} + labels: + {{- include "deployment.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "deployment.upgraderServiceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "deployment.upgraderClusterRoleName" . }} +{{- end }} {{- end }} === templates/secret.yaml === {{- $createManagementSecret := not .Values.management.existingSecret.name -}} {{- $createEncryptionSecret := not .Values.runtime.encryption.existingSecret.name -}} +{{- $encryptionKey := "" -}} +{{- if $createEncryptionSecret -}} + {{- $providedKey := .Values.runtime.encryption.key | default "" | trim -}} + {{- $existingKey := "" -}} + {{- $existingSecret := lookup "v1" "Secret" .Release.Namespace (include "deployment.fullname" .) -}} + {{- if and $existingSecret $existingSecret.data -}} + {{- with index $existingSecret.data "encryption-key" -}} + {{- $existingKey = b64dec . -}} + {{- end -}} + {{- end -}} + {{- if $providedKey -}} + {{- $encryptionKey = $providedKey -}} + {{- else if $existingKey -}} + {{- $encryptionKey = $existingKey -}} + {{- else -}} + {{- /* sprig randBytes returns base64; b64dec to raw bytes then hex */ -}} + {{- $encryptionKey = printf "%x" (b64dec (randBytes 32)) -}} + {{- end -}} +{{- end -}} {{- if or $createManagementSecret $createEncryptionSecret .Values.infrastructure .Values.logCollector.enabled }} apiVersion: v1 kind: Secret @@ -942,10 +1128,10 @@ metadata: type: Opaque stringData: {{- if $createManagementSecret }} - sync-token: {{ .Values.management.token | quote }} + sync-token: {{ required "management.token or management.existingSecret.name is required — pass the full values document" .Values.management.token | quote }} {{- end }} {{- if $createEncryptionSecret }} - encryption-key: {{ required "runtime.encryption.key or runtime.encryption.existingSecret.name is required" .Values.runtime.encryption.key | quote }} + encryption-key: {{ $encryptionKey | quote }} {{- end }} {{- if .Values.infrastructure }} external-bindings.json: {{ toJson .Values.infrastructure | quote }} @@ -979,6 +1165,11 @@ metadata: {{- include "deployment.labels" . | nindent 4 }} spec: replicas: {{ .Values.runtime.replicas }} + # Recreate guarantees exactly one agent runs at any time, so the + # InstanceLock is never contended — even during a self-update. + strategy: + type: Recreate + progressDeadlineSeconds: {{ .Values.runtime.progressDeadlineSeconds | default 120 }} selector: matchLabels: app.kubernetes.io/name: {{ include "deployment.name" . }} @@ -1064,16 +1255,62 @@ spec: - name: AZURE_REGION value: {{ .Values.basePlatformConfig.azure.location | quote }} {{- end }} + # `required` chart guardrail: any helm upgrade that does not + # carry the full values document fails to render (Helm aborts + # before touching the release). This is the protection against + # bare `helm upgrade` silently resetting the agent's manager + # config — manager-triggered, operator-triggered, or otherwise. - name: SYNC_URL - value: {{ .Values.management.url | quote }} + value: {{ required "management.url is required — pass the full values document" .Values.management.url | quote }} - name: OPERATOR_NAME - value: {{ .Values.management.name | quote }} + value: {{ required "management.name is required — pass the full values document" .Values.management.name | quote }} {{- if .Values.management.deploymentId }} - name: DEPLOYMENT_ID value: {{ .Values.management.deploymentId | quote }} {{- end }} - name: KUBERNETES_NAMESPACE value: {{ .Release.Namespace | quote }} + - name: KUBERNETES_HELM_RELEASE + value: {{ .Release.Name | quote }} + - name: ALIEN_OPERATOR_UPGRADER_SA + value: {{ include "deployment.upgraderServiceAccountName" . | quote }} + # Reported back on /v1/sync so the dashboard can surface the + # registry an admin will pull a new tag from when pinning a + # target operator version. + - name: ALIEN_OPERATOR_IMAGE_REPOSITORY + value: {{ .Values.runtime.image.repository | quote }} + {{- if .Values.runtime.upgrade.chartRef }} + # Used by the operator when it receives `operator_target.helm` and + # spawns a Helm-runner Job to apply the new version. The Job + # runs `helm upgrade --reuse-values` against this chart so only + # the manager-supplied `values` override (e.g. image.tag) flips. + - name: ALIEN_OPERATOR_CHART_REF + value: {{ .Values.runtime.upgrade.chartRef | quote }} + {{- end }} + {{- if .Values.runtime.upgrade.chartVersion }} + - name: ALIEN_OPERATOR_CHART_VERSION + value: {{ .Values.runtime.upgrade.chartVersion | quote }} + {{- end }} + {{- if .Values.runtime.upgrade.helmRunnerImage }} + - name: ALIEN_OPERATOR_HELM_RUNNER_IMAGE + value: {{ .Values.runtime.upgrade.helmRunnerImage | quote }} + {{- end }} + {{- if .Values.runtime.upgrade.extraArgs }} + # Extra flags spliced into the `helm upgrade` command the + # operator's helm-runner Job runs. Use sparingly — exists for + # local-dev/insecure OCI registries (`--plain-http`) and + # similar one-off escape hatches; production should leave empty. + - name: ALIEN_OPERATOR_HELM_EXTRA_ARGS + value: {{ .Values.runtime.upgrade.extraArgs | quote }} + {{- end }} + {{- if .Values.serviceAccountPrefix }} + # Pin the deployment's resource_prefix to the same value used for + # ServiceAccount naming, so Helm-created SAs and vault secret names + # stay aligned across operator restarts (pull-model storage is ephemeral + # and the operator would otherwise regenerate a random prefix each time). + - name: ALIEN_RESOURCE_PREFIX + value: {{ .Values.serviceAccountPrefix | quote }} + {{- end }} - name: OPERATOR_SETUP_METHOD value: "helm" - name: DATA_DIR diff --git a/crates/alien-infra/src/core/executor.rs b/crates/alien-infra/src/core/executor.rs index 55bc5451b..f2b7a18ab 100644 --- a/crates/alien-infra/src/core/executor.rs +++ b/crates/alien-infra/src/core/executor.rs @@ -384,6 +384,17 @@ impl StackExecutor { let resource_type = resource_entry.config.resource_type(); let controller_platform = controller_platform_for_entry(platform, base_platform, resource_entry.lifecycle); + + // Kubernetes ServiceAccounts are created by the Helm chart (with cloud + // identity annotations); worker/daemon/container controllers reference + // them by name. The agent does not provision them, and there is no k8s + // ServiceAccount controller, so skip the controller requirement here. + if matches!(controller_platform, Platform::Kubernetes) + && resource_type.0.as_ref() == "service-account" + { + continue; + } + resource_registry .get_controller(resource_type.clone(), controller_platform) .context(ErrorData::ControllerNotAvailable { @@ -516,6 +527,26 @@ impl StackExecutor { self.deployment_config.external_bindings.has(resource_id) } + /// Resources the platform provides externally, so the agent neither owns a + /// controller for them nor provisions them: Kubernetes ServiceAccounts, + /// which the Helm chart creates (worker/daemon/container controllers + /// reference them by name). Treated like external bindings throughout — + /// short-circuited to Running and exempt from the controller requirement. + fn is_platform_provided_resource(&self, resource_id: &str, platform: Platform) -> bool { + let Some(entry) = self.desired_stack.resources.get(resource_id) else { + return false; + }; + if entry.config.resource_type().0.as_ref() != "service-account" { + return false; + } + let controller_platform = controller_platform_for_entry( + platform, + self.deployment_config.base_platform, + entry.lifecycle, + ); + matches!(controller_platform, Platform::Kubernetes) + } + fn resource_lifecycle( &self, resource_id: &str, @@ -1088,6 +1119,30 @@ impl StackExecutor { continue; } + // Platform-provided resources (Kubernetes ServiceAccounts, created by + // the Helm chart) have no agent controller. Mark them Running, like an + // external binding, so dependents unblock and initial setup completes. + if self.is_platform_provided_resource(resource_id, next_state.platform) { + info!( + "Platform-provided resource '{}' (Helm-managed) -> Running", + resource_id + ); + let mut resource_state = StackResourceState::new_pending( + desired_config.resource.resource_type().to_string(), + desired_config.resource.clone(), + resource_lifecycle, + desired_config.dependencies.clone(), + ); + resource_state.status = ResourceStatus::Running; + resource_state.controller_platform = Some(controller_platform_for_entry( + next_state.platform, + self.deployment_config.base_platform, + desired_config.lifecycle, + )); + initial_transitions.insert(resource_id.clone(), resource_state); + continue; + } + debug!("Preparing CREATE for '{}' -> Pending", resource_id); let pending_view = StackResourceState::new_pending( desired_config.resource.resource_type().to_string(), @@ -1505,10 +1560,12 @@ impl StackExecutor { // We don't need to get a separate controller since the controller is stored // in the internal_state and handles its own stepping if !current_resource_state.has_internal_state() { - if self.is_external_binding_resource(&resource_id) { + if self.is_external_binding_resource(&resource_id) + || self.is_platform_provided_resource(&resource_id, next_state.platform) + { debug!( resource_id = %resource_id, - "External binding resource has no controller state; skipping step" + "Externally-provided resource has no controller state; skipping step" ); } else { warn!( diff --git a/crates/alien-launcher/Cargo.toml b/crates/alien-launcher/Cargo.toml new file mode 100644 index 000000000..ca4d8f850 --- /dev/null +++ b/crates/alien-launcher/Cargo.toml @@ -0,0 +1,58 @@ +[package] +name = "alien-launcher" +version.workspace = true +edition = "2021" +license-file.workspace = true +publish = false +description = "Supervisor for the alien-operator OS-service packaging — health-gated version swaps with last-stable rollback" + +[[bin]] +name = "alien-launcher" +path = "src/main.rs" + +[dependencies] +# Alien crates +alien-core = { workspace = true } +alien-error = { workspace = true } + +# OS-agnostic core dependencies ONLY. The launcher is deliberately synchronous +# (no tokio): it supervises one child and polls one endpoint, and the smallest +# possible dependency tree is a design goal for the binary that must never break. +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +semver = { workspace = true, features = ["serde"] } +ureq = { workspace = true } +sha2 = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +tracing = { workspace = true } +# stderr logging for the supervisor (journald/launchd capture stderr). +tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } + +[dev-dependencies] +tempfile = { workspace = true } + +# Platform shims are target-gated so they are physically unbuildable from +# `src/core/` on the wrong OS. Populated as each shim lands. +[target.'cfg(unix)'.dependencies] +nix = { workspace = true } +command-group = { workspace = true } +signal-hook = { workspace = true } + +[target.'cfg(target_os = "linux")'.dependencies] +sd-notify = { workspace = true } + +[target.'cfg(windows)'.dependencies] +# SCM service host (T3.1) + Job-Object child supervisor (T3.2) + +# junction-pointer version store (T3.4). +windows-service = { workspace = true } +windows-sys = { workspace = true, features = [ + "Win32_Foundation", + "Win32_System_Console", + "Win32_System_Threading", + # T3.4 store: MoveFileExW (reboot-delete) + GetDiskFreeSpaceExW (preflight). + "Win32_Storage_FileSystem", +] } +# Job Object (kill-on-close) child supervisor — same crate the Unix shim uses. +command-group = { workspace = true } +# Directory-junction pointers (`current` / `last-stable`) for the version store. +junction = { workspace = true } diff --git a/crates/alien-launcher/src/core/health.rs b/crates/alien-launcher/src/core/health.rs new file mode 100644 index 000000000..d2a394f52 --- /dev/null +++ b/crates/alien-launcher/src/core/health.rs @@ -0,0 +1,153 @@ +//! The readiness-gate client: a blocking GET against the operator's local +//! `/readyz` endpoint, polled by the state machine during probation. +//! +//! Deliberately minimal: one request, one timeout, no retries — the state +//! machine's gate loop IS the retry policy. Anything that is not a clean +//! `200` (connection refused, timeout, 5xx, transport error) is simply +//! "not ready yet". + +// Skeleton staging: constructed by the CLI wiring in the Linux phase; +// consumed by tests until then. +#![allow(dead_code)] + +use std::time::Duration; + +use super::traits::HealthProbe; + +/// Default per-request timeout. Well under the probe interval × 2 so a +/// hanging endpoint can never starve the gate loop. +pub const DEFAULT_PROBE_TIMEOUT: Duration = Duration::from_secs(2); + +/// Blocking `HealthProbe` over `ureq` — tiny, synchronous, no async runtime, +/// matching the launcher's deliberately-synchronous design. Shared by all +/// three OSes. +pub struct UreqProbe { + agent: ureq::Agent, +} + +impl UreqProbe { + /// A probe with an explicit per-request timeout (covers connect + read). + pub fn new(timeout: Duration) -> Self { + let config = ureq::Agent::config_builder() + .timeout_global(Some(timeout)) + // 4xx/5xx are readiness answers, not transport errors — we want + // to see the status, not an Err. + .http_status_as_error(false) + .build(); + Self { + agent: config.into(), + } + } +} + +impl Default for UreqProbe { + fn default() -> Self { + Self::new(DEFAULT_PROBE_TIMEOUT) + } +} + +impl HealthProbe for UreqProbe { + fn is_ready(&self, url: &str) -> bool { + match self.agent.get(url).call() { + Ok(response) => response.status().as_u16() == 200, + // Refused / timed out / reset / malformed — all mean "not ready + // yet" during probation; the gate keeps polling. + Err(_) => false, + } + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::time::Instant; + + /// One-shot HTTP responder on 127.0.0.1: reads the request head, writes + /// `response`, closes. Returns the URL to probe. + fn one_shot_server(response: &'static str) -> String { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind should succeed"); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("accept should succeed"); + let mut buf = [0u8; 1024]; + let _ = stream.read(&mut buf); // consume the request head + stream + .write_all(response.as_bytes()) + .expect("response write should succeed"); + }); + format!("http://{addr}/readyz") + } + + #[test] + fn ready_on_200() { + let url = one_shot_server("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok"); + assert!(UreqProbe::default().is_ready(&url)); + } + + #[test] + fn not_ready_on_503() { + let url = + one_shot_server("HTTP/1.1 503 Service Unavailable\r\ncontent-length: 0\r\n\r\n"); + assert!(!UreqProbe::default().is_ready(&url)); + } + + #[test] + fn not_ready_on_connection_refused() { + // Bind to learn a free port, then drop the listener so the connect + // is refused. + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + drop(listener); + + let started = Instant::now(); + assert!(!UreqProbe::default().is_ready(&format!("http://{addr}/readyz"))); + // A refused connection resolves to "not ready" within the probe timeout. + // On Unix the OS RSTs immediately (near-instant); on Windows a connect to + // a just-released loopback port can instead run to the connect timeout + // before failing — still not-ready, just not instant. Assert the probe's + // actual contract (bounded by `timeout_global`), not the OS's fast-fail + // optimization, so this holds cross-platform while still catching a + // never-returns regression. + assert!( + started.elapsed() < DEFAULT_PROBE_TIMEOUT + Duration::from_secs(1), + "refused connection must resolve within the probe timeout, took {:?}", + started.elapsed() + ); + } + + /// A hanging endpoint (accepts, never responds) returns false within the + /// timeout — the done-when bound is ≤ 3 s with the 2 s default. + #[test] + fn not_ready_on_hang_within_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + let (stream, _) = listener.accept().expect("accept should succeed"); + // Hold the socket open, never respond. + std::thread::sleep(Duration::from_secs(5)); + drop(stream); + }); + + let started = Instant::now(); + let ready = UreqProbe::default().is_ready(&format!("http://{addr}/readyz")); + let elapsed = started.elapsed(); + assert!(!ready, "a hanging endpoint is not ready"); + assert!( + elapsed <= Duration::from_secs(3), + "probe must give up within 3s on a hang, took {elapsed:?}" + ); + } + + /// A garbage response (not HTTP) is "not ready", never a panic. + #[test] + fn not_ready_on_malformed_response() { + let url = one_shot_server("definitely not http\r\n\r\n"); + assert!(!UreqProbe::default().is_ready(&url)); + } +} diff --git a/crates/alien-launcher/src/core/mod.rs b/crates/alien-launcher/src/core/mod.rs new file mode 100644 index 000000000..b87f447b0 --- /dev/null +++ b/crates/alien-launcher/src/core/mod.rs @@ -0,0 +1,158 @@ +//! OS-agnostic core of the launcher: the update state machine, the health +//! gate, failure reporting, and the trait boundary the platform shims +//! implement. +//! +//! HARD RULE — this module tree is platform-blind. Nothing under `src/core/` +//! may import platform modules or crates, name a syscall or a signal, or know +//! whether a version pointer is a symlink or a junction. All platform behavior +//! goes through `traits` and lives in `crate::platform`. The rule is enforced +//! mechanically by the `tests/platform_blind.rs` integration test, which scans +//! this directory's sources for forbidden tokens. + +pub mod health; +pub mod report; +pub mod state_machine; +pub mod store_common; +pub mod traits; + +#[cfg(test)] +pub mod testing; + +use std::sync::mpsc::{self, RecvTimeoutError}; +use std::time::Duration; + +use crate::error::Result; +use state_machine::{classify_startup, LoopExit, Machine, RunConfig}; +use traits::{ChildSupervisor, HealthProbe, ServiceHost, VersionStore}; + +/// The launcher entry point: classify the store, execute the startup action, +/// then supervise — with a dedicated watchdog-heartbeat thread that ticks for +/// the launcher's whole life (including through probation windows longer than +/// the watchdog interval; see the rule on `ServiceHost::heartbeat`). +/// +/// Store errors propagate out: the OS init respawns the launcher and startup +/// classification recovers from whatever the crash left on disk. +// Wired to the CLI in the Linux phase; consumed by tests until then. +#[allow(dead_code)] +pub fn run( + store: &S, + child: &mut C, + probe: &P, + host: &H, + config: &RunConfig, +) -> Result +where + S: VersionStore, + C: ChildSupervisor, + P: HealthProbe, + H: ServiceHost + Sync, +{ + let action = classify_startup(store, config)?; + let (shutdown_tx, shutdown_rx) = mpsc::channel::<()>(); + + std::thread::scope(|scope| { + scope.spawn(|| heartbeat_loop(host, config.heartbeat_interval, shutdown_rx)); + + let mut machine = Machine { + store, + child, + probe, + host, + config, + }; + let result = machine + .execute_startup(action) + .and_then(|handle| machine.supervise(handle)); + + // Wake the heartbeat thread immediately so the scope can close. + drop(shutdown_tx); + result + }) +} + +/// Tick `host.heartbeat()` every `interval` until the shutdown channel closes. +/// `recv_timeout` doubles as the sleep, so a shutdown wakes it instantly. +fn heartbeat_loop(host: &H, interval: Duration, shutdown: mpsc::Receiver<()>) { + loop { + host.heartbeat(); + match shutdown.recv_timeout(interval) { + Err(RecvTimeoutError::Timeout) => continue, + Ok(()) | Err(RecvTimeoutError::Disconnected) => return, + } + } +} + +#[cfg(test)] +mod run_tests { + use super::state_machine::RunConfig; + use super::testing::{SpawnOutcome, StubChild, StubHost, StubProbe, StubStore}; + use super::traits::{Control, PendingMarker, Version}; + use super::*; + use crate::core::store_common; + use std::sync::atomic::Ordering; + use std::time::{Duration, Instant}; + + fn version(s: &str) -> Version { + Version::parse(s).unwrap() + } + + /// The watchdog heartbeat must keep ticking DURING a probation + /// window (probation 150 ms >> heartbeat 10 ms), and READY must be + /// reported after the promote. + #[test] + fn watchdog_ticks_through_probation_and_run_stops_on_control() { + let dir = tempfile::tempdir().unwrap(); + let store = StubStore::new(dir.path()); + store.install_version(&version("1.3.5")); + store.set_current(&version("1.3.5")).unwrap(); + store.set_last_stable(&version("1.3.5")).unwrap(); + std::fs::write(store.state_dir().join("db"), b"state-v1").unwrap(); + + let config = RunConfig { + probation_window: Duration::from_millis(150), + probe_interval: Duration::from_millis(5), + poll_interval: Duration::from_millis(5), + heartbeat_interval: Duration::from_millis(10), + stop_grace: Duration::from_millis(50), + restart_backoff_base: Duration::from_millis(5), + restart_backoff_cap: Duration::from_millis(40), + healthy_reset: Duration::from_millis(150), + max_swap_attempts: 3, + operator_binary: "alien-operator".to_string(), + ..RunConfig::default() + }; + + // Stage a valid 1.4.0 so run() starts with a swap whose probation + // only passes near the end of the window. + store.install_version(&version("1.4.0")); + let binary = store.stage_dir(&version("1.4.0")).join("alien-operator"); + let pending = PendingMarker { + version: version("1.4.0"), + sha256: store_common::file_sha256(&binary).unwrap(), + staged_at: chrono::Utc::now(), + }; + store.write_pending(&pending).unwrap(); + + let mut child = StubChild::new([SpawnOutcome::UpNotReady]); + let probe = StubProbe::ReadyAt(Instant::now() + Duration::from_millis(100)); + let host = StubHost::new(); + let controls = host.controls_tx.clone(); + + std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(250)); + controls.send(Control::Stop).unwrap(); + }); + + let exit = run(&store, &mut child, &probe, &host, &config).unwrap(); + assert_eq!(exit, LoopExit::ControlStop(Control::Stop)); + + // ≥ 5 heartbeats must have landed during the ~100 ms probation alone; + // require a conservative floor to avoid timing flake. + let beats = host.heartbeat_calls.load(Ordering::SeqCst); + assert!(beats >= 5, "heartbeat starved during probation: {beats} ticks"); + assert!(host.ready_calls.load(Ordering::SeqCst) >= 1, "READY after promote"); + assert!(host.stopping_calls.load(Ordering::SeqCst) >= 1); + assert_eq!(store.current().unwrap(), Some(version("1.4.0"))); + assert_eq!(store.last_stable().unwrap(), Some(version("1.4.0"))); + } +} diff --git a/crates/alien-launcher/src/core/report.rs b/crates/alien-launcher/src/core/report.rs new file mode 100644 index 000000000..d6ea041fa --- /dev/null +++ b/crates/alien-launcher/src/core/report.rs @@ -0,0 +1,8 @@ +//! Failure reporting: maps launcher-local update outcomes onto the sync-wire +//! `OperatorUpdateReport` shape via on-disk failure records. +//! +//! The launcher has no network path to the manager, so the failure record in +//! the version store is the handoff — the operator translates the newest +//! record into its `SyncRequest.operator_update` on every sync. +//! +//! Implementation lands with the failure-records work. diff --git a/crates/alien-launcher/src/core/state_machine.rs b/crates/alien-launcher/src/core/state_machine.rs new file mode 100644 index 000000000..67f246e52 --- /dev/null +++ b/crates/alien-launcher/src/core/state_machine.rs @@ -0,0 +1,891 @@ +//! The update state machine: +//! Running → Staged → Swapping → Probation → Promoted / RollingBack. +//! +//! Everything here drives the `traits` boundary — no direct filesystem access +//! (enforced by `tests/platform_blind.rs`, which additionally forbids +//! `std::fs` in this file) and no platform knowledge. The machine is +//! reconstructable from the on-disk store alone: `classify_startup` maps +//! every reachable intermediate state to a recovery action, so a launcher +//! killed at ANY point resumes to promote or rollback. Store errors +//! deliberately propagate out — the OS init respawns the launcher and +//! classification recovers; self-healing in place would mask corruption. + +use std::time::{Duration, Instant}; + +use alien_error::AlienError; +use alien_core::sync::OperatorUpdatePhase; +use chrono::Utc; +use tracing::{info, warn}; + +use crate::error::{ErrorData, Result}; + +use super::store_common; +use super::traits::{ + ChildSupervisor, Control, ExitStatus, FailureRecord, HealthProbe, OperatorHandle, + PendingMarker, ProbationMarker, ServiceHost, UpdateEnv, Version, VersionStore, + EXIT_CODE_UPDATE_HANDOFF, +}; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +/// All tunables, injected so tests run the identical machine with millisecond +/// windows. +#[derive(Debug, Clone)] +pub struct RunConfig { + /// Probation window after a swap (~90 s in production). + pub probation_window: Duration, + /// How often the gate polls `/readyz`. + pub probe_interval: Duration, + /// Supervise-loop tick (child poll + control poll). + pub poll_interval: Duration, + /// Watchdog heartbeat interval (systemd: `WatchdogSec/3`). + pub heartbeat_interval: Duration, + /// Grace given to the operator between SIGTERM-equivalent and force-kill. + pub stop_grace: Duration, + /// Crash-respawn backoff: base doubling per consecutive crash… + pub restart_backoff_base: Duration, + /// …capped here… + pub restart_backoff_cap: Duration, + /// …and reset after the child has run healthy this long. + pub healthy_reset: Duration, + /// Give up resuming a crashed swap after this many attempts. + pub max_swap_attempts: u32, + /// File name of the operator binary inside `versions//`. + pub operator_binary: String, + /// Environment handed to every spawned operator. + pub update_env: UpdateEnv, +} + +impl RunConfig { + /// The `/readyz` URL derived from the health address the launcher itself + /// hands to the operator on spawn. + pub fn readyz_url(&self) -> String { + format!("http://{}/readyz", self.update_env.health_addr) + } + + /// The `/livez` URL. Liveness is manager-INDEPENDENT ("up and running, + /// possibly just waiting for the manager"), so the startup gate uses it + /// for the systemd READY decision — a manager outage at boot must not + /// look like a launch failure. The update gate keeps using `/readyz`. + pub fn livez_url(&self) -> String { + format!("http://{}/livez", self.update_env.health_addr) + } +} + +impl Default for RunConfig { + fn default() -> Self { + Self { + probation_window: Duration::from_secs(90), + probe_interval: Duration::from_secs(2), + poll_interval: Duration::from_millis(100), + heartbeat_interval: Duration::from_secs(20), + stop_grace: Duration::from_secs(10), + restart_backoff_base: Duration::from_secs(1), + restart_backoff_cap: Duration::from_secs(30), + healthy_reset: Duration::from_secs(60), + max_swap_attempts: 3, + operator_binary: "alien-operator".to_string(), + update_env: UpdateEnv { + health_addr: std::net::SocketAddr::from(([127, 0, 0, 1], 7799)), + launcher_version: env!("CARGO_PKG_VERSION").to_string(), + }, + } + } +} + +/// Once the operator has run healthy for at least `healthy_reset`, its prior +/// crash streak no longer counts against the respawn backoff, so the counter is +/// cleared. Extracted as a pure decision so the reset condition is unit-tested +/// deterministically (the `supervise` loop feeds it real wall-clock uptime). +fn should_reset_crash_backoff( + consecutive_crashes: u32, + up_for: Duration, + healthy_reset: Duration, +) -> bool { + consecutive_crashes > 0 && up_for >= healthy_reset +} + +// --------------------------------------------------------------------------- +// Startup classification (the normative recovery table for the on-disk protocol) +// --------------------------------------------------------------------------- + +/// What the launcher must do first, decided purely from the on-disk store. +#[derive(Debug, Clone, PartialEq)] +pub enum StartupAction { + /// Row 1 — steady state: spawn `current` (first install additionally + /// promotes it to `last-stable` after one passed gate). + SpawnCurrent, + /// Row 2 — staged, swap not begun: run the swap from step 1. + RunSwap { pending: PendingMarker }, + /// Row 2 guard — pending names the version that is already current AND + /// stable (crash after promote removed probation but not pending): + /// just clean up. + DiscardLeftoverPending { pending: PendingMarker }, + /// Row 3 — partial stage (binary missing / digest mismatch): delete the + /// marker and spawn `current`; the operator will re-stage. + DiscardInvalidPending { pending: PendingMarker }, + /// Row 4 — crashed mid-probation: spawn `current` (= the new version) and + /// resume the gate with the remaining window (`0` ⇒ roll back now). + ResumeProbation { + probation: ProbationMarker, + remaining: Duration, + }, + /// Row 4b — promote already began (`last-stable == current == new`): + /// finish the idempotent promote cleanup. + FinishPromote { probation: ProbationMarker }, + /// Row 5 — crashed after the probation marker, before the flip: + /// resume the swap at the flip (attempt budget permitting). + ResumeSwapAtFlip { + probation: ProbationMarker, + pending: PendingMarker, + }, + /// Row 5 cap — attempt budget exhausted (or the stage went invalid): + /// run the rollback cleanup and stay on `current`. + AbortSwap { probation: ProbationMarker }, + /// Row 6 — crashed mid-rollback (failure record already written): + /// re-run the idempotent rollback steps. + FinishRollback { probation: ProbationMarker }, +} + +/// Classify the store per the startup-recovery table of the on-disk handoff +/// protocol. The two crashed-swap rows share pointer +/// state (`current == probation.old`); they are discriminated by the failure +/// record — rollback's first persistent effect after the flip-back is writing +/// `failed/.json` with `attempts >= probation.attempt`, so its presence +/// means a rollback was underway. +pub fn classify_startup(store: &S, config: &RunConfig) -> Result { + let current = store.current()?; + let last_stable = store.last_stable()?; + + if let Some(probation) = store.read_probation()? { + let current = current.ok_or_else(|| corrupt(store, "probation exists but no current"))?; + + if current == probation.new { + if last_stable.as_ref() == Some(&probation.new) { + return Ok(StartupAction::FinishPromote { probation }); + } + let elapsed = (Utc::now() - probation.started_at) + .to_std() + .unwrap_or(Duration::ZERO); // NTP stepped backwards → clamp + let remaining = config.probation_window.saturating_sub(elapsed); + return Ok(StartupAction::ResumeProbation { + probation, + remaining, + }); + } + + if current == probation.old { + let rollback_recorded = store + .read_failure(&probation.new)? + .is_some_and(|record| record.attempts >= probation.attempt); + if rollback_recorded { + return Ok(StartupAction::FinishRollback { probation }); + } + if probation.attempt >= config.max_swap_attempts { + return Ok(StartupAction::AbortSwap { probation }); + } + // Resuming the swap needs a valid stage to flip to. + match store.read_pending()? { + Some(pending) if pending.version == probation.new => { + return Ok(StartupAction::ResumeSwapAtFlip { probation, pending }); + } + _ => return Ok(StartupAction::AbortSwap { probation }), + } + } + + return Err(corrupt( + store, + "probation marker matches neither current nor its old version", + )); + } + + if let Some(pending) = store.read_pending()? { + if Some(&pending.version) == current.as_ref() + && Some(&pending.version) == last_stable.as_ref() + { + return Ok(StartupAction::DiscardLeftoverPending { pending }); + } + let binary = store + .stage_dir(&pending.version) + .join(&config.operator_binary); + if store_common::validate_staged_binary(&binary, &pending.sha256)? { + return Ok(StartupAction::RunSwap { pending }); + } + return Ok(StartupAction::DiscardInvalidPending { pending }); + } + + Ok(StartupAction::SpawnCurrent) +} + +fn corrupt(store: &S, message: &str) -> AlienError { + AlienError::new(ErrorData::StoreCorrupt { + path: store.stage_dir(&Version::parse("0.0.0").expect("static version parses")) + .parent() + .map(|p| p.display().to_string()) + .unwrap_or_default(), + message: message.to_string(), + }) +} + +// --------------------------------------------------------------------------- +// The machine +// --------------------------------------------------------------------------- + +/// Why the probation gate ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GateOutcome { + Ready, + TimedOut, + ChildExited(ExitStatus), +} + +/// Why `supervise` returned. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LoopExit { + /// The OS supervisor asked us to stop; the child was stopped gracefully. + ControlStop(Control), +} + +/// The launcher core, generic over the platform boundary. Borrows its +/// components so the run loop, the heartbeat thread, and tests share them. +pub struct Machine<'a, S, C, P, H> +where + S: VersionStore, + C: ChildSupervisor, + P: HealthProbe, + H: ServiceHost, +{ + pub store: &'a S, + pub child: &'a mut C, + pub probe: &'a P, + pub host: &'a H, + pub config: &'a RunConfig, +} + +impl Machine<'_, S, C, P, H> +where + S: VersionStore, + C: ChildSupervisor, + P: HealthProbe, + H: ServiceHost, +{ + // -- startup ---------------------------------------------------------- + + /// Execute a startup classification to a supervised steady state and + /// return the handle of the running operator. + pub fn execute_startup(&mut self, action: StartupAction) -> Result { + match action { + StartupAction::SpawnCurrent => self.spawn_current_and_gate(), + StartupAction::DiscardLeftoverPending { pending } => { + info!(version = %pending.version, "discarding leftover pending marker from a completed promote"); + self.store.clear_pending()?; + self.spawn_current_and_gate() + } + StartupAction::DiscardInvalidPending { pending } => { + warn!( + version = %pending.version, + "pending stage is invalid (missing binary or digest mismatch); discarding — the operator will re-stage" + ); + self.store.clear_pending()?; + self.spawn_current_and_gate() + } + StartupAction::RunSwap { pending } => { + let attempt = self.next_attempt(&pending)?; + self.perform_swap(&pending, attempt) + } + StartupAction::ResumeProbation { + probation, + remaining, + } => { + let handle = self.spawn_version(&probation.new)?; + if remaining.is_zero() { + return self.rollback(&probation, Some(&handle), "probation window expired before the launcher restart".to_string()); + } + // Resuming an update's probation: full-health gate (/readyz). + match self.gate(&handle, remaining, &self.config.readyz_url())? { + GateOutcome::Ready => { + self.promote(&probation)?; + Ok(handle) + } + outcome => self.rollback(&probation, Some(&handle), gate_failure_message(outcome)), + } + } + StartupAction::FinishPromote { probation } => { + self.promote(&probation)?; + self.spawn_current_and_gate() + } + StartupAction::ResumeSwapAtFlip { probation, pending } => { + // The new version never ran; give the gate a fresh window + // (the marker's started_at predates the crash) but keep the + // attempt number. + let refreshed = ProbationMarker { + started_at: Utc::now(), + ..probation + }; + self.store.write_probation(&refreshed)?; + self.flip_spawn_and_gate(&refreshed, &pending) + } + StartupAction::AbortSwap { probation } => { + let message = format!( + "giving up on swap to {} after {} attempts", + probation.new, probation.attempt + ); + self.rollback(&probation, None, message) + } + StartupAction::FinishRollback { probation } => { + // Re-run the idempotent rollback tail; the failure record is + // already on disk (that is how this row was classified). + self.store.set_current(&probation.old)?; + self.store.restore_state(&probation.old)?; + self.store.clear_probation()?; + self.store.clear_pending()?; + self.spawn_current_and_gate() + } + } + } + + // -- the supervise loop ------------------------------------------------ + + /// Supervise the operator until the OS asks us to stop. Handles the + /// exit-code contract: 0 = respawn, 10 = validated swap, other = crash + /// with exponential respawn backoff (reset after `healthy_reset` up). + pub fn supervise(&mut self, initial: OperatorHandle) -> Result { + let mut handle = initial; + let mut spawned_at = Instant::now(); + let mut consecutive_crashes: u32 = 0; + + loop { + if let Some(control) = self.host.poll_control() { + info!(?control, "stop requested; stopping the operator"); + self.host.report_stopping(); + self.child.stop(&handle, self.config.stop_grace)?; + return Ok(LoopExit::ControlStop(control)); + } + + let Some(status) = self.child.try_wait(&handle)? else { + if should_reset_crash_backoff( + consecutive_crashes, + spawned_at.elapsed(), + self.config.healthy_reset, + ) { + consecutive_crashes = 0; + } + std::thread::sleep(self.config.poll_interval); + continue; + }; + + match status { + ExitStatus::Code(0) => { + info!("operator exited cleanly without a stop request; respawning"); + consecutive_crashes = 0; + handle = self.spawn_current_and_gate()?; + spawned_at = Instant::now(); + } + ExitStatus::Code(EXIT_CODE_UPDATE_HANDOFF) => { + match self.validated_pending()? { + Some(pending) => { + info!(version = %pending.version, "update handoff received; swapping"); + let attempt = self.next_attempt(&pending)?; + handle = self.perform_swap(&pending, attempt)?; + spawned_at = Instant::now(); + consecutive_crashes = 0; + } + None => { + warn!("handoff exit (10) without a valid pending stage; treating as a crash"); + self.store.clear_pending()?; + consecutive_crashes += 1; + if let Some(exit) = self.backoff_pause(consecutive_crashes)? { + return Ok(exit); + } + handle = self.spawn_current_and_gate()?; + spawned_at = Instant::now(); + } + } + } + other => { + warn!(?other, "operator crashed; respawning with backoff"); + consecutive_crashes += 1; + if let Some(exit) = self.backoff_pause(consecutive_crashes)? { + return Ok(exit); + } + handle = self.spawn_current_and_gate()?; + spawned_at = Instant::now(); + } + } + } + } + + /// Sleep the crash backoff (1·2ⁿ⁻¹ × base, capped), staying responsive to + /// stop controls. Returns `Some(exit)` if a control arrived mid-backoff. + fn backoff_pause(&mut self, consecutive_crashes: u32) -> Result> { + let factor = 2u32.saturating_pow(consecutive_crashes.saturating_sub(1)); + let delay = self + .config + .restart_backoff_base + .saturating_mul(factor) + .min(self.config.restart_backoff_cap); + let deadline = Instant::now() + delay; + while Instant::now() < deadline { + if let Some(control) = self.host.poll_control() { + self.host.report_stopping(); + return Ok(Some(LoopExit::ControlStop(control))); + } + std::thread::sleep(self.config.poll_interval.min(deadline - Instant::now())); + } + Ok(None) + } + + // -- swap / promote / rollback (the protocol's swap ordering) ----------- + + /// Swap steps 1–6 of the handoff protocol for a validated pending stage. + fn perform_swap(&mut self, pending: &PendingMarker, attempt: u32) -> Result { + // Preflight — an out-of-space attempt aborts cleanly before any + // mutation, recorded as a spawn-phase failure. + if let Err(err) = self.store.free_space_for_snapshot() { + warn!(%err, "disk-space preflight failed; aborting the update attempt"); + self.record_failure(&pending.version, &pending.sha256, attempt, + OperatorUpdatePhase::Spawn, format!("disk-space preflight failed: {err}"))?; + self.store.clear_pending()?; + return self.spawn_current_and_gate(); + } + + let old = self.store.current()?.ok_or_else(|| { + corrupt(self.store, "swap requested but the store has no current version") + })?; + + // 1. snapshot state/ (logging the copy cost so growth is visible). + let state_bytes = self.store.state_size()?; + let snapshot_started = Instant::now(); + self.store.snapshot_state(&old)?; + info!( + state_bytes, + duration_ms = snapshot_started.elapsed().as_millis() as u64, + old = %old, + new = %pending.version, + "state snapshot taken" + ); + + // 2. probation marker — before the flip, so any later crash is + // classifiable. + let probation = ProbationMarker { + new: pending.version.clone(), + old, + started_at: Utc::now(), + attempt, + }; + self.store.write_probation(&probation)?; + + self.flip_spawn_and_gate(&probation, pending) + } + + /// Protocol steps 3–6: flip `current`, spawn, gate, then promote or roll back. + fn flip_spawn_and_gate( + &mut self, + probation: &ProbationMarker, + pending: &PendingMarker, + ) -> Result { + // 3. flip. + self.store.set_current(&probation.new)?; + + // 4. spawn — a spawn failure never ran the new binary, so the state + // is untouched: flip back, record, clean up, restart the old version. + let handle = match self.spawn_version(&probation.new) { + Ok(handle) => handle, + Err(err) => { + warn!(%err, new = %probation.new, "spawning the new operator failed; reverting"); + self.record_failure(&probation.new, &pending.sha256, probation.attempt, + OperatorUpdatePhase::Spawn, format!("spawn failed: {err}"))?; + self.store.set_current(&probation.old)?; + self.store.clear_probation()?; + self.store.clear_pending()?; + return self.spawn_current_and_gate(); + } + }; + + // 5. the probation gate — a NEW version must prove FULL health, + // including reaching the manager (/readyz), before we promote and + // discard the rollback target. + match self.gate(&handle, self.config.probation_window, &self.config.readyz_url())? { + GateOutcome::Ready => { + // 6a. promote. + self.promote(probation)?; + info!(version = %probation.new, "promoted"); + Ok(handle) + } + outcome => { + // 6b. rollback. + self.rollback(probation, Some(&handle), gate_failure_message(outcome)) + } + } + } + + /// Promote cleanup — idempotent, re-runnable from any crash point within. + fn promote(&mut self, probation: &ProbationMarker) -> Result<()> { + self.store.set_last_stable(&probation.new)?; + self.host.report_ready(); + self.store.clear_probation()?; + self.store.clear_pending()?; + self.store.drop_snapshot(&probation.old)?; + self.store.gc(&[])?; + Ok(()) + } + + /// Rollback — stop the failed child (if any), restore the (binary + + /// state) pair, record the failure, and restart the old version. + fn rollback( + &mut self, + probation: &ProbationMarker, + failed_child: Option<&OperatorHandle>, + message: String, + ) -> Result { + warn!(new = %probation.new, old = %probation.old, %message, "rolling back"); + if let Some(handle) = failed_child { + self.child.stop(handle, self.config.stop_grace)?; + } + let restore_to = self.store.last_stable()?.unwrap_or_else(|| probation.old.clone()); + self.store.set_current(&restore_to)?; + self.store.restore_state(&probation.old)?; + let sha256 = self + .store + .read_pending()? + .map(|pending| pending.sha256) + .unwrap_or_default(); + self.record_failure( + &probation.new, + &sha256, + probation.attempt, + OperatorUpdatePhase::Apply, + message, + )?; + self.store.clear_probation()?; + self.store.clear_pending()?; + self.spawn_current_and_gate() + } + + fn record_failure( + &mut self, + version: &Version, + sha256: &str, + attempts: u32, + phase: OperatorUpdatePhase, + message: String, + ) -> Result<()> { + self.store.write_failure(&FailureRecord { + version: version.clone(), + sha256: sha256.to_string(), + phase, + message, + attempts, + last_failed_at: Utc::now(), + }) + } + + // -- gate + spawn helpers ---------------------------------------------- + + /// Poll `/readyz` until ready, child exit, or the deadline. + fn gate(&mut self, handle: &OperatorHandle, window: Duration, url: &str) -> Result { + let deadline = Instant::now() + window; + loop { + if let Some(status) = self.child.try_wait(handle)? { + return Ok(GateOutcome::ChildExited(status)); + } + if self.probe.is_ready(url) { + return Ok(GateOutcome::Ready); + } + let now = Instant::now(); + if now >= deadline { + return Ok(GateOutcome::TimedOut); + } + std::thread::sleep(self.config.probe_interval.min(deadline - now)); + } + } + + /// Spawn `current` and run a best-effort readiness gate: on ready, report + /// ready to the host and — on a first install with no fallback yet — + /// promote `current` to `last-stable`. A gate timeout here keeps + /// supervising (the manager-side heartbeat is the ground truth); a child + /// exit is handed back for the supervise loop to classify. + fn spawn_current_and_gate(&mut self) -> Result { + let current = self.store.current()?.ok_or_else(|| { + corrupt(self.store, "no current version to spawn — install is incomplete") + })?; + let handle = self.spawn_version(¤t)?; + // Startup gate signals systemd READY on LIVENESS (/livez), not full + // readiness: the launcher is "started" once it is supervising a live + // operator. An operator that is up but can't reach the manager + // (/readyz 503) must still count as started — otherwise a manager + // outage at boot fails the launcher's Type=notify start and flaps it. + // A genuinely broken operator (crashes, or never serves /livez) does + // NOT reach Ready here, so systemd start correctly fails. + match self.gate(&handle, self.config.probation_window, &self.config.livez_url())? { + GateOutcome::Ready => { + self.host.report_ready(); + // last-stable is a PROVEN-good fallback, so it stays gated on + // FULL health (/readyz), not liveness. Normally the installer + // seeds last-stable; this only fires on a store missing it, + // and only once the operator has actually reached the manager. + if self.store.last_stable()?.is_none() + && self.probe.is_ready(&self.config.readyz_url()) + { + info!(version = %current, "first install reached full health; recording last-stable"); + self.store.set_last_stable(¤t)?; + } + } + GateOutcome::TimedOut => { + warn!(version = %current, "operator did not become live within the window; supervising anyway"); + } + GateOutcome::ChildExited(status) => { + // Hand the exit to the supervise loop's contract by just + // returning — try_wait will re-observe it immediately. + warn!(?status, "operator exited during the startup gate"); + } + } + Ok(handle) + } + + fn spawn_version(&mut self, version: &Version) -> Result { + let binary = self + .store + .stage_dir(version) + .join(&self.config.operator_binary); + self.child.spawn(&binary, &self.config.update_env) + } + + /// Read + validate `pending.json` in one step for the handoff path. + fn validated_pending(&mut self) -> Result> { + let Some(pending) = self.store.read_pending()? else { + return Ok(None); + }; + let binary = self + .store + .stage_dir(&pending.version) + .join(&self.config.operator_binary); + if store_common::validate_staged_binary(&binary, &pending.sha256)? { + Ok(Some(pending)) + } else { + Ok(None) + } + } + + /// Attempt numbering: continue the count from a prior failure of the SAME + /// artifact (version + sha256); a different artifact starts fresh. + fn next_attempt(&mut self, pending: &PendingMarker) -> Result { + Ok(match self.store.read_failure(&pending.version)? { + Some(record) if record.sha256 == pending.sha256 => record.attempts + 1, + _ => 1, + }) + } +} + +fn gate_failure_message(outcome: GateOutcome) -> String { + match outcome { + GateOutcome::TimedOut => "readyz never returned 200 within the probation window".to_string(), + GateOutcome::ChildExited(ExitStatus::Code(code)) => { + format!("operator exited during probation with code {code}") + } + GateOutcome::ChildExited(status) => { + format!("operator exited during probation ({status:?})") + } + GateOutcome::Ready => unreachable!("Ready is not a failure"), + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::testing::{ + self, scenario_classification_rows, scenario_crash_injection_promote, + scenario_crash_injection_rollback, scenario_happy_promote, + scenario_rollback_on_probation_crash, scenario_rollback_restores_state, SpawnOutcome, + StubChild, StubHost, StubProbe, StubStore, + }; + use std::path::Path; + + // The state-machine suite lives in `core::testing` as scenarios generic + // over any `VersionStore`, so the platform stores run the IDENTICAL + // tests (see `platform::unix_store`). Here they run against the stub. + + fn stub(dir: &Path) -> StubStore { + StubStore::new(dir) + } + + #[test] + fn happy_promote() { + scenario_happy_promote(stub); + } + + #[test] + fn rollback_when_probe_never_ready_and_state_restored() { + scenario_rollback_restores_state(stub); + } + + #[test] + fn rollback_when_child_crashes_during_probation() { + scenario_rollback_on_probation_crash(stub); + } + + #[test] + fn classification_rows() { + scenario_classification_rows(stub); + } + + #[test] + fn crash_backoff_resets_only_after_a_healthy_uptime() { + let healthy_reset = Duration::from_secs(60); + // No crash streak → nothing to reset, regardless of uptime. + assert!(!should_reset_crash_backoff(0, Duration::from_secs(120), healthy_reset)); + // A streak, but not up long enough → keep it (backoff stays elevated). + assert!(!should_reset_crash_backoff(3, Duration::from_secs(59), healthy_reset)); + // A streak and up at least `healthy_reset` → reset to zero. + assert!(should_reset_crash_backoff(3, healthy_reset, healthy_reset)); + assert!(should_reset_crash_backoff(1, Duration::from_secs(61), healthy_reset)); + } + + #[test] + fn crash_injection_matrix_converges_from_every_step() { + scenario_crash_injection_promote(stub); + } + + #[test] + fn crash_injection_on_rollback_path_converges() { + scenario_crash_injection_rollback(stub); + } + + // -- stub-only scenarios ------------------------------------------- + + /// Disk preflight aborts cleanly before any mutation. Stub-only: the + /// real stores query genuine statvfs numbers, which tests cannot script. + #[test] + fn disk_preflight_failure_aborts_cleanly_before_any_mutation() { + let dir = tempfile::tempdir().unwrap(); + let store = StubStore::new(dir.path()); + testing::seed_base(&store); + let config = testing::test_run_config(); + testing::stage_valid(&store, &config); + store + .available_bytes + .store(0, std::sync::atomic::Ordering::SeqCst); + + let mut child = StubChild::new([SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let mut m = Machine { + store: &store, + child: &mut child, + probe: &probe, + host: &host, + config: &config, + }; + + let action = classify_startup(&store, &config).unwrap(); + m.execute_startup(action).unwrap(); + + // Old version still running; nothing mutated beyond the cleared pending. + assert_eq!(store.current().unwrap().unwrap().as_str(), "1.3.5"); + assert!(store.read_pending().unwrap().is_none()); + assert!(store.read_probation().unwrap().is_none()); + let record = store + .read_failure(&Version::parse("1.4.0").unwrap()) + .unwrap() + .unwrap(); + assert_eq!(record.phase, OperatorUpdatePhase::Spawn); + assert_eq!( + child.spawned[0].0, + store.stage_dir(&Version::parse("1.3.5").unwrap()).join("alien-operator"), + "the OLD operator was (re)spawned" + ); + } + + // -- supervise loop: exit-code contract ------------------------------ + + #[test] + fn supervise_exit_code_contract() { + let dir = tempfile::tempdir().unwrap(); + let store = StubStore::new(dir.path()); + testing::seed_base(&store); + let config = testing::test_run_config(); + // Invalid pending present when a handoff-10 arrives → discarded. + store.install_version(&Version::parse("1.4.0").unwrap()); + store + .write_pending(&PendingMarker { + version: Version::parse("1.4.0").unwrap(), + sha256: "0".repeat(64), + staged_at: Utc::now(), + }) + .unwrap(); + + // Script: initial spawn exits 0 (clean) → respawn exits 10 with the + // INVALID pending → treated as crash, pending deleted, backoff respawn + // exits 7 (crash) → backoff respawn stays up → Stop. + let mut child = StubChild::new([ + SpawnOutcome::ExitImmediately(0), + SpawnOutcome::ExitImmediately(10), + SpawnOutcome::ExitImmediately(7), + SpawnOutcome::UpNotReady, + ]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let controls = host.controls_tx.clone(); + let mut m = Machine { + store: &store, + child: &mut child, + probe: &probe, + host: &host, + config: &config, + }; + + let first = m.spawn_current_and_gate().unwrap(); + std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(150)); + controls.send(Control::Stop).unwrap(); + }); + let exit = m.supervise(first).unwrap(); + assert_eq!(exit, LoopExit::ControlStop(Control::Stop)); + + assert_eq!(child.spawned.len(), 4, "0→respawn, 10-invalid→respawn, 7→respawn"); + assert!(store.read_pending().unwrap().is_none(), "invalid pending deleted"); + assert_eq!(store.current().unwrap().unwrap().as_str(), "1.3.5", "no swap happened"); + assert_eq!(child.stop_calls.len(), 1, "the running child was stopped gracefully"); + assert!(host.stopping_calls.load(std::sync::atomic::Ordering::SeqCst) >= 1); + } + + #[test] + fn supervise_valid_handoff_swaps() { + let dir = tempfile::tempdir().unwrap(); + let store = StubStore::new(dir.path()); + testing::seed_base(&store); + let config = testing::test_run_config(); + testing::stage_valid(&store, &config); + + // Initial operator exits 10 (it staged 1.4.0) → swap → new stays up. + let mut child = StubChild::new([ + SpawnOutcome::ExitImmediately(10), + SpawnOutcome::UpNotReady, + ]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let controls = host.controls_tx.clone(); + let mut m = Machine { + store: &store, + child: &mut child, + probe: &probe, + host: &host, + config: &config, + }; + + let first = m.spawn_current_and_gate().unwrap(); + std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(100)); + controls.send(Control::Stop).unwrap(); + }); + let exit = m.supervise(first).unwrap(); + assert_eq!(exit, LoopExit::ControlStop(Control::Stop)); + testing::assert_steady_promoted(&store); + } +} diff --git a/crates/alien-launcher/src/core/store_common.rs b/crates/alien-launcher/src/core/store_common.rs new file mode 100644 index 000000000..6ec463a8e --- /dev/null +++ b/crates/alien-launcher/src/core/store_common.rs @@ -0,0 +1,444 @@ +//! Portable building blocks shared by every `VersionStore` implementation: +//! atomic JSON marker I/O, `state/` snapshot copy + restore, gc-candidate +//! computation, and the disk-space check. +//! +//! Everything here is platform-blind. The one inherently platform-specific +//! piece of the disk-space story — querying *available* bytes — is supplied +//! by the platform store; this module only computes the *required* bytes +//! (`dir_size`) and applies the check (`check_space`). + +// Skeleton staging: outside of test cfg, only the validation helpers have a +// consumer (the state machine) until the platform VersionStores land. +#![allow(dead_code)] + +use std::path::Path; + +use serde::de::DeserializeOwned; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use crate::error::{ErrorData, Result}; +use alien_error::{AlienError, Context, IntoAlienError}; + +use super::traits::Version; + +// --------------------------------------------------------------------------- +// Atomic marker I/O +// --------------------------------------------------------------------------- + +// The atomic write/read/remove primitives live in `alien_core::self_update` +// (they are the normative protocol both binaries share). These wrappers add +// the launcher's error semantics: parse failure = StoreCorrupt, loudly. + +/// Atomically write a JSON marker (temp → fsync → rename; see +/// `alien_core::self_update::write_json_atomic` for the crash semantics). +pub fn write_marker_atomic(path: &Path, value: &T) -> Result<()> { + alien_core::self_update::write_json_atomic(path, value) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to write marker '{}'", path.display()), + }) +} + +/// Read a JSON marker. Absent file → `Ok(None)`. A file that exists but does +/// not parse is genuine corruption (atomic writes rule out torn markers) and +/// fails loudly as `StoreCorrupt` — never silently treated as absent. +pub fn read_marker(path: &Path) -> Result> { + match alien_core::self_update::read_json(path) { + Ok(value) => Ok(value), + Err(e) if e.kind() == std::io::ErrorKind::InvalidData => { + Err(e).into_alien_error().context(ErrorData::StoreCorrupt { + path: path.display().to_string(), + message: "marker exists but is not valid JSON for its schema".to_string(), + }) + } + Err(e) => Err(e).into_alien_error().context(ErrorData::Other { + message: format!("failed to read marker '{}'", path.display()), + }), + } +} + +/// Remove a marker. Idempotent — an absent marker is success, matching the +/// re-runnable promote/rollback step sequences. +pub fn remove_marker(path: &Path) -> Result<()> { + alien_core::self_update::remove_json(path) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to remove marker '{}'", path.display()), + }) +} + +// --------------------------------------------------------------------------- +// Snapshot copy + restore +// --------------------------------------------------------------------------- + +/// Copy `state_dir` to `snapshots_dir//` via a temp dir + rename, so a +/// crash never leaves a half-copied snapshot under the final name. An +/// existing snapshot for `tag` is replaced. +pub fn snapshot_state_dir(state_dir: &Path, snapshots_dir: &Path, tag: &Version) -> Result<()> { + let final_dir = snapshots_dir.join(tag.as_str()); + let tmp_dir = snapshots_dir.join(format!(".tmp-{tag}")); + + remove_dir_if_exists(&tmp_dir)?; + std::fs::create_dir_all(snapshots_dir) + .into_alien_error() + .context(ErrorData::SnapshotFailed { + message: format!("failed to create '{}'", snapshots_dir.display()), + })?; + copy_dir_recursive(state_dir, &tmp_dir).context(ErrorData::SnapshotFailed { + message: format!( + "failed to copy '{}' to snapshot temp dir", + state_dir.display() + ), + })?; + remove_dir_if_exists(&final_dir)?; + std::fs::rename(&tmp_dir, &final_dir) + .into_alien_error() + .context(ErrorData::SnapshotFailed { + message: format!("failed to commit snapshot '{}'", final_dir.display()), + }) +} + +/// Restore `state_dir` from `snapshots_dir//`. +/// +/// Re-runnable at every crash point (rollback steps are idempotent): copy the +/// snapshot to a temp dir first, then swap it into place. Order: stale temp +/// cleanup → copy → remove old state → rename. +pub fn restore_state_dir(state_dir: &Path, snapshots_dir: &Path, tag: &Version) -> Result<()> { + let snapshot_dir = snapshots_dir.join(tag.as_str()); + if !snapshot_dir.is_dir() { + return Err(AlienError::new(ErrorData::SnapshotFailed { + message: format!( + "snapshot '{}' does not exist — cannot restore state", + snapshot_dir.display() + ), + })); + } + let tmp_dir = state_dir.with_file_name(format!( + ".tmp-restore-{}", + state_dir + .file_name() + .expect("state dir always has a name") + .to_string_lossy() + )); + + remove_dir_if_exists(&tmp_dir)?; + copy_dir_recursive(&snapshot_dir, &tmp_dir).context(ErrorData::SnapshotFailed { + message: format!( + "failed to copy snapshot '{}' for restore", + snapshot_dir.display() + ), + })?; + remove_dir_if_exists(state_dir)?; + std::fs::rename(&tmp_dir, state_dir) + .into_alien_error() + .context(ErrorData::SnapshotFailed { + message: format!( + "failed to swap restored state into '{}'", + state_dir.display() + ), + }) +} + +/// Recursively copy a directory tree (regular files + dirs; the state dir +/// contains no symlinks). +fn copy_dir_recursive(src: &Path, dst: &Path) -> Result<()> { + std::fs::create_dir_all(dst) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to create '{}'", dst.display()), + })?; + for entry in std::fs::read_dir(src) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to read dir '{}'", src.display()), + })? + { + let entry = entry.into_alien_error().context(ErrorData::Other { + message: format!("failed to read dir entry under '{}'", src.display()), + })?; + let from = entry.path(); + let to = dst.join(entry.file_name()); + let file_type = entry.file_type().into_alien_error().context(ErrorData::Other { + message: format!("failed to stat '{}'", from.display()), + })?; + if file_type.is_dir() { + copy_dir_recursive(&from, &to)?; + } else { + std::fs::copy(&from, &to) + .into_alien_error() + .context(ErrorData::Other { + message: format!( + "failed to copy '{}' to '{}'", + from.display(), + to.display() + ), + })?; + } + } + Ok(()) +} + +fn remove_dir_if_exists(dir: &Path) -> Result<()> { + match std::fs::remove_dir_all(dir) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e).into_alien_error().context(ErrorData::Other { + message: format!("failed to remove '{}'", dir.display()), + }), + } +} + +// --------------------------------------------------------------------------- +// Staged-binary validation +// --------------------------------------------------------------------------- + +/// SHA-256 of a file, lowercase hex. Streams in 64 KiB chunks so a large +/// binary never sits in memory. +pub fn file_sha256(path: &Path) -> Result { + let mut file = std::fs::File::open(path) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to open '{}' for hashing", path.display()), + })?; + let mut hasher = Sha256::new(); + std::io::copy(&mut file, &mut hasher) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to read '{}' for hashing", path.display()), + })?; + Ok(format!("{:x}", hasher.finalize())) +} + +/// Validate a staged binary against its `pending.json` claim: the file must +/// exist and its re-computed SHA-256 must match. `Ok(false)` = invalid stage +/// (partial download, tampering, wrong file) — the caller discards the +/// pending marker; errors are real I/O failures. +pub fn validate_staged_binary(binary_path: &Path, expected_sha256: &str) -> Result { + if !binary_path.is_file() { + return Ok(false); + } + Ok(file_sha256(binary_path)? == expected_sha256) +} + +// --------------------------------------------------------------------------- +// GC candidates + disk space +// --------------------------------------------------------------------------- + +/// The versions safe to delete: everything in `all` that is not in `keep` +/// and is not what `current` / `last-stable` point at. Pure function — the +/// never-delete-the-pointers rule is enforced here, once, for every platform +/// store. +pub fn gc_candidates( + all: &[Version], + keep: &[Version], + current: Option<&Version>, + last_stable: Option<&Version>, +) -> Vec { + all.iter() + .filter(|v| !keep.contains(v)) + .filter(|v| Some(*v) != current) + .filter(|v| Some(*v) != last_stable) + .cloned() + .collect() +} + +/// Total size in bytes of all regular files under `dir` — the "required +/// bytes" side of the snapshot preflight. +pub fn dir_size(dir: &Path) -> Result { + let mut total = 0u64; + for entry in std::fs::read_dir(dir) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to read dir '{}'", dir.display()), + })? + { + let entry = entry.into_alien_error().context(ErrorData::Other { + message: format!("failed to read dir entry under '{}'", dir.display()), + })?; + let path = entry.path(); + if path.is_dir() { + total += dir_size(&path)?; + } else { + let meta = entry.metadata().into_alien_error().context(ErrorData::Other { + message: format!("failed to stat '{}'", path.display()), + })?; + total += meta.len(); + } + } + Ok(total) +} + +/// Apply the disk-space preflight: fail with `DiskSpace` unless +/// `available_bytes` covers `required_bytes` plus 20% headroom. The platform +/// store queries `available_bytes` (statvfs / GetDiskFreeSpaceEx — inherently +/// platform code); this check stays shared so the policy lives in one place. +pub fn check_space(required_bytes: u64, available_bytes: u64, what: &str) -> Result<()> { + let with_headroom = required_bytes.saturating_add(required_bytes / 5); + if available_bytes < with_headroom { + return Err(AlienError::new(ErrorData::DiskSpace { + required_bytes: with_headroom, + available_bytes, + message: format!("not enough free space for {what}"), + })); + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::traits::PendingMarker; + + fn version(s: &str) -> Version { + Version::parse(s).expect("test version should parse") + } + + fn marker(v: &str) -> PendingMarker { + PendingMarker { + version: version(v), + sha256: "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3f9c71a1e4a6f2e0e6d5c4b3a" + .to_string(), + staged_at: "2026-07-08T12:00:00Z".parse().unwrap(), + } + } + + /// Write → read round-trips, and no `.tmp` residue survives a successful + /// commit. + #[test] + fn marker_write_is_atomic_and_leaves_no_temp() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + + write_marker_atomic(&path, &marker("1.4.0")).expect("write should succeed"); + let back: Option = read_marker(&path).expect("read should succeed"); + assert_eq!(back, Some(marker("1.4.0"))); + + let leftovers: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name.ends_with(".tmp")) + .collect(); + assert!(leftovers.is_empty(), "temp residue: {leftovers:?}"); + + // Overwrite is also atomic and clean. + write_marker_atomic(&path, &marker("1.5.0")).expect("overwrite should succeed"); + let back: Option = read_marker(&path).unwrap(); + assert_eq!(back.unwrap().version, version("1.5.0")); + } + + /// Absent marker reads as None; corrupt marker fails loudly (never a + /// silent None). + #[test] + fn marker_read_absent_is_none_corrupt_is_error() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + + let absent: Option = read_marker(&path).expect("absent is not an error"); + assert!(absent.is_none()); + + std::fs::write(&path, b"{ definitely not a marker").unwrap(); + let err = read_marker::(&path).expect_err("corrupt marker must error"); + assert_eq!(err.code, "STORE_CORRUPT"); + + // remove_marker is idempotent. + remove_marker(&path).expect("remove existing"); + remove_marker(&path).expect("remove absent is still Ok"); + assert!(!path.exists()); + } + + /// Snapshot, mutate state (edit + add + delete), restore → byte-identical. + #[test] + fn snapshot_then_restore_is_byte_identical() { + let root = tempfile::tempdir().unwrap(); + let state = root.path().join("state"); + let snaps = root.path().join("state-snapshots"); + std::fs::create_dir_all(state.join("sub")).unwrap(); + std::fs::write(state.join("db.sqlite"), b"original-db-bytes").unwrap(); + std::fs::write(state.join("sub/token"), b"original-token").unwrap(); + + let tag = version("1.3.5"); + snapshot_state_dir(&state, &snaps, &tag).expect("snapshot should succeed"); + + // Mutate: edit one file, add one, delete one — simulating a migration. + std::fs::write(state.join("db.sqlite"), b"MIGRATED-db-bytes-longer").unwrap(); + std::fs::write(state.join("new-table"), b"added-by-new-version").unwrap(); + std::fs::remove_file(state.join("sub/token")).unwrap(); + + restore_state_dir(&state, &snaps, &tag).expect("restore should succeed"); + + assert_eq!( + std::fs::read(state.join("db.sqlite")).unwrap(), + b"original-db-bytes" + ); + assert_eq!( + std::fs::read(state.join("sub/token")).unwrap(), + b"original-token" + ); + assert!( + !state.join("new-table").exists(), + "files added after the snapshot must not survive restore" + ); + // No temp residue from either operation. + assert!(!snaps.join(".tmp-1.3.5").exists()); + assert!(!root.path().join(".tmp-restore-state").exists()); + } + + /// Restoring a missing snapshot fails loudly. + #[test] + fn restore_missing_snapshot_is_error() { + let root = tempfile::tempdir().unwrap(); + let state = root.path().join("state"); + std::fs::create_dir_all(&state).unwrap(); + let err = restore_state_dir(&state, &root.path().join("state-snapshots"), &version("9.9.9")) + .expect_err("missing snapshot must error"); + assert_eq!(err.code, "SNAPSHOT_FAILED"); + } + + /// gc candidates never include current / last-stable, regardless of `keep`. + #[test] + fn gc_candidates_never_include_pointers() { + let all = [version("1.0.0"), version("1.1.0"), version("1.2.0"), version("1.3.0")]; + let current = version("1.3.0"); + let last_stable = version("1.2.0"); + + let candidates = gc_candidates(&all, &[], Some(¤t), Some(&last_stable)); + assert_eq!(candidates, vec![version("1.0.0"), version("1.1.0")]); + + // Even an explicit empty keep-list with pointers unset keeps nothing. + let candidates = gc_candidates(&all, &[version("1.0.0")], Some(¤t), Some(&last_stable)); + assert_eq!(candidates, vec![version("1.1.0")]); + + // No pointers at all (fresh store): everything not kept is a candidate. + let candidates = gc_candidates(&all, &[version("1.1.0")], None, None); + assert_eq!( + candidates, + vec![version("1.0.0"), version("1.2.0"), version("1.3.0")] + ); + } + + /// The space check demands required + 20% headroom. + #[test] + fn check_space_enforces_headroom() { + check_space(1000, 1200, "state snapshot").expect("exactly at headroom passes"); + let err = check_space(1000, 1199, "state snapshot").expect_err("below headroom fails"); + assert_eq!(err.code, "DISK_SPACE"); + + check_space(0, 0, "empty state").expect("zero required always passes"); + } + + /// dir_size counts nested regular files. + #[test] + fn dir_size_counts_nested_files() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(root.path().join("a/b")).unwrap(); + std::fs::write(root.path().join("f1"), vec![0u8; 100]).unwrap(); + std::fs::write(root.path().join("a/f2"), vec![0u8; 50]).unwrap(); + std::fs::write(root.path().join("a/b/f3"), vec![0u8; 7]).unwrap(); + assert_eq!(dir_size(root.path()).unwrap(), 157); + } +} diff --git a/crates/alien-launcher/src/core/testing.rs b/crates/alien-launcher/src/core/testing.rs new file mode 100644 index 000000000..a91f1c0a0 --- /dev/null +++ b/crates/alien-launcher/src/core/testing.rs @@ -0,0 +1,1304 @@ +//! Test stubs for the trait boundary — the state machine's entire test suite +//! runs against these, with no OS service, no real child process, and no +//! symlinks (so the suite passes identically on Linux, macOS, and Windows). +//! +//! `StubStore` is a REAL `VersionStore` over a tempdir (pointer files instead +//! of symlinks) built on `store_common` — so exercising the state machine +//! against it also exercises the shared helpers every platform store uses. + +use std::collections::VecDeque; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::sync::mpsc::{Receiver, Sender}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use alien_error::AlienError; + +use crate::error::{ErrorData, Result}; + +use super::store_common; +use super::traits::{ + ChildSupervisor, Control, ExitStatus, FailureRecord, HealthProbe, OperatorHandle, + PendingMarker, ProbationMarker, ServiceHost, UpdateEnv, Version, VersionStore, +}; + +// --------------------------------------------------------------------------- +// StubStore — a full VersionStore over a tempdir, symlink-free +// --------------------------------------------------------------------------- + +/// Layout mirrors the real store; `current` / `last-stable` are pointer FILES +/// (`current.txt` / `last-stable.txt` holding the version string) so the stub +/// runs on every OS without symlink or junction support. +pub struct StubStore { + root: PathBuf, + /// Scripted "free disk space" for `free_space_for_snapshot`. + /// Defaults to effectively-unlimited. + pub available_bytes: AtomicU64, +} + +impl StubStore { + pub fn new(root: &Path) -> Self { + for dir in ["versions", "state", "state-snapshots", "failed"] { + std::fs::create_dir_all(root.join(dir)).expect("stub store layout should create"); + } + Self { + root: root.to_path_buf(), + available_bytes: AtomicU64::new(u64::MAX), + } + } + + pub fn root(&self) -> &Path { + &self.root + } + + pub fn state_dir(&self) -> PathBuf { + self.root.join("state") + } + + /// Create `versions//` with a placeholder binary, as staging would. + pub fn install_version(&self, version: &Version) { + let dir = self.stage_dir(version); + std::fs::create_dir_all(&dir).expect("version dir should create"); + std::fs::write(dir.join("alien-operator"), format!("binary-{version}")) + .expect("placeholder binary should write"); + } + + fn pointer_path(&self, name: &str) -> PathBuf { + self.root.join(format!("{name}.txt")) + } + + fn read_pointer(&self, name: &str) -> Result> { + let path = self.pointer_path(name); + let raw = match std::fs::read_to_string(&path) { + Ok(raw) => raw, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => { + return Err(AlienError::new(ErrorData::StoreCorrupt { + path: path.display().to_string(), + message: format!("failed to read pointer: {e}"), + })); + } + }; + Version::parse(raw.trim()) + .map(Some) + .map_err(|e| { + AlienError::new(ErrorData::StoreCorrupt { + path: path.display().to_string(), + message: format!("pointer holds an unparseable version: {e}"), + }) + }) + } + + fn write_pointer(&self, name: &str, version: &Version) -> Result<()> { + // Pointer writes go through the same atomic temp+rename discipline. + let path = self.pointer_path(name); + let tmp = self.root.join(format!("{name}.txt.tmp")); + std::fs::write(&tmp, version.as_str()).map_err(|e| { + AlienError::new(ErrorData::Other { + message: format!("failed to write pointer temp '{}': {e}", tmp.display()), + }) + })?; + std::fs::rename(&tmp, &path).map_err(|e| { + AlienError::new(ErrorData::Other { + message: format!("failed to commit pointer '{}': {e}", path.display()), + }) + }) + } + + fn marker_path(&self, name: &str) -> PathBuf { + self.root.join(name) + } + + fn failure_path(&self, version: &Version) -> PathBuf { + self.root.join("failed").join(format!("{version}.json")) + } +} + +impl VersionStore for StubStore { + fn stage_dir(&self, version: &Version) -> PathBuf { + self.root.join("versions").join(version.as_str()) + } + + fn current(&self) -> Result> { + self.read_pointer("current") + } + + fn last_stable(&self) -> Result> { + self.read_pointer("last-stable") + } + + fn set_current(&self, version: &Version) -> Result<()> { + self.write_pointer("current", version) + } + + fn set_last_stable(&self, version: &Version) -> Result<()> { + self.write_pointer("last-stable", version) + } + + fn snapshot_state(&self, tag: &Version) -> Result<()> { + store_common::snapshot_state_dir( + &self.state_dir(), + &self.root.join("state-snapshots"), + tag, + ) + } + + fn restore_state(&self, tag: &Version) -> Result<()> { + store_common::restore_state_dir( + &self.state_dir(), + &self.root.join("state-snapshots"), + tag, + ) + } + + fn drop_snapshot(&self, tag: &Version) -> Result<()> { + let dir = self.root.join("state-snapshots").join(tag.as_str()); + match std::fs::remove_dir_all(&dir) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(AlienError::new(ErrorData::Other { + message: format!("failed to drop snapshot '{}': {e}", dir.display()), + })), + } + } + + fn state_size(&self) -> Result { + store_common::dir_size(&self.state_dir()) + } + + fn gc(&self, keep: &[Version]) -> Result<()> { + let all = self.list_versions()?; + let current = self.current()?; + let last_stable = self.last_stable()?; + for candidate in store_common::gc_candidates( + &all, + keep, + current.as_ref(), + last_stable.as_ref(), + ) { + std::fs::remove_dir_all(self.stage_dir(&candidate)).map_err(|e| { + AlienError::new(ErrorData::Other { + message: format!("gc failed for version {candidate}: {e}"), + }) + })?; + } + Ok(()) + } + + fn read_pending(&self) -> Result> { + store_common::read_marker(&self.marker_path("pending.json")) + } + + fn write_pending(&self, marker: &PendingMarker) -> Result<()> { + store_common::write_marker_atomic(&self.marker_path("pending.json"), marker) + } + + fn clear_pending(&self) -> Result<()> { + store_common::remove_marker(&self.marker_path("pending.json")) + } + + fn read_probation(&self) -> Result> { + store_common::read_marker(&self.marker_path("probation.json")) + } + + fn write_probation(&self, marker: &ProbationMarker) -> Result<()> { + store_common::write_marker_atomic(&self.marker_path("probation.json"), marker) + } + + fn clear_probation(&self) -> Result<()> { + store_common::remove_marker(&self.marker_path("probation.json")) + } + + fn read_failure(&self, version: &Version) -> Result> { + store_common::read_marker(&self.failure_path(version)) + } + + fn write_failure(&self, record: &FailureRecord) -> Result<()> { + let path = self.failure_path(&record.version); + store_common::write_marker_atomic(&path, record) + } + + fn list_versions(&self) -> Result> { + let versions_dir = self.root.join("versions"); + let mut versions = Vec::new(); + for entry in std::fs::read_dir(&versions_dir).map_err(|e| { + AlienError::new(ErrorData::Other { + message: format!("failed to read '{}': {e}", versions_dir.display()), + }) + })? { + let entry = entry.map_err(|e| { + AlienError::new(ErrorData::Other { + message: format!("failed to read versions entry: {e}"), + }) + })?; + let name = entry.file_name().to_string_lossy().into_owned(); + let version = Version::parse(&name).map_err(|e| { + AlienError::new(ErrorData::StoreCorrupt { + path: entry.path().display().to_string(), + message: format!("versions/ entry is not a version: {e}"), + }) + })?; + versions.push(version); + } + versions.sort(); + Ok(versions) + } + + fn free_space_for_snapshot(&self) -> Result<()> { + let required = store_common::dir_size(&self.state_dir())?; + store_common::check_space( + required, + self.available_bytes.load(Ordering::SeqCst), + "state snapshot", + ) + } +} + +// --------------------------------------------------------------------------- +// StubChild — scripted ChildSupervisor +// --------------------------------------------------------------------------- + +/// What a scripted spawn does. Readiness is controlled separately by the paired +/// `StubProbe`, so a child that "stays up" is `UpNotReady` here regardless of +/// what the probe reports. +#[derive(Debug, Clone)] +pub enum SpawnOutcome { + /// Child runs and stays up; exits only on `stop`. + UpNotReady, + /// Child exits immediately with this code (e.g. crash `1`, handoff `10`). + ExitImmediately(i32), +} + +#[derive(Debug)] +struct ChildState { + outcome: SpawnOutcome, + stopped: bool, +} + +/// Scripted `ChildSupervisor`: each `spawn` consumes the next outcome from the +/// script and records what was spawned for assertions. +pub struct StubChild { + script: VecDeque, + children: Vec, + /// Every spawn call: (binary path, env), for assertions. + pub spawned: Vec<(PathBuf, UpdateEnv)>, + /// pids passed to `stop`, for assertions. + pub stop_calls: Vec, + /// Optional hook run on every spawn — lets a test simulate the "new + /// operator migrates the state DB" side effect between snapshot and + /// rollback (the stub child is not a real process and touches nothing + /// by itself). + pub on_spawn: Option>, +} + +impl StubChild { + pub fn new(script: impl IntoIterator) -> Self { + Self { + script: script.into_iter().collect(), + children: Vec::new(), + spawned: Vec::new(), + stop_calls: Vec::new(), + on_spawn: None, + } + } +} + +impl ChildSupervisor for StubChild { + fn spawn(&mut self, binary: &Path, env: &UpdateEnv) -> Result { + let outcome = self.script.pop_front().ok_or_else(|| { + AlienError::new(ErrorData::SpawnFailed { + binary_path: binary.display().to_string(), + message: "stub script exhausted — test spawned more children than scripted" + .to_string(), + }) + })?; + if let Some(hook) = self.on_spawn.as_mut() { + hook(binary); + } + self.spawned.push((binary.to_path_buf(), env.clone())); + self.children.push(ChildState { + outcome, + stopped: false, + }); + // pid is the 1-based child index — unique per spawn within a test. + Ok(OperatorHandle { + pid: self.children.len() as u32, + }) + } + + fn stop(&mut self, handle: &OperatorHandle, _grace: Duration) -> Result<()> { + self.stop_calls.push(handle.pid); + let child = self.child_mut(handle)?; + child.stopped = true; + Ok(()) + } + + fn try_wait(&mut self, handle: &OperatorHandle) -> Result> { + let child = self.child_mut(handle)?; + if child.stopped { + return Ok(Some(ExitStatus::Code(0))); + } + match child.outcome { + SpawnOutcome::ExitImmediately(code) => Ok(Some(ExitStatus::Code(code))), + SpawnOutcome::UpNotReady => Ok(None), + } + } +} + +impl StubChild { + fn child_mut(&mut self, handle: &OperatorHandle) -> Result<&mut ChildState> { + let index = handle.pid as usize - 1; + self.children.get_mut(index).ok_or_else(|| { + AlienError::new(ErrorData::Other { + message: format!("unknown stub child pid {}", handle.pid), + }) + }) + } +} + +// --------------------------------------------------------------------------- +// StubHost — call-recording ServiceHost +// --------------------------------------------------------------------------- + +/// Records lifecycle reporting for assertions; `next_control` blocks on a +/// channel the test feeds via `controls_tx`. +pub struct StubHost { + pub ready_calls: AtomicU32, + pub heartbeat_calls: AtomicU32, + pub stopping_calls: AtomicU32, + controls: Mutex>, + pub controls_tx: Sender, +} + +impl StubHost { + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + let (tx, rx) = std::sync::mpsc::channel(); + Self { + ready_calls: AtomicU32::new(0), + heartbeat_calls: AtomicU32::new(0), + stopping_calls: AtomicU32::new(0), + controls: Mutex::new(rx), + controls_tx: tx, + } + } +} + +impl ServiceHost for StubHost { + fn poll_control(&self) -> Option { + self.controls + .lock() + .expect("controls receiver lock should not be poisoned") + .try_recv() + .ok() + } + + fn report_ready(&self) { + self.ready_calls.fetch_add(1, Ordering::SeqCst); + } + + fn heartbeat(&self) { + self.heartbeat_calls.fetch_add(1, Ordering::SeqCst); + } + + fn report_stopping(&self) { + self.stopping_calls.fetch_add(1, Ordering::SeqCst); + } +} + +// --------------------------------------------------------------------------- +// StubProbe — scripted HealthProbe +// --------------------------------------------------------------------------- + +/// Scripted readiness for the probation gate. +pub enum StubProbe { + /// Always this value. + Always(bool), + /// `false` until the instant, then `true` — pairs with an `UpNotReady` + /// child to model an operator that becomes ready after a delay. + ReadyAt(Instant), +} + +impl HealthProbe for StubProbe { + fn is_ready(&self, _url: &str) -> bool { + match self { + StubProbe::Always(ready) => *ready, + StubProbe::ReadyAt(instant) => Instant::now() >= *instant, + } + } +} + +/// A loopback `UpdateEnv` for tests. +pub fn test_update_env() -> UpdateEnv { + UpdateEnv { + health_addr: SocketAddr::from(([127, 0, 0, 1], 7799)), + launcher_version: "0.1.0-test".to_string(), + } +} + + +// --------------------------------------------------------------------------- +// Parameterized state-machine scenarios +// --------------------------------------------------------------------------- +// +// The Phase-0 state-machine suite, written against ANY `VersionStore` so the +// platform stores prove they honor the on-disk protocol by running the exact +// same scenarios the stub runs. `TestStoreOps` supplies the test-only store +// knowledge (how to install a version, where `state/` lives). + +use crate::core::state_machine::{ + classify_startup, Machine, RunConfig, StartupAction, +}; +use alien_core::sync::OperatorUpdatePhase; +use chrono::Utc; + +/// Test-only operations every store under test must provide. +pub trait TestStoreOps: VersionStore { + /// Create `versions//alien-operator` with deterministic content. + fn install_version(&self, version: &Version); + /// The live `state/` directory (for mutation + restore assertions). + fn state_dir_path(&self) -> PathBuf; + /// The store root (for snapshot-existence assertions). + fn store_root(&self) -> PathBuf; +} + +impl TestStoreOps for StubStore { + fn install_version(&self, version: &Version) { + StubStore::install_version(self, version) + } + fn state_dir_path(&self) -> PathBuf { + self.state_dir() + } + fn store_root(&self) -> PathBuf { + self.root().to_path_buf() + } +} + +pub fn test_run_config() -> RunConfig { + RunConfig { + probation_window: Duration::from_millis(200), + probe_interval: Duration::from_millis(5), + poll_interval: Duration::from_millis(5), + heartbeat_interval: Duration::from_millis(10), + stop_grace: Duration::from_millis(50), + restart_backoff_base: Duration::from_millis(5), + restart_backoff_cap: Duration::from_millis(40), + healthy_reset: Duration::from_millis(150), + max_swap_attempts: 3, + operator_binary: "alien-operator".to_string(), + ..RunConfig::default() + } +} + +fn v(s: &str) -> Version { + Version::parse(s).expect("test version should parse") +} + +/// Seed the baseline: 1.3.5 installed, both pointers on it, a state DB. +pub fn seed_base(store: &S) { + store.install_version(&v("1.3.5")); + store.set_current(&v("1.3.5")).unwrap(); + store.set_last_stable(&v("1.3.5")).unwrap(); + std::fs::write(store.state_dir_path().join("db"), b"state-v1").unwrap(); +} + +/// Install 1.4.0 and write a valid pending marker (real sha256). +pub fn stage_valid(store: &S, config: &RunConfig) -> PendingMarker { + store.install_version(&v("1.4.0")); + let binary = store.stage_dir(&v("1.4.0")).join(&config.operator_binary); + let sha256 = store_common::file_sha256(&binary).unwrap(); + let pending = PendingMarker { + version: v("1.4.0"), + sha256, + staged_at: Utc::now(), + }; + store.write_pending(&pending).unwrap(); + pending +} + +pub fn probation_marker(attempt: u32, started_ago: Duration) -> ProbationMarker { + ProbationMarker { + new: v("1.4.0"), + old: v("1.3.5"), + started_at: Utc::now() - chrono::Duration::from_std(started_ago).unwrap(), + attempt, + } +} + +pub fn assert_steady_promoted(store: &S) { + assert_eq!(store.current().unwrap(), Some(v("1.4.0"))); + assert_eq!(store.last_stable().unwrap(), Some(v("1.4.0"))); + assert!(store.read_pending().unwrap().is_none(), "pending cleared"); + assert!(store.read_probation().unwrap().is_none(), "probation cleared"); + assert!( + !store.store_root().join("state-snapshots/1.3.5").exists(), + "snapshot dropped on promote" + ); +} + +pub fn assert_rolled_back(store: &S, expected_attempts: u32) { + assert_eq!(store.current().unwrap(), Some(v("1.3.5"))); + assert_eq!(store.last_stable().unwrap(), Some(v("1.3.5"))); + assert_eq!( + std::fs::read(store.state_dir_path().join("db")).unwrap(), + b"state-v1", + "state restored from the snapshot" + ); + assert!(store.read_pending().unwrap().is_none()); + assert!(store.read_probation().unwrap().is_none()); + let record = store + .read_failure(&v("1.4.0")) + .unwrap() + .expect("failure record written"); + assert_eq!(record.attempts, expected_attempts); + assert_eq!(record.phase, OperatorUpdatePhase::Apply); +} + +macro_rules! machine { + ($store:expr, $child:expr, $probe:expr, $host:expr, $config:expr) => { + Machine { + store: &$store, + child: &mut $child, + probe: &$probe, + host: &$host, + config: &$config, + } + }; +} + +// -- the scenarios --------------------------------------------------------- + +pub fn scenario_happy_promote(make: impl Fn(&Path) -> S) { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + let config = test_run_config(); + let pending = stage_valid(&store, &config); + + let mut child = StubChild::new([SpawnOutcome::UpNotReady]); + let probe = StubProbe::ReadyAt(Instant::now() + Duration::from_millis(30)); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + + let action = classify_startup(&store, &config).unwrap(); + assert_eq!(action, StartupAction::RunSwap { pending }); + let handle = m.execute_startup(action).unwrap(); + + assert_steady_promoted(&store); + assert_eq!( + store.list_versions().unwrap(), + vec![v("1.4.0")], + "old version gc'd after promote" + ); + assert_eq!(child.spawned.len(), 1, "exactly the new operator spawned"); + assert_eq!( + child.spawned[0].0, + store.stage_dir(&v("1.4.0")).join("alien-operator") + ); + assert!(child.try_wait(&handle).unwrap().is_none(), "still running"); + assert!( + host.ready_calls.load(Ordering::SeqCst) >= 1, + "READY reported on promote" + ); +} + +pub fn scenario_rollback_restores_state(make: impl Fn(&Path) -> S) { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + let config = test_run_config(); + stage_valid(&store, &config); + + // The "new operator" (1.4.0) migrates the DB on spawn; rollback must + // undo it. The old operator's respawn must NOT re-mutate (path filter). + let state_db = store.state_dir_path().join("db"); + let mut child = StubChild::new([SpawnOutcome::UpNotReady, SpawnOutcome::UpNotReady]); + child.on_spawn = Some(Box::new(move |binary: &Path| { + if binary.to_string_lossy().contains("1.4.0") { + std::fs::write(&state_db, b"MIGRATED").unwrap(); + } + })); + let probe = StubProbe::Always(false); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + + let action = classify_startup(&store, &config).unwrap(); + m.execute_startup(action).unwrap(); + + assert_rolled_back(&store, 1); + assert_eq!(child.stop_calls.len(), 1, "failed child was stopped"); + assert_eq!(child.spawned.len(), 2, "old operator respawned"); + let record = store.read_failure(&v("1.4.0")).unwrap().unwrap(); + assert!( + record.message.contains("probation window"), + "message explains the timeout: {}", + record.message + ); + + // Second identical attempt increments the count. + let config2 = test_run_config(); + stage_valid(&store, &config2); + let mut child2 = StubChild::new([SpawnOutcome::UpNotReady, SpawnOutcome::UpNotReady]); + let probe2 = StubProbe::Always(false); + let host2 = StubHost::new(); + let mut m2 = machine!(store, child2, probe2, host2, config2); + let action = classify_startup(&store, &config2).unwrap(); + m2.execute_startup(action).unwrap(); + assert_rolled_back(&store, 2); +} + +pub fn scenario_rollback_on_probation_crash(make: impl Fn(&Path) -> S) { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + let config = test_run_config(); + stage_valid(&store, &config); + + let mut child = StubChild::new([SpawnOutcome::ExitImmediately(1), SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(false); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + + let action = classify_startup(&store, &config).unwrap(); + m.execute_startup(action).unwrap(); + + assert_rolled_back(&store, 1); + let record = store.read_failure(&v("1.4.0")).unwrap().unwrap(); + assert!(record.message.contains("code 1"), "{}", record.message); +} + +/// Every startup-classification row, constructed on disk and executed to a +/// coherent terminal state. +pub fn scenario_classification_rows(make: impl Fn(&Path) -> S) { + let config = test_run_config(); + + // Row 1 — steady state. + { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + assert_eq!( + classify_startup(&store, &config).unwrap(), + StartupAction::SpawnCurrent + ); + } + + // Row 1, first install: last-stable recorded after a passed gate. + { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + store.install_version(&v("1.3.5")); + store.set_current(&v("1.3.5")).unwrap(); + assert_eq!( + classify_startup(&store, &config).unwrap(), + StartupAction::SpawnCurrent + ); + let mut child = StubChild::new([SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + m.execute_startup(StartupAction::SpawnCurrent).unwrap(); + assert_eq!(store.last_stable().unwrap(), Some(v("1.3.5"))); + assert!(host.ready_calls.load(Ordering::SeqCst) >= 1); + } + + // Row 2 guard — leftover pending after a completed promote. + { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + store.install_version(&v("1.4.0")); + store.set_current(&v("1.4.0")).unwrap(); + store.set_last_stable(&v("1.4.0")).unwrap(); + let binary = store.stage_dir(&v("1.4.0")).join(&config.operator_binary); + let pending = PendingMarker { + version: v("1.4.0"), + sha256: store_common::file_sha256(&binary).unwrap(), + staged_at: Utc::now(), + }; + store.write_pending(&pending).unwrap(); + + let action = classify_startup(&store, &config).unwrap(); + assert_eq!(action, StartupAction::DiscardLeftoverPending { pending }); + let mut child = StubChild::new([SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + m.execute_startup(action).unwrap(); + assert!(store.read_pending().unwrap().is_none()); + assert_eq!(store.current().unwrap(), Some(v("1.4.0"))); + } + + // Row 3 — invalid pending discarded, current spawned, no swap. + { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + store.install_version(&v("1.4.0")); + let pending = PendingMarker { + version: v("1.4.0"), + sha256: "0".repeat(64), + staged_at: Utc::now(), + }; + store.write_pending(&pending).unwrap(); + + let action = classify_startup(&store, &config).unwrap(); + assert_eq!(action, StartupAction::DiscardInvalidPending { pending }); + let mut child = StubChild::new([SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + m.execute_startup(action).unwrap(); + assert!(store.read_pending().unwrap().is_none()); + assert_eq!(store.current().unwrap(), Some(v("1.3.5"))); + assert_eq!( + child.spawned[0].0, + store.stage_dir(&v("1.3.5")).join("alien-operator") + ); + } + + // Row 4 — mid-probation crash: resume and promote. + { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + stage_valid(&store, &config); + store.snapshot_state(&v("1.3.5")).unwrap(); + store + .write_probation(&probation_marker(1, Duration::from_millis(50))) + .unwrap(); + store.set_current(&v("1.4.0")).unwrap(); + + let action = classify_startup(&store, &config).unwrap(); + let StartupAction::ResumeProbation { remaining, .. } = &action else { + panic!("expected ResumeProbation, got {action:?}"); + }; + assert!(*remaining > Duration::ZERO && *remaining < config.probation_window); + let mut child = StubChild::new([SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + m.execute_startup(action).unwrap(); + assert_steady_promoted(&store); + } + + // Row 4, expired window: roll back even with a ready probe. + { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + stage_valid(&store, &config); + store.snapshot_state(&v("1.3.5")).unwrap(); + store + .write_probation(&probation_marker(1, Duration::from_secs(10))) + .unwrap(); + store.set_current(&v("1.4.0")).unwrap(); + + let action = classify_startup(&store, &config).unwrap(); + let StartupAction::ResumeProbation { remaining, .. } = &action else { + panic!("expected ResumeProbation, got {action:?}"); + }; + assert_eq!(*remaining, Duration::ZERO); + let mut child = StubChild::new([SpawnOutcome::UpNotReady, SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + m.execute_startup(action).unwrap(); + assert_rolled_back(&store, 1); + } + + // Row 4b — promote began: finish cleanup. + { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + stage_valid(&store, &config); + store.snapshot_state(&v("1.3.5")).unwrap(); + store + .write_probation(&probation_marker(1, Duration::from_millis(10))) + .unwrap(); + store.set_current(&v("1.4.0")).unwrap(); + store.set_last_stable(&v("1.4.0")).unwrap(); + + let action = classify_startup(&store, &config).unwrap(); + assert!(matches!(action, StartupAction::FinishPromote { .. }), "got {action:?}"); + let mut child = StubChild::new([SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + m.execute_startup(action).unwrap(); + assert_steady_promoted(&store); + } + + // Row 5 — pre-flip crash: resume the swap and promote. + { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + stage_valid(&store, &config); + store.snapshot_state(&v("1.3.5")).unwrap(); + store + .write_probation(&probation_marker(1, Duration::from_millis(10))) + .unwrap(); + + let action = classify_startup(&store, &config).unwrap(); + assert!(matches!(action, StartupAction::ResumeSwapAtFlip { .. }), "got {action:?}"); + let mut child = StubChild::new([SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + m.execute_startup(action).unwrap(); + assert_steady_promoted(&store); + } + + // Row 5 cap — attempt budget exhausted: abort to rollback. + { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + stage_valid(&store, &config); + store.snapshot_state(&v("1.3.5")).unwrap(); + store + .write_probation(&probation_marker(config.max_swap_attempts, Duration::from_millis(10))) + .unwrap(); + + let action = classify_startup(&store, &config).unwrap(); + assert!(matches!(action, StartupAction::AbortSwap { .. }), "got {action:?}"); + let mut child = StubChild::new([SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + m.execute_startup(action).unwrap(); + assert_rolled_back(&store, config.max_swap_attempts); + } + + // Row 6 — mid-rollback crash (failure record present): finish rollback. + { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + stage_valid(&store, &config); + store.snapshot_state(&v("1.3.5")).unwrap(); + store + .write_probation(&probation_marker(1, Duration::from_millis(10))) + .unwrap(); + store + .write_failure(&FailureRecord { + version: v("1.4.0"), + sha256: "beef".to_string(), + phase: OperatorUpdatePhase::Apply, + message: "gate failed before the crash".to_string(), + attempts: 1, + last_failed_at: Utc::now(), + }) + .unwrap(); + std::fs::write(store.state_dir_path().join("db"), b"MIGRATED").unwrap(); + + let action = classify_startup(&store, &config).unwrap(); + assert!(matches!(action, StartupAction::FinishRollback { .. }), "got {action:?}"); + let mut child = StubChild::new([SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + m.execute_startup(action).unwrap(); + assert_eq!(store.current().unwrap(), Some(v("1.3.5"))); + assert_eq!( + std::fs::read(store.state_dir_path().join("db")).unwrap(), + b"state-v1", + "restore re-ran" + ); + assert!(store.read_probation().unwrap().is_none()); + assert!(store.read_pending().unwrap().is_none()); + } +} + +// -- crash injection -------------------------------------------------------- + +/// A store decorator that fails the k-th MUTATING operation, simulating a +/// launcher crash at every swap-step boundary. Generic so the matrix runs +/// against any store under test. +pub struct FailingStore<'a, S: VersionStore> { + inner: &'a S, + fail_at: u32, + mutations: std::cell::Cell, +} + +impl<'a, S: VersionStore> FailingStore<'a, S> { + pub fn new(inner: &'a S, fail_at: u32) -> Self { + Self { + inner, + fail_at, + mutations: std::cell::Cell::new(0), + } + } + + fn trip(&self, op: &str) -> Result<()> { + let n = self.mutations.get() + 1; + self.mutations.set(n); + if n == self.fail_at { + Err(AlienError::new(ErrorData::Other { + message: format!("injected crash at mutation #{n} ({op})"), + })) + } else { + Ok(()) + } + } +} + +impl VersionStore for FailingStore<'_, S> { + fn stage_dir(&self, version: &Version) -> PathBuf { + self.inner.stage_dir(version) + } + fn current(&self) -> Result> { + self.inner.current() + } + fn last_stable(&self) -> Result> { + self.inner.last_stable() + } + fn set_current(&self, version: &Version) -> Result<()> { + self.trip("set_current")?; + self.inner.set_current(version) + } + fn set_last_stable(&self, version: &Version) -> Result<()> { + self.trip("set_last_stable")?; + self.inner.set_last_stable(version) + } + fn snapshot_state(&self, tag: &Version) -> Result<()> { + self.trip("snapshot_state")?; + self.inner.snapshot_state(tag) + } + fn restore_state(&self, tag: &Version) -> Result<()> { + self.trip("restore_state")?; + self.inner.restore_state(tag) + } + fn drop_snapshot(&self, tag: &Version) -> Result<()> { + self.trip("drop_snapshot")?; + self.inner.drop_snapshot(tag) + } + fn state_size(&self) -> Result { + self.inner.state_size() + } + fn gc(&self, keep: &[Version]) -> Result<()> { + self.trip("gc")?; + self.inner.gc(keep) + } + fn read_pending(&self) -> Result> { + self.inner.read_pending() + } + fn write_pending(&self, marker: &PendingMarker) -> Result<()> { + self.trip("write_pending")?; + self.inner.write_pending(marker) + } + fn clear_pending(&self) -> Result<()> { + self.trip("clear_pending")?; + self.inner.clear_pending() + } + fn read_probation(&self) -> Result> { + self.inner.read_probation() + } + fn write_probation(&self, marker: &ProbationMarker) -> Result<()> { + self.trip("write_probation")?; + self.inner.write_probation(marker) + } + fn clear_probation(&self) -> Result<()> { + self.trip("clear_probation")?; + self.inner.clear_probation() + } + fn read_failure(&self, version: &Version) -> Result> { + self.inner.read_failure(version) + } + fn write_failure(&self, record: &FailureRecord) -> Result<()> { + self.trip("write_failure")?; + self.inner.write_failure(record) + } + fn list_versions(&self) -> Result> { + self.inner.list_versions() + } + fn free_space_for_snapshot(&self) -> Result<()> { + self.inner.free_space_for_snapshot() + } +} + +/// Crash-inject at every mutating store call of the promote path, recover +/// with a fresh machine over the same store, and assert convergence. +pub fn scenario_crash_injection_promote(make: impl Fn(&Path) -> S) { + for fail_at in 1..=8u32 { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + let config = test_run_config(); + stage_valid(&store, &config); + + { + let failing = FailingStore::new(&store, fail_at); + let mut child = StubChild::new([SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let mut m = machine!(failing, child, probe, host, config); + let action = classify_startup(&failing, &config).unwrap(); + assert!( + m.execute_startup(action).is_err(), + "fail_at={fail_at}: the injected crash must surface" + ); + } + + let mut child = StubChild::new([ + SpawnOutcome::UpNotReady, + SpawnOutcome::UpNotReady, + SpawnOutcome::UpNotReady, + ]); + let probe = StubProbe::Always(true); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + let action = classify_startup(&store, &config).unwrap(); + m.execute_startup(action) + .unwrap_or_else(|e| panic!("fail_at={fail_at}: recovery must succeed: {e}")); + + assert!(store.read_probation().unwrap().is_none(), "fail_at={fail_at}"); + assert!(store.read_pending().unwrap().is_none(), "fail_at={fail_at}"); + let current = store.current().unwrap().expect("current set"); + let stable = store.last_stable().unwrap().expect("stable set"); + assert_eq!(current, stable, "fail_at={fail_at}: pointers agree"); + assert!( + current == v("1.4.0") || current == v("1.3.5"), + "fail_at={fail_at}: terminal version is one of the pair" + ); + if current == v("1.3.5") { + assert_eq!( + std::fs::read(store.state_dir_path().join("db")).unwrap(), + b"state-v1", + "fail_at={fail_at}: rolled back ⇒ state restored" + ); + } + } +} + +/// Same matrix on the rollback path (probe never ready). +pub fn scenario_crash_injection_rollback(make: impl Fn(&Path) -> S) { + for fail_at in 4..=8u32 { + let dir = tempfile::tempdir().unwrap(); + let store = make(dir.path()); + seed_base(&store); + let config = test_run_config(); + stage_valid(&store, &config); + + { + let failing = FailingStore::new(&store, fail_at); + let mut child = + StubChild::new([SpawnOutcome::UpNotReady, SpawnOutcome::UpNotReady]); + let probe = StubProbe::Always(false); + let host = StubHost::new(); + let mut m = machine!(failing, child, probe, host, config); + let action = classify_startup(&failing, &config).unwrap(); + assert!(m.execute_startup(action).is_err(), "fail_at={fail_at}"); + } + + let mut child = StubChild::new([ + SpawnOutcome::UpNotReady, + SpawnOutcome::UpNotReady, + SpawnOutcome::UpNotReady, + ]); + let probe = StubProbe::Always(false); + let host = StubHost::new(); + let mut m = machine!(store, child, probe, host, config); + let action = classify_startup(&store, &config).unwrap(); + m.execute_startup(action) + .unwrap_or_else(|e| panic!("fail_at={fail_at}: recovery must succeed: {e}")); + + assert!(store.read_probation().unwrap().is_none(), "fail_at={fail_at}"); + assert!(store.read_pending().unwrap().is_none(), "fail_at={fail_at}"); + assert_eq!(store.current().unwrap(), Some(v("1.3.5")), "fail_at={fail_at}"); + assert_eq!( + std::fs::read(store.state_dir_path().join("db")).unwrap(), + b"state-v1", + "fail_at={fail_at}: state restored" + ); + } +} + +// --------------------------------------------------------------------------- +// Tests — the stubs themselves must honor the trait contracts +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn version(s: &str) -> Version { + Version::parse(s).expect("test version should parse") + } + + fn store() -> (tempfile::TempDir, StubStore) { + let dir = tempfile::tempdir().expect("tempdir should create"); + let store = StubStore::new(dir.path()); + (dir, store) + } + + /// Pointers: unset → None; set → read back; flip → new value. No symlinks + /// anywhere (the whole point of the stub). + #[test] + fn stub_store_pointers_roundtrip_without_symlinks() { + let (_dir, store) = store(); + assert_eq!(store.current().unwrap(), None); + assert_eq!(store.last_stable().unwrap(), None); + + store.install_version(&version("1.3.5")); + store.set_current(&version("1.3.5")).unwrap(); + store.set_last_stable(&version("1.3.5")).unwrap(); + assert_eq!(store.current().unwrap(), Some(version("1.3.5"))); + assert_eq!(store.last_stable().unwrap(), Some(version("1.3.5"))); + + store.install_version(&version("1.4.0")); + store.set_current(&version("1.4.0")).unwrap(); + assert_eq!(store.current().unwrap(), Some(version("1.4.0"))); + assert_eq!( + store.last_stable().unwrap(), + Some(version("1.3.5")), + "flipping current must not move last-stable" + ); + } + + /// Markers ride the shared atomic helpers: write/read/clear round-trip and + /// clearing twice stays Ok (idempotent). + #[test] + fn stub_store_markers_roundtrip_and_clear_idempotent() { + let (_dir, store) = store(); + assert!(store.read_pending().unwrap().is_none()); + + let pending = PendingMarker { + version: version("1.4.0"), + sha256: "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3f9c71a1e4a6f2e0e6d5c4b3a" + .to_string(), + staged_at: "2026-07-08T12:00:00Z".parse().unwrap(), + }; + store.write_pending(&pending).unwrap(); + assert_eq!(store.read_pending().unwrap(), Some(pending)); + store.clear_pending().unwrap(); + store.clear_pending().unwrap(); + assert!(store.read_pending().unwrap().is_none()); + + let record = FailureRecord { + version: version("1.4.0"), + sha256: "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3f9c71a1e4a6f2e0e6d5c4b3a" + .to_string(), + phase: alien_core::sync::OperatorUpdatePhase::Apply, + message: "probe timeout".to_string(), + attempts: 1, + last_failed_at: "2026-07-08T12:05:00Z".parse().unwrap(), + }; + store.write_failure(&record).unwrap(); + assert_eq!(store.read_failure(&version("1.4.0")).unwrap(), Some(record)); + assert!(store.read_failure(&version("9.9.9")).unwrap().is_none()); + } + + /// gc through the store: current + last-stable survive, others go. + #[test] + fn stub_store_gc_preserves_pointer_targets() { + let (_dir, store) = store(); + for v in ["1.0.0", "1.1.0", "1.2.0"] { + store.install_version(&version(v)); + } + store.set_current(&version("1.2.0")).unwrap(); + store.set_last_stable(&version("1.1.0")).unwrap(); + + store.gc(&[]).unwrap(); + + assert_eq!( + store.list_versions().unwrap(), + vec![version("1.1.0"), version("1.2.0")], + "only the unpointed version is collected" + ); + } + + /// snapshot + restore through the store round-trips state bytes. + #[test] + fn stub_store_snapshot_restore_roundtrip() { + let (_dir, store) = store(); + std::fs::write(store.state_dir().join("db"), b"pre-migration").unwrap(); + store.snapshot_state(&version("1.3.5")).unwrap(); + std::fs::write(store.state_dir().join("db"), b"migrated!").unwrap(); + store.restore_state(&version("1.3.5")).unwrap(); + assert_eq!( + std::fs::read(store.state_dir().join("db")).unwrap(), + b"pre-migration" + ); + } + + /// The scripted free-space check trips the shared DiskSpace policy. + #[test] + fn stub_store_free_space_scriptable() { + let (_dir, store) = store(); + std::fs::write(store.state_dir().join("db"), vec![0u8; 1000]).unwrap(); + + store.free_space_for_snapshot().expect("unlimited space passes"); + + store.available_bytes.store(1100, Ordering::SeqCst); + let err = store + .free_space_for_snapshot() + .expect_err("1000 needed + 20% headroom > 1100 available"); + assert_eq!(err.code, "DISK_SPACE"); + } + + /// StubChild: outcomes script spawn-by-spawn; handoff code 10 and crash + /// codes surface through try_wait; stop() records and terminates. + #[test] + fn stub_child_scripts_outcomes() { + let env = test_update_env(); + let mut child = StubChild::new([ + SpawnOutcome::ExitImmediately(10), + SpawnOutcome::UpNotReady, + ]); + + let first = child.spawn(Path::new("/versions/1.3.5/op"), &env).unwrap(); + assert_eq!( + child.try_wait(&first).unwrap(), + Some(ExitStatus::Code(10)), + "scripted handoff exit" + ); + + let second = child.spawn(Path::new("/versions/1.4.0/op"), &env).unwrap(); + assert_eq!(child.try_wait(&second).unwrap(), None, "still running"); + child.stop(&second, Duration::from_secs(2)).unwrap(); + assert_eq!(child.try_wait(&second).unwrap(), Some(ExitStatus::Code(0))); + + assert_eq!(child.spawned.len(), 2); + assert_eq!(child.stop_calls, vec![second.pid]); + // Script exhausted → further spawns fail loudly. + let err = child + .spawn(Path::new("/versions/1.5.0/op"), &env) + .expect_err("exhausted script must not silently succeed"); + assert_eq!(err.code, "SPAWN_FAILED"); + } + + /// StubHost records lifecycle calls and delivers scripted controls. + #[test] + fn stub_host_records_and_delivers_controls() { + let host = StubHost::new(); + host.report_ready(); + host.heartbeat(); + host.heartbeat(); + host.report_stopping(); + assert_eq!(host.ready_calls.load(Ordering::SeqCst), 1); + assert_eq!(host.heartbeat_calls.load(Ordering::SeqCst), 2); + assert_eq!(host.stopping_calls.load(Ordering::SeqCst), 1); + + assert_eq!(host.poll_control(), None, "no control queued yet"); + host.controls_tx.send(Control::Stop).unwrap(); + assert_eq!(host.poll_control(), Some(Control::Stop)); + assert_eq!(host.poll_control(), None, "controls are drained once"); + } + + /// StubProbe: Always and ReadyAt behave as scripted. + #[test] + fn stub_probe_scripts_readiness() { + assert!(StubProbe::Always(true).is_ready("http://127.0.0.1:7799/readyz")); + assert!(!StubProbe::Always(false).is_ready("http://127.0.0.1:7799/readyz")); + + let soon = StubProbe::ReadyAt(Instant::now() + Duration::from_millis(50)); + assert!(!soon.is_ready("x"), "not ready before the instant"); + std::thread::sleep(Duration::from_millis(60)); + assert!(soon.is_ready("x"), "ready after the instant"); + } +} diff --git a/crates/alien-launcher/src/core/traits.rs b/crates/alien-launcher/src/core/traits.rs new file mode 100644 index 000000000..663716302 --- /dev/null +++ b/crates/alien-launcher/src/core/traits.rs @@ -0,0 +1,306 @@ +//! The trait boundary between the OS-agnostic core and the platform shims. +//! +//! Four traits isolate every platform difference; the core state machine is +//! written once against them and never names a syscall, a signal, a symlink, +//! or a Job Object. Signatures here are deliberately reviewed against the +//! constraints of ALL THREE OSes (no symlink assumption, no PDEATHSIG +//! assumption, Job-Object-compatible stop semantics, launchd's lack of a +//! notify protocol) so that no platform shim ever forces a signature change. +//! +//! Per-OS realization of each method: +//! +//! | Trait method | Linux (systemd) | macOS (launchd) | Windows (SCM) | +//! |---|---|---|---| +//! | `ServiceHost::poll_control` | signal thread → channel, drained non-blocking | same | SCM control handler → channel, drained non-blocking | +//! | `ServiceHost::report_ready` | `sd-notify` READY=1 | no-op | `SERVICE_RUNNING` | +//! | `ServiceHost::heartbeat` | `sd-notify` WATCHDOG=1 | no-op | checkpoint bump | +//! | `ChildSupervisor::spawn` | process group + parent-death signal | process group + operator-side parent watch | Job Object (kill-on-close) | +//! | `ChildSupervisor::stop` | SIGTERM then SIGKILL | SIGTERM then SIGKILL | CTRL_BREAK then Job terminate | +//! | `VersionStore::set_current` | atomic rename of a symlink | atomic rename of a symlink | directory-junction swap | +//! | `VersionStore::gc` | unlink | unlink | delayed delete for locked files | +//! | `HealthProbe::is_ready` | HTTP GET (shared impl) | shared | shared | + +// Skeleton staging: the env consts, ExitStatus/Control variants, and a few +// helpers are constructed by the platform shims; until the first shim lands +// the non-test build sees them as dead. Remove once wired. +#![allow(dead_code)] + +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::error::Result; + +// --------------------------------------------------------------------------- +// Core types +// --------------------------------------------------------------------------- + +// The pieces of the launcher↔operator protocol that BOTH binaries serialize +// (marker shapes, env-var names, the exit-code contract, the semver Version) +// live in the shared crate — `alien_core::self_update` — and are re-exported +// here so launcher code keeps its natural paths. `ProbationMarker` below is +// deliberately NOT shared: it is launcher-private bookkeeping the operator +// never reads. +pub use alien_core::self_update::{ + FailureRecord, PendingMarker, Version, EXIT_CODE_UPDATE_HANDOFF, +}; +// (The spawn env-var names — ENV_SELF_UPDATE / ENV_LAUNCHER_VERSION / +// ENV_HEALTH_ADDR — also live in `alien_core::self_update`; the platform +// shims import them from there when they map `UpdateEnv` onto the child's +// environment.) + +/// Opaque handle to a spawned operator child. Only meaningful to the +/// `ChildSupervisor` that produced it. +#[derive(Debug, Clone)] +pub struct OperatorHandle { + /// OS process id of the operator child (valid on all three OSes). + pub pid: u32, +} + +/// Environment the launcher passes to the operator on spawn. The +/// `ChildSupervisor` maps these onto the boundary's environment variables +/// (`ENV_SELF_UPDATE`, `ENV_LAUNCHER_VERSION`, `ENV_HEALTH_ADDR`). +#[derive(Debug, Clone)] +pub struct UpdateEnv { + /// Where the operator must serve `/readyz` + `/livez`, and where the + /// launcher probes during probation. Always a loopback address — the + /// endpoints are consumed only by the local launcher. + pub health_addr: SocketAddr, + /// The launcher's own version, reported by the operator on sync so the + /// manager can gate targets on `min_launcher_version`. + pub launcher_version: String, +} + +/// A lifecycle request from the OS supervisor. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Control { + /// Stop the service (systemctl stop / launchctl bootout / SCM Stop). + Stop, + /// The host is shutting down (SCM Shutdown; folded into SIGTERM on Unix). + Shutdown, +} + +/// How the operator child exited. +/// +/// Our own enum rather than `std::process::ExitStatus` so the same type fits +/// all three OSes: Windows has exit codes but no signals, Unix has both. The +/// state machine's exit-code contract (0 = clean stop, 10 = update handoff, +/// other = crash) is decided on `Code`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExitStatus { + /// Exited with a code. + Code(i32), + /// Killed by a signal (Unix only; never produced on Windows). + Signal, + /// The platform could not report how the process ended. + Unknown, +} + +// --------------------------------------------------------------------------- +// Marker files (the on-disk handoff protocol) +// --------------------------------------------------------------------------- + +/// `probation.json` — written by the LAUNCHER at the start of a swap, before +/// flipping `current`, so a crash at any later step is classifiable on +/// restart. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProbationMarker { + /// The version being promoted. + pub new: Version, + /// The version we swap away from (and roll back to on failure). + pub old: Version, + /// Wall-clock start of the probation window. A restarted launcher resumes + /// with the remaining time; negative elapsed (NTP step) clamps to zero. + pub started_at: DateTime, + /// 1-based swap attempt for this target version. + pub attempt: u32, +} + +// --------------------------------------------------------------------------- +// The four boundary traits +// --------------------------------------------------------------------------- + +/// Up-facing: how the launcher reports to, and receives control from, the OS +/// init system. +/// +/// | | Linux (systemd) | macOS (launchd) | Windows (SCM) | +/// |---|---|---|---| +/// | `poll_control` | signal thread → channel | same | control handler → channel | +/// | `report_ready` | READY=1 | no-op | `SERVICE_RUNNING` | +/// | `heartbeat` | WATCHDOG=1 | no-op | checkpoint bump | +/// | `report_stopping` | STOPPING=1 | no-op | `STOP_PENDING` → `STOPPED` | +pub trait ServiceHost { + /// Non-blocking: has the supervisor requested a lifecycle change since the + /// last poll? The run loop polls this every tick. (Deliberately not a + /// blocking wait: a blocked receive with no cancellation would leave the + /// control listener unjoinable on fatal-error paths. Platform impls + /// deliver signals / SCM controls onto an internal channel and this + /// method drains it.) + fn poll_control(&self) -> Option; + + /// A healthy operator is up. + fn report_ready(&self); + + /// Liveness ping. MUST be callable from a dedicated heartbeat thread that + /// ticks for the launcher's whole life — including during a probation + /// window longer than the watchdog interval — and must never be starved + /// by the update loop. + fn heartbeat(&self); + + /// We are shutting down. + fn report_stopping(&self); +} + +/// Down-facing: spawn/stop/observe the operator child. +/// +/// Die-with-parent is NORMATIVE, not hardening: an orphaned operator severs +/// the exit-code handoff channel, deadlocks the respawned launcher against +/// the operator's instance lock, and breaks stop/redeploy semantics. Every +/// implementation must guarantee the operator dies when the launcher dies +/// (kill-group / Job Object / parent watch), with the operator's instance +/// lock as the correctness backstop. +/// +/// | | Linux | macOS | Windows | +/// |---|---|---|---| +/// | `spawn` | process group + parent-death signal | process group + operator-side parent watch | Job Object, kill-on-close | +/// | `stop` | SIGTERM → SIGKILL | SIGTERM → SIGKILL | CTRL_BREAK → Job terminate | +pub trait ChildSupervisor { + /// Spawn the operator binary inside a kill-group, with the `UpdateEnv` + /// mapped onto `ENV_SELF_UPDATE=1`, `ENV_LAUNCHER_VERSION`, and + /// `ENV_HEALTH_ADDR`. + fn spawn(&mut self, binary: &Path, env: &UpdateEnv) -> Result; + + /// Graceful stop, escalating to force-kill after `grace`. + fn stop(&mut self, handle: &OperatorHandle, grace: Duration) -> Result<()>; + + /// Non-blocking: has the child exited, and how? + fn try_wait(&mut self, handle: &OperatorHandle) -> Result>; +} + +/// The on-disk version store: `versions/`, the `current` / `last-stable` +/// pointers, `state/` snapshots, the protocol marker files, and gc. +/// +/// Hides symlink-vs-junction and locked-file deletion. ALL marker writes MUST +/// be atomic — write `.tmp`, fsync, rename — so a crash never leaves a +/// half-written marker for startup classification to trip on. +/// +/// | | Linux / macOS | Windows | +/// |---|---|---| +/// | pointer flip | atomic `rename` of a symlink | directory-junction swap | +/// | `gc` | unlink | delayed delete for locked `.exe`s | +pub trait VersionStore { + /// Directory where a version's binary is staged: `versions//`. + fn stage_dir(&self, version: &Version) -> PathBuf; + + /// The version `current` points at. `None` on a store that has never had + /// a version installed. + fn current(&self) -> Result>; + + /// The proven-good fallback `last-stable` points at. + fn last_stable(&self) -> Result>; + + /// Atomically repoint `current`. + fn set_current(&self, version: &Version) -> Result<()>; + + /// Atomically repoint `last-stable`. + fn set_last_stable(&self, version: &Version) -> Result<()>; + + /// Copy `state/` to `state-snapshots//` (temp dir + rename). Called + /// before every swap so rollback restores a (binary + state) pair the old + /// binary can open. + fn snapshot_state(&self, tag: &Version) -> Result<()>; + + /// Restore `state/` from `state-snapshots//` during rollback. + fn restore_state(&self, tag: &Version) -> Result<()>; + + /// Remove `state-snapshots//` (idempotent) — promote's final step. + fn drop_snapshot(&self, tag: &Version) -> Result<()>; + + /// Size of `state/` in bytes — logged with every swap so snapshot-copy + /// cost growth is visible before it hurts. + fn state_size(&self) -> Result; + + /// Delete versions not in `keep`. Implementations must never delete the + /// versions `current` or `last-stable` point at, regardless of `keep`. + fn gc(&self, keep: &[Version]) -> Result<()>; + + // --- protocol marker files (atomic temp+fsync+rename writes) --- + + /// Read `pending.json`; `None` when absent. + fn read_pending(&self) -> Result>; + + /// Atomically write `pending.json`. + fn write_pending(&self, marker: &PendingMarker) -> Result<()>; + + /// Remove `pending.json` (idempotent — absent is not an error). + fn clear_pending(&self) -> Result<()>; + + /// Read `probation.json`; `None` when absent. + fn read_probation(&self) -> Result>; + + /// Atomically write `probation.json`. + fn write_probation(&self, marker: &ProbationMarker) -> Result<()>; + + /// Remove `probation.json` (idempotent). + fn clear_probation(&self) -> Result<()>; + + /// Read `failed/.json`; `None` when absent. + fn read_failure(&self, version: &Version) -> Result>; + + /// Atomically write `failed/.json` (overwrites an existing + /// record — the caller increments `attempts`). + fn write_failure(&self, record: &FailureRecord) -> Result<()>; + + /// All versions present under `versions/`. + fn list_versions(&self) -> Result>; + + /// Disk-space preflight: succeed iff there is enough free space to + /// snapshot `state/`. An out-of-space condition must abort the attempt + /// cleanly (`ErrorData::DiskSpace`), never corrupt the store. + fn free_space_for_snapshot(&self) -> Result<()>; +} + +/// The readiness-gate client: one blocking GET against the operator's local +/// `/readyz`. Shared implementation on every OS. +pub trait HealthProbe { + /// `true` iff `GET {url}` returned 200 within the probe timeout. + /// Connection refused, timeouts, and non-200s are all `false` — during + /// probation those simply mean "not ready yet". + fn is_ready(&self, url: &str) -> bool; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn version(s: &str) -> Version { + Version::parse(s).expect("test version should parse") + } + + /// `probation.json` round-trips with exact camelCase keys. + #[test] + fn probation_marker_roundtrip_exact_keys() { + let marker = ProbationMarker { + new: version("1.4.0"), + old: version("1.3.5"), + started_at: "2026-07-08T12:00:00Z".parse().unwrap(), + attempt: 2, + }; + let json = serde_json::to_value(&marker).unwrap(); + assert_eq!(json["new"], "1.4.0"); + assert_eq!(json["old"], "1.3.5"); + assert_eq!(json["startedAt"], "2026-07-08T12:00:00Z"); + assert_eq!(json["attempt"], 2); + let back: ProbationMarker = serde_json::from_value(json).unwrap(); + assert_eq!(back, marker); + } + +} diff --git a/crates/alien-launcher/src/error.rs b/crates/alien-launcher/src/error.rs new file mode 100644 index 000000000..68218ff9b --- /dev/null +++ b/crates/alien-launcher/src/error.rs @@ -0,0 +1,81 @@ +//! Error types for the launcher. +//! +//! The launcher is a local supervisor with no HTTP API of its own, so these +//! errors are for logs and for classification by the update state machine +//! (promote / rollback / retry decisions) — not for API responses. + +use alien_error::AlienErrorData; +use serde::{Deserialize, Serialize}; + +/// Convenient type alias for this crate's Result type. +#[allow(dead_code)] // consumed once the version store and state machine land +pub type Result = alien_error::Result; + +#[allow(dead_code)] // variants are constructed by upcoming core tasks +#[derive(Debug, Clone, AlienErrorData, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ErrorData { + /// The on-disk version store is in a state the startup classification + /// cannot map to a recovery action (missing pointers, unparseable markers). + #[error( + code = "STORE_CORRUPT", + message = "Version store at '{path}' is corrupt: {message}", + retryable = "false", + internal = "true" + )] + StoreCorrupt { path: String, message: String }, + + /// Spawning the operator binary failed (missing file, exec error). + #[error( + code = "SPAWN_FAILED", + message = "Failed to spawn operator '{binary_path}': {message}", + retryable = "false", + internal = "true" + )] + SpawnFailed { + binary_path: String, + message: String, + }, + + /// Copying `state/` to (or restoring it from) a pre-swap snapshot failed. + #[error( + code = "SNAPSHOT_FAILED", + message = "State snapshot operation failed: {message}", + retryable = "true", + internal = "true" + )] + SnapshotFailed { message: String }, + + /// The health probe could not be issued (bad address, client error) — + /// distinct from a healthy "not ready yet" 503, which is not an error. + #[error( + code = "PROBE_FAILED", + message = "Health probe against '{url}' failed: {message}", + retryable = "true", + internal = "true" + )] + ProbeFailed { url: String, message: String }, + + /// Not enough free disk space to proceed safely (snapshot / staging). + /// An out-of-space condition must abort the attempt cleanly, never corrupt. + #[error( + code = "DISK_SPACE", + message = "Insufficient disk space: need {required_bytes} bytes, {available_bytes} available. {message}", + retryable = "false", + internal = "true" + )] + DiskSpace { + required_bytes: u64, + available_bytes: u64, + message: String, + }, + + /// Generic catch-all for uncommon launcher errors. + #[error( + code = "LAUNCHER_ERROR", + message = "Launcher operation failed: {message}", + retryable = "true", + internal = "true" + )] + Other { message: String }, +} diff --git a/crates/alien-launcher/src/main.rs b/crates/alien-launcher/src/main.rs new file mode 100644 index 000000000..cec14054e --- /dev/null +++ b/crates/alien-launcher/src/main.rs @@ -0,0 +1,409 @@ +//! alien-launcher — supervisor for the alien-operator OS-service packaging. +//! +//! Sits between the OS init system (systemd / launchd / SCM) and the operator: +//! the init system keeps the launcher alive; the launcher keeps a healthy +//! operator running and owns version swaps + rollback over the on-disk +//! version store. The launcher itself is frozen — it never rewrites its own +//! binary; it is only replaced by a state-preserving reinstall. +//! +//! Layout: +//! - `core/` — the OS-agnostic update state machine, health gate, and the +//! trait boundary. Must stay platform-blind (see `core`'s module docs); +//! enforced mechanically by `tests/platform_blind.rs`. +//! - `platform/` — one shim per OS implementing the `core::traits` boundary. +//! +//! Deliberately dependency-light and synchronous: hand-rolled flag parsing, +//! no async runtime — this is the binary that must never break. + +mod core; +mod error; +mod platform; + +use std::process::ExitCode; + +/// Defaults; each is overridable by a flag. +const DEFAULT_DATA_DIR: &str = "/var/lib/alien-operator"; +const DEFAULT_PROBATION_SECS: u64 = 90; +const DEFAULT_HEALTH_PORT: u16 = 7799; + +const USAGE: &str = "\ +alien-launcher — supervises the alien-operator and performs health-gated +binary swaps with last-stable rollback. + +USAGE: + alien-launcher [OPTIONS] + +OPTIONS: + --data-dir Version-store root (default /var/lib/alien-operator) + --probation-secs Health-gate window after a swap (default 90) + --health-port Loopback port the operator serves /readyz on (default 7799) + --console Run in the foreground with a Ctrl-C handler instead + of under the OS service manager (Windows: bypass the + SCM dispatcher; no-op elsewhere). Drives the E2E suite. + --version Print the launcher version and exit + --help Print this help and exit +"; + +#[derive(Debug, PartialEq)] +struct Args { + data_dir: std::path::PathBuf, + probation_secs: u64, + health_port: u16, + /// Foreground/console mode (vs OS service manager). Only Windows branches on + /// it — it selects the SCM-dispatcher bypass; elsewhere it is a no-op. + console: bool, +} + +enum ParseOutcome { + Run(Args), + /// --version / --help: print and exit 0. + PrintAndExit(String), + Invalid(String), +} + +fn parse_args(argv: &[String]) -> ParseOutcome { + let mut data_dir = std::path::PathBuf::from(DEFAULT_DATA_DIR); + let mut probation_secs = DEFAULT_PROBATION_SECS; + let mut health_port = DEFAULT_HEALTH_PORT; + let mut console = false; + + let mut iter = argv.iter(); + while let Some(flag) = iter.next() { + let mut value_for = |flag: &str| { + iter.next() + .cloned() + .ok_or_else(|| format!("{flag} requires a value")) + }; + match flag.as_str() { + "--version" => { + return ParseOutcome::PrintAndExit(format!( + "alien-launcher {}", + env!("CARGO_PKG_VERSION") + )); + } + "--help" => return ParseOutcome::PrintAndExit(USAGE.to_string()), + "--console" => console = true, + "--data-dir" => match value_for("--data-dir") { + Ok(value) => data_dir = std::path::PathBuf::from(value), + Err(e) => return ParseOutcome::Invalid(e), + }, + "--probation-secs" => match value_for("--probation-secs") + .and_then(|v| v.parse::().map_err(|e| format!("--probation-secs: {e}"))) + { + Ok(value) => probation_secs = value, + Err(e) => return ParseOutcome::Invalid(e), + }, + "--health-port" => match value_for("--health-port") + .and_then(|v| v.parse::().map_err(|e| format!("--health-port: {e}"))) + { + Ok(value) => health_port = value, + Err(e) => return ParseOutcome::Invalid(e), + }, + other => return ParseOutcome::Invalid(format!("unknown flag '{other}'")), + } + } + ParseOutcome::Run(Args { + data_dir, + probation_secs, + health_port, + console, + }) +} + +fn main() -> ExitCode { + let argv: Vec = std::env::args().skip(1).collect(); + let args = match parse_args(&argv) { + ParseOutcome::Run(args) => args, + ParseOutcome::PrintAndExit(text) => { + println!("{text}"); + return ExitCode::SUCCESS; + } + ParseOutcome::Invalid(message) => { + eprintln!("alien-launcher: {message}\n\n{USAGE}"); + return ExitCode::FAILURE; + } + }; + run_supervisor(args) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn run_supervisor(args: Args) -> ExitCode { + use crate::core::health::UreqProbe; + use crate::core::state_machine::RunConfig; + use crate::core::traits::UpdateEnv; + + // Log to stderr; the init system captures it (systemd → journald via the + // unit's StandardError; launchd → the plist's StandardErrorPath). + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .with_writer(std::io::stderr) + .init(); + + let host = match platform::ActiveHost::new() { + Ok(host) => host, + Err(e) => { + eprintln!("alien-launcher: failed to initialize the service host: {e}"); + return ExitCode::FAILURE; + } + }; + let store = match platform::ActiveVersionStore::open(&args.data_dir) { + Ok(store) => store, + Err(e) => { + eprintln!( + "alien-launcher: failed to open the version store at '{}': {e}", + args.data_dir.display() + ); + return ExitCode::FAILURE; + } + }; + let mut child = platform::ActiveChildSupervisor::new(); + let probe = UreqProbe::default(); + + let config = RunConfig { + probation_window: std::time::Duration::from_secs(args.probation_secs), + heartbeat_interval: host.heartbeat_interval(), + update_env: UpdateEnv { + health_addr: std::net::SocketAddr::from(([127, 0, 0, 1], args.health_port)), + launcher_version: env!("CARGO_PKG_VERSION").to_string(), + }, + ..RunConfig::default() + }; + + tracing::info!( + version = env!("CARGO_PKG_VERSION"), + data_dir = %args.data_dir.display(), + probation_secs = args.probation_secs, + health_port = args.health_port, + console = args.console, + "alien-launcher starting" + ); + match core::run(&store, &mut child, &probe, &host, &config) { + Ok(exit) => { + tracing::info!(?exit, "alien-launcher stopped"); + ExitCode::SUCCESS + } + Err(e) => { + // Fatal (e.g. store corruption the startup classification cannot + // map): exit nonzero and let the init system respawn us — the + // fresh classification recovers from whatever is on disk. + tracing::error!(error = %e, "alien-launcher exiting on a fatal error"); + ExitCode::FAILURE + } + } +} + +/// Windows: run under the SCM (service_dispatcher → `service_main`), or in the +/// foreground with a Ctrl-C handler when `--console` (what the E2E suite drives). +#[cfg(target_os = "windows")] +fn run_supervisor(args: Args) -> ExitCode { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .with_writer(std::io::stderr) + .init(); + + if args.console { + match platform::windows::WindowsHost::console() { + Ok(host) => { + if run_core_loop(&args, &host) { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE + } + } + Err(e) => { + eprintln!("alien-launcher: failed to install the console Ctrl-C handler: {e}"); + ExitCode::FAILURE + } + } + } else { + windows_service_entry::run_as_service(args) + } +} + +/// Build the store/child/probe/config and drive `core::run` against a Windows +/// host (console or SCM). Returns whether the loop stopped cleanly — the caller +/// maps that to an `ExitCode` (console) or an SCM stop code (service). +#[cfg(target_os = "windows")] +fn run_core_loop(args: &Args, host: &platform::windows::WindowsHost) -> bool { + use crate::core::health::UreqProbe; + use crate::core::state_machine::RunConfig; + use crate::core::traits::UpdateEnv; + + let store = match platform::ActiveVersionStore::open(&args.data_dir) { + Ok(store) => store, + Err(e) => { + tracing::error!( + data_dir = %args.data_dir.display(), + error = %e, + "failed to open the version store" + ); + return false; + } + }; + let mut child = platform::ActiveChildSupervisor::new(); + let probe = UreqProbe::default(); + + let config = RunConfig { + probation_window: std::time::Duration::from_secs(args.probation_secs), + heartbeat_interval: host.heartbeat_interval(), + // The installer writes the operator binary with the platform exe suffix + // (`alien-operator.exe` on Windows); spawn the matching name. + operator_binary: format!("alien-operator{}", std::env::consts::EXE_SUFFIX), + update_env: UpdateEnv { + health_addr: std::net::SocketAddr::from(([127, 0, 0, 1], args.health_port)), + launcher_version: env!("CARGO_PKG_VERSION").to_string(), + }, + ..RunConfig::default() + }; + + tracing::info!( + version = env!("CARGO_PKG_VERSION"), + data_dir = %args.data_dir.display(), + probation_secs = args.probation_secs, + health_port = args.health_port, + console = args.console, + "alien-launcher starting" + ); + match core::run(&store, &mut child, &probe, host, &config) { + Ok(exit) => { + tracing::info!(?exit, "alien-launcher stopped"); + true + } + Err(e) => { + tracing::error!(error = %e, "alien-launcher exiting on a fatal error"); + false + } + } +} + +/// The SCM entry: `main()` parks the parsed `Args`, hands control to the service +/// dispatcher, and `service_main` (on the service thread) constructs the host, +/// runs the core loop, and reports the final stop status. +#[cfg(target_os = "windows")] +mod windows_service_entry { + use super::{run_core_loop, Args}; + use crate::platform::windows::WindowsHost; + use std::ffi::OsString; + use std::process::ExitCode; + use std::sync::Mutex; + use windows_service::{define_windows_service, service_dispatcher}; + + // For a SERVICE_WIN32_OWN_PROCESS service the SCM ignores the dispatch-table + // name; it is only a label for logs. + const SERVICE_NAME: &str = "alien-launcher"; + + // `service_main` is an extern callback that cannot capture, so `main` parks + // the parsed args here for the service thread to take. + static SERVICE_ARGS: Mutex> = Mutex::new(None); + + define_windows_service!(ffi_service_main, service_main); + + pub fn run_as_service(args: Args) -> ExitCode { + *SERVICE_ARGS.lock().expect("service args lock") = Some(args); + match service_dispatcher::start(SERVICE_NAME, ffi_service_main) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!( + "alien-launcher: could not connect to the service control manager: {e}\n\ + (run with --console to run in the foreground outside a service)" + ); + ExitCode::FAILURE + } + } + } + + fn service_main(_scm_args: Vec) { + let args = SERVICE_ARGS + .lock() + .expect("service args lock") + .take() + .expect("service args parked before dispatch"); + + // Registers the SCM control handler and reports StartPending. + let host = match WindowsHost::service(SERVICE_NAME) { + Ok(host) => host, + Err(e) => { + eprintln!("alien-launcher: failed to register with the SCM: {e}"); + return; + } + }; + let clean = run_core_loop(&args, &host); + // A nonzero stop code trips the SCM recovery config (doc-12 restarts). + host.report_stopped(if clean { 0 } else { 1 }); + } +} + +/// Other platforms have no service shim. Starting the supervisor there is a +/// hard, loud error — never a silent no-op an init system would respawn forever. +#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] +fn run_supervisor(_args: Args) -> ExitCode { + eprintln!( + "alien-launcher {}: this platform's service shim is not implemented (Linux, macOS, Windows only)", + env!("CARGO_PKG_VERSION") + ); + ExitCode::FAILURE +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(args: &[&str]) -> ParseOutcome { + parse_args(&args.iter().map(|s| s.to_string()).collect::>()) + } + + #[test] + fn defaults_and_overrides() { + let ParseOutcome::Run(args) = parse(&[]) else { + panic!("no flags parses to defaults"); + }; + assert_eq!(args.data_dir, std::path::PathBuf::from(DEFAULT_DATA_DIR)); + assert_eq!(args.probation_secs, DEFAULT_PROBATION_SECS); + assert_eq!(args.health_port, DEFAULT_HEALTH_PORT); + assert!(!args.console, "console is off by default"); + + let ParseOutcome::Run(args) = parse(&[ + "--data-dir", + "/tmp/store", + "--probation-secs", + "30", + "--health-port", + "9000", + "--console", + ]) else { + panic!("full flags parse"); + }; + assert_eq!(args.data_dir, std::path::PathBuf::from("/tmp/store")); + assert_eq!(args.probation_secs, 30); + assert_eq!(args.health_port, 9000); + assert!(args.console, "--console sets console mode"); + } + + #[test] + fn version_prints_cargo_version() { + let ParseOutcome::PrintAndExit(text) = parse(&["--version"]) else { + panic!("--version must print-and-exit"); + }; + assert_eq!(text, format!("alien-launcher {}", env!("CARGO_PKG_VERSION"))); + } + + #[test] + fn invalid_flags_are_loud() { + assert!(matches!(parse(&["--nope"]), ParseOutcome::Invalid(_))); + assert!(matches!(parse(&["--health-port"]), ParseOutcome::Invalid(_))); + assert!(matches!( + parse(&["--health-port", "not-a-port"]), + ParseOutcome::Invalid(_) + )); + assert!(matches!( + parse(&["--probation-secs", "-4"]), + ParseOutcome::Invalid(_) + )); + } +} diff --git a/crates/alien-launcher/src/platform/linux.rs b/crates/alien-launcher/src/platform/linux.rs new file mode 100644 index 000000000..a5dad98ad --- /dev/null +++ b/crates/alien-launcher/src/platform/linux.rs @@ -0,0 +1,120 @@ +//! Linux / systemd `ServiceHost`. +//! +//! The unit runs with `Type=notify` + `WatchdogSec` (see the installer's unit +//! template): the launcher signals `READY=1` once a healthy operator is up and +//! pings `WATCHDOG=1` from the core's dedicated heartbeat thread — including +//! through probation windows longer than the watchdog interval, or systemd +//! would kill the launcher mid-swap. +//! +//! Every notify call is a silent no-op when `NOTIFY_SOCKET` is unset (running +//! outside systemd — tests, the E2E harness, manual runs), so the same binary +//! behaves identically under and outside the init system. + +use std::time::Duration; + +use tracing::warn; + +use super::unix_signals::SignalControls; +use crate::core::traits::{Control, ServiceHost}; +use crate::error::Result; + +/// Fallback heartbeat interval when systemd did not provide `WATCHDOG_USEC` +/// (watchdog disabled or running outside systemd). +const DEFAULT_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20); + +pub struct LinuxHost { + signals: SignalControls, +} + +impl LinuxHost { + pub fn new() -> Result { + Ok(Self { + signals: SignalControls::register()?, + }) + } + + /// The interval the core's heartbeat thread should tick at: + /// `WATCHDOG_USEC / 3` when systemd supervises with a watchdog, else the + /// default. A third of the budget tolerates two missed/slow ticks before + /// systemd declares the launcher hung. + pub fn heartbeat_interval(&self) -> Duration { + heartbeat_interval_from(std::env::var("WATCHDOG_USEC").ok().as_deref()) + } +} + +/// Pure decision core, unit-tested: parse systemd's `WATCHDOG_USEC` (µs). +fn heartbeat_interval_from(watchdog_usec: Option<&str>) -> Duration { + match watchdog_usec.and_then(|raw| raw.parse::().ok()) { + Some(usec) if usec > 0 => Duration::from_micros(usec / 3), + Some(_) => DEFAULT_HEARTBEAT_INTERVAL, + None => { + if watchdog_usec.is_some() { + warn!("WATCHDOG_USEC is set but unparseable; using the default heartbeat interval"); + } + DEFAULT_HEARTBEAT_INTERVAL + } + } +} + +impl ServiceHost for LinuxHost { + fn poll_control(&self) -> Option { + self.signals.poll() + } + + fn report_ready(&self) { + // No-op (Ok) without NOTIFY_SOCKET; a genuine socket error is worth a + // log but never worth failing the launcher over. + if let Err(e) = sd_notify::notify(false, &[sd_notify::NotifyState::Ready]) { + warn!(error = %e, "failed to send READY=1 to systemd"); + } + } + + fn heartbeat(&self) { + if let Err(e) = sd_notify::notify(false, &[sd_notify::NotifyState::Watchdog]) { + warn!(error = %e, "failed to send WATCHDOG=1 to systemd"); + } + } + + fn report_stopping(&self) { + if let Err(e) = sd_notify::notify(false, &[sd_notify::NotifyState::Stopping]) { + warn!(error = %e, "failed to send STOPPING=1 to systemd"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `WATCHDOG_USEC / 3`, defaulting when absent, zero, or garbage. + #[test] + fn heartbeat_interval_is_a_third_of_the_watchdog_budget() { + // systemd's WatchdogSec=60 arrives as 60_000_000 µs → 20 s ticks. + assert_eq!( + heartbeat_interval_from(Some("60000000")), + Duration::from_secs(20) + ); + assert_eq!( + heartbeat_interval_from(Some("30000000")), + Duration::from_secs(10) + ); + assert_eq!(heartbeat_interval_from(None), DEFAULT_HEARTBEAT_INTERVAL); + assert_eq!( + heartbeat_interval_from(Some("garbage")), + DEFAULT_HEARTBEAT_INTERVAL + ); + assert_eq!(heartbeat_interval_from(Some("0")), DEFAULT_HEARTBEAT_INTERVAL); + } + + /// Outside systemd (no NOTIFY_SOCKET in the test env) every notify call + /// is a silent no-op — the host must never panic or error the launcher. + #[test] + fn notify_calls_are_noops_without_systemd() { + let host = LinuxHost::new().expect("host should construct"); + host.report_ready(); + host.heartbeat(); + host.report_stopping(); + // Reaching here without panic is the assertion; poll_control still works. + assert_eq!(host.poll_control(), None); + } +} diff --git a/crates/alien-launcher/src/platform/macos.rs b/crates/alien-launcher/src/platform/macos.rs new file mode 100644 index 000000000..87bc1dea2 --- /dev/null +++ b/crates/alien-launcher/src/platform/macos.rs @@ -0,0 +1,98 @@ +//! macOS / launchd `ServiceHost`. +//! +//! launchd has no readiness or watchdog protocol: a daemon is considered "up" +//! while its process is alive, and supervision is `KeepAlive` + exit codes (see +//! the installer's plist). So there is nothing for the launcher to notify — +//! every notify method is an intentional no-op, and the host reduces to the +//! shared Unix stop-signal bridge (`unix_signals`) for `poll_control`. +//! +//! `report_ready`/`heartbeat`/`report_stopping` exist only to satisfy the +//! `ServiceHost` contract the OS-agnostic core drives; on macOS they do nothing +//! and return immediately. + +use std::time::Duration; + +use super::unix_signals::SignalControls; +use crate::core::traits::{Control, ServiceHost}; +use crate::error::Result; + +/// Tick cadence for the core's heartbeat thread. launchd has no watchdog, so +/// `heartbeat` is a no-op; this only bounds how often that no-op wakes. Kept in +/// the same order of magnitude as the Linux default to avoid needless wakeups. +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20); + +pub struct MacosHost { + signals: SignalControls, +} + +impl MacosHost { + pub fn new() -> Result { + Ok(Self { + signals: SignalControls::register()?, + }) + } + + /// launchd provides no watchdog budget, so unlike the Linux host there is + /// nothing to derive — the core still runs a heartbeat thread, so hand it a + /// fixed, sane cadence for its no-op ping. + pub fn heartbeat_interval(&self) -> Duration { + HEARTBEAT_INTERVAL + } +} + +impl ServiceHost for MacosHost { + fn poll_control(&self) -> Option { + self.signals.poll() + } + + // launchd has no notify protocol; the following are intentional no-ops that + // return immediately (see the module docs). Supervision is KeepAlive + exit + // codes, so there is no READY / WATCHDOG / STOPPING to signal. + fn report_ready(&self) {} + fn heartbeat(&self) {} + fn report_stopping(&self) {} +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A real SIGTERM to our own process surfaces as `Control::Stop` on the + /// host's `poll_control` (which delegates to the shared Unix signal bridge). + /// Serialized against the other self-signal test so concurrent SIGTERMs + /// can't perturb either assertion. + #[test] + fn sigterm_maps_to_stop() { + let _guard = crate::platform::unix_signals::signal_test_guard(); + let host = MacosHost::new().expect("host should construct"); + + nix::sys::signal::kill(nix::unistd::Pid::this(), nix::sys::signal::Signal::SIGTERM) + .expect("self-signal should succeed"); + + // The handler runs asynchronously; poll until it lands (bounded). Extra + // SIGTERMs would only make Stop appear sooner, so this stays robust. + let mut seen = None; + for _ in 0..100 { + if let Some(control) = host.poll_control() { + seen = Some(control); + break; + } + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(seen, Some(Control::Stop), "SIGTERM must map to Stop"); + } + + /// launchd has no notify protocol: every notify method is a no-op that must + /// return immediately without panicking, and the heartbeat cadence is sane. + #[test] + fn notify_methods_are_noops_and_dont_panic() { + let host = MacosHost::new().expect("host should construct"); + host.report_ready(); + host.heartbeat(); + host.report_stopping(); + assert!( + host.heartbeat_interval() > Duration::ZERO, + "heartbeat cadence must be positive" + ); + } +} diff --git a/crates/alien-launcher/src/platform/mod.rs b/crates/alien-launcher/src/platform/mod.rs new file mode 100644 index 000000000..d8c6bcc17 --- /dev/null +++ b/crates/alien-launcher/src/platform/mod.rs @@ -0,0 +1,66 @@ +//! Platform shims — one module per OS, each implementing the +//! `crate::core::traits` boundary. This module is the single place where a +//! `#[cfg(...)]` selects the concrete implementation; nothing else in the +//! crate branches on the target OS. +//! +//! Wired for real: **Linux** (systemd host) and **macOS** (launchd host), both +//! over the shared Unix child supervisor and Unix symlink store — the host is +//! the only per-OS piece; supervision and the version store are identical Unix +//! code. The Windows shim (SCM + Job Object + junction store) lands in its own +//! phase. +//! +//! On any other target (no supported host) the `unix_*` modules are exercised +//! by tests only, hence the narrowed dead-code staging below. + +#[cfg(unix)] +#[cfg_attr( + not(any(target_os = "linux", target_os = "macos")), + allow(dead_code) +)] +pub mod unix_child; +#[cfg(unix)] +#[cfg_attr( + not(any(target_os = "linux", target_os = "macos")), + allow(dead_code) +)] +pub mod unix_signals; +#[cfg(unix)] +#[cfg_attr( + not(any(target_os = "linux", target_os = "macos")), + allow(dead_code) +)] +pub mod unix_store; + +#[cfg(target_os = "linux")] +pub mod linux; + +#[cfg(target_os = "macos")] +pub mod macos; + +#[cfg(target_os = "windows")] +pub mod windows; + +#[cfg(target_os = "windows")] +pub mod windows_child; + +#[cfg(target_os = "windows")] +pub mod windows_store; + +// The child supervisor and version store share one alias per OS. The host is +// NOT aliased on Windows: its constructors differ (`service()` / `console()` +// select SCM vs console mode), so `main.rs`'s Windows `run_supervisor` names +// `windows::WindowsHost` directly rather than a uniform `ActiveHost::new()`. +#[cfg(target_os = "linux")] +pub use linux::LinuxHost as ActiveHost; +#[cfg(target_os = "macos")] +pub use macos::MacosHost as ActiveHost; + +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub use unix_child::UnixChildSupervisor as ActiveChildSupervisor; +#[cfg(target_os = "windows")] +pub use windows_child::WindowsChildSupervisor as ActiveChildSupervisor; + +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub use unix_store::UnixVersionStore as ActiveVersionStore; +#[cfg(target_os = "windows")] +pub use windows_store::WindowsVersionStore as ActiveVersionStore; diff --git a/crates/alien-launcher/src/platform/unix_child.rs b/crates/alien-launcher/src/platform/unix_child.rs new file mode 100644 index 000000000..91d8c3b81 --- /dev/null +++ b/crates/alien-launcher/src/platform/unix_child.rs @@ -0,0 +1,256 @@ +//! Unix `ChildSupervisor` (Linux + macOS): spawn the operator in its own +//! process group, guarantee die-with-parent, stop gracefully with escalation. +//! +//! Die-with-parent is normative (see the trait docs). On Linux the kernel +//! delivers SIGTERM to the child when the launcher dies +//! (`PR_SET_PDEATHSIG`, installed in `pre_exec` between fork and exec). macOS +//! has no equivalent, so there the fast path is the operator-side parent +//! watch (it polls its parent pid and exits when reparented); the operator's +//! `InstanceLock` is the correctness backstop on both — a respawned +//! launcher's operator cannot run until an orphan exits. + +use std::collections::HashMap; +use std::path::Path; +use std::time::{Duration, Instant}; + +use command_group::{CommandGroup, GroupChild}; + +use crate::core::traits::{ + ChildSupervisor, ExitStatus, OperatorHandle, UpdateEnv, +}; +use crate::error::{ErrorData, Result}; +use alien_core::self_update::{ENV_HEALTH_ADDR, ENV_LAUNCHER_VERSION, ENV_SELF_UPDATE}; +use alien_error::{AlienError, Context, IntoAlienError}; + +/// How often `stop` re-polls the child while waiting out the grace period. +const STOP_POLL_INTERVAL: Duration = Duration::from_millis(50); + +#[derive(Default)] +pub struct UnixChildSupervisor { + /// Live children by pid (the group leader's pid IS the pgid). + children: HashMap, +} + +impl UnixChildSupervisor { + pub fn new() -> Self { + Self::default() + } + + fn child_mut(&mut self, handle: &OperatorHandle) -> Result<&mut GroupChild> { + self.children.get_mut(&handle.pid).ok_or_else(|| { + AlienError::new(ErrorData::Other { + message: format!("unknown child pid {}", handle.pid), + }) + }) + } +} + +impl ChildSupervisor for UnixChildSupervisor { + fn spawn(&mut self, binary: &Path, env: &UpdateEnv) -> Result { + let mut command = std::process::Command::new(binary); + command + .env(ENV_SELF_UPDATE, "1") + .env(ENV_LAUNCHER_VERSION, &env.launcher_version) + .env(ENV_HEALTH_ADDR, env.health_addr.to_string()); + + // Linux: ask the kernel to SIGTERM the child if we die. Runs between + // fork and exec; prctl is async-signal-safe. The tiny race (launcher + // dies before the prctl runs) is covered by the InstanceLock backstop. + #[cfg(target_os = "linux")] + { + use std::os::unix::process::CommandExt; + // SAFETY: only async-signal-safe calls (prctl, getppid) run in + // the pre-exec child context. + unsafe { + command.pre_exec(|| { + nix::sys::prctl::set_pdeathsig(nix::sys::signal::Signal::SIGTERM) + .map_err(std::io::Error::from)?; + // If the launcher died between fork and the prctl above, + // the death signal will never fire — detect the reparent + // and bail before exec. + if nix::unistd::getppid().as_raw() == 1 { + return Err(std::io::Error::other( + "launcher died before the operator exec'd", + )); + } + Ok(()) + }); + } + } + + let child = command + .group_spawn() + .into_alien_error() + .context(ErrorData::SpawnFailed { + binary_path: binary.display().to_string(), + message: "failed to spawn the operator in its own process group".to_string(), + })?; + let pid = child.id(); + self.children.insert(pid, child); + Ok(OperatorHandle { pid }) + } + + fn stop(&mut self, handle: &OperatorHandle, grace: Duration) -> Result<()> { + // Graceful first: SIGTERM the whole group (negative pgid; the group + // leader's pid is the pgid). ESRCH means it already exited — success. + let pgid = nix::unistd::Pid::from_raw(handle.pid as i32); + match nix::sys::signal::killpg(pgid, nix::sys::signal::Signal::SIGTERM) { + Ok(()) | Err(nix::errno::Errno::ESRCH) => {} + Err(e) => { + return Err(AlienError::new(ErrorData::Other { + message: format!("failed to SIGTERM process group {}: {e}", handle.pid), + })); + } + } + + // Wait out the grace period, then escalate to a group SIGKILL. + let deadline = Instant::now() + grace; + loop { + if self.try_wait(handle)?.is_some() { + return Ok(()); + } + if Instant::now() >= deadline { + break; + } + std::thread::sleep(STOP_POLL_INTERVAL.min(deadline - Instant::now())); + } + + let child = self.child_mut(handle)?; + child.kill().into_alien_error().context(ErrorData::Other { + message: format!("failed to SIGKILL process group {}", handle.pid), + })?; + // Reap the killed child so it does not linger as a zombie. + child.wait().into_alien_error().context(ErrorData::Other { + message: format!("failed to reap process group {}", handle.pid), + })?; + Ok(()) + } + + fn try_wait(&mut self, handle: &OperatorHandle) -> Result> { + let child = self.child_mut(handle)?; + let status = child + .try_wait() + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to poll child {}", handle.pid), + })?; + Ok(status.map(map_exit_status)) + } +} + +fn map_exit_status(status: std::process::ExitStatus) -> ExitStatus { + match status.code() { + Some(code) => ExitStatus::Code(code), + None => { + // No code on Unix means signal-terminated. + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if status.signal().is_some() { + return ExitStatus::Signal; + } + } + ExitStatus::Unknown + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::testing::test_update_env; + + fn spawn_sh(supervisor: &mut UnixChildSupervisor, script: &str) -> OperatorHandle { + // `sh -c` needs the binary to be `sh` with args; the trait spawns a + // bare binary, so write the script to a temp file and exec it. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("op.sh"); + std::fs::write(&path, format!("#!/bin/sh\n{script}\n")).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + // Leak the tempdir so the script survives until the child execs. + let _ = Box::leak(Box::new(dir)); + supervisor + .spawn(&path, &test_update_env()) + .expect("spawn should succeed") + } + + /// Exit codes pass through `try_wait` — including the handoff code 10. + #[test] + fn exit_codes_pass_through() { + let mut supervisor = UnixChildSupervisor::new(); + let handle = spawn_sh(&mut supervisor, "exit 10"); + + let mut status = None; + for _ in 0..100 { + if let Some(s) = supervisor.try_wait(&handle).unwrap() { + status = Some(s); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(status, Some(ExitStatus::Code(10))); + } + + /// The spawn env carries the handoff contract variables. + #[test] + fn spawn_env_carries_the_contract() { + let mut supervisor = UnixChildSupervisor::new(); + let dir = tempfile::tempdir().unwrap(); + let out = dir.path().join("env-dump"); + let handle = spawn_sh( + &mut supervisor, + &format!( + "printf '%s|%s|%s' \"$ALIEN_SELF_UPDATE\" \"$ALIEN_LAUNCHER_VERSION\" \"$ALIEN_HEALTH_ADDR\" > {}", + out.display() + ), + ); + for _ in 0..100 { + if supervisor.try_wait(&handle).unwrap().is_some() { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let dump = std::fs::read_to_string(&out).expect("child should have written the env dump"); + assert_eq!(dump, "1|0.1.0-test|127.0.0.1:7799"); + } + + /// A stubborn child (ignores TERM) is force-killed after the grace and + /// reported as signal-terminated; a cooperative child exits 0 in time. + #[test] + fn stop_escalates_after_grace() { + let mut supervisor = UnixChildSupervisor::new(); + // Ignores SIGTERM and sleeps far beyond the test. + let stubborn = spawn_sh(&mut supervisor, "trap '' TERM; sleep 30"); + std::thread::sleep(Duration::from_millis(100)); // let the trap install + let started = Instant::now(); + supervisor + .stop(&stubborn, Duration::from_millis(300)) + .expect("stop should succeed"); + assert!( + started.elapsed() < Duration::from_secs(5), + "stop must not wait for the 30s sleep" + ); + + // Cooperative child: exits 0 on TERM, well within the grace. + let cooperative = spawn_sh(&mut supervisor, "trap 'exit 0' TERM; sleep 30"); + std::thread::sleep(Duration::from_millis(100)); + supervisor + .stop(&cooperative, Duration::from_secs(5)) + .expect("stop should succeed"); + } + + // NOTE: die-with-parent (the operator must die when the launcher dies) is + // proven end-to-end by the os-service E2E, which SIGKILLs a real + // `alien-launcher` process and asserts its operator child exits. There is + // no sound *unit-level* test for it: the parent that dies must itself be a + // process that called `UnixChildSupervisor::spawn`, and forking one inside + // this multi-threaded test binary is unsafe (the forked child copies only + // the calling thread, so `Command::spawn`'s allocations/locks can deadlock + // post-fork). The production mechanism here — `PR_SET_PDEATHSIG` in + // `pre_exec`, with `command-group`'s plain `process_group(0)` spawn keeping + // the operator a DIRECT child of the launcher — is what makes that E2E pass. +} diff --git a/crates/alien-launcher/src/platform/unix_signals.rs b/crates/alien-launcher/src/platform/unix_signals.rs new file mode 100644 index 000000000..ba6df9742 --- /dev/null +++ b/crates/alien-launcher/src/platform/unix_signals.rs @@ -0,0 +1,94 @@ +//! Unix signal → `poll_control` bridge, shared by the Linux and macOS hosts. +//! +//! `signal-hook` (rather than a `sigwait` thread) because its handler +//! registration is async-signal-safe by construction and needs no process-wide +//! signal-mask coordination: the handler just flips an atomic, and the run +//! loop's `poll_control` drains it once per delivery. SIGTERM and SIGINT both +//! map to `Control::Stop` — on Unix a host shutdown arrives as SIGTERM, so +//! there is no separate `Shutdown` signal to distinguish. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use crate::core::traits::Control; +use crate::error::{ErrorData, Result}; +use alien_error::{Context, IntoAlienError}; + +/// Registered stop-signal flag. Cheap to clone-share with a host struct. +pub struct SignalControls { + stop_requested: Arc, +} + +impl SignalControls { + /// Register SIGTERM + SIGINT handlers. Call once, early in `main`. + pub fn register() -> Result { + let stop_requested = Arc::new(AtomicBool::new(false)); + for signal in [signal_hook::consts::SIGTERM, signal_hook::consts::SIGINT] { + signal_hook::flag::register(signal, stop_requested.clone()) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to register handler for signal {signal}"), + })?; + } + Ok(Self { stop_requested }) + } + + /// Drain the pending stop request, if any (at-most-once per delivery + /// burst — repeated signals before a poll collapse into one Stop, which + /// is exactly what the run loop wants). + pub fn poll(&self) -> Option { + if self.stop_requested.swap(false, Ordering::SeqCst) { + Some(Control::Stop) + } else { + None + } + } +} + +impl std::fmt::Debug for SignalControls { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SignalControls") + .field("stop_requested", &self.stop_requested.load(Ordering::SeqCst)) + .finish() + } +} + +/// Serializes tests that deliver a real signal to our own process. Signals are +/// process-wide, so two self-`SIGTERM` tests running concurrently would each +/// see the other's signal and break their strict assertions; taking this guard +/// makes them run one at a time. Test-only. +#[cfg(test)] +pub(crate) fn signal_test_guard() -> std::sync::MutexGuard<'static, ()> { + static GUARD: std::sync::Mutex<()> = std::sync::Mutex::new(()); + GUARD.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A real SIGTERM to our own process lands as exactly one `Control::Stop` + /// on the next poll, and the flag drains. + #[test] + fn sigterm_delivers_stop_once() { + let _guard = signal_test_guard(); + let controls = SignalControls::register().expect("registration should succeed"); + assert_eq!(controls.poll(), None, "no signal yet"); + + // Deliver SIGTERM to ourselves; the signal-hook handler flips the flag. + nix::sys::signal::kill(nix::unistd::Pid::this(), nix::sys::signal::Signal::SIGTERM) + .expect("self-signal should succeed"); + + // The handler runs asynchronously; poll until it lands (bounded). + let mut seen = None; + for _ in 0..100 { + if let Some(control) = controls.poll() { + seen = Some(control); + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert_eq!(seen, Some(Control::Stop), "SIGTERM must map to Stop"); + assert_eq!(controls.poll(), None, "drained after one poll"); + } +} diff --git a/crates/alien-launcher/src/platform/unix_store.rs b/crates/alien-launcher/src/platform/unix_store.rs new file mode 100644 index 000000000..58a375ef0 --- /dev/null +++ b/crates/alien-launcher/src/platform/unix_store.rs @@ -0,0 +1,424 @@ +//! Unix `VersionStore` (Linux + macOS): the on-disk version store with real +//! symlink pointers. +//! +//! `current` and `last-stable` are symlinks to `versions/`; a flip creates +//! the new symlink at a temp name and `rename(2)`s it over the pointer, which +//! is atomic — a reader (or a crashed launcher's restart classification) +//! always sees either the old or the new target, never a missing pointer. +//! Everything non-pointer (markers, snapshots, gc candidates, disk check) +//! comes from the shared `store_common` helpers. + +use std::path::{Path, PathBuf}; + +use crate::core::store_common; +use crate::core::traits::{ + FailureRecord, PendingMarker, ProbationMarker, Version, VersionStore, +}; +use crate::error::{ErrorData, Result}; +use alien_core::self_update as protocol; +use alien_error::{AlienError, Context, IntoAlienError}; + +pub struct UnixVersionStore { + root: PathBuf, +} + +impl UnixVersionStore { + /// Open a store rooted at the operator's data dir. Creates the layout + /// directories that may be missing (idempotent). + pub fn open(root: &Path) -> Result { + for dir in ["versions", "state", "state-snapshots", "failed", "download"] { + std::fs::create_dir_all(root.join(dir)) + .into_alien_error() + .context(ErrorData::StoreCorrupt { + path: root.display().to_string(), + message: format!("failed to create the '{dir}' directory"), + })?; + } + Ok(Self { + root: root.to_path_buf(), + }) + } + + fn pointer_path(&self, name: &str) -> PathBuf { + self.root.join(name) + } + + /// Read a pointer symlink → the version it names. Absent → None; a + /// pointer that exists but does not parse is store corruption. + fn read_pointer(&self, name: &str) -> Result> { + let path = self.pointer_path(name); + let target = match std::fs::read_link(&path) { + Ok(target) => target, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => { + return Err(e).into_alien_error().context(ErrorData::StoreCorrupt { + path: path.display().to_string(), + message: "failed to read the pointer symlink".to_string(), + }); + } + }; + let version_str = target + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| corrupt_pointer(&path, "symlink target has no version component"))?; + Version::parse(version_str) + .map(Some) + .map_err(|e| corrupt_pointer(&path, &format!("unparseable version in target: {e}"))) + } + + /// Atomically repoint `name` at `versions/`: create the symlink + /// at `.tmp` (removing any stale one from a crashed flip) and + /// rename it over the pointer. + fn write_pointer(&self, name: &str, version: &Version) -> Result<()> { + let path = self.pointer_path(name); + let tmp = self.pointer_path(&format!("{name}.tmp")); + let target = Path::new("versions").join(version.as_str()); + + match std::fs::remove_file(&tmp) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + return Err(e).into_alien_error().context(ErrorData::Other { + message: format!("failed to clear stale pointer temp '{}'", tmp.display()), + }); + } + } + std::os::unix::fs::symlink(&target, &tmp) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to create pointer temp '{}'", tmp.display()), + })?; + std::fs::rename(&tmp, &path) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to commit pointer '{}'", path.display()), + }) + } +} + +fn corrupt_pointer(path: &Path, message: &str) -> AlienError { + AlienError::new(ErrorData::StoreCorrupt { + path: path.display().to_string(), + message: message.to_string(), + }) +} + +impl VersionStore for UnixVersionStore { + fn stage_dir(&self, version: &Version) -> PathBuf { + protocol::version_dir(&self.root, version) + } + + fn current(&self) -> Result> { + self.read_pointer("current") + } + + fn last_stable(&self) -> Result> { + self.read_pointer("last-stable") + } + + fn set_current(&self, version: &Version) -> Result<()> { + self.write_pointer("current", version) + } + + fn set_last_stable(&self, version: &Version) -> Result<()> { + self.write_pointer("last-stable", version) + } + + fn snapshot_state(&self, tag: &Version) -> Result<()> { + store_common::snapshot_state_dir( + &self.root.join("state"), + &self.root.join("state-snapshots"), + tag, + ) + } + + fn restore_state(&self, tag: &Version) -> Result<()> { + store_common::restore_state_dir( + &self.root.join("state"), + &self.root.join("state-snapshots"), + tag, + ) + } + + fn drop_snapshot(&self, tag: &Version) -> Result<()> { + let dir = self.root.join("state-snapshots").join(tag.as_str()); + match std::fs::remove_dir_all(&dir) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e).into_alien_error().context(ErrorData::Other { + message: format!("failed to drop snapshot '{}'", dir.display()), + }), + } + } + + fn state_size(&self) -> Result { + store_common::dir_size(&self.root.join("state")) + } + + fn gc(&self, keep: &[Version]) -> Result<()> { + let all = self.list_versions()?; + let current = self.current()?; + let last_stable = self.last_stable()?; + for candidate in + store_common::gc_candidates(&all, keep, current.as_ref(), last_stable.as_ref()) + { + std::fs::remove_dir_all(self.stage_dir(&candidate)) + .into_alien_error() + .context(ErrorData::Other { + message: format!("gc failed for version {candidate}"), + })?; + } + Ok(()) + } + + fn read_pending(&self) -> Result> { + store_common::read_marker(&protocol::pending_path(&self.root)) + } + + fn write_pending(&self, marker: &PendingMarker) -> Result<()> { + store_common::write_marker_atomic(&protocol::pending_path(&self.root), marker) + } + + fn clear_pending(&self) -> Result<()> { + store_common::remove_marker(&protocol::pending_path(&self.root)) + } + + fn read_probation(&self) -> Result> { + store_common::read_marker(&self.root.join("probation.json")) + } + + fn write_probation(&self, marker: &ProbationMarker) -> Result<()> { + store_common::write_marker_atomic(&self.root.join("probation.json"), marker) + } + + fn clear_probation(&self) -> Result<()> { + store_common::remove_marker(&self.root.join("probation.json")) + } + + fn read_failure(&self, version: &Version) -> Result> { + store_common::read_marker(&protocol::failure_path(&self.root, version)) + } + + fn write_failure(&self, record: &FailureRecord) -> Result<()> { + store_common::write_marker_atomic( + &protocol::failure_path(&self.root, &record.version), + record, + ) + } + + fn list_versions(&self) -> Result> { + let versions_dir = self.root.join("versions"); + let mut versions = Vec::new(); + for entry in std::fs::read_dir(&versions_dir) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to read '{}'", versions_dir.display()), + })? + { + let entry = entry.into_alien_error().context(ErrorData::Other { + message: "failed to read a versions/ entry".to_string(), + })?; + let name = entry.file_name().to_string_lossy().into_owned(); + let version = Version::parse(&name).map_err(|e| { + AlienError::new(ErrorData::StoreCorrupt { + path: entry.path().display().to_string(), + message: format!("versions/ entry is not a version: {e}"), + }) + })?; + versions.push(version); + } + versions.sort(); + Ok(versions) + } + + fn free_space_for_snapshot(&self) -> Result<()> { + let required = self.state_size()?; + let stat = nix::sys::statvfs::statvfs(&self.root) + .into_alien_error() + .context(ErrorData::Other { + message: format!("statvfs failed for '{}'", self.root.display()), + })?; + let available = stat.blocks_available() as u64 * stat.fragment_size() as u64; + store_common::check_space(required, available, "state snapshot") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn version(s: &str) -> Version { + Version::parse(s).unwrap() + } + + fn store_with(dir: &Path, versions: &[&str]) -> UnixVersionStore { + let store = UnixVersionStore::open(dir).unwrap(); + for v in versions { + let stage = store.stage_dir(&version(v)); + std::fs::create_dir_all(&stage).unwrap(); + std::fs::write(stage.join("alien-operator"), format!("binary-{v}")).unwrap(); + } + store + } + + /// Pointers are real symlinks, and a rapid flip storm always leaves a + /// readable, parseable pointer (single-threaded — the launcher's own + /// access pattern: one supervise thread, reads and flips never race). + #[test] + fn pointer_flip_storm_is_always_readable() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &["1.0.0", "2.0.0"]); + store.set_current(&version("1.0.0")).unwrap(); + assert!( + dir.path().join("current").symlink_metadata().unwrap().file_type().is_symlink(), + "the pointer must be a real symlink" + ); + + for i in 0..1000 { + let v = if i % 2 == 0 { "2.0.0" } else { "1.0.0" }; + store.set_current(&version(v)).unwrap(); + assert_eq!(store.current().unwrap(), Some(version(v))); + } + assert!( + !dir.path().join("current.tmp").exists(), + "no temp residue after the storm" + ); + } + + /// Linux only (the Phase-1 target, exercised in CI): the flip is atomic + /// even under a CONCURRENT reader — every read during 1000 flips parses + /// to one of the two versions, never a missing/partial pointer. Gated + /// off macOS: APFS `readlink` can transiently return EINVAL while a + /// same-name `rename` is in flight — a platform quirk that does not + /// affect the launcher (whose reads and flips share one thread); crash + /// atomicity there is covered by the startup-classification E2E. + #[cfg(target_os = "linux")] + #[test] + fn pointer_flip_is_atomic_under_a_reader() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &["1.0.0", "2.0.0"]); + store.set_current(&version("1.0.0")).unwrap(); + + let reader_root = dir.path().to_path_buf(); + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let reader_stop = stop.clone(); + let reader = std::thread::spawn(move || { + let store = UnixVersionStore::open(&reader_root).unwrap(); + let mut reads = 0u32; + while !reader_stop.load(std::sync::atomic::Ordering::Relaxed) { + let current = store.current().expect("read must never fail mid-flip"); + let current = current.expect("pointer must never be missing mid-flip"); + assert!( + current == version("1.0.0") || current == version("2.0.0"), + "unexpected pointer target {current}" + ); + reads += 1; + } + reads + }); + + for i in 0..1000 { + let v = if i % 2 == 0 { "2.0.0" } else { "1.0.0" }; + store.set_current(&version(v)).unwrap(); + } + stop.store(true, std::sync::atomic::Ordering::Relaxed); + let reads = reader.join().expect("reader must not panic"); + assert!(reads > 0, "the reader must actually have observed flips"); + } + + /// A stale `current.tmp` from a crashed flip is cleaned up by the next + /// flip instead of failing it. + #[test] + fn stale_pointer_temp_is_cleared_on_the_next_flip() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &["1.0.0", "2.0.0"]); + // Simulate a crash between symlink-create and rename. + std::os::unix::fs::symlink("versions/1.0.0", dir.path().join("current.tmp")).unwrap(); + + store.set_current(&version("2.0.0")).expect("flip should clear the stale temp"); + assert_eq!(store.current().unwrap(), Some(version("2.0.0"))); + assert!(!dir.path().join("current.tmp").exists(), "temp consumed by the rename"); + } + + /// gc removes exactly the unpointed versions. + #[test] + fn gc_preserves_pointer_targets() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &["1.0.0", "1.1.0", "1.2.0"]); + store.set_current(&version("1.2.0")).unwrap(); + store.set_last_stable(&version("1.1.0")).unwrap(); + + store.gc(&[]).unwrap(); + assert_eq!( + store.list_versions().unwrap(), + vec![version("1.1.0"), version("1.2.0")] + ); + } + + /// The real disk has space for a tiny state dir — the preflight passes + /// against genuine statvfs numbers. + #[test] + fn free_space_preflight_passes_on_a_real_disk() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &[]); + std::fs::write(dir.path().join("state/db"), b"tiny").unwrap(); + store + .free_space_for_snapshot() + .expect("a 4-byte state dir must fit on any disk running this test"); + } + // -- the full Phase-0 state-machine suite against the REAL store ------- + + use crate::core::testing::{ + scenario_classification_rows, scenario_crash_injection_promote, + scenario_crash_injection_rollback, scenario_happy_promote, + scenario_rollback_on_probation_crash, scenario_rollback_restores_state, TestStoreOps, + }; + + impl TestStoreOps for UnixVersionStore { + fn install_version(&self, version: &Version) { + let stage = self.stage_dir(version); + std::fs::create_dir_all(&stage).unwrap(); + std::fs::write(stage.join("alien-operator"), format!("binary-{version}")).unwrap(); + } + fn state_dir_path(&self) -> PathBuf { + self.root.join("state") + } + fn store_root(&self) -> PathBuf { + self.root.clone() + } + } + + fn unix(dir: &Path) -> UnixVersionStore { + UnixVersionStore::open(dir).unwrap() + } + + #[test] + fn state_machine_happy_promote_on_real_store() { + scenario_happy_promote(unix); + } + + #[test] + fn state_machine_rollback_restores_state_on_real_store() { + scenario_rollback_restores_state(unix); + } + + #[test] + fn state_machine_rollback_on_probation_crash_on_real_store() { + scenario_rollback_on_probation_crash(unix); + } + + #[test] + fn state_machine_classification_rows_on_real_store() { + scenario_classification_rows(unix); + } + + #[test] + fn state_machine_crash_injection_promote_on_real_store() { + scenario_crash_injection_promote(unix); + } + + #[test] + fn state_machine_crash_injection_rollback_on_real_store() { + scenario_crash_injection_rollback(unix); + } +} diff --git a/crates/alien-launcher/src/platform/windows.rs b/crates/alien-launcher/src/platform/windows.rs new file mode 100644 index 000000000..0112dd75a --- /dev/null +++ b/crates/alien-launcher/src/platform/windows.rs @@ -0,0 +1,279 @@ +//! Windows service host — the `ServiceHost` boundary over the Service Control +//! Manager (SCM), mirroring the Linux/macOS hosts (`platform/linux.rs`, +//! `platform/macos.rs`). Control signals arrive on an internal cell drained by +//! `poll_control`; `report_ready` / `report_stopping` / `heartbeat` drive the +//! SCM status (`SERVICE_RUNNING` / `SERVICE_STOP_PENDING` / checkpoint bumps). +//! +//! A `--console` mode (what the E2E suite drives) swaps the SCM control handler +//! for a Ctrl-C handler and makes the status calls no-ops — no SCM to report to. +//! +//! `main.rs`'s Windows `run_supervisor` constructs this host — via `service()` +//! under the SCM dispatcher, or `console()` for the E2E suite — and drives it +//! through `core::run`. + +use std::sync::atomic::{AtomicU32, AtomicU8, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use alien_error::AlienError; +use tracing::warn; +use windows_service::service::{ + ServiceControl, ServiceControlAccept, ServiceExitCode, ServiceState, ServiceStatus, ServiceType, +}; +use windows_service::service_control_handler::{ + self, ServiceControlHandlerResult, ServiceStatusHandle, +}; + +use crate::core::traits::{Control, ServiceHost}; +use crate::error::{ErrorData, Result}; + +/// Fixed heartbeat cadence — SCM checkpoint bumps while in a `*Pending` state. +/// Mirrors the macOS host's 20 s; SCM has no `WATCHDOG_USEC` equivalent to read. +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20); + +/// Wait hint on a `*Pending` status — must exceed the heartbeat interval so the +/// SCM doesn't time the transition out between checkpoints. +const PENDING_WAIT_HINT: Duration = Duration::from_secs(30); + +// Pending control request. At-most-once per delivery burst (like the Unix flag): +// repeated controls before a poll collapse into the latest. +const CONTROL_NONE: u8 = 0; +const CONTROL_STOP: u8 = 1; +const CONTROL_SHUTDOWN: u8 = 2; + +// Current SCM state, tracked so `heartbeat` only bumps the checkpoint while +// pending (SCM ignores the checkpoint once the service is `Running`). +const STATE_START_PENDING: u8 = 0; +const STATE_RUNNING: u8 = 1; +const STATE_STOP_PENDING: u8 = 2; + +/// The console Ctrl-C handler is an `extern "system"` callback that cannot +/// capture, so the console host parks its control cell here for it to reach. +static CONSOLE_CONTROL: OnceLock> = OnceLock::new(); + +/// Up-facing host over the SCM (service) or a Ctrl-C handler (console). +pub struct WindowsHost { + /// SCM status handle; `None` in console mode (status calls are no-ops). + status_handle: Option, + /// Pending control request, set by the SCM handler / Ctrl-C handler and + /// drained by `poll_control`. + control: Arc, + /// Current SCM state, so `heartbeat` bumps the checkpoint only while pending. + state: AtomicU8, + /// Monotonic SCM checkpoint for `*Pending` transitions. + checkpoint: AtomicU32, +} + +impl WindowsHost { + /// Bind to the SCM: register the control handler (routing Stop/Shutdown onto + /// the control cell) and report `StartPending`. Call from the service main + /// after `service_dispatcher::start` hands control to it. + pub fn service(service_name: &str) -> Result { + let control = Arc::new(AtomicU8::new(CONTROL_NONE)); + let handler_control = control.clone(); + let status_handle = service_control_handler::register(service_name, move |ctrl| { + match map_service_control(ctrl) { + Some(Control::Stop) => { + handler_control.store(CONTROL_STOP, Ordering::SeqCst); + ServiceControlHandlerResult::NoError + } + Some(Control::Shutdown) => { + handler_control.store(CONTROL_SHUTDOWN, Ordering::SeqCst); + ServiceControlHandlerResult::NoError + } + // Interrogate must be acknowledged so the SCM sees us alive. + None if ctrl == ServiceControl::Interrogate => ServiceControlHandlerResult::NoError, + None => ServiceControlHandlerResult::NotImplemented, + } + }) + .map_err(|e| { + AlienError::new(ErrorData::Other { + message: format!("failed to register the SCM control handler: {e}"), + }) + })?; + + let host = Self { + status_handle: Some(status_handle), + control, + state: AtomicU8::new(STATE_START_PENDING), + checkpoint: AtomicU32::new(0), + }; + host.set_status(ServiceState::StartPending, ServiceControlAccept::empty()); + Ok(host) + } + + /// Console mode (`--console`, driven by the E2E suite): install a Ctrl-C + /// handler that requests Stop. No SCM, so the status calls are no-ops. + pub fn console() -> Result { + let control = Arc::new(AtomicU8::new(CONTROL_NONE)); + // Park the cell for the extern handler; ignore if already set (one + // console launcher per process). + let _ = CONSOLE_CONTROL.set(control.clone()); + install_console_ctrl_handler()?; + Ok(Self { + status_handle: None, + control, + state: AtomicU8::new(STATE_START_PENDING), + checkpoint: AtomicU32::new(0), + }) + } + + pub fn heartbeat_interval(&self) -> Duration { + HEARTBEAT_INTERVAL + } + + /// Report `SERVICE_STOPPED` — the service main calls this after `core::run` + /// returns. `exit_code` 0 is a clean stop; nonzero trips the SCM recovery + /// config (the doc-12 restart actions), the Windows analogue of systemd + /// respawning us on a nonzero exit. No-op in console mode. + pub fn report_stopped(&self, exit_code: u32) { + self.state.store(STATE_STOP_PENDING, Ordering::SeqCst); + let Some(handle) = self.status_handle.as_ref() else { + return; + }; + let status = ServiceStatus { + service_type: ServiceType::OWN_PROCESS, + current_state: ServiceState::Stopped, + controls_accepted: ServiceControlAccept::empty(), + exit_code: ServiceExitCode::Win32(exit_code), + checkpoint: 0, + wait_hint: Duration::default(), + process_id: None, + }; + if let Err(e) = handle.set_service_status(status) { + warn!(error = %e, "failed to report SERVICE_STOPPED"); + } + } + + /// Emit an SCM status (no-op in console mode). `checkpoint`/`wait_hint` only + /// matter for `*Pending` states; `Running`/`Stopped` report checkpoint 0. + fn set_status(&self, state: ServiceState, controls: ServiceControlAccept) { + let Some(handle) = self.status_handle.as_ref() else { + return; + }; + let pending = matches!( + state, + ServiceState::StartPending | ServiceState::StopPending + ); + let checkpoint = if pending { + self.checkpoint.fetch_add(1, Ordering::SeqCst) + 1 + } else { + 0 + }; + let status = ServiceStatus { + service_type: ServiceType::OWN_PROCESS, + current_state: state, + controls_accepted: controls, + exit_code: ServiceExitCode::Win32(0), + checkpoint, + wait_hint: if pending { + PENDING_WAIT_HINT + } else { + Duration::default() + }, + process_id: None, + }; + if let Err(e) = handle.set_service_status(status) { + warn!(error = %e, ?state, "failed to set the SCM service status"); + } + } +} + +impl ServiceHost for WindowsHost { + fn poll_control(&self) -> Option { + match self.control.swap(CONTROL_NONE, Ordering::SeqCst) { + CONTROL_STOP => Some(Control::Stop), + CONTROL_SHUTDOWN => Some(Control::Shutdown), + _ => None, + } + } + + fn report_ready(&self) { + self.state.store(STATE_RUNNING, Ordering::SeqCst); + self.set_status( + ServiceState::Running, + ServiceControlAccept::STOP | ServiceControlAccept::SHUTDOWN, + ); + } + + fn heartbeat(&self) { + // SCM ignores the checkpoint once `Running`; only bump while pending. + let state = match self.state.load(Ordering::SeqCst) { + STATE_RUNNING => return, + STATE_STOP_PENDING => ServiceState::StopPending, + _ => ServiceState::StartPending, + }; + self.set_status(state, ServiceControlAccept::empty()); + } + + fn report_stopping(&self) { + self.state.store(STATE_STOP_PENDING, Ordering::SeqCst); + self.set_status(ServiceState::StopPending, ServiceControlAccept::empty()); + } +} + +/// Pure mapping: an SCM control → the launcher's `Control`. Stop and Shutdown are +/// the only lifecycle controls we act on; the rest (Interrogate, Pause, …) are +/// not ours to translate. +fn map_service_control(control: ServiceControl) -> Option { + match control { + ServiceControl::Stop => Some(Control::Stop), + ServiceControl::Shutdown => Some(Control::Shutdown), + _ => None, + } +} + +/// Console Ctrl-C handler — `extern "system"`, can't capture, so it reads the +/// static control cell. Any close-ish console event requests Stop. +unsafe extern "system" fn console_ctrl_handler( + ctrl_type: u32, +) -> windows_sys::Win32::Foundation::BOOL { + use windows_sys::Win32::System::Console::{ + CTRL_BREAK_EVENT, CTRL_CLOSE_EVENT, CTRL_C_EVENT, CTRL_LOGOFF_EVENT, CTRL_SHUTDOWN_EVENT, + }; + let requested = ctrl_type == CTRL_C_EVENT + || ctrl_type == CTRL_BREAK_EVENT + || ctrl_type == CTRL_CLOSE_EVENT + || ctrl_type == CTRL_LOGOFF_EVENT + || ctrl_type == CTRL_SHUTDOWN_EVENT; + if requested { + if let Some(control) = CONSOLE_CONTROL.get() { + control.store(CONTROL_STOP, Ordering::SeqCst); + } + 1 // TRUE — handled + } else { + 0 // FALSE — pass to the next handler + } +} + +fn install_console_ctrl_handler() -> Result<()> { + // SAFETY: `console_ctrl_handler` is a static function that only touches the + // static atomic control cell — safe to run on the OS's Ctrl-C thread. + let ok = unsafe { + windows_sys::Win32::System::Console::SetConsoleCtrlHandler(Some(console_ctrl_handler), 1) + }; + if ok == 0 { + return Err(AlienError::new(ErrorData::Other { + message: "SetConsoleCtrlHandler failed".to_string(), + })); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scm_controls_map_to_lifecycle() { + assert_eq!( + map_service_control(ServiceControl::Stop), + Some(Control::Stop) + ); + assert_eq!( + map_service_control(ServiceControl::Shutdown), + Some(Control::Shutdown) + ); + assert_eq!(map_service_control(ServiceControl::Interrogate), None); + } +} diff --git a/crates/alien-launcher/src/platform/windows_child.rs b/crates/alien-launcher/src/platform/windows_child.rs new file mode 100644 index 000000000..ab9b51058 --- /dev/null +++ b/crates/alien-launcher/src/platform/windows_child.rs @@ -0,0 +1,234 @@ +//! Windows child supervisor — mirrors `unix_child.rs`. The operator is spawned +//! inside a Job Object (`command-group`, `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`: +//! die-with-parent for free — when the launcher dies the job closes and the +//! operator is terminated). Graceful `stop` sends `CTRL_BREAK` to the child's +//! process group, then escalates to Job termination after the grace window. +//! +//! Constructed by `main.rs`'s Windows `run_supervisor` (aliased +//! `ActiveChildSupervisor`) and driven through `core::run`. + +use std::collections::HashMap; +use std::path::Path; +use std::time::{Duration, Instant}; + +use alien_core::self_update::{ENV_HEALTH_ADDR, ENV_LAUNCHER_VERSION, ENV_SELF_UPDATE}; +use alien_error::{AlienError, Context, IntoAlienError}; +use command_group::{CommandGroup, GroupChild}; +use tracing::warn; +use windows_sys::Win32::System::Console::{GenerateConsoleCtrlEvent, CTRL_BREAK_EVENT}; + +use crate::core::traits::{ChildSupervisor, ExitStatus, OperatorHandle, UpdateEnv}; +use crate::error::{ErrorData, Result}; + +/// Poll cadence while waiting out a graceful stop's grace window. +const STOP_POLL_INTERVAL: Duration = Duration::from_millis(50); + +/// `CREATE_NEW_PROCESS_GROUP` — the child leads its own console process group so +/// `GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, pid)` targets only it, not the +/// launcher. (Die-with-parent is separate: command-group's kill-on-close Job.) +const CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200; + +/// `STILL_ACTIVE` — `GetExitCodeProcess` reports this while the process runs. +#[cfg(test)] +const STILL_ACTIVE: u32 = 259; + +#[derive(Default)] +pub struct WindowsChildSupervisor { + /// Live children by pid. The pid is also the console process-group id (the + /// child is a `CREATE_NEW_PROCESS_GROUP` leader), which is what + /// `GenerateConsoleCtrlEvent` targets on a graceful stop. + children: HashMap, +} + +impl WindowsChildSupervisor { + pub fn new() -> Self { + Self::default() + } + + fn child_mut(&mut self, handle: &OperatorHandle) -> Result<&mut GroupChild> { + self.children.get_mut(&handle.pid).ok_or_else(|| { + AlienError::new(ErrorData::Other { + message: format!("no live child for pid {}", handle.pid), + }) + }) + } +} + +impl ChildSupervisor for WindowsChildSupervisor { + fn spawn(&mut self, binary: &Path, env: &UpdateEnv) -> Result { + use std::os::windows::process::CommandExt; + + let mut command = std::process::Command::new(binary); + command + .env(ENV_SELF_UPDATE, "1") + .env(ENV_LAUNCHER_VERSION, &env.launcher_version) + .env(ENV_HEALTH_ADDR, env.health_addr.to_string()) + // Own console group so a later CTRL_BREAK targets only this child. + .creation_flags(CREATE_NEW_PROCESS_GROUP); + + // `group_spawn` puts the child in a Job Object with kill-on-close; on + // Windows >= 8 nested Jobs work, so this succeeds even when the launcher + // itself runs inside a Job (some CI) — command-group sets breakaway as + // needed. + let child = command + .group_spawn() + .into_alien_error() + .context(ErrorData::SpawnFailed { + binary_path: binary.display().to_string(), + message: "failed to spawn the operator in its own Job Object".to_string(), + })?; + let pid = child.id(); + self.children.insert(pid, child); + Ok(OperatorHandle { pid }) + } + + fn stop(&mut self, handle: &OperatorHandle, grace: Duration) -> Result<()> { + // Graceful: CTRL_BREAK to the child's process group (pid == pgid). + // Best-effort — a child that has already exited, ignores Ctrl events, or + // has no console just doesn't stop here; the grace-poll + Job-terminate + // escalation below is the hard guarantee. + // SAFETY: FFI call taking a plain pid; touches no shared state. + let sent = unsafe { GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, handle.pid) }; + if sent == 0 { + warn!( + pid = handle.pid, + "GenerateConsoleCtrlEvent(CTRL_BREAK) failed; escalating to Job termination" + ); + } + + let deadline = Instant::now() + grace; + loop { + if self.try_wait(handle)?.is_some() { + return Ok(()); + } + let now = Instant::now(); + if now >= deadline { + break; + } + std::thread::sleep(STOP_POLL_INTERVAL.min(deadline - now)); + } + + // Escalate: terminate the whole Job Object, then reap. + let child = self.child_mut(handle)?; + child.kill().into_alien_error().context(ErrorData::Other { + message: format!("failed to terminate the Job Object for pid {}", handle.pid), + })?; + child.wait().into_alien_error().context(ErrorData::Other { + message: format!("failed to reap the Job Object for pid {}", handle.pid), + })?; + Ok(()) + } + + fn try_wait(&mut self, handle: &OperatorHandle) -> Result> { + let child = self.child_mut(handle)?; + let status = child.try_wait().into_alien_error().context(ErrorData::Other { + message: format!("failed to poll child {}", handle.pid), + })?; + Ok(status.map(map_exit_status)) + } +} + +/// Map a process exit to our `ExitStatus`. On Windows `code()` is always `Some` +/// (no signals), so this never yields `Signal`. +fn map_exit_status(status: std::process::ExitStatus) -> ExitStatus { + match status.code() { + Some(code) => ExitStatus::Code(code), + None => ExitStatus::Unknown, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::windows::process::CommandExt; + + /// Spawn `cmd /c ` in a kill-on-close Job + its own process group, + /// and register it with the supervisor so `stop`/`try_wait` address it by + /// pid. (`ChildSupervisor::spawn` takes no args, so the tests build the + /// child directly — same as `unix_child`'s `spawn_sh`.) + fn spawn_cmd(sup: &mut WindowsChildSupervisor, cmdline: &str) -> OperatorHandle { + let child = std::process::Command::new("cmd") + .args(["/c", cmdline]) + .creation_flags(CREATE_NEW_PROCESS_GROUP) + .group_spawn() + .expect("group_spawn cmd"); + let pid = child.id(); + sup.children.insert(pid, child); + OperatorHandle { pid } + } + + /// Is `pid` still running? (`OpenProcess` + `GetExitCodeProcess` == STILL_ACTIVE.) + fn process_alive(pid: u32) -> bool { + use windows_sys::Win32::Foundation::CloseHandle; + use windows_sys::Win32::System::Threading::{ + GetExitCodeProcess, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, + }; + // SAFETY: standard Win32 query dance; the handle is closed on every path. + unsafe { + let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); + if handle.is_null() { + return false; + } + let mut code: u32 = 0; + let ok = GetExitCodeProcess(handle, &mut code); + CloseHandle(handle); + ok != 0 && code == STILL_ACTIVE + } + } + + #[test] + fn stop_terminates_a_long_running_child_within_grace() { + let mut sup = WindowsChildSupervisor::new(); + // `/nobreak` ignores Ctrl events, so this exercises the Job-terminate + // escalation, not the graceful path. + let handle = spawn_cmd(&mut sup, "timeout /t 30 /nobreak >nul"); + let started = Instant::now(); + sup.stop(&handle, Duration::from_secs(2)).expect("stop"); + assert!( + started.elapsed() < Duration::from_secs(3), + "stop must not wait for the 30s timeout" + ); + assert!( + sup.try_wait(&handle).expect("try_wait").is_some(), + "child should be gone after stop" + ); + } + + #[test] + fn try_wait_reports_the_exit_code() { + let mut sup = WindowsChildSupervisor::new(); + let handle = spawn_cmd(&mut sup, "exit 10"); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + if let Some(status) = sup.try_wait(&handle).expect("try_wait") { + assert_eq!(status, ExitStatus::Code(10)); + return; + } + assert!(Instant::now() < deadline, "child never exited"); + std::thread::sleep(Duration::from_millis(20)); + } + } + + #[test] + fn dropping_the_child_kills_it_via_job_close() { + // Spawn a long child, capture its pid, then drop the GroupChild — closing + // the Job handle fires JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE (the launcher's + // die-with-parent). The child must be gone shortly after. + let child = std::process::Command::new("cmd") + .args(["/c", "timeout /t 30 /nobreak >nul"]) + .creation_flags(CREATE_NEW_PROCESS_GROUP) + .group_spawn() + .expect("group_spawn"); + let pid = child.id(); + drop(child); + + let deadline = Instant::now() + Duration::from_secs(2); + while process_alive(pid) { + assert!( + Instant::now() < deadline, + "child survived the Job handle closing" + ); + std::thread::sleep(Duration::from_millis(50)); + } + } +} diff --git a/crates/alien-launcher/src/platform/windows_store.rs b/crates/alien-launcher/src/platform/windows_store.rs new file mode 100644 index 000000000..e2ca68b49 --- /dev/null +++ b/crates/alien-launcher/src/platform/windows_store.rs @@ -0,0 +1,624 @@ +//! Windows `VersionStore`: the on-disk version store with directory-junction +//! pointers (`junction` crate) instead of Unix symlinks. +//! +//! Unlike a Unix `rename(2)` over a symlink, a junction flip is **not atomic** — +//! Windows cannot rename over an existing directory, so a flip must +//! `create .new` → `remove ` → `rename .new` → ``. A +//! crash can therefore leave residue, which `current()` reconciles on the next +//! start (the launcher is single-threaded, so this is only ever a *restart* +//! concern, never a live race): +//! +//! - `.new` **and** `` present (crashed after create, before remove) +//! → finish the flip (remove old, rename new). +//! - `` missing, `.new` present (crashed mid-rename) → finish the +//! rename. +//! - `current` missing with no `.new` → reconstruct from `probation.json` (the +//! new version if probation is active, else `last-stable`) and log loudly. +//! +//! Everything non-pointer (markers, snapshots, gc candidates) comes from the +//! shared `store_common` helpers, identical to the Unix store. `gc` additionally +//! tolerates a locked binary (a just-swapped-away operator `.exe` still mapped): +//! it schedules the straggler for deletion on the next reboot rather than failing. +//! +//! Constructed by `main.rs`'s Windows `run_supervisor` (aliased +//! `ActiveVersionStore`) and driven through `core::run`. + +use std::path::{Path, PathBuf}; + +use alien_core::self_update as protocol; +use alien_error::{AlienError, Context, IntoAlienError}; +use tracing::warn; +use windows_sys::Win32::Storage::FileSystem::{ + GetDiskFreeSpaceExW, MoveFileExW, MOVEFILE_DELAY_UNTIL_REBOOT, +}; + +use crate::core::store_common; +use crate::core::traits::{FailureRecord, PendingMarker, ProbationMarker, Version, VersionStore}; +use crate::error::{ErrorData, Result}; + +pub struct WindowsVersionStore { + root: PathBuf, +} + +impl WindowsVersionStore { + /// Open a store rooted at the operator's data dir. Creates the layout + /// directories that may be missing (idempotent). Identical to the Unix store. + pub fn open(root: &Path) -> Result { + for dir in ["versions", "state", "state-snapshots", "failed", "download"] { + std::fs::create_dir_all(root.join(dir)) + .into_alien_error() + .context(ErrorData::StoreCorrupt { + path: root.display().to_string(), + message: format!("failed to create the '{dir}' directory"), + })?; + } + Ok(Self { + root: root.to_path_buf(), + }) + } + + fn pointer_path(&self, name: &str) -> PathBuf { + self.root.join(name) + } + + /// The absolute junction target for a version. Junctions (unlike symlinks) + /// require an absolute target, so this is `/versions/`. + fn version_target(&self, version: &Version) -> PathBuf { + protocol::version_dir(&self.root, version) + } + + /// Read a pointer junction → the version it names. Absent → None; a pointer + /// that exists but does not resolve/parse is store corruption. + fn read_pointer(&self, name: &str) -> Result> { + let path = self.pointer_path(name); + let target = match junction::get_target(&path) { + Ok(target) => target, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => { + return Err(e).into_alien_error().context(ErrorData::StoreCorrupt { + path: path.display().to_string(), + message: "failed to read the pointer junction".to_string(), + }); + } + }; + let version_str = target + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| corrupt_pointer(&path, "junction target has no version component"))?; + Version::parse(version_str) + .map(Some) + .map_err(|e| corrupt_pointer(&path, &format!("unparseable version in target: {e}"))) + } + + /// Repoint `name` at `versions/` via the non-atomic junction flip: + /// clear any stale `.new`, create the new junction, remove the old pointer + /// (Windows can't rename over it), then rename the new one into place. + fn write_pointer(&self, name: &str, version: &Version) -> Result<()> { + let path = self.pointer_path(name); + let staged = self.pointer_path(&format!("{name}.new")); + let target = self.version_target(version); + + remove_junction_if_present(&staged)?; + junction::create(&target, &staged) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to create pointer temp '{}'", staged.display()), + })?; + remove_junction_if_present(&path)?; + std::fs::rename(&staged, &path) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to commit pointer '{}'", path.display()), + }) + } + + /// Finish an interrupted flip: if `.new` survived a crash, complete it + /// (remove any leftover ``, rename `.new` into place). Covers the + /// "both present" and "mid-rename" residues; a no-op when there is no `.new`. + fn reconcile_flip(&self, name: &str) -> Result<()> { + let path = self.pointer_path(name); + let staged = self.pointer_path(&format!("{name}.new")); + if !exists_no_follow(&staged) { + return Ok(()); + } + warn!( + pointer = name, + "found a staged '.new' pointer from an interrupted flip; completing it" + ); + remove_junction_if_present(&path)?; + std::fs::rename(&staged, &path) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to complete an interrupted flip for '{}'", path.display()), + }) + } + + /// `current` is gone with no `.new` residue: rebuild it from the protocol + /// markers. An active probation means the new version was live; otherwise + /// fall back to `last-stable`. A truly fresh store (never installed) has + /// neither — that legitimately stays `None`. + fn reconstruct_current(&self) -> Result> { + let (target, source) = match self.read_probation()? { + Some(probation) => (probation.new, "probation.new"), + None => match self.last_stable()? { + Some(version) => (version, "last-stable"), + None => return Ok(None), + }, + }; + warn!( + version = %target, + source, + "current pointer is missing; reconstructing it from the recovery marker" + ); + self.write_pointer("current", &target)?; + Ok(Some(target)) + } +} + +fn corrupt_pointer(path: &Path, message: &str) -> AlienError { + AlienError::new(ErrorData::StoreCorrupt { + path: path.display().to_string(), + message: message.to_string(), + }) +} + +/// Presence check that does NOT follow the reparse point (so a junction counts +/// as present regardless of whether its target exists). +fn exists_no_follow(path: &Path) -> bool { + path.symlink_metadata().is_ok() +} + +/// Remove a junction (or plain empty dir) if present. `remove_dir` on a junction +/// deletes the reparse point itself, never the target's contents. +fn remove_junction_if_present(path: &Path) -> Result<()> { + match std::fs::remove_dir(path) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e).into_alien_error().context(ErrorData::Other { + message: format!("failed to remove pointer junction '{}'", path.display()), + }), + } +} + +impl VersionStore for WindowsVersionStore { + fn stage_dir(&self, version: &Version) -> PathBuf { + protocol::version_dir(&self.root, version) + } + + fn current(&self) -> Result> { + self.reconcile_flip("current")?; + match self.read_pointer("current")? { + Some(version) => Ok(Some(version)), + None => self.reconstruct_current(), + } + } + + fn last_stable(&self) -> Result> { + self.reconcile_flip("last-stable")?; + self.read_pointer("last-stable") + } + + fn set_current(&self, version: &Version) -> Result<()> { + self.write_pointer("current", version) + } + + fn set_last_stable(&self, version: &Version) -> Result<()> { + self.write_pointer("last-stable", version) + } + + fn snapshot_state(&self, tag: &Version) -> Result<()> { + store_common::snapshot_state_dir( + &self.root.join("state"), + &self.root.join("state-snapshots"), + tag, + ) + } + + fn restore_state(&self, tag: &Version) -> Result<()> { + store_common::restore_state_dir( + &self.root.join("state"), + &self.root.join("state-snapshots"), + tag, + ) + } + + fn drop_snapshot(&self, tag: &Version) -> Result<()> { + let dir = self.root.join("state-snapshots").join(tag.as_str()); + match std::fs::remove_dir_all(&dir) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e).into_alien_error().context(ErrorData::Other { + message: format!("failed to drop snapshot '{}'", dir.display()), + }), + } + } + + fn state_size(&self) -> Result { + store_common::dir_size(&self.root.join("state")) + } + + fn gc(&self, keep: &[Version]) -> Result<()> { + let all = self.list_versions()?; + let current = self.current()?; + let last_stable = self.last_stable()?; + for candidate in + store_common::gc_candidates(&all, keep, current.as_ref(), last_stable.as_ref()) + { + let dir = self.stage_dir(&candidate); + match std::fs::remove_dir_all(&dir) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + // A version's binary can still be mapped — a just-swapped-away + // operator `.exe` — and Windows refuses to delete an in-use + // file. Never fail gc for that: schedule the straggler for + // deletion on the next reboot (best-effort) and move on. + warn!( + version = %candidate, + error = %e, + "gc could not remove a version dir (likely a locked binary); scheduling reboot-delete" + ); + schedule_delete_on_reboot(&dir); + } + } + } + Ok(()) + } + + fn read_pending(&self) -> Result> { + store_common::read_marker(&protocol::pending_path(&self.root)) + } + + fn write_pending(&self, marker: &PendingMarker) -> Result<()> { + store_common::write_marker_atomic(&protocol::pending_path(&self.root), marker) + } + + fn clear_pending(&self) -> Result<()> { + store_common::remove_marker(&protocol::pending_path(&self.root)) + } + + fn read_probation(&self) -> Result> { + store_common::read_marker(&self.root.join("probation.json")) + } + + fn write_probation(&self, marker: &ProbationMarker) -> Result<()> { + store_common::write_marker_atomic(&self.root.join("probation.json"), marker) + } + + fn clear_probation(&self) -> Result<()> { + store_common::remove_marker(&self.root.join("probation.json")) + } + + fn read_failure(&self, version: &Version) -> Result> { + store_common::read_marker(&protocol::failure_path(&self.root, version)) + } + + fn write_failure(&self, record: &FailureRecord) -> Result<()> { + store_common::write_marker_atomic( + &protocol::failure_path(&self.root, &record.version), + record, + ) + } + + fn list_versions(&self) -> Result> { + let versions_dir = self.root.join("versions"); + let mut versions = Vec::new(); + for entry in std::fs::read_dir(&versions_dir) + .into_alien_error() + .context(ErrorData::Other { + message: format!("failed to read '{}'", versions_dir.display()), + })? + { + let entry = entry.into_alien_error().context(ErrorData::Other { + message: "failed to read a versions/ entry".to_string(), + })?; + let name = entry.file_name().to_string_lossy().into_owned(); + let version = Version::parse(&name).map_err(|e| { + AlienError::new(ErrorData::StoreCorrupt { + path: entry.path().display().to_string(), + message: format!("versions/ entry is not a version: {e}"), + }) + })?; + versions.push(version); + } + versions.sort(); + Ok(versions) + } + + fn free_space_for_snapshot(&self) -> Result<()> { + let required = self.state_size()?; + let available = free_bytes(&self.root)?; + store_common::check_space(required, available, "state snapshot") + } +} + +/// Free bytes available to the caller on the volume holding `path`. +fn free_bytes(path: &Path) -> Result { + let wide = to_wide_null(path); + let mut free_available: u64 = 0; + // SAFETY: `wide` is a valid NUL-terminated UTF-16 path; we pass a valid + // out-param for the caller-available free bytes and NULL for the two totals + // we don't need. + let ok = unsafe { + GetDiskFreeSpaceExW( + wide.as_ptr(), + &mut free_available, + std::ptr::null_mut(), + std::ptr::null_mut(), + ) + }; + if ok == 0 { + return Err(AlienError::new(ErrorData::Other { + message: format!("GetDiskFreeSpaceExW failed for '{}'", path.display()), + })); + } + Ok(free_available) +} + +/// Schedule `dir` and everything under it for deletion on the next reboot, +/// deepest entry first. Best-effort: `MOVEFILE_DELAY_UNTIL_REBOOT` needs admin, +/// so a scheduling failure is only logged — gc must not fail on a locked binary. +fn schedule_delete_on_reboot(dir: &Path) { + let mut paths = Vec::new(); + collect_depth_first(dir, &mut paths); + for path in &paths { + if !move_file_delay_until_reboot(path) { + warn!( + path = %path.display(), + "failed to schedule reboot-delete for a locked straggler (needs admin?)" + ); + } + } +} + +/// Collect `dir` and every descendant, children before their parent directory, +/// so reboot-deletion is scheduled in an order the OS accepts. A locked file +/// still enumerates (listing a directory does not open its files). +fn collect_depth_first(dir: &Path, out: &mut Vec) { + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if entry.file_type().map(|t| t.is_dir()).unwrap_or(false) { + collect_depth_first(&path, out); + } else { + out.push(path); + } + } + } + out.push(dir.to_path_buf()); +} + +/// `MoveFileExW(path, NULL, MOVEFILE_DELAY_UNTIL_REBOOT)` — mark `path` for +/// deletion at the next reboot. Returns whether the OS accepted the request. +fn move_file_delay_until_reboot(path: &Path) -> bool { + let wide = to_wide_null(path); + // SAFETY: `wide` is a valid NUL-terminated UTF-16 path; a NULL destination + // with DELAY_UNTIL_REBOOT schedules deletion rather than a move. + let ok = unsafe { MoveFileExW(wide.as_ptr(), std::ptr::null(), MOVEFILE_DELAY_UNTIL_REBOOT) }; + ok != 0 +} + +fn to_wide_null(path: &Path) -> Vec { + use std::os::windows::ffi::OsStrExt; + path.as_os_str().encode_wide().chain(std::iter::once(0)).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v(s: &str) -> Version { + Version::parse(s).unwrap() + } + + fn store_with(dir: &Path, versions: &[&str]) -> WindowsVersionStore { + let store = WindowsVersionStore::open(dir).unwrap(); + for ver in versions { + let stage = store.stage_dir(&v(ver)); + std::fs::create_dir_all(&stage).unwrap(); + std::fs::write(stage.join("alien-operator"), format!("binary-{ver}")).unwrap(); + } + store + } + + /// The pointer is a real junction, and a single-threaded flip storm always + /// leaves a readable, residue-free pointer. (The junction flip is not atomic, + /// so — unlike the Unix Linux test — there is no concurrent-reader variant: + /// the launcher reads and flips on one thread, and crash residue is covered + /// by the reconciliation tests below.) + #[test] + fn pointer_flip_storm_is_always_readable() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &["1.0.0", "2.0.0"]); + store.set_current(&v("1.0.0")).unwrap(); + assert!( + exists_no_follow(&dir.path().join("current")), + "the pointer must exist as a junction" + ); + + for i in 0..500 { + let ver = if i % 2 == 0 { "2.0.0" } else { "1.0.0" }; + store.set_current(&v(ver)).unwrap(); + assert_eq!(store.current().unwrap(), Some(v(ver))); + } + assert!( + !exists_no_follow(&dir.path().join("current.new")), + "no staged residue after the storm" + ); + } + + /// Residue (a): a crash left both `current` (old) and `current.new` (new). + /// `current()` finishes the flip → the new version, no residue. + #[test] + fn reconciles_current_and_new_both_present() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &["1.0.0", "2.0.0"]); + junction::create(&store.version_target(&v("1.0.0")), &dir.path().join("current")).unwrap(); + junction::create(&store.version_target(&v("2.0.0")), &dir.path().join("current.new")) + .unwrap(); + + assert_eq!(store.current().unwrap(), Some(v("2.0.0"))); + assert!(!exists_no_follow(&dir.path().join("current.new"))); + } + + /// Residue (b): a crash mid-rename left only `current.new`. `current()` + /// finishes the rename → the new version, and `current` now exists. + #[test] + fn reconciles_current_missing_new_present() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &["1.0.0", "2.0.0"]); + junction::create(&store.version_target(&v("2.0.0")), &dir.path().join("current.new")) + .unwrap(); + + assert_eq!(store.current().unwrap(), Some(v("2.0.0"))); + assert!(exists_no_follow(&dir.path().join("current"))); + assert!(!exists_no_follow(&dir.path().join("current.new"))); + } + + /// Residue (c) with probation: `current` is gone entirely. `current()` + /// reconstructs it from `probation.new` and re-creates the junction. + #[test] + fn reconstructs_current_from_probation() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &["1.0.0", "2.0.0"]); + store + .write_probation(&ProbationMarker { + new: v("2.0.0"), + old: v("1.0.0"), + started_at: chrono::Utc::now(), + attempt: 1, + }) + .unwrap(); + + assert_eq!(store.current().unwrap(), Some(v("2.0.0"))); + assert!(exists_no_follow(&dir.path().join("current"))); + } + + /// Residue (c) without probation: `current` gone, no probation → fall back to + /// `last-stable`. + #[test] + fn reconstructs_current_from_last_stable() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &["1.0.0", "2.0.0"]); + store.set_last_stable(&v("1.0.0")).unwrap(); + + assert_eq!(store.current().unwrap(), Some(v("1.0.0"))); + assert!(exists_no_follow(&dir.path().join("current"))); + } + + /// A fresh store (never installed) has no pointer and no recovery marker — + /// `current()` is legitimately `None`, not an error. + #[test] + fn fresh_store_has_no_current() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &[]); + assert_eq!(store.current().unwrap(), None); + } + + /// gc removes exactly the unpointed versions. + #[test] + fn gc_preserves_pointer_targets() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &["1.0.0", "1.1.0", "1.2.0"]); + store.set_current(&v("1.2.0")).unwrap(); + store.set_last_stable(&v("1.1.0")).unwrap(); + + store.gc(&[]).unwrap(); + assert_eq!(store.list_versions().unwrap(), vec![v("1.1.0"), v("1.2.0")]); + } + + /// gc must not fail when a candidate's binary is locked (an open handle with + /// no delete-share): the version is left in place (scheduled for reboot- + /// deletion — a no-op without admin), and gc still returns Ok. + #[test] + fn gc_tolerates_a_locked_binary() { + use std::os::windows::fs::OpenOptionsExt; + const FILE_SHARE_READ: u32 = 0x0000_0001; + + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &["1.0.0", "2.0.0"]); + store.set_current(&v("2.0.0")).unwrap(); + + let victim = store.stage_dir(&v("1.0.0")).join("alien-operator"); + // Open WITHOUT delete-share so remove_dir_all hits a sharing violation. + let _locked = std::fs::OpenOptions::new() + .read(true) + .share_mode(FILE_SHARE_READ) + .open(&victim) + .unwrap(); + + store.gc(&[]).expect("gc must not fail on a locked binary"); + assert!( + store.stage_dir(&v("1.0.0")).exists(), + "the locked version dir remains (deferred to reboot), not force-deleted" + ); + } + + /// The real disk has space for a tiny state dir — the preflight passes + /// against genuine free-space numbers. + #[test] + fn free_space_preflight_passes_on_a_real_disk() { + let dir = tempfile::tempdir().unwrap(); + let store = store_with(dir.path(), &[]); + std::fs::write(dir.path().join("state/db"), b"tiny").unwrap(); + store + .free_space_for_snapshot() + .expect("a 4-byte state dir must fit on any disk running this test"); + } + + // -- the full Phase-0 state-machine suite against the REAL store ----------- + + use crate::core::testing::{ + scenario_classification_rows, scenario_crash_injection_promote, + scenario_crash_injection_rollback, scenario_happy_promote, + scenario_rollback_on_probation_crash, scenario_rollback_restores_state, TestStoreOps, + }; + + impl TestStoreOps for WindowsVersionStore { + fn install_version(&self, version: &Version) { + let stage = self.stage_dir(version); + std::fs::create_dir_all(&stage).unwrap(); + std::fs::write(stage.join("alien-operator"), format!("binary-{version}")).unwrap(); + } + fn state_dir_path(&self) -> PathBuf { + self.root.join("state") + } + fn store_root(&self) -> PathBuf { + self.root.clone() + } + } + + fn windows(dir: &Path) -> WindowsVersionStore { + WindowsVersionStore::open(dir).unwrap() + } + + #[test] + fn state_machine_happy_promote_on_real_store() { + scenario_happy_promote(windows); + } + + #[test] + fn state_machine_rollback_restores_state_on_real_store() { + scenario_rollback_restores_state(windows); + } + + #[test] + fn state_machine_rollback_on_probation_crash_on_real_store() { + scenario_rollback_on_probation_crash(windows); + } + + #[test] + fn state_machine_classification_rows_on_real_store() { + scenario_classification_rows(windows); + } + + #[test] + fn state_machine_crash_injection_promote_on_real_store() { + scenario_crash_injection_promote(windows); + } + + #[test] + fn state_machine_crash_injection_rollback_on_real_store() { + scenario_crash_injection_rollback(windows); + } +} diff --git a/crates/alien-launcher/tests/platform_blind.rs b/crates/alien-launcher/tests/platform_blind.rs new file mode 100644 index 000000000..932ab5277 --- /dev/null +++ b/crates/alien-launcher/tests/platform_blind.rs @@ -0,0 +1,106 @@ +//! Mechanical guard: the launcher's `src/core/` must stay platform-blind. +//! +//! The core is written once and must compile and behave identically for +//! Linux, macOS, and Windows; all platform behavior belongs behind the +//! `core::traits` boundary in `src/platform/`. This test scans every source +//! file under `src/core/` for tokens that would leak a platform into the +//! core. It lives OUTSIDE `src/core/` on purpose — its own forbidden-token +//! list would otherwise trip the scan. +//! +//! Comments are stripped before matching so module docs may *describe* the +//! rule (and name the forbidden crates) without violating it. + +use std::path::{Path, PathBuf}; + +/// Tokens that must never appear in non-comment core source. Kept as +/// substrings so `use nix::...`, `nix::sys::...`, and `extern crate nix` +/// are all caught. +const FORBIDDEN: &[&str] = &[ + "std::os::unix", + "std::os::windows", + "nix::", + "rustix::", + "libc::", + "windows_sys", + "windows_service", + "command_group", + "sd_notify", + "junction::", + "cfg(unix)", + "cfg(windows)", + "cfg(target_os", +]; + +/// Strip `//`-style comments (line, doc, inner-doc). Good enough for this +/// codebase: the core contains no block comments or string literals holding +/// `//` (and a false *positive* here would only make the guard stricter). +fn strip_comments(source: &str) -> String { + source + .lines() + .map(|line| match line.find("//") { + Some(idx) => &line[..idx], + None => line, + }) + .collect::>() + .join("\n") +} + +fn collect_rs_files(dir: &Path, out: &mut Vec) { + for entry in std::fs::read_dir(dir) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", dir.display())) + { + let path = entry.expect("dir entry should be readable").path(); + if path.is_dir() { + collect_rs_files(&path, out); + } else if path.extension().is_some_and(|ext| ext == "rs") { + out.push(path); + } + } +} + +#[test] +fn core_is_platform_blind() { + let core_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/core"); + let mut files = Vec::new(); + collect_rs_files(&core_dir, &mut files); + assert!( + !files.is_empty(), + "no source files found under {} — the scan target moved?", + core_dir.display() + ); + + let mut violations = Vec::new(); + for file in &files { + let source = std::fs::read_to_string(file) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", file.display())); + let code = strip_comments(&source); + for token in FORBIDDEN { + if code.contains(token) { + violations.push(format!("{}: contains `{token}`", file.display())); + } + } + + // The state machine must drive the VersionStore trait exclusively — + // no direct filesystem access in its production code. Its #[cfg(test)] + // module legitimately inspects disk state, so the scan covers only the + // code BEFORE the first `#[cfg(test)]` — which relies on the Rust + // convention that the tests module ends the file. Don't add production + // code below the tests module; this guard would not see it. + if file.file_name().is_some_and(|name| name == "state_machine.rs") { + let production = code.split("#[cfg(test)]").next().unwrap_or(&code); + if production.contains("std::fs") { + violations.push(format!( + "{}: production code calls std::fs — go through VersionStore", + file.display() + )); + } + } + } + + assert!( + violations.is_empty(), + "src/core/ must stay platform-blind — move platform code behind the \ + core::traits boundary into src/platform/.\nViolations:\n{}", + violations.join("\n") + ); +} diff --git a/crates/alien-manager/Cargo.toml b/crates/alien-manager/Cargo.toml index d3c4492c8..efbc3c60d 100644 --- a/crates/alien-manager/Cargo.toml +++ b/crates/alien-manager/Cargo.toml @@ -76,6 +76,7 @@ tracing-subscriber = { workspace = true, features = ["env-filter"] } # HTTP http = { workspace = true } reqwest = { workspace = true, features = ["json"] } +semver = { workspace = true } # URL url = { workspace = true } diff --git a/crates/alien-manager/openapi.json b/crates/alien-manager/openapi.json index 3b2ffcadb..29a1f693c 100644 --- a/crates/alien-manager/openapi.json +++ b/crates/alien-manager/openapi.json @@ -886,6 +886,53 @@ } } }, + "/v1/deployments/{id}/target-operator-version": { + "put": { + "tags": [ + "deployments" + ], + "summary": "Admin-only knob behind the dashboard's \"Set target version\" control\n(and the equivalent flow on the SaaS API). Writes\n`target_operator_version` on the deployment row; the sync handler reads\nit on each /v1/sync and emits `operator_target` whenever it differs from\nthe operator's reported version, until they match.", + "operationId": "setDeploymentTargetOperatorVersion", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Deployment ID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetTargetOperatorVersionRequest" + } + } + }, + "required": true + }, + "responses": { + "202": { + "description": "Target operator version updated" + }, + "400": { + "description": "Invalid pin (not semver, build metadata, or a downgrade)" + }, + "401": { + "description": "Unauthorized" + }, + "403": { + "description": "Forbidden" + }, + "404": { + "description": "Not found" + } + } + } + }, "/v1/initialize": { "post": { "tags": [ @@ -932,6 +979,46 @@ ] } }, + "/v1/rejoin": { + "post": { + "tags": [ + "sync" + ], + "summary": "`POST /v1/rejoin` — re-acquire a deployment-scoped sync token for an\nexisting deployment by name. The OSS handler is intentionally lean:\nthe dg-token caller specifies the name, the store looks up the row,\nand a fresh `Deployment` token is minted and returned. Returns\n`404 Deployment not found` when no row matches — agents should fall\nback to `/v1/initialize` in that case.", + "description": "Distinct from `/v1/initialize`'s idempotency branch because the agent\nwants an explicit, distinguishable code path for \"I lost local state,\nplease re-attach me\" vs \"I'm a brand-new pod claiming a name\". The\nplatform-mode override forwards this to the SaaS rejoin endpoint\nwhere audit / event semantics differ.", + "operationId": "rejoin", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RejoinRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Existing deployment rejoined; fresh token returned", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RejoinResponse" + } + } + } + }, + "404": { + "description": "No deployment with that name in caller's deployment group" + } + }, + "security": [ + { + "bearer": [] + } + ] + } + }, "/v1/releases": { "get": { "tags": [ @@ -1418,18 +1505,56 @@ "deploymentId": { "type": "string" }, + "launcherVersion": { + "type": [ + "string", + "null" + ], + "description": "Version of the alien-launcher supervising this operator (os-service\nonly; reported, never driven). Gates binary targets against\nmin_launcher_version." + }, "observedInventoryBatches": { "type": "array", "items": { "$ref": "#/components/schemas/ObservedInventoryBatch" } }, + "operatorArch": { + "type": [ + "string", + "null" + ], + "description": "Operator host arch — `x86_64` / `aarch64`." + }, + "operatorImageRepository": { + "type": [ + "string", + "null" + ], + "description": "Image repository the operator was pulled from (no tag), chart-injected.\nSurfaced in the dashboard pin-version UI. Optional and Kubernetes-only." + }, + "operatorOs": { + "type": [ + "string", + "null" + ], + "description": "Operator host OS — `linux` / `macos` / `windows`." + }, + "operatorUpdate": { + "description": "Outcome of the operator's in-flight self-update (structured\n`OperatorUpdateReport`), forwarded verbatim to the platform reconcile so\nthe dashboard can show a truthful failed/in-progress state; opaque to the\nstandalone manager. Optional for back-compat." + }, "operatorVersion": { "type": [ "string", "null" ] }, + "packaging": { + "type": [ + "string", + "null" + ], + "description": "Supervisor packaging — `os-service` / `kubernetes`." + }, "resourceHeartbeats": { "type": "array", "items": { @@ -1452,6 +1577,9 @@ "currentState": { "description": "Authoritative deployment state from the manager.\n\nReturned when a pull deployment attaches with an empty local state while\nthe manager already has imported or previously reconciled state." }, + "operatorTarget": { + "description": "Desired agent self-update target. The payload carries either `binary`\n(OS-service flow) or `helm` (Kubernetes flow); the agent picks the\none matching its packaging." + }, "target": {} } }, @@ -6283,9 +6411,46 @@ } ] }, + "launcherVersion": { + "type": [ + "string", + "null" + ], + "description": "Version of the frozen alien-launcher supervising the operator\n(os-service only). Drives the \"redeploy required\" surface when it is\nbelow a target's minLauncherVersion." + }, "name": { "type": "string" }, + "operatorArch": { + "type": [ + "string", + "null" + ] + }, + "operatorImageRepository": { + "type": [ + "string", + "null" + ] + }, + "operatorOs": { + "type": [ + "string", + "null" + ] + }, + "operatorVersion": { + "type": [ + "string", + "null" + ] + }, + "packaging": { + "type": [ + "string", + "null" + ] + }, "platform": { "$ref": "#/components/schemas/Platform" }, @@ -6302,6 +6467,12 @@ "status": { "type": "string" }, + "targetOperatorVersion": { + "type": [ + "string", + "null" + ] + }, "updatedAt": { "type": [ "string", @@ -12128,6 +12299,33 @@ } } }, + "RejoinRequest": { + "type": "object", + "description": "`POST /v1/rejoin` — re-acquire a deployment-scoped sync token for an\nagent whose persistent state was wiped (e.g. emptyDir on pod restart).\n\nThe agent calls this when `/v1/initialize` returns a name-conflict\nerror: the deployment row already exists, so creating it would 409,\nbut the agent legitimately needs a new sync token to keep operating\nagainst it. Auth is the same dg bearer the chart originally mounted.\n\n`name` is required — without it the server can't disambiguate which\nrow to reattach to.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + } + } + }, + "RejoinResponse": { + "type": "object", + "required": [ + "deploymentId", + "token" + ], + "properties": { + "deploymentId": { + "type": "string" + }, + "token": { + "type": "string" + } + } + }, "ReleaseRequest": { "type": "object", "required": [ @@ -13050,6 +13248,18 @@ } } }, + "SetTargetOperatorVersionRequest": { + "type": "object", + "properties": { + "targetOperatorVersion": { + "type": [ + "string", + "null" + ], + "description": "Target operator version (semver, no `+build` metadata).\n`None`/omitted clears the target." + } + } + }, "StackByPlatform": { "type": "object", "description": "The release API accepts stacks keyed by platform.\nOnly one platform stack needs to be present.", diff --git a/crates/alien-manager/src/api.rs b/crates/alien-manager/src/api.rs index 9a1b27a10..0ea76b351 100644 --- a/crates/alien-manager/src/api.rs +++ b/crates/alien-manager/src/api.rs @@ -25,6 +25,7 @@ use utoipa::OpenApi; crate::routes::deployments::delete_deployment, crate::routes::deployments::retry_deployment, crate::routes::deployments::redeploy, + crate::routes::deployments::set_target_operator_version, // Releases crate::routes::releases::create_release, crate::routes::releases::list_releases, @@ -43,12 +44,14 @@ use utoipa::OpenApi; crate::routes::sync::release, crate::routes::sync::agent_sync, crate::routes::sync::initialize, + crate::routes::sync::rejoin, // Credentials crate::routes::credentials::resolve_credentials, ), components(schemas( // Deployment types crate::routes::deployments::CreateDeploymentRequest, + crate::routes::deployments::SetTargetOperatorVersionRequest, crate::routes::deployments::CreateDeploymentResponse, crate::routes::deployments::DeploymentResponse, crate::routes::deployments::DeploymentGroupMinimal, @@ -83,6 +86,8 @@ use utoipa::OpenApi; crate::routes::sync::AgentSyncResponse, crate::routes::sync::InitializeRequest, crate::routes::sync::InitializeResponse, + crate::routes::sync::RejoinRequest, + crate::routes::sync::RejoinResponse, // Credentials types crate::routes::credentials::ResolveCredentialsRequest, crate::routes::credentials::ResolveCredentialsResponse, diff --git a/crates/alien-manager/src/builder.rs b/crates/alien-manager/src/builder.rs index 8e0fe485c..dd8966b4e 100644 --- a/crates/alien-manager/src/builder.rs +++ b/crates/alien-manager/src/builder.rs @@ -30,6 +30,9 @@ pub struct AlienManagerBuilder { skip_initialize: bool, /// When `true`, the install script (`/v1/install`) is omitted from the router. skip_install: bool, + /// When `true`, the default `/v1/rejoin` route is omitted. Multi-tenant + /// embedders override this to forward to the SaaS rejoin endpoint. + skip_rejoin: bool, /// Override the bindings provider for cross-account registry access. /// When set in `with_standalone_defaults()`, this is stored in `ServerBindings` /// so `reconcile_registry_access()` can load the artifact registry and grant @@ -70,6 +73,7 @@ impl AlienManagerBuilder { log_buffer: None, skip_initialize: false, skip_install: false, + skip_rejoin: false, bindings_provider_override: None, target_bindings_providers_override: None, import_registry: None, @@ -162,6 +166,14 @@ impl AlienManagerBuilder { self } + /// Skip the default `/v1/rejoin` route. + /// Use this when embedding in a process that overrides rejoin via `extra_routes` + /// (e.g. multi-tenant managerx that forwards to the platform API). + pub fn skip_rejoin(mut self) -> Self { + self.skip_rejoin = true; + self + } + /// Inject a custom `/v1/stack/import` dispatch registry. Defaults to /// [`alien_infra::ImporterRegistry::built_in`]. /// @@ -629,6 +641,7 @@ impl AlienManagerBuilder { crate::routes::RouterOptions { include_initialize: !self.skip_initialize, include_install: !self.skip_install, + include_rejoin: !self.skip_rejoin, }, self.extra_routes, self.platform_routes, diff --git a/crates/alien-manager/src/lib.rs b/crates/alien-manager/src/lib.rs index 94d7f5515..00ba291a7 100644 --- a/crates/alien-manager/src/lib.rs +++ b/crates/alien-manager/src/lib.rs @@ -51,6 +51,7 @@ pub(crate) mod dev; pub mod loops; pub mod providers; pub mod registry_access; +pub mod release_manifest; pub mod routes; pub mod server; pub mod transports; diff --git a/crates/alien-manager/src/loops/deployment.rs b/crates/alien-manager/src/loops/deployment.rs index 4d2d8bfd9..953d85e1d 100644 --- a/crates/alien-manager/src/loops/deployment.rs +++ b/crates/alien-manager/src/loops/deployment.rs @@ -451,6 +451,16 @@ impl DeploymentLoop { observed_inventory_batches: Vec::new(), capabilities: Vec::new(), operator_version: None, + launcher_version: None, + // Background driver loop — no operator sync here, + // so leave the operator-inventory columns untouched. + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + operator_update: None, + redeploy_required: None, + min_launcher_version: None, }, ) .await?; @@ -980,6 +990,13 @@ mod tests { created_at: Utc::now(), updated_at: None, error: None, + operator_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + target_operator_version: None, + launcher_version: None, } } diff --git a/crates/alien-manager/src/providers/oss_authz.rs b/crates/alien-manager/src/providers/oss_authz.rs index 01c3c40d4..521277ece 100644 --- a/crates/alien-manager/src/providers/oss_authz.rs +++ b/crates/alien-manager/src/providers/oss_authz.rs @@ -288,6 +288,13 @@ mod tests { created_at: Utc::now(), updated_at: None, error: None, + operator_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + target_operator_version: None, + launcher_version: None, } } diff --git a/crates/alien-manager/src/release_manifest.rs b/crates/alien-manager/src/release_manifest.rs new file mode 100644 index 000000000..a3c0098fb --- /dev/null +++ b/crates/alien-manager/src/release_manifest.rs @@ -0,0 +1,251 @@ +//! Release-manifest loading for operator binary self-updates. +//! +//! The release pipeline publishes, per version, a `manifest.json` next to the +//! binaries it uploads: +//! +//! ```json +//! { +//! "version": "1.4.0", +//! "minLauncherVersion": "0.1.0", +//! "artifacts": { +//! "linux/x86_64": { "url": "https://…", "sha256": "…", "signature": "" }, +//! "darwin/aarch64": { "url": "https://…", "sha256": "…", "signature": "" } +//! } +//! } +//! ``` +//! +//! When an admin pins a target version for an os-service deployment, the sync +//! handler loads `//manifest.json`, resolves the +//! artifact for the host's reported `(os, arch)`, and emits a per-host +//! `operator_target.binary`. A manifest that fails to load or is missing the +//! host's platform yields NO target — never a partial one. +//! +//! Manifests are immutable once published, so they are cached per URL for the +//! manager's lifetime. The base may be `http(s)://…`, `file://…`, or a plain +//! filesystem path (tests, air-gapped mirrors). + +use std::collections::{BTreeMap, HashMap}; +use std::sync::{Arc, Mutex, OnceLock}; + +use alien_error::{AlienError, Context, IntoAlienError}; +use serde::Deserialize; + +use crate::error::{ErrorData, Result}; + +/// One downloadable binary in a release. +#[derive(Debug, Clone, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ManifestArtifact { + /// Download URL for this `(os, arch)`. + pub url: String, + /// SHA-256 of exactly that artifact, lowercase hex. + pub sha256: String, + /// ed25519 detached signature, base64. Empty until the signing + /// workstream lands. + #[serde(default)] + pub signature: String, +} + +/// The per-version release index the pipeline publishes. +#[derive(Debug, Clone, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ReleaseManifest { + /// The release version this manifest describes. + pub version: String, + /// The installed (frozen) launcher must be >= this for the manager to + /// advertise the target; otherwise "redeploy required". + pub min_launcher_version: String, + /// `"/"` (e.g. `"linux/x86_64"`) → artifact. + pub artifacts: BTreeMap, +} + +/// Outcome of checking an installed launcher against a target artifact's floor. +/// Shared by the sync-route target gate and the platform reconcile so both +/// agree on exactly one definition of "the launcher is too old". +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LauncherFloorCheck { + /// The launcher satisfies the floor — the target may be delivered. + Ok, + /// The launcher is below the floor (or unreported/unparseable) — the target + /// is withheld and the user must **redeploy** to get a newer launcher. + RedeployRequired, + /// The manifest's `minLauncherVersion` is itself unparseable — a manifest + /// error, not something a redeploy fixes. + InvalidFloor, +} + +/// Does the installed (frozen) launcher satisfy the artifact's minimum? The +/// launcher never self-updates, so a launcher below the floor cannot actuate +/// the newer handoff contract and the target must be withheld until the +/// operator is redeployed with a newer launcher. An unreported or unparseable +/// launcher is treated as below the floor (fail-closed). +pub fn check_launcher_floor( + launcher_version: Option<&str>, + min_launcher_version: &str, +) -> LauncherFloorCheck { + let Ok(min) = semver::Version::parse(min_launcher_version) else { + return LauncherFloorCheck::InvalidFloor; + }; + let meets_floor = launcher_version + .and_then(|v| semver::Version::parse(v).ok()) + .is_some_and(|v| v >= min); + if meets_floor { + LauncherFloorCheck::Ok + } else { + LauncherFloorCheck::RedeployRequired + } +} + +/// Load (and cache) the manifest for `version` under `base`. `base` accepts +/// `http(s)://`, `file://`, or a plain filesystem directory path. +pub async fn load(base: &str, version: &str) -> Result> { + let location = format!("{}/{}/manifest.json", base.trim_end_matches('/'), version); + + static CACHE: OnceLock>>> = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new())); + if let Some(hit) = cache + .lock() + .expect("manifest cache lock should not be poisoned") + .get(&location) + { + return Ok(hit.clone()); + } + + let bytes = fetch(&location).await?; + let manifest: ReleaseManifest = serde_json::from_slice(&bytes) + .into_alien_error() + .context(ErrorData::BadRequest { + reason: format!("release manifest at '{location}' is not valid manifest JSON"), + })?; + let manifest = Arc::new(manifest); + + cache + .lock() + .expect("manifest cache lock should not be poisoned") + .insert(location, manifest.clone()); + Ok(manifest) +} + +async fn fetch(location: &str) -> Result> { + if let Some(path) = location.strip_prefix("file://") { + return read_file(path).await; + } + if location.starts_with("http://") || location.starts_with("https://") { + let response = reqwest::get(location) + .await + .into_alien_error() + .context(ErrorData::BadRequest { + reason: format!("failed to fetch release manifest from '{location}'"), + })?; + if !response.status().is_success() { + return Err(AlienError::new(ErrorData::BadRequest { + reason: format!( + "release manifest at '{location}' returned HTTP {}", + response.status() + ), + })); + } + return response + .bytes() + .await + .map(|b| b.to_vec()) + .into_alien_error() + .context(ErrorData::BadRequest { + reason: format!("failed to read release manifest body from '{location}'"), + }); + } + // Plain filesystem path (tests, air-gapped mirrors). + read_file(location).await +} + +async fn read_file(path: &str) -> Result> { + tokio::fs::read(path) + .await + .into_alien_error() + .context(ErrorData::BadRequest { + reason: format!("failed to read release manifest file '{path}'"), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write_manifest(dir: &std::path::Path, version: &str, body: &str) { + let version_dir = dir.join(version); + std::fs::create_dir_all(&version_dir).unwrap(); + std::fs::write(version_dir.join("manifest.json"), body).unwrap(); + } + + const VALID: &str = r#"{ + "version": "1.4.0", + "minLauncherVersion": "0.2.0", + "artifacts": { + "linux/x86_64": { "url": "https://example.com/op-linux", "sha256": "aa11" }, + "macos/aarch64": { "url": "https://example.com/op-mac", "sha256": "bb22", "signature": "sig" } + } + }"#; + + #[tokio::test] + async fn loads_and_caches_from_a_plain_path() { + let dir = tempfile::tempdir().unwrap(); + write_manifest(dir.path(), "1.4.0", VALID); + + let base = dir.path().to_str().unwrap().to_string(); + let manifest = load(&base, "1.4.0").await.expect("manifest should load"); + assert_eq!(manifest.version, "1.4.0"); + assert_eq!(manifest.min_launcher_version, "0.2.0"); + let artifact = &manifest.artifacts["linux/x86_64"]; + assert_eq!(artifact.url, "https://example.com/op-linux"); + assert_eq!(artifact.sha256, "aa11"); + assert_eq!(artifact.signature, "", "signature defaults to empty"); + assert_eq!(manifest.artifacts["macos/aarch64"].signature, "sig"); + + // Cached: deleting the file no longer matters (manifests are immutable). + std::fs::remove_file(dir.path().join("1.4.0/manifest.json")).unwrap(); + let again = load(&base, "1.4.0").await.expect("cache should serve"); + assert_eq!(again.version, "1.4.0"); + } + + #[tokio::test] + async fn file_url_scheme_works() { + let dir = tempfile::tempdir().unwrap(); + write_manifest(dir.path(), "2.0.0", &VALID.replace("1.4.0", "2.0.0")); + let base = format!("file://{}", dir.path().to_str().unwrap()); + let manifest = load(&base, "2.0.0").await.expect("file:// should load"); + assert_eq!(manifest.version, "2.0.0"); + } + + #[tokio::test] + async fn missing_manifest_is_a_loud_error() { + let dir = tempfile::tempdir().unwrap(); + let base = dir.path().to_str().unwrap().to_string(); + let err = load(&base, "9.9.9").await.expect_err("missing must error"); + assert_eq!(err.code, "BAD_REQUEST"); + } + + #[tokio::test] + async fn invalid_json_is_a_loud_error() { + let dir = tempfile::tempdir().unwrap(); + write_manifest(dir.path(), "3.0.0", "{ not json"); + let base = dir.path().to_str().unwrap().to_string(); + let err = load(&base, "3.0.0").await.expect_err("garbage must error"); + assert_eq!(err.code, "BAD_REQUEST"); + assert!(err.to_string().contains("manifest"), "{err}"); + } + + #[test] + fn launcher_floor_check_rows() { + use LauncherFloorCheck::*; + // At or above the floor → Ok. + assert_eq!(check_launcher_floor(Some("0.2.0"), "0.2.0"), Ok); + assert_eq!(check_launcher_floor(Some("1.0.0"), "0.2.0"), Ok); + // Below the floor → redeploy required. + assert_eq!(check_launcher_floor(Some("0.1.9"), "0.2.0"), RedeployRequired); + // Unreported or unparseable launcher fails closed → redeploy required. + assert_eq!(check_launcher_floor(None, "0.2.0"), RedeployRequired); + assert_eq!(check_launcher_floor(Some("garbage"), "0.2.0"), RedeployRequired); + // Unparseable floor is a manifest error, not a redeploy case. + assert_eq!(check_launcher_floor(Some("1.0.0"), "not-semver"), InvalidFloor); + } +} diff --git a/crates/alien-manager/src/routes/deployments.rs b/crates/alien-manager/src/routes/deployments.rs index 654c64ead..33ec55c0d 100644 --- a/crates/alien-manager/src/routes/deployments.rs +++ b/crates/alien-manager/src/routes/deployments.rs @@ -4,7 +4,7 @@ use axum::{ extract::{Path, State}, http::{request::Parts, HeaderMap, StatusCode}, response::{IntoResponse, Response}, - routing::{get, post}, + routing::{get, post, put}, Json, Router, }; use serde::{Deserialize, Serialize}; @@ -91,6 +91,25 @@ pub struct DeploymentResponse { pub error: Option, #[serde(skip_serializing_if = "Option::is_none")] pub deployment_group: Option, + // Agent self-update inventory — populated by the sync handler + // on every operator /v1/sync. NULL until the operator has first reported. + #[serde(skip_serializing_if = "Option::is_none")] + pub operator_version: Option, + /// Version of the frozen alien-launcher supervising the operator + /// (os-service only). Drives the "redeploy required" surface when it is + /// below a target's minLauncherVersion. + #[serde(skip_serializing_if = "Option::is_none")] + pub launcher_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub operator_os: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub operator_arch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub packaging: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub operator_image_repository: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub target_operator_version: Option, } #[derive(Debug, Serialize, Clone)] @@ -218,6 +237,10 @@ pub fn router() -> Router { .route("/v1/deployments/{id}/info", get(get_deployment_info)) .route("/v1/deployments/{id}/retry", post(retry_deployment)) .route("/v1/deployments/{id}/redeploy", post(redeploy)) + .route( + "/v1/deployments/{id}/target-operator-version", + put(set_target_operator_version), + ) } // --- Helpers --- @@ -265,6 +288,14 @@ fn record_to_response( ), error: r.error.clone(), deployment_group, + // Surface the operator self-update inventory. + operator_version: r.operator_version.clone(), + launcher_version: r.launcher_version.clone(), + operator_os: r.operator_os.clone(), + operator_arch: r.operator_arch.clone(), + packaging: r.packaging.clone(), + operator_image_repository: r.operator_image_repository.clone(), + target_operator_version: r.target_operator_version.clone(), } } @@ -889,10 +920,175 @@ async fn redeploy( Json(serde_json::json!({ "success": true })).into_response() } +#[derive(Debug, Deserialize)] +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[serde(rename_all = "camelCase")] +pub struct SetTargetOperatorVersionRequest { + /// Target operator version (semver, no `+build` metadata). + /// `None`/omitted clears the target. + #[serde(default)] + pub target_operator_version: Option, +} + +/// Admin-only knob behind the dashboard's "Set target version" control +/// (and the equivalent flow on the SaaS API). Writes +/// `target_operator_version` on the deployment row; the sync handler reads +/// it on each /v1/sync and emits `operator_target` whenever it differs from +/// the operator's reported version, until they match. +#[cfg_attr(feature = "openapi", utoipa::path( + put, + operation_id = "setDeploymentTargetOperatorVersion", + path = "/v1/deployments/{id}/target-operator-version", + tag = "deployments", + params( + ("id" = String, Path, description = "Deployment ID"), + ), + request_body = SetTargetOperatorVersionRequest, + responses( + (status = 202, description = "Target operator version updated"), + (status = 400, description = "Invalid pin (not semver, build metadata, or a downgrade)"), + (status = 401, description = "Unauthorized"), + (status = 403, description = "Forbidden"), + (status = 404, description = "Not found"), + ) +))] +async fn set_target_operator_version( + State(state): State, + headers: HeaderMap, + Path(id): Path, + Json(req): Json, +) -> Response { + let subject = match auth::require_auth(&state, &headers).await { + Ok(s) => s, + Err(e) => return e.into_response(), + }; + + let deployment = match state.deployment_store.get_deployment(&subject, &id).await { + Ok(Some(d)) => d, + Ok(None) => return ErrorData::not_found_deployment(&id).into_response(), + Err(e) => return e.into_response(), + }; + if !state.authz.can_update_deployment(&subject, &deployment) { + return ErrorData::forbidden("Cannot set target operator version").into_response(); + } + + // Validate the pin early so admins get a 4xx rather than the operator + // silently ignoring a bad tag later (or a downgrade handing an old binary + // a state DB migrated past it). + if let Some(v) = req.target_operator_version.as_deref() { + if let Err(reason) = validate_target_pin(v, deployment.operator_version.as_deref()) { + return ErrorData::bad_request(reason).into_response(); + } + } + + if let Err(e) = state + .deployment_store + .set_target_operator_version(&subject, &id, req.target_operator_version.as_deref()) + .await + { + return e.into_response(); + } + + ( + StatusCode::ACCEPTED, + Json(serde_json::json!({ "success": true })), + ) + .into_response() +} + +/// Validate a target-version pin with the same parser used for ordering +/// everywhere else (`semver`): one grammar, no drift between "accepted" and +/// "comparable". +/// +/// Three rules: +/// - must parse as SemVer (prerelease pins like `1.5.0-rc.1` are fine — +/// canary channels need them); +/// - must not carry `+build` metadata: build metadata is semver-EQUAL but +/// string-UNEQUAL, and convergence compares exact strings, so such a pin +/// could never converge; +/// - must not be a downgrade below the reported operator version: the new +/// version may have migrated the state DB and the promote path discards +/// the pre-migration snapshot, so an older binary may be unable to open +/// the current state. Recovery from a bad release is rolling FORWARD. +/// An unreported/unparseable current version allows the pin (unknown is +/// not lower; the operator-side floor still protects). +fn validate_target_pin(pin: &str, reported_version: Option<&str>) -> Result<(), String> { + let pin_version = semver::Version::parse(pin).map_err(|e| { + format!("targetOperatorVersion must be a semver string (e.g. 1.4.0): {e}") + })?; + if !pin_version.build.is_empty() { + return Err( + "targetOperatorVersion must not contain build metadata (+…): convergence compares exact version strings, so a build-metadata pin can never converge" + .to_string(), + ); + } + if let Some(reported) = reported_version.and_then(|v| semver::Version::parse(v).ok()) { + if pin_version < reported { + return Err(format!( + "targetOperatorVersion {pin_version} is a downgrade below the reported operator version {reported}; downgrades are not supported (state may have been migrated past the target) — roll forward to a fixed version instead" + )); + } + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn pin_accepts_valid_versions_including_prerelease() { + for v in ["1.4.0", "0.0.1", "1.10.5", "1.4.0-rc.1", "2.0.0-alpha-1"] { + assert!( + validate_target_pin(v, None).is_ok(), + "should accept {v} with no reported version" + ); + assert!( + validate_target_pin(v, Some("0.0.1")).is_ok(), + "should accept {v} above reported 0.0.1" + ); + } + // Equal and higher pins are fine. + assert!(validate_target_pin("1.4.0", Some("1.4.0")).is_ok()); + assert!(validate_target_pin("1.5.0", Some("1.4.0")).is_ok()); + } + + #[test] + fn pin_rejects_garbage_semver() { + for v in ["", "1.4", "1.4.0.0", "v1.4.0", "1.4.x", "1.4.0 ", "01.2.3"] { + let err = validate_target_pin(v, None).expect_err(&format!("should reject {v:?}")); + assert!(err.contains("semver"), "{err}"); + } + } + + #[test] + fn pin_rejects_build_metadata() { + for v in ["1.4.0+build.123", "1.4.0-rc.1+build.123"] { + let err = validate_target_pin(v, None).expect_err("build metadata must be rejected"); + assert!(err.contains("build metadata"), "{err}"); + } + } + + #[test] + fn pin_rejects_downgrades_by_semver_precedence() { + let err = validate_target_pin("1.3.9", Some("1.4.0")).expect_err("downgrade"); + assert!(err.contains("downgrade"), "{err}"); + // Prerelease of the SAME version sorts below its release → downgrade. + let err = validate_target_pin("1.4.0-rc.1", Some("1.4.0")).expect_err("prerelease below"); + assert!(err.contains("downgrade"), "{err}"); + // Numeric (not lexical) comparison: 1.10.0 > 1.9.0 is NOT a downgrade. + assert!(validate_target_pin("1.10.0", Some("1.9.0")).is_ok()); + } + + #[test] + fn pin_allows_unknown_or_unparseable_reported_version() { + // Never-synced deployment: nothing to compare against. + assert!(validate_target_pin("1.4.0", None).is_ok()); + // A garbage reported version cannot be ordered — allow the pin + // (the operator-side floor still protects). + assert!(validate_target_pin("1.4.0", Some("not-a-version")).is_ok()); + } + #[test] fn local_deployments_default_to_push_for_embedded_dev_loop() { let settings = stack_settings_for_platform(Platform::Local, None) diff --git a/crates/alien-manager/src/routes/mod.rs b/crates/alien-manager/src/routes/mod.rs index 93187ffce..c45a0ab1d 100644 --- a/crates/alien-manager/src/routes/mod.rs +++ b/crates/alien-manager/src/routes/mod.rs @@ -90,6 +90,7 @@ pub fn create_router(state: AppState) -> Router { RouterOptions { include_initialize: true, include_install: true, + include_rejoin: true, }, ) .layer(cors) @@ -99,6 +100,11 @@ pub fn create_router(state: AppState) -> Router { pub struct RouterOptions { pub include_initialize: bool, pub include_install: bool, + /// Whether to mount `/v1/rejoin`. Multi-tenant embedders set this + /// to `false` and provide their own override that forwards to the + /// SaaS rejoin endpoint where audit / token-rotation semantics + /// differ. + pub include_rejoin: bool, } /// Like [`create_router`], but lets the caller opt-out of specific routes. @@ -147,6 +153,9 @@ pub fn create_router_inner(state: AppState, options: RouterOptions) -> Router { if options.include_initialize { router = router.merge(sync::initialize_router()); } + if options.include_rejoin { + router = router.merge(sync::rejoin_router()); + } router.with_state(state) } diff --git a/crates/alien-manager/src/routes/sync.rs b/crates/alien-manager/src/routes/sync.rs index 629e3f5cf..19d298659 100644 --- a/crates/alien-manager/src/routes/sync.rs +++ b/crates/alien-manager/src/routes/sync.rs @@ -130,6 +130,30 @@ pub struct AgentSyncRequest { pub capabilities: Vec, #[serde(default, rename = "operatorVersion")] pub operator_version: Option, + /// Version of the alien-launcher supervising this operator (os-service + /// only; reported, never driven). Gates binary targets against + /// min_launcher_version. + #[serde(default, rename = "launcherVersion")] + pub launcher_version: Option, + /// Operator host OS — `linux` / `macos` / `windows`. + #[serde(default)] + pub operator_os: Option, + /// Operator host arch — `x86_64` / `aarch64`. + #[serde(default)] + pub operator_arch: Option, + /// Supervisor packaging — `os-service` / `kubernetes`. + #[serde(default)] + pub packaging: Option, + /// Image repository the operator was pulled from (no tag), chart-injected. + /// Surfaced in the dashboard pin-version UI. Optional and Kubernetes-only. + #[serde(default)] + pub operator_image_repository: Option, + /// Outcome of the operator's in-flight self-update (structured + /// `OperatorUpdateReport`), forwarded verbatim to the platform reconcile so + /// the dashboard can show a truthful failed/in-progress state; opaque to the + /// standalone manager. Optional for back-compat. + #[serde(default)] + pub operator_update: Option, } #[derive(Debug, Serialize)] @@ -148,6 +172,11 @@ pub struct AgentSyncResponse { /// to poll for pending commands instead of the agent's local sync URL. #[serde(skip_serializing_if = "Option::is_none")] pub commands_url: Option, + /// Desired agent self-update target. The payload carries either `binary` + /// (OS-service flow) or `helm` (Kubernetes flow); the agent picks the + /// one matching its packaging. + #[serde(skip_serializing_if = "Option::is_none")] + pub operator_target: Option, } #[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq)] @@ -200,6 +229,31 @@ pub struct InitializeResponse { pub token: Option, } +/// `POST /v1/rejoin` — re-acquire a deployment-scoped sync token for an +/// agent whose persistent state was wiped (e.g. emptyDir on pod restart). +/// +/// The agent calls this when `/v1/initialize` returns a name-conflict +/// error: the deployment row already exists, so creating it would 409, +/// but the agent legitimately needs a new sync token to keep operating +/// against it. Auth is the same dg bearer the chart originally mounted. +/// +/// `name` is required — without it the server can't disambiguate which +/// row to reattach to. +#[derive(Debug, Deserialize)] +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[serde(rename_all = "camelCase")] +pub struct RejoinRequest { + pub name: String, +} + +#[derive(Debug, Serialize)] +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[serde(rename_all = "camelCase")] +pub struct RejoinResponse { + pub deployment_id: String, + pub token: String, +} + // --- Router --- pub fn router() -> Router { @@ -218,6 +272,16 @@ pub fn initialize_router() -> Router { Router::new().route("/v1/initialize", post(initialize)) } +/// Router for the `/v1/rejoin` endpoint only. +/// +/// Same separation rationale as `initialize_router` — multi-tenant +/// embedders override this with one that forwards to their upstream API +/// so token issuance lives on the SaaS side, not in the local manager +/// store. +pub fn rejoin_router() -> Router { + Router::new().route("/v1/rejoin", post(rejoin)) +} + // --- Handlers --- /// `POST /v1/sync/acquire` — Inbound: workspace / dg / deployment bearer. @@ -421,6 +485,16 @@ async fn reconcile( observed_inventory_batches: req.observed_inventory_batches, capabilities: req.capabilities, operator_version: req.operator_version, + // Non-agent reconcile path (push/platform-api) doesn't carry the + // operator self-update inventory; leave it out. + launcher_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + operator_update: None, + redeploy_required: None, + min_launcher_version: None, }, ) .await @@ -599,7 +673,6 @@ mod tests { "memory": null, "workload": null, "pods": [], - "instances": [], "events": [] } }, @@ -998,8 +1071,250 @@ mod tests { created_at: now, updated_at: Some(now), error: None, + operator_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + target_operator_version: None, + launcher_version: None, } } + + // --- operator_target resolution (binary path) ------------------------ + + /// A pinned os-service deployment with a matching manifest gets a + /// per-host binary target with the exact manifest values. + #[tokio::test] + async fn binary_target_resolves_for_the_hosts_platform() { + let dir = tempfile::tempdir().unwrap(); + write_manifest( + dir.path(), + "1.4.0", + r#"{ + "version": "1.4.0", + "minLauncherVersion": "0.2.0", + "artifacts": { + "linux/x86_64": { "url": "https://dl.example/op", "sha256": "abc123" } + } + }"#, + ); + let base = dir.path().to_str().unwrap(); + + let target = super::binary_operator_target( + base, + "1.4.0", + Some("linux"), + Some("x86_64"), + Some("0.2.0"), + ) + .await + .expect("target should be emitted"); + + assert_eq!(target.version, "1.4.0"); + assert!(target.helm.is_none(), "binary targets carry no helm payload"); + let binary = target.binary.expect("binary payload"); + assert_eq!(binary.url, "https://dl.example/op"); + assert_eq!(binary.sha256, "abc123"); + assert_eq!(binary.min_launcher_version, "0.2.0"); + } + + /// The frozen-launcher gate: an unreported launcher or one below the + /// manifest floor withholds the target ("redeploy required"); at or + /// above the floor it passes. + #[tokio::test] + async fn binary_target_gates_on_launcher_version() { + let dir = tempfile::tempdir().unwrap(); + write_manifest( + dir.path(), + "2.0.0", + r#"{ + "version": "2.0.0", + "minLauncherVersion": "0.5.0", + "artifacts": { + "linux/x86_64": { "url": "https://dl.example/op2", "sha256": "def" } + } + }"#, + ); + let base = dir.path().to_str().unwrap(); + + for too_old in [None, Some("0.4.9"), Some("garbage")] { + assert!( + super::binary_operator_target(base, "2.0.0", Some("linux"), Some("x86_64"), too_old) + .await + .is_none(), + "launcher {too_old:?} must be withheld" + ); + } + assert!( + super::binary_operator_target( + base, + "2.0.0", + Some("linux"), + Some("x86_64"), + Some("0.5.0") + ) + .await + .is_some(), + "launcher exactly at the floor passes" + ); + } + + /// Missing platform key, unreadable manifest, or unreported os/arch all + /// yield NO target — never a partial one, never a panic. + #[tokio::test] + async fn binary_target_withholds_on_missing_pieces() { + let dir = tempfile::tempdir().unwrap(); + write_manifest( + dir.path(), + "3.0.0", + r#"{ + "version": "3.0.0", + "minLauncherVersion": "0.1.0", + "artifacts": { + "linux/x86_64": { "url": "https://dl.example/op3", "sha256": "aaa" } + } + }"#, + ); + let base = dir.path().to_str().unwrap(); + + // Host platform missing from the manifest. + assert!( + super::binary_operator_target(base, "3.0.0", Some("windows"), Some("x86_64"), Some("1.0.0")) + .await + .is_none() + ); + // os/arch never reported. + assert!( + super::binary_operator_target(base, "3.0.0", None, None, Some("1.0.0")) + .await + .is_none() + ); + // No manifest published for the pinned version at all. + assert!( + super::binary_operator_target(base, "9.9.9", Some("linux"), Some("x86_64"), Some("1.0.0")) + .await + .is_none() + ); + } + + /// The "redeploy required" signal mirrors the target gate: an os-service + /// deployment with a pinned target whose launcher is below the manifest + /// floor reports `(Some(true), floor)`; at/above the floor `(Some(false), + /// floor)`; and it is inert `(None, None)` for kubernetes, no pin, or an + /// already-converged operator. + #[tokio::test] + async fn redeploy_status_tracks_the_launcher_floor() { + let dir = tempfile::tempdir().unwrap(); + write_manifest( + dir.path(), + "2.0.0", + r#"{ + "version": "2.0.0", + "minLauncherVersion": "0.5.0", + "artifacts": { + "linux/x86_64": { "url": "https://dl.example/op2", "sha256": "def" } + } + }"#, + ); + let config = crate::config::ManagerConfig { + releases_url: Some(dir.path().to_str().unwrap().to_string()), + ..Default::default() + }; + let mut deployment = deployment_record_with_state("running", None); + deployment.target_operator_version = Some("2.0.0".to_string()); + + let floor = || Some("0.5.0".to_string()); + // Below the floor → redeploy required, with the floor surfaced. + assert_eq!( + super::resolve_redeploy_status(&config, &deployment, Some("1.0.0"), Some("os-service"), Some("0.4.9")).await, + (Some(true), floor()) + ); + // At/above the floor → a target is pending but no redeploy needed. + assert_eq!( + super::resolve_redeploy_status(&config, &deployment, Some("1.0.0"), Some("os-service"), Some("0.5.0")).await, + (Some(false), floor()) + ); + // Not applicable: kubernetes, already converged, or no pin. + assert_eq!( + super::resolve_redeploy_status(&config, &deployment, Some("1.0.0"), Some("kubernetes"), Some("0.4.9")).await, + (None, None) + ); + assert_eq!( + super::resolve_redeploy_status(&config, &deployment, Some("2.0.0"), Some("os-service"), Some("0.4.9")).await, + (None, None), + "already converged (reported == pin) → nothing withheld" + ); + let mut unpinned = deployment_record_with_state("running", None); + unpinned.target_operator_version = None; + assert_eq!( + super::resolve_redeploy_status(&config, &unpinned, Some("1.0.0"), Some("os-service"), Some("0.4.9")).await, + (None, None) + ); + } + + /// The Kubernetes path is untouched by the binary work: a pinned + /// kubernetes deployment still gets the helm values overlay, and a + /// converged one gets nothing. + #[tokio::test] + async fn resolve_keeps_kubernetes_semantics() { + let config = crate::config::ManagerConfig::default(); + let mut deployment = deployment_record_with_state("running", None); + deployment.target_operator_version = Some("1.4.0".to_string()); + + let target = super::resolve_operator_target( + &config, + &deployment, + Some("1.3.5"), + Some("kubernetes"), + None, + None, + None, + ) + .await + .expect("kubernetes target"); + let helm = target.helm.expect("helm payload"); + assert_eq!( + helm.values, + serde_json::json!({ "runtime": { "image": { "tag": "1.4.0" } } }) + ); + assert!(target.binary.is_none()); + + // Converged (reported == pin) → no target, any packaging. + assert!( + super::resolve_operator_target( + &config, + &deployment, + Some("1.4.0"), + Some("kubernetes"), + None, + None, + None, + ) + .await + .is_none() + ); + // Unknown packaging → no target. + assert!( + super::resolve_operator_target( + &config, + &deployment, + Some("1.3.5"), + None, + None, + None, + None, + ) + .await + .is_none() + ); + } + + fn write_manifest(dir: &std::path::Path, version: &str, body: &str) { + let version_dir = dir.join(version); + std::fs::create_dir_all(&version_dir).unwrap(); + std::fs::write(version_dir.join("manifest.json"), body).unwrap(); + } } /// `POST /v1/sync` — Inbound: deployment bearer. The agent-driven sync @@ -1042,6 +1357,45 @@ async fn agent_sync( return ErrorData::forbidden("Access denied").into_response(); } + // os-service self-update: compute whether the pinned target is withheld + // because the installed launcher is below the target's floor ("redeploy + // required"). Forwarded on reconcile (below) so the dashboard can surface + // it. Shares the cached manifest load with `resolve_operator_target`. + let (redeploy_required, min_launcher_version) = resolve_redeploy_status( + &state.config, + &deployment, + req.operator_version.as_deref(), + req.packaging.as_deref(), + req.launcher_version.as_deref(), + ) + .await; + + // Persist the agent self-update inventory the agent reported on this sync + // (`operator_version`, `operator_os`, `operator_arch`, `packaging`). Runs on every + // sync regardless of whether the agent reported a state change, so the + // manager has a fleet-wide view of which version each host is on. Old + // agents that don't send these fields are no-ops. + if let Err(e) = state + .deployment_store + .update_agent_metadata( + &subject, + &req.deployment_id, + req.operator_version.as_deref(), + req.launcher_version.as_deref(), + req.operator_os.as_deref(), + req.operator_arch.as_deref(), + req.packaging.as_deref(), + req.operator_image_repository.as_deref(), + ) + .await + { + tracing::warn!( + deployment_id = %req.deployment_id, + error = %e, + "Failed to persist agent self-update inventory; continuing sync" + ); + } + // If the agent reported its current state, persist it to the deployment record. // This is how pull-mode agents propagate status changes (e.g. Pending → Running) // back to the manager so that API consumers can observe deployment progress. @@ -1079,7 +1433,18 @@ async fn agent_sync( observed_inventory_batches: req.observed_inventory_batches.clone(), capabilities: req.capabilities.clone(), operator_version: req.operator_version.clone(), + launcher_version: req.launcher_version.clone(), suggested_delay_ms: None, + // Forward the operator self-update inventory the + // operator reported on this sync so multi-tenant + // embedders can persist it in their own row. + operator_os: req.operator_os.clone(), + operator_arch: req.operator_arch.clone(), + packaging: req.packaging.clone(), + operator_image_repository: req.operator_image_repository.clone(), + operator_update: req.operator_update.clone(), + redeploy_required, + min_launcher_version: min_launcher_version.clone(), }, ) .await @@ -1262,6 +1627,14 @@ async fn agent_sync( observed_inventory_batches: req.observed_inventory_batches.clone(), capabilities: req.capabilities.clone(), operator_version: req.operator_version.clone(), + launcher_version: req.launcher_version.clone(), + operator_os: req.operator_os.clone(), + operator_arch: req.operator_arch.clone(), + packaging: req.packaging.clone(), + operator_image_repository: req.operator_image_repository.clone(), + operator_update: req.operator_update.clone(), + redeploy_required, + min_launcher_version: min_launcher_version.clone(), suggested_delay_ms: None, }, ) @@ -1290,6 +1663,21 @@ async fn agent_sync( None }; + // Agent upgrade decision: if the deployment has a pinned target version + // that differs from what the agent just reported, drive an upgrade via + // `operator_target` in the response. + let operator_target = resolve_operator_target( + &state.config, + &deployment, + req.operator_version.as_deref(), + req.packaging.as_deref(), + req.operator_os.as_deref(), + req.operator_arch.as_deref(), + req.launcher_version.as_deref(), + ) + .await + .and_then(|t| serde_json::to_value(t).ok()); + Json(AgentSyncResponse { current_state, target: match target.map(|t| serde_json::to_value(&t)).transpose() { @@ -1301,10 +1689,215 @@ async fn agent_sync( } }, commands_url: Some(state.config.commands_base_url()), + operator_target, }) .into_response() } +/// Resolve the `operator_target` for this sync: when +/// `deployment.target_operator_version` is set AND differs from what the +/// operator just reported, build the payload matching its packaging — `helm` +/// for Kubernetes, a per-host `binary` for os-service. +/// The "redeploy required" signal for an os-service self-update, mirrored from +/// the same floor gate `binary_operator_target` uses: a pinned target is +/// withheld when the installed (frozen) launcher is below the target's manifest +/// `minLauncherVersion`, and only a redeploy replaces the launcher. +/// +/// Returns `(redeploy_required, min_launcher_version)`, both forwarded on +/// reconcile so the dashboard can surface the state: +/// - `(Some(true), Some(floor))` — target withheld, redeploy needed. +/// - `(Some(false), Some(floor))` — target pending, launcher satisfies the floor. +/// - `(None, None)` — not applicable: not os-service, no pin, already converged, +/// or the manifest failed to load / has an unparseable floor. +/// +/// The manifest is cached (`release_manifest::load`), so this shares the load +/// `resolve_operator_target` performs for the same version. +async fn resolve_redeploy_status( + config: &crate::config::ManagerConfig, + deployment: &crate::traits::DeploymentRecord, + reported_version: Option<&str>, + packaging: Option<&str>, + launcher_version: Option<&str>, +) -> (Option, Option) { + if packaging != Some("os-service") { + return (None, None); + } + let Some(target_version) = deployment.target_operator_version.as_deref() else { + return (None, None); + }; + // Already converged — no target is pending, so there is nothing to withhold. + if reported_version == Some(target_version) { + return (None, None); + } + let manifest = match crate::release_manifest::load(&config.releases_url(), target_version).await + { + Ok(manifest) => manifest, + Err(_) => return (None, None), + }; + let floor = manifest.min_launcher_version.clone(); + match crate::release_manifest::check_launcher_floor(launcher_version, &floor) { + crate::release_manifest::LauncherFloorCheck::Ok => (Some(false), Some(floor)), + crate::release_manifest::LauncherFloorCheck::RedeployRequired => (Some(true), Some(floor)), + crate::release_manifest::LauncherFloorCheck::InvalidFloor => (None, None), + } +} + +async fn resolve_operator_target( + config: &crate::config::ManagerConfig, + deployment: &crate::traits::DeploymentRecord, + reported_version: Option<&str>, + packaging: Option<&str>, + operator_os: Option<&str>, + operator_arch: Option<&str>, + launcher_version: Option<&str>, +) -> Option { + let target_version = deployment.target_operator_version.as_deref()?; + if reported_version == Some(target_version) { + return None; + } + match packaging { + Some("kubernetes") => Some(helm_operator_target(target_version)), + Some("os-service") => { + binary_operator_target( + &config.releases_url(), + target_version, + operator_os, + operator_arch, + launcher_version, + ) + .await + } + other => { + tracing::debug!( + deployment_id = %deployment.id, + packaging = ?other, + "Target pinned but the operator reported no known packaging; not emitting operator_target" + ); + None + } + } +} + +/// Kubernetes payload. chart_repo / chart_version are emitted as empty +/// strings — the operator re-uses its current chart_ref (from the +/// `ALIEN_OPERATOR_CHART_REF` / `ALIEN_OPERATOR_CHART_VERSION` env vars the +/// chart injected at install time) and only the values overlay flips the +/// `runtime.image.tag`. This avoids per-version chart re-publication. +fn helm_operator_target(target_version: &str) -> alien_core::sync::OperatorTarget { + operator_target_shell( + target_version, + None, + Some(alien_core::sync::OperatorHelmTarget { + chart_repo: String::new(), + chart_version: String::new(), + values: serde_json::json!({ + "runtime": { "image": { "tag": target_version } } + }), + sensitive_values: Default::default(), + }), + ) +} + +/// os-service payload: resolve the artifact for THIS host's `(os, arch)` from +/// the release manifest and gate on the frozen launcher's version. Any +/// missing piece (no manifest, unknown platform, launcher below the floor or +/// unreported) yields NO target — never a partial one; the admin-facing +/// remedy for a launcher below the floor is a state-preserving redeploy. +async fn binary_operator_target( + releases_base: &str, + target_version: &str, + operator_os: Option<&str>, + operator_arch: Option<&str>, + launcher_version: Option<&str>, +) -> Option { + let (Some(os), Some(arch)) = (operator_os, operator_arch) else { + tracing::warn!( + target_version, + "os-service target pinned but the operator did not report os/arch; not emitting" + ); + return None; + }; + + let manifest = match crate::release_manifest::load(releases_base, target_version).await { + Ok(manifest) => manifest, + Err(e) => { + tracing::warn!( + target_version, + error = %e, + "Failed to load the release manifest; not emitting operator_target" + ); + return None; + } + }; + + // The frozen-launcher gate: the installed launcher must be able to + // actuate this artifact's handoff contract. Shares one definition with the + // platform's "redeploy required" signal (see `check_launcher_floor`). + match crate::release_manifest::check_launcher_floor( + launcher_version, + &manifest.min_launcher_version, + ) { + crate::release_manifest::LauncherFloorCheck::Ok => {} + crate::release_manifest::LauncherFloorCheck::RedeployRequired => { + tracing::info!( + target_version, + reported_launcher = ?launcher_version, + min_launcher_version = %manifest.min_launcher_version, + "Withholding operator_target: installed launcher is below the floor (or unreported) — redeploy required" + ); + return None; + } + crate::release_manifest::LauncherFloorCheck::InvalidFloor => { + tracing::warn!( + target_version, + min_launcher_version = %manifest.min_launcher_version, + "Release manifest carries an unparseable minLauncherVersion; not emitting" + ); + return None; + } + } + + let key = format!("{os}/{arch}"); + let Some(artifact) = manifest.artifacts.get(&key) else { + tracing::warn!( + target_version, + platform = %key, + "Release manifest has no artifact for this host's platform; not emitting" + ); + return None; + }; + + Some(operator_target_shell( + target_version, + Some(alien_core::sync::OperatorBinaryTarget { + url: artifact.url.clone(), + sha256: artifact.sha256.clone(), + signature: artifact.signature.clone(), + min_launcher_version: manifest.min_launcher_version.clone(), + }), + None, + )) +} + +fn operator_target_shell( + target_version: &str, + binary: Option, + helm: Option, +) -> alien_core::sync::OperatorTarget { + alien_core::sync::OperatorTarget { + version: target_version.to_string(), + // Floor on the operator's *current* version for it to accept this jump. + // MVP: any operator we ship (>= 1.0.0) can upgrade straight to the target, + // so keep the lowest floor. Setting it to `target_version` would (once the + // operator enforces it) make every operator that actually needs the upgrade + // refuse it — sub-target operators are exactly the ones we tell to upgrade. + // Raise this only when a target requires a stepping-stone upgrade. + min_supported_version: "1.0.0".to_string(), + binary, + helm, + } +} + fn release_stack_platform(platform: Platform) -> Platform { platform } @@ -1500,6 +2093,82 @@ fn release_info_from_record( }) } +/// `POST /v1/rejoin` — re-acquire a deployment-scoped sync token for an +/// existing deployment by name. The OSS handler is intentionally lean: +/// the dg-token caller specifies the name, the store looks up the row, +/// and a fresh `Deployment` token is minted and returned. Returns +/// `404 Deployment not found` when no row matches — agents should fall +/// back to `/v1/initialize` in that case. +/// +/// Distinct from `/v1/initialize`'s idempotency branch because the agent +/// wants an explicit, distinguishable code path for "I lost local state, +/// please re-attach me" vs "I'm a brand-new pod claiming a name". The +/// platform-mode override forwards this to the SaaS rejoin endpoint +/// where audit / event semantics differ. +#[cfg_attr(feature = "openapi", utoipa::path( + post, + path = "/v1/rejoin", + tag = "sync", + request_body = RejoinRequest, + responses( + (status = 200, description = "Existing deployment rejoined; fresh token returned", body = RejoinResponse), + (status = 404, description = "No deployment with that name in caller's deployment group"), + ), + security( + ("bearer" = []) + ) +))] +async fn rejoin( + State(state): State, + headers: HeaderMap, + Json(req): Json, +) -> Response { + let subject = match auth::require_auth(&state, &headers).await { + Ok(s) => s, + Err(e) => return e.into_response(), + }; + + let dg_id = match subject.scope.clone() { + crate::auth::Scope::DeploymentGroup { + deployment_group_id, + .. + } => deployment_group_id, + _ => { + return ErrorData::forbidden("Rejoin requires a deployment-group token").into_response(); + } + }; + + let existing = match state + .deployment_store + .get_deployment_by_name(&subject, &dg_id, &req.name) + .await + { + Ok(Some(d)) => d, + Ok(None) => return ErrorData::not_found_deployment(&req.name).into_response(), + Err(e) => return e.into_response(), + }; + + let (raw_token, key_prefix, key_hash) = ids::generate_token(TokenType::Deployment.prefix()); + match state + .token_store + .create_token(CreateTokenParams { + token_type: TokenType::Deployment, + key_prefix, + key_hash, + deployment_group_id: Some(dg_id), + deployment_id: Some(existing.id.clone()), + }) + .await + { + Ok(_) => Json(RejoinResponse { + deployment_id: existing.id, + token: raw_token, + }) + .into_response(), + Err(e) => e.into_response(), + } +} + /// `POST /v1/initialize` — Inbound: deployment-group bearer (typical), /// or workspace bearer for self-hosted operator workflows. New deployments /// are created via `DeploymentStore::create_deployment(caller, …)` so diff --git a/crates/alien-manager/src/stores/sqlite/command_registry.rs b/crates/alien-manager/src/stores/sqlite/command_registry.rs index 1d83286ad..8cbf9539a 100644 --- a/crates/alien-manager/src/stores/sqlite/command_registry.rs +++ b/crates/alien-manager/src/stores/sqlite/command_registry.rs @@ -409,6 +409,13 @@ mod tests { created_at: Utc::now(), updated_at: None, error: None, + operator_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + launcher_version: None, + operator_image_repository: None, + target_operator_version: None, } } diff --git a/crates/alien-manager/src/stores/sqlite/deployment.rs b/crates/alien-manager/src/stores/sqlite/deployment.rs index a7aec9a67..eae691075 100644 --- a/crates/alien-manager/src/stores/sqlite/deployment.rs +++ b/crates/alien-manager/src/stores/sqlite/deployment.rs @@ -125,13 +125,62 @@ impl SqliteDeploymentStore { } } + /// UPDATE statement persisting the inventory an operator reported on + /// sync. `None` when nothing was reported (old operator on the wire). + /// Built in a helper (not inline) so the non-Send sea_query statement is + /// dropped before any await — and so tests can assert the column set. + #[allow(clippy::too_many_arguments)] + fn update_agent_metadata_sql( + id: &str, + operator_version: Option<&str>, + launcher_version: Option<&str>, + operator_os: Option<&str>, + operator_arch: Option<&str>, + packaging: Option<&str>, + operator_image_repository: Option<&str>, + ) -> Option { + if operator_version.is_none() + && launcher_version.is_none() + && operator_os.is_none() + && operator_arch.is_none() + && packaging.is_none() + && operator_image_repository.is_none() + { + return None; + } + let mut q = Query::update(); + q.table(Deployments::Table); + if let Some(v) = operator_version { + q.value(Deployments::OperatorVersion, v); + } + if let Some(v) = launcher_version { + q.value(Deployments::LauncherVersion, v); + } + if let Some(v) = operator_os { + q.value(Deployments::OperatorOs, v); + } + if let Some(v) = operator_arch { + q.value(Deployments::OperatorArch, v); + } + if let Some(v) = packaging { + q.value(Deployments::Packaging, v); + } + if let Some(v) = operator_image_repository { + q.value(Deployments::OperatorImageRepository, v); + } + Some( + q.and_where(Expr::col(Deployments::Id).eq(id)) + .to_string(SqliteQueryBuilder), + ) + } + fn stale_lock_condition_sql() -> String { "\"locked_at\" IS NOT NULL AND julianday(\"locked_at\") < julianday('now', '-5 minutes')" .to_string() } /// All columns needed for deployment queries (must match parse_deployment order). - const DEPLOYMENT_COLUMNS: [Deployments; 27] = [ + const DEPLOYMENT_COLUMNS: [Deployments; 34] = [ Deployments::Id, Deployments::Name, Deployments::DeploymentGroupId, @@ -159,6 +208,16 @@ impl SqliteDeploymentStore { Deployments::Error, Deployments::WorkspaceId, Deployments::ProjectId, + // Operator self-update inventory: + Deployments::OperatorVersion, + Deployments::OperatorOs, + Deployments::OperatorArch, + Deployments::Packaging, + Deployments::OperatorImageRepository, + // Manager-driven upgrade target: + Deployments::TargetOperatorVersion, + // Supervising launcher version (os-service inventory): + Deployments::LauncherVersion, ]; fn parse_deployment(row: &turso::Row) -> Result { @@ -229,6 +288,14 @@ impl SqliteDeploymentStore { project_id: p .optional_string(26, "project_id")? .unwrap_or_else(|| "default".to_string()), + // Operator self-update inventory; indices match DEPLOYMENT_COLUMNS order. + operator_version: p.optional_string(27, "operator_version")?, + operator_os: p.optional_string(28, "operator_os")?, + operator_arch: p.optional_string(29, "operator_arch")?, + packaging: p.optional_string(30, "packaging")?, + operator_image_repository: p.optional_string(31, "operator_image_repository")?, + target_operator_version: p.optional_string(32, "target_operator_version")?, + launcher_version: p.optional_string(33, "launcher_version")?, }) } @@ -262,6 +329,43 @@ mod tests { use super::SqliteDeploymentStore; + /// The inventory UPDATE persists launcher_version alongside the operator + /// fields (and skips columns that weren't reported). + #[test] + fn update_agent_metadata_sql_persists_launcher_version() { + let sql = SqliteDeploymentStore::update_agent_metadata_sql( + "dep_1", + Some("1.4.0"), + Some("0.2.0"), + Some("linux"), + Some("x86_64"), + Some("os-service"), + None, + ) + .expect("reported fields produce an UPDATE"); + assert!(sql.contains("\"operator_version\""), "{sql}"); + assert!(sql.contains("\"launcher_version\""), "{sql}"); + assert!(sql.contains("'0.2.0'"), "{sql}"); + assert!(sql.contains("\"packaging\""), "{sql}"); + assert!( + !sql.contains("operator_image_repository"), + "unreported columns stay untouched: {sql}" + ); + + // Only the launcher version reported → still an UPDATE. + let sql = SqliteDeploymentStore::update_agent_metadata_sql( + "dep_1", None, Some("0.2.0"), None, None, None, None, + ) + .expect("launcher-only report persists"); + assert!(sql.contains("\"launcher_version\""), "{sql}"); + + // Nothing reported → no statement at all. + assert!(SqliteDeploymentStore::update_agent_metadata_sql( + "dep_1", None, None, None, None, None, None + ) + .is_none()); + } + #[test] fn stale_lock_condition_parses_rfc3339_timestamps() { let condition = SqliteDeploymentStore::stale_lock_condition_sql(); @@ -356,6 +460,13 @@ mod tests { created_at: now, updated_at: Some(now), error: None, + operator_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + target_operator_version: None, + launcher_version: None, } } } @@ -489,6 +600,14 @@ impl DeploymentStore for SqliteDeploymentStore { created_at: now, updated_at: None, error: None, + // Operator self-update inventory — NULL until the agent's first sync. + operator_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + target_operator_version: None, + launcher_version: None, }) } @@ -652,6 +771,14 @@ impl DeploymentStore for SqliteDeploymentStore { created_at: now, updated_at: None, error: None, + // Operator self-update inventory — NULL until the agent's first sync. + operator_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + target_operator_version: None, + launcher_version: None, }) } @@ -908,6 +1035,33 @@ impl DeploymentStore for SqliteDeploymentStore { self.db.execute(&sql).await } + async fn update_agent_metadata( + &self, + _caller: &crate::auth::Subject, + id: &str, + operator_version: Option<&str>, + launcher_version: Option<&str>, + operator_os: Option<&str>, + operator_arch: Option<&str>, + packaging: Option<&str>, + operator_image_repository: Option<&str>, + ) -> Result<(), AlienError> { + // Nothing to do if the agent didn't report any of these fields + // (e.g. an older agent on the wire). + let Some(sql) = Self::update_agent_metadata_sql( + id, + operator_version, + launcher_version, + operator_os, + operator_arch, + packaging, + operator_image_repository, + ) else { + return Ok(()); + }; + self.db.execute(&sql).await + } + async fn set_redeploy( &self, _caller: &crate::auth::Subject, @@ -921,6 +1075,26 @@ impl DeploymentStore for SqliteDeploymentStore { self.db.execute(&sql).await } + async fn set_target_operator_version( + &self, + _caller: &crate::auth::Subject, + id: &str, + target_operator_version: Option<&str>, + ) -> Result<(), AlienError> { + let sql = { + let mut q = Query::update(); + q.table(Deployments::Table); + match target_operator_version { + Some(v) => q.value(Deployments::TargetOperatorVersion, v), + // sea_query treats `Option::<&str>::None` as SQL NULL. + None => q.value(Deployments::TargetOperatorVersion, Option::<&str>::None), + }; + q.and_where(Expr::col(Deployments::Id).eq(id)) + .to_string(SqliteQueryBuilder) + }; + self.db.execute(&sql).await + } + async fn set_deployment_desired_release( &self, _caller: &crate::auth::Subject, diff --git a/crates/alien-manager/src/stores/sqlite/migrations.rs b/crates/alien-manager/src/stores/sqlite/migrations.rs index 78861ccd9..f5a626e24 100644 --- a/crates/alien-manager/src/stores/sqlite/migrations.rs +++ b/crates/alien-manager/src/stores/sqlite/migrations.rs @@ -42,6 +42,23 @@ pub(crate) enum Deployments { WorkspaceId, /// Project this deployment belongs to. Always `"default"` in this store. ProjectId, + // Operator self-update inventory. Populated by the sync handler from the + // matching SyncRequest fields on every /v1/sync. + /// Operator binary version (e.g. `"1.3.5"`). NULL until first sync. + OperatorVersion, + /// `linux` / `macos` / `windows`. NULL until first sync. + OperatorOs, + /// `x86_64` / `aarch64`. NULL until first sync. + OperatorArch, + /// Supervisor packaging — `os-service` / `kubernetes`. NULL until first sync. + Packaging, + /// Image repository the operator was pulled from (no tag), reported on + /// each sync. Drives the dashboard's pin-version registry display. + OperatorImageRepository, + /// Pinned target operator version. NULL = no pin. When set ≠ OperatorVersion, + /// sync handler emits `operator_target` to drive an upgrade. + TargetOperatorVersion, + LauncherVersion, } #[derive(Iden, Clone, Copy)] @@ -321,6 +338,24 @@ pub async fn run_migrations(db: &SqliteDatabase) -> Result<(), AlienError> { "ALTER TABLE releases ADD COLUMN project_id TEXT NOT NULL DEFAULT 'default'", "ALTER TABLE deployment_groups ADD COLUMN workspace_id TEXT NOT NULL DEFAULT 'default'", "ALTER TABLE deployment_groups ADD COLUMN project_id TEXT NOT NULL DEFAULT 'default'", + // Operator self-update inventory: populated by the sync handler from + // the new SyncRequest fields on every /v1/sync. + "ALTER TABLE deployments ADD COLUMN operator_version TEXT", + "ALTER TABLE deployments ADD COLUMN operator_os TEXT", + "ALTER TABLE deployments ADD COLUMN operator_arch TEXT", + "ALTER TABLE deployments ADD COLUMN packaging TEXT", + // Image repository the operator was pulled from, reported on sync. + // Surfaced in the dashboard pin-version UI so admins see the registry. + "ALTER TABLE deployments ADD COLUMN operator_image_repository TEXT", + // Pinned target operator version. Sync handler reads this on every + // request and emits operator_target when it differs from the agent's + // reported version. Drives the manager-directed upgrade flow. + "ALTER TABLE deployments ADD COLUMN target_operator_version TEXT", + // Version of the alien-launcher supervising the operator (os-service + // only; reported on sync, never driven — the launcher is frozen and + // only changes via redeploy). Gates binary targets against + // min_launcher_version. + "ALTER TABLE deployments ADD COLUMN launcher_version TEXT", ]; for sql in alter_statements { if let Err(e) = conn.execute(sql, ()).await { diff --git a/crates/alien-manager/src/traits/deployment_store.rs b/crates/alien-manager/src/traits/deployment_store.rs index 46a943504..5b53ecaae 100644 --- a/crates/alien-manager/src/traits/deployment_store.rs +++ b/crates/alien-manager/src/traits/deployment_store.rs @@ -74,6 +74,31 @@ pub struct DeploymentRecord { pub created_at: DateTime, pub updated_at: Option>, pub error: Option, + // Agent self-update inventory, written by the sync handler. + // All four are NULL until the agent has actually reported in. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_os: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_arch: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub packaging: Option, + /// Version of the supervising alien-launcher (os-service only; reported, + /// never driven). NULL for Kubernetes / launcher-less operators. Gates + /// binary targets: a target is withheld when this is below the + /// artifact's min_launcher_version ("redeploy required"). + pub launcher_version: Option, + // Image repository the agent was pulled from, reported on sync. + // Surfaced in the dashboard pin-version UI so admins see the registry. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operator_image_repository: Option, + + // Desired agent version. When set AND ≠ `operator_version`, the sync + // handler emits `operator_target` in the response so the agent triggers + // an upgrade. NULL = no pin, no upgrade. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_operator_version: Option, } impl std::fmt::Debug for DeploymentRecord { @@ -235,6 +260,34 @@ pub struct ReconcileData { pub observed_inventory_batches: Vec, pub capabilities: Vec, pub operator_version: Option, + /// Operator self-update inventory reported on this sync. Forwarded by + /// multi-tenant embedders into their platform-side reconcile request so the + /// dashboard can show fleet inventory. Each `None` means "leave the existing + /// column untouched" — back-compat with operators that don't report these. + pub operator_os: Option, + pub operator_arch: Option, + pub packaging: Option, + pub operator_image_repository: Option, + /// Supervising launcher version (os-service only) — forwarded like the + /// rest of the inventory so platform dashboards can surface + /// "redeploy required". + pub launcher_version: Option, + /// Structured `OperatorUpdateReport` the operator attached on this sync, + /// forwarded verbatim into the platform reconcile so the dashboard's + /// "Alien Operator Update" event reflects a real failure/progress signal. + /// `None` = no update in flight. + pub operator_update: Option, + /// os-service self-update: the manager withheld the pinned target because + /// the installed (frozen) launcher is below the target's + /// `minLauncherVersion` — a redeploy with a newer launcher is required. + /// Computed by the manager from the release manifest and forwarded so + /// platform dashboards can surface it. `None` = not applicable (kubernetes, + /// no pin, already converged, or manifest error); `Some(false)` = a target + /// is pending and the launcher is fine. + pub redeploy_required: Option, + /// The pinned target's `minLauncherVersion` (from the manifest), forwarded + /// alongside `redeploy_required` for a "needs ≥ X, has Y" affordance. + pub min_launcher_version: Option, } /// Persistence for deployments and deployment groups. @@ -338,6 +391,38 @@ pub trait DeploymentStore: Send + Sync { id: &str, ) -> Result<(), AlienError>; + /// Persist the agent self-update inventory reported on a `SyncRequest` + /// (`operator_version`, `operator_os`, `operator_arch`, `packaging`, image repo). + /// Called on every agent sync — alongside the heartbeat update — so the + /// manager has a fleet-wide view of which version + registry each host + /// is on and can decide whether to send an `operator_target` in the + /// response. A field of `None` leaves the corresponding column + /// untouched. + async fn update_agent_metadata( + &self, + caller: &crate::auth::Subject, + id: &str, + operator_version: Option<&str>, + launcher_version: Option<&str>, + operator_os: Option<&str>, + operator_arch: Option<&str>, + packaging: Option<&str>, + operator_image_repository: Option<&str>, + ) -> Result<(), AlienError>; + + /// Set (or clear with `None`) the deployment's target agent version. + /// When set AND different from the agent's reported `operator_version` the + /// sync handler emits `operator_target` on every /v1/sync until the agent + /// catches up. This is the admin-side knob behind the dashboard's + /// "Set target version" control and the standalone manager's + /// `PUT /v1/deployments/:id/target-operator-version` endpoint. + async fn set_target_operator_version( + &self, + caller: &crate::auth::Subject, + id: &str, + target_operator_version: Option<&str>, + ) -> Result<(), AlienError>; + async fn set_redeploy(&self, caller: &crate::auth::Subject, id: &str) -> Result<(), AlienError>; diff --git a/crates/alien-manager/src/transports/manager.rs b/crates/alien-manager/src/transports/manager.rs index cd58c76a0..72ecb4b50 100644 --- a/crates/alien-manager/src/transports/manager.rs +++ b/crates/alien-manager/src/transports/manager.rs @@ -85,6 +85,16 @@ impl DeploymentLoopTransport for ManagerTransport { observed_inventory_batches, capabilities: Vec::new(), operator_version: None, + launcher_version: None, + // Background reconciliation from inside the manager — no + // operator self-update inventory in this code path. + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + operator_update: None, + redeploy_required: None, + min_launcher_version: None, }, ) .await?; diff --git a/crates/alien-manager/tests/registry_proxy_cloud_test.rs b/crates/alien-manager/tests/registry_proxy_cloud_test.rs index cb7451b3a..f25307da9 100644 --- a/crates/alien-manager/tests/registry_proxy_cloud_test.rs +++ b/crates/alien-manager/tests/registry_proxy_cloud_test.rs @@ -466,6 +466,14 @@ impl CloudProxyTest { suggested_delay_ms: None, capabilities: vec![], operator_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + launcher_version: None, + operator_update: None, + redeploy_required: None, + min_launcher_version: None, }, ) .await diff --git a/crates/alien-manager/tests/registry_proxy_test.rs b/crates/alien-manager/tests/registry_proxy_test.rs index 7d0a4f859..37efaf65c 100644 --- a/crates/alien-manager/tests/registry_proxy_test.rs +++ b/crates/alien-manager/tests/registry_proxy_test.rs @@ -346,6 +346,14 @@ async fn setup() -> TestSetup { suggested_delay_ms: None, capabilities: vec![], operator_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + launcher_version: None, + operator_update: None, + redeploy_required: None, + min_launcher_version: None, }, ) .await @@ -821,6 +829,14 @@ async fn test_proxy_push_then_pull() { suggested_delay_ms: None, capabilities: vec![], operator_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + launcher_version: None, + operator_update: None, + redeploy_required: None, + min_launcher_version: None, }, ) .await diff --git a/crates/alien-manager/tests/sqlite_store_tests.rs b/crates/alien-manager/tests/sqlite_store_tests.rs index 066d271e4..61c097e45 100644 --- a/crates/alien-manager/tests/sqlite_store_tests.rs +++ b/crates/alien-manager/tests/sqlite_store_tests.rs @@ -765,6 +765,14 @@ async fn reconcile_succeeds_under_other_session_lock() { suggested_delay_ms: None, capabilities: vec![], operator_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + launcher_version: None, + operator_update: None, + redeploy_required: None, + min_launcher_version: None, }, ) .await @@ -832,6 +840,14 @@ async fn reconcile_refreshes_owned_lock_lease() { suggested_delay_ms: None, capabilities: vec![], operator_version: None, + operator_os: None, + operator_arch: None, + packaging: None, + operator_image_repository: None, + launcher_version: None, + operator_update: None, + redeploy_required: None, + min_launcher_version: None, }, ) .await @@ -1298,3 +1314,159 @@ async fn release_not_found() { .unwrap(); assert!(result.is_none()); } + +// ---------- Agent self-update inventory ---------------------------------- + +/// A freshly-created deployment has no agent inventory until the first sync. +#[tokio::test] +async fn agent_metadata_is_null_until_first_sync_report() { + let db = fresh_db().await; + let store = SqliteDeploymentStore::new(db.clone()); + let group_id = create_test_group(&store).await; + let dep = + create_test_deployment(&store, &group_id, "fresh", Platform::Kubernetes).await; + + let fetched = store + .get_deployment(&test_subject(), &dep.id) + .await + .unwrap() + .expect("deployment exists"); + assert!(fetched.operator_version.is_none(), "operator_version must be NULL pre-sync"); + assert!(fetched.operator_os.is_none()); + assert!(fetched.operator_arch.is_none()); + assert!(fetched.packaging.is_none()); +} + +/// The agent reports its full inventory on a sync; the manager writes +/// all four columns. A subsequent read sees the persisted values. +#[tokio::test] +async fn update_agent_metadata_persists_full_inventory() { + let db = fresh_db().await; + let store = SqliteDeploymentStore::new(db.clone()); + let group_id = create_test_group(&store).await; + let dep = + create_test_deployment(&store, &group_id, "k8s", Platform::Kubernetes).await; + + store + .update_agent_metadata( + &test_subject(), + &dep.id, + Some("1.4.0"), + Some("0.2.0"), + Some("linux"), + Some("aarch64"), + Some("kubernetes"), + Some("ghcr.io/alien-dev/alien-agent"), + ) + .await + .unwrap(); + + let fetched = store + .get_deployment(&test_subject(), &dep.id) + .await + .unwrap() + .unwrap(); + assert_eq!(fetched.operator_version.as_deref(), Some("1.4.0")); + assert_eq!(fetched.launcher_version.as_deref(), Some("0.2.0")); + assert_eq!(fetched.operator_os.as_deref(), Some("linux")); + assert_eq!(fetched.operator_arch.as_deref(), Some("aarch64")); + assert_eq!(fetched.packaging.as_deref(), Some("kubernetes")); + assert_eq!( + fetched.operator_image_repository.as_deref(), + Some("ghcr.io/alien-dev/alien-agent") + ); +} + +/// An old agent that doesn't send any of the new fields is a no-op: +/// previously-persisted values must be preserved (the call must not +/// blank existing inventory). +#[tokio::test] +async fn update_agent_metadata_with_all_none_is_a_noop() { + let db = fresh_db().await; + let store = SqliteDeploymentStore::new(db.clone()); + let group_id = create_test_group(&store).await; + let dep = + create_test_deployment(&store, &group_id, "k8s", Platform::Kubernetes).await; + + // First, populate. + store + .update_agent_metadata( + &test_subject(), + &dep.id, + Some("1.4.0"), + Some("0.2.0"), + Some("linux"), + Some("aarch64"), + Some("kubernetes"), + Some("ghcr.io/alien-dev/alien-agent"), + ) + .await + .unwrap(); + + // Then a "back-compat" old-agent sync: every field is None. + store + .update_agent_metadata(&test_subject(), &dep.id, None, None, None, None, None, None) + .await + .unwrap(); + + let fetched = store + .get_deployment(&test_subject(), &dep.id) + .await + .unwrap() + .unwrap(); + assert_eq!(fetched.operator_version.as_deref(), Some("1.4.0")); + assert_eq!(fetched.packaging.as_deref(), Some("kubernetes")); +} + +/// A partial update only touches the specified columns — useful for +/// agents that learn about new fields over time. +#[tokio::test] +async fn update_agent_metadata_partial_update_preserves_others() { + let db = fresh_db().await; + let store = SqliteDeploymentStore::new(db.clone()); + let group_id = create_test_group(&store).await; + let dep = + create_test_deployment(&store, &group_id, "k8s", Platform::Kubernetes).await; + + store + .update_agent_metadata( + &test_subject(), + &dep.id, + Some("1.3.5"), + Some("0.2.0"), + Some("linux"), + Some("aarch64"), + Some("kubernetes"), + Some("ghcr.io/alien-dev/alien-agent"), + ) + .await + .unwrap(); + + // Agent upgraded to 1.4.0; OS/arch/packaging/repo didn't change, so the + // handler only forwards operator_version this time (hypothetically — the + // real agent always sends all fields, but the contract supports + // partial updates). + store + .update_agent_metadata( + &test_subject(), + &dep.id, + Some("1.4.0"), + None, + None, + None, + None, + None, + ) + .await + .unwrap(); + + let fetched = store + .get_deployment(&test_subject(), &dep.id) + .await + .unwrap() + .unwrap(); + assert_eq!(fetched.operator_version.as_deref(), Some("1.4.0"), "version updated"); + assert_eq!(fetched.operator_os.as_deref(), Some("linux"), "os preserved"); + assert_eq!(fetched.operator_arch.as_deref(), Some("aarch64"), "arch preserved"); + assert_eq!(fetched.packaging.as_deref(), Some("kubernetes"), "packaging preserved"); +} diff --git a/crates/alien-operator/Cargo.toml b/crates/alien-operator/Cargo.toml index 411235777..caf8e55fc 100644 --- a/crates/alien-operator/Cargo.toml +++ b/crates/alien-operator/Cargo.toml @@ -24,6 +24,9 @@ alien-local = { workspace = true } alien-client-config = { workspace = true } alien-commands = { workspace = true, default-features = false, features = ["dispatchers"] } alien-manager-api = { path = "../../client-sdks/manager/rust" } +# Kubernetes API access for the self-update Helm-runner Jobs (in-pod SA config). +alien-k8s-clients = { workspace = true } +k8s-openapi = { version = "0.25.0", features = ["v1_33"] } # Async & HTTP tokio = { workspace = true, features = ["sync", "macros", "io-util", "rt-multi-thread", "time", "net", "fs", "signal"] } @@ -38,6 +41,7 @@ aegis = { workspace = true } # Serialization serde = { workspace = true, features = ["derive"] } +sha2 = { workspace = true } serde_json = { workspace = true } # CLI & Logging @@ -53,6 +57,11 @@ async-trait = { workspace = true } # Utilities chrono = { workspace = true, features = ["serde"] } +# Free-disk-space preflight for self-update downloads (cross-platform). +fs2 = "0.4" +# Self-update artifact signature verification; only compiled with the +# `enforce-signature` feature (off by default until the signing workstream). +ed25519-dalek = { workspace = true, optional = true } uuid = { workspace = true, features = ["v4"] } url = { workspace = true } hostname = { workspace = true } @@ -81,7 +90,16 @@ libc = "0.2" [target.'cfg(windows)'.dependencies] windows-service = "0.7" -fs2 = "0.4" + +[features] +# ed25519 verification of downloaded self-update artifacts. Default OFF until +# the signing workstream ships a pinned key; CI builds with it on so the +# enforcing path never bitrots. Off-path logs a loud "DISABLED" warning. +enforce-signature = ["dep:ed25519-dalek"] +# E2E test seam: allows ALIEN_OPERATOR_FAKE_VERSION to override the reported +# version so one compiled binary can impersonate "old" and "new" versions. +# Guarded by a compile_error! against release builds. +test-hooks = [] [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/alien-operator/src/cli.rs b/crates/alien-operator/src/cli.rs index f95c5da63..7a59823ae 100644 --- a/crates/alien-operator/src/cli.rs +++ b/crates/alien-operator/src/cli.rs @@ -11,7 +11,7 @@ use alien_core::{ validate_public_endpoint_urls, DeploymentState, DeploymentStatus, Platform, PublicEndpointUrls, DEPLOYMENT_PROTOCOL_VERSION, }; -use alien_error::{AlienError, Context, IntoAlienError}; +use alien_error::{AlienError, Context, ContextError, IntoAlienError}; use clap::{Parser, ValueEnum}; use std::net::IpAddr; use std::path::PathBuf; @@ -22,6 +22,7 @@ use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, Env #[derive(Parser, Debug)] #[command( name = "operator", + version, about = "Operator - Continuous deployment service (pull model)", long_about = "Run the Operator for continuous deployment using the pull model. @@ -199,6 +200,15 @@ pub fn cli_main_with_hooks(init_hook: InitHook, debug_loop_hook: DebugLoopHook) eprintln!("Error: {}", e); std::process::exit(1); } + + // A staged self-update requests the update-handoff exit code (10): the + // supervising launcher observes it via the exit status and performs the + // health-gated swap. Runs after the runtime has fully shut down so the + // InstanceLock and the state DB are released before the swap. + if let Some(code) = crate::self_update::requested_exit_code() { + drop(rt); + std::process::exit(code); + } } /// Convenience wrapper: [`cli_main_with_hook`] with a no-op init hook. @@ -245,6 +255,21 @@ async fn run(mut args: Args, init_hook: InitHook, debug_loop_hook: DebugLoopHook let cli_sync_token = load_sync_token(args.sync_token_file.as_deref()).await?; let collector_token = load_collector_token(args.collector_token_file.as_deref()).await?; + // Stack settings are loaded up front so they can be forwarded on the + // `initialize` call (multi-tenant managers like managerx require them + // to construct the deployment row; single-tenant OSS ignores them). + let stack_settings_json = load_config_value( + args.stack_settings.clone(), + args.stack_settings_file.as_deref(), + "stack settings", + false, + ) + .await?; + let initial_stack_settings = parse_json_opt::( + stack_settings_json.clone(), + "stack settings", + )?; + let effective_sync_url = args .sync_url .or_else(|| embedded_config.as_ref().and_then(|c| c.manager_url.clone())); @@ -277,6 +302,10 @@ async fn run(mut args: Args, init_hook: InitHook, debug_loop_hook: DebugLoopHook )? { StartupDeploymentId::Stored(stored_deployment_id) => { info!(" Using stored deployment ID: {}", stored_deployment_id); + // Prefer the deployment-scoped token returned by + // `/v1/initialize` over the chart-mounted deployment-group + // token (rejected by `/v1/sync/acquire`), so pod restarts + // don't silently lose sync. if let Some(stored_sync_token) = db.get_sync_token().await? { info!(" Using stored deployment-scoped sync token"); sync_token = stored_sync_token; @@ -289,7 +318,7 @@ async fn run(mut args: Args, init_hook: InitHook, debug_loop_hook: DebugLoopHook StartupDeploymentId::Initialize => { info!(" First startup, initializing with manager..."); - let (initialized_deployment_id, deployment_token) = initialize_with_manager( + let init_result = initialize_with_manager( &sync_url, &sync_token, args.platform, @@ -299,7 +328,40 @@ async fn run(mut args: Args, init_hook: InitHook, debug_loop_hook: DebugLoopHook operator_setup_method.as_deref(), args.initial_desired_release, ) - .await?; + .await; + + let (initialized_deployment_id, deployment_token) = match init_result { + Ok(v) => v, + // 409 from initialize means a deployment with our name + // already exists in this dg — the canonical cause is + // that we *had* state (deployment_id + dep-scoped token) + // in `data_dir` but it was wiped (emptyDir on chart + // default, manual reset, etc.). Re-acquiring a fresh + // deployment-scoped token over the existing row is + // exactly what `/v1/rejoin` exists for; fall through to + // it instead of crashing the agent. + Err(e) if e.http_status_code == Some(409) => { + info!( + " Name already exists — assuming local state was wiped, rejoining…" + ); + let agent_name = args.operator_name.clone().or_else(|| { + std::env::var("HOSTNAME").ok().or_else(|| { + hostname::get().ok().and_then(|h| h.into_string().ok()) + }) + }); + let name = agent_name.ok_or_else(|| { + AlienError::new(ErrorData::ConfigurationError { + message: "Cannot rejoin without a deployment name — pass --agent-name or set AGENT_NAME / HOSTNAME" + .to_string(), + }) + })?; + let (dep_id, dep_token) = + rejoin_with_manager(&sync_url, &sync_token, &name).await?; + info!(deployment_id = %dep_id, " Rejoined existing deployment"); + (dep_id, Some(dep_token)) + } + Err(e) => return Err(e), + }; db.set_deployment_id(&initialized_deployment_id).await?; if args.initial_desired_release == InitialDesiredReleaseArg::None { @@ -311,6 +373,9 @@ async fn run(mut args: Args, init_hook: InitHook, debug_loop_hook: DebugLoopHook info!(" Received deployment-scoped token from manager"); db.set_sync_token(dt).await?; sync_token = dt.clone(); + // Persist so pod restarts don't fall back to the + // chart-mounted deployment-group token. + db.set_sync_token(&sync_token).await?; } info!( @@ -366,13 +431,9 @@ async fn run(mut args: Args, init_hook: InitHook, debug_loop_hook: DebugLoopHook message: "Invalid public endpoints configuration".to_string(), })?; } - let stack_settings_json = load_config_value( - args.stack_settings, - args.stack_settings_file.as_deref(), - "stack settings", - false, - ) - .await?; + // `stack_settings_json` was loaded up front so it could be forwarded to + // `initialize`; reuse that same value here so the downstream config sees + // the settings that were sent to the manager. let mut stack_settings = parse_json_opt::(stack_settings_json, "stack settings")? .unwrap_or_default(); @@ -492,9 +553,22 @@ async fn wait_for_shutdown_signal() { #[cfg(windows)] async fn wait_for_shutdown_signal() { - tokio::signal::ctrl_c() - .await - .expect("failed to install Ctrl+C handler"); + use tokio::signal::windows; + + // Ctrl-C covers the legacy console / no-launcher run. Ctrl-Break is the + // launcher's graceful-stop signal to its operator child: the launcher spawns + // the operator in a new process group and sends `CTRL_BREAK_EVENT` (see the + // launcher's Windows child supervisor). Either drives the same graceful + // shutdown — releasing the InstanceLock, closing the DB, and stopping the app + // child. The `--service` (no-launcher SCM) path is handled separately by + // `windows_entry::run_as_service` and does not go through here. + let mut ctrl_c = windows::ctrl_c().expect("failed to install Ctrl+C handler"); + let mut ctrl_break = windows::ctrl_break().expect("failed to install Ctrl+Break handler"); + + tokio::select! { + _ = ctrl_c.recv() => {}, + _ = ctrl_break.recv() => {}, + } } fn install_panic_hook(data_dir: &PathBuf) { @@ -699,12 +773,28 @@ async fn initialize_with_manager( builder = builder.body_map(|b| b.setup_method(setup_method.to_string())); } + // NOTE (follow-up): the operator refactor's onboarding (operator_scope / + // permission / setup_method) replaced the previous stack-settings-on- + // initialize forwarding here. Multi-tenant `initialize` may still need the + // full stack settings — revisit when reconciling onboarding. + let response = builder .send() .await .map_err(alien_manager_api::convert_sdk_error) - .context(ErrorData::ConfigurationError { - message: "Failed to call initialize endpoint".to_string(), + .map_err(|e| { + // Preserve a 409 (deployment name already exists) verbatim + // so the caller's `e.http_status_code == Some(409)` arm can + // trigger the rejoin fall-through. Wrapping with + // `ConfigurationError` here would override the status to 500 + // and the agent would crash instead of recovering. + if e.http_status_code == Some(409) { + AlienError::new(ErrorData::DeploymentNameAlreadyExists) + } else { + e.context(ErrorData::ConfigurationError { + message: "Failed to call initialize endpoint".to_string(), + }) + } })?; let init_response = response.into_inner(); @@ -712,6 +802,54 @@ async fn initialize_with_manager( Ok((init_response.deployment_id, init_response.token)) } +/// Re-acquire a deployment-scoped sync token from the manager after the +/// agent's persistent state was wiped (chart-default emptyDir, manual +/// reset, etc.). Called only on a 409 from `initialize_with_manager` — +/// the existing deployment row is reattached to and a fresh sync token +/// is minted. +async fn rejoin_with_manager( + sync_url: &url::Url, + token: &str, + deployment_name: &str, +) -> Result<(String, String)> { + use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, USER_AGENT}; + + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {}", token)) + .into_alien_error() + .context(ErrorData::ConfigurationError { + message: "Invalid token format".to_string(), + })?, + ); + headers.insert(USER_AGENT, HeaderValue::from_static("alien-agent")); + + let http_client = reqwest::Client::builder() + .default_headers(headers) + .build() + .into_alien_error() + .context(ErrorData::ConfigurationError { + message: "Failed to create HTTP client".to_string(), + })?; + + let base_url = sync_url.as_str().trim_end_matches('/'); + let client = alien_manager_api::Client::new_with_client(base_url, http_client); + + let response = client + .rejoin() + .body_map(|b| b.name(deployment_name.to_string())) + .send() + .await + .map_err(alien_manager_api::convert_sdk_error) + .context(ErrorData::ConfigurationError { + message: "Failed to call rejoin endpoint".to_string(), + })?; + + let r = response.into_inner(); + Ok((r.deployment_id, r.token)) +} + fn setup_tracing(verbose: bool) { let filter = if verbose { EnvFilter::try_from_default_env() diff --git a/crates/alien-operator/src/db.rs b/crates/alien-operator/src/db.rs index 8ec905c17..b7756a14e 100644 --- a/crates/alien-operator/src/db.rs +++ b/crates/alien-operator/src/db.rs @@ -961,7 +961,9 @@ impl OperatorDb { Ok(()) } - /// Get the deployment-scoped sync token returned by initialization. + /// Get the deployment-scoped sync token returned by initialization. The + /// operator persists this so a pod restart doesn't fall back to the + /// chart-mounted deployment-group token (rejected by `/v1/sync/acquire`). pub async fn get_sync_token(&self) -> Result> { let conn = self.conn.lock().await; @@ -993,7 +995,8 @@ impl OperatorDb { } } - /// Persist the deployment-scoped sync token returned by initialization. + /// Persist the deployment-scoped sync token. Idempotent — overwrites any + /// prior value (e.g. when the manager rotates the token). pub async fn set_sync_token(&self, token: &str) -> Result<()> { let conn = self.conn.lock().await; diff --git a/crates/alien-operator/src/error.rs b/crates/alien-operator/src/error.rs index 0eb8f20e0..3134131d3 100644 --- a/crates/alien-operator/src/error.rs +++ b/crates/alien-operator/src/error.rs @@ -30,6 +30,20 @@ pub enum ErrorData { )] ConfigurationError { message: String }, + /// The manager rejected `/v1/initialize` because a deployment with the + /// requested `(deployment_group_id, name)` already exists. Distinct + /// from `ConfigurationError` so the caller can route this into the + /// `/v1/rejoin` fall-through (state-wipe recovery) instead of crashing + /// the agent. + #[error( + code = "DEPLOYMENT_NAME_ALREADY_EXISTS", + message = "Deployment name already exists in this deployment group", + retryable = "false", + internal = "false", + http_status_code = 409 + )] + DeploymentNameAlreadyExists, + #[error( code = "DATABASE_ERROR", message = "Database error: {message}", @@ -48,6 +62,17 @@ pub enum ErrorData { )] SyncFailed { message: String }, + /// The os-service self-update actuator failed (download, staging, or + /// marker I/O). Retryable — the manager keeps advertising the target and + /// the actuator backs off between attempts. + #[error( + code = "SELF_UPDATE_FAILED", + message = "Self-update failed: {message}", + retryable = "true", + internal = "true" + )] + SelfUpdateFailed { message: String }, + #[error( code = "DEPLOYMENT_FAILED", message = "{message}", diff --git a/crates/alien-operator/src/lib.rs b/crates/alien-operator/src/lib.rs index e0550fe21..be7c4cadf 100644 --- a/crates/alien-operator/src/lib.rs +++ b/crates/alien-operator/src/lib.rs @@ -28,6 +28,26 @@ pub mod error; pub mod lock; pub mod loops; pub mod otlp_server; +pub mod self_update; + +// The test seam must never ship: a release build with `test-hooks` on would +// let an env var falsify the fleet's version inventory. +#[cfg(all(feature = "test-hooks", not(debug_assertions)))] +compile_error!("the `test-hooks` feature must not be enabled in release builds"); + +/// The version this operator reports on sync and compares against update +/// targets. Normally `CARGO_PKG_VERSION`; under the `test-hooks` feature +/// (debug builds only) `ALIEN_OPERATOR_FAKE_VERSION` overrides it so E2E +/// suites can drive an update without compiling two binaries. +pub fn operator_version() -> String { + #[cfg(feature = "test-hooks")] + if let Ok(fake) = std::env::var("ALIEN_OPERATOR_FAKE_VERSION") { + if !fake.is_empty() { + return fake; + } + } + env!("CARGO_PKG_VERSION").to_string() +} pub use alien_core::{DeploymentState, DeploymentStatus, Platform, ReleaseInfo}; pub use config::{OperatorConfig, SyncConfig}; @@ -94,8 +114,19 @@ pub async fn run_operator_with_cancel_and_debug_loop( "Starting operator" ); + let readiness = otlp_server::ReadinessSignals::new(); + // Initialize encrypted database let db = Arc::new(db::OperatorDb::new(&config.data_dir, &config.encryption_key).await?); + readiness + .db_open + .store(true, std::sync::atomic::Ordering::Release); + // The CLI acquires the InstanceLock before calling run_operator and the + // guard lives for the process lifetime, so reaching this point means the + // lock is held. + readiness + .lock_held + .store(true, std::sync::atomic::Ordering::Release); // Create shared state let state = Arc::new(OperatorState { @@ -103,21 +134,41 @@ pub async fn run_operator_with_cancel_and_debug_loop( db: db.clone(), service_provider, cancel: cancel.clone(), + readiness: readiness.clone(), }); + // Die-with-parent: under the launcher, exit if our supervisor dies. macOS + // has no PR_SET_PDEATHSIG, and this also backstops the Linux fork→exec race; + // a no-op outside the launcher (Kubernetes, tests). Runs on a dedicated OS + // thread (not the async runtime, which can starve its timer under load); it + // self-exits when `cancel` is tripped, so the handle needs no explicit join. + let _parent_death_watch = self_update::spawn_parent_death_watch(cancel.clone()); + // Start OTLP server (for local functions to send telemetry). - // This is best-effort — a port conflict should not take down the operator. - let otlp_host = config.otlp_server_host; - let otlp_port = config.otlp_server_port; + // Also serves /livez and /readyz on the same port for Kubernetes probes. + // Best-effort — a port conflict should not take down the operator. + // Under the launcher, ALIEN_HEALTH_ADDR overrides the bind so the + // probation gate probes the exact address it handed us; an unparseable + // value is a startup error (a silent fallback would fail every probe by + // port mismatch). + let (otlp_host, otlp_port) = match otlp_server::health_addr_override()? { + Some(addr) => { + info!(address = %addr, "Health/OTLP bind overridden by the launcher (ALIEN_HEALTH_ADDR)"); + (addr.ip(), addr.port()) + } + None => (config.otlp_server_host, config.otlp_server_port), + }; let otlp_db = db.clone(); let otlp_namespace = config.namespace.clone(); let otlp_collector_token = config.collector_token.clone(); let otlp_cancel = cancel.clone(); + let probe_readiness = readiness.clone(); tokio::spawn(async move { if let Err(e) = otlp_server::start_otlp_server( otlp_host, otlp_port, otlp_db, + probe_readiness, otlp_namespace, otlp_collector_token, otlp_cancel, @@ -249,6 +300,20 @@ pub async fn run_operator_with_cancel_and_debug_loop( // Signal all loops to stop (idempotent if already cancelled) cancel.cancel(); + // Tear down the managed application before we exit. The operator owns the + // worker runtime that spawned the user's app; without this, a self-update + // handoff exit (like any operator exit) reparents the app to init and leaves + // it running — a second copy then starts under the swapped-in operator. + // `shutdown_all` signals the worker runtime, which kills the app child. The + // spawn-side `kill_on_drop`/`PR_SET_PDEATHSIG` are crash backstops; this is + // the clean-shutdown path. No-op when there is no local worker manager + // (e.g. Kubernetes, where the pod lifecycle owns the process tree). + if let Some(sp) = &state.service_provider { + if let Some(wm) = sp.get_local_worker_manager() { + wm.shutdown_all().await; + } + } + // Give loops a moment to finish current work tokio::time::sleep(std::time::Duration::from_secs(2)).await; @@ -265,4 +330,10 @@ pub struct OperatorState { pub service_provider: Option>, /// Cancellation token for graceful shutdown. pub cancel: CancellationToken, + /// Readiness signals consumed by the `/readyz` probe handler — the + /// explicit health conditions (DB open, InstanceLock held, first sync + /// completed, deployment loop progressing). /readyz returns 503 until + /// ALL hold, so a freshly-rolled operator isn't marked ready before it + /// has proven it can run and reach the manager. + pub readiness: otlp_server::ReadinessSignals, } diff --git a/crates/alien-operator/src/lock.rs b/crates/alien-operator/src/lock.rs index 7b7da1801..35cb0c3f2 100644 --- a/crates/alien-operator/src/lock.rs +++ b/crates/alien-operator/src/lock.rs @@ -3,6 +3,14 @@ //! Prevents multiple alien-operator instances from running concurrently on the //! same machine (or against the same data directory). Uses `flock` on Unix //! and `fs2::FileExt` on Windows. +//! +//! Under the os-service launcher this lock is also the **hard backstop for +//! die-with-parent**: if an operator is orphaned when its launcher dies, a +//! respawned launcher's fresh operator cannot acquire this lock until the +//! orphan exits and releases it. That guarantees at-most-one operator even if +//! every soft mechanism fails; the operator's parent-death watch +//! (`crate::self_update::spawn_parent_death_watch`) exists to keep that +//! blocking window short (one poll interval) rather than unbounded. use std::fs::{self, File}; use std::path::{Path, PathBuf}; diff --git a/crates/alien-operator/src/loops/deployment.rs b/crates/alien-operator/src/loops/deployment.rs index 58f3ebd0b..14e914503 100644 --- a/crates/alien-operator/src/loops/deployment.rs +++ b/crates/alien-operator/src/loops/deployment.rs @@ -104,6 +104,40 @@ impl DeploymentLoopTransport for OperatorTransport { /// 3. Runs alien-deployment::runner::run_step_loop() with OperatorTransport /// 4. OperatorTransport persists state and re-reads config after each step /// 5. Sync loop will pick up changes and report to manager +/// Health condition 5 ("the deployment loop is progressing") as a +/// consecutive-error tracker: a single failed tick is normal (transient cloud +/// errors, retries); the readiness flag drops only after +/// [`LoopHealth::UNREADY_THRESHOLD`] consecutive failures — a crash-looping +/// tick — and recovers on the next success. +pub struct LoopHealth { + flag: std::sync::Arc, + consecutive_errors: u32, +} + +impl LoopHealth { + /// Consecutive failed ticks after which the loop is considered unhealthy. + pub const UNREADY_THRESHOLD: u32 = 3; + + pub fn new(flag: std::sync::Arc) -> Self { + Self { + flag, + consecutive_errors: 0, + } + } + + pub fn record_ok(&mut self) { + self.consecutive_errors = 0; + self.flag.store(true, std::sync::atomic::Ordering::Release); + } + + pub fn record_err(&mut self) { + self.consecutive_errors = self.consecutive_errors.saturating_add(1); + if self.consecutive_errors >= Self::UNREADY_THRESHOLD { + self.flag.store(false, std::sync::atomic::Ordering::Release); + } + } +} + pub async fn run_deployment_loop(state: Arc) { let interval = Duration::from_secs(state.config.deployment_interval_seconds); @@ -112,15 +146,19 @@ pub async fn run_deployment_loop(state: Arc) { "Starting deployment loop" ); + let mut health = LoopHealth::new(state.readiness.deployment_loop_ok.clone()); + loop { match run_deployment_continuously(&state).await { Ok(steps) => { + health.record_ok(); if steps > 0 { info!(steps = steps, "Deployment completed"); } } Err(e) => { error!(error = %e, "Deployment failed"); + health.record_err(); } } @@ -283,7 +321,9 @@ async fn run_deployment_continuously(state: &OperatorState) -> Result { /// /// Applies public_endpoints and stack_settings from operator config, /// and injects commands polling env vars for K8s/Local platforms. -/// External bindings are part of stack_settings and flow through naturally. +/// External bindings live on a separate DeploymentConfig field that the +/// StackExecutor reads, so they do NOT flow through `stack_settings` alone — +/// the copy must be explicit (see also alien-deploy-cli's `up` command). async fn enrich_config( mut config: DeploymentConfig, operator_config: &OperatorConfig, @@ -298,6 +338,12 @@ async fn enrich_config( // Pass through stack settings from operator config (includes external_bindings) if let Some(ref stack_settings) = operator_config.stack_settings { config.stack_settings = stack_settings.clone(); + // external_bindings does NOT flow through stack_settings automatically: + // the StackExecutor reads DeploymentConfig::external_bindings, a separate + // field. Copy it explicitly, mirroring alien-deploy-cli's `up` command. + if let Some(ref ext_bindings) = stack_settings.external_bindings { + config.external_bindings = ext_bindings.clone(); + } } if config.base_platform.is_none() { config.base_platform = operator_config.base_platform; @@ -551,3 +597,36 @@ mod tests { assert_eq!(enriched.public_endpoints, Some(public_endpoints)); } } + +#[cfg(test)] +mod loop_health_tests { + use super::LoopHealth; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + /// One or two failed ticks keep readiness (transient errors are normal); + /// the third consecutive failure drops it; the next success restores it + /// and resets the streak. + #[test] + fn drops_after_threshold_and_recovers_on_success() { + let flag = Arc::new(AtomicBool::new(true)); + let mut health = LoopHealth::new(flag.clone()); + + health.record_err(); + health.record_err(); + assert!(flag.load(Ordering::Acquire), "below threshold stays ready"); + + health.record_err(); + assert!(!flag.load(Ordering::Acquire), "threshold reached drops readiness"); + + health.record_ok(); + assert!(flag.load(Ordering::Acquire), "success recovers readiness"); + + // The streak reset means it takes a full threshold again to drop. + health.record_err(); + health.record_err(); + assert!(flag.load(Ordering::Acquire), "streak was reset by the success"); + health.record_err(); + assert!(!flag.load(Ordering::Acquire)); + } +} diff --git a/crates/alien-operator/src/loops/mod.rs b/crates/alien-operator/src/loops/mod.rs index dab13e4a9..4c4c9630d 100644 --- a/crates/alien-operator/src/loops/mod.rs +++ b/crates/alien-operator/src/loops/mod.rs @@ -1,5 +1,6 @@ //! Background loops for the alien-operator +pub mod operator_upgrade; pub mod commands; pub mod debug_session; pub mod deployment; diff --git a/crates/alien-operator/src/loops/operator_upgrade.rs b/crates/alien-operator/src/loops/operator_upgrade.rs new file mode 100644 index 000000000..ec20f24dd --- /dev/null +++ b/crates/alien-operator/src/loops/operator_upgrade.rs @@ -0,0 +1,928 @@ +//! Agent self-update actuator (Kubernetes packaging). When `/v1/sync` returns +//! `operator_target.helm`, this module reconciles a Helm-runner Job that runs +//! `helm upgrade --reuse-values` to flip the agent's image tag (and any other +//! values overrides the manager sent). +//! +//! Unlike the original fire-and-forget version, this is **status-aware**: it +//! reads the live state of the upgrader Job(s) so it can (a) report progress and +//! failures back to the manager via `SyncRequest.operator_update`, and (b) retry +//! with exponential backoff without the old 409-on-a-dead-Job stall. Each retry +//! is a *distinct* Job (`-upgrader--`), so a failed +//! attempt never blocks the next one and its pod survives its TTL for debugging. +//! +//! Talks to the Kubernetes API through `alien-k8s-clients` (`JobApi`/`PodApi`) +//! using the in-pod ServiceAccount config — the projected token + CA and the +//! `KUBERNETES_SERVICE_*` env the chart's pod already has. +//! +//! `os-service` packaging is not in this MVP — the wire carries +//! `operator_target.binary` for it, but the actuator is unimplemented and this +//! module logs + skips that path (its failure reporting lands with that work). + +use std::time::Duration; + +use alien_core::sync::{OperatorHelmTarget, OperatorTarget, OperatorUpdatePhase, OperatorUpdateReport}; +use alien_error::AlienError; +use alien_k8s_clients::{KubernetesClient, KubernetesClientConfig, KubernetesClientConfigExt}; +use chrono::{DateTime, Utc}; +use k8s_openapi::api::batch::v1::Job; +use serde_json::Value; +use tracing::{info, warn}; + +use crate::error::{ErrorData, Result}; + +// Standard `app.kubernetes.io/*` labels on the upgrade Jobs the operator spawns. +// They carry no hardcoded brand — a white-labeled distribution never leaks +// `alien` into a customer's cluster — and line up with the chart's white-labeled +// resources. The vendor's install identity is the Helm release (chart-injected +// via `KUBERNETES_HELM_RELEASE` -> `app.kubernetes.io/instance`); the rest are +// generic. Attempt and target version are the operator's own bookkeeping, so +// they use un-prefixed keys rather than any branded domain. +const LABEL_NAME: &str = "app.kubernetes.io/name"; +const LABEL_INSTANCE: &str = "app.kubernetes.io/instance"; +const LABEL_COMPONENT: &str = "app.kubernetes.io/component"; +const LABEL_MANAGED_BY: &str = "app.kubernetes.io/managed-by"; +const LABEL_ATTEMPT: &str = "upgrade-attempt"; +const ANNOTATION_TARGET_VERSION: &str = "upgrade-target-version"; +const VALUE_NAME: &str = "operator"; +const VALUE_COMPONENT: &str = "upgrader"; +const VALUE_MANAGED_BY: &str = "operator"; +/// Label selector that finds the upgrade Jobs this operator owns. +const JOB_SELECTOR: &str = + "app.kubernetes.io/component=upgrader,app.kubernetes.io/managed-by=operator"; + +/// Exponential backoff between failed attempts: `30s * 2^(attempt-1)`, capped. +const BACKOFF_BASE_SECS: u64 = 30; +const BACKOFF_MAX_SECS: u64 = 300; + +/// Container `waiting.reason`s that mean "the image could not be pulled". +const IMAGE_PULL_REASONS: &[&str] = &[ + "ImagePullBackOff", + "ErrImagePull", + "InvalidImageName", + "ImageInspectError", + "RegistryUnavailable", + "ErrImageNeverPull", +]; + +// ============================================================================ +// Public API — called from the sync loop +// ============================================================================ + +/// Act on an `operator_target`: the `binary` payload goes to the os-service +/// actuator (`crate::self_update`), the `helm` payload reconciles the +/// Kubernetes upgrader Job below. Best-effort — never fails the sync; the +/// report the manager sees comes from [`current_update_report`] on the next +/// request. +pub async fn apply_operator_target(target: &OperatorTarget, state: &crate::OperatorState) { + if let Some(binary) = target.binary.as_ref() { + if !crate::self_update::actuator_enabled() { + warn!( + target_version = %target.version, + "Received operator_target.binary but the os-service actuator is not \ + enabled (not spawned by the launcher, or running in Kubernetes); ignoring" + ); + return; + } + let data_dir = std::path::Path::new(&state.config.data_dir); + match crate::self_update::apply_binary_target( + data_dir, + &crate::operator_version(), + &target.version, + &target.min_supported_version, + binary, + ) + .await + { + Ok(crate::self_update::BinaryActuation::Staged) => { + info!( + target_version = %target.version, + "New operator binary staged; shutting down gracefully for the \ + launcher to perform the health-gated swap" + ); + crate::self_update::request_update_handoff_exit(); + state.cancel.cancel(); + } + Ok(outcome) => { + info!(target_version = %target.version, ?outcome, "Binary target not actuated this tick"); + } + Err(e) => { + warn!(error = %e, target_version = %target.version, "Binary self-update attempt errored; will retry on a later sync"); + } + } + return; + } + + let Some(helm) = target.helm.as_ref() else { + warn!("Received operator_target with neither binary nor helm payload; ignoring"); + return; + }; + let namespace = match k8s_namespace() { + Ok(ns) => ns, + Err(e) => { + warn!(error = %e, "Cannot reconcile agent update — namespace unavailable"); + return; + } + }; + + let jobs = match list_upgrader_jobs(&namespace).await { + Ok(jobs) => jobs, + Err(e) => { + warn!(error = %e, "Failed to list upgrader Jobs; skipping this tick"); + return; + } + }; + let for_version: Vec = jobs + .into_iter() + .filter(|j| j.target_version == target.version) + .collect(); + + match decide_action(&for_version, Utc::now()) { + Action::Wait => {} + Action::Create(attempt) => { + if let Some(prev) = for_version.iter().find(|j| j.status == JobStatus::Failed) { + warn!( + target_version = %target.version, + failed_attempt = prev.attempt, + "Previous upgrader Job failed; spawning retry attempt {attempt}" + ); + } + match resolve_job_inputs(target, helm, attempt) { + Ok(inputs) => { + let (name, body) = build_job_body(&inputs); + match create_job(&namespace, &body).await { + Ok(()) => info!( + target_version = %target.version, + job_name = %name, + attempt, + "Spawned agent upgrader Job" + ), + Err(e) => warn!( + error = %e, + attempt, + "Failed to spawn agent upgrader Job; will retry on next sync" + ), + } + } + Err(e) => warn!(error = %e, "Failed to resolve upgrader Job inputs"), + } + } + } +} + +/// Derive the `operator_update` field for the next `SyncRequest`. Under the +/// launcher (os-service) this translates the on-disk markers — `pending.json` +/// → in-progress, the newest `failed/.json` → failed (the launcher has no +/// network path, so the record IS its report channel). On Kubernetes it reads +/// the live state of the newest upgrader Job. `None` when no update is in +/// flight. +pub async fn current_update_report(state: &crate::OperatorState) -> Option { + if crate::self_update::actuator_enabled() { + let data_dir = std::path::Path::new(&state.config.data_dir); + return crate::self_update::marker_update_report(data_dir); + } + let namespace = k8s_namespace().ok()?; + let jobs = list_upgrader_jobs(&namespace).await.ok()?; + let newest = jobs.iter().max_by_key(|j| j.created_at)?; + + match newest.status { + JobStatus::Active | JobStatus::Succeeded => Some(OperatorUpdateReport::InProgress { + target_version: newest.target_version.clone(), + attempt: newest.attempt, + }), + JobStatus::Failed => { + let release = std::env::var("KUBERNETES_HELM_RELEASE").unwrap_or_default(); + let (phase, message) = classify_failure(&namespace, &release, newest).await; + Some(OperatorUpdateReport::Failed { + target_version: newest.target_version.clone(), + phase, + message, + attempt: newest.attempt, + }) + } + } +} + +// ============================================================================ +// Pure decision core (unit-tested) +// ============================================================================ + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum JobStatus { + Active, + Succeeded, + Failed, +} + +#[derive(Debug, Clone)] +struct UpgraderJob { + #[allow(dead_code)] + name: String, + target_version: String, + attempt: u32, + status: JobStatus, + created_at: DateTime, + /// When the Job reached a terminal condition (for backoff). None if unknown. + finished_at: Option>, + /// Message from the `Failed` condition, if any. + failed_message: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Action { + /// Spawn a new attempt with this 1-based number. + Create(u32), + /// A Job is running, succeeded, or is within its backoff window. + Wait, +} + +/// Parse a k8s Job object into an [`UpgraderJob`]. Returns `None` if it is not a +/// recognizable upgrader Job (missing name). +fn parse_upgrader_job(job: &Value) -> Option { + let meta = job.get("metadata")?; + let name = meta.get("name")?.as_str()?.to_string(); + + let target_version = meta + .get("annotations") + .and_then(|a| a.get(ANNOTATION_TARGET_VERSION)) + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + let attempt = meta + .get("labels") + .and_then(|l| l.get(LABEL_ATTEMPT)) + .and_then(Value::as_str) + .and_then(|s| s.parse::().ok()) + .unwrap_or(1); + let created_at = meta + .get("creationTimestamp") + .and_then(Value::as_str) + .and_then(parse_ts) + .unwrap_or_else(Utc::now); + + let mut status = JobStatus::Active; + let mut finished_at = None; + let mut failed_message = None; + if let Some(conds) = job + .get("status") + .and_then(|s| s.get("conditions")) + .and_then(Value::as_array) + { + for cond in conds { + let ctype = cond.get("type").and_then(Value::as_str).unwrap_or_default(); + let ctrue = cond.get("status").and_then(Value::as_str) == Some("True"); + if !ctrue { + continue; + } + let ts = cond + .get("lastTransitionTime") + .and_then(Value::as_str) + .and_then(parse_ts); + match ctype { + "Failed" => { + status = JobStatus::Failed; + finished_at = ts; + failed_message = cond + .get("message") + .and_then(Value::as_str) + .map(str::to_string); + } + "Complete" => { + if status != JobStatus::Failed { + status = JobStatus::Succeeded; + finished_at = ts; + } + } + _ => {} + } + } + } + + Some(UpgraderJob { + name, + target_version, + attempt, + status, + created_at, + finished_at, + failed_message, + }) +} + +/// Backoff before retrying attempt `attempt` (1-based): `30s * 2^(attempt-1)`, +/// capped at [`BACKOFF_MAX_SECS`]. +fn backoff_delay(attempt: u32) -> Duration { + let shift = attempt.saturating_sub(1).min(20); + let secs = BACKOFF_BASE_SECS + .saturating_mul(1u64 << shift) + .min(BACKOFF_MAX_SECS); + Duration::from_secs(secs) +} + +/// Whether a failed Job's backoff window has elapsed and a retry is due. +fn should_retry(job: &UpgraderJob, now: DateTime) -> bool { + match job.finished_at { + Some(finished) => { + let elapsed = now.signed_duration_since(finished); + elapsed.to_std().map(|d| d >= backoff_delay(job.attempt)).unwrap_or(true) + } + // Failed but no timestamp — be permissive and allow the retry. + None => true, + } +} + +/// Decide what to do given the upgrader Jobs for a single target version. +fn decide_action(jobs_for_version: &[UpgraderJob], now: DateTime) -> Action { + match jobs_for_version.iter().max_by_key(|j| j.attempt) { + None => Action::Create(1), + Some(newest) => match newest.status { + JobStatus::Active | JobStatus::Succeeded => Action::Wait, + JobStatus::Failed => { + if should_retry(newest, now) { + Action::Create(newest.attempt + 1) + } else { + Action::Wait + } + } + }, + } +} + +fn parse_ts(s: &str) -> Option> { + DateTime::parse_from_rfc3339(s) + .ok() + .map(|dt| dt.with_timezone(&Utc)) +} + +fn is_image_pull_reason(reason: &str) -> bool { + IMAGE_PULL_REASONS.iter().any(|r| *r == reason) +} + +/// Scan a pod's container statuses for an image-pull `waiting` state. Returns +/// `(image, reason, message)` if found. Pure — takes the pod JSON. +fn image_pull_waiting(pod: &Value) -> Option<(String, String, String)> { + let statuses = ["containerStatuses", "initContainerStatuses"]; + let podspec_status = pod.get("status")?; + for key in statuses { + let Some(list) = podspec_status.get(key).and_then(Value::as_array) else { + continue; + }; + for cs in list { + let Some(waiting) = cs.get("state").and_then(|s| s.get("waiting")) else { + continue; + }; + let reason = waiting.get("reason").and_then(Value::as_str).unwrap_or_default(); + if is_image_pull_reason(reason) { + let image = cs.get("image").and_then(Value::as_str).unwrap_or_default(); + let message = waiting.get("message").and_then(Value::as_str).unwrap_or_default(); + return Some((image.to_string(), reason.to_string(), message.to_string())); + } + } + } + None +} + +// ============================================================================ +// Failure classification (I/O) +// ============================================================================ + +/// Determine whether a failed upgrade was a `Pull` failure (the new agent image +/// couldn't be pulled — the ImagePullBackOff case) or an `Apply` failure (helm +/// upgrade / rollback). Inspects the agent workload pods for an image-pull +/// waiting state; falls back to `Apply` with the Job's failure message. +async fn classify_failure( + namespace: &str, + release: &str, + job: &UpgraderJob, +) -> (OperatorUpdatePhase, String) { + if !release.is_empty() { + let selector = format!("app.kubernetes.io/instance={release}"); + if let Ok(pods) = list_pods(namespace, &selector).await { + for pod in &pods { + // Skip the upgrader runner pods; we want the agent workload pod. + let is_upgrader = pod + .get("metadata") + .and_then(|m| m.get("labels")) + .and_then(|l| l.get(LABEL_COMPONENT)) + .and_then(Value::as_str) + == Some(VALUE_COMPONENT); + if is_upgrader { + continue; + } + if let Some((image, reason, message)) = image_pull_waiting(pod) { + return ( + OperatorUpdatePhase::Pull, + format!("{image}: {reason} {message}").trim().to_string(), + ); + } + } + } + } + let message = job + .failed_message + .clone() + .unwrap_or_else(|| "helm upgrade failed".to_string()); + (OperatorUpdatePhase::Apply, message) +} + +// ============================================================================ +// Job body construction (pure) +// ============================================================================ + +/// Everything `build_job_body` needs that isn't a string literal. +struct JobInputs { + target_version: String, + attempt: u32, + namespace: String, + release: String, + upgrader_sa: String, + chart_ref: String, + /// `None` means "let helm pick latest" — omits the `--version` flag. + chart_version: Option, + runner_image: String, + /// Extra flags spliced into the `helm upgrade` command verbatim. + extra_args: String, + /// Pre-serialized JSON for `operator_target.helm.values`. + values_json: String, +} + +/// Read env vars + apply the manager-vs-env fallback for chart ref/version, and +/// serialise the values overlay. `attempt` is decided by the reconcile. +fn resolve_job_inputs(target: &OperatorTarget, helm: &OperatorHelmTarget, attempt: u32) -> Result { + let namespace = k8s_namespace()?; + let release = std::env::var("KUBERNETES_HELM_RELEASE").map_err(|_| { + AlienError::new(ErrorData::ConfigurationError { + message: "KUBERNETES_HELM_RELEASE env var missing — set by the chart at install time" + .to_string(), + }) + })?; + let upgrader_sa = std::env::var("ALIEN_OPERATOR_UPGRADER_SA").map_err(|_| { + AlienError::new(ErrorData::ConfigurationError { + message: "ALIEN_OPERATOR_UPGRADER_SA env var missing — set by the chart at install time" + .to_string(), + }) + })?; + + let chart_ref = non_empty(&helm.chart_repo) + .map(String::from) + .or_else(|| std::env::var("ALIEN_OPERATOR_CHART_REF").ok()) + .ok_or_else(|| { + AlienError::new(ErrorData::ConfigurationError { + message: + "operator_target.helm.chart_repo empty AND ALIEN_OPERATOR_CHART_REF unset — \ + can't run helm upgrade without a chart reference" + .to_string(), + }) + })?; + let chart_version = non_empty(&helm.chart_version) + .map(String::from) + .or_else(|| std::env::var("ALIEN_OPERATOR_CHART_VERSION").ok().filter(|s| !s.is_empty())); + let runner_image = std::env::var("ALIEN_OPERATOR_HELM_RUNNER_IMAGE") + .unwrap_or_else(|_| "alpine/helm:3.18.4".to_string()); + let extra_args = std::env::var("ALIEN_OPERATOR_HELM_EXTRA_ARGS").unwrap_or_default(); + + let values_json = serde_json::to_string(&helm.values).map_err(|e| { + AlienError::new(ErrorData::ConfigurationError { + message: format!("Failed to serialize operator_target.helm.values: {e}"), + }) + })?; + + Ok(JobInputs { + target_version: target.version.clone(), + attempt, + namespace, + release, + upgrader_sa, + chart_ref, + chart_version, + runner_image, + extra_args, + values_json, + }) +} + +/// Pure: build the Job name + the JSON body POSTed to the k8s API. The name is +/// unique per attempt so a failed attempt never blocks a retry; the raw target +/// version rides in an annotation (label values can't hold `+build` metadata). +fn build_job_body(inputs: &JobInputs) -> (String, serde_json::Value) { + let job_name = format!( + "{}-upgrader-{}-{}", + inputs.release, + sanitize_for_dns(&inputs.target_version, 16), + inputs.attempt + ); + + let version_flag = inputs + .chart_version + .as_deref() + .map(|v| format!(" --version {v}")) + .unwrap_or_default(); + let extra_args_flag = if inputs.extra_args.trim().is_empty() { + String::new() + } else { + format!(" {}", inputs.extra_args.trim()) + }; + // `helm upgrade` flags, and why each matters for a self-update: + // --reuse-values keep the deployment's existing values; the manager only + // overrides `image.tag` (the pinned operator version), so a + // version pin never silently drops other config. + // --atomic roll the release back to the prior revision if the upgrade + // fails, so a bad tag never leaves a half-swapped operator. + // That rollback IS the Apply-phase failure we report — the + // operator stays on its old version rather than a broken one. + // --wait block until the new pods pass their readiness probe (which + // gates on a first successful `/v1/sync`), so "upgrade done" + // means the new operator actually came up and reached the + // manager, not merely that the API objects were applied. + let helm_cmd = format!( + "set -e\n\ + printf '%s' \"$VALUES_JSON\" > /tmp/values.json\n\ + exec helm upgrade \"$RELEASE\" \"$CHART_REF\"{version_flag}{extra_args_flag} \ + --namespace \"$NAMESPACE\" \ + --reuse-values \ + --atomic --wait \ + --values /tmp/values.json" + ); + + let body = serde_json::json!({ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": job_name, + "namespace": inputs.namespace, + "labels": { + LABEL_NAME: VALUE_NAME, + LABEL_INSTANCE: inputs.release, + LABEL_COMPONENT: VALUE_COMPONENT, + LABEL_MANAGED_BY: VALUE_MANAGED_BY, + LABEL_ATTEMPT: inputs.attempt.to_string(), + }, + "annotations": { + ANNOTATION_TARGET_VERSION: inputs.target_version, + }, + }, + "spec": { + "backoffLimit": 1, + "ttlSecondsAfterFinished": 600, + "template": { + "metadata": { + "labels": { + LABEL_NAME: VALUE_NAME, + LABEL_INSTANCE: inputs.release, + LABEL_COMPONENT: VALUE_COMPONENT, + LABEL_MANAGED_BY: VALUE_MANAGED_BY, + }, + }, + "spec": { + "serviceAccountName": inputs.upgrader_sa, + "restartPolicy": "Never", + "containers": [{ + "name": "helm-upgrade", + "image": inputs.runner_image, + "command": ["sh", "-c", helm_cmd], + "env": [ + { "name": "RELEASE", "value": inputs.release }, + { "name": "NAMESPACE", "value": inputs.namespace }, + { "name": "CHART_REF", "value": inputs.chart_ref }, + { "name": "VALUES_JSON", "value": inputs.values_json }, + ], + }], + }, + }, + }, + }); + + (job_name, body) +} + +// ============================================================================ +// Kubernetes API I/O +// ============================================================================ + +fn k8s_namespace() -> Result { + std::env::var("KUBERNETES_NAMESPACE").map_err(|_| { + AlienError::new(ErrorData::ConfigurationError { + message: "KUBERNETES_NAMESPACE env var missing — agent self-update needs it".to_string(), + }) + }) +} + +/// In-pod Kubernetes client via `alien-k8s-clients`. `try_incluster` reads the +/// projected ServiceAccount token + CA and the `KUBERNETES_SERVICE_*` env the +/// chart's pod already has, so this module no longer hand-rolls auth/transport. +async fn k8s_client() -> Result { + let config = KubernetesClientConfig::try_incluster().await.map_err(|e| { + AlienError::new(ErrorData::ConfigurationError { + message: format!("Failed to load in-cluster Kubernetes config: {e}"), + }) + })?; + KubernetesClient::new(config).await.map_err(|e| { + AlienError::new(ErrorData::ConfigurationError { + message: format!("Failed to build Kubernetes client: {e}"), + }) + }) +} + +async fn list_upgrader_jobs(namespace: &str) -> Result> { + let client = k8s_client().await?; + let list = client + .list_jobs(namespace, Some(JOB_SELECTOR.to_string()), None) + .await + .map_err(|e| { + AlienError::new(ErrorData::ConfigurationError { + message: format!("Failed to list upgrader Jobs: {e}"), + }) + })?; + // Bridge the typed k8s-openapi Jobs back through JSON so the pure + // `parse_upgrader_job` (and its tests) keep working unchanged. + Ok(list + .items + .iter() + .filter_map(|job| serde_json::to_value(job).ok()) + .filter_map(|v| parse_upgrader_job(&v)) + .collect()) +} + +async fn list_pods(namespace: &str, selector: &str) -> Result> { + let client = k8s_client().await?; + let list = client + .list_pods(namespace, Some(selector.to_string()), None) + .await + .map_err(|e| { + AlienError::new(ErrorData::ConfigurationError { + message: format!("Failed to list pods: {e}"), + }) + })?; + Ok(list + .items + .iter() + .filter_map(|pod| serde_json::to_value(pod).ok()) + .collect()) +} + +/// Create the upgrader Job via `alien-k8s-clients`. Unique per-attempt names make +/// a 409 a rare same-sync race rather than the old dead-Job stall — treat it as +/// benign (the Job for this attempt is already in flight). +async fn create_job(namespace: &str, body: &serde_json::Value) -> Result<()> { + let job: Job = serde_json::from_value(body.clone()).map_err(|e| { + AlienError::new(ErrorData::ConfigurationError { + message: format!("Failed to build upgrader Job object: {e}"), + }) + })?; + let client = k8s_client().await?; + match client.create_job(namespace, &job).await { + Ok(_) => Ok(()), + Err(e) if e.http_status_code == Some(409) => { + warn!("Upgrader Job create returned 409 (race with a concurrent sync); leaving in place"); + Ok(()) + } + Err(e) => Err(AlienError::new(ErrorData::ConfigurationError { + message: format!("k8s API rejected Job creation: {e}"), + })), + } +} + +fn non_empty(s: &str) -> Option<&str> { + if s.is_empty() { None } else { Some(s) } +} + +/// Lowercase, DNS-1123-safe, max `cap` chars, trimmed of trailing hyphens. +fn sanitize_for_dns(value: &str, cap: usize) -> String { + let mut out: String = value + .to_lowercase() + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .collect(); + out.truncate(cap); + while out.ends_with('-') { + out.pop(); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn inputs() -> JobInputs { + JobInputs { + target_version: "1.4.0".to_string(), + attempt: 1, + namespace: "alien-agent".to_string(), + release: "alien".to_string(), + upgrader_sa: "alien-agent-upgrader".to_string(), + chart_ref: "oci://ghcr.io/alien-dev/alien".to_string(), + chart_version: Some("1.4.0".to_string()), + runner_image: "alpine/helm:3.18.4".to_string(), + extra_args: String::new(), + values_json: r#"{"runtime":{"image":{"tag":"1.4.0"}}}"#.to_string(), + } + } + + fn ts(s: &str) -> DateTime { + parse_ts(s).unwrap() + } + + #[test] + fn job_name_includes_release_version_and_attempt() { + let (name, _) = build_job_body(&inputs()); + assert_eq!(name, "alien-upgrader-1-4-0-1"); + let mut i = inputs(); + i.attempt = 3; + let (name, _) = build_job_body(&i); + assert_eq!(name, "alien-upgrader-1-4-0-3"); + } + + #[test] + fn job_name_caps_long_version_and_trims_trailing_hyphens() { + let mut i = inputs(); + i.target_version = "1.4.0-rc.1+build".to_string(); + let (name, _) = build_job_body(&i); + assert!(name.starts_with("alien-upgrader-"), "got {name}"); + assert!(name.ends_with("-1"), "attempt suffix should survive: {name}"); + // No double-hyphen from a trimmed sanitized core meeting the attempt. + assert!(!name.contains("--"), "no empty segment: {name}"); + } + + #[test] + fn job_metadata_is_white_labeled_with_standard_labels() { + let mut i = inputs(); + i.attempt = 2; + i.target_version = "1.4.0+build.7".to_string(); + let (_, body) = build_job_body(&i); + let labels = &body["metadata"]["labels"]; + // Standard app.kubernetes.io/* set; the vendor identity is the release. + assert_eq!(labels["app.kubernetes.io/name"], json!("operator")); + assert_eq!(labels["app.kubernetes.io/instance"], json!(i.release)); // == release + assert_eq!(labels["app.kubernetes.io/component"], json!("upgrader")); + assert_eq!(labels["app.kubernetes.io/managed-by"], json!("operator")); + assert_eq!(labels["upgrade-attempt"], json!("2")); + // Raw version (with '+') preserved in the annotation, not the label. + assert_eq!( + body["metadata"]["annotations"]["upgrade-target-version"], + json!("1.4.0+build.7") + ); + // No brand leaks into the customer's cluster via labels/annotations. + let stamped = format!("{}{}", labels, body["metadata"]["annotations"]); + assert!(!stamped.contains("alien.dev"), "no alien.dev keys: {stamped}"); + assert!(!stamped.contains("alien-agent"), "no alien-agent value: {stamped}"); + } + + #[test] + fn job_body_deserializes_into_a_typed_k8s_job() { + // create_job feeds build_job_body's JSON into a typed k8s-openapi Job + // before handing it to JobApi; guard that the hand-built body stays + // schema-compatible. + let (name, body) = build_job_body(&inputs()); + let job: k8s_openapi::api::batch::v1::Job = + serde_json::from_value(body).expect("body should deserialize into a typed Job"); + assert_eq!(job.metadata.name.as_deref(), Some(name.as_str())); + assert_eq!(job.metadata.namespace.as_deref(), Some("alien-agent")); + let containers = job + .spec + .expect("job spec") + .template + .spec + .expect("pod spec") + .containers; + assert_eq!(containers.len(), 1); + } + + #[test] + fn helm_command_omits_version_flag_when_unset() { + let mut i = inputs(); + i.chart_version = None; + let (_, body) = build_job_body(&i); + let cmd = body["spec"]["template"]["spec"]["containers"][0]["command"][2] + .as_str() + .unwrap(); + assert!(!cmd.contains("--version"), "got: {cmd}"); + assert!(cmd.contains("--reuse-values"), "got: {cmd}"); + assert!(cmd.contains("--atomic --wait"), "got: {cmd}"); + } + + #[test] + fn env_vars_carry_manager_sent_values_and_release_metadata() { + let (_, body) = build_job_body(&inputs()); + let envs = body["spec"]["template"]["spec"]["containers"][0]["env"] + .as_array() + .unwrap(); + let by_name: std::collections::HashMap<_, _> = envs + .iter() + .map(|e| (e["name"].as_str().unwrap(), e["value"].as_str().unwrap())) + .collect(); + assert_eq!(by_name["RELEASE"], "alien"); + assert_eq!(by_name["CHART_REF"], "oci://ghcr.io/alien-dev/alien"); + assert_eq!(by_name["VALUES_JSON"], r#"{"runtime":{"image":{"tag":"1.4.0"}}}"#); + } + + #[test] + fn helm_command_splices_extra_args_before_namespace() { + let mut i = inputs(); + i.extra_args = "--plain-http".to_string(); + let (_, body) = build_job_body(&i); + let cmd = body["spec"]["template"]["spec"]["containers"][0]["command"][2] + .as_str() + .unwrap(); + let plain_pos = cmd.find("--plain-http").expect("--plain-http present"); + let ns_pos = cmd.find("--namespace").expect("--namespace present"); + assert!(plain_pos < ns_pos, "--plain-http before --namespace: {cmd}"); + } + + #[test] + fn backoff_is_exponential_and_capped() { + assert_eq!(backoff_delay(1), Duration::from_secs(30)); + assert_eq!(backoff_delay(2), Duration::from_secs(60)); + assert_eq!(backoff_delay(3), Duration::from_secs(120)); + assert_eq!(backoff_delay(4), Duration::from_secs(240)); + assert_eq!(backoff_delay(5), Duration::from_secs(300)); // capped + assert_eq!(backoff_delay(50), Duration::from_secs(300)); // no overflow + } + + fn failed_job(attempt: u32, finished: &str) -> UpgraderJob { + UpgraderJob { + name: format!("alien-upgrader-1-4-0-{attempt}"), + target_version: "1.4.0".to_string(), + attempt, + status: JobStatus::Failed, + created_at: ts(finished), + finished_at: Some(ts(finished)), + failed_message: Some("BackoffLimitExceeded".to_string()), + } + } + + #[test] + fn decide_creates_first_attempt_when_no_jobs() { + assert_eq!(decide_action(&[], Utc::now()), Action::Create(1)); + } + + #[test] + fn decide_waits_while_active() { + let mut j = failed_job(1, "2026-07-05T12:00:00Z"); + j.status = JobStatus::Active; + assert_eq!(decide_action(&[j], Utc::now()), Action::Wait); + } + + #[test] + fn decide_retries_failed_after_backoff_elapses() { + let j = failed_job(1, "2026-07-05T12:00:00Z"); + // attempt 1 backoff = 30s; 40s later -> retry as attempt 2. + let now = ts("2026-07-05T12:00:40Z"); + assert_eq!(decide_action(&[j], now), Action::Create(2)); + } + + #[test] + fn decide_waits_within_backoff_window() { + let j = failed_job(2, "2026-07-05T12:00:00Z"); + // attempt 2 backoff = 60s; only 20s later -> wait. + let now = ts("2026-07-05T12:00:20Z"); + assert_eq!(decide_action(&[j], now), Action::Wait); + } + + #[test] + fn image_pull_waiting_detects_backoff_reason() { + let pod = json!({ + "status": { + "containerStatuses": [{ + "image": "public.ecr.aws/x/agent:1.4.0", + "state": { "waiting": { "reason": "ImagePullBackOff", "message": "not found" } } + }] + } + }); + let (image, reason, message) = image_pull_waiting(&pod).expect("should detect"); + assert_eq!(image, "public.ecr.aws/x/agent:1.4.0"); + assert_eq!(reason, "ImagePullBackOff"); + assert_eq!(message, "not found"); + } + + #[test] + fn image_pull_waiting_ignores_running_pod() { + let pod = json!({ + "status": { "containerStatuses": [{ "state": { "running": {} } }] } + }); + assert!(image_pull_waiting(&pod).is_none()); + } + + #[test] + fn parse_upgrader_job_reads_labels_annotations_and_failed_status() { + let job = json!({ + "metadata": { + "name": "alien-upgrader-1-4-0-2", + "creationTimestamp": "2026-07-05T12:00:00Z", + "labels": { "upgrade-attempt": "2" }, + "annotations": { "upgrade-target-version": "1.4.0+build" } + }, + "status": { + "conditions": [ + { "type": "Failed", "status": "True", "lastTransitionTime": "2026-07-05T12:05:00Z", "message": "BackoffLimitExceeded" } + ] + } + }); + let parsed = parse_upgrader_job(&job).unwrap(); + assert_eq!(parsed.attempt, 2); + assert_eq!(parsed.target_version, "1.4.0+build"); + assert_eq!(parsed.status, JobStatus::Failed); + assert_eq!(parsed.failed_message.as_deref(), Some("BackoffLimitExceeded")); + assert_eq!(parsed.finished_at, Some(ts("2026-07-05T12:05:00Z"))); + } +} diff --git a/crates/alien-operator/src/loops/sync.rs b/crates/alien-operator/src/loops/sync.rs index e9e9c98b6..a1ca8dfed 100644 --- a/crates/alien-operator/src/loops/sync.rs +++ b/crates/alien-operator/src/loops/sync.rs @@ -10,7 +10,10 @@ use crate::db::{Approval, ApprovalStatus}; use crate::OperatorState; use alien_core::{ - sync::{OperatorCapabilityReport, OperatorCapabilityState, SyncRequest, SyncResponse}, + sync::{ + OperatorArch, OperatorCapabilityReport, OperatorCapabilityState, OperatorOs, + OperatorPackaging, SyncRequest, SyncResponse, + }, DeploymentStatus, ObservedInventoryBatch, Platform, }; use alien_deployment::run_observe_pass; @@ -58,6 +61,14 @@ pub async fn run_sync_loop(state: Arc) { loop { match sync_with_manager(&state, &client, sync_config.url.as_str()).await { Ok(has_update) => { + // First successful sync turns /readyz from 503 → 200 — the + // gate Helm's --atomic --wait relies on so a freshly-rolled + // agent isn't marked ready until it has actually talked to + // the manager. Idempotent — only the first store matters. + state + .readiness + .first_sync_completed + .store(true, std::sync::atomic::Ordering::Release); if has_update { info!("Received update from manager"); } else { @@ -163,13 +174,31 @@ async fn sync_with_manager( } } + // Report the outcome of any in-flight self-update so the manager can show a + // truthful failed/in-progress state instead of inferring from a stalled + // version (markers under the launcher, upgrader-Job state on Kubernetes). + let operator_update = crate::loops::operator_upgrade::current_update_report(state).await; + let sync_request = SyncRequest { deployment_id: deployment_id.clone(), current_state: Some(deployment_state), heartbeats, observed_inventory_batches, capabilities: report_operator_capabilities(state), - operator_version: Some(env!("CARGO_PKG_VERSION").to_string()), + operator_version: Some(crate::operator_version()), + // The supervising launcher's version (ALIEN_LAUNCHER_VERSION, set on + // spawn) — the manager's min_launcher_version gate input. None on + // Kubernetes / launcher-less runs. + launcher_version: crate::self_update::launcher_version(), + // Operator self-update inventory — fleet visibility + upgrade gating. + operator_os: OperatorOs::detect(), + operator_arch: OperatorArch::detect(), + packaging: Some(detect_operator_packaging()), + // Chart-injected so admins see the registry the operator pulls a new tag from. + operator_image_repository: std::env::var("ALIEN_OPERATOR_IMAGE_REPOSITORY") + .ok() + .filter(|s| !s.is_empty()), + operator_update, }; // Call manager with deployment_id in request body. @@ -326,6 +355,13 @@ async fn sync_with_manager( } } + // Agent self-update: act on `operator_target` when the manager emits one. + // Best-effort — the actuator logs failures and the manager keeps + // sending the target until the agent reports the new version. + if let Some(target) = sync_response.operator_target.as_ref() { + crate::loops::operator_upgrade::apply_operator_target(target, state).await; + } + Ok(has_update || state_hydrated) } @@ -438,6 +474,17 @@ fn is_uninitialized_deployment_state(state: &alien_core::DeploymentState) -> boo && state.runtime_metadata.is_none() } +/// Detect the agent's supervisor packaging. `KUBERNETES_SERVICE_HOST` is the +/// kubelet-injected signal and takes precedence over any other hint; outside +/// k8s the agent is supervised as a native OS service via the launcher. +fn detect_operator_packaging() -> OperatorPackaging { + if std::env::var_os("KUBERNETES_SERVICE_HOST").is_some() { + OperatorPackaging::Kubernetes + } else { + OperatorPackaging::OsService + } +} + fn apply_manager_control_state( local_state: &mut alien_core::DeploymentState, manager_state: &alien_core::DeploymentState, diff --git a/crates/alien-operator/src/otlp_server.rs b/crates/alien-operator/src/otlp_server.rs index 80f4cea74..0cecd2a74 100644 --- a/crates/alien-operator/src/otlp_server.rs +++ b/crates/alien-operator/src/otlp_server.rs @@ -14,6 +14,7 @@ use axum::{ }; use serde::Serialize; use std::net::{IpAddr, SocketAddr}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info}; @@ -28,6 +29,68 @@ struct OtlpServerState { collector_token: Option, } +/// The operator's readiness, decomposed into the observable conditions of +/// the health contract ("an operator is healthy when…"). `/readyz` is 200 +/// iff ALL of them hold; condition 1 — the process is alive — is implicit in +/// the server answering at all. +#[derive(Clone)] +pub struct ReadinessSignals { + /// Condition 2: the encrypted state DB opened successfully. Set once at + /// startup after `OperatorDb::new` succeeds (a DB that dies later + /// surfaces through condition 5 — the loops start failing). + pub db_open: Arc, + /// Condition 3: the `InstanceLock` is held. The CLI acquires it before + /// the operator runs and the guard lives for the process lifetime. + pub lock_held: Arc, + /// Condition 4: flipped to `true` by the sync loop once at least one + /// /v1/sync round-trip with the manager has succeeded. + pub first_sync_completed: Arc, + /// Condition 5: the deployment loop is progressing — flipped to `false` + /// after it errors several consecutive ticks, back to `true` on the next + /// success (see `loops::deployment::LoopHealth`). + pub deployment_loop_ok: Arc, +} + +impl ReadinessSignals { + /// Fresh signals: nothing proven yet (DB, lock, sync all pending) except + /// the deployment loop, which is healthy-until-it-fails — it may not + /// even have work to do. + pub fn new() -> Self { + Self { + db_open: Arc::new(AtomicBool::new(false)), + lock_held: Arc::new(AtomicBool::new(false)), + first_sync_completed: Arc::new(AtomicBool::new(false)), + deployment_loop_ok: Arc::new(AtomicBool::new(true)), + } + } + + /// Liveness — the health conditions that do NOT depend on the manager: + /// DB open (2), lock held (3), deployment loop progressing (5). Condition + /// 1 (process alive) is implicit in the server answering at all. This is + /// "healthy except possibly for manager reachability": `/livez` reflects + /// it, and it's the signal the launcher uses to decide the operator is + /// up and running (so a manager outage at boot doesn't look like a start + /// failure). + pub fn local_ready(&self) -> bool { + self.db_open.load(Ordering::Acquire) + && self.lock_held.load(Ordering::Acquire) + && self.deployment_loop_ok.load(Ordering::Acquire) + } + + /// Full readiness = liveness PLUS at least one successful sync (condition + /// 4). `/readyz` reflects it; it gates the launcher's health-gated swap + /// and the Kubernetes `readinessProbe`. + pub fn all_ready(&self) -> bool { + self.local_ready() && self.first_sync_completed.load(Ordering::Acquire) + } +} + +impl Default for ReadinessSignals { + fn default() -> Self { + Self::new() + } +} + /// OTLP response #[derive(Serialize)] struct OtlpResponse { @@ -39,11 +102,14 @@ struct HealthResponse { status: &'static str, } -/// Start the OTLP server with graceful shutdown support. +/// Start the OTLP server with graceful shutdown support. Also serves +/// `/livez` and `/readyz` on the same port — the Kubernetes probes the +/// chart's deployment template references. pub async fn start_otlp_server( host: IpAddr, port: u16, db: Arc, + readiness: ReadinessSignals, namespace: Option, collector_token: Option, cancel: CancellationToken, @@ -52,6 +118,14 @@ pub async fn start_otlp_server( info!(address = %addr, "Starting OTLP server"); + // Probes ride on a separate Router so they don't need the OTLP body-limit + // layer and have a different (slim) state type; they are merged into the + // OTLP app at the end. + let probes = Router::new() + .route("/livez", get(handle_livez)) + .route("/readyz", get(handle_readyz)) + .with_state(readiness); + let app = Router::new() .route("/health", get(handle_health)) .route("/v1/logs", post(handle_logs)) @@ -63,7 +137,8 @@ pub async fn start_otlp_server( db, namespace, collector_token, - }); + }) + .merge(probes); let listener = tokio::net::TcpListener::bind(&addr) .await @@ -88,6 +163,64 @@ async fn handle_health() -> Json { Json(HealthResponse { status: "ok" }) } +/// Liveness probe: 200 iff the manager-independent health conditions hold — +/// DB open, InstanceLock held, deployment loop progressing (process-alive is +/// implicit in this handler answering). Deliberately does NOT require a +/// successful sync, so an operator that is up and running but cannot reach +/// the manager reports **live** (200) while `/readyz` stays 503. Consumed by +/// the Kubernetes `livenessProbe` and by the launcher's startup gate (it +/// signals systemd READY once the operator is live — a manager outage at +/// boot must not look like a launch failure). +async fn handle_livez( + State(readiness): State, +) -> Result, StatusCode> { + if !readiness.local_ready() { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + Ok(Json(HealthResponse { status: "ok" })) +} + +/// Readiness probe: 200 iff ALL health conditions hold — DB opened, +/// InstanceLock held, at least one successful `/v1/sync` round-trip, and the +/// deployment loop progressing (the process being alive is implicit in the +/// response itself). Consumed by the chart's `readinessProbe` (and therefore +/// Helm's `--atomic --wait`) on Kubernetes and by the launcher's probation +/// gate on os-service — a freshly-swapped operator isn't considered healthy +/// until it has actually proven it can run and reach the manager. +async fn handle_readyz( + State(readiness): State, +) -> Result, StatusCode> { + if !readiness.all_ready() { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + Ok(Json(HealthResponse { status: "ok" })) +} + +/// The health-endpoint bind override the launcher passes on spawn +/// (`ALIEN_HEALTH_ADDR=127.0.0.1:`). `None` when unset (Kubernetes / +/// standalone — the config's OTLP host+port apply). An unparseable value is +/// a hard error: the launcher probes the exact address it set, so falling +/// back to the config port would make every probation fail by mismatch — +/// better to crash loudly and let the launcher's backoff surface it. +pub fn health_addr_override() -> crate::error::Result> { + parse_health_addr(std::env::var(alien_core::self_update::ENV_HEALTH_ADDR).ok()) +} + +fn parse_health_addr(env: Option) -> crate::error::Result> { + let Some(raw) = env.filter(|value| !value.is_empty()) else { + return Ok(None); + }; + raw.parse::() + .map(Some) + .into_alien_error() + .context(crate::error::ErrorData::ConfigurationError { + message: format!( + "{} is set but is not a valid socket address: '{raw}'", + alien_core::self_update::ENV_HEALTH_ADDR + ), + }) +} + async fn handle_logs( State(state): State, body: Bytes, @@ -185,9 +318,22 @@ mod tests { listener.local_addr().unwrap().port() } - #[tokio::test] - async fn health_endpoint_returns_ok() { + /// Signals with every condition already satisfied — tests flip individual + /// flags off from here. + fn all_ready_signals() -> ReadinessSignals { + let signals = ReadinessSignals::new(); + signals.db_open.store(true, Ordering::Release); + signals.lock_held.store(true, Ordering::Release); + signals.first_sync_completed.store(true, Ordering::Release); + signals + } + + async fn start_test_server( + readiness: ReadinessSignals, + ) -> (u16, CancellationToken, tokio::task::JoinHandle>) { let data_dir = tempfile::tempdir().unwrap(); + // Leak the tempdir guard so it outlives the spawned server. + let data_dir = Box::leak(Box::new(data_dir)); let db = Arc::new( OperatorDb::new(data_dir.path().to_str().unwrap(), TEST_ENCRYPTION_KEY) .await @@ -196,38 +342,171 @@ mod tests { let port = free_port(); let cancel = CancellationToken::new(); let server_cancel = cancel.clone(); - let server = tokio::spawn(async move { start_otlp_server( IpAddr::V4(Ipv4Addr::LOCALHOST), port, db, + readiness, None, None, server_cancel, ) .await }); - + // Wait for the server to bind so subsequent GETs don't race. let url = format!("http://127.0.0.1:{port}/health"); let client = reqwest::Client::new(); - let mut response = None; for _ in 0..50 { - if let Ok(success) = client.get(&url).send().await { - response = Some(success); + if client.get(&url).send().await.is_ok() { break; } sleep(Duration::from_millis(20)).await; } + (port, cancel, server) + } - let response = response.expect("health endpoint did not become reachable"); + #[tokio::test] + async fn health_endpoint_returns_ok() { + let (port, cancel, server) = start_test_server(ReadinessSignals::new()).await; + let client = reqwest::Client::new(); + let response = client + .get(format!("http://127.0.0.1:{port}/health")) + .send() + .await + .unwrap(); assert!(response.status().is_success()); assert_eq!( response.json::().await.unwrap(), serde_json::json!({ "status": "ok" }) ); + cancel.cancel(); + server.await.unwrap().unwrap(); + } + + /// `/livez` is 200 once the LOCAL conditions hold (DB open, lock held, + /// loop progressing) even with NO successful sync — an operator that is + /// up but can't reach the manager is live but not ready. And each local + /// condition gates it (but first-sync does NOT). + #[tokio::test] + async fn livez_reflects_local_health_not_manager_reachability() { + let signals = all_ready_signals(); + // No sync yet: manager unreachable, but the operator is up. + signals.first_sync_completed.store(false, Ordering::Release); + let (port, cancel, server) = start_test_server(signals.clone()).await; + let client = reqwest::Client::new(); + let livez = format!("http://127.0.0.1:{port}/livez"); + let readyz = format!("http://127.0.0.1:{port}/readyz"); + + assert!( + client.get(&livez).send().await.unwrap().status().is_success(), + "live (up + local health) even before any sync" + ); + assert_eq!( + client.get(&readyz).send().await.unwrap().status().as_u16(), + 503, + "but NOT ready — no manager round-trip yet" + ); + + // first-sync does not gate liveness; the three local conditions do. + signals.first_sync_completed.store(true, Ordering::Release); + for (name, flag) in [ + ("db_open", &signals.db_open), + ("lock_held", &signals.lock_held), + ("deployment_loop_ok", &signals.deployment_loop_ok), + ] { + flag.store(false, Ordering::Release); + assert_eq!( + client.get(&livez).send().await.unwrap().status().as_u16(), + 503, + "{name}=false must gate liveness" + ); + flag.store(true, Ordering::Release); + assert!( + client.get(&livez).send().await.unwrap().status().is_success(), + "{name} restored must recover liveness" + ); + } cancel.cancel(); server.await.unwrap().unwrap(); } + + #[tokio::test] + async fn readyz_is_503_before_first_sync_and_200_after() { + let signals = all_ready_signals(); + signals.first_sync_completed.store(false, Ordering::Release); + let (port, cancel, server) = start_test_server(signals.clone()).await; + let client = reqwest::Client::new(); + let url = format!("http://127.0.0.1:{port}/readyz"); + + // Before the sync loop flips the flag, readyz must be 503 so that + // a freshly-rolled agent isn't marked ready until it has actually + // talked to the manager once. + let before = client.get(&url).send().await.unwrap(); + assert_eq!(before.status().as_u16(), 503, "expected 503 before first sync"); + + // Simulate the sync loop completing its first round-trip. + signals.first_sync_completed.store(true, Ordering::Release); + + let after = client.get(&url).send().await.unwrap(); + assert!(after.status().is_success(), "expected 200 after first sync"); + + cancel.cancel(); + server.await.unwrap().unwrap(); + } + + /// Every one of the four explicit conditions gates /readyz on its own: + /// flipping any single flag off must 503, restoring it must 200. + #[tokio::test] + async fn readyz_requires_every_condition() { + let signals = all_ready_signals(); + let (port, cancel, server) = start_test_server(signals.clone()).await; + let client = reqwest::Client::new(); + let url = format!("http://127.0.0.1:{port}/readyz"); + + let all_ok = client.get(&url).send().await.unwrap(); + assert!(all_ok.status().is_success(), "all conditions true → 200"); + + let flags = [ + ("db_open", &signals.db_open), + ("lock_held", &signals.lock_held), + ("first_sync_completed", &signals.first_sync_completed), + ("deployment_loop_ok", &signals.deployment_loop_ok), + ]; + for (name, flag) in flags { + flag.store(false, Ordering::Release); + let degraded = client.get(&url).send().await.unwrap(); + assert_eq!( + degraded.status().as_u16(), + 503, + "{name}=false must gate readiness" + ); + flag.store(true, Ordering::Release); + let recovered = client.get(&url).send().await.unwrap(); + assert!( + recovered.status().is_success(), + "{name} restored must recover readiness" + ); + } + + cancel.cancel(); + server.await.unwrap().unwrap(); + } + + /// The launcher-passed bind override parses strictly: valid → Some, + /// absent/empty → None, garbage → loud error (a silent fallback would + /// make every probation fail by port mismatch). + #[test] + fn health_addr_override_parses_strictly() { + assert_eq!( + parse_health_addr(Some("127.0.0.1:7799".to_string())).unwrap(), + Some("127.0.0.1:7799".parse().unwrap()) + ); + assert_eq!(parse_health_addr(None).unwrap(), None); + assert_eq!(parse_health_addr(Some(String::new())).unwrap(), None); + let err = parse_health_addr(Some("not-an-addr".to_string())) + .expect_err("garbage must be a hard error"); + assert_eq!(err.code, "CONFIGURATION_ERROR"); + } } diff --git a/crates/alien-operator/src/self_update.rs b/crates/alien-operator/src/self_update.rs new file mode 100644 index 000000000..074ec99ad --- /dev/null +++ b/crates/alien-operator/src/self_update.rs @@ -0,0 +1,1043 @@ +//! os-service self-update actuator: the operator's half of the on-disk +//! handoff protocol with `alien-launcher` (`alien_core::self_update`). +//! +//! When `/v1/sync` returns `operator_target.binary`, this module downloads +//! the artifact, verifies its SHA-256 while streaming, stages it under +//! `versions//`, writes `pending.json`, and requests a graceful exit with +//! the update-handoff code (10). The LAUNCHER then performs the health-gated +//! swap; on a failed probation it rolls back and records the failure in +//! `failed/.json`, which this module (a) translates into +//! `SyncRequest.operator_update` on every sync — the launcher has no network +//! path to the manager — and (b) uses for exponential backoff before +//! re-acting on the same artifact. +//! +//! Enabled only when spawned by the launcher (`ALIEN_SELF_UPDATE=1`); +//! Kubernetes detection wins even if that env leaks into a pod. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicI32, Ordering}; +use std::sync::Once; +// Only the Unix parent-death watch (and its tests) use `Duration`; on Windows +// the launcher signals the operator directly, so the import would be unused. +#[cfg(unix)] +use std::time::Duration; + +use alien_core::self_update::{ + backoff_delay, download_dir, failed_dir, failure_path, pending_path, read_json, version_dir, + write_json_atomic, FailureRecord, PendingMarker, Version, ENV_LAUNCHER_VERSION, + ENV_SELF_UPDATE, EXIT_CODE_UPDATE_HANDOFF, +}; +use alien_core::sync::{OperatorBinaryTarget, OperatorUpdatePhase, OperatorUpdateReport}; +use alien_error::{AlienError, Context, IntoAlienError}; +use chrono::Utc; +use sha2::{Digest, Sha256}; +use tokio::io::AsyncWriteExt; +use tokio_util::sync::CancellationToken; +use tracing::{info, warn}; + +use crate::error::{ErrorData, Result}; + +/// File name of the operator binary inside `versions//`. +const OPERATOR_BINARY: &str = "alien-operator"; +/// Disk-preflight headroom over the artifact size (20%). +const PREFLIGHT_HEADROOM_DIVISOR: u64 = 5; + +// --------------------------------------------------------------------------- +// Process-exit plumbing (exit code 10 = update handoff) +// --------------------------------------------------------------------------- + +/// 0 = no special exit requested. +static REQUESTED_EXIT_CODE: AtomicI32 = AtomicI32::new(0); + +/// Ask the process to exit with the update-handoff code once the runtime +/// shuts down gracefully (the CLI checks this after `run` returns). +pub fn request_update_handoff_exit() { + REQUESTED_EXIT_CODE.store(EXIT_CODE_UPDATE_HANDOFF, Ordering::SeqCst); +} + +/// The exit code a graceful shutdown should use, if a handoff was staged. +pub fn requested_exit_code() -> Option { + match REQUESTED_EXIT_CODE.load(Ordering::SeqCst) { + 0 => None, + code => Some(code), + } +} + +// --------------------------------------------------------------------------- +// Environment gates +// --------------------------------------------------------------------------- + +/// Is the os-service actuator enabled for this process? +pub fn actuator_enabled() -> bool { + enabled_from( + std::env::var_os("KUBERNETES_SERVICE_HOST").is_some(), + std::env::var(ENV_SELF_UPDATE).ok().as_deref(), + ) +} + +/// Pure decision core: Kubernetes detection wins; otherwise the launcher's +/// explicit opt-in (`ALIEN_SELF_UPDATE=1`) is required. +fn enabled_from(in_kubernetes: bool, self_update_env: Option<&str>) -> bool { + !in_kubernetes && self_update_env == Some("1") +} + +/// The supervising launcher's version (from `ALIEN_LAUNCHER_VERSION`, set on +/// spawn), reported on every sync for the `min_launcher_version` gate. +pub fn launcher_version() -> Option { + launcher_version_from(std::env::var(ENV_LAUNCHER_VERSION).ok()) +} + +fn launcher_version_from(env: Option) -> Option { + env.filter(|value| !value.is_empty()) +} + +// --------------------------------------------------------------------------- +// Die-with-parent watch +// --------------------------------------------------------------------------- +// +// The operator must never outlive its launcher: an orphaned operator would keep +// the InstanceLock and the state DB, so a respawned launcher's fresh operator +// could not start. On Linux the launcher installs `PR_SET_PDEATHSIG` between +// fork and exec, but macOS has no such mechanism, and even on Linux there is a +// tiny fork→exec race where the launcher can die before the prctl runs. This +// poll covers both: it samples our parent pid and, the moment the kernel +// reparents us (launcher gone), triggers a graceful shutdown. The InstanceLock +// (`crate::lock`) remains the hard backstop — the new operator blocks on it +// until this orphan exits — and this watch bounds that window to one interval. + +/// How often the die-with-parent watch samples `getppid()`. Runs on a dedicated +/// OS thread (see `spawn_parent_death_watch`), so a short interval is cheap and +/// bounds how long an orphaned operator lingers. +#[cfg(unix)] +const PARENT_WATCH_INTERVAL: Duration = Duration::from_secs(1); + +/// Pure decision: the operator is spawned as a direct child of the launcher, so +/// any change from the parent pid captured at startup means the launcher has +/// exited and the kernel reparented us (to `init`/`launchd` or a subreaper). +#[cfg(unix)] +fn parent_is_gone(original_ppid: i32, current_ppid: i32) -> bool { + current_ppid != original_ppid +} + +/// The watch loop, factored out with an injectable `read_ppid` for testing. +/// +/// Synchronous by design: it runs on a dedicated OS thread (see +/// `spawn_parent_death_watch`) and MUST NOT depend on the async runtime, so it +/// sleeps with `std::thread::sleep` and polls the cancel token rather than +/// awaiting it. Returns when the parent is gone (after cancelling `cancel`) or +/// when `cancel` is tripped externally (normal shutdown), the latter within one +/// interval. +#[cfg(unix)] +fn run_parent_death_watch( + original_ppid: i32, + read_ppid: impl Fn() -> i32, + interval: Duration, + cancel: CancellationToken, +) { + loop { + std::thread::sleep(interval); + if cancel.is_cancelled() { + return; + } + let current_ppid = read_ppid(); + if parent_is_gone(original_ppid, current_ppid) { + warn!( + original_ppid, + current_ppid, "launcher parent exited; triggering graceful operator shutdown" + ); + cancel.cancel(); + return; + } + } +} + +/// Arm the die-with-parent watch on a **dedicated OS thread**. +/// +/// Deliberately NOT a tokio task: a `tokio::time::sleep` timer only fires when +/// the runtime's time driver is scheduled, so under heavy load — blocking DB +/// work, stalled sync loops occupying every worker — the timer can be *starved* +/// and fire seconds-to-minutes late, leaving an orphaned operator alive far +/// longer than the poll interval (observed: minutes, under the concurrent E2E +/// suite). A plain OS thread with `std::thread::sleep` is immune to runtime +/// starvation, which is exactly the property a die-with-parent guard needs. +/// +/// Enabled only when launcher-supervised (`ALIEN_SELF_UPDATE=1`, not +/// Kubernetes) — a no-op otherwise, since K8s uses pod lifecycle and tests / +/// manual runs have no launcher to outlive. Returns the thread handle (or +/// `None` when disabled, or if the thread cannot be spawned — the `InstanceLock` +/// backstop still bounds correctness). The thread self-exits within one interval +/// of a normal shutdown and is killed on process exit, so the handle can be +/// dropped without joining. +#[cfg(unix)] +pub fn spawn_parent_death_watch(cancel: CancellationToken) -> Option> { + if !actuator_enabled() { + return None; + } + // SAFETY: `getppid` takes no arguments, cannot fail, and only reads process + // state — it is async-signal-safe and always sound to call. + let original_ppid = unsafe { libc::getppid() }; + info!(original_ppid, "die-with-parent watch armed"); + match std::thread::Builder::new() + .name("parent-death-watch".to_string()) + .spawn(move || { + run_parent_death_watch( + original_ppid, + // SAFETY: as above. + || unsafe { libc::getppid() }, + PARENT_WATCH_INTERVAL, + cancel, + ) + }) { + Ok(handle) => Some(handle), + Err(e) => { + warn!(error = %e, "failed to spawn the die-with-parent watch thread; relying on the InstanceLock backstop"); + None + } + } +} + +/// Windows die-with-parent is a Job Object (`KILL_ON_JOB_CLOSE`), wired in the +/// Windows phase; there is no `getppid` to poll, so the watch is a no-op here. +#[cfg(not(unix))] +pub fn spawn_parent_death_watch(_cancel: CancellationToken) -> Option> { + None +} + +// --------------------------------------------------------------------------- +// The actuator +// --------------------------------------------------------------------------- + +/// Outcome of acting on an `operator_target.binary`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BinaryActuation { + /// New binary staged + `pending.json` written — the caller must trigger a + /// graceful shutdown; the process must exit with code 10. + Staged, + /// The target names the version we are already running. + AlreadyCurrent, + /// Our version is below the target's `min_supported_version` floor + /// (stepping-stone upgrade required); recorded, not retried. + RefusedFloor, + /// A previous attempt for this exact artifact failed and its backoff + /// window has not elapsed; nothing was downloaded this tick. + SkippedBackoff, + /// This attempt failed (download / digest / staging); a failure record + /// was written and the next sync after backoff will retry. + Failed, +} + +/// Act on a binary target. `own_version` is injected (the caller passes +/// `CARGO_PKG_VERSION`) so the flow is testable with arbitrary versions. +pub async fn apply_binary_target( + data_dir: &Path, + own_version: &str, + target_version: &str, + min_supported_version: &str, + binary: &OperatorBinaryTarget, +) -> Result { + log_signature_posture(); + + if target_version == own_version { + return Ok(BinaryActuation::AlreadyCurrent); + } + let target = Version::parse(target_version).map_err(|e| { + AlienError::new(ErrorData::SelfUpdateFailed { + message: format!("target version '{target_version}' is not valid semver: {e}"), + }) + })?; + + // Floor: refuse a target our version is too old to jump to. Recorded + // once per artifact (idempotent — no attempt-count churn while floored). + if below_floor(own_version, min_supported_version) { + let message = format!( + "operator {own_version} is below the min_supported_version floor \ + {min_supported_version} for target {target_version}; a stepping-stone \ + upgrade is required" + ); + warn!(%message, "refusing operator_target"); + ensure_floor_record(data_dir, &target, &binary.sha256, &message)?; + return Ok(BinaryActuation::RefusedFloor); + } + + // Backoff: an identical artifact that failed recently is not retried + // until its exponential window elapses. + if let Some(record) = read_failure(data_dir, &target)? { + if record.sha256 == binary.sha256 { + let retry_at = record.last_failed_at + + chrono::Duration::from_std(backoff_delay(record.attempts)) + .unwrap_or(chrono::Duration::zero()); + if Utc::now() < retry_at { + return Ok(BinaryActuation::SkippedBackoff); + } + } + } + + match download_verify_stage(data_dir, &target, binary).await { + Ok(()) => { + info!(version = %target, "staged new operator binary; requesting update handoff"); + Ok(BinaryActuation::Staged) + } + Err(StageFailure { message }) => { + warn!(%message, version = %target, "self-update staging attempt failed"); + record_attempt_failure(data_dir, &target, &binary.sha256, &message)?; + Ok(BinaryActuation::Failed) + } + } +} + +/// `own < floor`, with unparseable inputs treated as "no floor" (the manager +/// validates these fields; blocking all updates on a malformed optional floor +/// would be worse than skipping the check — logged for diagnosis). +fn below_floor(own_version: &str, min_supported_version: &str) -> bool { + match ( + Version::parse(own_version), + Version::parse(min_supported_version), + ) { + (Ok(own), Ok(floor)) => own < floor, + (own, floor) => { + warn!( + own_parse_ok = own.is_ok(), + floor_parse_ok = floor.is_ok(), + "unparseable version in floor check; skipping the floor gate" + ); + false + } + } +} + +/// A staging failure that should be recorded + retried after backoff (all of +/// these are `Spawn`-phase in the wire vocabulary: the swap never started). +struct StageFailure { + message: String, +} + +async fn download_verify_stage( + data_dir: &Path, + target: &Version, + binary: &OperatorBinaryTarget, +) -> std::result::Result<(), StageFailure> { + let fail = |message: String| StageFailure { message }; + + let response = reqwest::Client::new() + .get(&binary.url) + .send() + .await + .map_err(|e| fail(format!("download request failed: {e}")))?; + if !response.status().is_success() { + return Err(fail(format!( + "artifact download returned HTTP {}", + response.status() + ))); + } + + // Disk preflight: artifact size + 20% headroom must fit the free space. + if let Some(length) = response.content_length() { + let required = length.saturating_add(length / PREFLIGHT_HEADROOM_DIVISOR); + let available = fs2::available_space(data_dir) + .map_err(|e| fail(format!("disk-space preflight failed: {e}")))?; + if available < required { + return Err(fail(format!( + "not enough free disk space for the artifact: need {required} bytes \ + (incl. headroom), {available} available" + ))); + } + } + + // Stream to download/.partial, hashing as we go. + let staging_dir = download_dir(data_dir); + tokio::fs::create_dir_all(&staging_dir) + .await + .map_err(|e| fail(format!("failed to create download dir: {e}")))?; + let partial = staging_dir.join(format!("{target}.partial")); + + let stream_result = stream_to_file(response, &partial).await; + let sha256 = match stream_result { + Ok(sha256) => sha256, + Err(message) => { + remove_best_effort(&partial).await; + return Err(fail(message)); + } + }; + if sha256 != binary.sha256 { + remove_best_effort(&partial).await; + return Err(fail(format!( + "artifact digest mismatch: manifest says {}, downloaded bytes hash to {sha256}", + binary.sha256 + ))); + } + + #[cfg(feature = "enforce-signature")] + if let Err(message) = signature::verify_file(&partial, &binary.signature).await { + remove_best_effort(&partial).await; + return Err(fail(format!("signature verification failed: {message}"))); + } + + // Promote the verified download into versions//. + let dest_dir = version_dir(data_dir, target); + tokio::fs::create_dir_all(&dest_dir) + .await + .map_err(|e| fail(format!("failed to create version dir: {e}")))?; + let dest = dest_dir.join(OPERATOR_BINARY); + tokio::fs::rename(&partial, &dest) + .await + .map_err(|e| fail(format!("failed to move staged binary into place: {e}")))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(&dest, std::fs::Permissions::from_mode(0o755)) + .await + .map_err(|e| fail(format!("failed to mark staged binary executable: {e}")))?; + } + + // The handoff marker — the launcher validates this against the staged + // bytes before swapping. + let marker = PendingMarker { + version: target.clone(), + sha256, + staged_at: Utc::now(), + }; + write_json_atomic(&pending_path(data_dir), &marker) + .map_err(|e| fail(format!("failed to write pending.json: {e}")))?; + Ok(()) +} + +/// Stream the response body to `path`, returning the SHA-256 (lowercase hex) +/// of the bytes written. +async fn stream_to_file( + mut response: reqwest::Response, + path: &Path, +) -> std::result::Result { + let mut file = tokio::fs::File::create(path) + .await + .map_err(|e| format!("failed to create '{}': {e}", path.display()))?; + let mut hasher = Sha256::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|e| format!("download stream failed: {e}"))? + { + hasher.update(&chunk); + file.write_all(&chunk) + .await + .map_err(|e| format!("failed to write '{}': {e}", path.display()))?; + } + file.sync_all() + .await + .map_err(|e| format!("failed to sync '{}': {e}", path.display()))?; + Ok(format!("{:x}", hasher.finalize())) +} + +async fn remove_best_effort(path: &Path) { + if let Err(e) = tokio::fs::remove_file(path).await { + if e.kind() != std::io::ErrorKind::NotFound { + warn!(path = %path.display(), error = %e, "failed to remove partial download"); + } + } +} + +// --------------------------------------------------------------------------- +// Failure records (shared shapes; the report handoff) +// --------------------------------------------------------------------------- + +fn read_failure(data_dir: &Path, version: &Version) -> Result> { + read_json(&failure_path(data_dir, version)) + .into_alien_error() + .context(ErrorData::SelfUpdateFailed { + message: format!("failed to read failure record for {version}"), + }) +} + +/// Record a failed staging attempt: same artifact increments the count (the +/// backoff input), a different artifact starts fresh at 1. +fn record_attempt_failure( + data_dir: &Path, + version: &Version, + sha256: &str, + message: &str, +) -> Result<()> { + let attempts = match read_failure(data_dir, version)? { + Some(prior) if prior.sha256 == sha256 => prior.attempts + 1, + _ => 1, + }; + write_failure( + data_dir, + &FailureRecord { + version: version.clone(), + sha256: sha256.to_string(), + phase: OperatorUpdatePhase::Spawn, + message: message.to_string(), + attempts, + last_failed_at: Utc::now(), + }, + ) +} + +/// Floor refusals are a persistent condition, not an attempt: write the +/// record once per artifact and leave it untouched afterwards (no +/// attempt-count churn, stable backoff clock). +fn ensure_floor_record( + data_dir: &Path, + version: &Version, + sha256: &str, + message: &str, +) -> Result<()> { + if let Some(existing) = read_failure(data_dir, version)? { + if existing.sha256 == sha256 { + return Ok(()); + } + } + write_failure( + data_dir, + &FailureRecord { + version: version.clone(), + sha256: sha256.to_string(), + phase: OperatorUpdatePhase::Spawn, + message: message.to_string(), + attempts: 1, + last_failed_at: Utc::now(), + }, + ) +} + +fn write_failure(data_dir: &Path, record: &FailureRecord) -> Result<()> { + std::fs::create_dir_all(failed_dir(data_dir)) + .into_alien_error() + .context(ErrorData::SelfUpdateFailed { + message: "failed to create the failed/ records dir".to_string(), + })?; + write_json_atomic(&failure_path(data_dir, &record.version), record) + .into_alien_error() + .context(ErrorData::SelfUpdateFailed { + message: format!("failed to write failure record for {}", record.version), + }) +} + +// --------------------------------------------------------------------------- +// Report translation (SyncRequest.operator_update for os-service) +// --------------------------------------------------------------------------- + +/// Derive `operator_update` from the on-disk markers: a staged `pending.json` +/// is an in-progress update; otherwise the newest failure record (written by +/// the launcher on rollback, or by us on a staging failure) is reported until +/// convergence or a new target supersedes it. +pub fn marker_update_report(data_dir: &Path) -> Option { + if let Ok(Some(pending)) = read_json::(&pending_path(data_dir)) { + let attempt = read_json::(&failure_path(data_dir, &pending.version)) + .ok() + .flatten() + .filter(|record| record.sha256 == pending.sha256) + .map(|record| record.attempts + 1) + .unwrap_or(1); + return Some(OperatorUpdateReport::InProgress { + target_version: pending.version.as_str(), + attempt, + }); + } + + let newest = newest_failure_record(data_dir)?; + Some(OperatorUpdateReport::Failed { + target_version: newest.version.as_str(), + phase: newest.phase, + message: newest.message, + attempt: newest.attempts, + }) +} + +fn newest_failure_record(data_dir: &Path) -> Option { + let entries = std::fs::read_dir(failed_dir(data_dir)).ok()?; + let mut newest: Option = None; + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + if path.extension().is_none_or(|ext| ext != "json") { + continue; + } + match read_json::(&path) { + Ok(Some(record)) => { + if newest + .as_ref() + .is_none_or(|best| record.last_failed_at > best.last_failed_at) + { + newest = Some(record); + } + } + Ok(None) => {} + Err(e) => warn!(path = %path.display(), error = %e, "unreadable failure record"), + } + } + newest +} + +// --------------------------------------------------------------------------- +// Signature verification (feature-gated until the signing workstream lands) +// --------------------------------------------------------------------------- + +/// Log the signature posture once, loudly, so a disabled verifier can never +/// be mistaken for an enforced one. +fn log_signature_posture() { + static LOGGED: Once = Once::new(); + LOGGED.call_once(|| { + #[cfg(feature = "enforce-signature")] + info!("self-update artifact signature verification is ENFORCED (ed25519)"); + #[cfg(not(feature = "enforce-signature"))] + warn!( + "self-update artifact signature verification is DISABLED — trusting \ + SHA-256 + HTTPS only (enable the `enforce-signature` feature once \ + the signing workstream ships)" + ); + }); +} + +#[cfg(feature = "enforce-signature")] +mod signature { + use base64::Engine; + use ed25519_dalek::{Signature, Verifier, VerifyingKey}; + use std::path::Path; + + /// PLACEHOLDER — replaced when the signing workstream lands (release + /// pipeline signing, key rotation policy). All-zeros is not a valid + /// ed25519 point, so an enforcement-on build refuses every artifact + /// until the real key ships — fail closed, never silently open. + pub const PINNED_PUBKEY: [u8; 32] = [0u8; 32]; + + pub async fn verify_file(path: &Path, signature_b64: &str) -> Result<(), String> { + let bytes = tokio::fs::read(path) + .await + .map_err(|e| format!("failed to read staged artifact: {e}"))?; + verify(&bytes, signature_b64, &PINNED_PUBKEY) + } + + pub fn verify(bytes: &[u8], signature_b64: &str, pubkey: &[u8; 32]) -> Result<(), String> { + let key = VerifyingKey::from_bytes(pubkey) + .map_err(|e| format!("pinned public key is invalid: {e}"))?; + let raw = base64::engine::general_purpose::STANDARD + .decode(signature_b64) + .map_err(|e| format!("signature is not valid base64: {e}"))?; + let signature = Signature::from_slice(&raw) + .map_err(|e| format!("signature has the wrong shape: {e}"))?; + key.verify(bytes, &signature) + .map_err(|e| format!("ed25519 verification failed: {e}")) + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use axum::routing::get; + use axum::Router; + use std::sync::atomic::AtomicU32; + use std::sync::Arc; + + fn sha256_hex(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) + } + + fn version(s: &str) -> Version { + Version::parse(s).unwrap() + } + + /// Serve `bytes` at a local URL, counting hits. + async fn artifact_server(bytes: Vec) -> (String, Arc) { + let hits = Arc::new(AtomicU32::new(0)); + let app = Router::new().route( + "/artifact", + get({ + let hits = hits.clone(); + move || { + let hits = hits.clone(); + let bytes = bytes.clone(); + async move { + hits.fetch_add(1, Ordering::SeqCst); + bytes + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (format!("http://{addr}/artifact"), hits) + } + + fn binary_target(url: &str, sha256: &str) -> alien_core::sync::OperatorBinaryTarget { + alien_core::sync::OperatorBinaryTarget { + url: url.to_string(), + sha256: sha256.to_string(), + signature: String::new(), + min_launcher_version: "0.1.0".to_string(), + } + } + + /// Staging succeeds end-to-end. Default build only: with + /// `enforce-signature` the placeholder pinned key fails closed and no + /// artifact can stage (see `enforcement_fails_closed_with_placeholder_key`). + #[cfg(not(feature = "enforce-signature"))] + #[tokio::test] + async fn happy_staging_writes_pending_and_binary() { + let dir = tempfile::tempdir().unwrap(); + let artifact = b"the-new-operator-binary".to_vec(); + let sha = sha256_hex(&artifact); + let (url, hits) = artifact_server(artifact.clone()).await; + + let outcome = apply_binary_target( + dir.path(), + "1.0.0", + "9.9.9", + "1.0.0", + &binary_target(&url, &sha), + ) + .await + .expect("staging should succeed"); + + assert_eq!(outcome, BinaryActuation::Staged); + assert_eq!(hits.load(Ordering::SeqCst), 1); + + let staged = version_dir(dir.path(), &version("9.9.9")).join(OPERATOR_BINARY); + assert_eq!(std::fs::read(&staged).unwrap(), artifact); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&staged).unwrap().permissions().mode(); + assert_eq!(mode & 0o111, 0o111, "staged binary must be executable"); + } + + let pending: PendingMarker = read_json(&pending_path(dir.path())).unwrap().unwrap(); + assert_eq!(pending.version, version("9.9.9")); + assert_eq!(pending.sha256, sha); + // No partial left behind. + assert!(!download_dir(dir.path()).join("9.9.9.partial").exists()); + } + + #[tokio::test] + async fn sha_mismatch_records_spawn_failure_and_cleans_up() { + let dir = tempfile::tempdir().unwrap(); + let (url, _hits) = artifact_server(b"actual-bytes".to_vec()).await; + let claimed_sha = sha256_hex(b"different-bytes"); + + let outcome = apply_binary_target( + dir.path(), + "1.0.0", + "9.9.9", + "1.0.0", + &binary_target(&url, &claimed_sha), + ) + .await + .expect("mismatch is a recorded failure, not an Err"); + + assert_eq!(outcome, BinaryActuation::Failed); + assert!( + !version_dir(dir.path(), &version("9.9.9")) + .join(OPERATOR_BINARY) + .exists(), + "nothing staged" + ); + assert!(read_json::(&pending_path(dir.path())).unwrap().is_none()); + assert!( + !download_dir(dir.path()).join("9.9.9.partial").exists(), + "partial removed" + ); + + let record: FailureRecord = read_json(&failure_path(dir.path(), &version("9.9.9"))) + .unwrap() + .expect("failure recorded"); + assert_eq!(record.phase, OperatorUpdatePhase::Spawn); + assert_eq!(record.attempts, 1); + assert_eq!(record.sha256, claimed_sha); + assert!(record.message.contains("digest mismatch"), "{}", record.message); + } + + #[tokio::test] + async fn backoff_skips_matching_artifact_without_downloading() { + let dir = tempfile::tempdir().unwrap(); + let artifact = b"artifact".to_vec(); + let sha = sha256_hex(&artifact); + let (url, hits) = artifact_server(artifact).await; + + // A fresh failure for the SAME artifact: attempts=3 → 2m window. + write_failure( + dir.path(), + &FailureRecord { + version: version("9.9.9"), + sha256: sha.clone(), + phase: OperatorUpdatePhase::Apply, + message: "rolled back".to_string(), + attempts: 3, + last_failed_at: Utc::now(), + }, + ) + .unwrap(); + + let outcome = apply_binary_target( + dir.path(), + "1.0.0", + "9.9.9", + "1.0.0", + &binary_target(&url, &sha), + ) + .await + .unwrap(); + + assert_eq!(outcome, BinaryActuation::SkippedBackoff); + assert_eq!(hits.load(Ordering::SeqCst), 0, "no download during backoff"); + } + + #[cfg(not(feature = "enforce-signature"))] + #[tokio::test] + async fn new_artifact_ignores_old_failure_record() { + let dir = tempfile::tempdir().unwrap(); + let artifact = b"fixed-artifact".to_vec(); + let sha = sha256_hex(&artifact); + let (url, hits) = artifact_server(artifact).await; + + // A failure for the same VERSION but a different sha (old broken build). + write_failure( + dir.path(), + &FailureRecord { + version: version("9.9.9"), + sha256: sha256_hex(b"old-broken-artifact"), + phase: OperatorUpdatePhase::Apply, + message: "rolled back".to_string(), + attempts: 5, + last_failed_at: Utc::now(), + }, + ) + .unwrap(); + + let outcome = apply_binary_target( + dir.path(), + "1.0.0", + "9.9.9", + "1.0.0", + &binary_target(&url, &sha), + ) + .await + .unwrap(); + + assert_eq!(outcome, BinaryActuation::Staged, "new artifact = fresh start"); + assert_eq!(hits.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn floor_refusal_records_once_without_churn() { + let dir = tempfile::tempdir().unwrap(); + let target = binary_target("http://127.0.0.1:9/unreachable", &"ab".repeat(32)); + + let outcome = + apply_binary_target(dir.path(), "1.0.0", "9.9.9", "5.0.0", &target) + .await + .unwrap(); + assert_eq!(outcome, BinaryActuation::RefusedFloor); + + let record: FailureRecord = read_json(&failure_path(dir.path(), &version("9.9.9"))) + .unwrap() + .expect("floor refusal recorded"); + assert_eq!(record.attempts, 1); + assert!(record.message.contains("min_supported_version"), "{}", record.message); + let first_time = record.last_failed_at; + + // Second sync tick: still floored — record untouched (no churn). + let outcome = + apply_binary_target(dir.path(), "1.0.0", "9.9.9", "5.0.0", &target) + .await + .unwrap(); + assert_eq!(outcome, BinaryActuation::RefusedFloor); + let record: FailureRecord = read_json(&failure_path(dir.path(), &version("9.9.9"))) + .unwrap() + .unwrap(); + assert_eq!(record.attempts, 1); + assert_eq!(record.last_failed_at, first_time, "no rewrite while floored"); + } + + #[tokio::test] + async fn already_current_is_a_noop() { + let dir = tempfile::tempdir().unwrap(); + let target = binary_target("http://127.0.0.1:9/unreachable", &"ab".repeat(32)); + let outcome = apply_binary_target(dir.path(), "1.4.0", "1.4.0", "1.0.0", &target) + .await + .unwrap(); + assert_eq!(outcome, BinaryActuation::AlreadyCurrent); + assert!(read_json::(&pending_path(dir.path())).unwrap().is_none()); + } + + #[test] + fn marker_report_translates_failure_then_pending() { + let dir = tempfile::tempdir().unwrap(); + assert!(marker_update_report(dir.path()).is_none(), "clean store = no report"); + + write_failure( + dir.path(), + &FailureRecord { + version: version("1.4.0"), + sha256: "aa".repeat(32), + phase: OperatorUpdatePhase::Apply, + message: "probation failed".to_string(), + attempts: 2, + last_failed_at: Utc::now(), + }, + ) + .unwrap(); + + let report = marker_update_report(dir.path()).expect("failure translates"); + assert_eq!( + report, + OperatorUpdateReport::Failed { + target_version: "1.4.0".to_string(), + phase: OperatorUpdatePhase::Apply, + message: "probation failed".to_string(), + attempt: 2, + } + ); + + // A staged pending for the same artifact wins: InProgress, attempt 3. + write_json_atomic( + &pending_path(dir.path()), + &PendingMarker { + version: version("1.4.0"), + sha256: "aa".repeat(32), + staged_at: Utc::now(), + }, + ) + .unwrap(); + let report = marker_update_report(dir.path()).unwrap(); + assert_eq!( + report, + OperatorUpdateReport::InProgress { + target_version: "1.4.0".to_string(), + attempt: 3, + } + ); + } + + #[test] + fn env_gates_pure_logic() { + // Kubernetes detection wins even with the opt-in present. + assert!(!enabled_from(true, Some("1"))); + assert!(enabled_from(false, Some("1"))); + assert!(!enabled_from(false, Some("0"))); + assert!(!enabled_from(false, None)); + + assert_eq!( + launcher_version_from(Some("1.2.3".to_string())), + Some("1.2.3".to_string()) + ); + assert_eq!(launcher_version_from(Some(String::new())), None); + assert_eq!(launcher_version_from(None), None); + } + + #[test] + fn exit_code_plumbing() { + // Note: process-global — this is the only test touching it. + assert_eq!(requested_exit_code(), None); + request_update_handoff_exit(); + assert_eq!(requested_exit_code(), Some(EXIT_CODE_UPDATE_HANDOFF)); + } + + #[cfg(unix)] + #[test] + fn parent_is_gone_decision() { + assert!(!parent_is_gone(1000, 1000), "unchanged parent is still alive"); + assert!(parent_is_gone(1000, 1), "reparented to init means launcher gone"); + assert!(parent_is_gone(1000, 2000), "any change means launcher gone"); + } + + /// A changed parent (launcher gone) trips the cancellation token, driving + /// the operator's graceful shutdown. The `read_ppid` fn is injected; the + /// watch is synchronous (runs on its own OS thread in production). + #[cfg(unix)] + #[test] + fn parent_change_triggers_shutdown() { + let cancel = CancellationToken::new(); + run_parent_death_watch( + 1000, + || 1, // reparented to init: our launcher exited + Duration::from_millis(5), + cancel.clone(), + ); + assert!(cancel.is_cancelled(), "a changed parent must trigger shutdown"); + } + + /// A stable parent never trips shutdown; the watch only exits when the token + /// is cancelled externally (normal operator shutdown), within one interval. + #[cfg(unix)] + #[test] + fn stable_parent_keeps_running_until_external_cancel() { + let cancel = CancellationToken::new(); + let watcher = cancel.clone(); + let watch = std::thread::spawn(move || { + run_parent_death_watch( + 1000, + || 1000, // parent unchanged across polls + Duration::from_millis(5), + watcher, + ); + }); + + // Let several poll intervals elapse; the watch must not shut us down. + std::thread::sleep(Duration::from_millis(40)); + assert!( + !cancel.is_cancelled(), + "a stable parent must not trigger shutdown" + ); + + // External shutdown ends the watch within one interval. + cancel.cancel(); + watch.join().expect("watch thread must not panic"); + } + + /// With enforcement on and only the placeholder key available, staging + /// FAILS CLOSED: the artifact is refused, recorded, and never staged. + #[cfg(feature = "enforce-signature")] + #[tokio::test] + async fn enforcement_fails_closed_with_placeholder_key() { + let dir = tempfile::tempdir().unwrap(); + let artifact = b"the-new-operator-binary".to_vec(); + let sha = sha256_hex(&artifact); + let (url, _hits) = artifact_server(artifact).await; + + let outcome = apply_binary_target( + dir.path(), + "1.0.0", + "9.9.9", + "1.0.0", + &binary_target(&url, &sha), + ) + .await + .unwrap(); + + assert_eq!(outcome, BinaryActuation::Failed, "placeholder key refuses"); + assert!(read_json::(&pending_path(dir.path())).unwrap().is_none()); + let record: FailureRecord = read_json(&failure_path(dir.path(), &version("9.9.9"))) + .unwrap() + .expect("refusal recorded"); + assert!(record.message.contains("signature"), "{}", record.message); + } + + /// The enforcing path works end-to-end with a real (deterministic) + /// keypair, and rejects tampering. Compiled only with the feature. + #[cfg(feature = "enforce-signature")] + #[test] + fn signature_verify_roundtrip_and_tamper() { + use base64::Engine; + use ed25519_dalek::Signer; + + let signing = ed25519_dalek::SigningKey::from_bytes(&[7u8; 32]); + let verifying = signing.verifying_key().to_bytes(); + let artifact = b"artifact-bytes"; + let sig = signing.sign(artifact); + let sig_b64 = base64::engine::general_purpose::STANDARD.encode(sig.to_bytes()); + + signature::verify(artifact, &sig_b64, &verifying).expect("valid signature verifies"); + signature::verify(b"tampered-bytes", &sig_b64, &verifying) + .expect_err("tampered bytes must fail"); + signature::verify(artifact, "not-base64!!!", &verifying) + .expect_err("garbage signature must fail"); + // The placeholder all-zeros key fails closed. + signature::verify(artifact, &sig_b64, &signature::PINNED_PUBKEY) + .expect_err("placeholder key must refuse everything"); + } +} diff --git a/crates/alien-runtime/Cargo.toml b/crates/alien-runtime/Cargo.toml index 2eb6785e0..3dd9ef35f 100644 --- a/crates/alien-runtime/Cargo.toml +++ b/crates/alien-runtime/Cargo.toml @@ -67,6 +67,14 @@ cloudevents-sdk = { workspace = true, features = ["reqwest"], optional = true } port_check = { workspace = true } strip-ansi-escapes = { workspace = true } +# Die-with-parent for the user application: place it in its own kill-on-close +# Job Object so it can't outlive the operator (Windows has no pdeathsig). Same +# crate the launcher uses to supervise the operator. +[target.'cfg(windows)'.dependencies] +# `with-tokio` for the async child API (`AsyncCommandGroup`/`AsyncGroupChild`) — +# the runtime waits on the app child inside a `tokio::select!`. +command-group = { workspace = true, features = ["with-tokio"] } + [dev-dependencies] port_check = { workspace = true } alien-bindings = { workspace = true, features = ["grpc", "local"] } diff --git a/crates/alien-runtime/src/app_child.rs b/crates/alien-runtime/src/app_child.rs new file mode 100644 index 000000000..ef1577d63 --- /dev/null +++ b/crates/alien-runtime/src/app_child.rs @@ -0,0 +1,95 @@ +//! The spawned user-application process, with a per-OS die-with-parent backstop. +//! +//! The operator must never let the workload outlive it: an operator self-update +//! handoff (or any operator exit/crash) otherwise orphans the app, and a second +//! copy then starts under the swapped-in operator. The backstop differs per OS, +//! and this wrapper hides that behind one API so `runtime.rs` handles a single +//! child type: +//! +//! - **Linux:** `PR_SET_PDEATHSIG=SIGKILL` on the child (set on the `Command` +//! before spawn) kills it the instant the operator dies, plus `kill_on_drop`. +//! - **Windows:** the child runs in its own Job Object with +//! `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE` (via `command-group`, exactly as the +//! launcher supervises the operator). When the operator exits or crashes, its +//! Job handle closes and the OS terminates the app. `kill_on_drop` +//! (TerminateProcess on handle drop) is only a partial cover — it does nothing +//! on a hard crash — so the Job Object is the robust mechanism. This is a +//! nested Job (the app's Job inside the launcher's operator Job); fine on +//! Windows >= 8. +//! - **macOS:** no pdeathsig; relies on `kill_on_drop` + the graceful teardown. + +use tokio::process::{ChildStderr, ChildStdout, Command}; + +use alien_error::AlienError; + +use crate::error::{ErrorData, Result}; + +/// A spawned user-application process. On Windows it is wrapped in a kill-on-close +/// Job Object; elsewhere it is a plain child (the caller sets the Linux pdeathsig +/// and `kill_on_drop` backstops on the `Command`). +pub struct AppChild { + #[cfg(windows)] + inner: command_group::AsyncGroupChild, + #[cfg(not(windows))] + inner: tokio::process::Child, +} + +impl AppChild { + /// Spawn `cmd`. On Windows the child is placed in its own kill-on-close Job + /// Object; elsewhere it is spawned directly. + pub fn spawn(cmd: &mut Command) -> Result { + #[cfg(windows)] + let inner = { + use command_group::AsyncCommandGroup; + cmd.group_spawn().map_err(spawn_error)? + }; + #[cfg(not(windows))] + let inner = cmd.spawn().map_err(spawn_error)?; + Ok(Self { inner }) + } + + /// The OS process id, while the child is still known to the OS. + pub fn id(&self) -> Option { + self.inner.id() + } + + /// Take the piped stdout stream (once). + pub fn take_stdout(&mut self) -> Option { + self.child_mut().stdout.take() + } + + /// Take the piped stderr stream (once). + pub fn take_stderr(&mut self) -> Option { + self.child_mut().stderr.take() + } + + /// Wait for the child to exit. + pub async fn wait(&mut self) -> std::io::Result { + self.inner.wait().await + } + + /// Kill the child. On Windows this terminates the whole Job Object. + pub async fn kill(&mut self) -> std::io::Result<()> { + self.inner.kill().await + } + + /// Mutable access to the underlying tokio child, for its stdio streams. + fn child_mut(&mut self) -> &mut tokio::process::Child { + #[cfg(windows)] + { + self.inner.inner() + } + #[cfg(not(windows))] + { + &mut self.inner + } + } +} + +/// Map a spawn failure to the runtime's `ProcessFailed` error. +fn spawn_error(e: std::io::Error) -> AlienError { + AlienError::new(ErrorData::ProcessFailed { + exit_code: None, + message: format!("Failed to start application: {}", e), + }) +} diff --git a/crates/alien-runtime/src/lib.rs b/crates/alien-runtime/src/lib.rs index 650ecbfa0..dcc1d8608 100644 --- a/crates/alien-runtime/src/lib.rs +++ b/crates/alien-runtime/src/lib.rs @@ -1,3 +1,4 @@ +mod app_child; pub mod config; pub mod error; pub mod events; diff --git a/crates/alien-runtime/src/runtime.rs b/crates/alien-runtime/src/runtime.rs index fef51b85e..64b08d363 100644 --- a/crates/alien-runtime/src/runtime.rs +++ b/crates/alien-runtime/src/runtime.rs @@ -26,7 +26,7 @@ use reqwest::Url; use serde::Deserialize; use tokio::{ io::{AsyncBufReadExt, BufReader}, - process::{Child, Command}, + process::Command, signal, sync::{broadcast, OnceCell}, task::JoinHandle, @@ -34,6 +34,7 @@ use tokio::{ use tracing::{debug, error, info, warn}; use crate::{ + app_child::AppChild, config::{RuntimeConfig, TransportType}, error::{ErrorData, Result}, otlp::{flush_otlp_logs, init_otlp_logging_from_config, shutdown_otlp_logs}, @@ -301,7 +302,7 @@ async fn run_transport( control_server: Arc, app_http_port: Option, wait_until_server: Arc, - child: &mut Child, + child: &mut AppChild, mut shutdown_rx: broadcast::Receiver<()>, ) -> Result<()> { // Create separate shutdown receiver for transport @@ -423,7 +424,7 @@ fn spawn_transport( async fn handle_shutdown( shutdown_result: std::result::Result<(), broadcast::error::RecvError>, wait_until_server: Arc, - child: &mut Child, + child: &mut AppChild, ) -> Result<()> { match shutdown_result { Ok(_) | Err(broadcast::error::RecvError::Closed) => { @@ -528,7 +529,7 @@ async fn start_application( config: &RuntimeConfig, secrets: &HashMap, log_exporter: crate::config::LogExporter, -) -> Result { +) -> Result { if config.command.is_empty() { return Err(AlienError::new(ErrorData::ConfigurationInvalid { message: "Application command is empty".to_string(), @@ -587,17 +588,37 @@ async fn start_application( cmd.stdout(Stdio::piped()); cmd.stderr(Stdio::piped()); - let mut child = cmd.spawn().map_err(|e| { - AlienError::new(ErrorData::ProcessFailed { - exit_code: None, - message: format!("Failed to start application: {}", e), - }) - })?; + // Die-with-parent for the workload: the app must never outlive this operator. + // Without it, an operator self-update handoff (or any operator exit) reparents + // the app to init and leaves it running — a second copy then starts under the + // swapped-in operator. `kill_on_drop` handles the child handle being dropped; + // on Linux `PR_SET_PDEATHSIG` also covers an operator crash that skips graceful + // shutdown, and on Windows the app runs in its own kill-on-close Job Object + // (see `AppChild::spawn`) which covers the same crash case. The clean path + // still kills the child explicitly (the operator calls `shutdown_all` on + // shutdown) — these are backstops. macOS has no pdeathsig, so its crash case + // falls back to `kill_on_drop` + the graceful-shutdown teardown. + cmd.kill_on_drop(true); + #[cfg(target_os = "linux")] + // SAFETY: the closure runs in the forked child before exec; `prctl` is + // async-signal-safe and touches no shared state, meeting `pre_exec`'s contract. + unsafe { + cmd.pre_exec(|| { + // SIGKILL (not SIGTERM) — a hard backstop for an operator that died + // without draining; the graceful path already SIGKILLs via child.kill(). + if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL as libc::c_ulong) != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + + let mut child = AppChild::spawn(&mut cmd)?; info!(pid = child.id(), "Application started"); // Always capture and stream stdout/stderr - if let Some(stdout) = child.stdout.take() { + if let Some(stdout) = child.take_stdout() { let exporter = log_exporter.clone(); tokio::spawn(async move { stream_output(stdout, true, exporter).await; @@ -606,7 +627,7 @@ async fn start_application( warn!("Failed to capture stdout - will miss application logs"); } - if let Some(stderr) = child.stderr.take() { + if let Some(stderr) = child.take_stderr() { let exporter = log_exporter; tokio::spawn(async move { stream_output(stderr, false, exporter).await; @@ -787,7 +808,7 @@ async fn stream_output( /// Graceful shutdown. async fn graceful_shutdown( wait_until_server: Arc, - child: &mut Child, + child: &mut AppChild, ) -> Result<()> { info!("Initiating graceful shutdown"); diff --git a/crates/alien-test/Cargo.toml b/crates/alien-test/Cargo.toml index 02bbe7719..5673a7e9b 100644 --- a/crates/alien-test/Cargo.toml +++ b/crates/alien-test/Cargo.toml @@ -83,3 +83,9 @@ async-trait = { workspace = true } chrono = { workspace = true } test-context = { workspace = true } temp-env = { workspace = true } + +[features] +# os-service self-update E2E (launcher + operator + manager, Linux-only at +# runtime). Kept behind a feature so the heavyweight suite runs only when CI +# (or a developer) asks for it. +e2e-os-service = [] diff --git a/crates/alien-test/src/lib.rs b/crates/alien-test/src/lib.rs index a8b549a1a..c284506a9 100644 --- a/crates/alien-test/src/lib.rs +++ b/crates/alien-test/src/lib.rs @@ -30,6 +30,8 @@ pub mod managed_secret; pub mod manager; pub mod ngrok; pub mod operator; +#[cfg(feature = "e2e-os-service")] +pub mod os_service; pub mod setup; // Re-exports for convenience diff --git a/crates/alien-test/src/manager.rs b/crates/alien-test/src/manager.rs index 5780b4c6a..b0524bd81 100644 --- a/crates/alien-test/src/manager.rs +++ b/crates/alien-test/src/manager.rs @@ -125,7 +125,16 @@ impl TestManager { config: &TestConfig, platforms: &[Platform], ) -> Result> { - Self::start_inner(Some(config), platforms).await + Self::start_inner(Some(config), platforms, None).await + } + + /// Start a standalone manager whose release-manifest base points at a + /// custom location (URL, `file://`, or plain path). Used by the + /// os-service self-update E2E to serve test release manifests. + pub async fn start_with_releases_url( + releases_url: String, + ) -> Result> { + Self::start_inner(None, &[], Some(releases_url)).await } /// Start a standalone manager with no cloud credentials. @@ -133,13 +142,14 @@ impl TestManager { /// Useful for tests that only exercise the manager API surface without /// deploying to a real cloud environment. pub async fn start() -> Result> { - Self::start_inner(None, &[]).await + Self::start_inner(None, &[], None).await } /// Internal constructor shared by both `start()` and `start_with_config()`. async fn start_inner( config: Option<&TestConfig>, platforms: &[Platform], + releases_url: Option, ) -> Result> { // 1. Ephemeral state directory let state_dir = tempfile::tempdir()?; @@ -283,6 +293,9 @@ impl TestManager { // or sets differently for production use. manager_config.targets = targets; manager_config.disable_heartbeat_loop = true; + if releases_url.is_some() { + manager_config.releases_url = releases_url; + } manager_config.response_signing_key = derive_response_signing_key(&raw_token); // 7. Build the server (reuses the pre-created token store). diff --git a/crates/alien-test/src/os_service.rs b/crates/alien-test/src/os_service.rs new file mode 100644 index 000000000..9bedb17b5 --- /dev/null +++ b/crates/alien-test/src/os_service.rs @@ -0,0 +1,691 @@ +//! E2E harness for the os-service self-update flow: a real `alien-launcher` +//! supervising a real `alien-operator` (built with `test-hooks`) against an +//! in-process standalone manager, with release manifests served from a temp +//! dir and artifacts from a local HTTP server. +//! +//! Version identity trick: every "operator artifact" is a tiny wrapper script +//! that exports `ALIEN_OPERATOR_FAKE_VERSION=` (honored by `test-hooks` +//! debug builds only) and `exec`s the ONE compiled operator binary — so a +//! single build impersonates any version, each artifact has a distinct +//! sha256, and per-version behavior (e.g. "never becomes ready") is a +//! one-line script change. + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use anyhow::Context as _; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +use crate::manager::TestManager; + +/// Served artifacts: URL path → (bytes, hit counter). +type ArtifactMap = Arc, Arc)>>>; + +pub struct OsServiceRig { + pub manager: TestManager, + pub deployment_id: String, + pub data_dir: tempfile::TempDir, + releases_dir: tempfile::TempDir, + artifacts: ArtifactMap, + artifact_addr: SocketAddr, + operator_binary: PathBuf, + launcher_binary: PathBuf, + pub health_port: u16, + launcher: Option, + /// Env the launcher (and by inheritance the operator) runs with. + launcher_env: Vec<(String, String)>, + /// Kept alive so the operator can read the seeded workload's OCI (built under + /// this dir) for the rig's lifetime. `None` unless started via + /// `start_with_workload`. + _workload_oci: Option, +} + +impl OsServiceRig { + /// Boot the full rig: manager + deployment + artifact server + version + /// store seeded with `initial_version` + a running launcher. + pub async fn start(initial_version: &str) -> anyhow::Result { + Self::start_inner(initial_version, false).await + } + + /// Like [`start`], but also seeds a real workload: `alien-test-app` built to + /// a local OCI, published as a release BEFORE the deployment is created — so + /// `create_deployment` attaches it as the desired release (directly, not via + /// the `running`-gated `set_desired_release`) and the operator spawns an + /// observable app child. The OCI build needs `cargo` + `zig` (musl target). + pub async fn start_with_workload(initial_version: &str) -> anyhow::Result { + Self::start_inner(initial_version, true).await + } + + async fn start_inner(initial_version: &str, seed_workload: bool) -> anyhow::Result { + let operator_binary = find_operator_binary()?; + let launcher_binary = find_launcher_binary()?; + + // Artifact server. + let artifacts: ArtifactMap = Arc::new(Mutex::new(HashMap::new())); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let artifact_addr = listener.local_addr()?; + tokio::spawn(serve_artifacts(listener, artifacts.clone())); + + // Manager with the manifest base pointing at our temp releases dir. + let releases_dir = tempfile::tempdir()?; + let manager = TestManager::start_with_releases_url( + releases_dir.path().to_string_lossy().into_owned(), + ) + .await + .map_err(|e| anyhow::anyhow!("manager start failed: {e}"))?; + + // Seed the workload release BEFORE the deployment is created, so + // `create_deployment` attaches it as the desired release (via the direct, + // non-`running`-gated path). Kept alive on the rig for the operator. + let workload_oci = if seed_workload { + Some(publish_workload_release(&manager).await?) + } else { + None + }; + + // A deployment group is required before a deployment (the admin token + // is workspace-scoped). Create one, then a deployment in it — the + // create endpoint returns a deployment-scoped sync token. + let group: serde_json::Value = post_json( + &manager, + "/v1/deployment-groups", + serde_json::json!({ "name": "e2e-os-service" }), + ) + .await + .context("create deployment group")?; + let group_id = group["id"] + .as_str() + .context("deployment group id in response")? + .to_string(); + + let created = post_json( + &manager, + "/v1/deployments", + serde_json::json!({ + "name": "e2e-os-service", + "platform": "local", + "deploymentGroupId": group_id, + }), + ) + .await + .context("create deployment")?; + let deployment_id = created["deployment"]["id"] + .as_str() + .context("deployment id in create response")? + .to_string(); + let token = created["token"] + .as_str() + .context("token in create response")? + .to_string(); + + // Data dir: secrets + the version store seeded with the initial version. + // The operator rejects secret files that are group/world-accessible + // (argv/perms hardening), so both must be 0600. + let data_dir = tempfile::tempdir()?; + let sync_token_path = data_dir.path().join("sync-token"); + write_secret(&sync_token_path, token.as_bytes())?; + let encryption_key_path = data_dir.path().join("encryption-key"); + write_secret(&encryption_key_path, "0123456789abcdef".repeat(4).as_bytes())?; + + let rig_partial = Self { + manager, + deployment_id, + data_dir, + releases_dir, + artifacts, + artifact_addr, + operator_binary, + launcher_binary, + health_port: free_port(), + launcher: None, + launcher_env: Vec::new(), + _workload_oci: workload_oci, + }; + + let initial_script = rig_partial.wrapper_script(initial_version, &[]); + let version_dir = rig_partial.data_dir.path().join("versions").join(initial_version); + std::fs::create_dir_all(&version_dir)?; + write_executable(&version_dir.join("alien-operator"), &initial_script)?; + for name in ["current", "last-stable"] { + std::os::unix::fs::symlink( + Path::new("versions").join(initial_version), + rig_partial.data_dir.path().join(name), + )?; + } + + let mut rig = rig_partial; + rig.launcher_env = vec![ + ("PLATFORM".into(), "local".into()), + ("SYNC_URL".into(), rig.manager.url.clone()), + ( + "SYNC_TOKEN_FILE".into(), + sync_token_path.to_string_lossy().into_owned(), + ), + ( + "DATA_DIR".into(), + rig.data_dir.path().to_string_lossy().into_owned(), + ), + ( + "OPERATOR_ENCRYPTION_KEY_FILE".into(), + encryption_key_path.to_string_lossy().into_owned(), + ), + ("DEPLOYMENT_ID".into(), rig.deployment_id.clone()), + // Fast loops for the test. + ("SYNC_INTERVAL".into(), "1".into()), + ("RUST_LOG".into(), "info".into()), + ]; + rig.spawn_launcher()?; + Ok(rig) + } + + /// The wrapper-script "artifact" for a version. `extra_env` lines let a + /// version misbehave (e.g. a SYNC_URL blackhole = never becomes ready). + /// `exec ... || exit 1` keeps a broken exec an ordinary crash. + pub fn wrapper_script(&self, version: &str, extra_env: &[(&str, &str)]) -> Vec { + let mut script = String::from("#!/bin/sh\n"); + script.push_str(&format!("export ALIEN_OPERATOR_FAKE_VERSION={version}\n")); + for (key, value) in extra_env { + script.push_str(&format!("export {key}={value}\n")); + } + script.push_str(&format!("exec {} \"$@\"\n", self.operator_binary.display())); + script.into_bytes() + } + + /// A script that exits immediately — a "broken build" artifact. + pub fn broken_script(&self) -> Vec { + b"#!/bin/sh\nexit 1\n".to_vec() + } + + /// Publish a release: serve the artifact over HTTP and write the + /// per-version `manifest.json` the manager resolves targets from. + /// Returns the artifact's hit counter. + pub fn publish_release( + &self, + version: &str, + artifact: Vec, + min_launcher_version: &str, + ) -> anyhow::Result> { + let sha256 = { + use sha2::Digest; + format!("{:x}", sha2::Sha256::digest(&artifact)) + }; + let path = format!("/artifacts/operator-{version}"); + let hits = Arc::new(AtomicU32::new(0)); + self.artifacts + .lock() + .expect("artifact map lock") + .insert(path.clone(), (artifact, hits.clone())); + + let platform_key = format!("{}/{}", std::env::consts::OS, std::env::consts::ARCH); + let manifest = serde_json::json!({ + "version": version, + "minLauncherVersion": min_launcher_version, + "artifacts": { + platform_key: { + "url": format!("http://{}{path}", self.artifact_addr), + "sha256": sha256, + } + } + }); + let version_dir = self.releases_dir.path().join(version); + std::fs::create_dir_all(&version_dir)?; + std::fs::write( + version_dir.join("manifest.json"), + serde_json::to_vec_pretty(&manifest)?, + )?; + Ok(hits) + } + + /// Swap the SERVED bytes for a published version without touching its + /// manifest — the downloaded artifact then fails the digest check. + pub fn replace_artifact( + &self, + version: &str, + artifact: Vec, + ) -> anyhow::Result> { + let path = format!("/artifacts/operator-{version}"); + let hits = Arc::new(AtomicU32::new(0)); + self.artifacts + .lock() + .expect("artifact map lock") + .insert(path, (artifact, hits.clone())); + Ok(hits) + } + + /// Pin (or clear) the target operator version via the admin API. + pub async fn pin(&self, version: Option<&str>) -> anyhow::Result<()> { + let response = self + .manager + .http_client() + .put(format!( + "{}/v1/deployments/{}/target-operator-version", + self.manager.url, self.deployment_id + )) + .json(&serde_json::json!({ "targetOperatorVersion": version })) + .send() + .await?; + anyhow::ensure!( + response.status().is_success(), + "pin failed: {} — {}", + response.status(), + response.text().await.unwrap_or_default() + ); + Ok(()) + } + + /// The operator version + launcher version currently on the deployment row. + pub async fn reported_versions(&self) -> anyhow::Result<(Option, Option)> { + let row: serde_json::Value = self + .manager + .http_client() + .get(format!( + "{}/v1/deployments/{}", + self.manager.url, self.deployment_id + )) + .send() + .await? + .error_for_status()? + .json() + .await?; + let get = |key: &str| row[key].as_str().map(str::to_string); + Ok((get("operatorVersion"), get("launcherVersion"))) + } + + /// Poll until the deployment row reports `version` (or fail loudly with + /// the last observation). + pub async fn wait_for_reported_version( + &self, + version: &str, + timeout: Duration, + ) -> anyhow::Result<()> { + let deadline = Instant::now() + timeout; + let mut last = None; + while Instant::now() < deadline { + last = self.reported_versions().await?.0; + if last.as_deref() == Some(version) { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + anyhow::bail!("deployment never reported {version}; last seen {last:?}") + } + + /// Poll until the launcher has fully PROMOTED `version`: current + + /// last-stable both point at it and the transient markers are cleared. + /// The manager reports the new operator_version the instant the new + /// operator syncs, which is up to one probe-interval BEFORE the launcher + /// finishes promote cleanup — so store-state assertions must wait for + /// this, not just for the reported version. + pub async fn wait_for_promote(&self, version: &str, timeout: Duration) -> anyhow::Result<()> { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if self.current_version().as_deref() == Some(version) + && self.last_stable_version().as_deref() == Some(version) + && !self.pending_exists() + && !self.probation_exists() + { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + anyhow::bail!( + "promote to {version} never completed: current={:?} last_stable={:?} pending={} probation={}", + self.current_version(), + self.last_stable_version(), + self.pending_exists(), + self.probation_exists(), + ) + } + + // -- workload + orphan inspection (for the self-update orphan guard) ----- + + /// PIDs of the workload app processes — direct children of the operator (the + /// operator runs the worker runtime in-process, which `cmd.spawn()`s the app, + /// so the app is the operator's child / the launcher's grandchild). + pub fn app_pids(&mut self) -> anyhow::Result> { + let operator_pid = self.operator_pid()?; + Ok(child_pids(operator_pid)) + } + + /// Poll until exactly one app child runs under the current operator. + pub async fn wait_for_one_app(&mut self, timeout: Duration) -> anyhow::Result { + let deadline = Instant::now() + timeout; + let mut last = Vec::new(); + while Instant::now() < deadline { + last = self.app_pids().unwrap_or_default(); + if last.len() == 1 { + return Ok(last[0]); + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + anyhow::bail!("never converged to exactly one app child; last seen {last:?}") + } + + /// True if `pid` is still alive AND reparented to init (ppid == 1) — the + /// orphaned-workload signature. A terminated process returns false (gone, + /// not orphaned). + pub fn is_orphaned(&self, pid: u32) -> bool { + ppid_of(pid) == Some(1) + } + + // -- launcher process control ------------------------------------------ + + pub fn spawn_launcher(&mut self) -> anyhow::Result<()> { + anyhow::ensure!(self.launcher.is_none(), "launcher already running"); + let child = std::process::Command::new(&self.launcher_binary) + .args([ + "--data-dir", + &self.data_dir.path().to_string_lossy(), + "--probation-secs", + "20", + "--health-port", + &self.health_port.to_string(), + ]) + .envs(self.launcher_env.iter().map(|(k, v)| (k, v))) + .spawn() + .context("spawning alien-launcher")?; + self.launcher = Some(child); + Ok(()) + } + + /// SIGKILL the launcher (simulating a crash — no graceful anything). + pub fn kill_launcher(&mut self) -> anyhow::Result { + let mut child = self.launcher.take().context("no launcher running")?; + let pid = child.id(); + child.kill().context("SIGKILL launcher")?; + child.wait().context("reap launcher")?; + Ok(pid) + } + + /// Pid of the operator child the launcher spawned (via the process table). + pub fn operator_pid(&mut self) -> anyhow::Result { + let launcher_pid = self + .launcher + .as_ref() + .context("no launcher running")? + .id(); + let output = std::process::Command::new("pgrep") + .args(["-P", &launcher_pid.to_string()]) + .output() + .context("pgrep")?; + let stdout = String::from_utf8_lossy(&output.stdout); + stdout + .split_whitespace() + .next() + .and_then(|pid| pid.parse().ok()) + .context("launcher has no child yet") + } + + // -- store inspection ---------------------------------------------------- + + pub fn current_version(&self) -> Option { + pointer_target(&self.data_dir.path().join("current")) + } + + pub fn last_stable_version(&self) -> Option { + pointer_target(&self.data_dir.path().join("last-stable")) + } + + pub fn pending_exists(&self) -> bool { + self.data_dir.path().join("pending.json").exists() + } + + pub fn probation_exists(&self) -> bool { + self.data_dir.path().join("probation.json").exists() + } + + pub fn failure_record(&self, version: &str) -> Option { + alien_core::self_update::read_json( + &self + .data_dir + .path() + .join("failed") + .join(format!("{version}.json")), + ) + .ok() + .flatten() + } + + /// Best-effort teardown: kill the launcher (its process group dies with + /// it via the supervisor's own mechanisms + kill fallback). + pub async fn shutdown(mut self) { + if let Some(mut child) = self.launcher.take() { + let _ = child.kill(); + let _ = child.wait(); + } + self.manager.stop().await; + } +} + +/// POST JSON to the manager with the admin token; surface the body on error +/// (the SDK/`error_for_status` hides it, which makes 400s undebuggable). +async fn post_json( + manager: &TestManager, + path: &str, + body: serde_json::Value, +) -> anyhow::Result { + let response = manager + .http_client() + .post(format!("{}{path}", manager.url)) + .json(&body) + .send() + .await?; + let status = response.status(); + let text = response.text().await.unwrap_or_default(); + anyhow::ensure!(status.is_success(), "POST {path} → {status}: {text}"); + Ok(serde_json::from_str(&text).unwrap_or(serde_json::Value::Null)) +} + +fn pointer_target(path: &Path) -> Option { + std::fs::read_link(path) + .ok()? + .file_name() + .map(|name| name.to_string_lossy().into_owned()) +} + +/// Direct children of `pid` (via `pgrep -P`). +fn child_pids(pid: u32) -> Vec { + std::process::Command::new("pgrep") + .args(["-P", &pid.to_string()]) + .output() + .ok() + .map(|o| { + String::from_utf8_lossy(&o.stdout) + .split_whitespace() + .filter_map(|p| p.parse().ok()) + .collect() + }) + .unwrap_or_default() +} + +/// Parent PID of `pid`, or None if the process no longer exists. +fn ppid_of(pid: u32) -> Option { + let out = std::process::Command::new("ps") + .args(["-o", "ppid=", "-p", &pid.to_string()]) + .output() + .ok()?; + if !out.status.success() { + return None; + } + String::from_utf8_lossy(&out.stdout).trim().parse().ok() +} + +/// Build `alien-test-app` (a Rust SDK worker) into a local OCI via `alien-build` +/// (cargo + zig for the musl target — no bun/docker) and publish it as a `local` +/// release. When called before the deployment is created, `create_deployment` +/// attaches it as the deployment's desired release. Returns the OCI build dir — +/// keep it alive for the operator to read. +async fn publish_workload_release(manager: &TestManager) -> anyhow::Result { + use alien_build::settings::{BuildSettings, PlatformBuildSettings}; + use alien_core::permissions::{PermissionProfile, PermissionsConfig}; + use alien_core::{ + BinaryTarget, ResourceLifecycle, Stack, ToolchainConfig, Worker, WorkerCode, + }; + + let app_src = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .context("crates dir")? + .join("alien-test-app"); + anyhow::ensure!( + app_src.is_dir(), + "alien-test-app source not found at {}", + app_src.display() + ); + + let worker = Worker::new("app".to_string()) + .code(WorkerCode::Source { + src: app_src.to_string_lossy().into_owned(), + toolchain: ToolchainConfig::Rust { + binary_name: "alien-test-app".to_string(), + }, + }) + .memory_mb(512) + .timeout_seconds(60) + .environment(HashMap::new()) + .permissions("execution".to_string()) + .build(); + let permissions = PermissionsConfig { + profiles: [("execution".to_string(), PermissionProfile::default())] + .into_iter() + .collect(), + management: Default::default(), + }; + let stack = Stack::new("orphan-guard".to_string()) + .add(worker, ResourceLifecycle::Live) + .permissions(permissions) + .build(); + + let build_dir = tempfile::tempdir()?; + let settings = BuildSettings { + output_directory: build_dir.path().to_string_lossy().into_owned(), + platform: PlatformBuildSettings::Local {}, + targets: Some(vec![BinaryTarget::current_os()]), + cache_url: None, + override_base_image: None, + debug_mode: false, + }; + let built = alien_build::build_stack(stack, &settings) + .await + .context("build alien-test-app into a local OCI")?; + + // The release's `local` stack is the built stack — its Worker.code is now a + // local OCI Image path the operator extracts + runs. + let stack_json = serde_json::to_value(&built).context("serialize built stack")?; + let resp = post_json( + manager, + "/v1/releases", + serde_json::json!({ "stack": { "local": stack_json }, "projectId": "default" }), + ) + .await?; + anyhow::ensure!( + resp.get("id").and_then(|v| v.as_str()).is_some(), + "release response missing id: {resp}" + ); + Ok(build_dir) +} + +fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .expect("bind :0") + .local_addr() + .expect("local addr") + .port() +} + +fn write_secret(path: &Path, bytes: &[u8]) -> anyhow::Result<()> { + std::fs::write(path, bytes)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?; + } + Ok(()) +} + +fn write_executable(path: &Path, bytes: &[u8]) -> anyhow::Result<()> { + std::fs::write(path, bytes)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755))?; + } + Ok(()) +} + +fn find_operator_binary() -> anyhow::Result { + if let Ok(path) = std::env::var("ALIEN_OPERATOR_BINARY") { + let path = PathBuf::from(path); + anyhow::ensure!(path.is_file(), "ALIEN_OPERATOR_BINARY does not exist"); + return Ok(path); + } + workspace_binary("alien-operator") +} + +fn find_launcher_binary() -> anyhow::Result { + if let Ok(path) = std::env::var("ALIEN_LAUNCHER_BINARY") { + let path = PathBuf::from(path); + anyhow::ensure!(path.is_file(), "ALIEN_LAUNCHER_BINARY does not exist"); + return Ok(path); + } + workspace_binary("alien-launcher") +} + +fn workspace_binary(name: &str) -> anyhow::Result { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../target/debug") + .join(name); + anyhow::ensure!( + path.is_file(), + "{name} not found at {} — build it first (the E2E CI job does) or set the \ + ALIEN_{}_BINARY env var", + path.display(), + name.trim_start_matches("alien-").to_uppercase() + ); + Ok(path) +} + +/// Minimal static-artifact HTTP server: GET → mapped bytes + hit count. +async fn serve_artifacts(listener: tokio::net::TcpListener, artifacts: ArtifactMap) { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let artifacts = artifacts.clone(); + tokio::spawn(async move { + let mut buf = vec![0u8; 4096]; + let n = stream.read(&mut buf).await.unwrap_or(0); + let request = String::from_utf8_lossy(&buf[..n]); + let path = request + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .unwrap_or("/") + .to_string(); + let hit = artifacts.lock().expect("artifact map lock").get(&path).map( + |(bytes, hits)| { + hits.fetch_add(1, Ordering::SeqCst); + bytes.clone() + }, + ); + let response = match hit { + Some(bytes) => { + let mut response = format!( + "HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", + bytes.len() + ) + .into_bytes(); + response.extend_from_slice(&bytes); + response + } + None => b"HTTP/1.1 404 Not Found\r\ncontent-length: 0\r\nconnection: close\r\n\r\n" + .to_vec(), + }; + let _ = stream.write_all(&response).await; + let _ = stream.shutdown().await; + }); + } +} diff --git a/crates/alien-test/tests/e2e_os_service.rs b/crates/alien-test/tests/e2e_os_service.rs new file mode 100644 index 000000000..1da1f27ee --- /dev/null +++ b/crates/alien-test/tests/e2e_os_service.rs @@ -0,0 +1,312 @@ +//! E2E: the os-service self-update flow — a real launcher supervising a real +//! operator against an in-process manager. Runs on Linux and macOS (the +//! launcher runs as a direct child — no systemd/launchd needed, so the suite is +//! hermetic and identical on both); the Windows job joins in its phase. Run +//! with: +//! +//! ```sh +//! cargo build -p alien-launcher && cargo build -p alien-operator --features test-hooks +//! cargo test -p alien-test --features e2e-os-service --test e2e_os_service +//! ``` +#![cfg(all( + feature = "e2e-os-service", + any(target_os = "linux", target_os = "macos") +))] + +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use alien_core::sync::OperatorUpdatePhase; +use alien_test::os_service::OsServiceRig; + +const CONVERGE: Duration = Duration::from_secs(60); + +/// 1. Happy update: pin → download → stage → exit(10) → swap → probation → +/// promote → converge, with the store fully cleaned up. +#[tokio::test] +async fn happy_update_converges_and_promotes() { + let rig = OsServiceRig::start("1.0.0").await.expect("rig"); + rig.wait_for_reported_version("1.0.0", CONVERGE) + .await + .expect("initial version reported"); + let (_, launcher_version) = rig.reported_versions().await.expect("row"); + assert!( + launcher_version.is_some(), + "the launcher's version must be reported for the min-launcher gate" + ); + + let hits = rig + .publish_release("2.0.0", rig.wrapper_script("2.0.0", &[]), "0.1.0") + .expect("publish"); + rig.pin(Some("2.0.0")).await.expect("pin"); + + rig.wait_for_reported_version("2.0.0", CONVERGE) + .await + .expect("converge to 2.0.0"); + rig.wait_for_promote("2.0.0", CONVERGE) + .await + .expect("promote completes"); + assert_eq!(hits.load(Ordering::SeqCst), 1, "exactly one download"); + assert!( + !rig.data_dir.path().join("state-snapshots/1.0.0").exists(), + "snapshot dropped after promote" + ); + rig.shutdown().await; +} + +/// 2+3. Rollback + backoff: a broken artifact (exits immediately) swaps, dies +/// in probation, rolls back to the old version, records the failure — and is +/// NOT re-downloaded while its backoff window runs. +#[tokio::test] +async fn broken_artifact_rolls_back_and_backs_off() { + let rig = OsServiceRig::start("1.0.0").await.expect("rig"); + rig.wait_for_reported_version("1.0.0", CONVERGE) + .await + .expect("initial version reported"); + + let hits = rig + .publish_release("3.0.0", rig.broken_script(), "0.1.0") + .expect("publish"); + rig.pin(Some("3.0.0")).await.expect("pin"); + + // Wait for the rollback: failure record written + old version back. + let deadline = Instant::now() + CONVERGE; + loop { + if let Some(record) = rig.failure_record("3.0.0") { + assert_eq!(record.phase, OperatorUpdatePhase::Apply, "died in probation"); + assert!(record.attempts >= 1); + break; + } + assert!(Instant::now() < deadline, "rollback never recorded"); + tokio::time::sleep(Duration::from_millis(500)).await; + } + assert_eq!(rig.current_version().as_deref(), Some("1.0.0"), "rolled back"); + assert_eq!(rig.last_stable_version().as_deref(), Some("1.0.0")); + + // Backoff: the manager keeps advertising, but the same artifact must not + // be re-downloaded before its 30s window (we observe a 10s slice). + let downloads_after_rollback = hits.load(Ordering::SeqCst); + assert_eq!(downloads_after_rollback, 1, "one download before the rollback"); + tokio::time::sleep(Duration::from_secs(10)).await; + assert_eq!( + hits.load(Ordering::SeqCst), + downloads_after_rollback, + "no re-download inside the backoff window" + ); + // The old operator is alive and reporting throughout. + rig.wait_for_reported_version("1.0.0", CONVERGE) + .await + .expect("old version still reporting"); + rig.pin(None).await.expect("unpin"); + rig.shutdown().await; +} + +/// 4. Digest mismatch: nothing staged, no swap, a spawn-phase failure record. +#[tokio::test] +async fn digest_mismatch_never_swaps() { + let rig = OsServiceRig::start("1.0.0").await.expect("rig"); + rig.wait_for_reported_version("1.0.0", CONVERGE) + .await + .expect("initial version reported"); + + // Publish a valid release, then swap the SERVED bytes: the manifest's + // sha256 no longer matches what the download returns. + rig.publish_release("4.0.0", rig.wrapper_script("4.0.0", &[]), "0.1.0") + .expect("publish"); + let tampered_hits = rig + .replace_artifact("4.0.0", rig.broken_script()) + .expect("tamper"); + rig.pin(Some("4.0.0")).await.expect("pin"); + + let deadline = Instant::now() + CONVERGE; + loop { + if let Some(record) = rig.failure_record("4.0.0") { + assert_eq!(record.phase, OperatorUpdatePhase::Spawn, "failed pre-swap"); + assert!(record.message.contains("digest"), "{}", record.message); + break; + } + assert!(Instant::now() < deadline, "mismatch never recorded"); + tokio::time::sleep(Duration::from_millis(500)).await; + } + assert!(tampered_hits.load(Ordering::SeqCst) >= 1); + assert_eq!(rig.current_version().as_deref(), Some("1.0.0"), "no swap"); + assert!(!rig.pending_exists(), "nothing staged"); + assert!( + !rig.data_dir.path().join("versions/4.0.0").exists(), + "no version dir for the refused artifact" + ); + rig.pin(None).await.expect("unpin"); + rig.shutdown().await; +} + +/// 5. Launcher killed mid-probation: the restart classifies the store and +/// converges (the never-ready target eventually rolls back). +#[tokio::test] +async fn launcher_crash_mid_probation_recovers() { + let mut rig = OsServiceRig::start("1.0.0").await.expect("rig"); + rig.wait_for_reported_version("1.0.0", CONVERGE) + .await + .expect("initial version reported"); + + // 5.0.0 points its sync at a blackhole → never completes a sync → never + // ready → probation runs its full window. + let _hits = rig + .publish_release( + "5.0.0", + rig.wrapper_script("5.0.0", &[("SYNC_URL", "http://127.0.0.1:1")]), + "0.1.0", + ) + .expect("publish"); + rig.pin(Some("5.0.0")).await.expect("pin"); + + // Wait until the swap is mid-probation, then SIGKILL the launcher. + let deadline = Instant::now() + CONVERGE; + while !rig.probation_exists() { + assert!(Instant::now() < deadline, "probation never started"); + tokio::time::sleep(Duration::from_millis(250)).await; + } + rig.kill_launcher().expect("kill mid-probation"); + + // Restart: startup classification resumes the gate; the target never + // becomes ready → rollback; the old operator reports again. + rig.spawn_launcher().expect("restart launcher"); + let deadline = Instant::now() + CONVERGE; + loop { + if rig.failure_record("5.0.0").is_some() + && rig.current_version().as_deref() == Some("1.0.0") + && !rig.probation_exists() + && !rig.pending_exists() + { + break; + } + assert!( + Instant::now() < deadline, + "no convergence after the mid-probation crash: current={:?} probation={} pending={}", + rig.current_version(), + rig.probation_exists(), + rig.pending_exists() + ); + tokio::time::sleep(Duration::from_millis(500)).await; + } + rig.wait_for_reported_version("1.0.0", CONVERGE) + .await + .expect("old version reporting after recovery"); + rig.pin(None).await.expect("unpin"); + rig.shutdown().await; +} + +/// 6. Die-with-parent: SIGKILL the launcher in steady state → the operator +/// must die with it (PDEATHSIG); a restarted launcher supervises cleanly. +#[tokio::test] +async fn launcher_death_kills_the_operator() { + let mut rig = OsServiceRig::start("1.0.0").await.expect("rig"); + rig.wait_for_reported_version("1.0.0", CONVERGE) + .await + .expect("initial version reported"); + + let operator_pid = rig.operator_pid().expect("operator child pid"); + rig.kill_launcher().expect("kill launcher"); + + // The kernel delivers SIGTERM to the operator on parent death. + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let alive = std::process::Command::new("kill") + .args(["-0", &operator_pid.to_string()]) + .status() + .expect("kill -0") + .success(); + if !alive { + break; + } + assert!( + Instant::now() < deadline, + "operator {operator_pid} survived its launcher — die-with-parent failed" + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } + + // A fresh launcher takes over without lock contention. + rig.spawn_launcher().expect("restart launcher"); + rig.wait_for_reported_version("1.0.0", CONVERGE) + .await + .expect("supervision resumed"); + rig.shutdown().await; +} + +/// 7. The frozen-launcher gate: a manifest demanding a newer launcher is +/// withheld by the manager — nothing is downloaded, nothing changes. +#[tokio::test] +async fn min_launcher_gate_withholds_the_target() { + let rig = OsServiceRig::start("1.0.0").await.expect("rig"); + rig.wait_for_reported_version("1.0.0", CONVERGE) + .await + .expect("initial version reported"); + + let hits = rig + .publish_release("6.0.0", rig.wrapper_script("6.0.0", &[]), "999.0.0") + .expect("publish"); + rig.pin(Some("6.0.0")).await.expect("pin"); + + // Several sync cycles: the gate must withhold the target entirely. + tokio::time::sleep(Duration::from_secs(8)).await; + assert_eq!(hits.load(Ordering::SeqCst), 0, "no download — redeploy required"); + assert_eq!(rig.current_version().as_deref(), Some("1.0.0")); + assert!(!rig.pending_exists()); + assert!(rig.failure_record("6.0.0").is_none(), "no failed attempt either"); + let (reported, _) = rig.reported_versions().await.expect("row"); + assert_eq!(reported.as_deref(), Some("1.0.0")); + rig.pin(None).await.expect("unpin"); + rig.shutdown().await; +} + +/// 8. Orphan guard: an operator self-update swap must NOT leave the user +/// workload orphaned. The old operator terminates its app child before handing +/// off (`shutdown_all` in the operator + `kill_on_drop`/`PR_SET_PDEATHSIG` in +/// alien-runtime), and the new operator re-spawns exactly ONE app — never two, +/// never one reparented to init. Regression guard for the two-app orphan bug. +#[tokio::test] +async fn app_child_not_orphaned_after_swap() { + // Seed a real workload up front (release published before the deployment is + // created, so the operator picks it up) → the operator spawns an app child. + let mut rig = OsServiceRig::start_with_workload("1.0.0") + .await + .expect("rig with workload"); + rig.wait_for_reported_version("1.0.0", CONVERGE) + .await + .expect("initial version reported"); + let app_before = rig + .wait_for_one_app(CONVERGE) + .await + .expect("workload app running under the operator"); + + // Trigger the operator self-update swap 1.0.0 → 2.0.0. + rig.publish_release("2.0.0", rig.wrapper_script("2.0.0", &[]), "0.1.0") + .expect("publish 2.0.0"); + rig.pin(Some("2.0.0")).await.expect("pin 2.0.0"); + rig.wait_for_reported_version("2.0.0", CONVERGE) + .await + .expect("converge to 2.0.0"); + rig.wait_for_promote("2.0.0", CONVERGE) + .await + .expect("promote 2.0.0"); + + // The fix under test: the pre-swap app was TERMINATED by the old operator's + // shutdown — never reparented to init (the orphan bug left it running). + assert!( + !rig.is_orphaned(app_before), + "pre-swap app {app_before} survived as an orphan (reparented to init) after the swap" + ); + + // The new operator re-synced and re-spawned exactly ONE app — a fresh + // process (the old one was killed, not adopted). + let app_after = rig + .wait_for_one_app(CONVERGE) + .await + .expect("exactly one app under the new operator"); + assert_ne!( + app_after, app_before, + "expected a fresh app child after the swap, not the old one" + ); + + rig.shutdown().await; +}