test: automate two-node HA failover and TCP survival (#534)

.github/workflows/ha-failover.yml

@@ -0,0 +1,36 @@
+name: HA failover qualification
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  two-node:
+    name: CARP, pfsync, and TCP survival
+    runs-on: [self-hosted, freebsd-ha]
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+
+      - name: Validate inventory
+        env:
+          AIFW_HA_INVENTORY: ${{ secrets.AIFW_HA_INVENTORY }}
+        run: |
+          set -eu
+          test -n "$AIFW_HA_INVENTORY"
+          printf '%s' "$AIFW_HA_INVENTORY" > /tmp/aifw-ha-inventory.json
+          python3 -m json.tool /tmp/aifw-ha-inventory.json >/dev/null
+
+      - name: Exercise controlled failover
+        run: python3 scripts/ha-failover-ci.py /tmp/aifw-ha-inventory.json --results artifacts/ha
+
+      - name: Upload raw HA evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: ha-failover-${{ github.sha }}
+          path: artifacts/ha/
+          if-no-files-found: warn
+          retention-days: 30

.gitignore

@@ -14,11 +14,10 @@ target
 **/mutants.out*/
 
 # RustRover
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+# or merged into this file. For a nuclear option (not recommended) uncomment:
+# .idea/
 
 # Next.js
 .next/
@@ -31,3 +30,4 @@ freebsd/ui-export/
 
 # CLAUDE.md is project policy and is tracked in the repo — do not ignore it.
 Cargo.lock
+scripts/__pycache__/

scripts/ha-failover-ci.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Two-node AiFw HA failover and TCP-session acceptance harness.
+
+The harness is provider-neutral: nodes and endpoints may be FreeBSD VMs,
+bhyve guests, or physical lab hosts.  A small JSON inventory supplies SSH
+targets and interface names.  Commands, CARP/pfsync state, timing samples,
+and traffic results are retained for CI artifact upload.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import shlex
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+
+
+def run(argv: list[str], *, check: bool = True, timeout: int = 30) -> subprocess.CompletedProcess[str]:
+    result = subprocess.run(argv, text=True, capture_output=True, timeout=timeout, check=False)
+    if check and result.returncode:
+        raise RuntimeError(f"command failed ({result.returncode}): {shlex.join(argv)}\n{result.stderr}")
+    return result
+
+
+@dataclass(frozen=True)
+class Host:
+    name: str
+    ssh: str
+
+    def command(self, command: str, *, check: bool = True, timeout: int = 30) -> subprocess.CompletedProcess[str]:
+        return run(
+            ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", self.ssh, command],
+            check=check,
+            timeout=timeout,
+        )
+
+
+def capture(host: Host, command: str, path: pathlib.Path) -> None:
+    result = host.command(command, check=False)
+    path.write_text(f"$ {command}\nexit={result.returncode}\n{result.stdout}\n{result.stderr}")
+
+
+def role(host: Host, carp_iface: str) -> str:
+    output = host.command(f"ifconfig {shlex.quote(carp_iface)}").stdout
+    for value in ("MASTER", "BACKUP", "INIT"):
+        if f"carp: {value}" in output:
+            return value
+    return "UNKNOWN"
+
+
+def await_single_master(nodes: list[Host], carp_iface: str, deadline: float) -> tuple[Host, Host, float]:
+    started = time.monotonic()
+    while time.monotonic() < deadline:
+        states = [(node, role(node, carp_iface)) for node in nodes]
+        masters = [node for node, state in states if state == "MASTER"]
+        backups = [node for node, state in states if state == "BACKUP"]
+        if len(masters) == 1 and len(backups) == 1:
+            return masters[0], backups[0], time.monotonic() - started
+        time.sleep(0.2)
+    raise RuntimeError("HA pair did not converge to exactly one MASTER and one BACKUP")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("inventory", type=pathlib.Path)
+    parser.add_argument("--results", type=pathlib.Path, default=pathlib.Path("ha-results"))
+    args = parser.parse_args()
+    cfg = json.loads(args.inventory.read_text())
+    args.results.mkdir(parents=True, exist_ok=True)
+
+    nodes = [Host("node-a", cfg["node_a"]), Host("node-b", cfg["node_b"])]
+    client = Host("client", cfg["client"])
+    server = Host("server", cfg["server"])
+    carp_iface = cfg["carp_interface"]
+    pfsync_iface = cfg["pfsync_interface"]
+    vip = cfg["lan_vip"]
+    server_addr = cfg["server_address"]
+    port = int(cfg.get("tcp_port", 18080))
+
+    # Preflight both kernels and retain the authoritative state.
+    for node in nodes:
+        capture(node, f"ifconfig {shlex.quote(carp_iface)}", args.results / f"{node.name}-carp-before.txt")
+        capture(node, f"ifconfig {shlex.quote(pfsync_iface)}", args.results / f"{node.name}-pfsync-before.txt")
+        capture(node, "pfctl -ss -vv", args.results / f"{node.name}-states-before.txt")
+    master, backup, initial_convergence = await_single_master(nodes, carp_iface, time.monotonic() + 20)
+
+    # The server emits an unbounded byte stream.  The client records it to a
+    # file across failover; byte growth after the transition proves the same
+    # TCP connection survived (a reconnecting probe cannot satisfy this).
+    server.command(f"pkill -f 'nc -l {port}' || true", check=False)
+    server.command(f"sh -c 'yes aifw-ha | nc -l {port} >/dev/null 2>&1 &'", check=True)
+    remote_out = cfg.get("client_output", "/tmp/aifw-ha-stream.out")
+    client.command(f"rm -f {shlex.quote(remote_out)}; sh -c 'nc {shlex.quote(server_addr)} {port} >{shlex.quote(remote_out)} 2>/tmp/aifw-ha-stream.err &'", check=True)
+    time.sleep(2)
+    size_before = int(client.command(f"stat -f %z {shlex.quote(remote_out)}").stdout.strip())
+    if size_before <= 0:
+        raise RuntimeError("long-lived TCP stream did not start")
+
+    # Demotion is controlled and reversible; unlike killing pf/pfsync it
+    # exercises the normal CARP transition and health-check recovery path.
+    transition_start = time.monotonic()
+    master.command("sysctl net.inet.carp.demotion=240")
+    new_master, new_backup, election_elapsed = await_single_master(nodes, carp_iface, time.monotonic() + 20)
+    if new_master == master:
+        raise RuntimeError("CARP master did not move to the peer after demotion")
+    time.sleep(2)
+    size_after = int(client.command(f"stat -f %z {shlex.quote(remote_out)}").stdout.strip())
+    if size_after <= size_before:
+        raise RuntimeError("long-lived TCP session stopped transferring during failover")
+
+    # Restore/preempt and verify the pair converges without split brain.
+    master.command("sysctl net.inet.carp.demotion=0")
+    final_master, _, recovery_elapsed = await_single_master(nodes, carp_iface, time.monotonic() + 30)
+    transition_elapsed = time.monotonic() - transition_start
+
+    for node in nodes:
+        capture(node, f"ifconfig {shlex.quote(carp_iface)}", args.results / f"{node.name}-carp-after.txt")
+        capture(node, f"ifconfig {shlex.quote(pfsync_iface)}", args.results / f"{node.name}-pfsync-after.txt")
+        capture(node, "pfctl -ss -vv", args.results / f"{node.name}-states-after.txt")
+        capture(node, "netstat -s -p pfsync", args.results / f"{node.name}-pfsync-counters.txt")
+
+    result = {
+        "status": "passed",
+        "initial_master": master.name,
+        "failover_master": new_master.name,
+        "final_master": final_master.name,
+        "initial_convergence_seconds": round(initial_convergence, 3),
+        "election_seconds": round(election_elapsed, 3),
+        "recovery_seconds": round(recovery_elapsed, 3),
+        "scenario_seconds": round(transition_elapsed, 3),
+        "tcp_bytes_before": size_before,
+        "tcp_bytes_after": size_after,
+        "tcp_session_survived": True,
+        "single_master": True,
+    }
+    (args.results / "result.json").write_text(json.dumps(result, indent=2) + "\n")
+    print(json.dumps(result, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except (RuntimeError, subprocess.TimeoutExpired, KeyError, ValueError) as error:
+        print(f"FAIL: {error}", file=sys.stderr)
+        raise SystemExit(1)

scripts/ha-inventory.example.json

@@ -0,0 +1,11 @@
+{
+  "node_a": "root@aifw-a",
+  "node_b": "root@aifw-b",
+  "client": "root@ha-client",
+  "server": "root@ha-server",
+  "carp_interface": "vtnet0",
+  "pfsync_interface": "pfsync0",
+  "lan_vip": "192.0.2.1",
+  "server_address": "198.51.100.2",
+  "tcp_port": 18080
+}