diff --git a/build/usb/includes.chroot/etc/systemd/system/rigos-state.service b/build/usb/includes.chroot/etc/systemd/system/rigos-state.service index aedfef6..017dc20 100644 --- a/build/usb/includes.chroot/etc/systemd/system/rigos-state.service +++ b/build/usb/includes.chroot/etc/systemd/system/rigos-state.service @@ -8,7 +8,7 @@ Before=local-fs.target rigos-state-ready.service Type=oneshot ExecStartPre=/usr/bin/systemd-tmpfiles --create /usr/lib/tmpfiles.d/rigos.conf ExecStart=/usr/local/sbin/rigos-state-orchestrate -TimeoutStartSec=12min +TimeoutStartSec=20min RemainAfterExit=yes [Install] diff --git a/build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate b/build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate index 2963114..3568003 100644 --- a/build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate +++ b/build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate @@ -15,6 +15,7 @@ BOOT_ID = Path("/proc/sys/kernel/random/boot_id") PARTUUID_ROOT = Path("/dev/disk/by-partuuid") PARTUUID_RE = re.compile(r"^[0-9A-Fa-f-]{1,128}$") FILESYSTEM_TIMEOUT_SECONDS = 300 +E2FSCK_UNCORRECTED_EXIT = 4 MAX_COMMAND_OUTPUT = 64 * 1024 @@ -111,6 +112,15 @@ def verified_state_device() -> tuple[Path | None, str | None]: return device, None +def revalidate_state_device(expected: Path) -> tuple[Path | None, str | None]: + observed, error = verified_state_device() + if error is not None or observed is None: + return None, error or "verified state device is unavailable" + if observed != expected: + return None, "verified state device changed during repair" + return observed, None + + def mark_repair_required(message: str) -> None: status = read_json(STATUS) attestation = read_json(ATTESTATION) @@ -164,18 +174,41 @@ def run_repair_command(argv: list[str], accepted: tuple[int, ...]) -> tuple[bool return True, None +def repair_ext4(device: Path, failure_prefix: str) -> bool: + checked, check_failure = run_repair_command( + ["/usr/sbin/e2fsck", "-f", "-p", str(device)], (0, 1) + ) + if checked: + return True + uncorrected = f"e2fsck: exit {E2FSCK_UNCORRECTED_EXIT}" + if check_failure is None or not check_failure.startswith(uncorrected): + mark_repair_required(f"{failure_prefix}: {check_failure}") + return False + + revalidated, error = revalidate_state_device(device) + if error is not None or revalidated is None: + mark_repair_required(error or "verified state device is unavailable before repair") + return False + repaired, repair_failure = run_repair_command( + ["/usr/sbin/e2fsck", "-f", "-y", str(revalidated)], (0, 1) + ) + if not repaired: + mark_repair_required(f"automatic ext4 repair failed: {repair_failure}") + return False + + revalidated, error = revalidate_state_device(device) + if error is not None or revalidated is None: + mark_repair_required(error or "verified state device is unavailable after repair") + return False + return True + + def forced_check() -> bool: device, error = verified_state_device() if error is not None or device is None: mark_repair_required(error or "verified state device is unavailable") return False - ok, failure = run_repair_command( - ["/usr/sbin/e2fsck", "-f", "-p", str(device)], (0, 1) - ) - if ok: - return True - mark_repair_required(f"forced ext4 check failed: {failure}") - return False + return repair_ext4(device, "forced ext4 check failed") def complete_resize_after_timeout() -> bool: @@ -183,14 +216,14 @@ def complete_resize_after_timeout() -> bool: if error is not None or device is None: mark_repair_required(error or "verified state device is unavailable") return False - checked, check_failure = run_repair_command( - ["/usr/sbin/e2fsck", "-f", "-p", str(device)], (0, 1) - ) - if not checked: - mark_repair_required(f"post-timeout ext4 check failed: {check_failure}") + if not repair_ext4(device, "post-timeout ext4 check failed"): + return False + revalidated, error = revalidate_state_device(device) + if error is not None or revalidated is None: + mark_repair_required(error or "verified state device is unavailable before resize") return False resized, resize_failure = run_repair_command( - ["/usr/sbin/resize2fs", str(device)], (0,) + ["/usr/sbin/resize2fs", str(revalidated)], (0,) ) if not resized: mark_repair_required(f"state filesystem resize failed: {resize_failure}") @@ -220,7 +253,12 @@ def main() -> int: retried_missing_device = True time.sleep(1) continue - if recoverable and ("e2fsck -f" in message or "e2fsck: timeout" in message): + e2fsck_failure = ( + "e2fsck -f" in message + or "e2fsck: timeout" in message + or f"e2fsck: exit {E2FSCK_UNCORRECTED_EXIT}" in message + ) + if recoverable and e2fsck_failure: if repaired_filesystem: mark_repair_required( "state core ext4 check remained incomplete after bounded repair" diff --git a/build/usb/version.env b/build/usb/version.env index aa1456c..d70e22f 100644 --- a/build/usb/version.env +++ b/build/usb/version.env @@ -1,8 +1,8 @@ -RIGOS_PRODUCT_VERSION=0.0.4-alpha.11 -RIGOS_IMAGE_VERSION=0.0.4-alpha.11 +RIGOS_PRODUCT_VERSION=0.0.4-alpha.12 +RIGOS_IMAGE_VERSION=0.0.4-alpha.12 RIGOS_IMAGE_ID=rigos-usb-amd64 RIGOS_IMAGE_CHANNEL=alpha -RIGOS_BUILD_ORDINAL=11 +RIGOS_BUILD_ORDINAL=12 RIGOS_XMRIG_VERSION=6.26.0 RIGOS_XMRIG_ARCHIVE_SHA256=fc6f8ae5f64e4f17481f7e3be29a1c56949f216a998414188003eae1db20c9e5 RIGOS_XMRIG_BINARY_SHA256=b20f39fc00d242e706b6c30367ad811c676e0575050a4ec2f30104b696944b49 diff --git a/crates/rigos-config/tests/diagnostic_ssh.rs b/crates/rigos-config/tests/diagnostic_ssh.rs index b0c9fb0..c3a4760 100644 --- a/crates/rigos-config/tests/diagnostic_ssh.rs +++ b/crates/rigos-config/tests/diagnostic_ssh.rs @@ -55,8 +55,8 @@ fn diagnostic_ssh_keeps_host_identity_mandatory_without_requiring_ready_state() ); } - assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.11")); - assert!(version.contains("RIGOS_BUILD_ORDINAL=11")); + assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.12")); + assert!(version.contains("RIGOS_BUILD_ORDINAL=12")); } #[test] diff --git a/crates/rigos-config/tests/firstboot_image_gate.rs b/crates/rigos-config/tests/firstboot_image_gate.rs index a4c4b7d..0ef5143 100644 --- a/crates/rigos-config/tests/firstboot_image_gate.rs +++ b/crates/rigos-config/tests/firstboot_image_gate.rs @@ -10,15 +10,15 @@ fn repo_file(path: &str) -> String { } #[test] -fn alpha11_build_runs_source_and_exact_image_firstboot_gates() { +fn alpha12_build_runs_source_and_exact_image_firstboot_gates() { let version = repo_file("build/usb/version.env"); let entrypoint = repo_file("scripts/build-usb-image-entrypoint.sh"); let verifier = repo_file("scripts/verify-firstboot-image.sh"); let hook = repo_file("build/usb/hooks/010-rigos.chroot"); - assert!(version.contains("RIGOS_PRODUCT_VERSION=0.0.4-alpha.11")); - assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.11")); - assert!(version.contains("RIGOS_BUILD_ORDINAL=11")); + assert!(version.contains("RIGOS_PRODUCT_VERSION=0.0.4-alpha.12")); + assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.12")); + assert!(version.contains("RIGOS_BUILD_ORDINAL=12")); assert!(entrypoint.contains("--test firstboot_tty")); assert!(entrypoint.contains("--test state_resize_recovery")); diff --git a/crates/rigos-config/tests/state_resize_recovery.rs b/crates/rigos-config/tests/state_resize_recovery.rs index 13473f7..55cc160 100644 --- a/crates/rigos-config/tests/state_resize_recovery.rs +++ b/crates/rigos-config/tests/state_resize_recovery.rs @@ -26,11 +26,15 @@ fn state_resize_timeout_has_a_bounded_verified_recovery_path() { for required in [ "FILESYSTEM_TIMEOUT_SECONDS = 300", + "E2FSCK_UNCORRECTED_EXIT = 4", + "def repair_ext4(device: Path, failure_prefix: str) -> bool:", "def complete_resize_after_timeout() -> bool:", "timeout=FILESYSTEM_TIMEOUT_SECONDS", - "post-timeout ext4 check failed", + "automatic ext4 repair failed", "state filesystem resize failed", "resize2fs: timeout", + "[\"/usr/sbin/e2fsck\", \"-f\", \"-y\"", + "f\"e2fsck: exit {E2FSCK_UNCORRECTED_EXIT}\" in message", ] { assert!( orchestrator.contains(required), @@ -41,7 +45,7 @@ fn state_resize_timeout_has_a_bounded_verified_recovery_path() { "exact-image verifier is missing: {required}" ); } - assert!(service.contains("TimeoutStartSec=12min")); + assert!(service.contains("TimeoutStartSec=20min")); assert!(image_verifier.contains("losetup --find --show --read-only")); assert!(image_verifier.contains("mount -o ro")); assert!(!image_verifier.contains("mount -o rw")); @@ -49,7 +53,7 @@ fn state_resize_timeout_has_a_bounded_verified_recovery_path() { } #[test] -fn resize_timeout_recovery_retries_core_and_reports_failed_repair_truthfully() { +fn resize_timeout_recovery_retries_core_and_repairs_only_e2fsck_exit_four() { let unique = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() @@ -78,25 +82,70 @@ statuses = [ {'outcome': 'ready', 'message': None}, ] core_calls = [] -repair_calls = [] +resize_calls = [] g['run_core'] = lambda: core_calls.append(True) or 0 def read_json(path): if path == g['STATUS']: return statuses.pop(0) return {} g['read_json'] = read_json -g['complete_resize_after_timeout'] = lambda: repair_calls.append(True) or True +g['complete_resize_after_timeout'] = lambda: resize_calls.append(True) or True assert namespace['main']() == 0 assert len(core_calls) == 2 -assert repair_calls == [True] +assert resize_calls == [True] assert statuses == [] -# A failed long repair is not reclassified as limited capacity. +# The exact exit-4 message emitted by the core reaches the verified fsck path. +statuses = [ + { + 'outcome': 'limited_capacity', + 'message': 'bounded command failed: e2fsck: exit 4: unexpected inconsistency', + }, + {'outcome': 'ready', 'message': None}, +] +core_calls.clear() +forced_calls = [] +g['forced_check'] = lambda: forced_calls.append(True) or True +assert namespace['main']() == 0 +assert len(core_calls) == 2 +assert forced_calls == [True] +assert statuses == [] + +# Preen exit 4 is the only condition that escalates to bounded -y repair. recorded = [] +calls = [] g['verified_state_device'] = lambda: (Path('/dev/test-state'), None) -results = iter([(True, None), (False, 'resize2fs: timeout after 300s')]) -g['run_repair_command'] = lambda _argv, _accepted: next(results) +results = iter([ + (False, 'e2fsck: exit 4: unexpected inconsistency'), + (True, None), + (True, None), +]) +def run_repair(argv, accepted): + calls.append((argv, accepted)) + return next(results) +g['run_repair_command'] = run_repair g['mark_repair_required'] = lambda message: recorded.append(message) +assert namespace['complete_resize_after_timeout']() is True +assert [call[0][0:3] for call in calls] == [ + ['/usr/sbin/e2fsck', '-f', '-p'], + ['/usr/sbin/e2fsck', '-f', '-y'], + ['/usr/sbin/resize2fs', '/dev/test-state'], +] +assert recorded == [] + +# Operational e2fsck failures never escalate to -y. +recorded.clear() +calls.clear() +results = iter([(False, 'e2fsck: exit 8: operational error')]) +assert namespace['complete_resize_after_timeout']() is False +assert len(calls) == 1 +assert calls[0][0][0:3] == ['/usr/sbin/e2fsck', '-f', '-p'] +assert recorded == ['post-timeout ext4 check failed: e2fsck: exit 8: operational error'] + +# A failed resize remains repair_required and is never reclassified as capacity. +recorded.clear() +calls.clear() +results = iter([(True, None), (False, 'resize2fs: timeout after 300s')]) assert namespace['complete_resize_after_timeout']() is False assert recorded == ['state filesystem resize failed: resize2fs: timeout after 300s'] "#; diff --git a/scripts/verify-state-recovery-image.sh b/scripts/verify-state-recovery-image.sh index fab7186..0e514a2 100644 --- a/scripts/verify-state-recovery-image.sh +++ b/scripts/verify-state-recovery-image.sh @@ -60,21 +60,34 @@ release="$root/etc/rigos-release" [[ -f "$release" ]] || die 'release metadata is missing' python3 -m py_compile "$orchestrator" "$recovery" "$gate" -grep -Fqx 'VERSION_ID="0.0.4-alpha.11"' "$release" \ - || die 'embedded alpha.11 version is missing' -grep -Fqx 'TimeoutStartSec=12min' "$state_service" \ - || die 'state service recovery window is missing' +grep -Fqx 'VERSION_ID="0.0.4-alpha.12"' "$release" \ + || die 'embedded alpha.12 version is missing' +grep -Fqx 'TimeoutStartSec=20min' "$state_service" \ + || die 'state service full repair window is missing' for required in \ 'FILESYSTEM_TIMEOUT_SECONDS = 300' \ + 'E2FSCK_UNCORRECTED_EXIT = 4' \ + 'def repair_ext4(device: Path, failure_prefix: str) -> bool:' \ 'def complete_resize_after_timeout() -> bool:' \ 'timeout=FILESYSTEM_TIMEOUT_SECONDS' \ - 'post-timeout ext4 check failed' \ + 'automatic ext4 repair failed' \ 'state filesystem resize failed' \ - 'resize2fs: timeout' + 'resize2fs: timeout' \ + '["/usr/sbin/e2fsck", "-f", "-y"' \ + 'f"e2fsck: exit {E2FSCK_UNCORRECTED_EXIT}" in message' do grep -Fq "$required" "$orchestrator" \ - || die "state resize recovery contract is missing: $required" + || die "state repair contract is missing: $required" +done + +for required in \ + 'def revalidate_state_device(expected: Path)' \ + 'verified state device changed during repair' \ + 'if check_failure is None or not check_failure.startswith(uncorrected):' +do + grep -Fq "$required" "$orchestrator" \ + || die "state repair safety boundary is missing: $required" done for required in \