Alpha.12 verified ext4 repair escalation

build/usb/includes.chroot/etc/systemd/system/rigos-state.service

@@ -8,7 +8,7 @@ Before=local-fs.target rigos-state-ready.service
 Type=oneshot
 ExecStartPre=/usr/bin/systemd-tmpfiles --create /usr/lib/tmpfiles.d/rigos.conf
 ExecStart=/usr/local/sbin/rigos-state-orchestrate
-TimeoutStartSec=12min
+TimeoutStartSec=20min
 RemainAfterExit=yes
 
 [Install]

build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate

@@ -15,6 +15,7 @@ BOOT_ID = Path("/proc/sys/kernel/random/boot_id")
 PARTUUID_ROOT = Path("/dev/disk/by-partuuid")
 PARTUUID_RE = re.compile(r"^[0-9A-Fa-f-]{1,128}$")
 FILESYSTEM_TIMEOUT_SECONDS = 300
+E2FSCK_UNCORRECTED_EXIT = 4
 MAX_COMMAND_OUTPUT = 64 * 1024
 
 
@@ -111,6 +112,15 @@ def verified_state_device() -> tuple[Path | None, str | None]:
     return device, None
 
 
+def revalidate_state_device(expected: Path) -> tuple[Path | None, str | None]:
+    observed, error = verified_state_device()
+    if error is not None or observed is None:
+        return None, error or "verified state device is unavailable"
+    if observed != expected:
+        return None, "verified state device changed during repair"
+    return observed, None
+
+
 def mark_repair_required(message: str) -> None:
     status = read_json(STATUS)
     attestation = read_json(ATTESTATION)
@@ -164,33 +174,56 @@ def run_repair_command(argv: list[str], accepted: tuple[int, ...]) -> tuple[bool
     return True, None
 
 
+def repair_ext4(device: Path, failure_prefix: str) -> bool:
+    checked, check_failure = run_repair_command(
+        ["/usr/sbin/e2fsck", "-f", "-p", str(device)], (0, 1)
+    )
+    if checked:
+        return True
+    uncorrected = f"e2fsck: exit {E2FSCK_UNCORRECTED_EXIT}"
+    if check_failure is None or not check_failure.startswith(uncorrected):
+        mark_repair_required(f"{failure_prefix}: {check_failure}")
+        return False
+
+    revalidated, error = revalidate_state_device(device)
+    if error is not None or revalidated is None:
+        mark_repair_required(error or "verified state device is unavailable before repair")
+        return False
+    repaired, repair_failure = run_repair_command(
+        ["/usr/sbin/e2fsck", "-f", "-y", str(revalidated)], (0, 1)
+    )
+    if not repaired:
+        mark_repair_required(f"automatic ext4 repair failed: {repair_failure}")
+        return False
+
+    revalidated, error = revalidate_state_device(device)
+    if error is not None or revalidated is None:
+        mark_repair_required(error or "verified state device is unavailable after repair")
+        return False
+    return True
+
+
 def forced_check() -> bool:
     device, error = verified_state_device()
     if error is not None or device is None:
         mark_repair_required(error or "verified state device is unavailable")
         return False
-    ok, failure = run_repair_command(
-        ["/usr/sbin/e2fsck", "-f", "-p", str(device)], (0, 1)
-    )
-    if ok:
-        return True
-    mark_repair_required(f"forced ext4 check failed: {failure}")
-    return False
+    return repair_ext4(device, "forced ext4 check failed")
 
 
 def complete_resize_after_timeout() -> bool:
     device, error = verified_state_device()
     if error is not None or device is None:
         mark_repair_required(error or "verified state device is unavailable")
         return False
-    checked, check_failure = run_repair_command(
-        ["/usr/sbin/e2fsck", "-f", "-p", str(device)], (0, 1)
-    )
-    if not checked:
-        mark_repair_required(f"post-timeout ext4 check failed: {check_failure}")
+    if not repair_ext4(device, "post-timeout ext4 check failed"):
+        return False
+    revalidated, error = revalidate_state_device(device)
+    if error is not None or revalidated is None:
+        mark_repair_required(error or "verified state device is unavailable before resize")
         return False
     resized, resize_failure = run_repair_command(
-        ["/usr/sbin/resize2fs", str(device)], (0,)
+        ["/usr/sbin/resize2fs", str(revalidated)], (0,)
     )
     if not resized:
         mark_repair_required(f"state filesystem resize failed: {resize_failure}")
@@ -220,7 +253,12 @@ def main() -> int:
             retried_missing_device = True
             time.sleep(1)
             continue
-        if recoverable and ("e2fsck -f" in message or "e2fsck: timeout" in message):
+        e2fsck_failure = (
+            "e2fsck -f" in message
+            or "e2fsck: timeout" in message
+            or f"e2fsck: exit {E2FSCK_UNCORRECTED_EXIT}" in message
+        )
+        if recoverable and e2fsck_failure:
             if repaired_filesystem:
                 mark_repair_required(
                     "state core ext4 check remained incomplete after bounded repair"

build/usb/version.env

@@ -1,8 +1,8 @@
-RIGOS_PRODUCT_VERSION=0.0.4-alpha.11
-RIGOS_IMAGE_VERSION=0.0.4-alpha.11
+RIGOS_PRODUCT_VERSION=0.0.4-alpha.12
+RIGOS_IMAGE_VERSION=0.0.4-alpha.12
 RIGOS_IMAGE_ID=rigos-usb-amd64
 RIGOS_IMAGE_CHANNEL=alpha
-RIGOS_BUILD_ORDINAL=11
+RIGOS_BUILD_ORDINAL=12
 RIGOS_XMRIG_VERSION=6.26.0
 RIGOS_XMRIG_ARCHIVE_SHA256=fc6f8ae5f64e4f17481f7e3be29a1c56949f216a998414188003eae1db20c9e5
 RIGOS_XMRIG_BINARY_SHA256=b20f39fc00d242e706b6c30367ad811c676e0575050a4ec2f30104b696944b49

crates/rigos-config/tests/diagnostic_ssh.rs

@@ -55,8 +55,8 @@ fn diagnostic_ssh_keeps_host_identity_mandatory_without_requiring_ready_state()
         );
     }
 
-    assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.11"));
-    assert!(version.contains("RIGOS_BUILD_ORDINAL=11"));
+    assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.12"));
+    assert!(version.contains("RIGOS_BUILD_ORDINAL=12"));
 }
 
 #[test]

crates/rigos-config/tests/firstboot_image_gate.rs

@@ -10,15 +10,15 @@ fn repo_file(path: &str) -> String {
 }
 
 #[test]
-fn alpha11_build_runs_source_and_exact_image_firstboot_gates() {
+fn alpha12_build_runs_source_and_exact_image_firstboot_gates() {
     let version = repo_file("build/usb/version.env");
     let entrypoint = repo_file("scripts/build-usb-image-entrypoint.sh");
     let verifier = repo_file("scripts/verify-firstboot-image.sh");
     let hook = repo_file("build/usb/hooks/010-rigos.chroot");
 
-    assert!(version.contains("RIGOS_PRODUCT_VERSION=0.0.4-alpha.11"));
-    assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.11"));
-    assert!(version.contains("RIGOS_BUILD_ORDINAL=11"));
+    assert!(version.contains("RIGOS_PRODUCT_VERSION=0.0.4-alpha.12"));
+    assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.12"));
+    assert!(version.contains("RIGOS_BUILD_ORDINAL=12"));
 
     assert!(entrypoint.contains("--test firstboot_tty"));
     assert!(entrypoint.contains("--test state_resize_recovery"));

crates/rigos-config/tests/state_resize_recovery.rs

@@ -26,11 +26,15 @@ fn state_resize_timeout_has_a_bounded_verified_recovery_path() {
 
     for required in [
         "FILESYSTEM_TIMEOUT_SECONDS = 300",
+        "E2FSCK_UNCORRECTED_EXIT = 4",
+        "def repair_ext4(device: Path, failure_prefix: str) -> bool:",
         "def complete_resize_after_timeout() -> bool:",
         "timeout=FILESYSTEM_TIMEOUT_SECONDS",
-        "post-timeout ext4 check failed",
+        "automatic ext4 repair failed",
         "state filesystem resize failed",
         "resize2fs: timeout",
+        "[\"/usr/sbin/e2fsck\", \"-f\", \"-y\"",
+        "f\"e2fsck: exit {E2FSCK_UNCORRECTED_EXIT}\" in message",
     ] {
         assert!(
             orchestrator.contains(required),
@@ -41,15 +45,15 @@ fn state_resize_timeout_has_a_bounded_verified_recovery_path() {
             "exact-image verifier is missing: {required}"
         );
     }
-    assert!(service.contains("TimeoutStartSec=12min"));
+    assert!(service.contains("TimeoutStartSec=20min"));
     assert!(image_verifier.contains("losetup --find --show --read-only"));
     assert!(image_verifier.contains("mount -o ro"));
     assert!(!image_verifier.contains("mount -o rw"));
     assert!(entrypoint.contains("bash ./scripts/verify-state-recovery-image.sh \"$image\""));
 }
 
 #[test]
-fn resize_timeout_recovery_retries_core_and_reports_failed_repair_truthfully() {
+fn resize_timeout_recovery_retries_core_and_repairs_only_e2fsck_exit_four() {
     let unique = SystemTime::now()
         .duration_since(UNIX_EPOCH)
         .unwrap()
@@ -78,25 +82,70 @@ statuses = [
     {'outcome': 'ready', 'message': None},
 ]
 core_calls = []
-repair_calls = []
+resize_calls = []
 g['run_core'] = lambda: core_calls.append(True) or 0
 def read_json(path):
     if path == g['STATUS']:
         return statuses.pop(0)
     return {}
 g['read_json'] = read_json
-g['complete_resize_after_timeout'] = lambda: repair_calls.append(True) or True
+g['complete_resize_after_timeout'] = lambda: resize_calls.append(True) or True
 assert namespace['main']() == 0
 assert len(core_calls) == 2
-assert repair_calls == [True]
+assert resize_calls == [True]
 assert statuses == []
 
-# A failed long repair is not reclassified as limited capacity.
+# The exact exit-4 message emitted by the core reaches the verified fsck path.
+statuses = [
+    {
+        'outcome': 'limited_capacity',
+        'message': 'bounded command failed: e2fsck: exit 4: unexpected inconsistency',
+    },
+    {'outcome': 'ready', 'message': None},
+]
+core_calls.clear()
+forced_calls = []
+g['forced_check'] = lambda: forced_calls.append(True) or True
+assert namespace['main']() == 0
+assert len(core_calls) == 2
+assert forced_calls == [True]
+assert statuses == []
+
+# Preen exit 4 is the only condition that escalates to bounded -y repair.
 recorded = []
+calls = []
 g['verified_state_device'] = lambda: (Path('/dev/test-state'), None)
-results = iter([(True, None), (False, 'resize2fs: timeout after 300s')])
-g['run_repair_command'] = lambda _argv, _accepted: next(results)
+results = iter([
+    (False, 'e2fsck: exit 4: unexpected inconsistency'),
+    (True, None),
+    (True, None),
+])
+def run_repair(argv, accepted):
+    calls.append((argv, accepted))
+    return next(results)
+g['run_repair_command'] = run_repair
 g['mark_repair_required'] = lambda message: recorded.append(message)
+assert namespace['complete_resize_after_timeout']() is True
+assert [call[0][0:3] for call in calls] == [
+    ['/usr/sbin/e2fsck', '-f', '-p'],
+    ['/usr/sbin/e2fsck', '-f', '-y'],
+    ['/usr/sbin/resize2fs', '/dev/test-state'],
+]
+assert recorded == []
+
+# Operational e2fsck failures never escalate to -y.
+recorded.clear()
+calls.clear()
+results = iter([(False, 'e2fsck: exit 8: operational error')])
+assert namespace['complete_resize_after_timeout']() is False
+assert len(calls) == 1
+assert calls[0][0][0:3] == ['/usr/sbin/e2fsck', '-f', '-p']
+assert recorded == ['post-timeout ext4 check failed: e2fsck: exit 8: operational error']
+
+# A failed resize remains repair_required and is never reclassified as capacity.
+recorded.clear()
+calls.clear()
+results = iter([(True, None), (False, 'resize2fs: timeout after 300s')])
 assert namespace['complete_resize_after_timeout']() is False
 assert recorded == ['state filesystem resize failed: resize2fs: timeout after 300s']
 "#;

scripts/verify-state-recovery-image.sh

@@ -60,21 +60,34 @@ release="$root/etc/rigos-release"
 [[ -f "$release" ]] || die 'release metadata is missing'
 
 python3 -m py_compile "$orchestrator" "$recovery" "$gate"
-grep -Fqx 'VERSION_ID="0.0.4-alpha.11"' "$release" \
-    || die 'embedded alpha.11 version is missing'
-grep -Fqx 'TimeoutStartSec=12min' "$state_service" \
-    || die 'state service recovery window is missing'
+grep -Fqx 'VERSION_ID="0.0.4-alpha.12"' "$release" \
+    || die 'embedded alpha.12 version is missing'
+grep -Fqx 'TimeoutStartSec=20min' "$state_service" \
+    || die 'state service full repair window is missing'
 
 for required in \
     'FILESYSTEM_TIMEOUT_SECONDS = 300' \
+    'E2FSCK_UNCORRECTED_EXIT = 4' \
+    'def repair_ext4(device: Path, failure_prefix: str) -> bool:' \
     'def complete_resize_after_timeout() -> bool:' \
     'timeout=FILESYSTEM_TIMEOUT_SECONDS' \
-    'post-timeout ext4 check failed' \
+    'automatic ext4 repair failed' \
     'state filesystem resize failed' \
-    'resize2fs: timeout'
+    'resize2fs: timeout' \
+    '["/usr/sbin/e2fsck", "-f", "-y"' \
+    'f"e2fsck: exit {E2FSCK_UNCORRECTED_EXIT}" in message'
 do
     grep -Fq "$required" "$orchestrator" \
-        || die "state resize recovery contract is missing: $required"
+        || die "state repair contract is missing: $required"
+done
+
+for required in \
+    'def revalidate_state_device(expected: Path)' \
+    'verified state device changed during repair' \
+    'if check_failure is None or not check_failure.startswith(uncorrected):'
+do
+    grep -Fq "$required" "$orchestrator" \
+        || die "state repair safety boundary is missing: $required"
 done
 
 for required in \