Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ ATTESTATION = RUNTIME / "boot-device.json"
BOOT_ID = Path("/proc/sys/kernel/random/boot_id")
PARTUUID_ROOT = Path("/dev/disk/by-partuuid")
PARTUUID_RE = re.compile(r"^[0-9A-Fa-f-]{1,128}$")
E2FSCK_EXIT_RE = re.compile(
r"\be2fsck: exit (?:(?P<plain>[0-9]+)|Some\((?P<option>[0-9]+)\))(?=[:\s]|$)"
)
FILESYSTEM_TIMEOUT_SECONDS = 300
E2FSCK_UNCORRECTED_EXIT = 4
MAX_COMMAND_OUTPUT = 64 * 1024
Expand Down Expand Up @@ -62,6 +65,14 @@ def properties(output: str) -> dict[str, str]:
return result


def e2fsck_exit_code(message: str) -> int | None:
match = E2FSCK_EXIT_RE.search(message)
if match is None:
return None
value = match.group("plain") or match.group("option")
return int(value)


def verified_state_device() -> tuple[Path | None, str | None]:
attestation = read_json(ATTESTATION)
try:
Expand Down Expand Up @@ -180,8 +191,10 @@ def repair_ext4(device: Path, failure_prefix: str) -> bool:
)
if checked:
return True
uncorrected = f"e2fsck: exit {E2FSCK_UNCORRECTED_EXIT}"
if check_failure is None or not check_failure.startswith(uncorrected):
if (
check_failure is None
or e2fsck_exit_code(check_failure) != E2FSCK_UNCORRECTED_EXIT
):
mark_repair_required(f"{failure_prefix}: {check_failure}")
return False

Expand Down Expand Up @@ -253,10 +266,11 @@ def main() -> int:
retried_missing_device = True
time.sleep(1)
continue
e2fsck_exit = e2fsck_exit_code(message)
e2fsck_failure = (
"e2fsck -f" in message
or "e2fsck: timeout" in message
or f"e2fsck: exit {E2FSCK_UNCORRECTED_EXIT}" in message
or e2fsck_exit == E2FSCK_UNCORRECTED_EXIT
)
if recoverable and e2fsck_failure:
if repaired_filesystem:
Expand Down
6 changes: 3 additions & 3 deletions build/usb/version.env
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
RIGOS_PRODUCT_VERSION=0.0.4-alpha.12
RIGOS_IMAGE_VERSION=0.0.4-alpha.12
RIGOS_PRODUCT_VERSION=0.0.4-alpha.13
RIGOS_IMAGE_VERSION=0.0.4-alpha.13
RIGOS_IMAGE_ID=rigos-usb-amd64
RIGOS_IMAGE_CHANNEL=alpha
RIGOS_BUILD_ORDINAL=12
RIGOS_BUILD_ORDINAL=13
RIGOS_XMRIG_VERSION=6.26.0
RIGOS_XMRIG_ARCHIVE_SHA256=fc6f8ae5f64e4f17481f7e3be29a1c56949f216a998414188003eae1db20c9e5
RIGOS_XMRIG_BINARY_SHA256=b20f39fc00d242e706b6c30367ad811c676e0575050a4ec2f30104b696944b49
4 changes: 2 additions & 2 deletions crates/rigos-config/tests/diagnostic_ssh.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ fn diagnostic_ssh_keeps_host_identity_mandatory_without_requiring_ready_state()
);
}

assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.12"));
assert!(version.contains("RIGOS_BUILD_ORDINAL=12"));
assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.13"));
assert!(version.contains("RIGOS_BUILD_ORDINAL=13"));
}

#[test]
Expand Down
8 changes: 4 additions & 4 deletions crates/rigos-config/tests/firstboot_image_gate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,15 @@ fn repo_file(path: &str) -> String {
}

#[test]
fn alpha12_build_runs_source_and_exact_image_firstboot_gates() {
fn alpha13_build_runs_source_and_exact_image_firstboot_gates() {
let version = repo_file("build/usb/version.env");
let entrypoint = repo_file("scripts/build-usb-image-entrypoint.sh");
let verifier = repo_file("scripts/verify-firstboot-image.sh");
let hook = repo_file("build/usb/hooks/010-rigos.chroot");

assert!(version.contains("RIGOS_PRODUCT_VERSION=0.0.4-alpha.12"));
assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.12"));
assert!(version.contains("RIGOS_BUILD_ORDINAL=12"));
assert!(version.contains("RIGOS_PRODUCT_VERSION=0.0.4-alpha.13"));
assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.13"));
assert!(version.contains("RIGOS_BUILD_ORDINAL=13"));

assert!(entrypoint.contains("--test firstboot_tty"));
assert!(entrypoint.contains("--test state_resize_recovery"));
Expand Down
28 changes: 25 additions & 3 deletions crates/rigos-config/tests/state_resize_recovery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,16 @@ fn state_resize_timeout_has_a_bounded_verified_recovery_path() {
for required in [
"FILESYSTEM_TIMEOUT_SECONDS = 300",
"E2FSCK_UNCORRECTED_EXIT = 4",
"E2FSCK_EXIT_RE = re.compile",
"def e2fsck_exit_code(message: str) -> int | None:",
"def repair_ext4(device: Path, failure_prefix: str) -> bool:",
"def complete_resize_after_timeout() -> bool:",
"timeout=FILESYSTEM_TIMEOUT_SECONDS",
"automatic ext4 repair failed",
"state filesystem resize failed",
"resize2fs: timeout",
"[\"/usr/sbin/e2fsck\", \"-f\", \"-y\"",
"f\"e2fsck: exit {E2FSCK_UNCORRECTED_EXIT}\" in message",
"e2fsck_exit == E2FSCK_UNCORRECTED_EXIT",
] {
assert!(
orchestrator.contains(required),
Expand Down Expand Up @@ -75,6 +77,12 @@ g['ATTESTATION'] = root / 'boot-device.json'
g['BOOT_ID'] = root / 'boot-id'
g['BOOT_ID'].write_text('boot-test\n', encoding='ascii')

# The parser accepts both the Python helper and Rust Option exit formats only.
assert g['e2fsck_exit_code']('e2fsck: exit 4: inconsistency') == 4
assert g['e2fsck_exit_code']('e2fsck: exit Some(4): inconsistency') == 4
assert g['e2fsck_exit_code']('e2fsck: exit Some(8): operational error') == 8
assert g['e2fsck_exit_code']('e2fsck: exit None: signal') is None

# A core resize timeout is completed under the longer verified repair budget,
# then core is rerun to perform the normal mount and initialization path.
statuses = [
Expand All @@ -95,11 +103,11 @@ assert len(core_calls) == 2
assert resize_calls == [True]
assert statuses == []

# The exact exit-4 message emitted by the core reaches the verified fsck path.
# The exact Rust exit Some(4) message emitted physically reaches verified fsck.
statuses = [
{
'outcome': 'limited_capacity',
'message': 'bounded command failed: e2fsck: exit 4: unexpected inconsistency',
'message': 'bounded command failed: e2fsck: exit Some(4): unexpected inconsistency',
},
{'outcome': 'ready', 'message': None},
]
Expand All @@ -111,6 +119,20 @@ assert len(core_calls) == 2
assert forced_calls == [True]
assert statuses == []

# Rust exit Some(8) remains operational failure and never enters repair.
statuses = [
{
'outcome': 'limited_capacity',
'message': 'bounded command failed: e2fsck: exit Some(8): operational error',
},
]
core_calls.clear()
forced_calls.clear()
assert namespace['main']() == 0
assert len(core_calls) == 1
assert forced_calls == []
assert statuses == []

# Preen exit 4 is the only condition that escalates to bounded -y repair.
recorded = []
calls = []
Expand Down
11 changes: 7 additions & 4 deletions scripts/verify-state-recovery-image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -60,31 +60,34 @@ release="$root/etc/rigos-release"
[[ -f "$release" ]] || die 'release metadata is missing'

python3 -m py_compile "$orchestrator" "$recovery" "$gate"
grep -Fqx 'VERSION_ID="0.0.4-alpha.12"' "$release" \
|| die 'embedded alpha.12 version is missing'
grep -Fqx 'VERSION_ID="0.0.4-alpha.13"' "$release" \
|| die 'embedded alpha.13 version is missing'
grep -Fqx 'TimeoutStartSec=20min' "$state_service" \
|| die 'state service full repair window is missing'

for required in \
'FILESYSTEM_TIMEOUT_SECONDS = 300' \
'E2FSCK_UNCORRECTED_EXIT = 4' \
'E2FSCK_EXIT_RE = re.compile' \
'def e2fsck_exit_code(message: str) -> int | None:' \
'def repair_ext4(device: Path, failure_prefix: str) -> bool:' \
'def complete_resize_after_timeout() -> bool:' \
'timeout=FILESYSTEM_TIMEOUT_SECONDS' \
'automatic ext4 repair failed' \
'state filesystem resize failed' \
'resize2fs: timeout' \
'["/usr/sbin/e2fsck", "-f", "-y"' \
'f"e2fsck: exit {E2FSCK_UNCORRECTED_EXIT}" in message'
'e2fsck_exit == E2FSCK_UNCORRECTED_EXIT'
do
grep -Fq "$required" "$orchestrator" \
|| die "state repair contract is missing: $required"
done

for required in \
'Some\((?P<option>[0-9]+)\)' \
'def revalidate_state_device(expected: Path)' \
'verified state device changed during repair' \
'if check_failure is None or not check_failure.startswith(uncorrected):'
'or e2fsck_exit_code(check_failure) != E2FSCK_UNCORRECTED_EXIT'
do
grep -Fq "$required" "$orchestrator" \
|| die "state repair safety boundary is missing: $required"
Expand Down