Hive Exit stability hardening

.gitattributes

@@ -1,7 +1,19 @@
 * text=auto
 *.sh text eol=lf
+*.py text eol=lf
 *.rs text eol=lf
 *.toml text eol=lf
 *.json text eol=lf
 *.md text eol=lf
+*.service text eol=lf
+*.timer text eol=lf
+*.conf text eol=lf
 *.ps1 text eol=crlf
+
+build/usb/hooks/* text eol=lf
+build/usb/package-lists/rigos.list.chroot text eol=lf
+build/usb/includes.chroot/usr/local/bin/* text eol=lf
+build/usb/includes.chroot/usr/local/sbin/rigos-* text eol=lf
+build/usb/includes.chroot/usr/lib/rigos/rigos-runtime-* text eol=lf
+build/usb/includes.chroot/usr/lib/rigos/rigos-miner-* text eol=lf
+build/usb/includes.chroot/usr/lib/rigos/rigos-remote-* text eol=lf

build/usb/hooks/010-rigos.chroot

@@ -6,12 +6,10 @@ useradd --create-home --shell /bin/bash --groups sudo rigosadmin
 passwd --lock rigosadmin
 install -d -o rigos -g rigos -m 0750 /var/lib/rigos
 install -d -m 0755 /usr/lib/rigos /usr/local/bin
-ln -sfn /usr/lib/rigos/rigosd /usr/local/bin/rigosd
-ln -sfn /usr/lib/rigos/rigosctl /usr/local/bin/rigosctl
 systemd-tmpfiles --create /usr/lib/tmpfiles.d/rigos.conf
 
-chmod 0755 /usr/local/sbin/rigos-firstboot /usr/local/sbin/rigos-recovery-access /usr/local/sbin/rigos-state-orchestrate /usr/lib/rigos/rigos-miner-gate /usr/lib/rigos/rigos-runtime-render /usr/lib/rigos/rigos-runtime-gate /usr/lib/rigos/rigos-state-init /usr/lib/rigos/rigos-state-ready /usr/lib/rigos/rigos-config /usr/lib/rigos/rigos-performance /usr/lib/rigos/rigos-lifecycle-cycles /usr/lib/rigos/rigos-identity-seed /usr/lib/rigos/xmrig
-systemctl enable NetworkManager.service ssh.service rigos-state.service rigos-recovery-access.service rigos-state-ready.service rigos-profile-apply.service rigos-runtime-render.service rigos-hugepages.service rigos-firstboot.service rigos-miner.service tmp.mount
+chmod 0755 /usr/local/bin/rigosd /usr/local/bin/rigosctl /usr/local/sbin/rigos-firstboot /usr/local/sbin/rigos-recovery-access /usr/local/sbin/rigos-state-orchestrate /usr/lib/rigos/rigos-miner-gate /usr/lib/rigos/rigos-miner-health /usr/lib/rigos/rigos-runtime-render /usr/lib/rigos/rigos-runtime-publish /usr/lib/rigos/rigos-runtime-gate /usr/lib/rigos/rigos-state-init /usr/lib/rigos/rigos-state-ready /usr/lib/rigos/rigos-config /usr/lib/rigos/rigos-performance /usr/lib/rigos/rigos-lifecycle-cycles /usr/lib/rigos/rigos-identity-seed /usr/lib/rigos/xmrig
+systemctl enable NetworkManager.service ssh.service rigos-state.service rigos-recovery-access.service rigos-state-ready.service rigos-profile-apply.service rigos-runtime-render.service rigos-hugepages.service rigos-firstboot.service rigos-miner.service rigos-miner-health.timer tmp.mount
 systemctl disable ssh.socket 2>/dev/null || true
 systemctl disable apt-daily.timer apt-daily-upgrade.timer logrotate.timer fstrim.timer 2>/dev/null || true
 systemctl disable systemd-journald-audit.socket 2>/dev/null || true

build/usb/includes.chroot/etc/systemd/system/rigos-miner-health.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Observe RIGOS miner health truth
+After=rigos-state-ready.service rigos-runtime-render.service rigos-miner.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/python3 /usr/lib/rigos/rigos-miner-health
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectHome=yes
+ProtectSystem=strict
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectControlGroups=yes
+RestrictAddressFamilies=AF_UNIX
+RestrictNamespaces=yes
+LockPersonality=yes
+ReadWritePaths=/run/rigos

build/usb/includes.chroot/etc/systemd/system/rigos-miner-health.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Sample RIGOS miner health truth
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=1min
+AccuracySec=5s
+Unit=rigos-miner-health.service
+
+[Install]
+WantedBy=timers.target

build/usb/includes.chroot/etc/systemd/system/rigos-miner.service.d/runtime-render.conf

@@ -5,7 +5,7 @@ ConditionPathExists=
 ConditionPathExists=/var/lib/rigos/current
 
 [Service]
-ExecCondition=+/usr/lib/rigos/rigos-runtime-render
+ExecCondition=+/usr/lib/rigos/rigos-runtime-authority
 ExecCondition=/usr/lib/rigos/rigos-runtime-gate
 ExecStart=
-ExecStart=/usr/lib/rigos/xmrig --config=/run/rigos/xmrig.json
+ExecStart=/usr/lib/rigos/xmrig -c /run/rigos/xmrig.json

build/usb/includes.chroot/etc/systemd/system/rigos-miner.service.d/stability.conf

@@ -0,0 +1,17 @@
+[Unit]
+StartLimitIntervalSec=10min
+StartLimitBurst=5
+
+[Service]
+Restart=on-failure
+RestartSec=15s
+TimeoutStartSec=2min
+TimeoutStopSec=30s
+KillSignal=SIGINT
+FinalKillSignal=SIGKILL
+SendSIGKILL=yes
+SuccessExitStatus=SIGINT SIGTERM
+OOMPolicy=stop
+OOMScoreAdjust=-500
+TasksMax=64
+LimitNOFILE=4096

build/usb/includes.chroot/etc/systemd/system/rigos-runtime-render.service

@@ -1,13 +1,13 @@
 [Unit]
-Description=Render RIGOS runtime miner configuration
+Description=Render and publish RIGOS runtime miner configuration
 After=rigos-state-ready.service rigos-profile-apply.service
 Requires=rigos-state-ready.service rigos-profile-apply.service
 Before=rigos-hugepages.service rigos-miner.service
 ConditionPathExists=/var/lib/rigos/current
 
 [Service]
 Type=oneshot
-ExecStart=/usr/lib/rigos/rigos-runtime-render
+ExecStart=/usr/lib/rigos/rigos-runtime-authority
 NoNewPrivileges=yes
 PrivateTmp=yes
 ProtectHome=yes

build/usb/includes.chroot/usr/lib/rigos/rigos-miner-health

@@ -0,0 +1,268 @@
+#!/usr/bin/python3
+import datetime as dt
+import json
+import os
+import re
+import subprocess
+import sys
+import uuid
+from pathlib import Path
+
+RUNTIME = Path(os.environ.get("RIGOS_RUNTIME_PATH", "/run/rigos"))
+BOOT_ID = Path(os.environ.get("RIGOS_BOOT_ID_PATH", "/proc/sys/kernel/random/boot_id"))
+CURRENT = Path(os.environ.get("RIGOS_CURRENT_REVISION_PATH", "/var/lib/rigos/current"))
+PROC = Path(os.environ.get("RIGOS_PROC_ROOT", "/proc"))
+SYSTEMCTL = os.environ.get("RIGOS_SYSTEMCTL", "/usr/bin/systemctl")
+JOURNALCTL = os.environ.get("RIGOS_JOURNALCTL", "/usr/bin/journalctl")
+STATUS = RUNTIME / "miner-health-status.json"
+WINDOW_SECONDS = int(os.environ.get("RIGOS_MINER_HEALTH_WINDOW_SECONDS", "180"))
+WARMUP_SECONDS = int(os.environ.get("RIGOS_MINER_WARMUP_SECONDS", "240"))
+MAX_JOURNAL_LINES = 500
+
+EXTERNAL_MARKERS = (
+    "connect error",
+    "connection refused",
+    "connection reset",
+    "network error",
+    "dns error",
+    "no active pools",
+    "operation timed out",
+    "connection timeout",
+)
+SHARE_RE = re.compile(r"accepted \((\d+)/(\d+)\)")
+
+
+def atomic_json(path: Path, value: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
+    data = json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n"
+    fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            handle.write(data)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary, path)
+        directory = os.open(path.parent, os.O_RDONLY)
+        try:
+            os.fsync(directory)
+        finally:
+            os.close(directory)
+    finally:
+        try:
+            temporary.unlink()
+        except FileNotFoundError:
+            pass
+
+
+def command(argv: list[str]) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        argv,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        timeout=10,
+        check=False,
+    )
+
+
+def systemd_properties() -> dict[str, str]:
+    result = command([
+        SYSTEMCTL,
+        "show",
+        "rigos-miner.service",
+        "--property=ActiveState",
+        "--property=SubState",
+        "--property=MainPID",
+        "--property=NRestarts",
+        "--property=Result",
+        "--property=ExecMainStatus",
+        "--property=ActiveEnterTimestampMonotonic",
+        "--no-pager",
+    ])
+    if result.returncode != 0:
+        raise RuntimeError("systemctl_show_failed")
+    values = {}
+    for line in result.stdout.splitlines():
+        if "=" in line:
+            key, value = line.split("=", 1)
+            values[key] = value
+    return values
+
+
+def current_revision() -> str | None:
+    try:
+        return CURRENT.resolve(strict=True).name
+    except OSError:
+        return None
+
+
+def runtime_revision() -> tuple[str | None, str | None]:
+    try:
+        value = json.loads((RUNTIME / "runtime-config-status.json").read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, json.JSONDecodeError):
+        return None, None
+    if not isinstance(value, dict):
+        return None, None
+    return value.get("outcome"), value.get("revision")
+
+
+def process_state(pid: int) -> str | None:
+    try:
+        raw = (PROC / str(pid) / "stat").read_text(encoding="ascii")
+    except (OSError, UnicodeError):
+        return None
+    end = raw.rfind(")")
+    if end < 0:
+        return None
+    fields = raw[end + 1 :].split()
+    return fields[0] if fields else None
+
+
+def active_seconds(properties: dict[str, str]) -> int | None:
+    try:
+        entered = int(properties.get("ActiveEnterTimestampMonotonic", "0"))
+        uptime = float((PROC / "uptime").read_text(encoding="ascii").split()[0])
+    except (OSError, UnicodeError, ValueError, IndexError):
+        return None
+    if entered <= 0:
+        return None
+    return max(0, int(uptime - entered / 1_000_000))
+
+
+def recent_journal() -> tuple[str, bool]:
+    result = command([
+        JOURNALCTL,
+        "-b",
+        "-u",
+        "rigos-miner.service",
+        "--since",
+        f"-{WINDOW_SECONDS} seconds",
+        "--lines",
+        str(MAX_JOURNAL_LINES),
+        "--no-pager",
+        "-o",
+        "cat",
+    ])
+    return result.stdout, result.returncode == 0
+
+
+def last_share_counts(text: str) -> tuple[int | None, int | None]:
+    matches = SHARE_RE.findall(text)
+    if not matches:
+        return None, None
+    accepted, rejected = matches[-1]
+    return int(accepted), int(rejected)
+
+
+def classify(
+    properties: dict[str, str],
+    proc_state: str | None,
+    active_for: int | None,
+    canonical_revision: str | None,
+    rendered_outcome: str | None,
+    rendered_revision: str | None,
+    journal: str,
+    journal_available: bool,
+) -> tuple[str, str | None]:
+    if properties.get("ActiveState") != "active" or properties.get("SubState") != "running":
+        return "failed", "miner_unit_not_running"
+    try:
+        pid = int(properties.get("MainPID", "0"))
+    except ValueError:
+        pid = 0
+    if pid <= 0 or proc_state is None or proc_state in ("X", "Z"):
+        return "failed", "miner_process_unavailable"
+    if canonical_revision is None:
+        return "blocked", "current_revision_unavailable"
+    if rendered_outcome != "ready" or rendered_revision != canonical_revision:
+        return "blocked", "runtime_revision_mismatch"
+    if not journal_available:
+        return "unknown", "journal_unavailable"
+
+    lowered = journal.lower()
+    if "miner    speed" in lowered:
+        return "ready", None
+    if any(marker in lowered for marker in EXTERNAL_MARKERS):
+        return "waiting_external", "pool_or_network_unavailable"
+    if active_for is not None and active_for < WARMUP_SECONDS:
+        return "warming_up", None
+    return "degraded", "no_recent_speed_evidence"
+
+
+def main() -> int:
+    try:
+        boot_id = BOOT_ID.read_text(encoding="ascii").strip()
+        if not boot_id:
+            raise RuntimeError("boot_id_unavailable")
+        properties = systemd_properties()
+        try:
+            pid = int(properties.get("MainPID", "0"))
+        except ValueError:
+            pid = 0
+        proc_state = process_state(pid) if pid > 0 else None
+        active_for = active_seconds(properties)
+        canonical = current_revision()
+        rendered_outcome, rendered_revision = runtime_revision()
+        journal, journal_available = recent_journal()
+        accepted, rejected = last_share_counts(journal)
+        state, reason = classify(
+            properties,
+            proc_state,
+            active_for,
+            canonical,
+            rendered_outcome,
+            rendered_revision,
+            journal,
+            journal_available,
+        )
+        lowered_journal = journal.lower()
+        value = {
+            "schema": "rigos.miner-health-status/v1",
+            "generated_at": dt.datetime.now(dt.timezone.utc).isoformat(timespec="milliseconds"),
+            "boot_id": boot_id,
+            "state": state,
+            "reason": reason,
+            "unit": {
+                "active_state": properties.get("ActiveState"),
+                "sub_state": properties.get("SubState"),
+                "result": properties.get("Result"),
+                "main_pid": pid or None,
+                "process_state": proc_state,
+                "restart_count": int(properties.get("NRestarts", "0") or 0),
+                "exec_main_status": int(properties.get("ExecMainStatus", "0") or 0),
+                "active_seconds": active_for,
+            },
+            "config": {
+                "current_revision": canonical,
+                "runtime_outcome": rendered_outcome,
+                "runtime_revision": rendered_revision,
+            },
+            "evidence": {
+                "window_seconds": WINDOW_SECONDS,
+                "journal_available": journal_available,
+                "journal_line_limit": MAX_JOURNAL_LINES,
+                "recent_speed": "miner    speed" in lowered_journal,
+                "external_wait_marker": any(
+                    marker in lowered_journal for marker in EXTERNAL_MARKERS
+                ),
+                "accepted_shares": accepted,
+                "rejected_shares": rejected,
+            },
+            "remediation": "observe_only",
+        }
+        atomic_json(STATUS, value)
+        print(json.dumps({
+            "schema": "rigos.miner-health-observer/v1",
+            "state": state,
+            "reason": reason,
+            "remediation": "observe_only",
+        }, sort_keys=True))
+        return 0
+    except (OSError, UnicodeError, RuntimeError, subprocess.SubprocessError, ValueError) as error:
+        print(f"rigos-miner-health: {error}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())