From 541c7af3c7205d0f4dc7de27d7faba6eabc3bed2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 20 Jul 2026 15:57:51 -0400 Subject: [PATCH 1/3] Add vidimus dependency --- pyproject.toml | 1 + uv.lock | 134 +++++++++++++++++++++++++++---------------------- 2 files changed, 75 insertions(+), 60 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ff835bf..65dd055 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "odfpy>=1.4.1", "xlrd>=2.0.1", "pypdf>=6.0.0", + "vidimus==0.1.2", ] [project.optional-dependencies] diff --git a/uv.lock b/uv.lock index 3a7c2b8..214f9e6 100644 --- a/uv.lock +++ b/uv.lock @@ -280,66 +280,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] -[[package]] -name = "policyengine-arch-data" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "cryptography" }, - { name = "httpx" }, - { name = "microplex" }, - { name = "numpy" }, - { name = "odfpy" }, - { name = "openpyxl" }, - { name = "pandas" }, - { name = "pyarrow" }, - { name = "pypdf" }, - { name = "pyyaml" }, - { name = "scipy" }, - { name = "sqlmodel" }, - { name = "supabase" }, - { name = "xlrd" }, -] - -[package.optional-dependencies] -dev = [ - { name = "pytest" }, - { name = "ruff" }, -] -policyengine = [ - { name = "policyengine-us" }, -] - -[package.dev-dependencies] -dev = [ - { name = "pytest" }, -] - -[package.metadata] -requires-dist = [ - { name = "cryptography", specifier = ">=46.0.3" }, - { name = "httpx", specifier = ">=0.27.0" }, - { name = "microplex", specifier = ">=0.1.0" }, - { name = "numpy", specifier = ">=1.24.0" }, - { name = "odfpy", specifier = ">=1.4.1" }, - { name = "openpyxl", specifier = ">=3.1.0" }, - { name = "pandas", specifier = ">=2.0.0" }, - { name = "policyengine-us", marker = "extra == 'policyengine'", specifier = ">=1.670.2" }, - { name = "pyarrow", specifier = ">=22.0.0" }, - { name = "pypdf", specifier = ">=6.0.0" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0,<9" }, - { name = "pyyaml", specifier = ">=6.0" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.5.0" }, - { name = "scipy", specifier = ">=1.11.0" }, - { name = "sqlmodel", specifier = ">=0.0.22" }, - { name = "supabase", specifier = ">=2.0.0" }, - { name = "xlrd", specifier = ">=2.0.1" }, -] -provides-extras = ["dev", "policyengine"] - -[package.metadata.requires-dev] -dev = [{ name = "pytest", specifier = ">=8.0.0,<9" }] - [[package]] name = "cryptography" version = "46.0.3" @@ -1697,6 +1637,68 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "policyengine-arch-data" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "cryptography" }, + { name = "httpx" }, + { name = "microplex" }, + { name = "numpy" }, + { name = "odfpy" }, + { name = "openpyxl" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pypdf" }, + { name = "pyyaml" }, + { name = "scipy" }, + { name = "sqlmodel" }, + { name = "supabase" }, + { name = "vidimus" }, + { name = "xlrd" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] +policyengine = [ + { name = "policyengine-us" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "cryptography", specifier = ">=46.0.3" }, + { name = "httpx", specifier = ">=0.27.0" }, + { name = "microplex", specifier = ">=0.1.0" }, + { name = "numpy", specifier = ">=1.24.0" }, + { name = "odfpy", specifier = ">=1.4.1" }, + { name = "openpyxl", specifier = ">=3.1.0" }, + { name = "pandas", specifier = ">=2.0.0" }, + { name = "policyengine-us", marker = "extra == 'policyengine'", specifier = ">=1.670.2" }, + { name = "pyarrow", specifier = ">=22.0.0" }, + { name = "pypdf", specifier = ">=6.0.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0,<9" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.5.0" }, + { name = "scipy", specifier = ">=1.11.0" }, + { name = "sqlmodel", specifier = ">=0.0.22" }, + { name = "supabase", specifier = ">=2.0.0" }, + { name = "vidimus", specifier = "==0.1.2" }, + { name = "xlrd", specifier = ">=2.0.1" }, +] +provides-extras = ["dev", "policyengine"] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.0.0,<9" }] + [[package]] name = "policyengine-core" version = "3.25.3" @@ -2864,6 +2866,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/65/a8/1791660a87f03d10a3bce00401a66035999c91f5a9a6987569b84df5719d/us-3.2.0-py3-none-any.whl", hash = "sha256:571714ad6d473c72bbd2058a53404cdf4ecc0129e4f19adfcbeb4e2d7e3dc3e7", size = 13775, upload-time = "2024-07-22T01:09:41.432Z" }, ] +[[package]] +name = "vidimus" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/4b/83079ebfa8a96e47e73743b7a7f7d18ca6e7d41b8ef99d94a57e6da44cab/vidimus-0.1.2.tar.gz", hash = "sha256:ef0f798ecea994ce8796358442c046ddd77a9f7fc5608ca1bf35f9aa73a05b3f", size = 58045, upload-time = "2026-07-20T17:17:16.793Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/0d/fa474cfbd58439eacd8268d2d956c7440667b1b67252188c20454f17607c/vidimus-0.1.2-py3-none-any.whl", hash = "sha256:f6662a73b31bb7c67b038b7179680c48de93c368ad2d9fd0bcde34fd3f6841ba", size = 28938, upload-time = "2026-07-20T17:17:15.392Z" }, +] + [[package]] name = "wcwidth" version = "0.6.0" From cb2b3afaefadf30bd2326ce5d29d7af68bebbd41 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 20 Jul 2026 16:06:34 -0400 Subject: [PATCH 2/3] Replace integrity scripts with vidimus shims --- scripts/canonical_json.py | 158 +-- scripts/check_thesis_facts_append.py | 816 ++------------- scripts/verify_release_chain.py | 1413 +++----------------------- scripts/vidimus_pins.py | 94 ++ 4 files changed, 381 insertions(+), 2100 deletions(-) create mode 100644 scripts/vidimus_pins.py diff --git a/scripts/canonical_json.py b/scripts/canonical_json.py index a20a78f..259a4e3 100644 --- a/scripts/canonical_json.py +++ b/scripts/canonical_json.py @@ -1,142 +1,24 @@ #!/usr/bin/env python3 -"""Canonical JSON compatible with site/src/data/canonical-json.ts. - -Object keys are sorted by UTF-16 code units, not Unicode code points. Python's -normal string ordering differs for astral-plane keys, so callers must use the -key below rather than ``sort_keys=True``. Number formatting follows -ECMAScript JSON.stringify's fixed/scientific thresholds. -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import math -import sys -from decimal import Decimal -from typing import Any - - -def utf16_sort_key(value: str) -> bytes: - """Return the big-endian UTF-16 code units used by JavaScript ordering.""" - - return value.encode("utf-16-be", errors="surrogatepass") - - -def _serialize_string(value: str) -> str: - pieces = ['"'] - short_escapes = { - "\b": "\\b", - "\t": "\\t", - "\n": "\\n", - "\f": "\\f", - "\r": "\\r", - '"': '\\"', - "\\": "\\\\", - } - for char in value: - if char in short_escapes: - pieces.append(short_escapes[char]) - elif ord(char) < 0x20 or 0xD800 <= ord(char) <= 0xDFFF: - pieces.append(f"\\u{ord(char):04x}") - else: - pieces.append(char) - pieces.append('"') - return "".join(pieces) - - -def _serialize_float(value: float) -> str: - if not math.isfinite(value): - raise ValueError(f"Canonical JSON cannot serialize non-finite number: {value}") - if value == 0: - return "0" - - absolute = abs(value) - rendered = repr(value).lower() - if 1e-6 <= absolute < 1e21: - fixed = format(Decimal(rendered), "f") - if "." in fixed: - fixed = fixed.rstrip("0").rstrip(".") - return fixed - - if "e" not in rendered: - rendered = format(value, ".15e") - mantissa, exponent_text = rendered.split("e", 1) - mantissa = mantissa.rstrip("0").rstrip(".") - exponent = int(exponent_text) - sign = "+" if exponent >= 0 else "" - return f"{mantissa}e{sign}{exponent}" - - -def canonical_stringify(value: Any) -> str: - """Serialize JSON-compatible data exactly like canonicalStringify in TS.""" - - ancestors: set[int] = set() - - def serialize(item: Any) -> str: - if item is None: - return "null" - if item is True: - return "true" - if item is False: - return "false" - if isinstance(item, str): - return _serialize_string(item) - if isinstance(item, int): - if abs(item) > (2**53 - 1): - try: - return _serialize_float(float(item)) - except OverflowError as exc: - raise ValueError( - f"Canonical JSON integer exceeds Number range: {item}" - ) from exc - return str(item) - if isinstance(item, float): - return _serialize_float(item) - - if isinstance(item, (list, dict)): - identity = id(item) - if identity in ancestors: - raise ValueError("Canonical JSON cannot serialize circular structures") - ancestors.add(identity) - try: - if isinstance(item, list): - return "[" + ",".join(serialize(entry) for entry in item) + "]" - entries = [] - for key in sorted(item, key=utf16_sort_key): - if not isinstance(key, str): - raise TypeError("Canonical JSON object keys must be strings") - entries.append(f"{_serialize_string(key)}:{serialize(item[key])}") - return "{" + ",".join(entries) + "}" - finally: - ancestors.remove(identity) - - raise TypeError( - f"Canonical JSON cannot serialize value of type {type(item).__name__}" - ) - - return serialize(value) - - -def canonical_bytes(value: Any) -> bytes: - return canonical_stringify(value).encode("utf-8") - - -def canonical_sha256(value: Any) -> str: - return hashlib.sha256(canonical_bytes(value)).hexdigest() - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument( - "--sha256", action="store_true", help="print SHA-256 instead of JSON" - ) - args = parser.parse_args() - value = json.load(sys.stdin) - output = canonical_sha256(value) if args.sha256 else canonical_stringify(value) - sys.stdout.write(output + "\n") - return 0 +# Thin shim over vidimus==0.1.2 (hash-pinned in uv.lock). Any vidimus upgrade +# requires a fresh byte-equivalence proof at this repo's then-current pin BEFORE +# the bump. +"""Canonical JSON compatibility surface backed by vidimus.""" + +from vidimus.canonical import ( + canonical_bytes, + canonical_sha256, + canonical_stringify, + main, + utf16_sort_key, +) + +__all__ = [ + "canonical_bytes", + "canonical_sha256", + "canonical_stringify", + "main", + "utf16_sort_key", +] if __name__ == "__main__": diff --git a/scripts/check_thesis_facts_append.py b/scripts/check_thesis_facts_append.py index 907da4b..773bbfd 100644 --- a/scripts/check_thesis_facts_append.py +++ b/scripts/check_thesis_facts_append.py @@ -1,578 +1,91 @@ #!/usr/bin/env python3 -"""Gate every change to the thesis-facts observation ledger. - -The observation file is append-only with an immutable frozen prefix -(``ledger/immutable_prefix.json``). Resolver appends arrive as pull -requests; this checker is the deterministic review each proposal must pass -before merge: - -- the frozen prefix is byte-identical (no rewrite, no truncation); -- against a base ref, the change only appends whole lines; -- every appended row parses, and carries the post-quarantine bindings: - ``assertionVersion`` (content-addressed, recomputed here), ``retrievedAt``, - ``sourceVintage``, ``ledgerRepoSha``, and a ``responseArchive`` digest; -- ``targetContentHash`` and ``sourceBindingProjection`` appear together or - not at all, the projection's response digest matches the archive, and its - unit matches the row's measure unit; -- a duplicate ``source_record_id`` is legal only as an explicit correction: - the later row's ``assertionVersion.supersedes`` must name the version ID - of the row it replaces. -- after witnessed genesis, every byte append carries exactly one next canonical - release manifest, its producer signature, and two independently anchored - RFC 3161 receipts; all prior release files remain byte-immutable against the - PR's base commit. - -Usage: - python3 scripts/check_thesis_facts_append.py [--base-ref REF] - -With ``--base-ref`` (CI: the pull request's base commit) the append-only -diff is enforced; without it only the full-file invariants run. -""" +# Thin shim over vidimus==0.1.2 (hash-pinned in uv.lock). Any vidimus upgrade +# requires a fresh byte-equivalence proof at this repo's then-current pin BEFORE +# the bump. +"""Gate every change to the thesis-facts observation ledger.""" from __future__ import annotations import argparse -import hashlib -import json -import os import pathlib -import re -import subprocess import sys from typing import Any -from canonical_json import canonical_sha256 -from verify_release_chain import ( - ANCHORS, - MANIFEST_RE, - PRODUCER_PUBLIC_KEY_FILENAME, - ReleaseChainError, - git_blob_bytes, - git_file_entry, - verify_base_release_chain, - verify_release_chain, - verify_release_history_immutable, -) +import vidimus.append_gate as _vidimus +from vidimus.release_chain import MANIFEST_RE, ReleaseChainError + +try: + from vidimus_pins import APPEND_GATE_SPEC, LEDGER_SPEC +except ModuleNotFoundError as exc: + if exc.name != "vidimus_pins": + raise + # The test suite copies the legacy three-script surface into temporary + # repositories. The editable consumer tree remains the sole pin owner. + from scripts.vidimus_pins import APPEND_GATE_SPEC, LEDGER_SPEC + CODE_ROOT = pathlib.Path(__file__).resolve().parents[1] ROOT = CODE_ROOT -LEDGER_PATH = ROOT / "ledger" / "official_observations.jsonl" -PREFIX_PATH = ROOT / "ledger" / "immutable_prefix.json" -RELEASE_MANIFEST_PREFIX = "releases/manifests/" -GENESIS_SUPPORT_FILES = { - "releases/README.md", - *(f"releases/anchors/{spec.filename}" for spec in ANCHORS.values()), - f"releases/anchors/{PRODUCER_PUBLIC_KEY_FILENAME}", -} - -GATE_SURFACE = frozenset( - { - "scripts/check_thesis_facts_append.py", - "scripts/verify_release_chain.py", - "scripts/canonical_json.py", - "scripts/cut_release_manifest.py", - ".github/workflows/thesis-facts-append.yml", - "releases/anchors/**", - } -) -DATA_SURFACE = frozenset( - { - "ledger/**", - "releases/manifests/**", - } -) - -ASSERTION_CONTENT_KEYS = ( - "source_record_id", - "value", - "observed_at", - "period", - "geography", - "entity", - "aggregation", - "filters", - "domain", -) - - -class AppendError(ValueError): - """The proposed ledger change violates an append invariant.""" +LEDGER_PATH = ROOT / LEDGER_SPEC.state_relative +PREFIX_PATH = ROOT / LEDGER_SPEC.prefix_relative +RELEASE_MANIFEST_PREFIX = APPEND_GATE_SPEC.release_manifest_prefix +GENESIS_SUPPORT_FILES = APPEND_GATE_SPEC.genesis_support_files +GATE_SURFACE = APPEND_GATE_SPEC.gate_surface +DATA_SURFACE = APPEND_GATE_SPEC.data_surface +ASSERTION_CONTENT_KEYS = APPEND_GATE_SPEC.assertion_content_keys +AppendError = _vidimus.AppendError +AppendGateSpec = _vidimus.AppendGateSpec +reject_non_append_bytes = _vidimus.reject_non_append_bytes -def _set_root(root: pathlib.Path) -> None: - """Select the candidate worktree without changing the trusted code root. - Pull-request CI executes this module from a detached checkout of the base - commit, while ``--root`` points at the checked-out PR merge tree. Imports - and production anchors therefore remain rooted at immutable ``CODE_ROOT``; - only candidate data paths and git comparisons use ``ROOT``. - """ +def _set_root(root: pathlib.Path) -> None: + """Select candidate paths while leaving the trusted code root unchanged.""" global ROOT, LEDGER_PATH, PREFIX_PATH ROOT = root.resolve() - LEDGER_PATH = ROOT / "ledger" / "official_observations.jsonl" - PREFIX_PATH = ROOT / "ledger" / "immutable_prefix.json" - - -def _git_output(arguments: list[str]) -> bytes: - try: - completed = subprocess.run( - ["git", *arguments], - cwd=ROOT, - check=False, - capture_output=True, - ) - except FileNotFoundError as exc: - raise AppendError("git is required for --base-ref verification") from exc - if completed.returncode != 0: - diagnostic = completed.stderr.decode( - "utf-8", errors="replace" - ).strip() - raise AppendError( - f"git {' '.join(arguments)} failed: {diagnostic}" - ) - return completed.stdout + LEDGER_PATH = ROOT / LEDGER_SPEC.state_relative + PREFIX_PATH = ROOT / LEDGER_SPEC.prefix_relative -def _resolve_base_commit(base_ref: str) -> str: - completed = _git_output( - ["rev-parse", "--verify", "--end-of-options", f"{base_ref}^{{commit}}"] - ) - commit = completed.decode("ascii").strip() - _git_output(["merge-base", "--is-ancestor", commit, "HEAD"]) - return commit - - -def _nul_paths(payload: bytes) -> set[str]: - return {os.fsdecode(path) for path in payload.split(b"\0") if path} - - -def _matches_surface(path: str, surface: frozenset[str]) -> bool: - for pattern in surface: - if pattern.endswith("/**"): - if path.startswith(pattern[:-2]): - return True - elif path == pattern: - return True - return False +def _candidate() -> Any: + return _vidimus._set_root(ROOT, APPEND_GATE_SPEC) def check_surface_separation(base_ref: str) -> tuple[set[str], set[str]]: - """Return data/gate changes and reject a proposal that combines them.""" - - commit = _resolve_base_commit(base_ref) - changed = _nul_paths( - _git_output( - [ - "diff", - "--name-only", - "-z", - "--no-renames", - "--no-ext-diff", - "--no-textconv", - commit, - "--", - ] - ) - ) - # ``git diff`` excludes untracked files. Tests mint release siblings before - # staging them, and a newly added anchor must still classify as gate code. - changed.update( - _nul_paths( - _git_output( - ["ls-files", "--others", "--exclude-standard", "-z", "--"] - ) - ) - ) - data_changes = { - path for path in changed if _matches_surface(path, DATA_SURFACE) - } - gate_changes = { - path for path in changed if _matches_surface(path, GATE_SURFACE) - } - if data_changes and gate_changes: - raise AppendError( - "mixed data/gate proposal is forbidden: DATA_SURFACE changes=" - f"{sorted(data_changes)}; GATE_SURFACE changes=" - f"{sorted(gate_changes)}; split them into separate pull requests" - ) - return data_changes, gate_changes - - -def _lines(text: str) -> list[str]: - return [line for line in text.split("\n") if line.strip()] - - -def reject_non_append_bytes(text: str) -> None: - """Reject blank/whitespace-only lines and any non-single trailing newline. - - ``_lines`` drops blank lines so row parsing is convenient, but that means a - blank line inserted into the frozen JSONL would normalize away and pass both - the prefix hash and the append-only diff. A JSONL row is exactly one - non-empty line: a blank/whitespace-only line inside the covered region is a - byte tamper, and the file must end with exactly one trailing newline. - """ - parts = text.split("\n") - if parts[-1] != "": - raise AppendError("ledger must end with exactly one trailing newline") - for index, part in enumerate(parts[:-1], start=1): - if not part.strip(): - raise AppendError( - f"line {index} is blank or whitespace-only; a JSONL row is one " - "non-empty line and a stray blank line is a tamper" - ) + return _vidimus.check_surface_separation(base_ref, _candidate()) def expected_assertion_version_id(row: dict[str, Any]) -> str: - """Recompute the content address the resolver must have written. - - Mirrors ``assertion_version`` in the Thesis resolver (av1 v2 spec): the ID - commits to everything that changes what the assertion MEANS — identity, - value, timing, population, the complete measure concept mapping, exact - source lineage/digest, row/cell lineage, and the archived response digest — - so an in-place edit is detectable and a correction must supersede - explicitly. This projection must stay byte-identical to the Brier writer's - ``assertion_version`` (both fed to the shared ``canonical_sha256``), so any - change here is a coordinated schema migration on both sides. - """ - measure = row.get("measure") or {} - source = row.get("source") or {} - projection = {key: row.get(key) for key in ASSERTION_CONTENT_KEYS} - projection["measure"] = { - "concept": measure.get("concept"), - "unit": measure.get("unit"), - "source_concept": measure.get("source_concept"), - "concept_relation": measure.get("concept_relation"), - "concept_authority": measure.get("concept_authority"), - "legal_vintage": measure.get("legal_vintage"), - } - projection["source"] = { - "source_name": source.get("source_name"), - "source_table": source.get("source_table"), - "source_file": source.get("source_file"), - "url": source.get("url"), - "vintage": source.get("vintage"), - "source_sha256": source.get("source_sha256"), - } - projection["lineage"] = { - "source_row_keys": row.get("source_row_keys"), - "source_cell_keys": row.get("source_cell_keys"), - } - projection["responseArchiveSha256"] = (row.get("responseArchive") or {}).get( - "sha256" - ) - return f"av2:{canonical_sha256(projection)}" - - -def _effective_assertion_id(row: dict[str, Any]) -> str: - """Return the row's effective assertion version ID. + return _vidimus.expected_assertion_version_id(row, APPEND_GATE_SPEC) - Post-cutover rows carry an explicit ``assertionVersion.id`` (validated - against the recomputed content address in :func:`check_rows`); legacy - pre-versioning rows are addressable by their recomputed content address. - Either way every row has exactly one effective ID that a correction must - name and that no later row may reissue. - """ - version = row.get("assertionVersion") - if isinstance(version, dict) and version.get("id"): - return str(version["id"]) - return expected_assertion_version_id(row) - -def effective_current_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Return the latest non-superseded row per assertion identity. - - A correction names the version it replaces via - ``assertionVersion.supersedes``; the replaced row drops out of the current - view. Aggregate-fact validation runs on this supersede-aware view so a - legitimate correction (same semantic key, new value) is not mistaken for a - duplicate key. - """ - superseded: set[str] = set() - for row in rows: - version = row.get("assertionVersion") - if isinstance(version, dict) and version.get("supersedes"): - superseded.add(str(version["supersedes"])) - return [row for row in rows if _effective_assertion_id(row) not in superseded] +def effective_current_rows( + rows: list[dict[str, Any]], +) -> list[dict[str, Any]]: + return _vidimus.effective_current_rows(rows, APPEND_GATE_SPEC) def check_prefix(lines: list[str]) -> dict[str, Any]: - prefix = json.loads(PREFIX_PATH.read_text()) - if prefix.get("schemaVersion") != "thesis_facts_immutable_prefix_v1": - raise AppendError( - f"unsupported prefix manifest schema {prefix.get('schemaVersion')!r}" - ) - count = int(prefix["prefixLineCount"]) - hashes = prefix["lineSha256s"] - if len(hashes) != count: - raise AppendError("prefix manifest line hashes disagree with its count") - if len(lines) < count: - raise AppendError( - f"ledger has {len(lines)} rows but the immutable prefix " - f"requires at least {count}" - ) - for index in range(count): - digest = hashlib.sha256(lines[index].encode("utf-8")).hexdigest() - if digest != hashes[index]: - row_id = json.loads(lines[index]).get("source_record_id", "?") - raise AppendError( - f"immutable prefix line {index + 1} ({row_id}) was rewritten" - ) - joined = hashlib.sha256( - ("\n".join(lines[:count]) + "\n").encode("utf-8") - ).hexdigest() - if joined != prefix["prefixSha256"]: - raise AppendError("immutable prefix cumulative hash mismatch") - return prefix + return _vidimus.check_prefix(lines, _candidate()) def check_rows(lines: list[str], prefix_count: int) -> None: - versions: dict[str, int] = {} - active_by_record_id: dict[str, tuple[int, str | None]] = {} - for number, line in enumerate(lines, start=1): - try: - row = json.loads(line) - except json.JSONDecodeError as exc: - raise AppendError(f"line {number} is not valid JSON: {exc}") from exc - if not isinstance(row, dict): - raise AppendError(f"line {number} is not a JSON object") - record_id = row.get("source_record_id") - if not record_id: - raise AppendError(f"line {number} lacks source_record_id") - if not isinstance(row.get("value"), (int, float)): - raise AppendError(f"line {number} ({record_id}) has no numeric value") - if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(row.get("observed_at", ""))): - raise AppendError(f"line {number} ({record_id}) has no observed_at date") - unit = (row.get("measure") or {}).get("unit") - if not unit: - raise AppendError(f"line {number} ({record_id}) has no measure unit") - - recomputed = expected_assertion_version_id(row) - version = row.get("assertionVersion") - supersedes = None - if version is not None: - if not isinstance(version, dict): - raise AppendError(f"line {number} assertionVersion is not an object") - version_id = str(version.get("id", "")) - supersedes = version.get("supersedes") - if version_id != recomputed: - raise AppendError( - f"line {number} ({record_id}) assertionVersion.id does not " - f"match its content ({version_id} != {recomputed})" - ) - effective_id = version_id - else: - # Pre-versioning rows are addressable by their recomputed content - # address; that ID is reserved just like an explicit one so a legacy - # synthetic ID cannot be silently reissued. - effective_id = recomputed - - # Reserve the effective ID of EVERY row. A collision means two rows - # claim the same assertion version — a duplicate legacy ID or an - # A->B->A chain trying to restore a superseded value. - if effective_id in versions: - raise AppendError( - f"line {number} restates assertion version {effective_id} " - f"from line {versions[effective_id]}" - ) - versions[effective_id] = number - - if number > prefix_count: - for field in ( - "retrievedAt", - "sourceVintage", - "ledgerRepoSha", - "responseArchive", - "assertionVersion", - ): - if not row.get(field): - raise AppendError( - f"appended line {number} ({record_id}) lacks {field}" - ) - archive = row["responseArchive"] - if not isinstance(archive, dict) or not archive.get("sha256"): - raise AppendError( - f"appended line {number} responseArchive lacks a digest" - ) - # Key PRESENCE pairs the binding, and present values must be - # shape-valid: truthiness accepted targetContentHash "" with a - # missing (or {}) projection, silently waiving the contract - # binding (found during the vidimus extraction review). - has_hash = "targetContentHash" in row - has_projection = "sourceBindingProjection" in row - if has_hash != has_projection: - raise AppendError( - f"appended line {number} ({record_id}) must carry " - "targetContentHash and sourceBindingProjection together" - ) - if has_hash: - content_hash = row["targetContentHash"] - if not isinstance(content_hash, str) or not re.fullmatch( - r"[0-9a-f]{64}", content_hash - ): - raise AppendError( - f"appended line {number} ({record_id}) " - "targetContentHash is not a SHA-256 hex digest" - ) - projection = row["sourceBindingProjection"] - if not isinstance(projection, dict) or not projection: - raise AppendError( - f"appended line {number} ({record_id}) " - "sourceBindingProjection must be a non-empty object" - ) - if projection.get("responseSha256") != archive.get("sha256"): - raise AppendError( - f"appended line {number} projection digest does not " - "match its archived response" - ) - if projection.get("unit") != unit: - raise AppendError( - f"appended line {number} projection unit " - f"{projection.get('unit')!r} contradicts the row unit " - f"{unit!r}" - ) - - previous = active_by_record_id.get(str(record_id)) - if previous is not None: - previous_line, previous_version = previous - if supersedes is None: - raise AppendError( - f"line {number} duplicates {record_id} (line " - f"{previous_line}) without superseding an assertion " - "version — corrections must be explicit" - ) - if supersedes != previous_version: - raise AppendError( - f"line {number} supersedes {supersedes} but the active " - f"version of {record_id} is {previous_version}" - ) - elif supersedes is not None: - raise AppendError( - f"line {number} supersedes {supersedes} but {record_id} has " - "no earlier row" - ) - active_by_record_id[str(record_id)] = (number, effective_id) + return _vidimus.check_rows(lines, prefix_count, APPEND_GATE_SPEC) def check_append_only(base_ref: str, lines: list[str]) -> int: - relative = LEDGER_PATH.relative_to(ROOT).as_posix() - try: - base_text = subprocess.check_output( - ["git", "show", f"{base_ref}:{relative}"], cwd=ROOT, text=True - ) - except subprocess.CalledProcessError as exc: - raise AppendError(f"cannot read {relative} at base {base_ref}") from exc - base_lines = _lines(base_text) - if len(lines) < len(base_lines): - raise AppendError( - f"change truncates the ledger: {len(base_lines)} -> {len(lines)} rows" - ) - for index, line in enumerate(base_lines): - if lines[index] != line: - row_id = json.loads(line).get("source_record_id", "?") - raise AppendError( - f"change rewrites existing line {index + 1} ({row_id}); " - "the ledger is append-only — supersede instead" - ) - return len(lines) - len(base_lines) - - -def _manifest_at_ref(base_ref: str) -> dict[str, Any]: - relative = PREFIX_PATH.relative_to(ROOT).as_posix() - try: - text = subprocess.check_output( - ["git", "show", f"{base_ref}:{relative}"], cwd=ROOT, text=True - ) - except subprocess.CalledProcessError as exc: - raise AppendError( - f"cannot read {relative} at base {base_ref}" - ) from exc - return json.loads(text) - - -def check_prefix_anchored_to_base(base_ref: str, candidate_prefix: dict[str, Any]) -> int: - """Require the frozen prefix manifest to be unchanged from the base. - - The immutable-prefix manifest lives beside the ledger and is candidate- - controlled, so a PR could grow ``prefixLineCount`` over its own append and - have every post-cutover binding skipped (the appended row would count as - "prefix"). Growing the frozen prefix is an explicit, separately reviewed - migration — never part of the automated append path — so under a base ref - the count, cumulative hash, and per-line hashes must match the base exactly. - Returns the BASE prefix line count, which callers use as the post-cutover - binding boundary so a candidate-controlled count can never move it. - """ - base_prefix = _manifest_at_ref(base_ref) - for field in ("prefixLineCount", "prefixSha256", "lineSha256s"): - if candidate_prefix.get(field) != base_prefix.get(field): - raise AppendError( - f"immutable prefix manifest {field} changed vs base {base_ref}; " - "the frozen prefix cannot grow through the automated append path " - "— growing it is an explicit reviewed migration" - ) - return int(base_prefix["prefixLineCount"]) - - -def _release_triple( - new_files: set[str], - expected_index: int, - *, - allowed_support_files: set[str] | None = None, -) -> pathlib.Path: - """Require exactly one manifest, producer signature, and two receipts.""" - - manifest_files = [ - relative - for relative in new_files - if relative.startswith(RELEASE_MANIFEST_PREFIX) - and MANIFEST_RE.fullmatch(pathlib.PurePosixPath(relative).name) - ] - if len(manifest_files) != 1: - raise AppendError( - f"release proposal must add exactly one manifest for index " - f"{expected_index}; found {sorted(manifest_files)}" - ) - manifest_relative = manifest_files[0] - manifest_name = pathlib.PurePosixPath(manifest_relative).name - match = MANIFEST_RE.fullmatch(manifest_name) - assert match is not None - if int(match.group("index")) != expected_index: - raise AppendError( - f"release proposal index must be {expected_index}, not " - f"{int(match.group('index'))}" - ) - stem = pathlib.PurePosixPath(manifest_name).stem - expected = { - manifest_relative, - *(f"{RELEASE_MANIFEST_PREFIX}{stem}.{tsa}.tsr" for tsa in ANCHORS), - f"{RELEASE_MANIFEST_PREFIX}{stem}.producer.sig", - } - allowed = expected | (allowed_support_files or set()) - if new_files != expected and not ( - expected <= new_files and new_files <= allowed - ): - raise AppendError( - "release proposal must add its manifest, producer signature, and " - "exactly the freetsa and digicert receipts with no other releases/ " - "changes; " - f"missing={sorted(expected - new_files)}, " - f"extra={sorted(new_files - allowed)}" - ) - return ROOT / pathlib.PurePosixPath(manifest_relative) + return _vidimus.check_append_only(base_ref, lines, _candidate()) -def _base_ledger_bytes(commit: str) -> bytes: - relative = LEDGER_PATH.relative_to(ROOT).as_posix() - return git_blob_bytes(ROOT, git_file_entry(ROOT, commit, relative)) - - -def _check_exact_byte_append(base_bytes: bytes, candidate_bytes: bytes) -> bytes: - if not candidate_bytes.startswith(base_bytes): - raise AppendError( - "change is not an exact byte append to the base JSONL; existing " - "bytes, including line endings, are immutable" - ) - return candidate_bytes[len(base_bytes) :] +def check_prefix_anchored_to_base( + base_ref: str, candidate_prefix: dict[str, Any] +) -> int: + return _vidimus.check_prefix_anchored_to_base( + base_ref, + candidate_prefix, + _candidate(), + ) def check_release_proposal( @@ -581,106 +94,12 @@ def check_release_proposal( anchor_dir: pathlib.Path | None = None, enforce_production_pins: bool | None = None, ) -> int | None: - """Verify the base chain and the one allowed candidate transition. - - A pre-genesis base keeps legacy append proposals valid only while they do - not touch ``releases/``. Genesis may add the prescribed anchors and README - alongside its exact manifest/signature/receipt bundle. Once genesis exists, - all base release files are byte- and mode-immutable and a ledger byte - append must carry exactly one next release bundle. - """ - - try: - commit, new_files, base_release_entries = ( - verify_release_history_immutable(ROOT, base_ref) - ) - except ReleaseChainError as exc: - raise AppendError(str(exc)) from exc - - base_has_chain = any( - relative.startswith(RELEASE_MANIFEST_PREFIX) - for relative in base_release_entries + return _vidimus.check_release_proposal( + base_ref, + candidate=_candidate(), + anchor_dir=anchor_dir, + enforce_production_pins=enforce_production_pins, ) - candidate_has_chain = any( - path.is_file() - for path in (ROOT / "releases" / "manifests").glob("*.json") - ) if (ROOT / "releases" / "manifests").is_dir() else False - base_bytes = _base_ledger_bytes(commit) - candidate_bytes = LEDGER_PATH.read_bytes() - appended_bytes = _check_exact_byte_append(base_bytes, candidate_bytes) - ledger_changed = bool(appended_bytes) - if enforce_production_pins is None: - enforce_production_pins = anchor_dir is None - - if not base_has_chain: - if not candidate_has_chain: - if new_files: - raise AppendError( - "legacy pre-genesis proposal must not change releases/; " - "add a complete genesis manifest, producer signature, and " - "both receipts or no release files at all " - f"(changed={sorted(new_files)})" - ) - return None - _release_triple( - new_files, - 0, - allowed_support_files=GENESIS_SUPPORT_FILES, - ) - try: - verification = verify_release_chain( - ROOT, - anchor_dir=anchor_dir, - require_chain=True, - verify_state=True, - enforce_production_pins=enforce_production_pins, - ) - except ReleaseChainError as exc: - raise AppendError(str(exc)) from exc - if len(verification.releases) != 1: - raise AppendError( - "genesis proposal must create exactly one release at index 0" - ) - return 0 - - try: - base_verification = verify_base_release_chain( - ROOT, - commit, - base_release_entries, - anchor_dir=anchor_dir, - enforce_production_pins=enforce_production_pins, - ) - except ReleaseChainError as exc: - raise AppendError(f"base release chain is invalid: {exc}") from exc - assert base_verification.head is not None - expected_index = base_verification.head.release_index + 1 - - if ledger_changed: - _release_triple(new_files, expected_index) - elif new_files: - raise AppendError( - "release-only proposal is forbidden after genesis; a next release " - "must witness an actual ledger byte append" - ) - - try: - candidate_verification = verify_release_chain( - ROOT, - anchor_dir=anchor_dir, - require_chain=True, - verify_state=True, - enforce_production_pins=enforce_production_pins, - ) - except ReleaseChainError as exc: - raise AppendError(str(exc)) from exc - expected_length = len(base_verification.releases) + (1 if ledger_changed else 0) - if len(candidate_verification.releases) != expected_length: - raise AppendError( - f"release chain length must be {expected_length} for this proposal; " - f"found {len(candidate_verification.releases)}" - ) - return candidate_verification.head.release_index def check_release_chain_without_base( @@ -688,25 +107,27 @@ def check_release_chain_without_base( anchor_dir: pathlib.Path | None = None, enforce_production_pins: bool | None = None, ) -> int | None: - """On push, verify any initialized chain against working-tree state.""" + return _vidimus.check_release_chain_without_base( + candidate=_candidate(), + anchor_dir=anchor_dir, + enforce_production_pins=enforce_production_pins, + ) - manifest_directory = ROOT / "releases" / "manifests" - if not manifest_directory.is_dir() or not any(manifest_directory.iterdir()): - return None - if enforce_production_pins is None: - enforce_production_pins = anchor_dir is None - try: - verification = verify_release_chain( - ROOT, - anchor_dir=anchor_dir, - require_chain=True, - verify_state=True, - enforce_production_pins=enforce_production_pins, - ) - except ReleaseChainError as exc: - raise AppendError(str(exc)) from exc - assert verification.head is not None - return verification.head.release_index + +def verify_append_gate( + root: pathlib.Path = ROOT, + *, + base_ref: str | None = None, + trusted_code_root: pathlib.Path = CODE_ROOT, + release_anchor_dir: pathlib.Path | None = None, +) -> str: + return _vidimus.verify_append_gate( + root, + spec=APPEND_GATE_SPEC, + base_ref=base_ref, + trusted_code_root=trusted_code_root, + release_anchor_dir=release_anchor_dir, + ) def main() -> int: @@ -728,65 +149,48 @@ def main() -> int: ) args = parser.parse_args() try: - _set_root(args.root) - if args.base_ref: - _data_changes, gate_changes = check_surface_separation(args.base_ref) - if gate_changes: - print( - "thesis-facts append check OK: gate-only proposal; " - "DATA_SURFACE unchanged; GATE_SURFACE changes=" - f"{sorted(gate_changes)}" - ) - return 0 - - text = LEDGER_PATH.read_text(encoding="utf-8") - reject_non_append_bytes(text) - lines = _lines(text) - prefix = check_prefix(lines) - # The post-cutover binding boundary is the BASE prefix count under a - # base ref, so a PR cannot grandfather an unbound append by growing the - # candidate manifest over it. Without a base ref (push) there is nothing - # to anchor against, so the candidate manifest is trusted for the - # full-file invariants only — base-anchoring requires the PR path. - binding_boundary = int(prefix["prefixLineCount"]) - appended = None - if args.base_ref: - binding_boundary = check_prefix_anchored_to_base(args.base_ref, prefix) - appended = check_append_only(args.base_ref, lines) - check_rows(lines, binding_boundary) - # On the PR path, CODE_ROOT is the detached base checkout. Production - # verification must use those immutable anchors and the base verifier's - # pins, never files supplied by the candidate worktree. The hidden test - # override remains unpinned and continues to use generated test anchors. - production_pins = args.release_anchor_dir is None - anchor_dir = args.release_anchor_dir or ( - CODE_ROOT / "releases" / "anchors" - ) - release_index = ( - check_release_proposal( - args.base_ref, - anchor_dir=anchor_dir, - enforce_production_pins=production_pins, - ) - if args.base_ref - else check_release_chain_without_base( - anchor_dir=anchor_dir, - enforce_production_pins=production_pins, - ) + summary = verify_append_gate( + args.root.resolve(), + base_ref=args.base_ref, + trusted_code_root=CODE_ROOT.resolve(), + release_anchor_dir=args.release_anchor_dir, ) except AppendError as exc: print(f"thesis-facts append check failed: {exc}", file=sys.stderr) return 1 - suffix = f", +{appended} appended vs base" if appended is not None else "" - release_suffix = ( - f", release {release_index}" if release_index is not None else "" - ) - print( - f"thesis-facts append check OK: {len(lines)} rows, immutable prefix " - f"{prefix['prefixLineCount']}{suffix}{release_suffix}" - ) + print(summary) return 0 +__all__ = [ + "APPEND_GATE_SPEC", + "ASSERTION_CONTENT_KEYS", + "AppendError", + "AppendGateSpec", + "CODE_ROOT", + "DATA_SURFACE", + "GATE_SURFACE", + "GENESIS_SUPPORT_FILES", + "LEDGER_PATH", + "MANIFEST_RE", + "PREFIX_PATH", + "RELEASE_MANIFEST_PREFIX", + "ROOT", + "ReleaseChainError", + "check_append_only", + "check_prefix", + "check_prefix_anchored_to_base", + "check_release_chain_without_base", + "check_release_proposal", + "check_rows", + "check_surface_separation", + "effective_current_rows", + "expected_assertion_version_id", + "main", + "reject_non_append_bytes", + "verify_append_gate", +] + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/verify_release_chain.py b/scripts/verify_release_chain.py index 31c66d1..fd182ab 100644 --- a/scripts/verify_release_chain.py +++ b/scripts/verify_release_chain.py @@ -1,619 +1,79 @@ #!/usr/bin/env python3 -"""Offline verification for the witnessed thesis-ledger release chain. - -The verifier treats manifest, signature, and receipt bytes as an append-only -journal. It does not trust manifest provenance or timestamps supplied by the -producer: each manifest is canonical and content-addressed, every state and -append digest is recomputed from the current append-only JSONL, every manifest -has a valid signature from the pinned producer key, and both RFC 3161 receipts -are verified against separate, committed trust anchors. -""" +# Thin shim over vidimus==0.1.2 (hash-pinned in uv.lock). Any vidimus upgrade +# requires a fresh byte-equivalence proof at this repo's then-current pin BEFORE +# the bump. +"""Offline verification for the witnessed thesis-ledger release chain.""" from __future__ import annotations import argparse -import hashlib -import json -import os import pathlib -import re -import subprocess import sys -import tempfile -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone +from datetime import datetime from typing import Any -from canonical_json import canonical_bytes +import vidimus.release_chain as _vidimus try: - from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm - from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey - from cryptography.hazmat.primitives.serialization import ( - Encoding, - PublicFormat, - load_pem_public_key, - ) -except ImportError: # Bare pre-sync CI falls back to the OpenSSL 3 CLI below. - CRYPTOGRAPHY_AVAILABLE = False -else: - CRYPTOGRAPHY_AVAILABLE = True - -ROOT = pathlib.Path(__file__).resolve().parents[1] -MANIFEST_RELATIVE = pathlib.PurePosixPath("releases/manifests") -LEDGER_RELATIVE = pathlib.PurePosixPath("ledger/official_observations.jsonl") -PREFIX_RELATIVE = pathlib.PurePosixPath("ledger/immutable_prefix.json") -STATE_PATH = LEDGER_RELATIVE.as_posix() -SCHEMA_VERSION = "thesis_ledger_release_v1" -MAX_RELEASE_INDEX = 9_999 -DEFAULT_CLOCK_SKEW_SECONDS = 300 -MAX_FUTURE_SECONDS = 300 -SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") -MANIFEST_RE = re.compile(r"(?P[0-9]{4})-(?P[0-9a-f]{16})\.json\Z") -RECEIPT_RE = re.compile( - r"(?P[0-9]{4}-[0-9a-f]{16})" - r"\.(?Pfreetsa|digicert)\.tsr\Z" -) -PRODUCER_SIGNATURE_RE = re.compile( - r"(?P[0-9]{4}-[0-9a-f]{16})\.producer\.sig\Z" -) -PRODUCER_PUBLIC_KEY_FILENAME = "producer-ed25519.pub" -PRODUCER_SPKI_SHA256 = ( - "4a90eff40455ce0d853d4bab1608efbdae1efaf8c06054ead6e396c5b0c4846e" -) -PRODUCER_SIGNATURE_BYTES = 64 -STRICT_UTC_RE = re.compile( - r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:" - r"[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z\Z" -) -TIME_STAMP_RE = re.compile( - r"(?P[A-Z][a-z]{2})\s+" - r"(?P[0-9]{1,2})\s+" - r"(?P[0-9]{2}):(?P[0-9]{2}):" - r"(?P[0-9]{2})(?P\.[0-9]+)?\s+" - r"(?P[0-9]{4})\s+GMT\Z" -) - - -@dataclass(frozen=True) -class AnchorSpec: - filename: str - pem_sha256: str - policy_oid: str - signer_certificate_sha256: str - signer_spki_sha256: str - - -ANCHORS = { - "freetsa": AnchorSpec( - filename="freetsa-root-2016.pem", - pem_sha256=("2151b61137ffa86bf664691ba67e7da0b19f98c758e3d228d5d8ebf27e044438"), - policy_oid="1.2.3.4.1", - signer_certificate_sha256=( - "32e841a95cc1164101ffde41298ef2fc75c1c4372ef095e88a6bbd47dfb191fc" - ), - signer_spki_sha256=( - "fa02bd555e3e483d62b4e70be6218692068d2b0b0a7525db58dcbf2901cdb072" - ), - ), - "digicert": AnchorSpec( - filename="digicert-trusted-root-g4.pem", - pem_sha256=("ce7d6b44f5d510391be98c8d76b18709400a30cd87659bfebe1c6f97ff5181ee"), - policy_oid="2.16.840.1.114412.7.1", - signer_certificate_sha256=( - "4aa03fa22cd75c84c55c938f828e676b9caecab33fe36d269aa334f146110a33" - ), - signer_spki_sha256=( - "7abda95ed7301ac94bded350babc319903d0b4f16c4e7e39346dba5f9e992b72" - ), - ), -} - - -class ReleaseChainError(ValueError): - """The release journal is malformed, inconsistent, or untrusted.""" - - -@dataclass(frozen=True) -class GitEntry: - mode: str - object_type: str - object_id: str - path: str - - -@dataclass(frozen=True) -class ReleaseRecord: - path: pathlib.Path - raw: bytes - sha256: str - manifest: dict[str, Any] - receipt_paths: dict[str, pathlib.Path] - receipt_times: dict[str, datetime] - producer_signature_path: pathlib.Path - - @property - def release_index(self) -> int: - return int(self.manifest["releaseIndex"]) - - -@dataclass(frozen=True) -class ChainVerification: - releases: tuple[ReleaseRecord, ...] - - @property - def head(self) -> ReleaseRecord | None: - return self.releases[-1] if self.releases else None - - -def sha256_bytes(payload: bytes) -> str: - return hashlib.sha256(payload).hexdigest() - - -def _fail_json_constant(value: str) -> None: - raise ReleaseChainError(f"manifest contains non-JSON number {value!r}") - - -def _object_without_duplicates( - pairs: list[tuple[str, Any]], -) -> dict[str, Any]: - result: dict[str, Any] = {} - for key, value in pairs: - if key in result: - raise ReleaseChainError(f"manifest has duplicate key {key!r}") - result[key] = value - return result - - -def _exact_keys(value: Any, expected: set[str], label: str) -> dict[str, Any]: - if type(value) is not dict: - raise ReleaseChainError(f"{label} must be an object") - actual = set(value) - if actual != expected: - missing = sorted(expected - actual) - unknown = sorted(actual - expected) - raise ReleaseChainError( - f"{label} keys are not closed-world: missing={missing}, unknown={unknown}" - ) - return value + from vidimus_pins import LEDGER_SPEC +except ModuleNotFoundError as exc: + if exc.name != "vidimus_pins": + raise + # The test suite copies the legacy three-script surface into temporary + # repositories. The editable consumer tree remains the sole pin owner. + from scripts.vidimus_pins import LEDGER_SPEC -def _strict_int(value: Any, label: str, *, minimum: int = 0) -> int: - if type(value) is not int: - raise ReleaseChainError(f"{label} must be an integer, not a boolean") - if value < minimum: - raise ReleaseChainError(f"{label} must be >= {minimum}") - return value - - -def _strict_string(value: Any, label: str, *, nonempty: bool = True) -> str: - if type(value) is not str or (nonempty and not value): - suffix = " and non-empty" if nonempty else "" - raise ReleaseChainError(f"{label} must be a string{suffix}") - return value - - -def _sha256(value: Any, label: str) -> str: - if type(value) is not str or SHA256_RE.fullmatch(value) is None: - raise ReleaseChainError( - f"{label} must be exactly 64 lowercase hexadecimal characters" - ) - return value - - -def parse_created_at(value: Any, label: str = "createdAtUtc") -> datetime: - text = _strict_string(value, label) - if STRICT_UTC_RE.fullmatch(text) is None: - raise ReleaseChainError(f"{label} must be a strict UTC timestamp ending in Z") - try: - parsed = datetime.fromisoformat(text[:-1] + "+00:00") - except ValueError as exc: - raise ReleaseChainError(f"{label} is not a real UTC time: {text!r}") from exc - return parsed.astimezone(timezone.utc) +ROOT = pathlib.Path(__file__).resolve().parents[1] +MANIFEST_RELATIVE = LEDGER_SPEC.manifest_relative +LEDGER_RELATIVE = LEDGER_SPEC.state_relative +PREFIX_RELATIVE = LEDGER_SPEC.prefix_relative +STATE_PATH = LEDGER_SPEC.state_path +SCHEMA_VERSION = LEDGER_SPEC.schema_version +ANCHORS = LEDGER_SPEC.anchors +PRODUCER_PUBLIC_KEY_FILENAME = LEDGER_SPEC.producer_public_key_filename +PRODUCER_SPKI_SHA256 = LEDGER_SPEC.producer_spki_sha256 + +CRYPTOGRAPHY_AVAILABLE = _vidimus.CRYPTOGRAPHY_AVAILABLE +DEFAULT_CLOCK_SKEW_SECONDS = _vidimus.DEFAULT_CLOCK_SKEW_SECONDS +MAX_FUTURE_SECONDS = _vidimus.MAX_FUTURE_SECONDS +MAX_RELEASE_INDEX = _vidimus.MAX_RELEASE_INDEX +MANIFEST_RE = _vidimus.MANIFEST_RE +PRODUCER_SIGNATURE_BYTES = _vidimus.PRODUCER_SIGNATURE_BYTES +PRODUCER_SIGNATURE_RE = _vidimus.PRODUCER_SIGNATURE_RE +RECEIPT_RE = _vidimus._receipt_re(LEDGER_SPEC) +SHA256_RE = _vidimus.SHA256_RE +STRICT_UTC_RE = _vidimus.STRICT_UTC_RE +TIME_STAMP_RE = _vidimus.TIME_STAMP_RE + +AnchorSpec = _vidimus.AnchorSpec +ChainSpec = _vidimus.ChainSpec +ChainVerification = _vidimus.ChainVerification +GitEntry = _vidimus.GitEntry +ReleaseChainError = _vidimus.ReleaseChainError +ReleaseRecord = _vidimus.ReleaseRecord + +git_blob_bytes = _vidimus.git_blob_bytes +git_file_entry = _vidimus.git_file_entry +git_tree_entries = _vidimus.git_tree_entries +jsonl_line_offsets = _vidimus.jsonl_line_offsets +manifest_filename = _vidimus.manifest_filename +parse_created_at = _vidimus.parse_created_at +producer_signature_path_for_manifest = _vidimus.producer_signature_path_for_manifest +resolve_base_commit = _vidimus.resolve_base_commit +sha256_bytes = _vidimus.sha256_bytes def validate_manifest_schema(manifest: Any) -> dict[str, Any]: - """Validate the closed-world ``thesis_ledger_release_v1`` schema.""" - - payload = _exact_keys( - manifest, - { - "schemaVersion", - "releaseIndex", - "previousManifestSha256", - "state", - "append", - "createdAtUtc", - "producer", - }, - "manifest", - ) - if payload["schemaVersion"] != SCHEMA_VERSION: - raise ReleaseChainError( - f"unsupported manifest schema {payload['schemaVersion']!r}" - ) - index = _strict_int(payload["releaseIndex"], "releaseIndex") - if index > MAX_RELEASE_INDEX: - raise ReleaseChainError( - f"releaseIndex {index} exceeds the four-digit filename limit" - ) - - previous = payload["previousManifestSha256"] - if index == 0: - if previous is not None: - raise ReleaseChainError("genesis previousManifestSha256 must be null") - else: - _sha256(previous, "previousManifestSha256") - - state = _exact_keys( - payload["state"], - { - "path", - "jsonlSha256", - "lineCount", - "immutablePrefixSha256", - }, - "state", - ) - if state["path"] != STATE_PATH: - raise ReleaseChainError(f"state.path must be exactly {STATE_PATH!r}") - _sha256(state["jsonlSha256"], "state.jsonlSha256") - _strict_int(state["lineCount"], "state.lineCount") - _sha256( - state["immutablePrefixSha256"], - "state.immutablePrefixSha256", - ) - - append = payload["append"] - if index == 0: - if append is not None: - raise ReleaseChainError("genesis append must be null") - else: - append_block = _exact_keys( - append, - { - "previousLineCount", - "appendedRowCount", - "appendedBytesSha256", - }, - "append", - ) - _strict_int( - append_block["previousLineCount"], - "append.previousLineCount", - ) - _strict_int( - append_block["appendedRowCount"], - "append.appendedRowCount", - minimum=1, - ) - _sha256( - append_block["appendedBytesSha256"], - "append.appendedBytesSha256", - ) - - parse_created_at(payload["createdAtUtc"]) - producer = _exact_keys(payload["producer"], {"repo", "branch"}, "producer") - _strict_string(producer["repo"], "producer.repo") - _strict_string(producer["branch"], "producer.branch") - return payload + return _vidimus.validate_manifest_schema(manifest, LEDGER_SPEC) def load_manifest(path: pathlib.Path) -> tuple[dict[str, Any], bytes, str]: - if path.is_symlink() or not path.is_file(): - raise ReleaseChainError(f"manifest is not a regular file: {path}") - raw = path.read_bytes() - try: - text = raw.decode("utf-8") - except UnicodeDecodeError as exc: - raise ReleaseChainError(f"manifest is not UTF-8: {path}") from exc - try: - parsed = json.loads( - text, - object_pairs_hook=_object_without_duplicates, - parse_constant=_fail_json_constant, - ) - except json.JSONDecodeError as exc: - raise ReleaseChainError(f"manifest is not valid JSON: {path}: {exc}") from exc - payload = validate_manifest_schema(parsed) - expected = canonical_bytes(payload) + b"\n" - if raw != expected: - raise ReleaseChainError( - f"manifest bytes are not canonical JSON plus one newline: {path}" - ) - return payload, raw, sha256_bytes(raw) - - -def manifest_filename(index: int, raw: bytes) -> str: - _strict_int(index, "releaseIndex") - if index > MAX_RELEASE_INDEX: - raise ReleaseChainError( - f"releaseIndex {index} exceeds the four-digit filename limit" - ) - return f"{index:04d}-{sha256_bytes(raw)[:16]}.json" + return _vidimus.load_manifest(path, LEDGER_SPEC) def receipt_paths_for_manifest(path: pathlib.Path) -> dict[str, pathlib.Path]: - stem = path.stem - return {tsa: path.with_name(f"{stem}.{tsa}.tsr") for tsa in ANCHORS} - - -def producer_signature_path_for_manifest(path: pathlib.Path) -> pathlib.Path: - return path.with_name(f"{path.stem}.producer.sig") - - -def _enumerate_manifest_files( - root: pathlib.Path, -) -> list[tuple[pathlib.Path, dict[str, pathlib.Path], pathlib.Path]]: - directory = root / MANIFEST_RELATIVE - if not directory.exists(): - return [] - if directory.is_symlink() or not directory.is_dir(): - raise ReleaseChainError( - f"release manifest path is not a regular directory: {directory}" - ) - - manifests: dict[str, pathlib.Path] = {} - receipts: dict[str, dict[str, pathlib.Path]] = {} - producer_signatures: dict[str, pathlib.Path] = {} - for entry in directory.iterdir(): - if entry.is_symlink() or not entry.is_file(): - raise ReleaseChainError( - f"release manifest directory contains a non-regular entry: {entry}" - ) - manifest_match = MANIFEST_RE.fullmatch(entry.name) - if manifest_match is not None: - manifests[entry.stem] = entry - continue - receipt_match = RECEIPT_RE.fullmatch(entry.name) - if receipt_match is not None: - stem = receipt_match.group("stem") - tsa = receipt_match.group("tsa") - receipts.setdefault(stem, {})[tsa] = entry - continue - signature_match = PRODUCER_SIGNATURE_RE.fullmatch(entry.name) - if signature_match is not None: - producer_signatures[signature_match.group("stem")] = entry - continue - raise ReleaseChainError( - f"unknown file in closed release manifest directory: {entry.name}" - ) - - orphan_receipts = sorted(set(receipts) - set(manifests)) - if orphan_receipts: - raise ReleaseChainError( - f"orphan release receipts for manifest stems: {orphan_receipts}" - ) - orphan_signatures = sorted(set(producer_signatures) - set(manifests)) - if orphan_signatures: - raise ReleaseChainError( - "orphan producer signatures for manifest stems: " - f"{orphan_signatures}" - ) - result: list[ - tuple[pathlib.Path, dict[str, pathlib.Path], pathlib.Path] - ] = [] - seen_indices: dict[int, str] = {} - for stem, path in manifests.items(): - match = MANIFEST_RE.fullmatch(path.name) - assert match is not None - index = int(match.group("index")) - if index in seen_indices: - raise ReleaseChainError( - f"duplicate release index {index}: {seen_indices[index]}, {path.name}" - ) - seen_indices[index] = path.name - actual_receipts = receipts.get(stem, {}) - if set(actual_receipts) != set(ANCHORS): - raise ReleaseChainError( - f"manifest {path.name} must have exactly freetsa and digicert " - f"receipts; found={sorted(actual_receipts)}" - ) - producer_signature = producer_signatures.get(stem) - if producer_signature is None: - raise ReleaseChainError( - f"manifest {path.name} is missing its producer signature " - f"{stem}.producer.sig" - ) - result.append((path, actual_receipts, producer_signature)) - return sorted( - result, - key=lambda item: int(MANIFEST_RE.fullmatch(item[0].name).group("index")), - ) - - -def _openssl_environment(empty_ca_dir: pathlib.Path) -> dict[str, str]: - environment = os.environ.copy() - environment.update( - { - "LC_ALL": "C", - "OPENSSL_CONF": "/dev/null", - "SSL_CERT_DIR": str(empty_ca_dir), - "SSL_CERT_FILE": "/dev/null", - } - ) - return environment - - -def _command_error(completed: subprocess.CompletedProcess[str]) -> str: - details = (completed.stderr or completed.stdout).strip() - return details[-1000:] if details else "no OpenSSL diagnostic" - - -def _parse_receipt_text(output: str, receipt: pathlib.Path) -> tuple[datetime, str]: - status_lines = [ - line.strip() for line in output.splitlines() if line.startswith("Status:") - ] - if status_lines != ["Status: Granted."]: - raise ReleaseChainError( - f"RFC 3161 receipt is not granted for {receipt}: {status_lines}" - ) - hash_lines = [ - line.split(":", 1)[1].strip() - for line in output.splitlines() - if line.startswith("Hash Algorithm:") - ] - if hash_lines != ["sha256"]: - raise ReleaseChainError( - f"RFC 3161 receipt does not use SHA-256 for {receipt}: {hash_lines}" - ) - policy_lines = [ - line.split(":", 1)[1].strip() - for line in output.splitlines() - if line.startswith("Policy OID:") - ] - if len(policy_lines) != 1: - raise ReleaseChainError( - f"RFC 3161 receipt has no unique policy OID for {receipt}" - ) - time_lines = [ - line.split(":", 1)[1].strip() - for line in output.splitlines() - if line.startswith("Time stamp:") - ] - if len(time_lines) != 1: - raise ReleaseChainError(f"RFC 3161 receipt has no unique genTime for {receipt}") - match = TIME_STAMP_RE.fullmatch(time_lines[0]) - if match is None: - raise ReleaseChainError( - f"unsupported RFC 3161 genTime for {receipt}: {time_lines[0]!r}" - ) - timestamp = ( - f"{match.group('month')} {match.group('day')} " - f"{match.group('hour')}:{match.group('minute')}:" - f"{match.group('second')} {match.group('year')} GMT" - ) - try: - parsed = datetime.strptime(timestamp, "%b %d %H:%M:%S %Y GMT").replace( - tzinfo=timezone.utc - ) - except ValueError as exc: - raise ReleaseChainError( - f"invalid RFC 3161 genTime for {receipt}: {timestamp!r}" - ) from exc - fraction = match.group("fraction") - if fraction: - parsed = parsed.replace(microsecond=int((fraction[1:] + "000000")[:6])) - return parsed, policy_lines[0] - - -def _openssl_binary( - arguments: list[str], - *, - environment: dict[str, str], - label: str, -) -> bytes: - try: - completed = subprocess.run( - ["openssl", *arguments], - check=False, - capture_output=True, - env=environment, - ) - except FileNotFoundError as exc: - raise ReleaseChainError( - "openssl is required for RFC 3161 verification" - ) from exc - if completed.returncode != 0: - diagnostic = (completed.stderr or completed.stdout).decode( - "utf-8", errors="replace" - ) - raise ReleaseChainError( - f"OpenSSL {label} failed (exit {completed.returncode}): " - f"{diagnostic.strip()[-1000:]}" - ) - return completed.stdout - - -def _producer_openssl_binary( - arguments: list[str], - *, - environment: dict[str, str], - label: str, -) -> bytes: - try: - completed = subprocess.run( - ["openssl", *arguments], - check=False, - capture_output=True, - env=environment, - ) - except FileNotFoundError as exc: - raise ReleaseChainError( - "producer signature verification requires cryptography or OpenSSL 3" - ) from exc - if completed.returncode != 0: - diagnostic = (completed.stderr or completed.stdout).decode( - "utf-8", errors="replace" - ) - raise ReleaseChainError( - f"OpenSSL producer {label} failed (exit {completed.returncode}): " - f"{diagnostic.strip()[-1000:]}" - ) - return completed.stdout - - -def _verify_producer_signature_with_openssl( - manifest: bytes, - signature: bytes, - public_key_pem: bytes, - *, - enforce_production_pin: bool, - label: str, -) -> None: - with tempfile.TemporaryDirectory(prefix="thesis-release-producer-") as name: - temporary = pathlib.Path(name) - empty_ca_dir = temporary / "empty-ca" - empty_ca_dir.mkdir() - environment = _openssl_environment(empty_ca_dir) - manifest_path = temporary / "manifest.json" - signature_path = temporary / "producer.sig" - public_key_path = temporary / PRODUCER_PUBLIC_KEY_FILENAME - manifest_path.write_bytes(manifest) - signature_path.write_bytes(signature) - public_key_path.write_bytes(public_key_pem) - - spki_der = _producer_openssl_binary( - [ - "pkey", - "-pubin", - "-in", - str(public_key_path), - "-outform", - "DER", - ], - environment=environment, - label=f"public-key decoding for {label}", - ) - if enforce_production_pin: - spki_sha256 = sha256_bytes(spki_der) - if spki_sha256 != PRODUCER_SPKI_SHA256: - raise ReleaseChainError( - "producer public-key SPKI is not code-pinned: " - f"{spki_sha256}" - ) - - try: - _producer_openssl_binary( - [ - "pkeyutl", - "-verify", - "-pubin", - "-inkey", - str(public_key_path), - "-rawin", - "-in", - str(manifest_path), - "-sigfile", - str(signature_path), - ], - environment=environment, - label=f"Ed25519 signature verification for {label}", - ) - except ReleaseChainError as exc: - raise ReleaseChainError( - f"producer Ed25519 signature verification failed for {label}" - ) from exc + return _vidimus.receipt_paths_for_manifest(path, LEDGER_SPEC) def verify_producer_signature_bytes( @@ -624,59 +84,14 @@ def verify_producer_signature_bytes( enforce_production_pin: bool, label: str, ) -> None: - """Verify one raw Ed25519 signature over exact manifest bytes.""" - - if type(manifest) is not bytes: - raise ReleaseChainError("producer-signed manifest payload must be bytes") - if type(signature) is not bytes or len(signature) != PRODUCER_SIGNATURE_BYTES: - actual = len(signature) if isinstance(signature, bytes) else "non-bytes" - raise ReleaseChainError( - f"producer signature for {label} must be exactly " - f"{PRODUCER_SIGNATURE_BYTES} raw bytes; found={actual}" - ) - public_key_path = anchor_dir / PRODUCER_PUBLIC_KEY_FILENAME - if public_key_path.is_symlink() or not public_key_path.is_file(): - raise ReleaseChainError( - f"missing or non-regular producer public key: {public_key_path}" - ) - public_key_pem = public_key_path.read_bytes() - - if not CRYPTOGRAPHY_AVAILABLE: - _verify_producer_signature_with_openssl( - manifest, - signature, - public_key_pem, - enforce_production_pin=enforce_production_pin, - label=label, - ) - return - - try: - public_key = load_pem_public_key(public_key_pem) - except (TypeError, ValueError, UnsupportedAlgorithm) as exc: - raise ReleaseChainError( - f"cannot decode producer Ed25519 public key: {public_key_path}" - ) from exc - if not isinstance(public_key, Ed25519PublicKey): - raise ReleaseChainError( - f"producer public key is not Ed25519: {public_key_path}" - ) - spki_der = public_key.public_bytes( - Encoding.DER, - PublicFormat.SubjectPublicKeyInfo, + return _vidimus.verify_producer_signature_bytes( + manifest, + signature, + spec=LEDGER_SPEC, + anchor_dir=anchor_dir, + enforce_production_pin=enforce_production_pin, + label=label, ) - if enforce_production_pin: - spki_sha256 = sha256_bytes(spki_der) - if spki_sha256 != PRODUCER_SPKI_SHA256: - raise ReleaseChainError( - f"producer public-key SPKI is not code-pinned: {spki_sha256}" - ) - try: - public_key.verify(signature, manifest) - except InvalidSignature as exc: - raise ReleaseChainError( - f"producer Ed25519 signature verification failed for {label}" - ) from exc def verify_producer_signature( @@ -686,99 +101,15 @@ def verify_producer_signature( anchor_dir: pathlib.Path, enforce_production_pin: bool, ) -> None: - if signature_path.is_symlink() or not signature_path.is_file(): - raise ReleaseChainError( - f"missing or non-regular producer signature: {signature_path}" - ) - verify_producer_signature_bytes( + return _vidimus.verify_producer_signature( manifest, - signature_path.read_bytes(), + signature_path, + spec=LEDGER_SPEC, anchor_dir=anchor_dir, enforce_production_pin=enforce_production_pin, - label=signature_path.name, ) -def _verify_production_signer( - receipt: pathlib.Path, - anchor: pathlib.Path, - spec: AnchorSpec, - gen_time: datetime, - temporary: pathlib.Path, - environment: dict[str, str], -) -> None: - token = temporary / "token.der" - signer = temporary / "signer.pem" - content = temporary / "tst-info.der" - _openssl_binary( - [ - "ts", - "-reply", - "-config", - "/dev/null", - "-in", - str(receipt), - "-token_out", - "-out", - str(token), - ], - environment=environment, - label=f"token extraction for {receipt.name}", - ) - _openssl_binary( - [ - "cms", - "-verify", - "-inform", - "DER", - "-in", - str(token), - "-CAfile", - str(anchor), - "-no-CApath", - "-no-CAstore", - "-purpose", - "timestampsign", - "-attime", - str(int(gen_time.timestamp())), - "-signer", - str(signer), - "-out", - str(content), - ], - environment=environment, - label=f"signer extraction for {receipt.name}", - ) - certificate_der = _openssl_binary( - ["x509", "-in", str(signer), "-outform", "DER"], - environment=environment, - label=f"signer certificate decoding for {receipt.name}", - ) - public_key_pem = _openssl_binary( - ["x509", "-in", str(signer), "-pubkey", "-noout"], - environment=environment, - label=f"signer public-key extraction for {receipt.name}", - ) - public_key = temporary / "signer-public-key.pem" - public_key.write_bytes(public_key_pem) - public_key_der = _openssl_binary( - ["pkey", "-pubin", "-in", str(public_key), "-outform", "DER"], - environment=environment, - label=f"signer SPKI decoding for {receipt.name}", - ) - certificate_sha256 = sha256_bytes(certificate_der) - spki_sha256 = sha256_bytes(public_key_der) - if certificate_sha256 != spec.signer_certificate_sha256: - raise ReleaseChainError( - f"RFC 3161 signer certificate is not pinned for {receipt.name}: " - f"{certificate_sha256}" - ) - if spki_sha256 != spec.signer_spki_sha256: - raise ReleaseChainError( - f"RFC 3161 signer SPKI is not pinned for {receipt.name}: {spki_sha256}" - ) - - def verify_receipt( manifest_digest: str, receipt: pathlib.Path, @@ -788,107 +119,15 @@ def verify_receipt( enforce_production_pins: bool, now: datetime | None = None, ) -> datetime: - """Cryptographically verify one receipt and return its signed genTime.""" - - if tsa not in ANCHORS: - raise ReleaseChainError(f"unknown TSA receipt kind {tsa!r}") - _sha256(manifest_digest, "manifest digest") - if receipt.is_symlink() or not receipt.is_file(): - raise ReleaseChainError(f"missing or non-regular RFC 3161 receipt: {receipt}") - spec = ANCHORS[tsa] - anchor = anchor_dir / spec.filename - if anchor.is_symlink() or not anchor.is_file(): - raise ReleaseChainError(f"missing or non-regular TSA anchor: {anchor}") - if enforce_production_pins: - anchor_digest = sha256_bytes(anchor.read_bytes()) - if anchor_digest != spec.pem_sha256: - raise ReleaseChainError( - f"production TSA anchor bytes are not code-pinned for {tsa}: " - f"{anchor_digest}" - ) - - with tempfile.TemporaryDirectory(prefix="thesis-release-tsa-") as name: - temporary = pathlib.Path(name) - empty_ca_dir = temporary / "empty-ca" - empty_ca_dir.mkdir() - environment = _openssl_environment(empty_ca_dir) - try: - text_result = subprocess.run( - [ - "openssl", - "ts", - "-reply", - "-config", - "/dev/null", - "-in", - str(receipt), - "-text", - ], - check=False, - capture_output=True, - text=True, - env=environment, - ) - except FileNotFoundError as exc: - raise ReleaseChainError( - "openssl is required for RFC 3161 verification" - ) from exc - if text_result.returncode != 0: - raise ReleaseChainError( - f"cannot inspect RFC 3161 receipt {receipt} " - f"(exit {text_result.returncode}): {_command_error(text_result)}" - ) - gen_time, policy_oid = _parse_receipt_text(text_result.stdout, receipt) - if enforce_production_pins and policy_oid != spec.policy_oid: - raise ReleaseChainError( - f"RFC 3161 policy is not pinned for {tsa}: {policy_oid!r}" - ) - current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) - if gen_time > current + timedelta(seconds=MAX_FUTURE_SECONDS): - raise ReleaseChainError( - f"RFC 3161 genTime {gen_time.isoformat()} for {receipt.name} " - f"postdates verifier time {current.isoformat()}" - ) - - verify_result = subprocess.run( - [ - "openssl", - "ts", - "-verify", - "-config", - "/dev/null", - "-digest", - manifest_digest, - "-in", - str(receipt), - "-CAfile", - str(anchor), - "-CApath", - str(empty_ca_dir), - "-attime", - str(int(gen_time.timestamp())), - ], - check=False, - capture_output=True, - text=True, - env=environment, - ) - if verify_result.returncode != 0: - raise ReleaseChainError( - f"RFC 3161 verification failed for {receipt.name} " - f"(exit {verify_result.returncode}): " - f"{_command_error(verify_result)}" - ) - if enforce_production_pins: - _verify_production_signer( - receipt, - anchor, - spec, - gen_time, - temporary, - environment, - ) - return gen_time + return _vidimus.verify_receipt( + manifest_digest, + receipt, + tsa, + spec=LEDGER_SPEC, + anchor_dir=anchor_dir, + enforce_production_pins=enforce_production_pins, + now=now, + ) def verify_release_receipts( @@ -902,158 +141,17 @@ def verify_release_receipts( previous_times: dict[str, datetime] | None = None, now: datetime | None = None, ) -> dict[str, datetime]: - """Verify both receipts and their chronology for one manifest.""" - - if set(receipt_paths) != set(ANCHORS): - raise ReleaseChainError( - "release must have exactly freetsa and digicert receipt paths" - ) - receipt_times = { - tsa: verify_receipt( - manifest_digest, - receipt_path, - tsa, - anchor_dir=anchor_dir, - enforce_production_pins=enforce_production_pins, - now=now, - ) - for tsa, receipt_path in receipt_paths.items() - } - created_at = parse_created_at(manifest["createdAtUtc"]) - earliest_allowed = created_at - timedelta(seconds=clock_skew_seconds) - release_index = manifest["releaseIndex"] - for tsa, gen_time in receipt_times.items(): - if gen_time < earliest_allowed: - raise ReleaseChainError( - f"release {release_index} {tsa} genTime " - f"{gen_time.isoformat()} impossibly precedes createdAtUtc " - f"{created_at.isoformat()}" - ) - if previous_times is not None: - lower_bound = max(previous_times.values()) - timedelta( - seconds=clock_skew_seconds - ) - current_earliest = min(receipt_times.values()) - if current_earliest < lower_bound: - raise ReleaseChainError( - f"release {release_index} receipt chronology regresses: " - f"earliest current genTime {current_earliest.isoformat()} " - f"precedes latest prior genTime " - f"{max(previous_times.values()).isoformat()} beyond " - f"{clock_skew_seconds}s skew" - ) - return receipt_times - - -def jsonl_line_offsets(payload: bytes, label: str) -> list[int]: - """Return exact byte offsets after each non-empty LF-terminated row.""" - - try: - payload.decode("utf-8") - except UnicodeDecodeError as exc: - raise ReleaseChainError(f"{label} is not UTF-8") from exc - if not payload.endswith(b"\n"): - raise ReleaseChainError( - f"{label} must end with exactly one LF after its final JSONL row" - ) - rows = payload.split(b"\n") - if rows[-1] != b"": - raise AssertionError("split invariant") - rows = rows[:-1] - offsets = [0] - position = 0 - for number, row in enumerate(rows, start=1): - if not row.strip(): - raise ReleaseChainError(f"{label} row {number} is blank") - if row.endswith(b"\r"): - raise ReleaseChainError(f"{label} row {number} uses CRLF, not exact LF") - position += len(row) + 1 - offsets.append(position) - return offsets - - -def _regular_file_bytes(root: pathlib.Path, relative: pathlib.PurePosixPath) -> bytes: - path = root / relative - if path.is_symlink() or not path.is_file(): - raise ReleaseChainError( - f"required state file is missing or non-regular: {path}" - ) - return path.read_bytes() - - -def _verify_state_history( - records: list[ReleaseRecord], - root: pathlib.Path, - *, - require_head_current: bool, -) -> None: - ledger = _regular_file_bytes(root, LEDGER_RELATIVE) - prefix = _regular_file_bytes(root, PREFIX_RELATIVE) - offsets = jsonl_line_offsets(ledger, STATE_PATH) - total_lines = len(offsets) - 1 - prefix_digest = sha256_bytes(prefix) - - previous_line_count: int | None = None - for record in records: - state = record.manifest["state"] - line_count = int(state["lineCount"]) - if line_count > total_lines: - raise ReleaseChainError( - f"release {record.release_index} lineCount {line_count} exceeds " - f"working-tree line count {total_lines}" - ) - historical_bytes = ledger[: offsets[line_count]] - historical_digest = sha256_bytes(historical_bytes) - if historical_digest != state["jsonlSha256"]: - raise ReleaseChainError( - f"release {record.release_index} state.jsonlSha256 does not " - "match the exact historical JSONL prefix" - ) - if state["immutablePrefixSha256"] != prefix_digest: - raise ReleaseChainError( - f"release {record.release_index} immutablePrefixSha256 does " - "not match ledger/immutable_prefix.json" - ) - - if previous_line_count is not None: - append = record.manifest["append"] - assert isinstance(append, dict) - if line_count <= previous_line_count: - raise ReleaseChainError( - f"release {record.release_index} lineCount must strictly increase" - ) - if append["previousLineCount"] != previous_line_count: - raise ReleaseChainError( - f"release {record.release_index} append.previousLineCount " - "does not match the previous manifest" - ) - row_delta = line_count - previous_line_count - if append["appendedRowCount"] != row_delta: - raise ReleaseChainError( - f"release {record.release_index} appendedRowCount " - f"{append['appendedRowCount']} does not match line delta " - f"{row_delta}" - ) - suffix = ledger[offsets[previous_line_count] : offsets[line_count]] - suffix_digest = sha256_bytes(suffix) - if append["appendedBytesSha256"] != suffix_digest: - raise ReleaseChainError( - f"release {record.release_index} appendedBytesSha256 does " - "not match the exact byte suffix" - ) - previous_line_count = line_count - - if require_head_current: - head = records[-1] - if head.manifest["state"]["lineCount"] != total_lines: - raise ReleaseChainError( - f"HEAD release lineCount {head.manifest['state']['lineCount']} " - f"does not match working-tree line count {total_lines}" - ) - if head.manifest["state"]["jsonlSha256"] != sha256_bytes(ledger): - raise ReleaseChainError( - "HEAD release state.jsonlSha256 does not match working-tree bytes" - ) + return _vidimus.verify_release_receipts( + manifest, + manifest_digest, + receipt_paths, + spec=LEDGER_SPEC, + anchor_dir=anchor_dir, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + previous_times=previous_times, + now=now, + ) def verify_release_chain( @@ -1067,255 +165,23 @@ def verify_release_chain( clock_skew_seconds: int = DEFAULT_CLOCK_SKEW_SECONDS, now: datetime | None = None, ) -> ChainVerification: - """Verify all manifests, signatures, receipts, links, and state bytes.""" - - root = root.resolve() - default_anchor_dir = root / "releases" / "anchors" - selected_anchors = (anchor_dir or default_anchor_dir).resolve() - if enforce_production_pins is None: - enforce_production_pins = selected_anchors == default_anchor_dir - if type(clock_skew_seconds) is not int or clock_skew_seconds < 0: - raise ReleaseChainError("clock_skew_seconds must be a non-negative integer") - - enumerated = _enumerate_manifest_files(root) - if not enumerated: - if require_chain: - raise ReleaseChainError("release chain is absent; genesis is required") - return ChainVerification(()) - - records: list[ReleaseRecord] = [] - previous_hash: str | None = None - previous_times: dict[str, datetime] | None = None - verification_now = now or datetime.now(timezone.utc) - for expected_index, (path, receipt_paths, producer_signature_path) in enumerate( - enumerated - ): - manifest, raw, digest = load_manifest(path) - filename_match = MANIFEST_RE.fullmatch(path.name) - assert filename_match is not None - filename_index = int(filename_match.group("index")) - if filename_index != expected_index: - raise ReleaseChainError( - f"release indices are not contiguous from 0: expected " - f"{expected_index:04d}, found {filename_index:04d}" - ) - if manifest["releaseIndex"] != expected_index: - raise ReleaseChainError( - f"manifest releaseIndex {manifest['releaseIndex']} does not " - f"match filename index {expected_index}" - ) - if filename_match.group("digest") != digest[:16]: - raise ReleaseChainError( - f"manifest filename hash does not match exact file bytes: {path.name}" - ) - if manifest["previousManifestSha256"] != previous_hash: - raise ReleaseChainError( - f"release {expected_index} previousManifestSha256 does not " - "match the previous manifest file bytes" - ) - verify_producer_signature( - raw, - producer_signature_path, - anchor_dir=selected_anchors, - enforce_production_pin=enforce_production_pins, - ) - if records: - previous_line_count = records[-1].manifest["state"]["lineCount"] - line_count = manifest["state"]["lineCount"] - append = manifest["append"] - assert isinstance(append, dict) - if line_count <= previous_line_count: - raise ReleaseChainError( - f"release {expected_index} lineCount must strictly increase" - ) - if append["previousLineCount"] != previous_line_count: - raise ReleaseChainError( - f"release {expected_index} append.previousLineCount does " - "not match the previous manifest" - ) - row_delta = line_count - previous_line_count - if append["appendedRowCount"] != row_delta: - raise ReleaseChainError( - f"release {expected_index} appendedRowCount " - f"{append['appendedRowCount']} does not match line delta " - f"{row_delta}" - ) - - receipt_times = verify_release_receipts( - manifest, - digest, - receipt_paths, - anchor_dir=selected_anchors, - enforce_production_pins=enforce_production_pins, - clock_skew_seconds=clock_skew_seconds, - previous_times=previous_times, - now=verification_now, - ) - - records.append( - ReleaseRecord( - path=path, - raw=raw, - sha256=digest, - manifest=manifest, - receipt_paths=receipt_paths, - receipt_times=receipt_times, - producer_signature_path=producer_signature_path, - ) - ) - previous_hash = digest - previous_times = receipt_times - - if type(allow_pending_append) is not bool: - raise ReleaseChainError("allow_pending_append must be a boolean") - if allow_pending_append and not verify_state: - raise ReleaseChainError( - "allow_pending_append requires historical state verification" - ) - if verify_state: - _verify_state_history( - records, - root, - require_head_current=not allow_pending_append, - ) - return ChainVerification(tuple(records)) - - -def _git_run( - root: pathlib.Path, - arguments: list[str], - *, - text: bool = False, -) -> subprocess.CompletedProcess[Any]: - try: - return subprocess.run( - ["git", *arguments], - cwd=root, - check=False, - capture_output=True, - text=text, - ) - except FileNotFoundError as exc: - raise ReleaseChainError("git is required for --base-ref verification") from exc - - -def resolve_base_commit(root: pathlib.Path, base_ref: str) -> str: - completed = _git_run( + return _vidimus.verify_release_chain( root, - ["rev-parse", "--verify", "--end-of-options", f"{base_ref}^{{commit}}"], - text=True, - ) - if completed.returncode != 0: - raise ReleaseChainError( - f"cannot resolve base ref {base_ref!r} to a commit: " - f"{completed.stderr.strip()}" - ) - commit = completed.stdout.strip() - ancestor = _git_run(root, ["merge-base", "--is-ancestor", commit, "HEAD"]) - if ancestor.returncode != 0: - raise ReleaseChainError(f"base commit {commit} is not an ancestor of HEAD") - return commit - - -def git_tree_entries( - root: pathlib.Path, commit: str, pathspec: str -) -> dict[str, GitEntry]: - completed = _git_run( - root, - ["ls-tree", "-r", "-z", "--full-tree", commit, "--", pathspec], + spec=LEDGER_SPEC, + anchor_dir=anchor_dir, + require_chain=require_chain, + verify_state=verify_state, + allow_pending_append=allow_pending_append, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + now=now, ) - if completed.returncode != 0: - diagnostic = completed.stderr.decode("utf-8", errors="replace").strip() - raise ReleaseChainError( - f"cannot enumerate {pathspec} at base {commit}: {diagnostic}" - ) - entries: dict[str, GitEntry] = {} - for record in completed.stdout.split(b"\0"): - if not record: - continue - try: - metadata, raw_path = record.split(b"\t", 1) - mode, object_type, object_id = metadata.decode("ascii").split(" ") - path = raw_path.decode("utf-8") - except (ValueError, UnicodeDecodeError) as exc: - raise ReleaseChainError( - f"cannot parse git tree entry under {pathspec}" - ) from exc - if path in entries: - raise ReleaseChainError(f"duplicate git tree entry for {path}") - entries[path] = GitEntry(mode, object_type, object_id, path) - return entries - - -def git_blob_bytes(root: pathlib.Path, entry: GitEntry) -> bytes: - if entry.object_type != "blob": - raise ReleaseChainError( - f"base release entry is not a blob: {entry.path} ({entry.object_type})" - ) - completed = _git_run(root, ["cat-file", "blob", entry.object_id]) - if completed.returncode != 0: - diagnostic = completed.stderr.decode("utf-8", errors="replace").strip() - raise ReleaseChainError(f"cannot read base blob for {entry.path}: {diagnostic}") - return completed.stdout - - -def git_file_entry(root: pathlib.Path, commit: str, path: str) -> GitEntry: - entries = git_tree_entries(root, commit, path) - entry = entries.get(path) - if entry is None: - raise ReleaseChainError(f"required file {path} is absent at base {commit}") - return entry - - -def _working_release_files(root: pathlib.Path) -> dict[str, pathlib.Path]: - release_root = root / "releases" - if not release_root.exists(): - return {} - if release_root.is_symlink() or not release_root.is_dir(): - raise ReleaseChainError("releases must be a real directory, not a symlink") - files: dict[str, pathlib.Path] = {} - for path in release_root.rglob("*"): - relative = path.relative_to(root).as_posix() - if path.is_symlink(): - raise ReleaseChainError(f"release path is a symlink: {relative}") - if path.is_dir(): - continue - if not path.is_file(): - raise ReleaseChainError(f"release path is not regular: {relative}") - files[relative] = path - return files def verify_release_history_immutable( root: pathlib.Path, base_ref: str ) -> tuple[str, set[str], dict[str, GitEntry]]: - """Compare every base ``releases/`` file byte and mode to the candidate.""" - - root = root.resolve() - commit = resolve_base_commit(root, base_ref) - base_entries = git_tree_entries(root, commit, "releases") - current_files = _working_release_files(root) - for relative, entry in base_entries.items(): - if entry.mode not in {"100644", "100755"}: - raise ReleaseChainError( - f"base release entry has non-regular git mode {entry.mode}: {relative}" - ) - current = current_files.get(relative) - if current is None: - raise ReleaseChainError( - f"existing release file was deleted relative to {commit}: {relative}" - ) - candidate_mode = "100755" if current.stat().st_mode & 0o111 else "100644" - if candidate_mode != entry.mode: - raise ReleaseChainError( - f"existing release file mode changed relative to {commit}: " - f"{relative} ({entry.mode} -> {candidate_mode})" - ) - if current.read_bytes() != git_blob_bytes(root, entry): - raise ReleaseChainError( - f"existing release file bytes changed relative to {commit}: {relative}" - ) - return commit, set(current_files) - set(base_entries), base_entries + return _vidimus.verify_release_history_immutable(root, base_ref, LEDGER_SPEC) def materialize_base_tree( @@ -1324,17 +190,13 @@ def materialize_base_tree( destination: pathlib.Path, release_entries: dict[str, GitEntry], ) -> None: - entries = dict(release_entries) - for relative in (LEDGER_RELATIVE.as_posix(), PREFIX_RELATIVE.as_posix()): - entries[relative] = git_file_entry(root, commit, relative) - for relative, entry in entries.items(): - if entry.mode not in {"100644", "100755"}: - raise ReleaseChainError( - f"base tree entry has non-regular mode {entry.mode}: {relative}" - ) - output = destination / pathlib.PurePosixPath(relative) - output.parent.mkdir(parents=True, exist_ok=True) - output.write_bytes(git_blob_bytes(root, entry)) + return _vidimus.materialize_base_tree( + root, + commit, + destination, + release_entries, + LEDGER_SPEC, + ) def verify_base_release_chain( @@ -1346,22 +208,15 @@ def verify_base_release_chain( enforce_production_pins: bool = True, clock_skew_seconds: int = DEFAULT_CLOCK_SKEW_SECONDS, ) -> ChainVerification: - with tempfile.TemporaryDirectory(prefix="thesis-release-base-") as name: - base_root = pathlib.Path(name) - materialize_base_tree(root, commit, base_root, release_entries) - base_anchor_dir = anchor_dir or (base_root / "releases" / "anchors") - return verify_release_chain( - base_root, - anchor_dir=base_anchor_dir, - require_chain=True, - verify_state=True, - enforce_production_pins=enforce_production_pins, - clock_skew_seconds=clock_skew_seconds, - ) - - -def _format_time(value: datetime) -> str: - return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + return _vidimus.verify_base_release_chain( + root, + commit, + release_entries, + spec=LEDGER_SPEC, + anchor_dir=anchor_dir, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + ) def main() -> int: @@ -1417,7 +272,7 @@ def main() -> int: return 0 head = verification.releases[-1] receipt_summary = ", ".join( - f"{tsa}={_format_time(value)}" + f"{tsa}={_vidimus._format_time(value)}" for tsa, value in sorted(head.receipt_times.items()) ) print( @@ -1427,5 +282,51 @@ def main() -> int: return 0 +__all__ = [ + "ANCHORS", + "AnchorSpec", + "ChainSpec", + "ChainVerification", + "CRYPTOGRAPHY_AVAILABLE", + "DEFAULT_CLOCK_SKEW_SECONDS", + "GitEntry", + "LEDGER_RELATIVE", + "MANIFEST_RE", + "MANIFEST_RELATIVE", + "PREFIX_RELATIVE", + "PRODUCER_PUBLIC_KEY_FILENAME", + "PRODUCER_SIGNATURE_BYTES", + "PRODUCER_SIGNATURE_RE", + "PRODUCER_SPKI_SHA256", + "RECEIPT_RE", + "ROOT", + "ReleaseChainError", + "ReleaseRecord", + "SCHEMA_VERSION", + "STATE_PATH", + "git_blob_bytes", + "git_file_entry", + "git_tree_entries", + "jsonl_line_offsets", + "load_manifest", + "main", + "manifest_filename", + "materialize_base_tree", + "parse_created_at", + "producer_signature_path_for_manifest", + "receipt_paths_for_manifest", + "resolve_base_commit", + "sha256_bytes", + "validate_manifest_schema", + "verify_base_release_chain", + "verify_producer_signature", + "verify_producer_signature_bytes", + "verify_receipt", + "verify_release_chain", + "verify_release_history_immutable", + "verify_release_receipts", +] + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/vidimus_pins.py b/scripts/vidimus_pins.py new file mode 100644 index 0000000..49ee590 --- /dev/null +++ b/scripts/vidimus_pins.py @@ -0,0 +1,94 @@ +"""Consumer-owned trust and append-gate configuration for vidimus.""" + +from __future__ import annotations + +import pathlib + +from vidimus.append_gate import AppendGateSpec +from vidimus.release_chain import AnchorSpec, ChainSpec + + +LEDGER_SPEC = ChainSpec( + manifest_relative=pathlib.PurePosixPath("releases/manifests"), + state_relative=pathlib.PurePosixPath("ledger/official_observations.jsonl"), + prefix_relative=pathlib.PurePosixPath("ledger/immutable_prefix.json"), + anchor_relative=pathlib.PurePosixPath("releases/anchors"), + release_root_relative=pathlib.PurePosixPath("releases"), + schema_version="thesis_ledger_release_v1", + producer_public_key_filename="producer-ed25519.pub", + producer_spki_sha256=( + "4a90eff40455ce0d853d4bab1608efbdae1efaf8c06054ead6e396c5b0c4846e" + ), + anchors={ + "freetsa": AnchorSpec( + filename="freetsa-root-2016.pem", + pem_sha256=( + "2151b61137ffa86bf664691ba67e7da0b19f98c758e3d228d5d8ebf27e044438" + ), + policy_oid="1.2.3.4.1", + signer_certificate_sha256=( + "32e841a95cc1164101ffde41298ef2fc75c1c4372ef095e88a6bbd47dfb191fc" + ), + signer_spki_sha256=( + "fa02bd555e3e483d62b4e70be6218692068d2b0b0a7525db58dcbf2901cdb072" + ), + ), + "digicert": AnchorSpec( + filename="digicert-trusted-root-g4.pem", + pem_sha256=( + "ce7d6b44f5d510391be98c8d76b18709400a30cd87659bfebe1c6f97ff5181ee" + ), + policy_oid="2.16.840.1.114412.7.1", + signer_certificate_sha256=( + "4aa03fa22cd75c84c55c938f828e676b9caecab33fe36d269aa334f146110a33" + ), + signer_spki_sha256=( + "7abda95ed7301ac94bded350babc319903d0b4f16c4e7e39346dba5f9e992b72" + ), + ), + }, +) + + +APPEND_GATE_SPEC = AppendGateSpec( + chain=LEDGER_SPEC, + prefix_schema_version="thesis_facts_immutable_prefix_v1", + release_manifest_prefix="releases/manifests/", + genesis_support_files=frozenset( + { + "releases/README.md", + *( + f"releases/anchors/{anchor.filename}" + for anchor in LEDGER_SPEC.anchors.values() + ), + (f"releases/anchors/{LEDGER_SPEC.producer_public_key_filename}"), + } + ), + gate_surface=frozenset( + { + "scripts/check_thesis_facts_append.py", + "scripts/verify_release_chain.py", + "scripts/canonical_json.py", + "scripts/cut_release_manifest.py", + ".github/workflows/thesis-facts-append.yml", + "releases/anchors/**", + } + ), + data_surface=frozenset( + { + "ledger/**", + "releases/manifests/**", + } + ), + assertion_content_keys=( + "source_record_id", + "value", + "observed_at", + "period", + "geography", + "entity", + "aggregation", + "filters", + "domain", + ), +) From 78f65abd273a01a7f837b288a67af8b8ace3ea58 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 20 Jul 2026 16:09:52 -0400 Subject: [PATCH 3/3] Prove vidimus shim byte transparency --- .../vidimus_shim_originals/canonical_json.py | 143 ++ .../check_thesis_facts_append.py | 792 +++++++++ .../verify_release_chain.py | 1431 +++++++++++++++++ tests/test_vidimus_shim_transparency.py | 401 +++++ 4 files changed, 2767 insertions(+) create mode 100644 tests/fixtures/vidimus_shim_originals/canonical_json.py create mode 100644 tests/fixtures/vidimus_shim_originals/check_thesis_facts_append.py create mode 100644 tests/fixtures/vidimus_shim_originals/verify_release_chain.py create mode 100644 tests/test_vidimus_shim_transparency.py diff --git a/tests/fixtures/vidimus_shim_originals/canonical_json.py b/tests/fixtures/vidimus_shim_originals/canonical_json.py new file mode 100644 index 0000000..a20a78f --- /dev/null +++ b/tests/fixtures/vidimus_shim_originals/canonical_json.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Canonical JSON compatible with site/src/data/canonical-json.ts. + +Object keys are sorted by UTF-16 code units, not Unicode code points. Python's +normal string ordering differs for astral-plane keys, so callers must use the +key below rather than ``sort_keys=True``. Number formatting follows +ECMAScript JSON.stringify's fixed/scientific thresholds. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import sys +from decimal import Decimal +from typing import Any + + +def utf16_sort_key(value: str) -> bytes: + """Return the big-endian UTF-16 code units used by JavaScript ordering.""" + + return value.encode("utf-16-be", errors="surrogatepass") + + +def _serialize_string(value: str) -> str: + pieces = ['"'] + short_escapes = { + "\b": "\\b", + "\t": "\\t", + "\n": "\\n", + "\f": "\\f", + "\r": "\\r", + '"': '\\"', + "\\": "\\\\", + } + for char in value: + if char in short_escapes: + pieces.append(short_escapes[char]) + elif ord(char) < 0x20 or 0xD800 <= ord(char) <= 0xDFFF: + pieces.append(f"\\u{ord(char):04x}") + else: + pieces.append(char) + pieces.append('"') + return "".join(pieces) + + +def _serialize_float(value: float) -> str: + if not math.isfinite(value): + raise ValueError(f"Canonical JSON cannot serialize non-finite number: {value}") + if value == 0: + return "0" + + absolute = abs(value) + rendered = repr(value).lower() + if 1e-6 <= absolute < 1e21: + fixed = format(Decimal(rendered), "f") + if "." in fixed: + fixed = fixed.rstrip("0").rstrip(".") + return fixed + + if "e" not in rendered: + rendered = format(value, ".15e") + mantissa, exponent_text = rendered.split("e", 1) + mantissa = mantissa.rstrip("0").rstrip(".") + exponent = int(exponent_text) + sign = "+" if exponent >= 0 else "" + return f"{mantissa}e{sign}{exponent}" + + +def canonical_stringify(value: Any) -> str: + """Serialize JSON-compatible data exactly like canonicalStringify in TS.""" + + ancestors: set[int] = set() + + def serialize(item: Any) -> str: + if item is None: + return "null" + if item is True: + return "true" + if item is False: + return "false" + if isinstance(item, str): + return _serialize_string(item) + if isinstance(item, int): + if abs(item) > (2**53 - 1): + try: + return _serialize_float(float(item)) + except OverflowError as exc: + raise ValueError( + f"Canonical JSON integer exceeds Number range: {item}" + ) from exc + return str(item) + if isinstance(item, float): + return _serialize_float(item) + + if isinstance(item, (list, dict)): + identity = id(item) + if identity in ancestors: + raise ValueError("Canonical JSON cannot serialize circular structures") + ancestors.add(identity) + try: + if isinstance(item, list): + return "[" + ",".join(serialize(entry) for entry in item) + "]" + entries = [] + for key in sorted(item, key=utf16_sort_key): + if not isinstance(key, str): + raise TypeError("Canonical JSON object keys must be strings") + entries.append(f"{_serialize_string(key)}:{serialize(item[key])}") + return "{" + ",".join(entries) + "}" + finally: + ancestors.remove(identity) + + raise TypeError( + f"Canonical JSON cannot serialize value of type {type(item).__name__}" + ) + + return serialize(value) + + +def canonical_bytes(value: Any) -> bytes: + return canonical_stringify(value).encode("utf-8") + + +def canonical_sha256(value: Any) -> str: + return hashlib.sha256(canonical_bytes(value)).hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--sha256", action="store_true", help="print SHA-256 instead of JSON" + ) + args = parser.parse_args() + value = json.load(sys.stdin) + output = canonical_sha256(value) if args.sha256 else canonical_stringify(value) + sys.stdout.write(output + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/vidimus_shim_originals/check_thesis_facts_append.py b/tests/fixtures/vidimus_shim_originals/check_thesis_facts_append.py new file mode 100644 index 0000000..907da4b --- /dev/null +++ b/tests/fixtures/vidimus_shim_originals/check_thesis_facts_append.py @@ -0,0 +1,792 @@ +#!/usr/bin/env python3 +"""Gate every change to the thesis-facts observation ledger. + +The observation file is append-only with an immutable frozen prefix +(``ledger/immutable_prefix.json``). Resolver appends arrive as pull +requests; this checker is the deterministic review each proposal must pass +before merge: + +- the frozen prefix is byte-identical (no rewrite, no truncation); +- against a base ref, the change only appends whole lines; +- every appended row parses, and carries the post-quarantine bindings: + ``assertionVersion`` (content-addressed, recomputed here), ``retrievedAt``, + ``sourceVintage``, ``ledgerRepoSha``, and a ``responseArchive`` digest; +- ``targetContentHash`` and ``sourceBindingProjection`` appear together or + not at all, the projection's response digest matches the archive, and its + unit matches the row's measure unit; +- a duplicate ``source_record_id`` is legal only as an explicit correction: + the later row's ``assertionVersion.supersedes`` must name the version ID + of the row it replaces. +- after witnessed genesis, every byte append carries exactly one next canonical + release manifest, its producer signature, and two independently anchored + RFC 3161 receipts; all prior release files remain byte-immutable against the + PR's base commit. + +Usage: + python3 scripts/check_thesis_facts_append.py [--base-ref REF] + +With ``--base-ref`` (CI: the pull request's base commit) the append-only +diff is enforced; without it only the full-file invariants run. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import re +import subprocess +import sys +from typing import Any + +from canonical_json import canonical_sha256 +from verify_release_chain import ( + ANCHORS, + MANIFEST_RE, + PRODUCER_PUBLIC_KEY_FILENAME, + ReleaseChainError, + git_blob_bytes, + git_file_entry, + verify_base_release_chain, + verify_release_chain, + verify_release_history_immutable, +) + +CODE_ROOT = pathlib.Path(__file__).resolve().parents[1] +ROOT = CODE_ROOT +LEDGER_PATH = ROOT / "ledger" / "official_observations.jsonl" +PREFIX_PATH = ROOT / "ledger" / "immutable_prefix.json" +RELEASE_MANIFEST_PREFIX = "releases/manifests/" +GENESIS_SUPPORT_FILES = { + "releases/README.md", + *(f"releases/anchors/{spec.filename}" for spec in ANCHORS.values()), + f"releases/anchors/{PRODUCER_PUBLIC_KEY_FILENAME}", +} + +GATE_SURFACE = frozenset( + { + "scripts/check_thesis_facts_append.py", + "scripts/verify_release_chain.py", + "scripts/canonical_json.py", + "scripts/cut_release_manifest.py", + ".github/workflows/thesis-facts-append.yml", + "releases/anchors/**", + } +) +DATA_SURFACE = frozenset( + { + "ledger/**", + "releases/manifests/**", + } +) + +ASSERTION_CONTENT_KEYS = ( + "source_record_id", + "value", + "observed_at", + "period", + "geography", + "entity", + "aggregation", + "filters", + "domain", +) + + +class AppendError(ValueError): + """The proposed ledger change violates an append invariant.""" + + +def _set_root(root: pathlib.Path) -> None: + """Select the candidate worktree without changing the trusted code root. + + Pull-request CI executes this module from a detached checkout of the base + commit, while ``--root`` points at the checked-out PR merge tree. Imports + and production anchors therefore remain rooted at immutable ``CODE_ROOT``; + only candidate data paths and git comparisons use ``ROOT``. + """ + + global ROOT, LEDGER_PATH, PREFIX_PATH + ROOT = root.resolve() + LEDGER_PATH = ROOT / "ledger" / "official_observations.jsonl" + PREFIX_PATH = ROOT / "ledger" / "immutable_prefix.json" + + +def _git_output(arguments: list[str]) -> bytes: + try: + completed = subprocess.run( + ["git", *arguments], + cwd=ROOT, + check=False, + capture_output=True, + ) + except FileNotFoundError as exc: + raise AppendError("git is required for --base-ref verification") from exc + if completed.returncode != 0: + diagnostic = completed.stderr.decode( + "utf-8", errors="replace" + ).strip() + raise AppendError( + f"git {' '.join(arguments)} failed: {diagnostic}" + ) + return completed.stdout + + +def _resolve_base_commit(base_ref: str) -> str: + completed = _git_output( + ["rev-parse", "--verify", "--end-of-options", f"{base_ref}^{{commit}}"] + ) + commit = completed.decode("ascii").strip() + _git_output(["merge-base", "--is-ancestor", commit, "HEAD"]) + return commit + + +def _nul_paths(payload: bytes) -> set[str]: + return {os.fsdecode(path) for path in payload.split(b"\0") if path} + + +def _matches_surface(path: str, surface: frozenset[str]) -> bool: + for pattern in surface: + if pattern.endswith("/**"): + if path.startswith(pattern[:-2]): + return True + elif path == pattern: + return True + return False + + +def check_surface_separation(base_ref: str) -> tuple[set[str], set[str]]: + """Return data/gate changes and reject a proposal that combines them.""" + + commit = _resolve_base_commit(base_ref) + changed = _nul_paths( + _git_output( + [ + "diff", + "--name-only", + "-z", + "--no-renames", + "--no-ext-diff", + "--no-textconv", + commit, + "--", + ] + ) + ) + # ``git diff`` excludes untracked files. Tests mint release siblings before + # staging them, and a newly added anchor must still classify as gate code. + changed.update( + _nul_paths( + _git_output( + ["ls-files", "--others", "--exclude-standard", "-z", "--"] + ) + ) + ) + data_changes = { + path for path in changed if _matches_surface(path, DATA_SURFACE) + } + gate_changes = { + path for path in changed if _matches_surface(path, GATE_SURFACE) + } + if data_changes and gate_changes: + raise AppendError( + "mixed data/gate proposal is forbidden: DATA_SURFACE changes=" + f"{sorted(data_changes)}; GATE_SURFACE changes=" + f"{sorted(gate_changes)}; split them into separate pull requests" + ) + return data_changes, gate_changes + + +def _lines(text: str) -> list[str]: + return [line for line in text.split("\n") if line.strip()] + + +def reject_non_append_bytes(text: str) -> None: + """Reject blank/whitespace-only lines and any non-single trailing newline. + + ``_lines`` drops blank lines so row parsing is convenient, but that means a + blank line inserted into the frozen JSONL would normalize away and pass both + the prefix hash and the append-only diff. A JSONL row is exactly one + non-empty line: a blank/whitespace-only line inside the covered region is a + byte tamper, and the file must end with exactly one trailing newline. + """ + parts = text.split("\n") + if parts[-1] != "": + raise AppendError("ledger must end with exactly one trailing newline") + for index, part in enumerate(parts[:-1], start=1): + if not part.strip(): + raise AppendError( + f"line {index} is blank or whitespace-only; a JSONL row is one " + "non-empty line and a stray blank line is a tamper" + ) + + +def expected_assertion_version_id(row: dict[str, Any]) -> str: + """Recompute the content address the resolver must have written. + + Mirrors ``assertion_version`` in the Thesis resolver (av1 v2 spec): the ID + commits to everything that changes what the assertion MEANS — identity, + value, timing, population, the complete measure concept mapping, exact + source lineage/digest, row/cell lineage, and the archived response digest — + so an in-place edit is detectable and a correction must supersede + explicitly. This projection must stay byte-identical to the Brier writer's + ``assertion_version`` (both fed to the shared ``canonical_sha256``), so any + change here is a coordinated schema migration on both sides. + """ + measure = row.get("measure") or {} + source = row.get("source") or {} + projection = {key: row.get(key) for key in ASSERTION_CONTENT_KEYS} + projection["measure"] = { + "concept": measure.get("concept"), + "unit": measure.get("unit"), + "source_concept": measure.get("source_concept"), + "concept_relation": measure.get("concept_relation"), + "concept_authority": measure.get("concept_authority"), + "legal_vintage": measure.get("legal_vintage"), + } + projection["source"] = { + "source_name": source.get("source_name"), + "source_table": source.get("source_table"), + "source_file": source.get("source_file"), + "url": source.get("url"), + "vintage": source.get("vintage"), + "source_sha256": source.get("source_sha256"), + } + projection["lineage"] = { + "source_row_keys": row.get("source_row_keys"), + "source_cell_keys": row.get("source_cell_keys"), + } + projection["responseArchiveSha256"] = (row.get("responseArchive") or {}).get( + "sha256" + ) + return f"av2:{canonical_sha256(projection)}" + + +def _effective_assertion_id(row: dict[str, Any]) -> str: + """Return the row's effective assertion version ID. + + Post-cutover rows carry an explicit ``assertionVersion.id`` (validated + against the recomputed content address in :func:`check_rows`); legacy + pre-versioning rows are addressable by their recomputed content address. + Either way every row has exactly one effective ID that a correction must + name and that no later row may reissue. + """ + version = row.get("assertionVersion") + if isinstance(version, dict) and version.get("id"): + return str(version["id"]) + return expected_assertion_version_id(row) + + +def effective_current_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Return the latest non-superseded row per assertion identity. + + A correction names the version it replaces via + ``assertionVersion.supersedes``; the replaced row drops out of the current + view. Aggregate-fact validation runs on this supersede-aware view so a + legitimate correction (same semantic key, new value) is not mistaken for a + duplicate key. + """ + superseded: set[str] = set() + for row in rows: + version = row.get("assertionVersion") + if isinstance(version, dict) and version.get("supersedes"): + superseded.add(str(version["supersedes"])) + return [row for row in rows if _effective_assertion_id(row) not in superseded] + + +def check_prefix(lines: list[str]) -> dict[str, Any]: + prefix = json.loads(PREFIX_PATH.read_text()) + if prefix.get("schemaVersion") != "thesis_facts_immutable_prefix_v1": + raise AppendError( + f"unsupported prefix manifest schema {prefix.get('schemaVersion')!r}" + ) + count = int(prefix["prefixLineCount"]) + hashes = prefix["lineSha256s"] + if len(hashes) != count: + raise AppendError("prefix manifest line hashes disagree with its count") + if len(lines) < count: + raise AppendError( + f"ledger has {len(lines)} rows but the immutable prefix " + f"requires at least {count}" + ) + for index in range(count): + digest = hashlib.sha256(lines[index].encode("utf-8")).hexdigest() + if digest != hashes[index]: + row_id = json.loads(lines[index]).get("source_record_id", "?") + raise AppendError( + f"immutable prefix line {index + 1} ({row_id}) was rewritten" + ) + joined = hashlib.sha256( + ("\n".join(lines[:count]) + "\n").encode("utf-8") + ).hexdigest() + if joined != prefix["prefixSha256"]: + raise AppendError("immutable prefix cumulative hash mismatch") + return prefix + + +def check_rows(lines: list[str], prefix_count: int) -> None: + versions: dict[str, int] = {} + active_by_record_id: dict[str, tuple[int, str | None]] = {} + for number, line in enumerate(lines, start=1): + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise AppendError(f"line {number} is not valid JSON: {exc}") from exc + if not isinstance(row, dict): + raise AppendError(f"line {number} is not a JSON object") + record_id = row.get("source_record_id") + if not record_id: + raise AppendError(f"line {number} lacks source_record_id") + if not isinstance(row.get("value"), (int, float)): + raise AppendError(f"line {number} ({record_id}) has no numeric value") + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(row.get("observed_at", ""))): + raise AppendError(f"line {number} ({record_id}) has no observed_at date") + unit = (row.get("measure") or {}).get("unit") + if not unit: + raise AppendError(f"line {number} ({record_id}) has no measure unit") + + recomputed = expected_assertion_version_id(row) + version = row.get("assertionVersion") + supersedes = None + if version is not None: + if not isinstance(version, dict): + raise AppendError(f"line {number} assertionVersion is not an object") + version_id = str(version.get("id", "")) + supersedes = version.get("supersedes") + if version_id != recomputed: + raise AppendError( + f"line {number} ({record_id}) assertionVersion.id does not " + f"match its content ({version_id} != {recomputed})" + ) + effective_id = version_id + else: + # Pre-versioning rows are addressable by their recomputed content + # address; that ID is reserved just like an explicit one so a legacy + # synthetic ID cannot be silently reissued. + effective_id = recomputed + + # Reserve the effective ID of EVERY row. A collision means two rows + # claim the same assertion version — a duplicate legacy ID or an + # A->B->A chain trying to restore a superseded value. + if effective_id in versions: + raise AppendError( + f"line {number} restates assertion version {effective_id} " + f"from line {versions[effective_id]}" + ) + versions[effective_id] = number + + if number > prefix_count: + for field in ( + "retrievedAt", + "sourceVintage", + "ledgerRepoSha", + "responseArchive", + "assertionVersion", + ): + if not row.get(field): + raise AppendError( + f"appended line {number} ({record_id}) lacks {field}" + ) + archive = row["responseArchive"] + if not isinstance(archive, dict) or not archive.get("sha256"): + raise AppendError( + f"appended line {number} responseArchive lacks a digest" + ) + # Key PRESENCE pairs the binding, and present values must be + # shape-valid: truthiness accepted targetContentHash "" with a + # missing (or {}) projection, silently waiving the contract + # binding (found during the vidimus extraction review). + has_hash = "targetContentHash" in row + has_projection = "sourceBindingProjection" in row + if has_hash != has_projection: + raise AppendError( + f"appended line {number} ({record_id}) must carry " + "targetContentHash and sourceBindingProjection together" + ) + if has_hash: + content_hash = row["targetContentHash"] + if not isinstance(content_hash, str) or not re.fullmatch( + r"[0-9a-f]{64}", content_hash + ): + raise AppendError( + f"appended line {number} ({record_id}) " + "targetContentHash is not a SHA-256 hex digest" + ) + projection = row["sourceBindingProjection"] + if not isinstance(projection, dict) or not projection: + raise AppendError( + f"appended line {number} ({record_id}) " + "sourceBindingProjection must be a non-empty object" + ) + if projection.get("responseSha256") != archive.get("sha256"): + raise AppendError( + f"appended line {number} projection digest does not " + "match its archived response" + ) + if projection.get("unit") != unit: + raise AppendError( + f"appended line {number} projection unit " + f"{projection.get('unit')!r} contradicts the row unit " + f"{unit!r}" + ) + + previous = active_by_record_id.get(str(record_id)) + if previous is not None: + previous_line, previous_version = previous + if supersedes is None: + raise AppendError( + f"line {number} duplicates {record_id} (line " + f"{previous_line}) without superseding an assertion " + "version — corrections must be explicit" + ) + if supersedes != previous_version: + raise AppendError( + f"line {number} supersedes {supersedes} but the active " + f"version of {record_id} is {previous_version}" + ) + elif supersedes is not None: + raise AppendError( + f"line {number} supersedes {supersedes} but {record_id} has " + "no earlier row" + ) + active_by_record_id[str(record_id)] = (number, effective_id) + + +def check_append_only(base_ref: str, lines: list[str]) -> int: + relative = LEDGER_PATH.relative_to(ROOT).as_posix() + try: + base_text = subprocess.check_output( + ["git", "show", f"{base_ref}:{relative}"], cwd=ROOT, text=True + ) + except subprocess.CalledProcessError as exc: + raise AppendError(f"cannot read {relative} at base {base_ref}") from exc + base_lines = _lines(base_text) + if len(lines) < len(base_lines): + raise AppendError( + f"change truncates the ledger: {len(base_lines)} -> {len(lines)} rows" + ) + for index, line in enumerate(base_lines): + if lines[index] != line: + row_id = json.loads(line).get("source_record_id", "?") + raise AppendError( + f"change rewrites existing line {index + 1} ({row_id}); " + "the ledger is append-only — supersede instead" + ) + return len(lines) - len(base_lines) + + +def _manifest_at_ref(base_ref: str) -> dict[str, Any]: + relative = PREFIX_PATH.relative_to(ROOT).as_posix() + try: + text = subprocess.check_output( + ["git", "show", f"{base_ref}:{relative}"], cwd=ROOT, text=True + ) + except subprocess.CalledProcessError as exc: + raise AppendError( + f"cannot read {relative} at base {base_ref}" + ) from exc + return json.loads(text) + + +def check_prefix_anchored_to_base(base_ref: str, candidate_prefix: dict[str, Any]) -> int: + """Require the frozen prefix manifest to be unchanged from the base. + + The immutable-prefix manifest lives beside the ledger and is candidate- + controlled, so a PR could grow ``prefixLineCount`` over its own append and + have every post-cutover binding skipped (the appended row would count as + "prefix"). Growing the frozen prefix is an explicit, separately reviewed + migration — never part of the automated append path — so under a base ref + the count, cumulative hash, and per-line hashes must match the base exactly. + Returns the BASE prefix line count, which callers use as the post-cutover + binding boundary so a candidate-controlled count can never move it. + """ + base_prefix = _manifest_at_ref(base_ref) + for field in ("prefixLineCount", "prefixSha256", "lineSha256s"): + if candidate_prefix.get(field) != base_prefix.get(field): + raise AppendError( + f"immutable prefix manifest {field} changed vs base {base_ref}; " + "the frozen prefix cannot grow through the automated append path " + "— growing it is an explicit reviewed migration" + ) + return int(base_prefix["prefixLineCount"]) + + +def _release_triple( + new_files: set[str], + expected_index: int, + *, + allowed_support_files: set[str] | None = None, +) -> pathlib.Path: + """Require exactly one manifest, producer signature, and two receipts.""" + + manifest_files = [ + relative + for relative in new_files + if relative.startswith(RELEASE_MANIFEST_PREFIX) + and MANIFEST_RE.fullmatch(pathlib.PurePosixPath(relative).name) + ] + if len(manifest_files) != 1: + raise AppendError( + f"release proposal must add exactly one manifest for index " + f"{expected_index}; found {sorted(manifest_files)}" + ) + manifest_relative = manifest_files[0] + manifest_name = pathlib.PurePosixPath(manifest_relative).name + match = MANIFEST_RE.fullmatch(manifest_name) + assert match is not None + if int(match.group("index")) != expected_index: + raise AppendError( + f"release proposal index must be {expected_index}, not " + f"{int(match.group('index'))}" + ) + stem = pathlib.PurePosixPath(manifest_name).stem + expected = { + manifest_relative, + *(f"{RELEASE_MANIFEST_PREFIX}{stem}.{tsa}.tsr" for tsa in ANCHORS), + f"{RELEASE_MANIFEST_PREFIX}{stem}.producer.sig", + } + allowed = expected | (allowed_support_files or set()) + if new_files != expected and not ( + expected <= new_files and new_files <= allowed + ): + raise AppendError( + "release proposal must add its manifest, producer signature, and " + "exactly the freetsa and digicert receipts with no other releases/ " + "changes; " + f"missing={sorted(expected - new_files)}, " + f"extra={sorted(new_files - allowed)}" + ) + return ROOT / pathlib.PurePosixPath(manifest_relative) + + +def _base_ledger_bytes(commit: str) -> bytes: + relative = LEDGER_PATH.relative_to(ROOT).as_posix() + return git_blob_bytes(ROOT, git_file_entry(ROOT, commit, relative)) + + +def _check_exact_byte_append(base_bytes: bytes, candidate_bytes: bytes) -> bytes: + if not candidate_bytes.startswith(base_bytes): + raise AppendError( + "change is not an exact byte append to the base JSONL; existing " + "bytes, including line endings, are immutable" + ) + return candidate_bytes[len(base_bytes) :] + + +def check_release_proposal( + base_ref: str, + *, + anchor_dir: pathlib.Path | None = None, + enforce_production_pins: bool | None = None, +) -> int | None: + """Verify the base chain and the one allowed candidate transition. + + A pre-genesis base keeps legacy append proposals valid only while they do + not touch ``releases/``. Genesis may add the prescribed anchors and README + alongside its exact manifest/signature/receipt bundle. Once genesis exists, + all base release files are byte- and mode-immutable and a ledger byte + append must carry exactly one next release bundle. + """ + + try: + commit, new_files, base_release_entries = ( + verify_release_history_immutable(ROOT, base_ref) + ) + except ReleaseChainError as exc: + raise AppendError(str(exc)) from exc + + base_has_chain = any( + relative.startswith(RELEASE_MANIFEST_PREFIX) + for relative in base_release_entries + ) + candidate_has_chain = any( + path.is_file() + for path in (ROOT / "releases" / "manifests").glob("*.json") + ) if (ROOT / "releases" / "manifests").is_dir() else False + base_bytes = _base_ledger_bytes(commit) + candidate_bytes = LEDGER_PATH.read_bytes() + appended_bytes = _check_exact_byte_append(base_bytes, candidate_bytes) + ledger_changed = bool(appended_bytes) + if enforce_production_pins is None: + enforce_production_pins = anchor_dir is None + + if not base_has_chain: + if not candidate_has_chain: + if new_files: + raise AppendError( + "legacy pre-genesis proposal must not change releases/; " + "add a complete genesis manifest, producer signature, and " + "both receipts or no release files at all " + f"(changed={sorted(new_files)})" + ) + return None + _release_triple( + new_files, + 0, + allowed_support_files=GENESIS_SUPPORT_FILES, + ) + try: + verification = verify_release_chain( + ROOT, + anchor_dir=anchor_dir, + require_chain=True, + verify_state=True, + enforce_production_pins=enforce_production_pins, + ) + except ReleaseChainError as exc: + raise AppendError(str(exc)) from exc + if len(verification.releases) != 1: + raise AppendError( + "genesis proposal must create exactly one release at index 0" + ) + return 0 + + try: + base_verification = verify_base_release_chain( + ROOT, + commit, + base_release_entries, + anchor_dir=anchor_dir, + enforce_production_pins=enforce_production_pins, + ) + except ReleaseChainError as exc: + raise AppendError(f"base release chain is invalid: {exc}") from exc + assert base_verification.head is not None + expected_index = base_verification.head.release_index + 1 + + if ledger_changed: + _release_triple(new_files, expected_index) + elif new_files: + raise AppendError( + "release-only proposal is forbidden after genesis; a next release " + "must witness an actual ledger byte append" + ) + + try: + candidate_verification = verify_release_chain( + ROOT, + anchor_dir=anchor_dir, + require_chain=True, + verify_state=True, + enforce_production_pins=enforce_production_pins, + ) + except ReleaseChainError as exc: + raise AppendError(str(exc)) from exc + expected_length = len(base_verification.releases) + (1 if ledger_changed else 0) + if len(candidate_verification.releases) != expected_length: + raise AppendError( + f"release chain length must be {expected_length} for this proposal; " + f"found {len(candidate_verification.releases)}" + ) + return candidate_verification.head.release_index + + +def check_release_chain_without_base( + *, + anchor_dir: pathlib.Path | None = None, + enforce_production_pins: bool | None = None, +) -> int | None: + """On push, verify any initialized chain against working-tree state.""" + + manifest_directory = ROOT / "releases" / "manifests" + if not manifest_directory.is_dir() or not any(manifest_directory.iterdir()): + return None + if enforce_production_pins is None: + enforce_production_pins = anchor_dir is None + try: + verification = verify_release_chain( + ROOT, + anchor_dir=anchor_dir, + require_chain=True, + verify_state=True, + enforce_production_pins=enforce_production_pins, + ) + except ReleaseChainError as exc: + raise AppendError(str(exc)) from exc + assert verification.head is not None + return verification.head.release_index + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--root", + type=pathlib.Path, + default=CODE_ROOT, + help="candidate worktree root (defaults to the checker's repository)", + ) + parser.add_argument( + "--base-ref", + help="enforce an append-only diff against this git ref", + ) + parser.add_argument( + "--release-anchor-dir", + type=pathlib.Path, + help=argparse.SUPPRESS, + ) + args = parser.parse_args() + try: + _set_root(args.root) + if args.base_ref: + _data_changes, gate_changes = check_surface_separation(args.base_ref) + if gate_changes: + print( + "thesis-facts append check OK: gate-only proposal; " + "DATA_SURFACE unchanged; GATE_SURFACE changes=" + f"{sorted(gate_changes)}" + ) + return 0 + + text = LEDGER_PATH.read_text(encoding="utf-8") + reject_non_append_bytes(text) + lines = _lines(text) + prefix = check_prefix(lines) + # The post-cutover binding boundary is the BASE prefix count under a + # base ref, so a PR cannot grandfather an unbound append by growing the + # candidate manifest over it. Without a base ref (push) there is nothing + # to anchor against, so the candidate manifest is trusted for the + # full-file invariants only — base-anchoring requires the PR path. + binding_boundary = int(prefix["prefixLineCount"]) + appended = None + if args.base_ref: + binding_boundary = check_prefix_anchored_to_base(args.base_ref, prefix) + appended = check_append_only(args.base_ref, lines) + check_rows(lines, binding_boundary) + # On the PR path, CODE_ROOT is the detached base checkout. Production + # verification must use those immutable anchors and the base verifier's + # pins, never files supplied by the candidate worktree. The hidden test + # override remains unpinned and continues to use generated test anchors. + production_pins = args.release_anchor_dir is None + anchor_dir = args.release_anchor_dir or ( + CODE_ROOT / "releases" / "anchors" + ) + release_index = ( + check_release_proposal( + args.base_ref, + anchor_dir=anchor_dir, + enforce_production_pins=production_pins, + ) + if args.base_ref + else check_release_chain_without_base( + anchor_dir=anchor_dir, + enforce_production_pins=production_pins, + ) + ) + except AppendError as exc: + print(f"thesis-facts append check failed: {exc}", file=sys.stderr) + return 1 + suffix = f", +{appended} appended vs base" if appended is not None else "" + release_suffix = ( + f", release {release_index}" if release_index is not None else "" + ) + print( + f"thesis-facts append check OK: {len(lines)} rows, immutable prefix " + f"{prefix['prefixLineCount']}{suffix}{release_suffix}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/vidimus_shim_originals/verify_release_chain.py b/tests/fixtures/vidimus_shim_originals/verify_release_chain.py new file mode 100644 index 0000000..31c66d1 --- /dev/null +++ b/tests/fixtures/vidimus_shim_originals/verify_release_chain.py @@ -0,0 +1,1431 @@ +#!/usr/bin/env python3 +"""Offline verification for the witnessed thesis-ledger release chain. + +The verifier treats manifest, signature, and receipt bytes as an append-only +journal. It does not trust manifest provenance or timestamps supplied by the +producer: each manifest is canonical and content-addressed, every state and +append digest is recomputed from the current append-only JSONL, every manifest +has a valid signature from the pinned producer key, and both RFC 3161 receipts +are verified against separate, committed trust anchors. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import re +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any + +from canonical_json import canonical_bytes + +try: + from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + from cryptography.hazmat.primitives.serialization import ( + Encoding, + PublicFormat, + load_pem_public_key, + ) +except ImportError: # Bare pre-sync CI falls back to the OpenSSL 3 CLI below. + CRYPTOGRAPHY_AVAILABLE = False +else: + CRYPTOGRAPHY_AVAILABLE = True + +ROOT = pathlib.Path(__file__).resolve().parents[1] +MANIFEST_RELATIVE = pathlib.PurePosixPath("releases/manifests") +LEDGER_RELATIVE = pathlib.PurePosixPath("ledger/official_observations.jsonl") +PREFIX_RELATIVE = pathlib.PurePosixPath("ledger/immutable_prefix.json") +STATE_PATH = LEDGER_RELATIVE.as_posix() +SCHEMA_VERSION = "thesis_ledger_release_v1" +MAX_RELEASE_INDEX = 9_999 +DEFAULT_CLOCK_SKEW_SECONDS = 300 +MAX_FUTURE_SECONDS = 300 +SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") +MANIFEST_RE = re.compile(r"(?P[0-9]{4})-(?P[0-9a-f]{16})\.json\Z") +RECEIPT_RE = re.compile( + r"(?P[0-9]{4}-[0-9a-f]{16})" + r"\.(?Pfreetsa|digicert)\.tsr\Z" +) +PRODUCER_SIGNATURE_RE = re.compile( + r"(?P[0-9]{4}-[0-9a-f]{16})\.producer\.sig\Z" +) +PRODUCER_PUBLIC_KEY_FILENAME = "producer-ed25519.pub" +PRODUCER_SPKI_SHA256 = ( + "4a90eff40455ce0d853d4bab1608efbdae1efaf8c06054ead6e396c5b0c4846e" +) +PRODUCER_SIGNATURE_BYTES = 64 +STRICT_UTC_RE = re.compile( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:" + r"[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z\Z" +) +TIME_STAMP_RE = re.compile( + r"(?P[A-Z][a-z]{2})\s+" + r"(?P[0-9]{1,2})\s+" + r"(?P[0-9]{2}):(?P[0-9]{2}):" + r"(?P[0-9]{2})(?P\.[0-9]+)?\s+" + r"(?P[0-9]{4})\s+GMT\Z" +) + + +@dataclass(frozen=True) +class AnchorSpec: + filename: str + pem_sha256: str + policy_oid: str + signer_certificate_sha256: str + signer_spki_sha256: str + + +ANCHORS = { + "freetsa": AnchorSpec( + filename="freetsa-root-2016.pem", + pem_sha256=("2151b61137ffa86bf664691ba67e7da0b19f98c758e3d228d5d8ebf27e044438"), + policy_oid="1.2.3.4.1", + signer_certificate_sha256=( + "32e841a95cc1164101ffde41298ef2fc75c1c4372ef095e88a6bbd47dfb191fc" + ), + signer_spki_sha256=( + "fa02bd555e3e483d62b4e70be6218692068d2b0b0a7525db58dcbf2901cdb072" + ), + ), + "digicert": AnchorSpec( + filename="digicert-trusted-root-g4.pem", + pem_sha256=("ce7d6b44f5d510391be98c8d76b18709400a30cd87659bfebe1c6f97ff5181ee"), + policy_oid="2.16.840.1.114412.7.1", + signer_certificate_sha256=( + "4aa03fa22cd75c84c55c938f828e676b9caecab33fe36d269aa334f146110a33" + ), + signer_spki_sha256=( + "7abda95ed7301ac94bded350babc319903d0b4f16c4e7e39346dba5f9e992b72" + ), + ), +} + + +class ReleaseChainError(ValueError): + """The release journal is malformed, inconsistent, or untrusted.""" + + +@dataclass(frozen=True) +class GitEntry: + mode: str + object_type: str + object_id: str + path: str + + +@dataclass(frozen=True) +class ReleaseRecord: + path: pathlib.Path + raw: bytes + sha256: str + manifest: dict[str, Any] + receipt_paths: dict[str, pathlib.Path] + receipt_times: dict[str, datetime] + producer_signature_path: pathlib.Path + + @property + def release_index(self) -> int: + return int(self.manifest["releaseIndex"]) + + +@dataclass(frozen=True) +class ChainVerification: + releases: tuple[ReleaseRecord, ...] + + @property + def head(self) -> ReleaseRecord | None: + return self.releases[-1] if self.releases else None + + +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _fail_json_constant(value: str) -> None: + raise ReleaseChainError(f"manifest contains non-JSON number {value!r}") + + +def _object_without_duplicates( + pairs: list[tuple[str, Any]], +) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ReleaseChainError(f"manifest has duplicate key {key!r}") + result[key] = value + return result + + +def _exact_keys(value: Any, expected: set[str], label: str) -> dict[str, Any]: + if type(value) is not dict: + raise ReleaseChainError(f"{label} must be an object") + actual = set(value) + if actual != expected: + missing = sorted(expected - actual) + unknown = sorted(actual - expected) + raise ReleaseChainError( + f"{label} keys are not closed-world: missing={missing}, unknown={unknown}" + ) + return value + + +def _strict_int(value: Any, label: str, *, minimum: int = 0) -> int: + if type(value) is not int: + raise ReleaseChainError(f"{label} must be an integer, not a boolean") + if value < minimum: + raise ReleaseChainError(f"{label} must be >= {minimum}") + return value + + +def _strict_string(value: Any, label: str, *, nonempty: bool = True) -> str: + if type(value) is not str or (nonempty and not value): + suffix = " and non-empty" if nonempty else "" + raise ReleaseChainError(f"{label} must be a string{suffix}") + return value + + +def _sha256(value: Any, label: str) -> str: + if type(value) is not str or SHA256_RE.fullmatch(value) is None: + raise ReleaseChainError( + f"{label} must be exactly 64 lowercase hexadecimal characters" + ) + return value + + +def parse_created_at(value: Any, label: str = "createdAtUtc") -> datetime: + text = _strict_string(value, label) + if STRICT_UTC_RE.fullmatch(text) is None: + raise ReleaseChainError(f"{label} must be a strict UTC timestamp ending in Z") + try: + parsed = datetime.fromisoformat(text[:-1] + "+00:00") + except ValueError as exc: + raise ReleaseChainError(f"{label} is not a real UTC time: {text!r}") from exc + return parsed.astimezone(timezone.utc) + + +def validate_manifest_schema(manifest: Any) -> dict[str, Any]: + """Validate the closed-world ``thesis_ledger_release_v1`` schema.""" + + payload = _exact_keys( + manifest, + { + "schemaVersion", + "releaseIndex", + "previousManifestSha256", + "state", + "append", + "createdAtUtc", + "producer", + }, + "manifest", + ) + if payload["schemaVersion"] != SCHEMA_VERSION: + raise ReleaseChainError( + f"unsupported manifest schema {payload['schemaVersion']!r}" + ) + index = _strict_int(payload["releaseIndex"], "releaseIndex") + if index > MAX_RELEASE_INDEX: + raise ReleaseChainError( + f"releaseIndex {index} exceeds the four-digit filename limit" + ) + + previous = payload["previousManifestSha256"] + if index == 0: + if previous is not None: + raise ReleaseChainError("genesis previousManifestSha256 must be null") + else: + _sha256(previous, "previousManifestSha256") + + state = _exact_keys( + payload["state"], + { + "path", + "jsonlSha256", + "lineCount", + "immutablePrefixSha256", + }, + "state", + ) + if state["path"] != STATE_PATH: + raise ReleaseChainError(f"state.path must be exactly {STATE_PATH!r}") + _sha256(state["jsonlSha256"], "state.jsonlSha256") + _strict_int(state["lineCount"], "state.lineCount") + _sha256( + state["immutablePrefixSha256"], + "state.immutablePrefixSha256", + ) + + append = payload["append"] + if index == 0: + if append is not None: + raise ReleaseChainError("genesis append must be null") + else: + append_block = _exact_keys( + append, + { + "previousLineCount", + "appendedRowCount", + "appendedBytesSha256", + }, + "append", + ) + _strict_int( + append_block["previousLineCount"], + "append.previousLineCount", + ) + _strict_int( + append_block["appendedRowCount"], + "append.appendedRowCount", + minimum=1, + ) + _sha256( + append_block["appendedBytesSha256"], + "append.appendedBytesSha256", + ) + + parse_created_at(payload["createdAtUtc"]) + producer = _exact_keys(payload["producer"], {"repo", "branch"}, "producer") + _strict_string(producer["repo"], "producer.repo") + _strict_string(producer["branch"], "producer.branch") + return payload + + +def load_manifest(path: pathlib.Path) -> tuple[dict[str, Any], bytes, str]: + if path.is_symlink() or not path.is_file(): + raise ReleaseChainError(f"manifest is not a regular file: {path}") + raw = path.read_bytes() + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise ReleaseChainError(f"manifest is not UTF-8: {path}") from exc + try: + parsed = json.loads( + text, + object_pairs_hook=_object_without_duplicates, + parse_constant=_fail_json_constant, + ) + except json.JSONDecodeError as exc: + raise ReleaseChainError(f"manifest is not valid JSON: {path}: {exc}") from exc + payload = validate_manifest_schema(parsed) + expected = canonical_bytes(payload) + b"\n" + if raw != expected: + raise ReleaseChainError( + f"manifest bytes are not canonical JSON plus one newline: {path}" + ) + return payload, raw, sha256_bytes(raw) + + +def manifest_filename(index: int, raw: bytes) -> str: + _strict_int(index, "releaseIndex") + if index > MAX_RELEASE_INDEX: + raise ReleaseChainError( + f"releaseIndex {index} exceeds the four-digit filename limit" + ) + return f"{index:04d}-{sha256_bytes(raw)[:16]}.json" + + +def receipt_paths_for_manifest(path: pathlib.Path) -> dict[str, pathlib.Path]: + stem = path.stem + return {tsa: path.with_name(f"{stem}.{tsa}.tsr") for tsa in ANCHORS} + + +def producer_signature_path_for_manifest(path: pathlib.Path) -> pathlib.Path: + return path.with_name(f"{path.stem}.producer.sig") + + +def _enumerate_manifest_files( + root: pathlib.Path, +) -> list[tuple[pathlib.Path, dict[str, pathlib.Path], pathlib.Path]]: + directory = root / MANIFEST_RELATIVE + if not directory.exists(): + return [] + if directory.is_symlink() or not directory.is_dir(): + raise ReleaseChainError( + f"release manifest path is not a regular directory: {directory}" + ) + + manifests: dict[str, pathlib.Path] = {} + receipts: dict[str, dict[str, pathlib.Path]] = {} + producer_signatures: dict[str, pathlib.Path] = {} + for entry in directory.iterdir(): + if entry.is_symlink() or not entry.is_file(): + raise ReleaseChainError( + f"release manifest directory contains a non-regular entry: {entry}" + ) + manifest_match = MANIFEST_RE.fullmatch(entry.name) + if manifest_match is not None: + manifests[entry.stem] = entry + continue + receipt_match = RECEIPT_RE.fullmatch(entry.name) + if receipt_match is not None: + stem = receipt_match.group("stem") + tsa = receipt_match.group("tsa") + receipts.setdefault(stem, {})[tsa] = entry + continue + signature_match = PRODUCER_SIGNATURE_RE.fullmatch(entry.name) + if signature_match is not None: + producer_signatures[signature_match.group("stem")] = entry + continue + raise ReleaseChainError( + f"unknown file in closed release manifest directory: {entry.name}" + ) + + orphan_receipts = sorted(set(receipts) - set(manifests)) + if orphan_receipts: + raise ReleaseChainError( + f"orphan release receipts for manifest stems: {orphan_receipts}" + ) + orphan_signatures = sorted(set(producer_signatures) - set(manifests)) + if orphan_signatures: + raise ReleaseChainError( + "orphan producer signatures for manifest stems: " + f"{orphan_signatures}" + ) + result: list[ + tuple[pathlib.Path, dict[str, pathlib.Path], pathlib.Path] + ] = [] + seen_indices: dict[int, str] = {} + for stem, path in manifests.items(): + match = MANIFEST_RE.fullmatch(path.name) + assert match is not None + index = int(match.group("index")) + if index in seen_indices: + raise ReleaseChainError( + f"duplicate release index {index}: {seen_indices[index]}, {path.name}" + ) + seen_indices[index] = path.name + actual_receipts = receipts.get(stem, {}) + if set(actual_receipts) != set(ANCHORS): + raise ReleaseChainError( + f"manifest {path.name} must have exactly freetsa and digicert " + f"receipts; found={sorted(actual_receipts)}" + ) + producer_signature = producer_signatures.get(stem) + if producer_signature is None: + raise ReleaseChainError( + f"manifest {path.name} is missing its producer signature " + f"{stem}.producer.sig" + ) + result.append((path, actual_receipts, producer_signature)) + return sorted( + result, + key=lambda item: int(MANIFEST_RE.fullmatch(item[0].name).group("index")), + ) + + +def _openssl_environment(empty_ca_dir: pathlib.Path) -> dict[str, str]: + environment = os.environ.copy() + environment.update( + { + "LC_ALL": "C", + "OPENSSL_CONF": "/dev/null", + "SSL_CERT_DIR": str(empty_ca_dir), + "SSL_CERT_FILE": "/dev/null", + } + ) + return environment + + +def _command_error(completed: subprocess.CompletedProcess[str]) -> str: + details = (completed.stderr or completed.stdout).strip() + return details[-1000:] if details else "no OpenSSL diagnostic" + + +def _parse_receipt_text(output: str, receipt: pathlib.Path) -> tuple[datetime, str]: + status_lines = [ + line.strip() for line in output.splitlines() if line.startswith("Status:") + ] + if status_lines != ["Status: Granted."]: + raise ReleaseChainError( + f"RFC 3161 receipt is not granted for {receipt}: {status_lines}" + ) + hash_lines = [ + line.split(":", 1)[1].strip() + for line in output.splitlines() + if line.startswith("Hash Algorithm:") + ] + if hash_lines != ["sha256"]: + raise ReleaseChainError( + f"RFC 3161 receipt does not use SHA-256 for {receipt}: {hash_lines}" + ) + policy_lines = [ + line.split(":", 1)[1].strip() + for line in output.splitlines() + if line.startswith("Policy OID:") + ] + if len(policy_lines) != 1: + raise ReleaseChainError( + f"RFC 3161 receipt has no unique policy OID for {receipt}" + ) + time_lines = [ + line.split(":", 1)[1].strip() + for line in output.splitlines() + if line.startswith("Time stamp:") + ] + if len(time_lines) != 1: + raise ReleaseChainError(f"RFC 3161 receipt has no unique genTime for {receipt}") + match = TIME_STAMP_RE.fullmatch(time_lines[0]) + if match is None: + raise ReleaseChainError( + f"unsupported RFC 3161 genTime for {receipt}: {time_lines[0]!r}" + ) + timestamp = ( + f"{match.group('month')} {match.group('day')} " + f"{match.group('hour')}:{match.group('minute')}:" + f"{match.group('second')} {match.group('year')} GMT" + ) + try: + parsed = datetime.strptime(timestamp, "%b %d %H:%M:%S %Y GMT").replace( + tzinfo=timezone.utc + ) + except ValueError as exc: + raise ReleaseChainError( + f"invalid RFC 3161 genTime for {receipt}: {timestamp!r}" + ) from exc + fraction = match.group("fraction") + if fraction: + parsed = parsed.replace(microsecond=int((fraction[1:] + "000000")[:6])) + return parsed, policy_lines[0] + + +def _openssl_binary( + arguments: list[str], + *, + environment: dict[str, str], + label: str, +) -> bytes: + try: + completed = subprocess.run( + ["openssl", *arguments], + check=False, + capture_output=True, + env=environment, + ) + except FileNotFoundError as exc: + raise ReleaseChainError( + "openssl is required for RFC 3161 verification" + ) from exc + if completed.returncode != 0: + diagnostic = (completed.stderr or completed.stdout).decode( + "utf-8", errors="replace" + ) + raise ReleaseChainError( + f"OpenSSL {label} failed (exit {completed.returncode}): " + f"{diagnostic.strip()[-1000:]}" + ) + return completed.stdout + + +def _producer_openssl_binary( + arguments: list[str], + *, + environment: dict[str, str], + label: str, +) -> bytes: + try: + completed = subprocess.run( + ["openssl", *arguments], + check=False, + capture_output=True, + env=environment, + ) + except FileNotFoundError as exc: + raise ReleaseChainError( + "producer signature verification requires cryptography or OpenSSL 3" + ) from exc + if completed.returncode != 0: + diagnostic = (completed.stderr or completed.stdout).decode( + "utf-8", errors="replace" + ) + raise ReleaseChainError( + f"OpenSSL producer {label} failed (exit {completed.returncode}): " + f"{diagnostic.strip()[-1000:]}" + ) + return completed.stdout + + +def _verify_producer_signature_with_openssl( + manifest: bytes, + signature: bytes, + public_key_pem: bytes, + *, + enforce_production_pin: bool, + label: str, +) -> None: + with tempfile.TemporaryDirectory(prefix="thesis-release-producer-") as name: + temporary = pathlib.Path(name) + empty_ca_dir = temporary / "empty-ca" + empty_ca_dir.mkdir() + environment = _openssl_environment(empty_ca_dir) + manifest_path = temporary / "manifest.json" + signature_path = temporary / "producer.sig" + public_key_path = temporary / PRODUCER_PUBLIC_KEY_FILENAME + manifest_path.write_bytes(manifest) + signature_path.write_bytes(signature) + public_key_path.write_bytes(public_key_pem) + + spki_der = _producer_openssl_binary( + [ + "pkey", + "-pubin", + "-in", + str(public_key_path), + "-outform", + "DER", + ], + environment=environment, + label=f"public-key decoding for {label}", + ) + if enforce_production_pin: + spki_sha256 = sha256_bytes(spki_der) + if spki_sha256 != PRODUCER_SPKI_SHA256: + raise ReleaseChainError( + "producer public-key SPKI is not code-pinned: " + f"{spki_sha256}" + ) + + try: + _producer_openssl_binary( + [ + "pkeyutl", + "-verify", + "-pubin", + "-inkey", + str(public_key_path), + "-rawin", + "-in", + str(manifest_path), + "-sigfile", + str(signature_path), + ], + environment=environment, + label=f"Ed25519 signature verification for {label}", + ) + except ReleaseChainError as exc: + raise ReleaseChainError( + f"producer Ed25519 signature verification failed for {label}" + ) from exc + + +def verify_producer_signature_bytes( + manifest: bytes, + signature: bytes, + *, + anchor_dir: pathlib.Path, + enforce_production_pin: bool, + label: str, +) -> None: + """Verify one raw Ed25519 signature over exact manifest bytes.""" + + if type(manifest) is not bytes: + raise ReleaseChainError("producer-signed manifest payload must be bytes") + if type(signature) is not bytes or len(signature) != PRODUCER_SIGNATURE_BYTES: + actual = len(signature) if isinstance(signature, bytes) else "non-bytes" + raise ReleaseChainError( + f"producer signature for {label} must be exactly " + f"{PRODUCER_SIGNATURE_BYTES} raw bytes; found={actual}" + ) + public_key_path = anchor_dir / PRODUCER_PUBLIC_KEY_FILENAME + if public_key_path.is_symlink() or not public_key_path.is_file(): + raise ReleaseChainError( + f"missing or non-regular producer public key: {public_key_path}" + ) + public_key_pem = public_key_path.read_bytes() + + if not CRYPTOGRAPHY_AVAILABLE: + _verify_producer_signature_with_openssl( + manifest, + signature, + public_key_pem, + enforce_production_pin=enforce_production_pin, + label=label, + ) + return + + try: + public_key = load_pem_public_key(public_key_pem) + except (TypeError, ValueError, UnsupportedAlgorithm) as exc: + raise ReleaseChainError( + f"cannot decode producer Ed25519 public key: {public_key_path}" + ) from exc + if not isinstance(public_key, Ed25519PublicKey): + raise ReleaseChainError( + f"producer public key is not Ed25519: {public_key_path}" + ) + spki_der = public_key.public_bytes( + Encoding.DER, + PublicFormat.SubjectPublicKeyInfo, + ) + if enforce_production_pin: + spki_sha256 = sha256_bytes(spki_der) + if spki_sha256 != PRODUCER_SPKI_SHA256: + raise ReleaseChainError( + f"producer public-key SPKI is not code-pinned: {spki_sha256}" + ) + try: + public_key.verify(signature, manifest) + except InvalidSignature as exc: + raise ReleaseChainError( + f"producer Ed25519 signature verification failed for {label}" + ) from exc + + +def verify_producer_signature( + manifest: bytes, + signature_path: pathlib.Path, + *, + anchor_dir: pathlib.Path, + enforce_production_pin: bool, +) -> None: + if signature_path.is_symlink() or not signature_path.is_file(): + raise ReleaseChainError( + f"missing or non-regular producer signature: {signature_path}" + ) + verify_producer_signature_bytes( + manifest, + signature_path.read_bytes(), + anchor_dir=anchor_dir, + enforce_production_pin=enforce_production_pin, + label=signature_path.name, + ) + + +def _verify_production_signer( + receipt: pathlib.Path, + anchor: pathlib.Path, + spec: AnchorSpec, + gen_time: datetime, + temporary: pathlib.Path, + environment: dict[str, str], +) -> None: + token = temporary / "token.der" + signer = temporary / "signer.pem" + content = temporary / "tst-info.der" + _openssl_binary( + [ + "ts", + "-reply", + "-config", + "/dev/null", + "-in", + str(receipt), + "-token_out", + "-out", + str(token), + ], + environment=environment, + label=f"token extraction for {receipt.name}", + ) + _openssl_binary( + [ + "cms", + "-verify", + "-inform", + "DER", + "-in", + str(token), + "-CAfile", + str(anchor), + "-no-CApath", + "-no-CAstore", + "-purpose", + "timestampsign", + "-attime", + str(int(gen_time.timestamp())), + "-signer", + str(signer), + "-out", + str(content), + ], + environment=environment, + label=f"signer extraction for {receipt.name}", + ) + certificate_der = _openssl_binary( + ["x509", "-in", str(signer), "-outform", "DER"], + environment=environment, + label=f"signer certificate decoding for {receipt.name}", + ) + public_key_pem = _openssl_binary( + ["x509", "-in", str(signer), "-pubkey", "-noout"], + environment=environment, + label=f"signer public-key extraction for {receipt.name}", + ) + public_key = temporary / "signer-public-key.pem" + public_key.write_bytes(public_key_pem) + public_key_der = _openssl_binary( + ["pkey", "-pubin", "-in", str(public_key), "-outform", "DER"], + environment=environment, + label=f"signer SPKI decoding for {receipt.name}", + ) + certificate_sha256 = sha256_bytes(certificate_der) + spki_sha256 = sha256_bytes(public_key_der) + if certificate_sha256 != spec.signer_certificate_sha256: + raise ReleaseChainError( + f"RFC 3161 signer certificate is not pinned for {receipt.name}: " + f"{certificate_sha256}" + ) + if spki_sha256 != spec.signer_spki_sha256: + raise ReleaseChainError( + f"RFC 3161 signer SPKI is not pinned for {receipt.name}: {spki_sha256}" + ) + + +def verify_receipt( + manifest_digest: str, + receipt: pathlib.Path, + tsa: str, + *, + anchor_dir: pathlib.Path, + enforce_production_pins: bool, + now: datetime | None = None, +) -> datetime: + """Cryptographically verify one receipt and return its signed genTime.""" + + if tsa not in ANCHORS: + raise ReleaseChainError(f"unknown TSA receipt kind {tsa!r}") + _sha256(manifest_digest, "manifest digest") + if receipt.is_symlink() or not receipt.is_file(): + raise ReleaseChainError(f"missing or non-regular RFC 3161 receipt: {receipt}") + spec = ANCHORS[tsa] + anchor = anchor_dir / spec.filename + if anchor.is_symlink() or not anchor.is_file(): + raise ReleaseChainError(f"missing or non-regular TSA anchor: {anchor}") + if enforce_production_pins: + anchor_digest = sha256_bytes(anchor.read_bytes()) + if anchor_digest != spec.pem_sha256: + raise ReleaseChainError( + f"production TSA anchor bytes are not code-pinned for {tsa}: " + f"{anchor_digest}" + ) + + with tempfile.TemporaryDirectory(prefix="thesis-release-tsa-") as name: + temporary = pathlib.Path(name) + empty_ca_dir = temporary / "empty-ca" + empty_ca_dir.mkdir() + environment = _openssl_environment(empty_ca_dir) + try: + text_result = subprocess.run( + [ + "openssl", + "ts", + "-reply", + "-config", + "/dev/null", + "-in", + str(receipt), + "-text", + ], + check=False, + capture_output=True, + text=True, + env=environment, + ) + except FileNotFoundError as exc: + raise ReleaseChainError( + "openssl is required for RFC 3161 verification" + ) from exc + if text_result.returncode != 0: + raise ReleaseChainError( + f"cannot inspect RFC 3161 receipt {receipt} " + f"(exit {text_result.returncode}): {_command_error(text_result)}" + ) + gen_time, policy_oid = _parse_receipt_text(text_result.stdout, receipt) + if enforce_production_pins and policy_oid != spec.policy_oid: + raise ReleaseChainError( + f"RFC 3161 policy is not pinned for {tsa}: {policy_oid!r}" + ) + current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) + if gen_time > current + timedelta(seconds=MAX_FUTURE_SECONDS): + raise ReleaseChainError( + f"RFC 3161 genTime {gen_time.isoformat()} for {receipt.name} " + f"postdates verifier time {current.isoformat()}" + ) + + verify_result = subprocess.run( + [ + "openssl", + "ts", + "-verify", + "-config", + "/dev/null", + "-digest", + manifest_digest, + "-in", + str(receipt), + "-CAfile", + str(anchor), + "-CApath", + str(empty_ca_dir), + "-attime", + str(int(gen_time.timestamp())), + ], + check=False, + capture_output=True, + text=True, + env=environment, + ) + if verify_result.returncode != 0: + raise ReleaseChainError( + f"RFC 3161 verification failed for {receipt.name} " + f"(exit {verify_result.returncode}): " + f"{_command_error(verify_result)}" + ) + if enforce_production_pins: + _verify_production_signer( + receipt, + anchor, + spec, + gen_time, + temporary, + environment, + ) + return gen_time + + +def verify_release_receipts( + manifest: dict[str, Any], + manifest_digest: str, + receipt_paths: dict[str, pathlib.Path], + *, + anchor_dir: pathlib.Path, + enforce_production_pins: bool, + clock_skew_seconds: int, + previous_times: dict[str, datetime] | None = None, + now: datetime | None = None, +) -> dict[str, datetime]: + """Verify both receipts and their chronology for one manifest.""" + + if set(receipt_paths) != set(ANCHORS): + raise ReleaseChainError( + "release must have exactly freetsa and digicert receipt paths" + ) + receipt_times = { + tsa: verify_receipt( + manifest_digest, + receipt_path, + tsa, + anchor_dir=anchor_dir, + enforce_production_pins=enforce_production_pins, + now=now, + ) + for tsa, receipt_path in receipt_paths.items() + } + created_at = parse_created_at(manifest["createdAtUtc"]) + earliest_allowed = created_at - timedelta(seconds=clock_skew_seconds) + release_index = manifest["releaseIndex"] + for tsa, gen_time in receipt_times.items(): + if gen_time < earliest_allowed: + raise ReleaseChainError( + f"release {release_index} {tsa} genTime " + f"{gen_time.isoformat()} impossibly precedes createdAtUtc " + f"{created_at.isoformat()}" + ) + if previous_times is not None: + lower_bound = max(previous_times.values()) - timedelta( + seconds=clock_skew_seconds + ) + current_earliest = min(receipt_times.values()) + if current_earliest < lower_bound: + raise ReleaseChainError( + f"release {release_index} receipt chronology regresses: " + f"earliest current genTime {current_earliest.isoformat()} " + f"precedes latest prior genTime " + f"{max(previous_times.values()).isoformat()} beyond " + f"{clock_skew_seconds}s skew" + ) + return receipt_times + + +def jsonl_line_offsets(payload: bytes, label: str) -> list[int]: + """Return exact byte offsets after each non-empty LF-terminated row.""" + + try: + payload.decode("utf-8") + except UnicodeDecodeError as exc: + raise ReleaseChainError(f"{label} is not UTF-8") from exc + if not payload.endswith(b"\n"): + raise ReleaseChainError( + f"{label} must end with exactly one LF after its final JSONL row" + ) + rows = payload.split(b"\n") + if rows[-1] != b"": + raise AssertionError("split invariant") + rows = rows[:-1] + offsets = [0] + position = 0 + for number, row in enumerate(rows, start=1): + if not row.strip(): + raise ReleaseChainError(f"{label} row {number} is blank") + if row.endswith(b"\r"): + raise ReleaseChainError(f"{label} row {number} uses CRLF, not exact LF") + position += len(row) + 1 + offsets.append(position) + return offsets + + +def _regular_file_bytes(root: pathlib.Path, relative: pathlib.PurePosixPath) -> bytes: + path = root / relative + if path.is_symlink() or not path.is_file(): + raise ReleaseChainError( + f"required state file is missing or non-regular: {path}" + ) + return path.read_bytes() + + +def _verify_state_history( + records: list[ReleaseRecord], + root: pathlib.Path, + *, + require_head_current: bool, +) -> None: + ledger = _regular_file_bytes(root, LEDGER_RELATIVE) + prefix = _regular_file_bytes(root, PREFIX_RELATIVE) + offsets = jsonl_line_offsets(ledger, STATE_PATH) + total_lines = len(offsets) - 1 + prefix_digest = sha256_bytes(prefix) + + previous_line_count: int | None = None + for record in records: + state = record.manifest["state"] + line_count = int(state["lineCount"]) + if line_count > total_lines: + raise ReleaseChainError( + f"release {record.release_index} lineCount {line_count} exceeds " + f"working-tree line count {total_lines}" + ) + historical_bytes = ledger[: offsets[line_count]] + historical_digest = sha256_bytes(historical_bytes) + if historical_digest != state["jsonlSha256"]: + raise ReleaseChainError( + f"release {record.release_index} state.jsonlSha256 does not " + "match the exact historical JSONL prefix" + ) + if state["immutablePrefixSha256"] != prefix_digest: + raise ReleaseChainError( + f"release {record.release_index} immutablePrefixSha256 does " + "not match ledger/immutable_prefix.json" + ) + + if previous_line_count is not None: + append = record.manifest["append"] + assert isinstance(append, dict) + if line_count <= previous_line_count: + raise ReleaseChainError( + f"release {record.release_index} lineCount must strictly increase" + ) + if append["previousLineCount"] != previous_line_count: + raise ReleaseChainError( + f"release {record.release_index} append.previousLineCount " + "does not match the previous manifest" + ) + row_delta = line_count - previous_line_count + if append["appendedRowCount"] != row_delta: + raise ReleaseChainError( + f"release {record.release_index} appendedRowCount " + f"{append['appendedRowCount']} does not match line delta " + f"{row_delta}" + ) + suffix = ledger[offsets[previous_line_count] : offsets[line_count]] + suffix_digest = sha256_bytes(suffix) + if append["appendedBytesSha256"] != suffix_digest: + raise ReleaseChainError( + f"release {record.release_index} appendedBytesSha256 does " + "not match the exact byte suffix" + ) + previous_line_count = line_count + + if require_head_current: + head = records[-1] + if head.manifest["state"]["lineCount"] != total_lines: + raise ReleaseChainError( + f"HEAD release lineCount {head.manifest['state']['lineCount']} " + f"does not match working-tree line count {total_lines}" + ) + if head.manifest["state"]["jsonlSha256"] != sha256_bytes(ledger): + raise ReleaseChainError( + "HEAD release state.jsonlSha256 does not match working-tree bytes" + ) + + +def verify_release_chain( + root: pathlib.Path = ROOT, + *, + anchor_dir: pathlib.Path | None = None, + require_chain: bool = True, + verify_state: bool = True, + allow_pending_append: bool = False, + enforce_production_pins: bool | None = None, + clock_skew_seconds: int = DEFAULT_CLOCK_SKEW_SECONDS, + now: datetime | None = None, +) -> ChainVerification: + """Verify all manifests, signatures, receipts, links, and state bytes.""" + + root = root.resolve() + default_anchor_dir = root / "releases" / "anchors" + selected_anchors = (anchor_dir or default_anchor_dir).resolve() + if enforce_production_pins is None: + enforce_production_pins = selected_anchors == default_anchor_dir + if type(clock_skew_seconds) is not int or clock_skew_seconds < 0: + raise ReleaseChainError("clock_skew_seconds must be a non-negative integer") + + enumerated = _enumerate_manifest_files(root) + if not enumerated: + if require_chain: + raise ReleaseChainError("release chain is absent; genesis is required") + return ChainVerification(()) + + records: list[ReleaseRecord] = [] + previous_hash: str | None = None + previous_times: dict[str, datetime] | None = None + verification_now = now or datetime.now(timezone.utc) + for expected_index, (path, receipt_paths, producer_signature_path) in enumerate( + enumerated + ): + manifest, raw, digest = load_manifest(path) + filename_match = MANIFEST_RE.fullmatch(path.name) + assert filename_match is not None + filename_index = int(filename_match.group("index")) + if filename_index != expected_index: + raise ReleaseChainError( + f"release indices are not contiguous from 0: expected " + f"{expected_index:04d}, found {filename_index:04d}" + ) + if manifest["releaseIndex"] != expected_index: + raise ReleaseChainError( + f"manifest releaseIndex {manifest['releaseIndex']} does not " + f"match filename index {expected_index}" + ) + if filename_match.group("digest") != digest[:16]: + raise ReleaseChainError( + f"manifest filename hash does not match exact file bytes: {path.name}" + ) + if manifest["previousManifestSha256"] != previous_hash: + raise ReleaseChainError( + f"release {expected_index} previousManifestSha256 does not " + "match the previous manifest file bytes" + ) + verify_producer_signature( + raw, + producer_signature_path, + anchor_dir=selected_anchors, + enforce_production_pin=enforce_production_pins, + ) + if records: + previous_line_count = records[-1].manifest["state"]["lineCount"] + line_count = manifest["state"]["lineCount"] + append = manifest["append"] + assert isinstance(append, dict) + if line_count <= previous_line_count: + raise ReleaseChainError( + f"release {expected_index} lineCount must strictly increase" + ) + if append["previousLineCount"] != previous_line_count: + raise ReleaseChainError( + f"release {expected_index} append.previousLineCount does " + "not match the previous manifest" + ) + row_delta = line_count - previous_line_count + if append["appendedRowCount"] != row_delta: + raise ReleaseChainError( + f"release {expected_index} appendedRowCount " + f"{append['appendedRowCount']} does not match line delta " + f"{row_delta}" + ) + + receipt_times = verify_release_receipts( + manifest, + digest, + receipt_paths, + anchor_dir=selected_anchors, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + previous_times=previous_times, + now=verification_now, + ) + + records.append( + ReleaseRecord( + path=path, + raw=raw, + sha256=digest, + manifest=manifest, + receipt_paths=receipt_paths, + receipt_times=receipt_times, + producer_signature_path=producer_signature_path, + ) + ) + previous_hash = digest + previous_times = receipt_times + + if type(allow_pending_append) is not bool: + raise ReleaseChainError("allow_pending_append must be a boolean") + if allow_pending_append and not verify_state: + raise ReleaseChainError( + "allow_pending_append requires historical state verification" + ) + if verify_state: + _verify_state_history( + records, + root, + require_head_current=not allow_pending_append, + ) + return ChainVerification(tuple(records)) + + +def _git_run( + root: pathlib.Path, + arguments: list[str], + *, + text: bool = False, +) -> subprocess.CompletedProcess[Any]: + try: + return subprocess.run( + ["git", *arguments], + cwd=root, + check=False, + capture_output=True, + text=text, + ) + except FileNotFoundError as exc: + raise ReleaseChainError("git is required for --base-ref verification") from exc + + +def resolve_base_commit(root: pathlib.Path, base_ref: str) -> str: + completed = _git_run( + root, + ["rev-parse", "--verify", "--end-of-options", f"{base_ref}^{{commit}}"], + text=True, + ) + if completed.returncode != 0: + raise ReleaseChainError( + f"cannot resolve base ref {base_ref!r} to a commit: " + f"{completed.stderr.strip()}" + ) + commit = completed.stdout.strip() + ancestor = _git_run(root, ["merge-base", "--is-ancestor", commit, "HEAD"]) + if ancestor.returncode != 0: + raise ReleaseChainError(f"base commit {commit} is not an ancestor of HEAD") + return commit + + +def git_tree_entries( + root: pathlib.Path, commit: str, pathspec: str +) -> dict[str, GitEntry]: + completed = _git_run( + root, + ["ls-tree", "-r", "-z", "--full-tree", commit, "--", pathspec], + ) + if completed.returncode != 0: + diagnostic = completed.stderr.decode("utf-8", errors="replace").strip() + raise ReleaseChainError( + f"cannot enumerate {pathspec} at base {commit}: {diagnostic}" + ) + entries: dict[str, GitEntry] = {} + for record in completed.stdout.split(b"\0"): + if not record: + continue + try: + metadata, raw_path = record.split(b"\t", 1) + mode, object_type, object_id = metadata.decode("ascii").split(" ") + path = raw_path.decode("utf-8") + except (ValueError, UnicodeDecodeError) as exc: + raise ReleaseChainError( + f"cannot parse git tree entry under {pathspec}" + ) from exc + if path in entries: + raise ReleaseChainError(f"duplicate git tree entry for {path}") + entries[path] = GitEntry(mode, object_type, object_id, path) + return entries + + +def git_blob_bytes(root: pathlib.Path, entry: GitEntry) -> bytes: + if entry.object_type != "blob": + raise ReleaseChainError( + f"base release entry is not a blob: {entry.path} ({entry.object_type})" + ) + completed = _git_run(root, ["cat-file", "blob", entry.object_id]) + if completed.returncode != 0: + diagnostic = completed.stderr.decode("utf-8", errors="replace").strip() + raise ReleaseChainError(f"cannot read base blob for {entry.path}: {diagnostic}") + return completed.stdout + + +def git_file_entry(root: pathlib.Path, commit: str, path: str) -> GitEntry: + entries = git_tree_entries(root, commit, path) + entry = entries.get(path) + if entry is None: + raise ReleaseChainError(f"required file {path} is absent at base {commit}") + return entry + + +def _working_release_files(root: pathlib.Path) -> dict[str, pathlib.Path]: + release_root = root / "releases" + if not release_root.exists(): + return {} + if release_root.is_symlink() or not release_root.is_dir(): + raise ReleaseChainError("releases must be a real directory, not a symlink") + files: dict[str, pathlib.Path] = {} + for path in release_root.rglob("*"): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + raise ReleaseChainError(f"release path is a symlink: {relative}") + if path.is_dir(): + continue + if not path.is_file(): + raise ReleaseChainError(f"release path is not regular: {relative}") + files[relative] = path + return files + + +def verify_release_history_immutable( + root: pathlib.Path, base_ref: str +) -> tuple[str, set[str], dict[str, GitEntry]]: + """Compare every base ``releases/`` file byte and mode to the candidate.""" + + root = root.resolve() + commit = resolve_base_commit(root, base_ref) + base_entries = git_tree_entries(root, commit, "releases") + current_files = _working_release_files(root) + for relative, entry in base_entries.items(): + if entry.mode not in {"100644", "100755"}: + raise ReleaseChainError( + f"base release entry has non-regular git mode {entry.mode}: {relative}" + ) + current = current_files.get(relative) + if current is None: + raise ReleaseChainError( + f"existing release file was deleted relative to {commit}: {relative}" + ) + candidate_mode = "100755" if current.stat().st_mode & 0o111 else "100644" + if candidate_mode != entry.mode: + raise ReleaseChainError( + f"existing release file mode changed relative to {commit}: " + f"{relative} ({entry.mode} -> {candidate_mode})" + ) + if current.read_bytes() != git_blob_bytes(root, entry): + raise ReleaseChainError( + f"existing release file bytes changed relative to {commit}: {relative}" + ) + return commit, set(current_files) - set(base_entries), base_entries + + +def materialize_base_tree( + root: pathlib.Path, + commit: str, + destination: pathlib.Path, + release_entries: dict[str, GitEntry], +) -> None: + entries = dict(release_entries) + for relative in (LEDGER_RELATIVE.as_posix(), PREFIX_RELATIVE.as_posix()): + entries[relative] = git_file_entry(root, commit, relative) + for relative, entry in entries.items(): + if entry.mode not in {"100644", "100755"}: + raise ReleaseChainError( + f"base tree entry has non-regular mode {entry.mode}: {relative}" + ) + output = destination / pathlib.PurePosixPath(relative) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(git_blob_bytes(root, entry)) + + +def verify_base_release_chain( + root: pathlib.Path, + commit: str, + release_entries: dict[str, GitEntry], + *, + anchor_dir: pathlib.Path | None = None, + enforce_production_pins: bool = True, + clock_skew_seconds: int = DEFAULT_CLOCK_SKEW_SECONDS, +) -> ChainVerification: + with tempfile.TemporaryDirectory(prefix="thesis-release-base-") as name: + base_root = pathlib.Path(name) + materialize_base_tree(root, commit, base_root, release_entries) + base_anchor_dir = anchor_dir or (base_root / "releases" / "anchors") + return verify_release_chain( + base_root, + anchor_dir=base_anchor_dir, + require_chain=True, + verify_state=True, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + ) + + +def _format_time(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="verify the offline thesis-ledger release journal" + ) + parser.add_argument( + "--full", + action="store_true", + help="require a genesis-to-HEAD chain and exact working-tree state", + ) + parser.add_argument( + "--base-ref", + help="also reject any changed/deleted existing releases file vs this ref", + ) + parser.add_argument( + "--root", + type=pathlib.Path, + default=ROOT, + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--anchor-dir", + type=pathlib.Path, + help="override releases/anchors (intended for offline test fixtures)", + ) + parser.add_argument( + "--clock-skew-seconds", + type=int, + default=DEFAULT_CLOCK_SKEW_SECONDS, + ) + args = parser.parse_args() + + root = args.root.resolve() + anchor_dir = args.anchor_dir.resolve() if args.anchor_dir else None + enforce_pins = anchor_dir is None + try: + if args.base_ref: + verify_release_history_immutable(root, args.base_ref) + verification = verify_release_chain( + root, + anchor_dir=anchor_dir, + require_chain=args.full or bool(args.base_ref), + verify_state=True, + enforce_production_pins=enforce_pins, + clock_skew_seconds=args.clock_skew_seconds, + ) + except (OSError, ReleaseChainError) as exc: + print(f"release chain verification failed: {exc}", file=sys.stderr) + return 1 + if not verification.releases: + print("release chain absent (legacy pre-genesis state)") + return 0 + head = verification.releases[-1] + receipt_summary = ", ".join( + f"{tsa}={_format_time(value)}" + for tsa, value in sorted(head.receipt_times.items()) + ) + print( + f"release chain OK: {len(verification.releases)} releases, " + f"HEAD={head.path.name}, {receipt_summary}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_vidimus_shim_transparency.py b/tests/test_vidimus_shim_transparency.py new file mode 100644 index 0000000..468d0c0 --- /dev/null +++ b/tests/test_vidimus_shim_transparency.py @@ -0,0 +1,401 @@ +"""Byte-level differential tests for the vidimus consumer shims.""" + +from __future__ import annotations + +import hashlib +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys +from collections.abc import Callable + +import pytest + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SHIM_SCRIPTS = ROOT / "scripts" +ORIGINAL_FIXTURES = ROOT / "tests" / "fixtures" / "vidimus_shim_originals" +ORIGINAL_HASHES = { + "canonical_json.py": ( + "562bf267b7686bce8cb71f3c13f34825c21cd4ef0aba1c0c46aff16962a6cadd" + ), + "check_thesis_facts_append.py": ( + "46727ab22186b8f150fc7dbee8222cee729a6ddb4ba8e8cbe4a3dda702cbc427" + ), + "verify_release_chain.py": ( + "7f73e6921ca40e41e556c8e37a634e2780e7e8eeb3ab203ecdb9b7bd4b15a844" + ), +} +OPENSSL_QUEUE_ID = re.compile(rb"(?m)^[0-9A-Fa-f]{8,16}(?=:error:)") + +BASE_LINE_COUNT = 145 +CANDIDATE_LINE_COUNT = 147 +NEW_RELEASE_STEM = "0002-a69272175b73c83b" +RELEASE_FILE_SUFFIXES = ( + ".json", + ".producer.sig", + ".freetsa.tsr", + ".digicert.tsr", +) + +Mutation = Callable[[pathlib.Path], None] + + +def _sha256(path: pathlib.Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +@pytest.mark.parametrize("name", sorted(ORIGINAL_HASHES)) +def test_original_oracle_fixtures_are_authenticated(name: str) -> None: + assert _sha256(ORIGINAL_FIXTURES / name) == ORIGINAL_HASHES[name] + + +@pytest.fixture(scope="session") +def original_oracle(tmp_path_factory: pytest.TempPathFactory) -> pathlib.Path: + """Copy the authenticated original scripts into one executable tree.""" + + oracle = tmp_path_factory.mktemp("vidimus-original-oracle") + scripts = oracle / "scripts" + scripts.mkdir() + for name, expected in ORIGINAL_HASHES.items(): + source = ORIGINAL_FIXTURES / name + assert _sha256(source) == expected + shutil.copyfile(source, scripts / name) + shutil.copytree( + ROOT / "releases" / "anchors", + oracle / "releases" / "anchors", + ) + return oracle + + +def _run_script( + script: pathlib.Path, + *arguments: str, + cwd: pathlib.Path = ROOT, + stdin: bytes | None = None, +) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + [sys.executable, str(script), *arguments], + cwd=cwd, + input=stdin, + capture_output=True, + check=False, + ) + + +def _normalized_stderr(value: bytes) -> bytes: + """Mask only OpenSSL 3's necessarily per-process error-queue prefix.""" + + return OPENSSL_QUEUE_ID.sub(b"", value) + + +def _assert_byte_identical( + original: subprocess.CompletedProcess[bytes], + shim: subprocess.CompletedProcess[bytes], + *, + expected_code: int, +) -> None: + assert original.returncode == expected_code + assert shim.returncode == expected_code + assert shim.stdout == original.stdout + assert _normalized_stderr(shim.stderr) == _normalized_stderr(original.stderr) + + +@pytest.mark.parametrize( + ("arguments", "stdin", "expected_code"), + [ + ( + (), + b'{"\\ud83d\\ude00":1,"\\ue000":2,"fixed":1e-6,"scientific":1e21}\n', + 0, + ), + ( + ("--sha256",), + b'{"\\ud83d\\ude00":1,"\\ue000":2,"fixed":1e-6,"scientific":1e21}\n', + 0, + ), + (("--help",), None, 0), + (("--not-an-option",), None, 2), + ], +) +def test_canonical_json_cli_is_byte_identical( + original_oracle: pathlib.Path, + arguments: tuple[str, ...], + stdin: bytes | None, + expected_code: int, +) -> None: + original = _run_script( + original_oracle / "scripts" / "canonical_json.py", + *arguments, + stdin=stdin, + ) + shim = _run_script( + SHIM_SCRIPTS / "canonical_json.py", + *arguments, + stdin=stdin, + ) + _assert_byte_identical(original, shim, expected_code=expected_code) + + +def test_release_chain_cli_help_is_byte_identical( + original_oracle: pathlib.Path, +) -> None: + original = _run_script( + original_oracle / "scripts" / "verify_release_chain.py", + "--help", + ) + shim = _run_script(SHIM_SCRIPTS / "verify_release_chain.py", "--help") + _assert_byte_identical(original, shim, expected_code=0) + + +def test_live_full_release_chain_is_byte_identical( + original_oracle: pathlib.Path, +) -> None: + arguments = ("--full", "--root", str(ROOT)) + original = _run_script( + original_oracle / "scripts" / "verify_release_chain.py", + *arguments, + ) + shim = _run_script(SHIM_SCRIPTS / "verify_release_chain.py", *arguments) + _assert_byte_identical(original, shim, expected_code=0) + assert shim.stderr == b"" + assert shim.stdout == ( + b"release chain OK: 3 releases, " + b"HEAD=0002-a69272175b73c83b.json, " + b"digicert=2026-07-18T16:39:11Z, " + b"freetsa=2026-07-18T16:39:11Z\n" + ) + + +def _copy_custody_tree(destination: pathlib.Path) -> pathlib.Path: + root = destination / "root" + shutil.copytree(ROOT / "ledger", root / "ledger") + shutil.copytree(ROOT / "releases", root / "releases") + return root + + +def _flip_middle_byte(path: pathlib.Path) -> None: + payload = bytearray(path.read_bytes()) + payload[len(payload) // 2] ^= 0x01 + path.write_bytes(bytes(payload)) + + +def _append_unwitnessed_row(root: pathlib.Path) -> None: + ledger = root / "ledger" / "official_observations.jsonl" + ledger.write_bytes(ledger.read_bytes() + b"{}\n") + + +def _corrupt_producer_signature(root: pathlib.Path) -> None: + _flip_middle_byte( + root + / "releases" + / "manifests" + / "0001-916626696d034b80.producer.sig" + ) + + +def _corrupt_freetsa_receipt(root: pathlib.Path) -> None: + _flip_middle_byte( + root + / "releases" + / "manifests" + / "0001-916626696d034b80.freetsa.tsr" + ) + + +@pytest.mark.parametrize( + ("case", "mutation", "marker"), + [ + ( + "unwitnessed-row", + _append_unwitnessed_row, + b"HEAD release lineCount 147 does not match working-tree line count 148", + ), + ( + "producer-signature", + _corrupt_producer_signature, + b"producer Ed25519 signature verification failed", + ), + ( + "freetsa-receipt", + _corrupt_freetsa_receipt, + b"cannot inspect RFC 3161 receipt", + ), + ], +) +def test_corrupt_release_chain_refusals_are_byte_identical( + original_oracle: pathlib.Path, + tmp_path: pathlib.Path, + case: str, + mutation: Mutation, + marker: bytes, +) -> None: + custody = _copy_custody_tree(tmp_path / case) + mutation(custody) + arguments = ("--full", "--root", str(custody)) + original = _run_script( + original_oracle / "scripts" / "verify_release_chain.py", + *arguments, + ) + shim = _run_script(SHIM_SCRIPTS / "verify_release_chain.py", *arguments) + _assert_byte_identical(original, shim, expected_code=1) + assert shim.stdout == b"" + assert marker in _normalized_stderr(shim.stderr) + + +def _git(root: pathlib.Path, *arguments: str) -> str: + environment = os.environ.copy() + environment.update( + { + "GIT_CONFIG_GLOBAL": "/dev/null", + "GIT_CONFIG_SYSTEM": "/dev/null", + "GIT_CONFIG_NOSYSTEM": "1", + } + ) + completed = subprocess.run( + ["git", *arguments], + cwd=root, + env=environment, + capture_output=True, + text=True, + check=True, + ) + return completed.stdout.strip() + + +def _release_file(root: pathlib.Path, suffix: str) -> pathlib.Path: + return root / "releases" / "manifests" / f"{NEW_RELEASE_STEM}{suffix}" + + +def _replay_release_two(destination: pathlib.Path) -> tuple[pathlib.Path, str]: + """Create the real release-1 base and restore the witnessed release-2 append.""" + + root = _copy_custody_tree(destination) + ledger = root / "ledger" / "official_observations.jsonl" + full_ledger = ledger.read_bytes() + rows = full_ledger.splitlines(keepends=True) + assert len(rows) == CANDIDATE_LINE_COUNT + assert all(row.endswith(b"\n") for row in rows) + ledger.write_bytes(b"".join(rows[:BASE_LINE_COUNT])) + for suffix in RELEASE_FILE_SUFFIXES: + _release_file(root, suffix).unlink() + + _git(root, "init", "--quiet") + _git(root, "config", "user.email", "shim-differential@example.invalid") + _git(root, "config", "user.name", "Shim Differential") + _git(root, "add", "-A") + _git(root, "commit", "--quiet", "-m", "release 1 base") + base = _git(root, "rev-parse", "HEAD") + + ledger.write_bytes(full_ledger) + for suffix in RELEASE_FILE_SUFFIXES: + shutil.copyfile( + _release_file(ROOT, suffix), + _release_file(root, suffix), + ) + return root, base + + +def _run_append_pair( + original_oracle: pathlib.Path, + candidate: pathlib.Path, + base: str, +) -> tuple[subprocess.CompletedProcess[bytes], subprocess.CompletedProcess[bytes]]: + arguments = ("--root", str(candidate), "--base-ref", base) + original = _run_script( + original_oracle / "scripts" / "check_thesis_facts_append.py", + *arguments, + cwd=candidate, + ) + shim = _run_script( + SHIM_SCRIPTS / "check_thesis_facts_append.py", + *arguments, + cwd=candidate, + ) + return original, shim + + +def test_append_gate_cli_help_is_byte_identical( + original_oracle: pathlib.Path, +) -> None: + original = _run_script( + original_oracle / "scripts" / "check_thesis_facts_append.py", + "--help", + ) + shim = _run_script(SHIM_SCRIPTS / "check_thesis_facts_append.py", "--help") + _assert_byte_identical(original, shim, expected_code=0) + + +def test_valid_base_ref_append_is_byte_identical( + original_oracle: pathlib.Path, + tmp_path: pathlib.Path, +) -> None: + candidate, base = _replay_release_two(tmp_path) + original, shim = _run_append_pair(original_oracle, candidate, base) + _assert_byte_identical(original, shim, expected_code=0) + assert shim.stderr == b"" + assert shim.stdout == ( + b"thesis-facts append check OK: 147 rows, immutable prefix 128, " + b"+2 appended vs base, release 2\n" + ) + + +def _rewrite_historical_row(root: pathlib.Path) -> None: + ledger = root / "ledger" / "official_observations.jsonl" + rows = ledger.read_bytes().splitlines(keepends=True) + rows[128] = b" " + rows[128] + ledger.write_bytes(b"".join(rows)) + + +def _remove_appended_assertion_version(root: pathlib.Path) -> None: + ledger = root / "ledger" / "official_observations.jsonl" + rows = ledger.read_bytes().splitlines(keepends=True) + row = json.loads(rows[145]) + row.pop("assertionVersion") + rows[145] = ( + json.dumps(row, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + + b"\n" + ) + ledger.write_bytes(b"".join(rows)) + + +def _remove_new_release_manifest(root: pathlib.Path) -> None: + _release_file(root, ".json").unlink() + + +@pytest.mark.parametrize( + ("case", "mutation", "marker"), + [ + ( + "historical-rewrite", + _rewrite_historical_row, + b"change rewrites existing line 129", + ), + ( + "missing-assertion-version", + _remove_appended_assertion_version, + b"appended line 146", + ), + ( + "missing-release-manifest", + _remove_new_release_manifest, + b"release proposal must add exactly one manifest for index 2", + ), + ], +) +def test_corrupt_base_ref_append_refusals_are_byte_identical( + original_oracle: pathlib.Path, + tmp_path: pathlib.Path, + case: str, + mutation: Mutation, + marker: bytes, +) -> None: + candidate, base = _replay_release_two(tmp_path / case) + mutation(candidate) + original, shim = _run_append_pair(original_oracle, candidate, base) + _assert_byte_identical(original, shim, expected_code=1) + assert shim.stdout == b"" + assert marker in _normalized_stderr(shim.stderr)