From 048b540b216e2ce7933ca0e2e38677a46d0f7092 Mon Sep 17 00:00:00 2001 From: stacknil Date: Fri, 3 Jul 2026 19:12:18 +0800 Subject: [PATCH] docs(release): add operator reproduction gate --- .../artifact-regeneration-failure.md | 33 ++++++++ .../ISSUE_TEMPLATE/demo-boundary-question.md | 29 +++++++ .github/ISSUE_TEMPLATE/feature_request.yml | 8 +- .github/ISSUE_TEMPLATE/schema-drift-report.md | 32 +++++++ README.md | 23 ++++- docs/README.md | 1 + docs/operator-reproduction.md | 57 +++++++++++++ docs/roadmap.md | 18 +++- scripts/check_release_contract.py | 83 +++++++++++++++++++ tests/test_release_contract_script.py | 63 ++++++++++++++ tests/test_reviewer_docs.py | 60 ++++++++++++++ 11 files changed, 399 insertions(+), 8 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/artifact-regeneration-failure.md create mode 100644 .github/ISSUE_TEMPLATE/demo-boundary-question.md create mode 100644 .github/ISSUE_TEMPLATE/schema-drift-report.md create mode 100644 docs/operator-reproduction.md create mode 100644 scripts/check_release_contract.py create mode 100644 tests/test_release_contract_script.py diff --git a/.github/ISSUE_TEMPLATE/artifact-regeneration-failure.md b/.github/ISSUE_TEMPLATE/artifact-regeneration-failure.md new file mode 100644 index 0000000..1537070 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/artifact-regeneration-failure.md @@ -0,0 +1,33 @@ +--- +name: Artifact regeneration failure +about: Report a failure in the committed artifact regeneration gate +title: "[Artifact regeneration]: " +labels: ["artifacts", "reviewer-contract"] +--- + +## Command + +```bash +python scripts/regenerate_artifacts.py --check +``` + +## Failure Summary + +Paste the failing job, artifact path, reason, and exit code. + +## Environment + +- OS: +- Python version: +- Fresh clone or existing checkout: + +## Expected Contract + +The committed artifact should match deterministic local output, or the drift +should be documented in a reviewer-facing artifact diff. + +## Boundary Check + +- [ ] This report uses only the committed synthetic sample data. +- [ ] No live AWS account, production telemetry, credentials, or private data are involved. +- [ ] This is not a request for a dashboard, SIEM, alert routing, case management, or autonomous response. diff --git a/.github/ISSUE_TEMPLATE/demo-boundary-question.md b/.github/ISSUE_TEMPLATE/demo-boundary-question.md new file mode 100644 index 0000000..2985eeb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/demo-boundary-question.md @@ -0,0 +1,29 @@ +--- +name: Demo boundary question +about: Ask whether a proposed change fits the local reviewer-contract scope +title: "[Boundary]: " +labels: ["scope", "reviewer-contract"] +--- + +## Question + +What boundary or scope decision do you want to clarify? + +## Related Demo Or Doc + +Path or demo name: + +## Why It Matters + +Explain the review or reproduction problem this question affects. + +## Proposed Direction + +Describe the smallest scoped change that would answer the question. + +## Non-Goals + +- [ ] No new demo expansion for v1.1. +- [ ] No live ingestion, production SIEM, dashboard, alert routing, or case-management scope. +- [ ] No autonomous response or final incident verdict. +- [ ] No real account IDs, credentials, or private data. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 7032bff..d9fbc2a 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,5 +1,5 @@ name: Feature request -description: Suggest a small improvement or next demo for the repository +description: Suggest a small reviewer-contract or reproduction improvement title: "[Feature]: " body: - type: textarea @@ -7,7 +7,7 @@ body: attributes: label: What would you like to add or improve? description: Keep it concrete and close to the repository's current scope. - placeholder: Add a host-centric investigation demo that reuses the current sample outputs. + placeholder: Make the local reproduction path easier to verify from a fresh clone. validations: required: true @@ -16,7 +16,7 @@ body: attributes: label: Why does it help? description: Explain how this improves the demo, docs, or portfolio narrative. - placeholder: It would make the rule outputs easier to connect to an analyst workflow. + placeholder: It would reduce reviewer guesswork without changing the demo matrix. validations: required: true @@ -25,4 +25,4 @@ body: attributes: label: Scope guardrails description: Call out anything this request should not turn into. - placeholder: Keep it file-based; no hosted dashboard or alert-routing work. + placeholder: Keep it file-based; no new demos, hosted dashboard, or alert-routing work. diff --git a/.github/ISSUE_TEMPLATE/schema-drift-report.md b/.github/ISSUE_TEMPLATE/schema-drift-report.md new file mode 100644 index 0000000..06739dc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/schema-drift-report.md @@ -0,0 +1,32 @@ +--- +name: Schema drift report +about: Report a mismatch between a reviewer-facing JSON/JSONL artifact and its schema +title: "[Schema drift]: " +labels: ["schema", "reviewer-contract"] +--- + +## Artifact + +Path: + +Schema: + +## Command + +```bash +python -m pytest tests/test_evidence_pipeline_schemas.py +``` + +## Failure + +Paste the failing schema validation path and message. + +## Expected Contract + +Describe which schema field or artifact path appears out of sync. + +## Boundary Check + +- [ ] This report is about committed synthetic reviewer artifacts only. +- [ ] This is not a request for live ingestion, production detection, dashboard, alert routing, or case management. +- [ ] No real account IDs, credentials, hostnames, or private data are included. diff --git a/README.md b/README.md index 95e5aa4..b521439 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A local, file-based detection workflow lab for reviewer-verifiable telemetry and detection demos. -Current focus: v1 reviewer contract stabilization for the five-demo matrix, with v1.0 treated as a five-demo contract freeze. +Current focus: v1 reviewer contract stabilization for the five-demo matrix. v1.1 is scoped as an Operator Reproduction Release, not another demo expansion. Latest tagged release: [v1.0 — reviewer contract release](https://github.com/stacknil/telemetry-lab/releases/tag/v1.0). @@ -13,6 +13,7 @@ Latest tagged release: [v1.0 — reviewer contract release](https://github.com/s - [`docs/reviewer-brief.md`](docs/reviewer-brief.md): scope, value, evidence, and boundaries - [`docs/reviewer-path.md`](docs/reviewer-path.md): choose the right demo by review question - [`docs/reviewer-pack.md`](docs/reviewer-pack.md): demo matrix, artifact contract, and v1 readiness gate +- [`docs/operator-reproduction.md`](docs/operator-reproduction.md): shortest local path from clone to artifact regeneration - [`docs/v1-contract-freeze.md`](docs/v1-contract-freeze.md): v1.0 five-demo contract freeze, release status, and contract scope - [`docs/v1-readiness-gate.md`](docs/v1-readiness-gate.md): fixed inputs, fixed outputs, schema validation, artifact regeneration, and test pass requirements - [`docs/release-v1.0.md`](docs/release-v1.0.md): v1.0 reviewer-contract release notes and explicit non-SIEM boundary @@ -62,8 +63,8 @@ Latest tagged release: [v1.0 — reviewer contract release](https://github.com/s `cloud-iam-change-investigation-demo` uses synthetic CloudTrail-like events to review IAM changes, failed console logins, CloudTrail logging changes, and security group ingress changes with bounded deterministic rules. It has no live AWS account, no real account ID, no production detection claim, and no final incident verdict. -## Quick Run - +## Quick Run + ```bash python -m pip install -e . python -m telemetry_window_demo.cli run --config configs/default.yaml @@ -72,6 +73,20 @@ python -m telemetry_window_demo.cli run --config configs/default.yaml Use the same Python interpreter for install, tests, and demo commands. On machines with multiple Python installs, replace `python` with the intended interpreter path. To run the test suite in a fresh environment, install the dev extra with `python -m pip install -e ".[dev]"`. +If you want to verify v1.0 locally, run these three commands. + +```bash +python scripts/regenerate_artifacts.py --check +python -m pytest tests/test_evidence_pipeline_schemas.py +python -m pytest +``` + +For the same reviewer-friendly gate with labeled steps, run: + +```bash +python scripts/check_release_contract.py +``` + Other demo entrypoints: - `python -m telemetry_window_demo.cli run-ai-demo` @@ -170,6 +185,7 @@ Cooldown behavior: - [`demos/config-change-investigation-demo/README.md`](demos/config-change-investigation-demo/README.md) explains the config-change investigation demo and its committed artifacts - [`demos/cloud-iam-change-investigation-demo/README.md`](demos/cloud-iam-change-investigation-demo/README.md) explains the synthetic CloudTrail-like IAM investigation demo and its committed artifacts - [`docs/README.md`](docs/README.md) indexes current reviewer docs, supporting design notes, and historical release evidence +- [`docs/operator-reproduction.md`](docs/operator-reproduction.md) gives the shortest local path from clone to artifact regeneration and the release contract gate - [`docs/reviewer-pack.md`](docs/reviewer-pack.md) is the top-level no-guessing reviewer pack and artifact naming contract - [`docs/v1-contract-freeze.md`](docs/v1-contract-freeze.md) defines the v1.0 five-demo contract freeze gate - [`docs/v1-readiness-gate.md`](docs/v1-readiness-gate.md) defines the fixed-input, fixed-output, schema-validation, artifact-regeneration, and test-pass readiness gate @@ -190,6 +206,7 @@ Cooldown behavior: ## v1 Reviewer Contract Stabilization - demo expansion is closed +- v1.1 is an Operator Reproduction Release, not a new-demo release - treat v1.0 as a five-demo contract freeze, not a feature expansion - stabilize the five-demo matrix and avoid broad platform expansion - freeze reviewer-visible artifact names unless a rename is intentionally coordinated across docs, tests, and sample outputs diff --git a/docs/README.md b/docs/README.md index 015d72f..33c6d3f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,6 +5,7 @@ This directory separates the current reviewer route from supporting design notes ## Current reviewer route - [`reviewer-pack.md`](reviewer-pack.md): top-level reviewer pack, demo matrix, artifact naming contract, and v1 readiness gate +- [`operator-reproduction.md`](operator-reproduction.md): shortest local path from clone to artifact regeneration - [`reviewer-path.md`](reviewer-path.md): choose a demo by review question - [`reviewer-brief.md`](reviewer-brief.md): short problem, value, evidence, and boundary summary - [`v1-contract-freeze.md`](v1-contract-freeze.md): v1.0 five-demo contract freeze, release status, and contract scope diff --git a/docs/operator-reproduction.md b/docs/operator-reproduction.md new file mode 100644 index 0000000..652a10c --- /dev/null +++ b/docs/operator-reproduction.md @@ -0,0 +1,57 @@ +# Operator Reproduction + +This is the shortest local path for reproducing the reviewer contract from a +fresh clone. It does not add a new demo and does not require any live service, +cloud account, SIEM, dashboard, alert routing, case-management system, or +production telemetry source. + +## Fresh Clone + +```bash +git clone https://github.com/stacknil/telemetry-lab.git +cd telemetry-lab +python -m pip install -e ".[dev]" +``` + +Use the same Python interpreter for install, artifact regeneration, schema +validation, and tests. + +## Reproduce v1.0 Artifacts + +```bash +python scripts/regenerate_artifacts.py --check +``` + +Expected result: + +- five demo jobs run from committed synthetic inputs +- `23` byte-stable committed artifacts match regenerated output +- `6` PNG visual snapshots regenerate as smoke checks without byte comparison + +If this command fails, open an artifact regeneration issue with the command, +exit code, failing artifact path, and whether the failure is a missing file or +content difference. + +## Full Reviewer Contract Gate + +The v1.0 reviewer contract is the exact three-command gate: + +```bash +python scripts/regenerate_artifacts.py --check +python -m pytest tests/test_evidence_pipeline_schemas.py +python -m pytest +``` + +For the same sequence with clearer step labels: + +```bash +python scripts/check_release_contract.py +``` + +The wrapper stops at the first failed step and reports which gate failed. + +## What This Does Not Prove + +Passing the gate means the committed reviewer contract is locally reproducible. +It does not claim production readiness, operational detection quality, final +incident verdicts, live ingestion, or cloud-environment coverage. diff --git a/docs/roadmap.md b/docs/roadmap.md index fc4969f..17c83bc 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -4,6 +4,8 @@ Demo expansion is closed. Next phase: v1 reviewer contract stabilization. +v1.1 theme: Operator Reproduction Release. + The concrete milestone is [`v1.0 Five-Demo Contract Freeze`](v1-contract-freeze.md). The repo now has five reviewer-verifiable demos and a clear [`docs/reviewer-path.md`](reviewer-path.md). The priority is to keep the demo matrix stable, preserve artifact names, keep release evidence explicit, and make review faster without implying a SIEM, dashboard, or production monitoring platform. @@ -20,6 +22,20 @@ Recently added: - [`docs/vocabulary.md`](vocabulary.md) defines cross-demo evidence workflow terms. - [`docs/reviewer-artifact-diff.md`](reviewer-artifact-diff.md) defines the release diff contract for reviewer-facing artifact changes. - [`docs/v0.6-to-v1-artifact-diff.md`](v0.6-to-v1-artifact-diff.md) records the additive fourth-to-fifth-demo artifact contract. +- [`docs/operator-reproduction.md`](operator-reproduction.md) records the shortest clone-to-regeneration path. +- `scripts/check_release_contract.py` wraps artifact regeneration, schema validation, and the full test suite. + +## v1.1 Operator Reproduction Release + +The v1.1 release theme is operator reproduction, not demo expansion. + +Deliverable focus: + +1. Keep the five-demo matrix unchanged. +2. Make clone-to-artifact-regeneration instructions short and explicit. +3. Provide issue templates for schema drift, artifact regeneration failures, and demo boundary questions. +4. Provide one reviewer-friendly release contract gate command. +5. Keep README verification commands copy/paste runnable. ## v1 Reviewer Contract Stabilization @@ -40,7 +56,7 @@ The consolidation gate lives in [`docs/v1-readiness-gate.md`](v1-readiness-gate. The optional final demo slot has been used by `cloud-iam-change-investigation-demo`. -Further work before v1 should focus on reviewer contract stability, documentation accuracy, tests, and committed artifact reproducibility rather than adding more workflow surface area. +Further work should focus on reviewer contract stability, documentation accuracy, tests, and committed artifact reproducibility rather than adding more workflow surface area. ## Non-Directions diff --git a/scripts/check_release_contract.py b/scripts/check_release_contract.py new file mode 100644 index 0000000..2eeb4c6 --- /dev/null +++ b/scripts/check_release_contract.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import argparse +import subprocess +import sys +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +@dataclass(frozen=True) +class GateStep: + name: str + command: tuple[str, ...] + + +def build_gate_steps(python_executable: str) -> list[GateStep]: + return [ + GateStep( + name="artifact regeneration", + command=(python_executable, "scripts/regenerate_artifacts.py", "--check"), + ), + GateStep( + name="schema validation", + command=( + python_executable, + "-m", + "pytest", + "tests/test_evidence_pipeline_schemas.py", + ), + ), + GateStep( + name="full test suite", + command=(python_executable, "-m", "pytest"), + ), + ] + + +def main(argv: Sequence[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + steps = build_gate_steps(args.python) + + print("telemetry-lab release contract gate", flush=True) + print("Scope: local reviewer contract reproduction; not a production SIEM gate.", flush=True) + print("", flush=True) + + for index, step in enumerate(steps, start=1): + display_command = " ".join(step.command) + print(f"[{index}/{len(steps)}] {step.name}", flush=True) + print(f"$ {display_command}", flush=True) + result = subprocess.run(step.command, cwd=REPO_ROOT, check=False) + if result.returncode != 0: + print("", flush=True) + print(f"[FAIL] {step.name} failed with exit code {result.returncode}.", flush=True) + print("The release contract gate stops at the first failing step.", flush=True) + return result.returncode + print(f"[OK] {step.name}", flush=True) + print("", flush=True) + + print("[OK] Release contract gate passed.", flush=True) + return 0 + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Run the reviewer-facing release contract gate: artifact " + "regeneration, schema validation, and the full test suite." + ), + ) + parser.add_argument( + "--python", + default=sys.executable, + help="Python executable used for all gate commands. Defaults to this interpreter.", + ) + return parser + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_release_contract_script.py b/tests/test_release_contract_script.py new file mode 100644 index 0000000..db01c94 --- /dev/null +++ b/tests/test_release_contract_script.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = REPO_ROOT / "scripts" / "check_release_contract.py" + + +def _load_release_contract_script(): + spec = importlib.util.spec_from_file_location( + "check_release_contract", + SCRIPT_PATH, + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_release_contract_gate_runs_expected_steps_with_configured_python(monkeypatch) -> None: + script = _load_release_contract_script() + calls: list[tuple[str, ...]] = [] + + def fake_run(command, cwd, check): + calls.append(tuple(command)) + assert cwd == REPO_ROOT + assert check is False + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(subprocess, "run", fake_run) + + exit_code = script.main(["--python", "python"]) + + assert exit_code == 0 + assert calls == [ + ("python", "scripts/regenerate_artifacts.py", "--check"), + ("python", "-m", "pytest", "tests/test_evidence_pipeline_schemas.py"), + ("python", "-m", "pytest"), + ] + + +def test_release_contract_gate_stops_at_first_failure(monkeypatch, capsys) -> None: + script = _load_release_contract_script() + calls: list[tuple[str, ...]] = [] + + def fake_run(command, cwd, check): + calls.append(tuple(command)) + return SimpleNamespace(returncode=7) + + monkeypatch.setattr(subprocess, "run", fake_run) + + exit_code = script.main(["--python", "python"]) + + assert exit_code == 7 + assert calls == [("python", "scripts/regenerate_artifacts.py", "--check")] + assert "[FAIL] artifact regeneration failed with exit code 7." in capsys.readouterr().out diff --git a/tests/test_reviewer_docs.py b/tests/test_reviewer_docs.py index 5c14943..31ace21 100644 --- a/tests/test_reviewer_docs.py +++ b/tests/test_reviewer_docs.py @@ -93,6 +93,12 @@ def _read_pyproject() -> dict[str, object]: return tomllib.loads(_read_repo_file("pyproject.toml")) +def _read_issue_template(name: str) -> str: + return (REPO_ROOT / ".github" / "ISSUE_TEMPLATE" / name).read_text( + encoding="utf-8" + ) + + def test_reviewer_path_keeps_detection_lab_positioning() -> None: reviewer_path = _read_repo_file("docs/reviewer-path.md") reviewer_brief = _read_repo_file("docs/reviewer-brief.md") @@ -152,6 +158,7 @@ def test_docs_index_separates_current_route_from_history() -> None: for current_doc in [ "reviewer-pack.md", + "operator-reproduction.md", "reviewer-path.md", "reviewer-brief.md", "v1-contract-freeze.md", @@ -251,6 +258,8 @@ def test_current_docs_use_v1_contract_stabilization_language() -> None: assert "Demo expansion is closed." in current_docs["docs/roadmap.md"] assert "Next phase: v1 reviewer contract stabilization." in current_docs["docs/roadmap.md"] + assert "v1.1 theme: Operator Reproduction Release." in current_docs["docs/roadmap.md"] + assert "v1.1 is an Operator Reproduction Release, not a new-demo release" in current_docs["README.md"] assert "v1.0 Five-Demo Contract Freeze" in current_docs["docs/roadmap.md"] assert "## v1 Reviewer Contract Stabilization" in current_docs["README.md"] @@ -515,3 +524,54 @@ def test_architecture_doc_keeps_local_file_based_boundaries() -> None: for _, demo_name, _ in REVIEWER_DEMO_MATRIX: assert f"`{demo_name}`" in architecture + + +def test_operator_reproduction_doc_and_readme_define_short_gate() -> None: + operator_doc = _read_repo_file("docs/operator-reproduction.md") + docs_index = _read_repo_file("docs/README.md") + readme = _read_repo_file("README.md") + roadmap = _read_repo_file("docs/roadmap.md") + + assert "# Operator Reproduction" in operator_doc + assert "git clone https://github.com/stacknil/telemetry-lab.git" in operator_doc + assert "python -m pip install -e \".[dev]\"" in operator_doc + assert "python scripts/regenerate_artifacts.py --check" in operator_doc + assert "python -m pytest tests/test_evidence_pipeline_schemas.py" in operator_doc + assert "python -m pytest" in operator_doc + assert "python scripts/check_release_contract.py" in operator_doc + assert "does not add a new demo" in operator_doc + assert "does not claim production readiness" in operator_doc + + assert "If you want to verify v1.0 locally, run these three commands." in readme + assert "docs/operator-reproduction.md" in readme + assert "operator-reproduction.md" in docs_index + assert "scripts/check_release_contract.py" in roadmap + + +def test_operator_issue_templates_keep_reviewer_contract_scope() -> None: + issue_template_dir = REPO_ROOT / ".github" / "ISSUE_TEMPLATE" + templates = { + "schema-drift-report.md": _read_issue_template("schema-drift-report.md"), + "artifact-regeneration-failure.md": _read_issue_template( + "artifact-regeneration-failure.md" + ), + "demo-boundary-question.md": _read_issue_template("demo-boundary-question.md"), + } + feature_template = _read_issue_template("feature_request.yml") + + assert issue_template_dir.is_dir() + for name, text in templates.items(): + assert (issue_template_dir / name).is_file() + assert "reviewer-contract" in text + assert "No real account IDs, credentials" in text or "No live AWS account" in text + assert "dashboard" in text + assert "case" in text.lower() + + assert "python -m pytest tests/test_evidence_pipeline_schemas.py" in templates[ + "schema-drift-report.md" + ] + assert "python scripts/regenerate_artifacts.py --check" in templates[ + "artifact-regeneration-failure.md" + ] + assert "No new demo expansion for v1.1." in templates["demo-boundary-question.md"] + assert "next demo" not in feature_template.lower()