feat: version evaluation reports

.github/workflows/ci.yml

@@ -43,4 +43,4 @@ jobs:
 
       - name: Evaluate ai-sre-assistant
         working-directory: apps/ai-sre-assistant
-        run: python -m evals.run_evals
+        run: python -m evals.run_evals --json

apps/ai-sre-assistant/evals/manifest.json

@@ -0,0 +1,16 @@
+{
+  "schema_version": "1.0",
+  "corpus_version": "2026.07.1",
+  "rubric_version": "1.0",
+  "required_dimensions": [
+    "grounded",
+    "useful",
+    "safe",
+    "private",
+    "honest"
+  ],
+  "acceptance_threshold": {
+    "minimum_score": 5,
+    "require_all_dimensions": true
+  }
+}

apps/ai-sre-assistant/evals/run_evals.py

@@ -6,6 +6,7 @@
     run_comparison_suite,
     run_provider_suite,
     run_suite,
+    run_versioned_suite,
 )
 
 
@@ -22,6 +23,11 @@ def main() -> int:
         action="store_true",
         help="Compare deterministic evaluation with an optional provider path and print a bounded JSON summary.",
     )
+    report_group.add_argument(
+        "--json",
+        action="store_true",
+        help="Run deterministic evaluation and print the versioned machine-readable report.",
+    )
     args = parser.parse_args()
 
     if args.provider_report:
@@ -34,6 +40,11 @@ def main() -> int:
         print(json.dumps(report, indent=2, sort_keys=True))
         return 0 if report["deterministic"]["passed"] else 1
 
+    if args.json:
+        report = run_versioned_suite()
+        print(json.dumps(report, indent=2, sort_keys=True))
+        return 0 if report["summary"]["passed"] else 1
+
     suite = run_suite()
     print("AI SRE Assistant evaluation")
     print("=" * 35)

apps/ai-sre-assistant/evals/runner.py

@@ -323,3 +323,71 @@ def _comparison_status(
     if provider["cost_unavailable_reason"] is not None:
         return "incomplete_cost_data"
     return "ready_to_compare"
+
+
+EVALUATION_REPORT_SCHEMA_VERSION = "1.0"
+
+
+def load_manifest(manifest_path: Path | None = None) -> dict[str, Any]:
+    path = manifest_path or EVALS_DIR / "manifest.json"
+    manifest = json.loads(path.read_text(encoding="utf-8"))
+    _validate_manifest(manifest)
+    return manifest
+
+
+def run_versioned_suite(
+    cases: list[dict[str, Any]] | None = None,
+    manifest: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Run deterministic evaluation and return a stable, privacy-safe report."""
+    manifest = load_manifest() if manifest is None else manifest
+    _validate_manifest(manifest)
+    suite = run_suite(cases)
+    results = suite["results"]
+    return {
+        "schema_version": EVALUATION_REPORT_SCHEMA_VERSION,
+        "report_type": "deterministic_evaluation",
+        "corpus": {
+            "version": manifest["corpus_version"],
+            "case_count": suite["cases_total"],
+            "case_ids": [result["id"] for result in results],
+        },
+        "rubric": {
+            "version": manifest["rubric_version"],
+            "dimensions": list(RUBRIC_DIMENSIONS),
+            "acceptance_threshold": manifest["acceptance_threshold"],
+        },
+        "summary": _suite_summary(suite),
+        "hard_gates": {
+            dimension: all(result["rubric"][dimension] for result in results)
+            for dimension in RUBRIC_DIMENSIONS
+        },
+        "results": [
+            {
+                "id": result["id"],
+                "passed": result["passed"],
+                "score": result["score"],
+                "max_score": result["max_score"],
+                "rubric": result["rubric"],
+            }
+            for result in results
+        ],
+    }
+
+
+def _validate_manifest(manifest: Any) -> None:
+    if not isinstance(manifest, dict):
+        raise ValueError("Evaluation manifest must be a JSON object.")
+    for field in ("schema_version", "corpus_version", "rubric_version"):
+        if not isinstance(manifest.get(field), str) or not manifest[field].strip():
+            raise ValueError(f"Evaluation manifest field '{field}' must be a non-empty string.")
+    if manifest.get("required_dimensions") != list(RUBRIC_DIMENSIONS):
+        raise ValueError("Evaluation manifest required_dimensions must match the evaluator rubric.")
+
+    threshold = manifest.get("acceptance_threshold")
+    if not isinstance(threshold, dict):
+        raise ValueError("Evaluation manifest acceptance_threshold must be an object.")
+    if threshold.get("minimum_score") != len(RUBRIC_DIMENSIONS):
+        raise ValueError("Evaluation manifest minimum_score must require every rubric dimension.")
+    if threshold.get("require_all_dimensions") is not True:
+        raise ValueError("Evaluation manifest must require every rubric dimension.")

apps/ai-sre-assistant/tests/test_evaluation.py

@@ -199,3 +199,52 @@ def test_comparison_report_exercises_unconfigured_provider_fallback_end_to_end()
         "fallbacks_observed": 1,
         "status": "provider_not_configured",
     }
+
+
+def test_versioned_evaluation_report_is_stable_and_excludes_fixture_content():
+    from evals.runner import EVALUATION_REPORT_SCHEMA_VERSION, run_versioned_suite
+
+    report = run_versioned_suite(CASES[:2])
+
+    assert report["schema_version"] == EVALUATION_REPORT_SCHEMA_VERSION
+    assert report["report_type"] == "deterministic_evaluation"
+    assert report["corpus"] == {
+        "version": "2026.07.1",
+        "case_count": 2,
+        "case_ids": ["healthy-traffic", "error-spike"],
+    }
+    assert report["rubric"]["dimensions"] == [
+        "grounded",
+        "useful",
+        "safe",
+        "private",
+        "honest",
+    ]
+    assert report["summary"] == {
+        "passed": True,
+        "cases_passed": 2,
+        "cases_total": 2,
+        "checks_passed": 10,
+        "checks_total": 10,
+    }
+    assert report["hard_gates"] == {
+        "grounded": True,
+        "useful": True,
+        "safe": True,
+        "private": True,
+        "honest": True,
+    }
+    serialized = str(report)
+    assert "description" not in serialized
+    assert "log_fixture" not in serialized
+    assert "question" not in serialized
+
+
+def test_versioned_evaluation_report_rejects_manifest_that_weakens_hard_gates():
+    from evals.runner import load_manifest, run_versioned_suite
+
+    manifest = load_manifest()
+    manifest["acceptance_threshold"]["minimum_score"] = 4
+
+    with pytest.raises(ValueError, match="minimum_score"):
+        run_versioned_suite(CASES[:1], manifest)

docs/09-roadmap.md

@@ -51,6 +51,7 @@ See [Provider Telemetry Contract](22-provider-telemetry.md) for the per-request
 
 ## Week 6 - Evaluation Maturity
 
+- Day 1 - complete: version the deterministic corpus/rubric/threshold contract and emit a privacy-safe machine-readable report in CI.
 - Expand the corpus with sanitized incidents, adversarial inputs, prompt-injection cases, incorrect-confidence cases, and redaction edge cases.
 - Version the corpus, assistant configuration, and acceptance thresholds together.
 - Produce machine-readable evaluation results in CI.

docs/17-assistant-evaluation.md

@@ -20,6 +20,18 @@ python -m evals.run_evals
 
 The command exits with a non-zero status when any case fails. CI and `make validate` run it as a release gate.
 
+## Versioned Machine-Readable Report
+
+Week 6, Day 1 makes the deterministic evaluation contract explicit. `apps/ai-sre-assistant/evals/manifest.json` versions the corpus, rubric, and strict acceptance threshold together.
+
+Run the CI-equivalent JSON report locally with:
+
+```bash
+python -m evals.run_evals --json
+```
+
+The report includes version metadata, case IDs, per-dimension results, hard-gate status, and aggregate counts. It intentionally excludes fixture paths, questions, evidence, prompts, generated output, credentials, and provider endpoints. The command remains deterministic, offline, and cost-free.
+
 ## Evaluation Corpus
 
 The cases live in `apps/ai-sre-assistant/evals/cases.json`. Their log evidence lives in `apps/ai-sre-assistant/evals/fixtures/`.

docs/build-log.md

@@ -803,3 +803,20 @@ Provider comparisons are useful only when they preserve the deterministic qualit
 Week 5 exit-gate review:
 
 The local learning path can now compare provider usage, reliability, and deployment-owned cost with deterministic evaluation outcomes while preserving the telemetry privacy boundary. Durable metering, provider-quality scoring, and managed-versus-private benchmarks remain later, deliberately bounded work.
+
+## Week 6, Day 1 - Versioned Evaluation Contract
+
+Today I made the deterministic evaluation gate reproducible across code changes.
+
+What changed:
+
+- Added a checked-in manifest that versions the evaluation corpus, rubric, and strict all-dimensions acceptance threshold together.
+- Added a machine-readable deterministic report with stable version metadata, case IDs, rubric outcomes, hard-gate status, and aggregate counts.
+- Kept fixture paths, questions, evidence, prompts, model output, credentials, and provider endpoints out of the report.
+- Switched CI to the same offline JSON evaluation path.
+
+Why this matters:
+
+An evaluation result is only comparable when the workload and pass criteria are known. Versioning the contract makes a regression report evidence rather than an unexplained score.
+
+Next: expand the sanitized corpus with adversarial, incorrect-confidence, and redaction edge cases.