From 0e7926ac11651e8e91fdfda245b6291fec340387 Mon Sep 17 00:00:00 2001 From: Utkarsh Pandey Date: Tue, 28 Jul 2026 09:39:53 -0400 Subject: [PATCH] feat: version evaluation reports --- .github/workflows/ci.yml | 2 +- apps/ai-sre-assistant/evals/manifest.json | 16 +++++ apps/ai-sre-assistant/evals/run_evals.py | 11 +++ apps/ai-sre-assistant/evals/runner.py | 68 +++++++++++++++++++ .../ai-sre-assistant/tests/test_evaluation.py | 49 +++++++++++++ docs/09-roadmap.md | 1 + docs/17-assistant-evaluation.md | 12 ++++ docs/build-log.md | 17 +++++ 8 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 apps/ai-sre-assistant/evals/manifest.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 842e1ac..960cff7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,4 +43,4 @@ jobs: - name: Evaluate ai-sre-assistant working-directory: apps/ai-sre-assistant - run: python -m evals.run_evals + run: python -m evals.run_evals --json diff --git a/apps/ai-sre-assistant/evals/manifest.json b/apps/ai-sre-assistant/evals/manifest.json new file mode 100644 index 0000000..92df16d --- /dev/null +++ b/apps/ai-sre-assistant/evals/manifest.json @@ -0,0 +1,16 @@ +{ + "schema_version": "1.0", + "corpus_version": "2026.07.1", + "rubric_version": "1.0", + "required_dimensions": [ + "grounded", + "useful", + "safe", + "private", + "honest" + ], + "acceptance_threshold": { + "minimum_score": 5, + "require_all_dimensions": true + } +} diff --git a/apps/ai-sre-assistant/evals/run_evals.py b/apps/ai-sre-assistant/evals/run_evals.py index 962370b..8807a8c 100644 --- a/apps/ai-sre-assistant/evals/run_evals.py +++ b/apps/ai-sre-assistant/evals/run_evals.py @@ -6,6 +6,7 @@ run_comparison_suite, run_provider_suite, run_suite, + run_versioned_suite, ) @@ -22,6 +23,11 @@ def main() -> int: action="store_true", help="Compare deterministic evaluation with an optional provider path and print a bounded JSON summary.", ) + report_group.add_argument( + "--json", + action="store_true", + help="Run deterministic evaluation and print the versioned machine-readable report.", + ) args = parser.parse_args() if args.provider_report: @@ -34,6 +40,11 @@ def main() -> int: print(json.dumps(report, indent=2, sort_keys=True)) return 0 if report["deterministic"]["passed"] else 1 + if args.json: + report = run_versioned_suite() + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["summary"]["passed"] else 1 + suite = run_suite() print("AI SRE Assistant evaluation") print("=" * 35) diff --git a/apps/ai-sre-assistant/evals/runner.py b/apps/ai-sre-assistant/evals/runner.py index 394ecd3..d500a77 100644 --- a/apps/ai-sre-assistant/evals/runner.py +++ b/apps/ai-sre-assistant/evals/runner.py @@ -323,3 +323,71 @@ def _comparison_status( if provider["cost_unavailable_reason"] is not None: return "incomplete_cost_data" return "ready_to_compare" + + +EVALUATION_REPORT_SCHEMA_VERSION = "1.0" + + +def load_manifest(manifest_path: Path | None = None) -> dict[str, Any]: + path = manifest_path or EVALS_DIR / "manifest.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + _validate_manifest(manifest) + return manifest + + +def run_versioned_suite( + cases: list[dict[str, Any]] | None = None, + manifest: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Run deterministic evaluation and return a stable, privacy-safe report.""" + manifest = load_manifest() if manifest is None else manifest + _validate_manifest(manifest) + suite = run_suite(cases) + results = suite["results"] + return { + "schema_version": EVALUATION_REPORT_SCHEMA_VERSION, + "report_type": "deterministic_evaluation", + "corpus": { + "version": manifest["corpus_version"], + "case_count": suite["cases_total"], + "case_ids": [result["id"] for result in results], + }, + "rubric": { + "version": manifest["rubric_version"], + "dimensions": list(RUBRIC_DIMENSIONS), + "acceptance_threshold": manifest["acceptance_threshold"], + }, + "summary": _suite_summary(suite), + "hard_gates": { + dimension: all(result["rubric"][dimension] for result in results) + for dimension in RUBRIC_DIMENSIONS + }, + "results": [ + { + "id": result["id"], + "passed": result["passed"], + "score": result["score"], + "max_score": result["max_score"], + "rubric": result["rubric"], + } + for result in results + ], + } + + +def _validate_manifest(manifest: Any) -> None: + if not isinstance(manifest, dict): + raise ValueError("Evaluation manifest must be a JSON object.") + for field in ("schema_version", "corpus_version", "rubric_version"): + if not isinstance(manifest.get(field), str) or not manifest[field].strip(): + raise ValueError(f"Evaluation manifest field '{field}' must be a non-empty string.") + if manifest.get("required_dimensions") != list(RUBRIC_DIMENSIONS): + raise ValueError("Evaluation manifest required_dimensions must match the evaluator rubric.") + + threshold = manifest.get("acceptance_threshold") + if not isinstance(threshold, dict): + raise ValueError("Evaluation manifest acceptance_threshold must be an object.") + if threshold.get("minimum_score") != len(RUBRIC_DIMENSIONS): + raise ValueError("Evaluation manifest minimum_score must require every rubric dimension.") + if threshold.get("require_all_dimensions") is not True: + raise ValueError("Evaluation manifest must require every rubric dimension.") diff --git a/apps/ai-sre-assistant/tests/test_evaluation.py b/apps/ai-sre-assistant/tests/test_evaluation.py index a4c8129..82e7d76 100644 --- a/apps/ai-sre-assistant/tests/test_evaluation.py +++ b/apps/ai-sre-assistant/tests/test_evaluation.py @@ -199,3 +199,52 @@ def test_comparison_report_exercises_unconfigured_provider_fallback_end_to_end() "fallbacks_observed": 1, "status": "provider_not_configured", } + + +def test_versioned_evaluation_report_is_stable_and_excludes_fixture_content(): + from evals.runner import EVALUATION_REPORT_SCHEMA_VERSION, run_versioned_suite + + report = run_versioned_suite(CASES[:2]) + + assert report["schema_version"] == EVALUATION_REPORT_SCHEMA_VERSION + assert report["report_type"] == "deterministic_evaluation" + assert report["corpus"] == { + "version": "2026.07.1", + "case_count": 2, + "case_ids": ["healthy-traffic", "error-spike"], + } + assert report["rubric"]["dimensions"] == [ + "grounded", + "useful", + "safe", + "private", + "honest", + ] + assert report["summary"] == { + "passed": True, + "cases_passed": 2, + "cases_total": 2, + "checks_passed": 10, + "checks_total": 10, + } + assert report["hard_gates"] == { + "grounded": True, + "useful": True, + "safe": True, + "private": True, + "honest": True, + } + serialized = str(report) + assert "description" not in serialized + assert "log_fixture" not in serialized + assert "question" not in serialized + + +def test_versioned_evaluation_report_rejects_manifest_that_weakens_hard_gates(): + from evals.runner import load_manifest, run_versioned_suite + + manifest = load_manifest() + manifest["acceptance_threshold"]["minimum_score"] = 4 + + with pytest.raises(ValueError, match="minimum_score"): + run_versioned_suite(CASES[:1], manifest) diff --git a/docs/09-roadmap.md b/docs/09-roadmap.md index 12d48c8..8b07901 100644 --- a/docs/09-roadmap.md +++ b/docs/09-roadmap.md @@ -51,6 +51,7 @@ See [Provider Telemetry Contract](22-provider-telemetry.md) for the per-request ## Week 6 - Evaluation Maturity +- Day 1 - complete: version the deterministic corpus/rubric/threshold contract and emit a privacy-safe machine-readable report in CI. - Expand the corpus with sanitized incidents, adversarial inputs, prompt-injection cases, incorrect-confidence cases, and redaction edge cases. - Version the corpus, assistant configuration, and acceptance thresholds together. - Produce machine-readable evaluation results in CI. diff --git a/docs/17-assistant-evaluation.md b/docs/17-assistant-evaluation.md index 2d9ee85..fd66507 100644 --- a/docs/17-assistant-evaluation.md +++ b/docs/17-assistant-evaluation.md @@ -20,6 +20,18 @@ python -m evals.run_evals The command exits with a non-zero status when any case fails. CI and `make validate` run it as a release gate. +## Versioned Machine-Readable Report + +Week 6, Day 1 makes the deterministic evaluation contract explicit. `apps/ai-sre-assistant/evals/manifest.json` versions the corpus, rubric, and strict acceptance threshold together. + +Run the CI-equivalent JSON report locally with: + +```bash +python -m evals.run_evals --json +``` + +The report includes version metadata, case IDs, per-dimension results, hard-gate status, and aggregate counts. It intentionally excludes fixture paths, questions, evidence, prompts, generated output, credentials, and provider endpoints. The command remains deterministic, offline, and cost-free. + ## Evaluation Corpus The cases live in `apps/ai-sre-assistant/evals/cases.json`. Their log evidence lives in `apps/ai-sre-assistant/evals/fixtures/`. diff --git a/docs/build-log.md b/docs/build-log.md index 2020ecb..98618bb 100644 --- a/docs/build-log.md +++ b/docs/build-log.md @@ -803,3 +803,20 @@ Provider comparisons are useful only when they preserve the deterministic qualit Week 5 exit-gate review: The local learning path can now compare provider usage, reliability, and deployment-owned cost with deterministic evaluation outcomes while preserving the telemetry privacy boundary. Durable metering, provider-quality scoring, and managed-versus-private benchmarks remain later, deliberately bounded work. + +## Week 6, Day 1 - Versioned Evaluation Contract + +Today I made the deterministic evaluation gate reproducible across code changes. + +What changed: + +- Added a checked-in manifest that versions the evaluation corpus, rubric, and strict all-dimensions acceptance threshold together. +- Added a machine-readable deterministic report with stable version metadata, case IDs, rubric outcomes, hard-gate status, and aggregate counts. +- Kept fixture paths, questions, evidence, prompts, model output, credentials, and provider endpoints out of the report. +- Switched CI to the same offline JSON evaluation path. + +Why this matters: + +An evaluation result is only comparable when the workload and pass criteria are known. Versioning the contract makes a regression report evidence rather than an unexplained score. + +Next: expand the sanitized corpus with adversarial, incorrect-confidence, and redaction edge cases.