feat: add local provider comparison report

apps/ai-sre-assistant/README.md

@@ -134,3 +134,13 @@ python -m evals.run_evals --provider-report
 ```
 
 This explicit command makes one provider call per evaluation case and prints a bounded JSON report, including estimated cost per successful evaluated analysis when complete token and price data is available. It remains outside CI so the normal release gate stays offline and cost-free.
+
+## Local Provider Comparison
+
+To compare the deterministic release gate with the configured provider path, run:
+
+```bash
+python -m evals.run_evals --comparison-report
+```
+
+The report contains only counts, bounded provider/model identity, quality-gate parity, provider outcomes, fallback count, and cost summaries. It does not print fixture evidence, prompts, provider output, credentials, or endpoints. A configured provider makes one call per fixture; with `LLM_PROVIDER=none`, the same command is an offline end-to-end fallback check.

apps/ai-sre-assistant/evals/run_evals.py

@@ -1,23 +1,39 @@
 import argparse
 import json
 
-from evals.runner import RUBRIC_DIMENSIONS, run_provider_suite, run_suite
+from evals.runner import (
+    RUBRIC_DIMENSIONS,
+    run_comparison_suite,
+    run_provider_suite,
+    run_suite,
+)
 
 
 def main() -> int:
     parser = argparse.ArgumentParser(description="Run AI SRE Assistant evaluations.")
-    parser.add_argument(
+    report_group = parser.add_mutually_exclusive_group()
+    report_group.add_argument(
         "--provider-report",
         action="store_true",
         help="Run the corpus with optional provider enrichment and print a bounded JSON cost report.",
     )
+    report_group.add_argument(
+        "--comparison-report",
+        action="store_true",
+        help="Compare deterministic evaluation with an optional provider path and print a bounded JSON summary.",
+    )
     args = parser.parse_args()
 
     if args.provider_report:
         suite = run_provider_suite()
         print(json.dumps(suite, indent=2, sort_keys=True))
         return 0 if suite["passed"] else 1
 
+    if args.comparison_report:
+        report = run_comparison_suite()
+        print(json.dumps(report, indent=2, sort_keys=True))
+        return 0 if report["deterministic"]["passed"] else 1
+
     suite = run_suite()
     print("AI SRE Assistant evaluation")
     print("=" * 35)

apps/ai-sre-assistant/evals/runner.py

@@ -52,9 +52,13 @@ def evaluate_provider_case(case: dict[str, Any], config: LLMConfig) -> dict[str,
     }
 
 
-def _analyze_case_with_logs(case: dict[str, Any]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+def _analyze_case_with_logs(
+    case: dict[str, Any],
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
     fixture_name = case.get("log_fixture")
-    fixture_path = FIXTURES_DIR / fixture_name if fixture_name else FIXTURES_DIR / "missing.log"
+    fixture_path = (
+        FIXTURES_DIR / fixture_name if fixture_name else FIXTURES_DIR / "missing.log"
+    )
     logs = read_recent_logs(log_path=fixture_path, max_lines=100)
     return logs, analyze_logs(logs, question=case.get("question"))
 
@@ -69,7 +73,8 @@ def evaluate_output(case: dict[str, Any], output: dict[str, Any]) -> dict[str, A
 
     rubric = {
         "grounded": (
-            expected["summary_contains"].lower() in str(output.get("summary", "")).lower()
+            expected["summary_contains"].lower()
+            in str(output.get("summary", "")).lower()
             and _contains_all(facts, expected.get("facts_contain", []))
             and len(output.get("evidence", [])) >= expected.get("min_evidence", 0)
         ),
@@ -80,14 +85,20 @@ def evaluate_output(case: dict[str, Any], output: dict[str, Any]) -> dict[str, A
         ),
         "safe": not _contains_any(serialized, UNSAFE_OUTPUT_PATTERNS),
         "private": (
-            not _contains_any(serialized, expected.get("forbidden_output", []), case_sensitive=True)
-            and (not expected.get("requires_redaction", False) or REDACTED in serialized)
+            not _contains_any(
+                serialized, expected.get("forbidden_output", []), case_sensitive=True
+            )
+            and (
+                not expected.get("requires_redaction", False) or REDACTED in serialized
+            )
         ),
         "honest": (
             bool(facts)
             and bool(guesses)
             and _contains_all(guesses, expected.get("guesses_contain", []))
-            and not _contains_any(" ".join(str(item) for item in facts), SPECULATION_MARKERS)
+            and not _contains_any(
+                " ".join(str(item) for item in facts), SPECULATION_MARKERS
+            )
         ),
     }
 
@@ -107,7 +118,9 @@ def evaluate_case(case: dict[str, Any]) -> dict[str, Any]:
 
 
 def run_suite(cases: list[dict[str, Any]] | None = None) -> dict[str, Any]:
-    results = [evaluate_case(case) for case in (cases if cases is not None else load_cases())]
+    results = [
+        evaluate_case(case) for case in (cases if cases is not None else load_cases())
+    ]
     checks_passed = sum(result["score"] for result in results)
     checks_total = sum(result["max_score"] for result in results)
     return {
@@ -130,7 +143,10 @@ def run_provider_suite(
     normal deterministic CI stays offline and has no token cost.
     """
     config = config or load_config()
-    results = [evaluate_provider_case(case, config) for case in (cases if cases is not None else load_cases())]
+    results = [
+        evaluate_provider_case(case, config)
+        for case in (cases if cases is not None else load_cases())
+    ]
     checks_passed = sum(result["score"] for result in results)
     checks_total = sum(result["max_score"] for result in results)
     successful = [
@@ -140,10 +156,13 @@ def run_provider_suite(
     ]
     known_costs = [
         cost
-        for cost in (_cost_decimal(result["llm_cost_estimate"].get("estimated_total_cost_usd")) for result in successful)
+        for cost in (
+            _cost_decimal(result["llm_cost_estimate"].get("estimated_total_cost_usd"))
+            for result in successful
+        )
         if cost is not None
     ]
-    costs_complete = len(known_costs) == len(successful)
+    costs_complete = bool(successful) and len(known_costs) == len(successful)
     total_cost = sum(known_costs, Decimal("0")) if costs_complete else None
     provider_outcomes = _count_provider_outcomes(results)
 
@@ -155,7 +174,8 @@ def run_provider_suite(
         cost_unavailable_reason = None
 
     return {
-        "passed": all(result["passed"] for result in results) and provider_outcomes["success"] == len(results),
+        "passed": all(result["passed"] for result in results)
+        and provider_outcomes["success"] == len(results),
         "provider": _provider_identity(results),
         "cases_passed": sum(result["passed"] for result in results),
         "cases_total": len(results),
@@ -166,7 +186,9 @@ def run_provider_suite(
         "costed_successful_evaluated_analyses": len(known_costs),
         "estimated_total_cost_usd": _decimal_string(total_cost),
         "estimated_cost_per_successful_evaluated_analysis_usd": (
-            _decimal_string(total_cost / len(successful)) if total_cost is not None and successful else None
+            _decimal_string(total_cost / len(successful))
+            if total_cost is not None and successful
+            else None
         ),
         "cost_unavailable_reason": cost_unavailable_reason,
         "results": results,
@@ -178,7 +200,9 @@ def _contains_all(values: list[Any], needles: list[str]) -> bool:
     return all(needle.lower() in text for needle in needles)
 
 
-def _contains_any(text: str, needles: tuple[str, ...] | list[str], case_sensitive: bool = False) -> bool:
+def _contains_any(
+    text: str, needles: tuple[str, ...] | list[str], case_sensitive: bool = False
+) -> bool:
     if not case_sensitive:
         text = text.lower()
         needles = [needle.lower() for needle in needles]
@@ -212,4 +236,90 @@ def _cost_decimal(value: Any) -> Decimal | None:
 
 
 def _decimal_string(value: Decimal | None) -> str | None:
-    return format(value.quantize(Decimal("0.00000001")), "f") if value is not None else None
+    return (
+        format(value.quantize(Decimal("0.00000001")), "f")
+        if value is not None
+        else None
+    )
+
+
+def run_comparison_suite(
+    cases: list[dict[str, Any]] | None = None,
+    config: LLMConfig | None = None,
+) -> dict[str, Any]:
+    """Compare the offline quality gate with one optional provider run.
+
+    The report contains summaries only, avoiding fixture content, prompts, model
+    output, or endpoint information.
+    """
+    selected_cases = cases if cases is not None else load_cases()
+    deterministic = run_suite(selected_cases)
+    provider = run_provider_suite(selected_cases, config)
+    quality_results_match = _quality_results_match(
+        deterministic["results"], provider["results"]
+    )
+    provider_outcomes = provider["provider_outcomes"]
+
+    return {
+        "schema_version": "week5-comparison-v1",
+        "deterministic": _suite_summary(deterministic),
+        "provider": {
+            "identity": provider["provider"],
+            "passed": provider["passed"],
+            "provider_outcomes": provider_outcomes,
+            "successful_evaluated_analyses": provider["successful_evaluated_analyses"],
+            "costed_successful_evaluated_analyses": provider[
+                "costed_successful_evaluated_analyses"
+            ],
+            "estimated_total_cost_usd": provider["estimated_total_cost_usd"],
+            "estimated_cost_per_successful_evaluated_analysis_usd": provider[
+                "estimated_cost_per_successful_evaluated_analysis_usd"
+            ],
+            "cost_unavailable_reason": provider["cost_unavailable_reason"],
+        },
+        "comparison": {
+            "quality_results_match": quality_results_match,
+            "fallbacks_observed": provider_outcomes["failure"]
+            + provider_outcomes["not_configured"],
+            "status": _comparison_status(
+                deterministic, provider, quality_results_match
+            ),
+        },
+    }
+
+
+def _suite_summary(suite: dict[str, Any]) -> dict[str, int | bool]:
+    return {
+        "passed": suite["passed"],
+        "cases_passed": suite["cases_passed"],
+        "cases_total": suite["cases_total"],
+        "checks_passed": suite["checks_passed"],
+        "checks_total": suite["checks_total"],
+    }
+
+
+def _quality_results_match(
+    deterministic: list[dict[str, Any]], provider: list[dict[str, Any]]
+) -> bool:
+    deterministic_scores = [
+        (result["id"], result["score"], result["rubric"]) for result in deterministic
+    ]
+    provider_scores = [
+        (result["id"], result["score"], result["rubric"]) for result in provider
+    ]
+    return deterministic_scores == provider_scores
+
+
+def _comparison_status(
+    deterministic: dict[str, Any], provider: dict[str, Any], quality_results_match: bool
+) -> str:
+    if not deterministic["passed"] or not quality_results_match:
+        return "deterministic_quality_gate_failed"
+    outcomes = provider["provider_outcomes"]
+    if outcomes["not_configured"]:
+        return "provider_not_configured"
+    if outcomes["failure"]:
+        return "provider_request_failed"
+    if provider["cost_unavailable_reason"] is not None:
+        return "incomplete_cost_data"
+    return "ready_to_compare"

apps/ai-sre-assistant/tests/test_evaluation.py

@@ -78,15 +78,23 @@ def successful_enrichment(**_kwargs):
     report = run_provider_suite(CASES[:2], config)
 
     assert report["passed"] is True
-    assert report["provider_outcomes"] == {"success": 2, "failure": 0, "not_configured": 0}
+    assert report["provider_outcomes"] == {
+        "success": 2,
+        "failure": 0,
+        "not_configured": 0,
+    }
     assert report["successful_evaluated_analyses"] == 2
     assert report["costed_successful_evaluated_analyses"] == 2
     assert report["estimated_total_cost_usd"] == "0.00030000"
-    assert report["estimated_cost_per_successful_evaluated_analysis_usd"] == "0.00015000"
+    assert (
+        report["estimated_cost_per_successful_evaluated_analysis_usd"] == "0.00015000"
+    )
     assert report["cost_unavailable_reason"] is None
 
 
-def test_provider_report_keeps_cost_per_success_unknown_with_incomplete_usage(monkeypatch):
+def test_provider_report_keeps_cost_per_success_unknown_with_incomplete_usage(
+    monkeypatch,
+):
     from decimal import Decimal
 
     from app.llm import LLMCallResult, LLMConfig
@@ -121,3 +129,73 @@ def enrichment_without_usage(**_kwargs):
     assert report["estimated_total_cost_usd"] is None
     assert report["estimated_cost_per_successful_evaluated_analysis_usd"] is None
     assert report["cost_unavailable_reason"] == "incomplete_cost_data"
+
+
+def test_comparison_report_is_ready_when_provider_quality_and_cost_are_complete(
+    monkeypatch,
+):
+    from decimal import Decimal
+
+    from app.llm import LLMCallResult, LLMConfig
+    from evals.runner import run_comparison_suite
+
+    def successful_enrichment(**_kwargs):
+        return LLMCallResult(
+            analysis="optional provider analysis",
+            notice=None,
+            telemetry={
+                "provider": "openai",
+                "model": "eval-model",
+                "outcome": "success",
+                "usage": {"input_tokens": 100, "output_tokens": 50},
+            },
+        )
+
+    monkeypatch.setattr("evals.runner.analyze_with_llm", successful_enrichment)
+    config = LLMConfig(
+        provider="openai",
+        api_key="test-key",
+        base_url="https://example.invalid/v1",
+        model="eval-model",
+        input_usd_per_million_tokens=Decimal("1"),
+        output_usd_per_million_tokens=Decimal("1"),
+    )
+
+    report = run_comparison_suite(CASES[:2], config)
+
+    assert report["schema_version"] == "week5-comparison-v1"
+    assert report["deterministic"]["passed"] is True
+    assert report["provider"]["provider_outcomes"] == {
+        "success": 2,
+        "failure": 0,
+        "not_configured": 0,
+    }
+    assert report["comparison"] == {
+        "quality_results_match": True,
+        "fallbacks_observed": 0,
+        "status": "ready_to_compare",
+    }
+
+
+def test_comparison_report_exercises_unconfigured_provider_fallback_end_to_end():
+    from app.llm import LLMConfig
+    from evals.runner import run_comparison_suite
+
+    report = run_comparison_suite(
+        CASES[:1],
+        LLMConfig(provider="none", api_key="", base_url="", model=""),
+    )
+
+    assert report["deterministic"]["passed"] is True
+    assert report["provider"]["provider_outcomes"] == {
+        "success": 0,
+        "failure": 0,
+        "not_configured": 1,
+    }
+    assert report["provider"]["successful_evaluated_analyses"] == 0
+    assert report["provider"]["estimated_total_cost_usd"] is None
+    assert report["comparison"] == {
+        "quality_results_match": True,
+        "fallbacks_observed": 1,
+        "status": "provider_not_configured",
+    }

docs/09-roadmap.md

@@ -43,7 +43,7 @@ This is the canonical technical execution order for Reliability Lab. Each week a
 - Day 2 - complete: add aggregate counters and latency distributions with bounded labels.
 - Day 3 - complete: accept explicit deployment-owned pricing inputs and return per-call estimated cost metadata only when prices and token directions are known.
 - Day 4 - complete: join provider outcomes and deployment-owned cost estimates with deterministic evaluation results, including cost per successful evaluated analysis when data is complete.
-- Add a local comparison report across deterministic and configured provider paths.
+- Day 5 - complete: add a bounded local comparison report across deterministic and configured provider paths, including the end-to-end fallback state.
 
 See [Provider Telemetry Contract](22-provider-telemetry.md) for the per-request contract, aggregate metrics, privacy boundary, and remaining-week sequence.
 

docs/17-assistant-evaluation.md

@@ -76,3 +76,15 @@ This makes one optional provider-enrichment call per fixture and emits a bounded
 A successful evaluated analysis requires a passing deterministic rubric and a successful provider response. The cost-per-success value is emitted only when every successful evaluated analysis has complete price and token data. Otherwise it is `null` with `cost_unavailable_reason`; unknown usage is never treated as zero cost. The report does not write a usage ledger or include fixture evidence, prompts, provider output, endpoints, or credentials.
 
 The normal evaluation command remains deterministic, offline, and cost-free. The provider report is deliberately opt-in and remains outside CI until a controlled provider test environment and explicit budget exist.
+
+## Local Provider Comparison
+
+Run the local comparison report with:
+
+```bash
+python -m evals.run_evals --comparison-report
+```
+
+It runs the deterministic corpus and the selected optional-provider path against the same fixtures, then reports the two quality summaries, bounded provider/model identity, outcome counts, fallback count, and estimated cost summary. `comparison.status` is `ready_to_compare` only when deterministic quality passes, the provider succeeds for every case, and cost is complete. Other explicit states distinguish unconfigured providers, request failures, incomplete cost data, and deterministic-gate failures.
+
+The command prints summaries only: it excludes fixture evidence, prompts, provider output, credentials, and endpoints. It is opt-in and outside CI because a configured provider makes one call per fixture. With `LLM_PROVIDER=none`, it is an offline end-to-end check that verifies the deterministic fallback remains available.