diff --git a/apps/ai-sre-assistant/README.md b/apps/ai-sre-assistant/README.md index b94c4e7..fe23d14 100644 --- a/apps/ai-sre-assistant/README.md +++ b/apps/ai-sre-assistant/README.md @@ -134,3 +134,13 @@ python -m evals.run_evals --provider-report ``` This explicit command makes one provider call per evaluation case and prints a bounded JSON report, including estimated cost per successful evaluated analysis when complete token and price data is available. It remains outside CI so the normal release gate stays offline and cost-free. + +## Local Provider Comparison + +To compare the deterministic release gate with the configured provider path, run: + +```bash +python -m evals.run_evals --comparison-report +``` + +The report contains only counts, bounded provider/model identity, quality-gate parity, provider outcomes, fallback count, and cost summaries. It does not print fixture evidence, prompts, provider output, credentials, or endpoints. A configured provider makes one call per fixture; with `LLM_PROVIDER=none`, the same command is an offline end-to-end fallback check. diff --git a/apps/ai-sre-assistant/evals/run_evals.py b/apps/ai-sre-assistant/evals/run_evals.py index f70e244..962370b 100644 --- a/apps/ai-sre-assistant/evals/run_evals.py +++ b/apps/ai-sre-assistant/evals/run_evals.py @@ -1,16 +1,27 @@ import argparse import json -from evals.runner import RUBRIC_DIMENSIONS, run_provider_suite, run_suite +from evals.runner import ( + RUBRIC_DIMENSIONS, + run_comparison_suite, + run_provider_suite, + run_suite, +) def main() -> int: parser = argparse.ArgumentParser(description="Run AI SRE Assistant evaluations.") - parser.add_argument( + report_group = parser.add_mutually_exclusive_group() + report_group.add_argument( "--provider-report", action="store_true", help="Run the corpus with optional provider enrichment and print a bounded JSON cost report.", ) + report_group.add_argument( + "--comparison-report", + action="store_true", + help="Compare deterministic evaluation with an optional provider path and print a bounded JSON summary.", + ) args = parser.parse_args() if args.provider_report: @@ -18,6 +29,11 @@ def main() -> int: print(json.dumps(suite, indent=2, sort_keys=True)) return 0 if suite["passed"] else 1 + if args.comparison_report: + report = run_comparison_suite() + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["deterministic"]["passed"] else 1 + suite = run_suite() print("AI SRE Assistant evaluation") print("=" * 35) diff --git a/apps/ai-sre-assistant/evals/runner.py b/apps/ai-sre-assistant/evals/runner.py index f9acd77..394ecd3 100644 --- a/apps/ai-sre-assistant/evals/runner.py +++ b/apps/ai-sre-assistant/evals/runner.py @@ -52,9 +52,13 @@ def evaluate_provider_case(case: dict[str, Any], config: LLMConfig) -> dict[str, } -def _analyze_case_with_logs(case: dict[str, Any]) -> tuple[list[dict[str, Any]], dict[str, Any]]: +def _analyze_case_with_logs( + case: dict[str, Any], +) -> tuple[list[dict[str, Any]], dict[str, Any]]: fixture_name = case.get("log_fixture") - fixture_path = FIXTURES_DIR / fixture_name if fixture_name else FIXTURES_DIR / "missing.log" + fixture_path = ( + FIXTURES_DIR / fixture_name if fixture_name else FIXTURES_DIR / "missing.log" + ) logs = read_recent_logs(log_path=fixture_path, max_lines=100) return logs, analyze_logs(logs, question=case.get("question")) @@ -69,7 +73,8 @@ def evaluate_output(case: dict[str, Any], output: dict[str, Any]) -> dict[str, A rubric = { "grounded": ( - expected["summary_contains"].lower() in str(output.get("summary", "")).lower() + expected["summary_contains"].lower() + in str(output.get("summary", "")).lower() and _contains_all(facts, expected.get("facts_contain", [])) and len(output.get("evidence", [])) >= expected.get("min_evidence", 0) ), @@ -80,14 +85,20 @@ def evaluate_output(case: dict[str, Any], output: dict[str, Any]) -> dict[str, A ), "safe": not _contains_any(serialized, UNSAFE_OUTPUT_PATTERNS), "private": ( - not _contains_any(serialized, expected.get("forbidden_output", []), case_sensitive=True) - and (not expected.get("requires_redaction", False) or REDACTED in serialized) + not _contains_any( + serialized, expected.get("forbidden_output", []), case_sensitive=True + ) + and ( + not expected.get("requires_redaction", False) or REDACTED in serialized + ) ), "honest": ( bool(facts) and bool(guesses) and _contains_all(guesses, expected.get("guesses_contain", [])) - and not _contains_any(" ".join(str(item) for item in facts), SPECULATION_MARKERS) + and not _contains_any( + " ".join(str(item) for item in facts), SPECULATION_MARKERS + ) ), } @@ -107,7 +118,9 @@ def evaluate_case(case: dict[str, Any]) -> dict[str, Any]: def run_suite(cases: list[dict[str, Any]] | None = None) -> dict[str, Any]: - results = [evaluate_case(case) for case in (cases if cases is not None else load_cases())] + results = [ + evaluate_case(case) for case in (cases if cases is not None else load_cases()) + ] checks_passed = sum(result["score"] for result in results) checks_total = sum(result["max_score"] for result in results) return { @@ -130,7 +143,10 @@ def run_provider_suite( normal deterministic CI stays offline and has no token cost. """ config = config or load_config() - results = [evaluate_provider_case(case, config) for case in (cases if cases is not None else load_cases())] + results = [ + evaluate_provider_case(case, config) + for case in (cases if cases is not None else load_cases()) + ] checks_passed = sum(result["score"] for result in results) checks_total = sum(result["max_score"] for result in results) successful = [ @@ -140,10 +156,13 @@ def run_provider_suite( ] known_costs = [ cost - for cost in (_cost_decimal(result["llm_cost_estimate"].get("estimated_total_cost_usd")) for result in successful) + for cost in ( + _cost_decimal(result["llm_cost_estimate"].get("estimated_total_cost_usd")) + for result in successful + ) if cost is not None ] - costs_complete = len(known_costs) == len(successful) + costs_complete = bool(successful) and len(known_costs) == len(successful) total_cost = sum(known_costs, Decimal("0")) if costs_complete else None provider_outcomes = _count_provider_outcomes(results) @@ -155,7 +174,8 @@ def run_provider_suite( cost_unavailable_reason = None return { - "passed": all(result["passed"] for result in results) and provider_outcomes["success"] == len(results), + "passed": all(result["passed"] for result in results) + and provider_outcomes["success"] == len(results), "provider": _provider_identity(results), "cases_passed": sum(result["passed"] for result in results), "cases_total": len(results), @@ -166,7 +186,9 @@ def run_provider_suite( "costed_successful_evaluated_analyses": len(known_costs), "estimated_total_cost_usd": _decimal_string(total_cost), "estimated_cost_per_successful_evaluated_analysis_usd": ( - _decimal_string(total_cost / len(successful)) if total_cost is not None and successful else None + _decimal_string(total_cost / len(successful)) + if total_cost is not None and successful + else None ), "cost_unavailable_reason": cost_unavailable_reason, "results": results, @@ -178,7 +200,9 @@ def _contains_all(values: list[Any], needles: list[str]) -> bool: return all(needle.lower() in text for needle in needles) -def _contains_any(text: str, needles: tuple[str, ...] | list[str], case_sensitive: bool = False) -> bool: +def _contains_any( + text: str, needles: tuple[str, ...] | list[str], case_sensitive: bool = False +) -> bool: if not case_sensitive: text = text.lower() needles = [needle.lower() for needle in needles] @@ -212,4 +236,90 @@ def _cost_decimal(value: Any) -> Decimal | None: def _decimal_string(value: Decimal | None) -> str | None: - return format(value.quantize(Decimal("0.00000001")), "f") if value is not None else None + return ( + format(value.quantize(Decimal("0.00000001")), "f") + if value is not None + else None + ) + + +def run_comparison_suite( + cases: list[dict[str, Any]] | None = None, + config: LLMConfig | None = None, +) -> dict[str, Any]: + """Compare the offline quality gate with one optional provider run. + + The report contains summaries only, avoiding fixture content, prompts, model + output, or endpoint information. + """ + selected_cases = cases if cases is not None else load_cases() + deterministic = run_suite(selected_cases) + provider = run_provider_suite(selected_cases, config) + quality_results_match = _quality_results_match( + deterministic["results"], provider["results"] + ) + provider_outcomes = provider["provider_outcomes"] + + return { + "schema_version": "week5-comparison-v1", + "deterministic": _suite_summary(deterministic), + "provider": { + "identity": provider["provider"], + "passed": provider["passed"], + "provider_outcomes": provider_outcomes, + "successful_evaluated_analyses": provider["successful_evaluated_analyses"], + "costed_successful_evaluated_analyses": provider[ + "costed_successful_evaluated_analyses" + ], + "estimated_total_cost_usd": provider["estimated_total_cost_usd"], + "estimated_cost_per_successful_evaluated_analysis_usd": provider[ + "estimated_cost_per_successful_evaluated_analysis_usd" + ], + "cost_unavailable_reason": provider["cost_unavailable_reason"], + }, + "comparison": { + "quality_results_match": quality_results_match, + "fallbacks_observed": provider_outcomes["failure"] + + provider_outcomes["not_configured"], + "status": _comparison_status( + deterministic, provider, quality_results_match + ), + }, + } + + +def _suite_summary(suite: dict[str, Any]) -> dict[str, int | bool]: + return { + "passed": suite["passed"], + "cases_passed": suite["cases_passed"], + "cases_total": suite["cases_total"], + "checks_passed": suite["checks_passed"], + "checks_total": suite["checks_total"], + } + + +def _quality_results_match( + deterministic: list[dict[str, Any]], provider: list[dict[str, Any]] +) -> bool: + deterministic_scores = [ + (result["id"], result["score"], result["rubric"]) for result in deterministic + ] + provider_scores = [ + (result["id"], result["score"], result["rubric"]) for result in provider + ] + return deterministic_scores == provider_scores + + +def _comparison_status( + deterministic: dict[str, Any], provider: dict[str, Any], quality_results_match: bool +) -> str: + if not deterministic["passed"] or not quality_results_match: + return "deterministic_quality_gate_failed" + outcomes = provider["provider_outcomes"] + if outcomes["not_configured"]: + return "provider_not_configured" + if outcomes["failure"]: + return "provider_request_failed" + if provider["cost_unavailable_reason"] is not None: + return "incomplete_cost_data" + return "ready_to_compare" diff --git a/apps/ai-sre-assistant/tests/test_evaluation.py b/apps/ai-sre-assistant/tests/test_evaluation.py index f247684..a4c8129 100644 --- a/apps/ai-sre-assistant/tests/test_evaluation.py +++ b/apps/ai-sre-assistant/tests/test_evaluation.py @@ -78,15 +78,23 @@ def successful_enrichment(**_kwargs): report = run_provider_suite(CASES[:2], config) assert report["passed"] is True - assert report["provider_outcomes"] == {"success": 2, "failure": 0, "not_configured": 0} + assert report["provider_outcomes"] == { + "success": 2, + "failure": 0, + "not_configured": 0, + } assert report["successful_evaluated_analyses"] == 2 assert report["costed_successful_evaluated_analyses"] == 2 assert report["estimated_total_cost_usd"] == "0.00030000" - assert report["estimated_cost_per_successful_evaluated_analysis_usd"] == "0.00015000" + assert ( + report["estimated_cost_per_successful_evaluated_analysis_usd"] == "0.00015000" + ) assert report["cost_unavailable_reason"] is None -def test_provider_report_keeps_cost_per_success_unknown_with_incomplete_usage(monkeypatch): +def test_provider_report_keeps_cost_per_success_unknown_with_incomplete_usage( + monkeypatch, +): from decimal import Decimal from app.llm import LLMCallResult, LLMConfig @@ -121,3 +129,73 @@ def enrichment_without_usage(**_kwargs): assert report["estimated_total_cost_usd"] is None assert report["estimated_cost_per_successful_evaluated_analysis_usd"] is None assert report["cost_unavailable_reason"] == "incomplete_cost_data" + + +def test_comparison_report_is_ready_when_provider_quality_and_cost_are_complete( + monkeypatch, +): + from decimal import Decimal + + from app.llm import LLMCallResult, LLMConfig + from evals.runner import run_comparison_suite + + def successful_enrichment(**_kwargs): + return LLMCallResult( + analysis="optional provider analysis", + notice=None, + telemetry={ + "provider": "openai", + "model": "eval-model", + "outcome": "success", + "usage": {"input_tokens": 100, "output_tokens": 50}, + }, + ) + + monkeypatch.setattr("evals.runner.analyze_with_llm", successful_enrichment) + config = LLMConfig( + provider="openai", + api_key="test-key", + base_url="https://example.invalid/v1", + model="eval-model", + input_usd_per_million_tokens=Decimal("1"), + output_usd_per_million_tokens=Decimal("1"), + ) + + report = run_comparison_suite(CASES[:2], config) + + assert report["schema_version"] == "week5-comparison-v1" + assert report["deterministic"]["passed"] is True + assert report["provider"]["provider_outcomes"] == { + "success": 2, + "failure": 0, + "not_configured": 0, + } + assert report["comparison"] == { + "quality_results_match": True, + "fallbacks_observed": 0, + "status": "ready_to_compare", + } + + +def test_comparison_report_exercises_unconfigured_provider_fallback_end_to_end(): + from app.llm import LLMConfig + from evals.runner import run_comparison_suite + + report = run_comparison_suite( + CASES[:1], + LLMConfig(provider="none", api_key="", base_url="", model=""), + ) + + assert report["deterministic"]["passed"] is True + assert report["provider"]["provider_outcomes"] == { + "success": 0, + "failure": 0, + "not_configured": 1, + } + assert report["provider"]["successful_evaluated_analyses"] == 0 + assert report["provider"]["estimated_total_cost_usd"] is None + assert report["comparison"] == { + "quality_results_match": True, + "fallbacks_observed": 1, + "status": "provider_not_configured", + } diff --git a/docs/09-roadmap.md b/docs/09-roadmap.md index e68afb6..12d48c8 100644 --- a/docs/09-roadmap.md +++ b/docs/09-roadmap.md @@ -43,7 +43,7 @@ This is the canonical technical execution order for Reliability Lab. Each week a - Day 2 - complete: add aggregate counters and latency distributions with bounded labels. - Day 3 - complete: accept explicit deployment-owned pricing inputs and return per-call estimated cost metadata only when prices and token directions are known. - Day 4 - complete: join provider outcomes and deployment-owned cost estimates with deterministic evaluation results, including cost per successful evaluated analysis when data is complete. -- Add a local comparison report across deterministic and configured provider paths. +- Day 5 - complete: add a bounded local comparison report across deterministic and configured provider paths, including the end-to-end fallback state. See [Provider Telemetry Contract](22-provider-telemetry.md) for the per-request contract, aggregate metrics, privacy boundary, and remaining-week sequence. diff --git a/docs/17-assistant-evaluation.md b/docs/17-assistant-evaluation.md index f6b1de4..2d9ee85 100644 --- a/docs/17-assistant-evaluation.md +++ b/docs/17-assistant-evaluation.md @@ -76,3 +76,15 @@ This makes one optional provider-enrichment call per fixture and emits a bounded A successful evaluated analysis requires a passing deterministic rubric and a successful provider response. The cost-per-success value is emitted only when every successful evaluated analysis has complete price and token data. Otherwise it is `null` with `cost_unavailable_reason`; unknown usage is never treated as zero cost. The report does not write a usage ledger or include fixture evidence, prompts, provider output, endpoints, or credentials. The normal evaluation command remains deterministic, offline, and cost-free. The provider report is deliberately opt-in and remains outside CI until a controlled provider test environment and explicit budget exist. + +## Local Provider Comparison + +Run the local comparison report with: + +```bash +python -m evals.run_evals --comparison-report +``` + +It runs the deterministic corpus and the selected optional-provider path against the same fixtures, then reports the two quality summaries, bounded provider/model identity, outcome counts, fallback count, and estimated cost summary. `comparison.status` is `ready_to_compare` only when deterministic quality passes, the provider succeeds for every case, and cost is complete. Other explicit states distinguish unconfigured providers, request failures, incomplete cost data, and deterministic-gate failures. + +The command prints summaries only: it excludes fixture evidence, prompts, provider output, credentials, and endpoints. It is opt-in and outside CI because a configured provider makes one call per fixture. With `LLM_PROVIDER=none`, it is an offline end-to-end check that verifies the deterministic fallback remains available. diff --git a/docs/22-provider-telemetry.md b/docs/22-provider-telemetry.md index ac8d2e7..f46e342 100644 --- a/docs/22-provider-telemetry.md +++ b/docs/22-provider-telemetry.md @@ -182,8 +182,16 @@ Do not attach user or workspace identifiers to Prometheus labels. Detailed audit - Unknown price or token information remains unknown and includes a reason rather than becoming zero. - Normal evaluation remains offline and cost-free; provider calls require an explicit command. -## Remaining Week 5 Sequence +## Day 5 Definition Of Done -1. Add a local comparison report across deterministic and configured provider paths. -2. Exercise provider failure and fallback behavior end to end. -3. Close the week with an exit-gate review against the technical roadmap. +- `python -m evals.run_evals --comparison-report` emits a bounded local comparison of the deterministic release gate and selected provider path. +- The report includes only quality summaries, provider/model identity, outcome counts, fallback count, and cost summary; it excludes fixture evidence, prompts, outputs, credentials, and endpoints. +- A configured provider remains opt-in and makes one call per fixture; the normal CI evaluation remains offline and cost-free. +- `LLM_PROVIDER=none` exercises the deterministic fallback end to end and reports `provider_not_configured` without attempting a network call. +- A provider is ready to compare only when the deterministic gate passes, every provider call succeeds, and cost data is complete. + +## Week 5 Exit Gate Review + +The Week 5 exit gate is met for the local learning path: provider identity, latency, usage, outcome, fallback, and deployment-owned cost estimates can be compared with deterministic evaluation outcomes without storing prompts, incident evidence, credentials, endpoints, or generated output. + +The comparison is intentionally a local point-in-time report, not a durable usage ledger, a provider-quality benchmark, or an approval to enable a provider in CI. Week 6 will expand and version the corpus; Week 8 will evaluate managed and private endpoints under a controlled benchmark plan. diff --git a/docs/build-log.md b/docs/build-log.md index 25b2667..2020ecb 100644 --- a/docs/build-log.md +++ b/docs/build-log.md @@ -783,3 +783,23 @@ What changed: - Kept the default deterministic evaluation command offline and cost-free for CI. Next: add a local comparison report, exercise fallback end to end, and close the Week 5 exit gate. + +## Week 5, Day 5 - Local Provider Comparison + +Today I closed the Week 5 measurement loop with a bounded local comparison report and an end-to-end fallback check. + +What changed: + +- Added `python -m evals.run_evals --comparison-report` to run the same corpus through the deterministic release gate and the selected optional-provider path. +- Reported only quality summaries, bounded provider/model identity, outcome counts, fallback count, and estimated cost summaries. +- Added an explicit comparison status so `ready_to_compare` requires deterministic quality to pass, every provider call to succeed, and cost data to be complete. +- Kept provider calls opt-in and out of CI; the default evaluation remains offline and cost-free. +- Exercised the `LLM_PROVIDER=none` path end to end to verify that provider-not-configured falls back safely without a network request. + +Why this matters: + +Provider comparisons are useful only when they preserve the deterministic quality gate and make unknowns visible. A local report makes the tradeoff inspectable without turning test fixtures or provider responses into a usage dataset. + +Week 5 exit-gate review: + +The local learning path can now compare provider usage, reliability, and deployment-owned cost with deterministic evaluation outcomes while preserving the telemetry privacy boundary. Durable metering, provider-quality scoring, and managed-versus-private benchmarks remain later, deliberately bounded work.