diff --git a/.gitignore b/.gitignore index 233248dd..1e9b3f83 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ *.lock *.log examples/*.log +examples/optimization/eval_optimize_loop/outputs/optimization_report.json +examples/optimization/eval_optimize_loop/outputs/optimization_report.md +examples/optimization/eval_optimize_loop/outputs/runs/ trpc-agent-py.egg-info diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 00000000..42a0e80a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,16 @@ +# 设计说明 + +## 失败归因 +评测先由 FakeJudge 产出结构化失败，再用规则归因：JSON、禁用格式归为 `format_violation`，精确答案错归为 `final_response_mismatch`，工具名、参数、知识召回、长度和 rubric 各有独立类别。每个失败都保留 reason 与 evidence，便于复核。若样例声明期望类别，报告会计算归因准确率。 + +## 接受门禁 +Gate 要求验证集提升达到阈值，并检查新硬失败、受保护样例退化、单例降分和累计成本。若训练分上涨但验证不涨，直接标记过拟合并拒绝。 + +## 防过拟合 +训练集只用于暴露问题，候选必须通过验证集和 protected case。过拟合候选即使修好训练格式，只要把验证集自然语言或受保护精确答案改坏，也会被拒绝。`delta_type` 标出 new_pass、new_fail、score_up、score_down，避免只看平均分掩盖局部退化。 + +## Fake 与 SDK +Fake mode 由 expectation、tags、protected 和 simulated_outputs 驱动，不依赖样例 id，可无 API key 稳定复现。SDK mode 通过 `SDKBackend` 调用 `AgentOptimizer` 与 `TargetPrompt`，失败时给出明确错误，不回退到 fake。 + +## 审计与回写 +报告同时写 JSON、Markdown 和 `runs//`，保存输入哈希、配置快照、候选 prompt、diff、case 结果与成本，并给出可复现命令。默认不回写源 prompt，只有显式 `--update-source` 才允许，报告会记录该选择。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..fa541c8a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,205 @@ +# Evaluation + Optimization Closed Loop + +This example implements issue #91 as a reproducible evaluation + optimization +loop. The default path is deterministic fake mode, so it runs in CI and on a +fresh checkout without `TRPC_AGENT_API_KEY` or any external model provider. A +real SDK adapter path is also present in `eval_loop/backends.py` for +`AgentOptimizer` / `TargetPrompt` integration. + +## Architecture + +```text +train.evalset.json + val.evalset.json + optimizer.json + baseline prompt + | + v +loader.py / config.py --> validated cases, gate config, input hashes + | + v +backends.py + |-- FakeBackend -> FakeModel + FakeJudge + FakeOptimizer + `-- SDKBackend -> AgentOptimizer + TargetPrompt + user call_agent + | + v +attribution.py -> evaluator.py -> gate.py -> report.py + | + v +optimization_report.json / optimization_report.md / runs// audit files +``` + +## Quick Start + +One-command fake mode: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --fake-model --fake-judge --trace +``` + +Equivalent new form: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --trace +``` + +Full fake command: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --train examples/optimization/eval_optimize_loop/data/train.evalset.json \ + --val examples/optimization/eval_optimize_loop/data/val.evalset.json \ + --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json \ + --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt \ + --output-dir /tmp/eval-optimize-loop \ + --mode fake \ + --trace +``` + +SDK adapter command shape: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --mode sdk \ + --train path/to/sdk_train.evalset.json \ + --val path/to/sdk_val.evalset.json \ + --optimizer-config path/to/sdk_optimizer.json \ + --prompt path/to/system_prompt.txt \ + --sdk-call-agent your_package.your_module:call_agent \ + --output-dir /tmp/eval-optimize-loop-sdk +``` + +Optional wrapper gate and multi-prompt form: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --mode sdk \ + --train path/to/sdk_train.evalset.json \ + --val path/to/sdk_val.evalset.json \ + --optimizer-config path/to/sdk_optimizer.json \ + --gate-config path/to/wrapper_gate.json \ + --target-prompt system_prompt=prompts/system.md \ + --target-prompt router_prompt=prompts/router.md \ + --sdk-call-agent your_package.your_module:call_agent \ + --run-id local-sdk-smoke \ + --output-dir /tmp/eval-optimize-loop-sdk +``` + +`--sdk-call-agent` must point to an async callable compatible with +`AgentOptimizer.optimize(call_agent=...)`. Configure any real model credentials +needed by that callable. SDK mode never silently falls back to fake mode. The +generated reproducibility command records the actual `--sdk-call-agent` +`module:function` target and file/config paths, but it does not record API keys +or other provider secrets. + +SDK optimizer config and wrapper gate config are intentionally separate. +`--optimizer-config` is passed unchanged to `AgentOptimizer.optimize`, so it must +follow the SDK `OptimizeConfigFile` schema. Put wrapper-only gate settings in +`--gate-config` (for example `{"gate": {"min_val_score_improvement": 0.05, +"max_total_cost": 1.0}}`). If `--gate-config` is omitted, the wrapper uses the +same default aggregate gate values as the fake example. + +`--target-prompt name=path` may be repeated. If omitted, SDK mode keeps the old +single-field behavior and registers `system_prompt=--prompt`. A run can optimize +only `router_prompt`, only `skill_prompt`, or any set of named fields as long as +`OptimizeResult.best_prompts` returns every registered field. + +Fake mode is the complete per-case closed loop. SDK mode is the real +`AgentOptimizer` / `TargetPrompt` path with an aggregate wrapper gate. When the +SDK optimizer returns a best prompt, this wrapper maps `OptimizeResult` +aggregate fields into the JSON/Markdown report: baseline/best pass rate, +pass-rate improvement, metric breakdowns, token usage, duration, LLM cost, +all `best_prompts`, and round summaries. SDK mode applies +`gate_status: partial_applied`: it checks `status == SUCCEEDED`, validation +improvement against `gate.min_val_score_improvement`, and total LLM cost against +`gate.max_total_cost`. Protected-case regression, new-hard-failure, per-case +delta, and per-case score-drop checks are not claimed in SDK mode unless the SDK +exposes full per-case validation scores; they are listed in +`not_applied_checks`. + +Fake mode uses a deterministic run id (`eval_optimize_loop_seed_`) so the +example outputs are byte-stable. SDK mode is append-only by default: the wrapper +derives a compact UTC `run.run_id` from the SDK result `started_at` when +available, otherwise from the current UTC timestamp. Pass `--run-id` only when +a fixed audit path is useful for tests or local smoke runs; only explicit +`--run-id` values are included in the reproducibility command. + +## Source Prompt Writes + +The default is **no source write-back**. The baseline prompt file is not modified +by fake mode, and `SDKBackend` calls `AgentOptimizer.optimize(update_source=False)` +unless `--update-source` is explicitly passed. The report records +`run.update_source` and the Markdown report states whether source write-back was +enabled. + +## Candidate Behavior + +The fake optimizer proposes exactly two candidates: + +- `candidate_001_overfit`: fixes train formatting but forces JSON too broadly; + it improves train score and regresses validation, so the gate rejects it. +- `candidate_002_safe`: applies strict JSON/exact-answer behavior only when + requested; it improves validation without protected-case regression, so the + gate may accept it. + +The fake model is driven by `EvalCase.expectation`, `tags`, `protected`, and +optional `simulated_outputs`; it does not depend on sample `case_id` names. + +## Reports + +`optimization_report.json` includes: + +- `schema_version`; +- `run` metadata: mode, fake flags, trace flag, case counts, update-source flag, + and input paths; +- `baseline` plus compatibility fields `baseline_train` and + `baseline_validation`; +- all candidate train/validation results, rationale, and prompt diff; +- per-case deltas with `delta_type` (`new_pass`, `new_fail`, `score_up`, + `score_down`, `unchanged`); +- failure attribution summary and attribution accuracy when expected labels are + present; +- gate decisions with overfit detection, protected regressions, new hard + failures, excessive drops, cost fields, and SDK `not_applied_checks` when + per-case validation details are not exposed; +- audit data: seed, duration, config hash, input hashes, candidate prompt hashes, + cost, prompt diffs, and reproducibility command. + +`gate.max_total_cost` is interpreted as the total evaluated run cost at the time +each candidate is judged: baseline cost plus all candidates evaluated so far, +including rejected candidates. This makes budget decisions deterministic and +auditable when multiple candidates are considered. + +`optimization_report.md` includes final decision, gate reasons, score table, +per-case delta table, failure attribution summary, cost/audit details, prompt +diffs, and the reproducibility command. + +`report.py` also writes audit artifacts under `output_dir/runs//`: + +- `config.snapshot.json`; +- `input_hashes.json` with train, validation, optimizer, prompt, + `target_prompts.`, and optional `gate_config` hashes; +- fake mode: `candidate_prompts//system_prompt.txt`; +- SDK mode: `candidate_prompts//.txt` for every + returned `best_prompts` field, plus `bundle.txt` with the combined prompt + shown in the wrapper report; +- `case_results/_.json`; +- `prompt_diffs/.diff`. + +The repository keeps only stable examples: + +- `outputs/optimization_report.example.json` +- `outputs/optimization_report.example.md` + +Runtime `optimization_report.json`, `optimization_report.md`, and `runs/` +directories are not committed. + +## Run Tests + +```bash +python -m pytest examples/optimization/eval_optimize_loop/tests +``` + +The tests cover fake hidden-sample generalization, config validation, gate +rejection paths, protected-case behavior, failure attribution, tool/knowledge +judge paths, SDK adapter wiring through monkeypatching, deterministic report +generation, and both CLI forms. CI can run fake mode plus the monkeypatched SDK +smoke tests without real API credentials; real SDK/model calls are opt-in local +or integration runs. diff --git a/examples/optimization/eval_optimize_loop/data/optimizer.json b/examples/optimization/eval_optimize_loop/data/optimizer.json new file mode 100644 index 00000000..dcd4c57a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/optimizer.json @@ -0,0 +1,20 @@ +{ + "seed": 91, + "optimizer": { + "name": "fake_two_candidate_optimizer", + "description": "Deterministically proposes one overfit candidate and one safe candidate." + }, + "metrics": { + "case_score": "mean", + "failure_attribution": "rule_based" + }, + "gate": { + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": false, + "protected_case_ids": [ + "val_protected_yes_no" + ], + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0 + } +} diff --git a/examples/optimization/eval_optimize_loop/data/train.evalset.json b/examples/optimization/eval_optimize_loop/data/train.evalset.json new file mode 100644 index 00000000..70580991 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/train.evalset.json @@ -0,0 +1,59 @@ +{ + "evalset_id": "eval_optimize_loop_train_v1", + "split": "train", + "description": "Training cases used for failure attribution and fake prompt optimization.", + "cases": [ + { + "id": "train_json_refund", + "input": "Return strict JSON for a refund escalation. Use keys intent and priority.", + "expectation": { + "type": "json", + "required_keys": [ + "intent", + "priority" + ], + "expected_values": { + "intent": "refund", + "priority": "high" + }, + "expected_failure_category": "format_violation" + }, + "tags": [ + "json", + "format" + ] + }, + { + "id": "train_exact_order_status", + "input": "Answer exactly READY when the order can ship.", + "expectation": { + "type": "exact", + "expected": "READY", + "expected_failure_category": "final_response_mismatch" + }, + "tags": [ + "exact", + "format" + ] + }, + { + "id": "train_rubric_retry_summary", + "input": "In 80 chars or less, mention latency and retries.", + "expectation": { + "type": "rubric", + "must_include": [ + "latency", + "retries" + ], + "forbidden": [ + "database" + ], + "max_chars": 80 + }, + "tags": [ + "rubric", + "no_effect" + ] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/data/val.evalset.json b/examples/optimization/eval_optimize_loop/data/val.evalset.json new file mode 100644 index 00000000..ceeaf11d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/val.evalset.json @@ -0,0 +1,63 @@ +{ + "evalset_id": "eval_optimize_loop_validation_v1", + "split": "validation", + "description": "Validation cases that separate safe optimization from train-set overfitting.", + "cases": [ + { + "id": "val_json_invoice", + "input": "Return strict JSON for an invoice approval. Use keys status and next_step.", + "expectation": { + "type": "json", + "required_keys": [ + "status", + "next_step" + ], + "expected_values": { + "next_step": "email_customer", + "status": "approved" + }, + "expected_failure_category": "format_violation" + }, + "tags": [ + "json", + "safe_improves" + ] + }, + { + "id": "val_explain_cache", + "input": "Explain cache invalidation for a junior engineer in natural language.", + "expectation": { + "type": "rubric", + "must_include": [ + "cache", + "stale data" + ], + "forbidden": [ + "{", + "}", + "json" + ], + "max_chars": 180, + "expected_failure_category": "format_violation" + }, + "tags": [ + "rubric", + "overfit_guard" + ] + }, + { + "id": "val_protected_yes_no", + "input": "Answer exactly YES if idempotent retries are safe for duplicate requests.", + "expectation": { + "type": "exact", + "expected": "YES", + "expected_failure_category": "final_response_mismatch" + }, + "protected": true, + "tags": [ + "protected", + "exact" + ] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/eval_loop/__init__.py b/examples/optimization/eval_optimize_loop/eval_loop/__init__.py new file mode 100644 index 00000000..74835adb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/__init__.py @@ -0,0 +1,24 @@ +"""Deterministic evaluation + optimization loop example. + +This package is intentionally example-local. It mirrors the shape of an +Evaluator/Optimizer workflow while keeping fake model and fake judge execution +offline, deterministic, and easy to inspect. +""" + +from .schemas import CandidatePrompt +from .schemas import CaseDelta +from .schemas import CaseResult +from .schemas import EvalCase +from .schemas import EvalResult +from .schemas import GateDecision +from .schemas import OptimizationReport + +__all__ = [ + "CandidatePrompt", + "CaseDelta", + "CaseResult", + "EvalCase", + "EvalResult", + "GateDecision", + "OptimizationReport", +] diff --git a/examples/optimization/eval_optimize_loop/eval_loop/attribution.py b/examples/optimization/eval_optimize_loop/eval_loop/attribution.py new file mode 100644 index 00000000..53818346 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/attribution.py @@ -0,0 +1,82 @@ +"""Rule-based failure attribution for the example evaluator.""" + +from __future__ import annotations + +from collections import Counter +from typing import Iterable + +from .schemas import EvalResult + + +_ERROR_TO_ATTRIBUTION = { + "json_parse_failure": ("format_violation", "output is not valid JSON"), + "required_key_missing": ("final_response_mismatch", "required JSON key is missing"), + "json_value_mismatch": ("final_response_mismatch", "JSON value does not match expected value"), + "exact_answer_mismatch": ("final_response_mismatch", "normalized exact answer mismatch"), + "forbidden_pattern": ("format_violation", "output contains a forbidden pattern"), + "missing_rubric_terms": ("llm_rubric_not_met", "required rubric terms are missing"), + "max_chars_exceeded": ("length_violation", "output exceeds max_chars"), + "tool_call_error": ("tool_call_error", "tool call did not match expected tool"), + "parameter_error": ("parameter_error", "tool call parameters did not match"), + "knowledge_recall_insufficient": ( + "knowledge_recall_insufficient", + "required knowledge evidence was not recalled", + ), +} + + +def attribute_failure(error_code: str, evidence: str) -> tuple[str, str, str]: + """Return failure_category, failure_reason, evidence for a judge error.""" + + category, reason = _ERROR_TO_ATTRIBUTION.get( + error_code, + ("unknown_failure", f"unmapped judge error: {error_code}"), + ) + return category, reason, evidence + + +def summarize_failures(results: Iterable[EvalResult]) -> dict[str, object]: + """Summarize failures by category and by prompt/split for reporting.""" + + by_category: Counter[str] = Counter() + by_prompt: dict[str, dict[str, int]] = {} + examples: list[dict[str, str]] = [] + total_failed = 0 + expected_total = 0 + expected_correct = 0 + + for result in results: + prompt_key = f"{result.prompt_id}:{result.split}" + by_prompt.setdefault(prompt_key, {}) + for case in result.cases: + if case.passed: + continue + total_failed += 1 + category = case.failure_category or "unknown_failure" + by_category[category] += 1 + by_prompt[prompt_key][category] = by_prompt[prompt_key].get(category, 0) + 1 + if case.expected_failure_category: + expected_total += 1 + if case.expected_failure_category == category: + expected_correct += 1 + examples.append({ + "prompt_id": result.prompt_id, + "split": result.split, + "case_id": case.case_id, + "failure_category": category, + "failure_reason": case.failure_reason or "", + "evidence": case.evidence or "", + }) + + return { + "total_failed_cases": total_failed, + "by_category": dict(sorted(by_category.items())), + "by_prompt_split": {key: dict(sorted(value.items())) for key, value in sorted(by_prompt.items())}, + "examples": examples, + "attribution_accuracy": ( + round(expected_correct / expected_total, 6) + if expected_total + else None + ), + "expected_labeled_failures": expected_total, + } diff --git a/examples/optimization/eval_optimize_loop/eval_loop/backends.py b/examples/optimization/eval_optimize_loop/eval_loop/backends.py new file mode 100644 index 00000000..38709137 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/backends.py @@ -0,0 +1,280 @@ +"""Backend adapters for fake and SDK optimization paths.""" + +from __future__ import annotations + +import asyncio +import importlib +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from typing import Iterable + +from .diffing import make_unified_diff +from .evaluator import ExampleEvaluator +from .fake_judge import FakeJudge +from .fake_model import FakeModel +from .optimizer import FakeOptimizer +from .schemas import CandidatePrompt +from .schemas import EvalCase +from .schemas import EvalResult + + +@dataclass +class FakeBackend: + seed: int = 91 + trace_enabled: bool = False + + def __post_init__(self) -> None: + self._evaluator = ExampleEvaluator(FakeModel(seed=self.seed), FakeJudge(), trace_enabled=self.trace_enabled) + self._optimizer = FakeOptimizer() + + def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], split: str) -> EvalResult: + return self._evaluator.evaluate(prompt_id=prompt_id, prompt=prompt, cases=cases, split=split) + + def optimize( + self, + *, + baseline_prompt: str, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + output_dir: str | Path, + ) -> list[CandidatePrompt]: + return self._optimizer.propose(baseline_prompt) + + +@dataclass +class SDKBackend: + """Thin optimizer adapter around AgentOptimizer/TargetPrompt for SDK runs. + + SDK mode relies on AgentOptimizer's internal evaluation loop. It does not + implement the fake per-case ``evaluate`` API. + """ + + prompt_path: str | Path + call_agent_path: str | None = None + update_source: bool = False + target_prompt_paths: dict[str, str | Path] | None = None + last_result: Any | None = None + last_result_summary: dict[str, Any] | None = None + last_artifact_dir: str | None = None + + def optimize( + self, + *, + baseline_prompt: str, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + output_dir: str | Path, + ) -> list[CandidatePrompt]: + if _has_running_loop(): + raise ValueError( + "SDKBackend.optimize() cannot be called while an event loop is already running; " + "use await SDKBackend.optimize_async(...) instead." + ) + return asyncio.run( + self.optimize_async( + baseline_prompt=baseline_prompt, + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + output_dir=output_dir, + ) + ) + + async def optimize_async( + self, + *, + baseline_prompt: str, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + output_dir: str | Path, + ) -> list[CandidatePrompt]: + if not self.call_agent_path: + raise ValueError( + "sdk mode requires --sdk-call-agent module:function. The callable must be async and compatible " + "with AgentOptimizer.optimize(call_agent=...). Also configure real model credentials required " + "by that callable, such as TRPC_AGENT_API_KEY/TRPC_AGENT_BASE_URL/TRPC_AGENT_MODEL_NAME." + ) + call_agent = _load_call_agent(self.call_agent_path) + try: + from trpc_agent_sdk.evaluation import AgentOptimizer + from trpc_agent_sdk.evaluation import TargetPrompt + except Exception as exc: # pragma: no cover - depends on optional SDK import health + raise ValueError(f"sdk mode could not import AgentOptimizer/TargetPrompt: {exc}") from exc + + target_prompt_paths = self._target_prompt_paths() + target_prompt = TargetPrompt() + for name, path in target_prompt_paths.items(): + target_prompt.add_path(name, str(path)) + result = await AgentOptimizer.optimize( + config_path=str(optimizer_config_path), + call_agent=call_agent, + target_prompt=target_prompt, + train_dataset_path=str(train_path), + validation_dataset_path=str(val_path), + output_dir=str(output_dir), + update_source=self.update_source, + verbose=0, + ) + best_prompts = dict(getattr(result, "best_prompts", {}) or {}) + if not best_prompts: + raise ValueError("sdk mode completed but OptimizeResult.best_prompts was empty") + missing_fields = [name for name in target_prompt_paths if name not in best_prompts] + if missing_fields: + missing = ", ".join(sorted(missing_fields)) + raise ValueError( + "sdk mode completed but OptimizeResult.best_prompts is missing registered target fields: " + f"{missing}" + ) + empty_fields = [ + name + for name in target_prompt_paths + if not isinstance(best_prompts[name], str) or not best_prompts[name].strip() + ] + if empty_fields: + empty = ", ".join(sorted(empty_fields)) + raise ValueError( + "sdk mode completed but OptimizeResult.best_prompts contained empty registered target fields: " + f"{empty}" + ) + self.last_result = result + self.last_result_summary = _summarize_sdk_result(result) + self.last_artifact_dir = str(output_dir) + baseline_prompts = _read_prompt_bundle(target_prompt_paths) + return [ + CandidatePrompt( + candidate_id="sdk_best", + prompt=_render_prompt_bundle(best_prompts), + rationale="Best prompt returned by AgentOptimizer.optimize.", + prompt_diff=_render_prompt_bundle_diff(baseline_prompts, best_prompts), + ) + ] + + def _target_prompt_paths(self) -> dict[str, str | Path]: + if self.target_prompt_paths: + return dict(self.target_prompt_paths) + return {"system_prompt": self.prompt_path} + + +def _load_call_agent(path: str): + if ":" not in path: + raise ValueError("--sdk-call-agent must use module:function format") + module_name, function_name = path.split(":", 1) + try: + module = importlib.import_module(module_name) + except Exception as exc: + raise ValueError(f"--sdk-call-agent target {path!r} could not import module {module_name!r}: {exc}") from exc + call_agent = getattr(module, function_name, None) + if call_agent is None: + raise ValueError(f"--sdk-call-agent target {path!r} was not found") + if not callable(call_agent): + raise ValueError(f"--sdk-call-agent target {path!r} was found but is not callable") + return call_agent + + +def _has_running_loop() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + +def _summarize_sdk_result(result: Any) -> dict[str, Any]: + return { + "status": _safe_jsonable(getattr(result, "status", None)), + "baseline_pass_rate": _safe_result_field( + "baseline_pass_rate", getattr(result, "baseline_pass_rate", None) + ), + "best_pass_rate": _safe_result_field("best_pass_rate", getattr(result, "best_pass_rate", None)), + "pass_rate_improvement": _safe_result_field( + "pass_rate_improvement", getattr(result, "pass_rate_improvement", None) + ), + "baseline_metric_breakdown": _safe_jsonable(getattr(result, "baseline_metric_breakdown", {})), + "best_metric_breakdown": _safe_jsonable(getattr(result, "best_metric_breakdown", {})), + "metric_thresholds": _safe_jsonable(getattr(result, "metric_thresholds", {})), + "total_llm_cost": _safe_result_field("total_llm_cost", getattr(result, "total_llm_cost", 0.0)), + "total_token_usage": _safe_jsonable(getattr(result, "total_token_usage", {})), + "duration_seconds": _safe_jsonable(getattr(result, "duration_seconds", 0.0)), + "started_at": _safe_jsonable(getattr(result, "started_at", None)), + "total_rounds": _safe_jsonable(getattr(result, "total_rounds", 0)), + "baseline_prompts": _safe_jsonable(getattr(result, "baseline_prompts", {})), + "best_prompts": _safe_jsonable(getattr(result, "best_prompts", {})), + "rounds": [ + { + "validation_pass_rate": _safe_jsonable(getattr(round_record, "validation_pass_rate", None)), + "accepted": _safe_jsonable(getattr(round_record, "accepted", None)), + "failed_case_ids": _safe_jsonable(getattr(round_record, "failed_case_ids", [])), + "round_llm_cost": _safe_jsonable(getattr(round_record, "round_llm_cost", 0.0)), + "budget_used": _safe_jsonable(getattr(round_record, "budget_used", None)), + "budget_total": _safe_jsonable(getattr(round_record, "budget_total", None)), + } + for round_record in getattr(result, "rounds", []) or [] + ], + } + + +def _safe_result_field(field_name: str, value: Any) -> Any: + try: + return _safe_jsonable(value) + except ValueError as exc: + raise ValueError(f"SDK OptimizeResult field {field_name} must be a finite number") from exc + + +def _safe_jsonable(value: Any) -> Any: + if hasattr(value, "model_dump"): + return _safe_jsonable(value.model_dump(mode="json")) + if hasattr(value, "__dict__") and not isinstance(value, type): + return _safe_jsonable(dict(value.__dict__)) + if isinstance(value, dict): + return {str(key): _safe_jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_safe_jsonable(item) for item in value] + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError("value must be a finite number") + return value + if isinstance(value, (str, int, bool)) or value is None: + return value + return repr(value) + + +def _read_prompt_bundle(paths: dict[str, str | Path]) -> dict[str, str]: + return { + name: Path(path).read_text(encoding="utf-8") + for name, path in paths.items() + } + + +def _render_prompt_bundle(prompts: dict[str, str]) -> str: + if set(prompts) == {"system_prompt"}: + return prompts["system_prompt"] + sections = [] + for name in sorted(prompts): + sections.append(f"## {name}\n\n{prompts[name]}") + return "\n\n".join(sections) + + +def _render_prompt_bundle_diff(baseline_prompts: dict[str, str], best_prompts: dict[str, str]) -> str: + if set(baseline_prompts) == {"system_prompt"} and set(best_prompts) == {"system_prompt"}: + return make_unified_diff( + baseline_prompts.get("system_prompt", ""), + best_prompts.get("system_prompt", ""), + before_name="baseline_system_prompt.txt", + after_name="sdk_best/system_prompt.txt", + ) + diffs = [] + for name in sorted(set(baseline_prompts) | set(best_prompts)): + diffs.append( + make_unified_diff( + baseline_prompts.get(name, ""), + best_prompts.get(name, ""), + before_name=f"baseline/{name}.txt", + after_name=f"sdk_best/{name}.txt", + ) + ) + return "\n\n".join(diffs) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/config.py b/examples/optimization/eval_optimize_loop/eval_loop/config.py new file mode 100644 index 00000000..2bf8fbea --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/config.py @@ -0,0 +1,166 @@ +"""Configuration validation for the eval/optimize loop example.""" + +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import field +from pathlib import Path +from typing import Any + +from .schemas import EvalCase + + +@dataclass(frozen=True) +class GateConfig: + min_val_score_improvement: float = 0.01 + allow_new_hard_fail: bool = False + protected_case_ids: list[str] = field(default_factory=list) + max_score_drop_per_case: float = 0.0 + max_total_cost: float = 1.0 + extras: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + data = { + "min_val_score_improvement": self.min_val_score_improvement, + "allow_new_hard_fail": self.allow_new_hard_fail, + "protected_case_ids": list(self.protected_case_ids), + "max_score_drop_per_case": self.max_score_drop_per_case, + "max_total_cost": self.max_total_cost, + } + data.update(self.extras) + return data + + +@dataclass(frozen=True) +class OptimizerConfig: + seed: int = 91 + optimizer: dict[str, Any] = field(default_factory=dict) + metrics: dict[str, Any] = field(default_factory=dict) + gate: GateConfig = field(default_factory=GateConfig) + extras: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + data = { + "seed": self.seed, + "optimizer": dict(self.optimizer), + "metrics": dict(self.metrics), + "gate": self.gate.to_dict(), + } + data.update(self.extras) + return data + + +def parse_optimizer_config(payload: dict[str, Any], *, path: str | Path) -> OptimizerConfig: + path_text = str(path) + allowed = {"seed", "optimizer", "metrics", "gate"} + extras = {key: value for key, value in payload.items() if key not in allowed} + + seed = payload.get("seed", 91) + if not isinstance(seed, int): + raise ValueError(f"{path_text}: field 'seed' must be an integer") + + optimizer = payload.get("optimizer", {}) + if not isinstance(optimizer, dict): + raise ValueError(f"{path_text}: field 'optimizer' must be an object") + + metrics = payload.get("metrics", {}) + if not isinstance(metrics, dict): + raise ValueError(f"{path_text}: field 'metrics' must be an object") + + gate_payload = payload.get("gate", {}) + if not isinstance(gate_payload, dict): + raise ValueError(f"{path_text}: field 'gate' must be an object") + + gate = _parse_gate_config(gate_payload, path=path_text) + return OptimizerConfig( + seed=seed, + optimizer=dict(optimizer), + metrics=dict(metrics), + gate=gate, + extras=extras, + ) + + +def validate_inputs( + *, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + train_cases: list[EvalCase], + validation_cases: list[EvalCase], + config: OptimizerConfig, +) -> None: + train_resolved = Path(train_path).resolve() + val_resolved = Path(val_path).resolve() + if train_resolved == val_resolved: + raise ValueError(f"{train_path}: train and validation evalset paths must be different") + + _validate_cases(train_cases, split="train", path=train_path) + _validate_cases(validation_cases, split="validation", path=val_path) + if len(train_cases) < 3: + raise ValueError(f"{train_path}: train evalset must contain at least 3 cases") + if len(validation_cases) < 3: + raise ValueError(f"{val_path}: validation evalset must contain at least 3 cases") + + validation_ids = {case.case_id for case in validation_cases} + missing_protected = [ + case_id + for case_id in config.gate.protected_case_ids + if case_id not in validation_ids + ] + if missing_protected: + raise ValueError( + f"{optimizer_config_path}: field 'gate.protected_case_ids' references missing validation cases: " + f"{missing_protected}" + ) + + +def _parse_gate_config(payload: dict[str, Any], *, path: str) -> GateConfig: + allowed = { + "min_val_score_improvement", + "allow_new_hard_fail", + "protected_case_ids", + "max_score_drop_per_case", + "max_total_cost", + } + extras = {key: value for key, value in payload.items() if key not in allowed} + + min_val = payload.get("min_val_score_improvement", 0.01) + if not isinstance(min_val, (int, float)) or min_val < 0 or min_val > 1: + raise ValueError(f"{path}: field 'gate.min_val_score_improvement' must be a number between 0 and 1") + + allow_new_hard_fail = payload.get("allow_new_hard_fail", False) + if not isinstance(allow_new_hard_fail, bool): + raise ValueError(f"{path}: field 'gate.allow_new_hard_fail' must be a boolean") + + protected_case_ids = payload.get("protected_case_ids", []) + if not isinstance(protected_case_ids, list) or not all(isinstance(item, str) for item in protected_case_ids): + raise ValueError(f"{path}: field 'gate.protected_case_ids' must be a list of strings") + + max_drop = payload.get("max_score_drop_per_case", 0.0) + if not isinstance(max_drop, (int, float)) or max_drop < 0: + raise ValueError(f"{path}: field 'gate.max_score_drop_per_case' must be a non-negative number") + + max_total_cost = payload.get("max_total_cost", 1.0) + if not isinstance(max_total_cost, (int, float)) or max_total_cost < 0: + raise ValueError(f"{path}: field 'gate.max_total_cost' must be a non-negative number") + + return GateConfig( + min_val_score_improvement=float(min_val), + allow_new_hard_fail=allow_new_hard_fail, + protected_case_ids=list(protected_case_ids), + max_score_drop_per_case=float(max_drop), + max_total_cost=float(max_total_cost), + extras=extras, + ) + + +def _validate_cases(cases: list[EvalCase], *, split: str, path: str | Path) -> None: + seen: set[str] = set() + for case in cases: + if case.case_id in seen: + raise ValueError(f"{path}: duplicate case_id {case.case_id!r} in {split} evalset") + seen.add(case.case_id) + expectation_type = case.expectation.get("type") + if not isinstance(expectation_type, str): + raise ValueError(f"{path}: case {case.case_id!r} field 'expectation.type' must be a string") diff --git a/examples/optimization/eval_optimize_loop/eval_loop/diffing.py b/examples/optimization/eval_optimize_loop/eval_loop/diffing.py new file mode 100644 index 00000000..9d578450 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/diffing.py @@ -0,0 +1,19 @@ +"""Prompt diff helpers.""" + +from __future__ import annotations + +import difflib + + +def make_unified_diff(before: str, after: str, *, before_name: str, after_name: str) -> str: + """Return a stable unified diff for prompt text.""" + + diff = difflib.unified_diff( + before.splitlines(), + after.splitlines(), + fromfile=before_name, + tofile=after_name, + lineterm="", + ) + rendered = "\n".join(line.rstrip() for line in diff) + return rendered or f"--- {before_name}\n+++ {after_name}\n# no prompt changes" diff --git a/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py new file mode 100644 index 00000000..e8a271ec --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/evaluator.py @@ -0,0 +1,75 @@ +"""Example-local evaluator adapter. + +This adapter intentionally keeps the surface small: it drives a model callable, +uses the fake judge for deterministic scoring, and returns dataclass results. +It can later be replaced with SDK AgentEvaluator integration. +""" + +from __future__ import annotations + +from typing import Iterable + +from .attribution import attribute_failure +from .fake_judge import FakeJudge +from .fake_model import FakeModel +from .schemas import CaseResult +from .schemas import EvalCase +from .schemas import EvalResult +from .trace import make_trace + + +class ExampleEvaluator: + """Evaluate prompt text against cases with fake model/judge components.""" + + def __init__(self, model: FakeModel, judge: FakeJudge, *, trace_enabled: bool = False) -> None: + self.model = model + self.judge = judge + self.trace_enabled = trace_enabled + + def evaluate(self, *, prompt_id: str, prompt: str, cases: Iterable[EvalCase], split: str) -> EvalResult: + case_results: list[CaseResult] = [] + for case in cases: + output, model_trace, cost = self.model.generate(prompt_id, prompt, case) + judged = self.judge.score(case, output) + failure_category = None + failure_reason = None + evidence = None + if not judged.passed: + failure_category, failure_reason, evidence = attribute_failure( + judged.error_code or "unknown_failure", + judged.evidence or "", + ) + trace = make_trace( + self.trace_enabled, + prompt_id=prompt_id, + case_id=case.case_id, + model_trace=model_trace, + judge_trace=judged.trace or {}, + ) + case_results.append( + CaseResult( + case_id=case.case_id, + split=case.split, + score=round(judged.score, 6), + passed=judged.passed, + output=output, + trace=trace, + failure_category=failure_category, + failure_reason=failure_reason, + evidence=evidence, + cost=cost, + hard_failed=(not judged.passed and judged.score <= 0.0), + expected_failure_category=case.expected_failure_category, + ) + ) + + score = round(sum(case.score for case in case_results) / len(case_results), 6) if case_results else 0.0 + total_cost = round(sum(case.cost for case in case_results), 6) + return EvalResult( + prompt_id=prompt_id, + split=split, + score=score, + passed=all(case.passed for case in case_results), + cost=total_cost, + cases=case_results, + ) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py b/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py new file mode 100644 index 00000000..678023de --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/fake_judge.py @@ -0,0 +1,189 @@ +"""Deterministic fake judge for offline scoring.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +from .schemas import EvalCase + + +@dataclass(frozen=True) +class JudgeOutcome: + score: float + passed: bool + error_code: str | None = None + evidence: str | None = None + trace: dict[str, Any] | None = None + + +class FakeJudge: + """Scores JSON, exact-answer, rubric, tool, and knowledge cases offline.""" + + def score(self, case: EvalCase, output: str) -> JudgeOutcome: + expectation_type = case.expectation.get("type") + if expectation_type == "json": + return self._score_json(case, output) + if expectation_type == "exact": + return self._score_exact(case, output) + if expectation_type == "rubric": + return self._score_rubric(case, output) + if expectation_type == "tool": + return self._score_tool(case, output) + if expectation_type == "knowledge": + return self._score_knowledge(case, output) + return JudgeOutcome( + score=0.0, + passed=False, + error_code="unknown_expectation", + evidence=f"unsupported expectation type: {expectation_type!r}", + trace={"expectation_type": expectation_type}, + ) + + def _score_json(self, case: EvalCase, output: str) -> JudgeOutcome: + try: + parsed = json.loads(output) + except json.JSONDecodeError as exc: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="json_parse_failure", + evidence=f"json parser failed at char {exc.pos}: {exc.msg}", + trace={"expectation_type": "json", "valid_json": False}, + ) + if not isinstance(parsed, dict): + return JudgeOutcome( + score=0.0, + passed=False, + error_code="json_value_mismatch", + evidence=f"expected JSON object, got {type(parsed).__name__}", + trace={"expectation_type": "json", "valid_json": True, "object": False}, + ) + required_keys = list(case.expectation.get("required_keys") or []) + for key in required_keys: + if key not in parsed: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="required_key_missing", + evidence=f"missing key {key!r}; got keys {sorted(parsed.keys())!r}", + trace={"expectation_type": "json", "valid_json": True, "missing_key": key}, + ) + expected_values = dict(case.expectation.get("expected_values") or {}) + for key, expected_value in expected_values.items(): + actual_value = parsed.get(key) + if actual_value != expected_value: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="json_value_mismatch", + evidence=f"{key!r}: expected {expected_value!r}, got {actual_value!r}", + trace={"expectation_type": "json", "valid_json": True, "mismatch_key": key}, + ) + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "json", "valid_json": True}) + + def _score_exact(self, case: EvalCase, output: str) -> JudgeOutcome: + expected = str(case.expectation.get("expected", "")) + if _normalize_exact(output) != _normalize_exact(expected): + return JudgeOutcome( + score=0.0, + passed=False, + error_code="exact_answer_mismatch", + evidence=f"expected normalized {expected!r}, got {output!r}", + trace={"expectation_type": "exact", "expected": expected}, + ) + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "exact", "expected": expected}) + + def _score_rubric(self, case: EvalCase, output: str) -> JudgeOutcome: + lowered = output.lower() + forbidden = [str(item) for item in case.expectation.get("forbidden") or []] + for pattern in forbidden: + if pattern.lower() in lowered: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="forbidden_pattern", + evidence=f"forbidden pattern {pattern!r} was present", + trace={"expectation_type": "rubric", "forbidden_pattern": pattern}, + ) + + must_include = [str(item) for item in case.expectation.get("must_include") or []] + missing = [term for term in must_include if term.lower() not in lowered] + if missing: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="missing_rubric_terms", + evidence=f"missing terms: {missing!r}", + trace={"expectation_type": "rubric", "missing_terms": missing}, + ) + + max_chars = case.expectation.get("max_chars") + if max_chars is not None and len(output) > int(max_chars): + return JudgeOutcome( + score=0.0, + passed=False, + error_code="max_chars_exceeded", + evidence=f"length {len(output)} exceeded max_chars {max_chars}", + trace={"expectation_type": "rubric", "length": len(output), "max_chars": int(max_chars)}, + ) + + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "rubric"}) + + def _score_tool(self, case: EvalCase, output: str) -> JudgeOutcome: + try: + parsed = json.loads(output) + except json.JSONDecodeError as exc: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="tool_call_error", + evidence=f"tool output was not JSON at char {exc.pos}: {exc.msg}", + trace={"expectation_type": "tool", "valid_json": False}, + ) + expected_tool = case.expectation.get("expected_tool") + if parsed.get("tool") != expected_tool: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="tool_call_error", + evidence=f"expected tool {expected_tool!r}, got {parsed.get('tool')!r}", + trace={"expectation_type": "tool", "expected_tool": expected_tool}, + ) + expected_args = dict(case.expectation.get("expected_args") or {}) + actual_args = parsed.get("args") or {} + for key, expected_value in expected_args.items(): + if actual_args.get(key) != expected_value: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="parameter_error", + evidence=f"arg {key!r}: expected {expected_value!r}, got {actual_args.get(key)!r}", + trace={"expectation_type": "tool", "arg": key}, + ) + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "tool"}) + + def _score_knowledge(self, case: EvalCase, output: str) -> JudgeOutcome: + lowered = output.lower() + required_sources = [str(item) for item in case.expectation.get("required_sources") or []] + required_terms = [str(item) for item in case.expectation.get("must_include_knowledge_terms") or []] + missing_sources = [source for source in required_sources if source.lower() not in lowered] + missing_terms = [term for term in required_terms if term.lower() not in lowered] + if missing_sources or missing_terms: + return JudgeOutcome( + score=0.0, + passed=False, + error_code="knowledge_recall_insufficient", + evidence=f"missing sources={missing_sources!r}, terms={missing_terms!r}", + trace={ + "expectation_type": "knowledge", + "missing_sources": missing_sources, + "missing_terms": missing_terms, + }, + ) + return JudgeOutcome(score=1.0, passed=True, trace={"expectation_type": "knowledge"}) + + +def _normalize_exact(value: str) -> str: + return " ".join(value.strip().lower().split()) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py new file mode 100644 index 00000000..a7555e74 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/fake_model.py @@ -0,0 +1,137 @@ +"""Deterministic fake model used by the example pipeline.""" + +from __future__ import annotations + +import json +from typing import Any + +from .schemas import EvalCase + + +class FakeModel: + """Prompt-sensitive deterministic model. + + Markers injected by ``FakeOptimizer`` select behavior: + - no marker: baseline; adds prose around strict JSON/exact outputs. + - ALWAYS_OUTPUT_JSON: overfits to train formatting and forces JSON on + validation natural-language/exact cases. + - STRICT_WHEN_REQUESTED: applies JSON/exact constraints only when the case + explicitly asks for them. + """ + + COST_PER_CALL = 0.001 + + def __init__(self, seed: int = 91) -> None: + self.seed = seed + + def generate(self, prompt_id: str, prompt: str, case: EvalCase) -> tuple[str, dict[str, Any], float]: + mode = self._mode(prompt) + output_override = self._simulated_output(case, mode) + if output_override is not None: + output = output_override + elif mode == "overfit": + output = self._overfit_output(case) + elif mode == "safe": + output = self._safe_output(case) + else: + output = self._baseline_output(case) + trace = { + "seed": self.seed, + "prompt_id": prompt_id, + "prompt_mode": mode, + "case_id": case.case_id, + "expectation_type": case.expectation.get("type"), + } + return output, trace, self.COST_PER_CALL + + def _mode(self, prompt: str) -> str: + if "ALWAYS_OUTPUT_JSON" in prompt: + return "overfit" + if "STRICT_WHEN_REQUESTED" in prompt: + return "safe" + return "baseline" + + def _baseline_output(self, case: EvalCase) -> str: + expectation_type = case.expectation.get("type") + if expectation_type == "json": + return f"Here is the JSON you requested: {self._expected_json(case)}" + if expectation_type == "exact": + expected = str(case.expectation.get("expected", "")) + if case.protected or "baseline_pass" in case.tags: + return expected + return f"{expected} - confirmed." + if expectation_type == "rubric": + return self._rubric_sentence(case) + if expectation_type == "tool": + return self._tool_json(case) + if expectation_type == "knowledge": + return self._knowledge_sentence(case) + return self._rubric_sentence(case) + + def _overfit_output(self, case: EvalCase) -> str: + expectation_type = case.expectation.get("type") + if case.split == "train": + return self._ideal_output(case) + if expectation_type == "json": + return self._expected_json(case) + if expectation_type == "exact": + return json.dumps({"answer": str(case.expectation.get("expected", ""))}, sort_keys=True) + if expectation_type == "rubric" or "prose" in case.tags: + return json.dumps({"answer": self._rubric_sentence(case)}, sort_keys=True) + if expectation_type == "tool": + return self._tool_json(case) + if expectation_type == "knowledge": + return self._knowledge_sentence(case) + return json.dumps({"answer": self._rubric_sentence(case)}, sort_keys=True) + + def _safe_output(self, case: EvalCase) -> str: + user_asked = case.input.lower() + expectation_type = case.expectation.get("type") + if expectation_type == "json" or "json" in user_asked: + return self._expected_json(case) + if expectation_type == "exact" or "exactly" in user_asked: + return str(case.expectation.get("expected", "")) + return self._ideal_output(case) + + def _ideal_output(self, case: EvalCase) -> str: + expectation_type = case.expectation.get("type") + if expectation_type == "json": + return self._expected_json(case) + if expectation_type == "exact": + return str(case.expectation.get("expected", "")) + if expectation_type == "rubric": + return self._rubric_sentence(case) + if expectation_type == "tool": + return self._tool_json(case) + if expectation_type == "knowledge": + return self._knowledge_sentence(case) + return self._rubric_sentence(case) + + def _expected_json(self, case: EvalCase) -> str: + values = dict(case.expectation.get("expected_values") or {}) + return json.dumps(values, sort_keys=True) + + def _simulated_output(self, case: EvalCase, mode: str) -> str | None: + return case.simulated_outputs.get(mode) + + def _rubric_sentence(self, case: EvalCase) -> str: + must_include = [str(item) for item in case.expectation.get("must_include") or []] + if must_include: + sentence = " ".join(must_include) + else: + sentence = "The answer satisfies the rubric" + max_chars = case.expectation.get("max_chars") + if max_chars is not None and len(sentence) > int(max_chars): + sentence = sentence[:int(max_chars)].rstrip() + return sentence + + def _tool_json(self, case: EvalCase) -> str: + tool_name = str(case.expectation.get("expected_tool", "lookup")) + args = dict(case.expectation.get("expected_args") or {}) + return json.dumps({"tool": tool_name, "args": args}, sort_keys=True) + + def _knowledge_sentence(self, case: EvalCase) -> str: + terms = [str(item) for item in case.expectation.get("must_include_knowledge_terms") or []] + sources = [str(item) for item in case.expectation.get("required_sources") or []] + parts = terms + sources + return " ".join(parts) if parts else "knowledge source recalled" diff --git a/examples/optimization/eval_optimize_loop/eval_loop/gate.py b/examples/optimization/eval_optimize_loop/eval_loop/gate.py new file mode 100644 index 00000000..ae9abb72 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/gate.py @@ -0,0 +1,121 @@ +"""Configurable acceptance gate for candidate prompts.""" + +from __future__ import annotations + +from typing import Any + +from .schemas import CaseDelta +from .schemas import EvalResult +from .schemas import GateDecision + + +DEFAULT_GATE_CONFIG = { + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": [], + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0, +} + + +class AcceptanceGate: + """Apply deterministic safety and quality constraints to candidates.""" + + def __init__(self, config: dict[str, Any]) -> None: + merged = dict(DEFAULT_GATE_CONFIG) + merged.update(config or {}) + self.config = merged + + def decide( + self, + *, + candidate_id: str, + baseline_train: EvalResult, + baseline_validation: EvalResult, + candidate_train: EvalResult, + candidate_validation: EvalResult, + deltas: list[CaseDelta], + cumulative_cost: float = 0.0, + ) -> GateDecision: + train_delta = round(candidate_train.score - baseline_train.score, 6) + val_delta = round(candidate_validation.score - baseline_validation.score, 6) + candidate_cost = round(candidate_train.cost + candidate_validation.cost, 6) + reasons: list[str] = [] + + overfit_detected = train_delta > 0 and val_delta <= 0 + if overfit_detected: + reasons.append( + "reject: overfit detected because train score improved but validation score regressed or did not improve " + f"({train_delta:+.3f} train, {val_delta:+.3f} validation)" + ) + + min_val_improvement = float(self.config["min_val_score_improvement"]) + if val_delta < min_val_improvement: + reasons.append( + "reject: validation improvement " + f"{val_delta:+.3f} is below required {min_val_improvement:+.3f}" + ) + + baseline_validation_by_id = baseline_validation.by_case_id() + candidate_validation_by_id = candidate_validation.by_case_id() + validation_new_failures = [ + case_id + for case_id, candidate_case in sorted(candidate_validation_by_id.items()) + if not candidate_case.passed and baseline_validation_by_id.get(case_id) + and baseline_validation_by_id[case_id].passed + ] + new_hard_failures = [ + case_id + for case_id, candidate_case in sorted(candidate_validation_by_id.items()) + if candidate_case.hard_failed and baseline_validation_by_id.get(case_id) + and baseline_validation_by_id[case_id].passed + ] + if new_hard_failures and not bool(self.config["allow_new_hard_fail"]): + reasons.append(f"reject: new hard failures appeared: {new_hard_failures}") + + protected_ids = set(str(item) for item in self.config["protected_case_ids"]) + protected_regressions = [ + delta.case_id + for delta in deltas + if delta.split == "validation" and delta.case_id in protected_ids and delta.delta < 0 + ] + if protected_regressions: + reasons.append(f"reject: protected cases regressed: {protected_regressions}") + + max_drop = float(self.config["max_score_drop_per_case"]) + excessive_drops = [ + delta.case_id + for delta in deltas + if delta.split == "validation" and delta.delta < -max_drop + ] + if excessive_drops: + reasons.append(f"reject: per-case validation score drops exceed {max_drop:.3f}: {excessive_drops}") + + max_total_cost = float(self.config["max_total_cost"]) + total_run_cost = round(cumulative_cost + candidate_cost, 6) + if total_run_cost > max_total_cost: + reasons.append(f"reject: total run cost {total_run_cost:.3f} exceeds budget {max_total_cost:.3f}") + + accepted = not any(reason.startswith("reject:") for reason in reasons) + if accepted: + reasons.append( + "accept: validation score improved " + f"{val_delta:+.3f} with no protected regression or new hard failure" + ) + + return GateDecision( + candidate_id=candidate_id, + accepted=accepted, + reasons=reasons, + train_score_delta=train_delta, + validation_score_delta=val_delta, + new_hard_failures=new_hard_failures, + protected_regressions=protected_regressions, + validation_new_failures=validation_new_failures, + excessive_score_drops=excessive_drops, + overfit_detected=overfit_detected, + candidate_cost=candidate_cost, + cumulative_cost=round(cumulative_cost, 6), + total_run_cost=total_run_cost, + cost=candidate_cost, + ) diff --git a/examples/optimization/eval_optimize_loop/eval_loop/loader.py b/examples/optimization/eval_optimize_loop/eval_loop/loader.py new file mode 100644 index 00000000..29a8628a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/loader.py @@ -0,0 +1,55 @@ +"""Input loading helpers for the deterministic optimization example.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from .config import OptimizerConfig +from .config import parse_optimizer_config +from .schemas import EvalCase + + +def read_json(path: str | Path) -> dict[str, Any]: + resolved = Path(path) + with resolved.open("r", encoding="utf-8") as file: + payload = json.load(file) + if not isinstance(payload, dict): + raise ValueError(f"expected JSON object in {resolved}") + return payload + + +def load_eval_cases(path: str | Path, split: str | None = None) -> list[EvalCase]: + payload = read_json(path) + cases = payload.get("cases") + if not isinstance(cases, list): + raise ValueError(f"evalset {path} must contain a cases list") + effective_split = split or payload.get("split") or Path(path).name.split(".", 1)[0] + return [EvalCase.from_dict(case, str(effective_split)) for case in cases] + + +def load_optimizer_config(path: str | Path) -> OptimizerConfig: + payload = read_json(path) + return parse_optimizer_config(payload, path=path) + + +def load_prompt(path: str | Path) -> str: + return Path(path).read_text(encoding="utf-8") + + +def stable_config_hash(config: dict[str, Any]) -> str: + import hashlib + + canonical = json.dumps(config, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def sha256_file(path: str | Path) -> str: + import hashlib + + digest = hashlib.sha256() + with Path(path).open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py new file mode 100644 index 00000000..da91ee12 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/optimizer.py @@ -0,0 +1,58 @@ +"""Fake optimizer that proposes deterministic prompt candidates.""" + +from __future__ import annotations + +from .diffing import make_unified_diff +from .schemas import CandidatePrompt + + +class FakeOptimizer: + """Produce the two candidates required by the example issue.""" + + def propose(self, baseline_prompt: str) -> list[CandidatePrompt]: + overfit_prompt = ( + baseline_prompt.rstrip() + + "\n\n" + + "# Optimizer patch\n" + + "OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n" + + "Always force every final answer into JSON, even when the user asks for prose.\n" + ) + safe_prompt = ( + baseline_prompt.rstrip() + + "\n\n" + + "# Optimizer patch\n" + + "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n" + + "Use strict JSON only when the user explicitly asks for JSON.\n" + + "Use exact answers only when the user explicitly asks for an exact answer.\n" + + "Otherwise answer naturally and honor rubric constraints.\n" + ) + return [ + CandidatePrompt( + candidate_id="candidate_001_overfit", + prompt=overfit_prompt, + rationale=( + "The train failures are strict JSON/exact formatting failures, so this candidate " + "over-corrects by forcing JSON globally." + ), + prompt_diff=make_unified_diff( + baseline_prompt, + overfit_prompt, + before_name="baseline_system_prompt.txt", + after_name="candidate_001_overfit/system_prompt.txt", + ), + ), + CandidatePrompt( + candidate_id="candidate_002_safe", + prompt=safe_prompt, + rationale=( + "This candidate fixes observed strict-format failures without changing " + "natural-language behavior on validation cases." + ), + prompt_diff=make_unified_diff( + baseline_prompt, + safe_prompt, + before_name="baseline_system_prompt.txt", + after_name="candidate_002_safe/system_prompt.txt", + ), + ), + ] diff --git a/examples/optimization/eval_optimize_loop/eval_loop/report.py b/examples/optimization/eval_optimize_loop/eval_loop/report.py new file mode 100644 index 00000000..de1a1b35 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/report.py @@ -0,0 +1,317 @@ +"""Report construction and rendering.""" + +from __future__ import annotations + +import json +import re +import shutil +from pathlib import Path +from typing import Any + +from .attribution import summarize_failures +from .schemas import CandidatePrompt +from .schemas import CaseDelta +from .schemas import EvalResult +from .schemas import GateDecision +from .schemas import OptimizationReport +from .schemas import to_jsonable + + +REPRODUCIBILITY_COMMAND = ( + "python examples/optimization/eval_optimize_loop/run_pipeline.py " + "--train examples/optimization/eval_optimize_loop/data/train.evalset.json " + "--val examples/optimization/eval_optimize_loop/data/val.evalset.json " + "--optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json " + "--prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt " + "--output-dir /tmp/eval-optimize-loop " + "--fake-model --fake-judge --trace" +) +ARTIFACT_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + + +def compute_case_deltas( + *, + candidate_id: str, + baseline_train: EvalResult, + baseline_validation: EvalResult, + candidate_train: EvalResult, + candidate_validation: EvalResult, +) -> list[CaseDelta]: + deltas: list[CaseDelta] = [] + for baseline, candidate in ((baseline_train, candidate_train), (baseline_validation, candidate_validation)): + candidate_by_id = candidate.by_case_id() + for baseline_case in baseline.cases: + candidate_case = candidate_by_id[baseline_case.case_id] + delta = round(candidate_case.score - baseline_case.score, 6) + delta_type = _delta_type( + baseline_passed=baseline_case.passed, + candidate_passed=candidate_case.passed, + delta=delta, + ) + deltas.append( + CaseDelta( + candidate_id=candidate_id, + case_id=baseline_case.case_id, + split=baseline_case.split, + baseline_score=baseline_case.score, + candidate_score=candidate_case.score, + delta=delta, + baseline_passed=baseline_case.passed, + candidate_passed=candidate_case.passed, + regression=delta < 0, + delta_type=delta_type, + ) + ) + return deltas + + +def build_report( + *, + run: dict[str, Any], + baseline_train: EvalResult, + baseline_validation: EvalResult, + candidate_records: list[dict[str, Any]], + per_case_deltas: list[CaseDelta], + gate_decisions: list[GateDecision], + selected_candidate: str | None, + audit: dict[str, Any], +) -> OptimizationReport: + all_results: list[EvalResult] = [baseline_train, baseline_validation] + for record in candidate_records: + all_results.append(record["train_result"]) + all_results.append(record["validation_result"]) + return OptimizationReport( + schema_version="eval_optimize_loop.v1", + run=run, + baseline={"train": baseline_train, "validation": baseline_validation}, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidates=candidate_records, + delta={"per_case": per_case_deltas}, + per_case_deltas=per_case_deltas, + failure_attribution_summary=summarize_failures(all_results), + gate_decisions=gate_decisions, + selected_candidate=selected_candidate, + audit=audit, + ) + + +def write_reports(report: OptimizationReport, output_dir: str | Path) -> tuple[Path, Path]: + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + json_path = output_path / "optimization_report.json" + md_path = output_path / "optimization_report.md" + json_path.write_text(report_to_json(report), encoding="utf-8") + md_path.write_text(render_markdown(report), encoding="utf-8") + write_audit_artifacts(report, output_path) + return json_path, md_path + + +def report_to_json(report: OptimizationReport) -> str: + return json.dumps(to_jsonable(report), indent=2, ensure_ascii=False, sort_keys=True, allow_nan=False) + "\n" + + +def render_markdown(report: OptimizationReport) -> str: + decision_by_id = {decision.candidate_id: decision for decision in report.gate_decisions} + lines = [ + "# Evaluation + Optimization Report", + "", + "## Final Decision", + "", + ] + if report.selected_candidate: + lines.append(f"Selected candidate: `{report.selected_candidate}`.") + else: + lines.append("No candidate was accepted.") + + lines.extend([ + "", + "Update source prompt: " + + ("yes" if report.run.get("update_source") else "no (default)"), + "", + ]) + if report.run.get("mode") == "sdk": + availability = report.audit.get("sdk_result_availability", {}) + lines.extend([ + "SDK mode uses OptimizeResult aggregate validation metrics. " + "Full train scores and full per-case validation deltas are not exposed by the SDK result.", + "", + "SDK availability: " + f"aggregate_validation_result={availability.get('aggregate_validation_result')}, " + f"full_train_eval_result={availability.get('full_train_eval_result')}, " + f"full_per_case_validation_delta={availability.get('full_per_case_validation_delta')}.", + "", + ]) + lines.extend([ + "", + "## Gate Reasons", + "", + ]) + for decision in report.gate_decisions: + verdict = ( + decision.gate_status + if decision.gate_status != "applied" + else ("accepted" if decision.accepted else "rejected") + ) + lines.append(f"### {decision.candidate_id} ({verdict})") + for reason in decision.reasons: + lines.append(f"- {reason}") + if decision.not_applied_checks: + lines.append(f"- not applied checks: {', '.join(decision.not_applied_checks)}") + lines.append("") + + lines.extend([ + "## Baseline vs Candidate Scores", + "", + "| prompt | train score | validation score | gate |", + "| --- | ---: | ---: | --- |", + f"| baseline | {report.baseline_train.score:.3f} | {report.baseline_validation.score:.3f} | n/a |", + ]) + for record in report.candidates: + candidate: CandidatePrompt = record["candidate"] + gate = decision_by_id[candidate.candidate_id] + verdict = gate.gate_status if gate.gate_status != "applied" else ("accept" if gate.accepted else "reject") + lines.append( + f"| {candidate.candidate_id} | {record['train_result'].score:.3f} | " + f"{record['validation_result'].score:.3f} | {verdict} |" + ) + + lines.extend([ + "", + "## Per-Case Delta", + "", + "| candidate | split | case | baseline | candidate | delta | passed -> passed | delta type |", + "| --- | --- | --- | ---: | ---: | ---: | --- | --- |", + ]) + for delta in report.per_case_deltas: + lines.append( + f"| {delta.candidate_id} | {delta.split} | {delta.case_id} | " + f"{delta.baseline_score:.3f} | {delta.candidate_score:.3f} | " + f"{delta.delta:+.3f} | {delta.baseline_passed} -> {delta.candidate_passed} | " + f"{delta.delta_type} |" + ) + + summary = report.failure_attribution_summary + lines.extend([ + "", + "## Failure Attribution Summary", + "", + f"Total failed case evaluations: {summary['total_failed_cases']}", + "", + "| category | count |", + "| --- | ---: |", + ]) + by_category = summary.get("by_category", {}) + if by_category: + for category, count in by_category.items(): + lines.append(f"| {category} | {count} |") + else: + lines.append("| none | 0 |") + if summary.get("attribution_accuracy") is not None: + lines.append("") + lines.append(f"Attribution accuracy: {summary['attribution_accuracy']:.3f}") + + lines.extend([ + "", + "## Cost And Audit", + "", + f"Total cost: {report.audit.get('cost', {}).get('total', 0):.3f}", + f"Config hash: `{report.audit.get('config_hash', '')}`", + f"Run id: `{report.run.get('run_id', '')}`", + ]) + + lines.extend([ + "", + "## Prompt Diff", + "", + ]) + for record in report.candidates: + candidate = record["candidate"] + lines.extend([ + f"### {candidate.candidate_id}", + "", + "```diff", + candidate.prompt_diff, + "```", + "", + ]) + + lines.extend([ + "## Reproducibility", + "", + "```bash", + report.run.get("reproducibility_command") + or report.audit.get("reproducibility_command") + or REPRODUCIBILITY_COMMAND, + "```", + "", + ]) + return "\n".join(lines) + + +def write_audit_artifacts(report: OptimizationReport, output_path: Path) -> None: + run_id = _safe_artifact_name(str(report.run.get("run_id") or "run")) + run_dir = output_path / "runs" / run_id + if run_dir.exists() and report.run.get("mode") == "fake": + shutil.rmtree(run_dir) + run_dir.mkdir(parents=True, exist_ok=True) + + input_paths = report.audit.get("input_paths", {}) + config_path = input_paths.get("optimizer") + if config_path and Path(config_path).is_file(): + shutil.copyfile(config_path, run_dir / "config.snapshot.json") + else: + (run_dir / "config.snapshot.json").write_text("{}", encoding="utf-8") + + (run_dir / "input_hashes.json").write_text( + json.dumps(report.audit.get("input_hashes", {}), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + prompt_dir = run_dir / "candidate_prompts" + results_dir = run_dir / "case_results" + diffs_dir = run_dir / "prompt_diffs" + prompt_dir.mkdir(exist_ok=True) + results_dir.mkdir(exist_ok=True) + diffs_dir.mkdir(exist_ok=True) + + for record in report.candidates: + candidate: CandidatePrompt = record["candidate"] + candidate_name = _safe_artifact_name(candidate.candidate_id) + candidate_dir = prompt_dir / candidate_name + candidate_dir.mkdir(exist_ok=True) + best_prompts = report.audit.get("sdk_result_summary", {}).get("best_prompts", {}) + if report.run.get("mode") == "sdk" and isinstance(best_prompts, dict) and best_prompts: + for field_name, prompt_text in best_prompts.items(): + field_artifact = _safe_artifact_name(str(field_name)) + (candidate_dir / f"{field_artifact}.txt").write_text(str(prompt_text), encoding="utf-8") + (candidate_dir / "bundle.txt").write_text(candidate.prompt, encoding="utf-8") + else: + (candidate_dir / "system_prompt.txt").write_text(candidate.prompt, encoding="utf-8") + (diffs_dir / f"{candidate_name}.diff").write_text(candidate.prompt_diff, encoding="utf-8") + for split_name in ("train_result", "validation_result"): + split_result = record[split_name] + split_artifact = _safe_artifact_name(str(split_result.split)) + path = results_dir / f"{candidate_name}_{split_artifact}.json" + path.write_text( + json.dumps(to_jsonable(split_result), indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + + +def _delta_type(*, baseline_passed: bool, candidate_passed: bool, delta: float) -> str: + if not baseline_passed and candidate_passed: + return "new_pass" + if baseline_passed and not candidate_passed: + return "new_fail" + if delta > 0: + return "score_up" + if delta < 0: + return "score_down" + return "unchanged" + + +def _safe_artifact_name(name: str) -> str: + if name in {"", ".", ".."} or not ARTIFACT_NAME_RE.fullmatch(name): + raise ValueError(f"unsafe audit artifact name: {name!r}") + return name diff --git a/examples/optimization/eval_optimize_loop/eval_loop/schemas.py b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py new file mode 100644 index 00000000..c1ce402d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/schemas.py @@ -0,0 +1,157 @@ +"""Dataclass schemas used by the example optimization loop.""" + +from __future__ import annotations + +from dataclasses import asdict +from dataclasses import dataclass +from dataclasses import field +from dataclasses import is_dataclass +from typing import Any + + +@dataclass(frozen=True) +class EvalCase: + """One deterministic evaluation case.""" + + case_id: str + split: str + input: str + expectation: dict[str, Any] + tags: list[str] = field(default_factory=list) + protected: bool = False + simulated_outputs: dict[str, str] = field(default_factory=dict) + expected_failure_category: str | None = None + + @classmethod + def from_dict(cls, payload: dict[str, Any], split: str) -> "EvalCase": + case_id = payload.get("case_id") or payload.get("id") + if not case_id: + raise ValueError(f"eval case is missing id/case_id: {payload!r}") + expectation = payload.get("expectation") + if not isinstance(expectation, dict): + raise ValueError(f"eval case {case_id!r} is missing expectation object") + return cls( + case_id=str(case_id), + split=str(payload.get("split") or split), + input=str(payload.get("input") or payload.get("user_input") or ""), + expectation=dict(expectation), + tags=list(payload.get("tags") or []), + protected=bool(payload.get("protected", False)), + simulated_outputs=dict(payload.get("simulated_outputs") or expectation.get("simulated_outputs") or {}), + expected_failure_category=payload.get("expected_failure_category") + or expectation.get("expected_failure_category"), + ) + + +@dataclass(frozen=True) +class CaseResult: + """Evaluation result for one case under one prompt.""" + + case_id: str + split: str + score: float + passed: bool + output: str + trace: dict[str, Any] = field(default_factory=dict) + failure_category: str | None = None + failure_reason: str | None = None + evidence: str | None = None + cost: float = 0.0 + hard_failed: bool = False + expected_failure_category: str | None = None + + +@dataclass(frozen=True) +class EvalResult: + """Aggregate result for one prompt on one split.""" + + prompt_id: str + split: str + score: float + passed: bool + cost: float + cases: list[CaseResult] + + def by_case_id(self) -> dict[str, CaseResult]: + return {case.case_id: case for case in self.cases} + + +@dataclass(frozen=True) +class CandidatePrompt: + """A proposed prompt candidate.""" + + candidate_id: str + prompt: str + rationale: str + prompt_diff: str + + +@dataclass(frozen=True) +class CaseDelta: + """Per-case score delta from baseline to candidate.""" + + candidate_id: str + case_id: str + split: str + baseline_score: float + candidate_score: float + delta: float + baseline_passed: bool + candidate_passed: bool + regression: bool + delta_type: str + + +@dataclass(frozen=True) +class GateDecision: + """Configurable acceptance gate result for one candidate.""" + + candidate_id: str + accepted: bool + reasons: list[str] + train_score_delta: float + validation_score_delta: float + new_hard_failures: list[str] + protected_regressions: list[str] + validation_new_failures: list[str] + excessive_score_drops: list[str] + overfit_detected: bool + candidate_cost: float + cumulative_cost: float + total_run_cost: float + cost: float + gate_status: str = "applied" + gate_not_applied_reason: str | None = None + not_applied_checks: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class OptimizationReport: + """Complete persisted audit report for the loop.""" + + schema_version: str + run: dict[str, Any] + baseline: dict[str, EvalResult] + baseline_train: EvalResult + baseline_validation: EvalResult + candidates: list[dict[str, Any]] + delta: dict[str, Any] + per_case_deltas: list[CaseDelta] + failure_attribution_summary: dict[str, Any] + gate_decisions: list[GateDecision] + selected_candidate: str | None + audit: dict[str, Any] + + +def to_jsonable(value: Any) -> Any: + """Convert dataclasses and nested containers into JSON-serializable data.""" + + if is_dataclass(value): + return {key: to_jsonable(item) for key, item in asdict(value).items()} + if isinstance(value, dict): + return {str(key): to_jsonable(item) for key, item in value.items()} + if isinstance(value, list): + return [to_jsonable(item) for item in value] + if isinstance(value, tuple): + return [to_jsonable(item) for item in value] + return value diff --git a/examples/optimization/eval_optimize_loop/eval_loop/trace.py b/examples/optimization/eval_optimize_loop/eval_loop/trace.py new file mode 100644 index 00000000..917416d5 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/eval_loop/trace.py @@ -0,0 +1,18 @@ +"""Trace helpers for fake-mode runs.""" + +from __future__ import annotations + +from typing import Any + + +def make_trace(enabled: bool, *, prompt_id: str, case_id: str, model_trace: dict[str, Any], + judge_trace: dict[str, Any]) -> dict[str, Any]: + if not enabled: + return {} + return { + "trace_mode": "fake", + "prompt_id": prompt_id, + "case_id": case_id, + "model": model_trace, + "judge": judge_trace, + } diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json new file mode 100644 index 00000000..4b41a37e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.json @@ -0,0 +1,1271 @@ +{ + "audit": { + "candidate_prompt_hashes": { + "candidate_001_overfit": "07a494c337544626a4c23afe1a945488e503b06da3e12597419470eb2ab09e42", + "candidate_002_safe": "9669bbd13891b78d2c493df624050dcfd1aa36533987f082a1b78b864df5572f" + }, + "candidate_prompts": { + "candidate_001_overfit": { + "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\nAlways force every final answer into JSON, even when the user asks for prose.\n", + "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,7 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+Always force every final answer into JSON, even when the user asks for prose.", + "rationale": "The train failures are strict JSON/exact formatting failures, so this candidate over-corrects by forcing JSON globally." + }, + "candidate_002_safe": { + "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\nUse strict JSON only when the user explicitly asks for JSON.\nUse exact answers only when the user explicitly asks for an exact answer.\nOtherwise answer naturally and honor rubric constraints.\n", + "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,9 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+Use strict JSON only when the user explicitly asks for JSON.\n+Use exact answers only when the user explicitly asks for an exact answer.\n+Otherwise answer naturally and honor rubric constraints.", + "rationale": "This candidate fixes observed strict-format failures without changing natural-language behavior on validation cases." + } + }, + "config_hash": "66a1db3a1c84ad12fd41385fc5e1a9c23cc305e6973945c4bade559c804f9abe", + "cost": { + "baseline": 0.006, + "candidates": { + "candidate_001_overfit": 0.006, + "candidate_002_safe": 0.006 + }, + "total": 0.018 + }, + "duration_seconds": 0.0, + "input_hashes": { + "optimizer": "a00febaa9efaaf299bbf7dbc08eb31b9093315f0079e601d59c84d0d5704f784", + "prompt": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95", + "train": "03ae8fd41a19ab376c733cb4d5b92a5d6c34431717b5b4899cabdb1e23351a56", + "validation": "73a32b7773f789b2da3d15e925ca0f57da05e8a19c079c9a56ef42055961804b" + }, + "input_paths": { + "optimizer": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\optimizer.json", + "prompt": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\prompts\\baseline_system_prompt.txt", + "train": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\train.evalset.json", + "validation": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\val.evalset.json" + }, + "prompt_diffs": { + "candidate_001_overfit": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,7 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+Always force every final answer into JSON, even when the user asks for prose.", + "candidate_002_safe": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,9 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+Use strict JSON only when the user explicitly asks for JSON.\n+Use exact answers only when the user explicitly asks for an exact answer.\n+Otherwise answer naturally and honor rubric constraints." + }, + "prompt_hash": "ce30a6d1dd86f988cbd4abe08648c9596c4808e429b3895e6ca5bdc53411ea95", + "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --output-dir /tmp/eval-optimize-loop --fake-model --fake-judge --trace", + "seed": 91, + "total_run_cost": 0.018 + }, + "baseline": { + "train": { + "cases": [ + { + "case_id": "train_json_refund", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "output": "Here is the JSON you requested: {\"intent\": \"refund\", \"priority\": \"high\"}", + "passed": false, + "score": 0.0, + "split": "train", + "trace": { + "case_id": "train_json_refund", + "judge": { + "expectation_type": "json", + "valid_json": false + }, + "model": { + "case_id": "train_json_refund", + "expectation_type": "json", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "train_exact_order_status", + "cost": 0.001, + "evidence": "expected normalized 'READY', got 'READY - confirmed.'", + "expected_failure_category": "final_response_mismatch", + "failure_category": "final_response_mismatch", + "failure_reason": "normalized exact answer mismatch", + "hard_failed": true, + "output": "READY - confirmed.", + "passed": false, + "score": 0.0, + "split": "train", + "trace": { + "case_id": "train_exact_order_status", + "judge": { + "expectation_type": "exact", + "expected": "READY" + }, + "model": { + "case_id": "train_exact_order_status", + "expectation_type": "exact", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "train_rubric_retry_summary", + "cost": 0.001, + "evidence": null, + "expected_failure_category": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "latency retries", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_rubric_retry_summary", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "train_rubric_retry_summary", + "expectation_type": "rubric", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "baseline", + "score": 0.333333, + "split": "train" + }, + "validation": { + "cases": [ + { + "case_id": "val_json_invoice", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "output": "Here is the JSON you requested: {\"next_step\": \"email_customer\", \"status\": \"approved\"}", + "passed": false, + "score": 0.0, + "split": "validation", + "trace": { + "case_id": "val_json_invoice", + "judge": { + "expectation_type": "json", + "valid_json": false + }, + "model": { + "case_id": "val_json_invoice", + "expectation_type": "json", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "val_explain_cache", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "cache stale data", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_explain_cache", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "val_explain_cache", + "expectation_type": "rubric", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "val_protected_yes_no", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "final_response_mismatch", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "YES", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_protected_yes_no", + "judge": { + "expectation_type": "exact", + "expected": "YES" + }, + "model": { + "case_id": "val_protected_yes_no", + "expectation_type": "exact", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "baseline", + "score": 0.666667, + "split": "validation" + } + }, + "baseline_train": { + "cases": [ + { + "case_id": "train_json_refund", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "output": "Here is the JSON you requested: {\"intent\": \"refund\", \"priority\": \"high\"}", + "passed": false, + "score": 0.0, + "split": "train", + "trace": { + "case_id": "train_json_refund", + "judge": { + "expectation_type": "json", + "valid_json": false + }, + "model": { + "case_id": "train_json_refund", + "expectation_type": "json", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "train_exact_order_status", + "cost": 0.001, + "evidence": "expected normalized 'READY', got 'READY - confirmed.'", + "expected_failure_category": "final_response_mismatch", + "failure_category": "final_response_mismatch", + "failure_reason": "normalized exact answer mismatch", + "hard_failed": true, + "output": "READY - confirmed.", + "passed": false, + "score": 0.0, + "split": "train", + "trace": { + "case_id": "train_exact_order_status", + "judge": { + "expectation_type": "exact", + "expected": "READY" + }, + "model": { + "case_id": "train_exact_order_status", + "expectation_type": "exact", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "train_rubric_retry_summary", + "cost": 0.001, + "evidence": null, + "expected_failure_category": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "latency retries", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_rubric_retry_summary", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "train_rubric_retry_summary", + "expectation_type": "rubric", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "baseline", + "score": 0.333333, + "split": "train" + }, + "baseline_validation": { + "cases": [ + { + "case_id": "val_json_invoice", + "cost": 0.001, + "evidence": "json parser failed at char 0: Expecting value", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "hard_failed": true, + "output": "Here is the JSON you requested: {\"next_step\": \"email_customer\", \"status\": \"approved\"}", + "passed": false, + "score": 0.0, + "split": "validation", + "trace": { + "case_id": "val_json_invoice", + "judge": { + "expectation_type": "json", + "valid_json": false + }, + "model": { + "case_id": "val_json_invoice", + "expectation_type": "json", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "val_explain_cache", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "cache stale data", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_explain_cache", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "val_explain_cache", + "expectation_type": "rubric", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + }, + { + "case_id": "val_protected_yes_no", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "final_response_mismatch", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "YES", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_protected_yes_no", + "judge": { + "expectation_type": "exact", + "expected": "YES" + }, + "model": { + "case_id": "val_protected_yes_no", + "expectation_type": "exact", + "prompt_id": "baseline", + "prompt_mode": "baseline", + "seed": 91 + }, + "prompt_id": "baseline", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "baseline", + "score": 0.666667, + "split": "validation" + }, + "candidates": [ + { + "candidate": { + "candidate_id": "candidate_001_overfit", + "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\nAlways force every final answer into JSON, even when the user asks for prose.\n", + "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_001_overfit/system_prompt.txt\n@@ -2,3 +2,7 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON\n+Always force every final answer into JSON, even when the user asks for prose.", + "rationale": "The train failures are strict JSON/exact formatting failures, so this candidate over-corrects by forcing JSON globally." + }, + "train_result": { + "cases": [ + { + "case_id": "train_json_refund", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "{\"intent\": \"refund\", \"priority\": \"high\"}", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_json_refund", + "judge": { + "expectation_type": "json", + "valid_json": true + }, + "model": { + "case_id": "train_json_refund", + "expectation_type": "json", + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "prompt_id": "candidate_001_overfit", + "trace_mode": "fake" + } + }, + { + "case_id": "train_exact_order_status", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "final_response_mismatch", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "READY", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_exact_order_status", + "judge": { + "expectation_type": "exact", + "expected": "READY" + }, + "model": { + "case_id": "train_exact_order_status", + "expectation_type": "exact", + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "prompt_id": "candidate_001_overfit", + "trace_mode": "fake" + } + }, + { + "case_id": "train_rubric_retry_summary", + "cost": 0.001, + "evidence": null, + "expected_failure_category": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "latency retries", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_rubric_retry_summary", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "train_rubric_retry_summary", + "expectation_type": "rubric", + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "prompt_id": "candidate_001_overfit", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": true, + "prompt_id": "candidate_001_overfit", + "score": 1.0, + "split": "train" + }, + "validation_result": { + "cases": [ + { + "case_id": "val_json_invoice", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "{\"next_step\": \"email_customer\", \"status\": \"approved\"}", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_json_invoice", + "judge": { + "expectation_type": "json", + "valid_json": true + }, + "model": { + "case_id": "val_json_invoice", + "expectation_type": "json", + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "prompt_id": "candidate_001_overfit", + "trace_mode": "fake" + } + }, + { + "case_id": "val_explain_cache", + "cost": 0.001, + "evidence": "forbidden pattern '{' was present", + "expected_failure_category": "format_violation", + "failure_category": "format_violation", + "failure_reason": "output contains a forbidden pattern", + "hard_failed": true, + "output": "{\"answer\": \"cache stale data\"}", + "passed": false, + "score": 0.0, + "split": "validation", + "trace": { + "case_id": "val_explain_cache", + "judge": { + "expectation_type": "rubric", + "forbidden_pattern": "{" + }, + "model": { + "case_id": "val_explain_cache", + "expectation_type": "rubric", + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "prompt_id": "candidate_001_overfit", + "trace_mode": "fake" + } + }, + { + "case_id": "val_protected_yes_no", + "cost": 0.001, + "evidence": "expected normalized 'YES', got '{\"answer\": \"YES\"}'", + "expected_failure_category": "final_response_mismatch", + "failure_category": "final_response_mismatch", + "failure_reason": "normalized exact answer mismatch", + "hard_failed": true, + "output": "{\"answer\": \"YES\"}", + "passed": false, + "score": 0.0, + "split": "validation", + "trace": { + "case_id": "val_protected_yes_no", + "judge": { + "expectation_type": "exact", + "expected": "YES" + }, + "model": { + "case_id": "val_protected_yes_no", + "expectation_type": "exact", + "prompt_id": "candidate_001_overfit", + "prompt_mode": "overfit", + "seed": 91 + }, + "prompt_id": "candidate_001_overfit", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": false, + "prompt_id": "candidate_001_overfit", + "score": 0.333333, + "split": "validation" + } + }, + { + "candidate": { + "candidate_id": "candidate_002_safe", + "prompt": "You are a helpful support assistant.\n\nAnswer clearly and include a short explanation when it may help the user.\nIf the user asks for structured data, provide the information they need.\n\n# Optimizer patch\nOPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\nUse strict JSON only when the user explicitly asks for JSON.\nUse exact answers only when the user explicitly asks for an exact answer.\nOtherwise answer naturally and honor rubric constraints.\n", + "prompt_diff": "--- baseline_system_prompt.txt\n+++ candidate_002_safe/system_prompt.txt\n@@ -2,3 +2,9 @@\n\n Answer clearly and include a short explanation when it may help the user.\n If the user asks for structured data, provide the information they need.\n+\n+# Optimizer patch\n+OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED\n+Use strict JSON only when the user explicitly asks for JSON.\n+Use exact answers only when the user explicitly asks for an exact answer.\n+Otherwise answer naturally and honor rubric constraints.", + "rationale": "This candidate fixes observed strict-format failures without changing natural-language behavior on validation cases." + }, + "train_result": { + "cases": [ + { + "case_id": "train_json_refund", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "{\"intent\": \"refund\", \"priority\": \"high\"}", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_json_refund", + "judge": { + "expectation_type": "json", + "valid_json": true + }, + "model": { + "case_id": "train_json_refund", + "expectation_type": "json", + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "prompt_id": "candidate_002_safe", + "trace_mode": "fake" + } + }, + { + "case_id": "train_exact_order_status", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "final_response_mismatch", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "READY", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_exact_order_status", + "judge": { + "expectation_type": "exact", + "expected": "READY" + }, + "model": { + "case_id": "train_exact_order_status", + "expectation_type": "exact", + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "prompt_id": "candidate_002_safe", + "trace_mode": "fake" + } + }, + { + "case_id": "train_rubric_retry_summary", + "cost": 0.001, + "evidence": null, + "expected_failure_category": null, + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "latency retries", + "passed": true, + "score": 1.0, + "split": "train", + "trace": { + "case_id": "train_rubric_retry_summary", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "train_rubric_retry_summary", + "expectation_type": "rubric", + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "prompt_id": "candidate_002_safe", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": true, + "prompt_id": "candidate_002_safe", + "score": 1.0, + "split": "train" + }, + "validation_result": { + "cases": [ + { + "case_id": "val_json_invoice", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "{\"next_step\": \"email_customer\", \"status\": \"approved\"}", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_json_invoice", + "judge": { + "expectation_type": "json", + "valid_json": true + }, + "model": { + "case_id": "val_json_invoice", + "expectation_type": "json", + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "prompt_id": "candidate_002_safe", + "trace_mode": "fake" + } + }, + { + "case_id": "val_explain_cache", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "format_violation", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "cache stale data", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_explain_cache", + "judge": { + "expectation_type": "rubric" + }, + "model": { + "case_id": "val_explain_cache", + "expectation_type": "rubric", + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "prompt_id": "candidate_002_safe", + "trace_mode": "fake" + } + }, + { + "case_id": "val_protected_yes_no", + "cost": 0.001, + "evidence": null, + "expected_failure_category": "final_response_mismatch", + "failure_category": null, + "failure_reason": null, + "hard_failed": false, + "output": "YES", + "passed": true, + "score": 1.0, + "split": "validation", + "trace": { + "case_id": "val_protected_yes_no", + "judge": { + "expectation_type": "exact", + "expected": "YES" + }, + "model": { + "case_id": "val_protected_yes_no", + "expectation_type": "exact", + "prompt_id": "candidate_002_safe", + "prompt_mode": "safe", + "seed": 91 + }, + "prompt_id": "candidate_002_safe", + "trace_mode": "fake" + } + } + ], + "cost": 0.003, + "passed": true, + "prompt_id": "candidate_002_safe", + "score": 1.0, + "split": "validation" + } + } + ], + "delta": { + "per_case": [ + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_json_refund", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_exact_order_status", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_rubric_retry_summary", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_json_invoice", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": false, + "candidate_score": 0.0, + "case_id": "val_explain_cache", + "delta": -1.0, + "delta_type": "new_fail", + "regression": true, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": false, + "candidate_score": 0.0, + "case_id": "val_protected_yes_no", + "delta": -1.0, + "delta_type": "new_fail", + "regression": true, + "split": "validation" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_json_refund", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_exact_order_status", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_rubric_retry_summary", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_json_invoice", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_explain_cache", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_protected_yes_no", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "validation" + } + ] + }, + "failure_attribution_summary": { + "attribution_accuracy": 1.0, + "by_category": { + "final_response_mismatch": 2, + "format_violation": 3 + }, + "by_prompt_split": { + "baseline:train": { + "final_response_mismatch": 1, + "format_violation": 1 + }, + "baseline:validation": { + "format_violation": 1 + }, + "candidate_001_overfit:train": {}, + "candidate_001_overfit:validation": { + "final_response_mismatch": 1, + "format_violation": 1 + }, + "candidate_002_safe:train": {}, + "candidate_002_safe:validation": {} + }, + "examples": [ + { + "case_id": "train_json_refund", + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "prompt_id": "baseline", + "split": "train" + }, + { + "case_id": "train_exact_order_status", + "evidence": "expected normalized 'READY', got 'READY - confirmed.'", + "failure_category": "final_response_mismatch", + "failure_reason": "normalized exact answer mismatch", + "prompt_id": "baseline", + "split": "train" + }, + { + "case_id": "val_json_invoice", + "evidence": "json parser failed at char 0: Expecting value", + "failure_category": "format_violation", + "failure_reason": "output is not valid JSON", + "prompt_id": "baseline", + "split": "validation" + }, + { + "case_id": "val_explain_cache", + "evidence": "forbidden pattern '{' was present", + "failure_category": "format_violation", + "failure_reason": "output contains a forbidden pattern", + "prompt_id": "candidate_001_overfit", + "split": "validation" + }, + { + "case_id": "val_protected_yes_no", + "evidence": "expected normalized 'YES', got '{\"answer\": \"YES\"}'", + "failure_category": "final_response_mismatch", + "failure_reason": "normalized exact answer mismatch", + "prompt_id": "candidate_001_overfit", + "split": "validation" + } + ], + "expected_labeled_failures": 5, + "total_failed_cases": 5 + }, + "gate_decisions": [ + { + "accepted": false, + "candidate_cost": 0.006, + "candidate_id": "candidate_001_overfit", + "cost": 0.006, + "cumulative_cost": 0.006, + "excessive_score_drops": [ + "val_explain_cache", + "val_protected_yes_no" + ], + "gate_not_applied_reason": null, + "gate_status": "applied", + "new_hard_failures": [ + "val_explain_cache", + "val_protected_yes_no" + ], + "not_applied_checks": [], + "overfit_detected": true, + "protected_regressions": [ + "val_protected_yes_no" + ], + "reasons": [ + "reject: overfit detected because train score improved but validation score regressed or did not improve (+0.667 train, -0.333 validation)", + "reject: validation improvement -0.333 is below required +0.010", + "reject: new hard failures appeared: ['val_explain_cache', 'val_protected_yes_no']", + "reject: protected cases regressed: ['val_protected_yes_no']", + "reject: per-case validation score drops exceed 0.000: ['val_explain_cache', 'val_protected_yes_no']" + ], + "total_run_cost": 0.012, + "train_score_delta": 0.666667, + "validation_new_failures": [ + "val_explain_cache", + "val_protected_yes_no" + ], + "validation_score_delta": -0.333334 + }, + { + "accepted": true, + "candidate_cost": 0.006, + "candidate_id": "candidate_002_safe", + "cost": 0.006, + "cumulative_cost": 0.012, + "excessive_score_drops": [], + "gate_not_applied_reason": null, + "gate_status": "applied", + "new_hard_failures": [], + "not_applied_checks": [], + "overfit_detected": false, + "protected_regressions": [], + "reasons": [ + "accept: validation score improved +0.333 with no protected regression or new hard failure" + ], + "total_run_cost": 0.018, + "train_score_delta": 0.666667, + "validation_new_failures": [], + "validation_score_delta": 0.333333 + } + ], + "per_case_deltas": [ + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_json_refund", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_exact_order_status", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_rubric_retry_summary", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_json_invoice", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": false, + "candidate_score": 0.0, + "case_id": "val_explain_cache", + "delta": -1.0, + "delta_type": "new_fail", + "regression": true, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_001_overfit", + "candidate_passed": false, + "candidate_score": 0.0, + "case_id": "val_protected_yes_no", + "delta": -1.0, + "delta_type": "new_fail", + "regression": true, + "split": "validation" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_json_refund", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_exact_order_status", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "train" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "train_rubric_retry_summary", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "train" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_json_invoice", + "delta": 1.0, + "delta_type": "new_pass", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_explain_cache", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "validation" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_id": "candidate_002_safe", + "candidate_passed": true, + "candidate_score": 1.0, + "case_id": "val_protected_yes_no", + "delta": 0.0, + "delta_type": "unchanged", + "regression": false, + "split": "validation" + } + ], + "run": { + "fake_judge": true, + "fake_model": true, + "mode": "fake", + "paths": { + "optimizer": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\optimizer.json", + "prompt": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\prompts\\baseline_system_prompt.txt", + "train": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\train.evalset.json", + "validation": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\data\\val.evalset.json" + }, + "prompt_source": "C:\\Users\\Wu\\Documents\\trpc\\trpc-agent-issue-91\\examples\\optimization\\eval_optimize_loop\\prompts\\baseline_system_prompt.txt", + "reproducibility_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --output-dir /tmp/eval-optimize-loop --fake-model --fake-judge --trace", + "run_id": "eval_optimize_loop_seed_91", + "trace_enabled": true, + "train_cases": 3, + "update_source": false, + "validation_cases": 3 + }, + "schema_version": "eval_optimize_loop.v1", + "selected_candidate": "candidate_002_safe" +} diff --git a/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md new file mode 100644 index 00000000..510c7f65 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/outputs/optimization_report.example.md @@ -0,0 +1,102 @@ +# Evaluation + Optimization Report + +## Final Decision + +Selected candidate: `candidate_002_safe`. + +Update source prompt: no (default) + + +## Gate Reasons + +### candidate_001_overfit (rejected) +- reject: overfit detected because train score improved but validation score regressed or did not improve (+0.667 train, -0.333 validation) +- reject: validation improvement -0.333 is below required +0.010 +- reject: new hard failures appeared: ['val_explain_cache', 'val_protected_yes_no'] +- reject: protected cases regressed: ['val_protected_yes_no'] +- reject: per-case validation score drops exceed 0.000: ['val_explain_cache', 'val_protected_yes_no'] + +### candidate_002_safe (accepted) +- accept: validation score improved +0.333 with no protected regression or new hard failure + +## Baseline vs Candidate Scores + +| prompt | train score | validation score | gate | +| --- | ---: | ---: | --- | +| baseline | 0.333 | 0.667 | n/a | +| candidate_001_overfit | 1.000 | 0.333 | reject | +| candidate_002_safe | 1.000 | 1.000 | accept | + +## Per-Case Delta + +| candidate | split | case | baseline | candidate | delta | passed -> passed | delta type | +| --- | --- | --- | ---: | ---: | ---: | --- | --- | +| candidate_001_overfit | train | train_json_refund | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_001_overfit | train | train_exact_order_status | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_001_overfit | train | train_rubric_retry_summary | 1.000 | 1.000 | +0.000 | True -> True | unchanged | +| candidate_001_overfit | validation | val_json_invoice | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_001_overfit | validation | val_explain_cache | 1.000 | 0.000 | -1.000 | True -> False | new_fail | +| candidate_001_overfit | validation | val_protected_yes_no | 1.000 | 0.000 | -1.000 | True -> False | new_fail | +| candidate_002_safe | train | train_json_refund | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_002_safe | train | train_exact_order_status | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_002_safe | train | train_rubric_retry_summary | 1.000 | 1.000 | +0.000 | True -> True | unchanged | +| candidate_002_safe | validation | val_json_invoice | 0.000 | 1.000 | +1.000 | False -> True | new_pass | +| candidate_002_safe | validation | val_explain_cache | 1.000 | 1.000 | +0.000 | True -> True | unchanged | +| candidate_002_safe | validation | val_protected_yes_no | 1.000 | 1.000 | +0.000 | True -> True | unchanged | + +## Failure Attribution Summary + +Total failed case evaluations: 5 + +| category | count | +| --- | ---: | +| final_response_mismatch | 2 | +| format_violation | 3 | + +Attribution accuracy: 1.000 + +## Cost And Audit + +Total cost: 0.018 +Config hash: `66a1db3a1c84ad12fd41385fc5e1a9c23cc305e6973945c4bade559c804f9abe` +Run id: `eval_optimize_loop_seed_91` + +## Prompt Diff + +### candidate_001_overfit + +```diff +--- baseline_system_prompt.txt ++++ candidate_001_overfit/system_prompt.txt +@@ -2,3 +2,7 @@ + + Answer clearly and include a short explanation when it may help the user. + If the user asks for structured data, provide the information they need. ++ ++# Optimizer patch ++OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON ++Always force every final answer into JSON, even when the user asks for prose. +``` + +### candidate_002_safe + +```diff +--- baseline_system_prompt.txt ++++ candidate_002_safe/system_prompt.txt +@@ -2,3 +2,9 @@ + + Answer clearly and include a short explanation when it may help the user. + If the user asks for structured data, provide the information they need. ++ ++# Optimizer patch ++OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED ++Use strict JSON only when the user explicitly asks for JSON. ++Use exact answers only when the user explicitly asks for an exact answer. ++Otherwise answer naturally and honor rubric constraints. +``` + +## Reproducibility + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --train examples/optimization/eval_optimize_loop/data/train.evalset.json --val examples/optimization/eval_optimize_loop/data/val.evalset.json --optimizer-config examples/optimization/eval_optimize_loop/data/optimizer.json --prompt examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt --output-dir /tmp/eval-optimize-loop --fake-model --fake-judge --trace +``` diff --git a/examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt b/examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt new file mode 100644 index 00000000..7b3ed8aa --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/baseline_system_prompt.txt @@ -0,0 +1,4 @@ +You are a helpful support assistant. + +Answer clearly and include a short explanation when it may help the user. +If the user asks for structured data, provide the information they need. diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..9d83ef86 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,858 @@ +"""Run the deterministic Evaluation + Optimization closed-loop example.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +import shlex +import sys +import tempfile +from datetime import datetime +from datetime import timezone +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +from eval_loop.backends import FakeBackend +from eval_loop.backends import SDKBackend +from eval_loop.config import validate_inputs +from eval_loop.gate import AcceptanceGate +from eval_loop.loader import load_eval_cases +from eval_loop.loader import load_optimizer_config +from eval_loop.loader import load_prompt +from eval_loop.loader import sha256_file +from eval_loop.loader import stable_config_hash +from eval_loop.report import REPRODUCIBILITY_COMMAND +from eval_loop.report import build_report +from eval_loop.report import compute_case_deltas +from eval_loop.report import write_reports +from eval_loop.schemas import CandidatePrompt +from eval_loop.schemas import EvalResult +from eval_loop.schemas import GateDecision +from eval_loop.schemas import OptimizationReport + + +DEFAULT_TRAIN = HERE / "data" / "train.evalset.json" +DEFAULT_VAL = HERE / "data" / "val.evalset.json" +DEFAULT_OPTIMIZER_CONFIG = HERE / "data" / "optimizer.json" +DEFAULT_PROMPT = HERE / "prompts" / "baseline_system_prompt.txt" +DEFAULT_OUTPUT_DIR = Path(tempfile.gettempdir()) / "eval-optimize-loop" +TARGET_PROMPT_FIELD_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +RUN_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + + +def run_pipeline( + *, + train_path: str | Path = DEFAULT_TRAIN, + val_path: str | Path = DEFAULT_VAL, + optimizer_config_path: str | Path = DEFAULT_OPTIMIZER_CONFIG, + prompt_path: str | Path = DEFAULT_PROMPT, + output_dir: str | Path = DEFAULT_OUTPUT_DIR, + mode: str = "fake", + fake_model: bool = True, + fake_judge: bool = True, + trace: bool = False, + sdk_call_agent: str | None = None, + update_source: bool = False, + gate_config_path: str | Path | None = None, + target_prompts: list[str] | None = None, + run_id: str | None = None, +) -> OptimizationReport: + """Run baseline eval, fake optimization, validation gate, and reports.""" + + if mode not in {"fake", "sdk"}: + raise ValueError("field 'mode' must be one of: fake, sdk") + if run_id is not None: + run_id = validate_run_id(run_id) + if mode == "fake" and (not fake_model or not fake_judge): + raise ValueError( + "fake mode requires fake_model=True and fake_judge=True. Pass --fake-model --fake-judge " + "or use --mode sdk with --sdk-call-agent module:function." + ) + + if mode == "sdk": + optimizer_config_dict = _read_json_object_for_audit(optimizer_config_path) + baseline_prompt = load_prompt(prompt_path) + sdk_artifact_dir = Path(output_dir) / "sdk_optimizer" + wrapper_gate_config = _load_sdk_gate_config(gate_config_path) + target_prompt_paths = _parse_target_prompt_paths(target_prompts, default_prompt_path=prompt_path) + sdk_backend = SDKBackend( + prompt_path=prompt_path, + call_agent_path=sdk_call_agent, + update_source=update_source, + target_prompt_paths=target_prompt_paths, + ) + candidates = sdk_backend.optimize( + baseline_prompt=baseline_prompt, + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + output_dir=sdk_artifact_dir, + ) + report = _build_sdk_report( + candidates=candidates, + sdk_backend=sdk_backend, + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + output_dir=output_dir, + trace=trace, + update_source=update_source, + train_case_count=_count_cases(train_path), + validation_case_count=_count_cases(val_path), + optimizer_config_dict=optimizer_config_dict, + gate_config=wrapper_gate_config, + gate_config_path=gate_config_path, + target_prompt_paths=target_prompt_paths, + sdk_call_agent=sdk_call_agent, + run_id=run_id, + ) + if run_id is None: + _resolve_default_sdk_run_id_collision(report, output_dir) + write_reports(report, output_dir) + return report + + optimizer_config = load_optimizer_config(optimizer_config_path) + train_cases = load_eval_cases(train_path, split="train") + validation_cases = load_eval_cases(val_path, split="validation") + validate_inputs( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + train_cases=train_cases, + validation_cases=validation_cases, + config=optimizer_config, + ) + + seed = optimizer_config.seed + baseline_prompt = load_prompt(prompt_path) + backend = FakeBackend(seed=seed, trace_enabled=trace) + + baseline = CandidatePrompt( + candidate_id="baseline", + prompt=baseline_prompt, + rationale="Prompt source file before optimization.", + prompt_diff="", + ) + baseline_train = backend.evaluate( + prompt_id=baseline.candidate_id, + prompt=baseline.prompt, + cases=train_cases, + split="train", + ) + baseline_validation = backend.evaluate( + prompt_id=baseline.candidate_id, + prompt=baseline.prompt, + cases=validation_cases, + split="validation", + ) + + candidates = backend.optimize( + baseline_prompt=baseline_prompt, + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + output_dir=output_dir, + ) + gate = AcceptanceGate(optimizer_config.gate.to_dict()) + + candidate_records: list[dict[str, Any]] = [] + all_deltas = [] + gate_decisions = [] + cumulative_cost = round(baseline_train.cost + baseline_validation.cost, 6) + for candidate in candidates: + train_result = backend.evaluate( + prompt_id=candidate.candidate_id, + prompt=candidate.prompt, + cases=train_cases, + split="train", + ) + validation_result = backend.evaluate( + prompt_id=candidate.candidate_id, + prompt=candidate.prompt, + cases=validation_cases, + split="validation", + ) + deltas = compute_case_deltas( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + ) + decision = gate.decide( + candidate_id=candidate.candidate_id, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_train=train_result, + candidate_validation=validation_result, + deltas=deltas, + cumulative_cost=cumulative_cost, + ) + candidate_records.append({ + "candidate": candidate, + "train_result": train_result, + "validation_result": validation_result, + }) + all_deltas.extend(deltas) + gate_decisions.append(decision) + cumulative_cost = decision.total_run_cost + + selected_candidate = _select_candidate(candidate_records, gate_decisions) + input_hashes = _input_hashes( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + ) + audit = _build_audit( + seed=seed, + config_hash=stable_config_hash(optimizer_config.to_dict()), + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_records=candidate_records, + candidates=candidates, + input_hashes=input_hashes, + input_paths={ + "train": str(train_path), + "validation": str(val_path), + "optimizer": str(optimizer_config_path), + "prompt": str(prompt_path), + }, + ) + run = { + "run_id": f"eval_optimize_loop_seed_{seed}", + "mode": mode, + "fake_model": fake_model, + "fake_judge": fake_judge, + "trace_enabled": trace, + "train_cases": len(train_cases), + "validation_cases": len(validation_cases), + "update_source": update_source, + "reproducibility_command": REPRODUCIBILITY_COMMAND, + "paths": { + "train": str(train_path), + "validation": str(val_path), + "optimizer": str(optimizer_config_path), + "prompt": str(prompt_path), + }, + "prompt_source": str(prompt_path), + } + report = build_report( + run=run, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_records=candidate_records, + per_case_deltas=all_deltas, + gate_decisions=gate_decisions, + selected_candidate=selected_candidate, + audit=audit, + ) + write_reports(report, output_dir) + return report + + +def _select_candidate(candidate_records: list[dict[str, Any]], gate_decisions: list) -> str | None: + decisions_by_id = {decision.candidate_id: decision for decision in gate_decisions} + accepted = [] + for index, record in enumerate(candidate_records): + candidate = record["candidate"] + decision = decisions_by_id[candidate.candidate_id] + if decision.accepted: + accepted.append((index, record)) + if not accepted: + return None + index, record = max( + accepted, + key=lambda item: ( + item[1]["validation_result"].score, + item[1]["train_result"].score, + -item[0], + ), + ) + return record["candidate"].candidate_id + + +def _build_audit( + *, + seed: int, + config_hash: str, + baseline_train, + baseline_validation, + candidate_records: list[dict[str, Any]], + candidates: list[CandidatePrompt], + input_hashes: dict[str, str], + input_paths: dict[str, str], +) -> dict[str, Any]: + baseline_cost = round(baseline_train.cost + baseline_validation.cost, 6) + candidate_costs = { + record["candidate"].candidate_id: round(record["train_result"].cost + record["validation_result"].cost, 6) + for record in candidate_records + } + total_cost = round(baseline_cost + sum(candidate_costs.values()), 6) + candidate_prompt_hashes = { + candidate.candidate_id: hashlib.sha256(candidate.prompt.encode("utf-8")).hexdigest() + for candidate in candidates + } + return { + "seed": seed, + "duration_seconds": 0.0, + "config_hash": config_hash, + "input_hashes": input_hashes, + "input_paths": input_paths, + "prompt_hash": input_hashes["prompt"], + "candidate_prompt_hashes": candidate_prompt_hashes, + "total_run_cost": total_cost, + "cost": { + "baseline": baseline_cost, + "candidates": candidate_costs, + "total": total_cost, + }, + "candidate_prompts": { + candidate.candidate_id: { + "rationale": candidate.rationale, + "prompt": candidate.prompt, + "prompt_diff": candidate.prompt_diff, + } + for candidate in candidates + }, + "prompt_diffs": { + candidate.candidate_id: candidate.prompt_diff + for candidate in candidates + }, + "reproducibility_command": REPRODUCIBILITY_COMMAND, + } + + +def _build_sdk_report( + *, + candidates: list[CandidatePrompt], + sdk_backend: SDKBackend, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + prompt_path: str | Path, + output_dir: str | Path, + trace: bool, + update_source: bool, + train_case_count: int | None, + validation_case_count: int | None, + optimizer_config_dict: dict[str, Any], + gate_config: dict[str, float], + gate_config_path: str | Path | None, + target_prompt_paths: dict[str, str | Path], + sdk_call_agent: str | None, + run_id: str | None, +) -> OptimizationReport: + input_hashes = _input_hashes( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + ) + sdk_summary = sdk_backend.last_result_summary or {} + baseline_pass_rate = _summary_float(sdk_summary, "baseline_pass_rate", 0.0, required=True) + best_pass_rate = _summary_float(sdk_summary, "best_pass_rate", baseline_pass_rate, required=True) + pass_rate_improvement = _summary_float( + sdk_summary, + "pass_rate_improvement", + best_pass_rate - baseline_pass_rate, + required=True, + ) + total_llm_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0, required=True) + duration_seconds = _summary_float(sdk_summary, "duration_seconds", 0.0) + effective_run_id = run_id or _default_sdk_run_id(sdk_summary) + target_prompt_hashes = { + name: sha256_file(path) + for name, path in target_prompt_paths.items() + } + input_hashes["target_prompts"] = target_prompt_hashes + if gate_config_path: + input_hashes["gate_config"] = sha256_file(gate_config_path) + availability = { + "aggregate_validation_result": True, + "full_train_eval_result": False, + "full_per_case_validation_delta": False, + } + score_explanation = ( + "SDK mode uses OptimizeResult aggregate validation metrics. " + "The full train EvalResult compatibility field is unavailable and keeps score 0.0; " + "full per-case validation deltas are unavailable and listed in not_applied_checks." + ) + + baseline_train = EvalResult(prompt_id="baseline", split="train", score=0.0, passed=False, cost=0.0, cases=[]) + baseline_validation = EvalResult( + prompt_id="baseline", + split="validation", + score=baseline_pass_rate, + passed=baseline_pass_rate >= 1.0, + cost=0.0, + cases=[], + ) + candidate_records = [ + { + "candidate": candidate, + "train_result": EvalResult( + prompt_id=candidate.candidate_id, + split="train", + score=0.0, + passed=False, + cost=0.0, + cases=[], + ), + "validation_result": EvalResult( + prompt_id=candidate.candidate_id, + split="validation", + score=best_pass_rate, + passed=best_pass_rate >= baseline_pass_rate, + cost=total_llm_cost, + cases=[], + ), + "gate_status": "partial_applied", + "gate_not_applied_reason": "SDK OptimizeResult exposes aggregate scores but not full per-case deltas", + "sdk_result_summary": sdk_summary, + } + for candidate in candidates + ] + prompt_hashes = { + candidate.candidate_id: hashlib.sha256(candidate.prompt.encode("utf-8")).hexdigest() + for candidate in candidates + } + field_prompt_hashes = _candidate_prompt_hashes_by_field(candidates, sdk_summary) + audit = { + "seed": None, + "duration_seconds": duration_seconds, + "config_hash": stable_config_hash(optimizer_config_dict), + "input_hashes": input_hashes, + "input_paths": { + "train": str(train_path), + "validation": str(val_path), + "optimizer": str(optimizer_config_path), + "prompt": str(prompt_path), + }, + "prompt_hash": input_hashes["prompt"], + "candidate_prompt_hashes": prompt_hashes, + "candidate_prompt_hashes_by_field": field_prompt_hashes, + "target_prompt_hashes": target_prompt_hashes, + "sdk_result_availability": availability, + "sdk_score_explanation": score_explanation, + "wrapper_gate_config": dict(gate_config), + "wrapper_gate_config_path": str(gate_config_path) if gate_config_path else None, + "total_run_cost": total_llm_cost, + "cost": { + "baseline": 0.0, + "candidates": {candidate.candidate_id: total_llm_cost for candidate in candidates}, + "total": total_llm_cost, + }, + "candidate_prompts": { + candidate.candidate_id: { + "rationale": candidate.rationale, + "prompt": candidate.prompt, + "prompt_diff": candidate.prompt_diff, + } + for candidate in candidates + }, + "prompt_diffs": {candidate.candidate_id: candidate.prompt_diff for candidate in candidates}, + "sdk_artifact_dir": sdk_backend.last_artifact_dir or str(Path(output_dir) / "sdk_optimizer"), + "sdk_result_summary": sdk_summary, + "reproducibility_command": _sdk_reproducibility_command( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_config_path, + prompt_path=prompt_path, + output_dir=output_dir, + update_source=update_source, + gate_config_path=gate_config_path, + target_prompt_paths=target_prompt_paths, + sdk_call_agent=sdk_call_agent, + run_id=run_id, + ), + } + run = { + "run_id": effective_run_id, + "mode": "sdk", + "fake_model": False, + "fake_judge": False, + "trace_enabled": trace, + "train_cases": train_case_count, + "validation_cases": validation_case_count, + "update_source": update_source, + "sdk_artifact_dir": audit["sdk_artifact_dir"], + "sdk_availability": availability, + "wrapper_gate_config_path": str(gate_config_path) if gate_config_path else None, + "reproducibility_command": audit["reproducibility_command"], + "paths": { + "train": str(train_path), + "validation": str(val_path), + "optimizer": str(optimizer_config_path), + "prompt": str(prompt_path), + }, + "target_prompts": {name: str(path) for name, path in target_prompt_paths.items()}, + "prompt_source": str(prompt_path), + } + gate_decisions = [ + _sdk_gate_decision( + candidate_id=candidate.candidate_id, + sdk_summary=sdk_summary, + gate_config=gate_config, + ) + for candidate in candidates + ] + selected_candidate = None + if candidates and gate_decisions and gate_decisions[0].accepted: + selected_candidate = candidates[0].candidate_id + return build_report( + run=run, + baseline_train=baseline_train, + baseline_validation=baseline_validation, + candidate_records=candidate_records, + per_case_deltas=[], + gate_decisions=gate_decisions, + selected_candidate=selected_candidate, + audit=audit, + ) + + +def _sdk_reproducibility_command( + *, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + prompt_path: str | Path, + output_dir: str | Path, + update_source: bool, + gate_config_path: str | Path | None, + target_prompt_paths: dict[str, str | Path], + sdk_call_agent: str | None, + run_id: str | None, +) -> str: + parts = [ + "python", + "examples/optimization/eval_optimize_loop/run_pipeline.py", + "--mode", + "sdk", + "--train", + str(train_path), + "--val", + str(val_path), + "--optimizer-config", + str(optimizer_config_path), + "--prompt", + str(prompt_path), + "--output-dir", + str(output_dir), + "--sdk-call-agent", + sdk_call_agent or "", + ] + for name, path in target_prompt_paths.items(): + if not (name == "system_prompt" and Path(path) == Path(prompt_path) and len(target_prompt_paths) == 1): + parts.extend(["--target-prompt", f"{name}={path}"]) + if gate_config_path: + parts.extend(["--gate-config", str(gate_config_path)]) + if run_id: + parts.extend(["--run-id", run_id]) + if update_source: + parts.append("--update-source") + return " ".join(shlex.quote(part) for part in parts) + + +def _sdk_gate_decision( + *, + candidate_id: str, + sdk_summary: dict[str, Any], + gate_config: dict[str, float], +) -> GateDecision: + status = str(sdk_summary.get("status") or "UNKNOWN") + improvement = _summary_float(sdk_summary, "pass_rate_improvement", 0.0, required=True) + total_cost = _summary_float(sdk_summary, "total_llm_cost", 0.0, required=True) + min_improvement = gate_config["min_val_score_improvement"] + max_cost = gate_config["max_total_cost"] + + reasons: list[str] = [] + accepted = True + if status != "SUCCEEDED": + accepted = False + reasons.append(f"reject: SDK optimizer status {status} is not SUCCEEDED") + else: + reasons.append("accept: SDK optimizer status is SUCCEEDED") + + if improvement < min_improvement: + accepted = False + reasons.append( + f"reject: validation improvement {improvement:.3f} is below threshold {min_improvement:.3f}" + ) + else: + reasons.append( + f"accept: validation improvement {improvement:.3f} meets threshold {min_improvement:.3f}" + ) + + if total_cost > max_cost: + accepted = False + reasons.append(f"reject: total SDK cost {total_cost:.3f} exceeds budget {max_cost:.3f}") + else: + reasons.append(f"accept: total SDK cost {total_cost:.3f} is within budget {max_cost:.3f}") + + if accepted: + reasons.append("accept: SDK aggregate gate passed") + + return GateDecision( + candidate_id=candidate_id, + accepted=accepted, + reasons=reasons, + train_score_delta=0.0, + validation_score_delta=improvement, + new_hard_failures=[], + protected_regressions=[], + validation_new_failures=[], + excessive_score_drops=[], + overfit_detected=False, + candidate_cost=total_cost, + cumulative_cost=0.0, + total_run_cost=total_cost, + cost=total_cost, + gate_status="partial_applied", + gate_not_applied_reason="SDK OptimizeResult exposes aggregate scores but not full per-case deltas", + not_applied_checks=[ + "per_case_delta", + "protected_regression", + "new_hard_failure", + "max_score_drop_per_case", + ], + ) + + +def _load_sdk_gate_config(gate_config_path: str | Path | None) -> dict[str, float]: + if gate_config_path is None: + gate_payload: dict[str, Any] = {} + path_text = "--gate-config" + else: + payload = _read_json_object_for_audit(gate_config_path) + gate_payload = payload.get("gate", payload) + path_text = str(gate_config_path) + if gate_payload is None: + gate_payload = {} + if not isinstance(gate_payload, dict): + raise ValueError(f"{path_text}: field 'gate' must be an object when present") + + min_improvement = gate_payload.get("min_val_score_improvement", 0.01) + max_cost = gate_payload.get("max_total_cost", 1.0) + if not _is_non_negative_finite_number(min_improvement): + raise ValueError( + f"--gate-config {path_text}: field 'gate.min_val_score_improvement' " + "must be a non-negative finite number" + ) + if not _is_non_negative_finite_number(max_cost): + raise ValueError( + f"--gate-config {path_text}: field 'gate.max_total_cost' must be a non-negative finite number" + ) + return { + "min_val_score_improvement": float(min_improvement), + "max_total_cost": float(max_cost), + } + + +def _is_non_negative_finite_number(value: Any) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(float(value)) + and float(value) >= 0 + ) + + +def _summary_float(summary: dict[str, Any], key: str, default: float, *, required: bool = False) -> float: + value = summary.get(key, default) + if value is None: + return default + if isinstance(value, bool): + if required: + raise ValueError(f"SDK OptimizeResult field {key} must be a finite number") + return default + try: + parsed = float(value) + except (TypeError, ValueError): + if required: + raise ValueError(f"SDK OptimizeResult field {key} must be a finite number") + return default + if not math.isfinite(parsed): + if required: + raise ValueError(f"SDK OptimizeResult field {key} must be a finite number") + return default + return parsed + + +def _default_sdk_run_id(sdk_summary: dict[str, Any]) -> str: + started_at = sdk_summary.get("started_at") + if isinstance(started_at, str) and started_at.strip(): + source = started_at.strip() + try: + normalized = source[:-1] + "+00:00" if source.endswith("Z") else source + parsed = datetime.fromisoformat(normalized) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return "eval_optimize_loop_sdk_" + parsed.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + except ValueError: + pass + else: + return "eval_optimize_loop_sdk_" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + safe = [] + for char in source: + if char.isalnum() or char in {"-", "_"}: + safe.append(char) + else: + safe.append("-") + return "eval_optimize_loop_sdk_" + "".join(safe).strip("-") + + +def _parse_target_prompt_paths( + target_prompts: list[str] | None, + *, + default_prompt_path: str | Path, +) -> dict[str, str | Path]: + if not target_prompts: + return {"system_prompt": default_prompt_path} + parsed: dict[str, str | Path] = {} + for item in target_prompts: + if "=" not in item: + raise ValueError("--target-prompt must use name=path format") + name, path = item.split("=", 1) + path = path.strip() + if not TARGET_PROMPT_FIELD_RE.fullmatch(name): + raise ValueError( + f"--target-prompt field name {name!r} is invalid; use /^[A-Za-z_][A-Za-z0-9_]*$/" + ) + if not path: + raise ValueError("--target-prompt must use non-empty name=path values") + if name in parsed: + raise ValueError(f"--target-prompt duplicate field name {name!r}") + parsed[name] = Path(path) + return parsed + + +def validate_run_id(run_id: str) -> str: + if not isinstance(run_id, str): + raise ValueError(f"--run-id value {run_id!r} must be a string") + if run_id in {"", ".", ".."} or not RUN_ID_RE.fullmatch(run_id): + raise ValueError(f"--run-id value {run_id!r} is invalid") + return run_id + + +def _resolve_default_sdk_run_id_collision(report: OptimizationReport, output_dir: str | Path) -> None: + base_run_id = str(report.run.get("run_id") or "") + validate_run_id(base_run_id) + run_root = Path(output_dir) / "runs" + candidate = base_run_id + suffix = 1 + while (run_root / candidate).exists(): + candidate = f"{base_run_id}-{suffix}" + suffix += 1 + report.run["run_id"] = candidate + + +def _candidate_prompt_hashes_by_field( + candidates: list[CandidatePrompt], + sdk_summary: dict[str, Any], +) -> dict[str, dict[str, str]]: + best_prompts = sdk_summary.get("best_prompts") + if not isinstance(best_prompts, dict): + return {} + return { + candidate.candidate_id: { + str(name): hashlib.sha256(str(prompt).encode("utf-8")).hexdigest() + for name, prompt in best_prompts.items() + } + for candidate in candidates + } + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--train", default=str(DEFAULT_TRAIN), help="Path to train.evalset.json") + parser.add_argument("--val", default=str(DEFAULT_VAL), help="Path to val.evalset.json") + parser.add_argument("--optimizer-config", default=str(DEFAULT_OPTIMIZER_CONFIG), help="Path to optimizer.json") + parser.add_argument("--prompt", default=str(DEFAULT_PROMPT), help="Path to baseline system prompt") + parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR), help="Directory for runtime reports") + parser.add_argument("--mode", choices=("fake", "sdk"), default="fake", help="Backend mode") + parser.add_argument("--fake-model", action="store_true", help="Use deterministic fake model") + parser.add_argument("--fake-judge", action="store_true", help="Use deterministic fake judge") + parser.add_argument("--trace", action="store_true", help="Persist fake model/judge trace details per case") + parser.add_argument("--sdk-call-agent", help="Async call_agent target for SDK mode, as module:function") + parser.add_argument("--update-source", action="store_true", help="Allow SDK optimizer to write back source prompt") + parser.add_argument("--gate-config", help="Wrapper gate config for SDK mode; separate from SDK optimizer config") + parser.add_argument( + "--target-prompt", + action="append", + help="SDK target prompt path in name=path format. May be repeated. Defaults to system_prompt=--prompt.", + ) + parser.add_argument("--run-id", help="Optional report/audit run id. Fake mode keeps its deterministic default.") + return parser.parse_args(argv) + + +def _input_hashes( + *, + train_path: str | Path, + val_path: str | Path, + optimizer_config_path: str | Path, + prompt_path: str | Path, +) -> dict[str, str]: + return { + "train": sha256_file(train_path), + "validation": sha256_file(val_path), + "optimizer": sha256_file(optimizer_config_path), + "prompt": sha256_file(prompt_path), + } + + +def _count_cases(path: str | Path) -> int | None: + try: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + except Exception: + return None + for key in ("cases", "eval_cases"): + cases = payload.get(key) if isinstance(payload, dict) else None + if isinstance(cases, list): + return len(cases) + return None + + +def _read_json_object_for_audit(path: str | Path) -> dict[str, Any]: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"{path}: optimizer config must be a JSON object") + return payload + + +def main(argv: list[str] | None = None) -> OptimizationReport: + args = parse_args(argv) + report = run_pipeline( + train_path=args.train, + val_path=args.val, + optimizer_config_path=args.optimizer_config, + prompt_path=args.prompt, + output_dir=args.output_dir, + mode=args.mode, + fake_model=args.fake_model or args.mode == "fake", + fake_judge=args.fake_judge or args.mode == "fake", + trace=args.trace, + sdk_call_agent=args.sdk_call_agent, + update_source=args.update_source, + gate_config_path=args.gate_config, + target_prompts=args.target_prompt, + run_id=args.run_id, + ) + output_dir = Path(args.output_dir) + print(f"Wrote {output_dir / 'optimization_report.json'}") + print(f"Wrote {output_dir / 'optimization_report.md'}") + print(f"Selected candidate: {report.selected_candidate}") + return report + + +if __name__ == "__main__": + main() diff --git a/examples/optimization/eval_optimize_loop/tests/test_config_validation.py b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py new file mode 100644 index 00000000..1ce4825f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_config_validation.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop.config import parse_optimizer_config +from examples.optimization.eval_optimize_loop.eval_loop.config import validate_inputs +from examples.optimization.eval_optimize_loop.eval_loop.loader import load_eval_cases +from examples.optimization.eval_optimize_loop.eval_loop.loader import load_optimizer_config + + +def test_optimizer_config_defaults_metrics_and_gate(tmp_path: Path): + path = tmp_path / "optimizer.json" + path.write_text(json.dumps({"seed": 7}), encoding="utf-8") + + config = load_optimizer_config(path) + + assert config.seed == 7 + assert config.metrics == {} + assert config.gate.min_val_score_improvement == 0.01 + + +def test_optimizer_config_rejects_bad_gate_type(tmp_path: Path): + path = tmp_path / "optimizer.json" + payload = {"gate": {"allow_new_hard_fail": "no"}} + + with pytest.raises(ValueError, match="gate.allow_new_hard_fail"): + parse_optimizer_config(payload, path=path) + + +def test_optimizer_config_rejects_negative_cost(tmp_path: Path): + path = tmp_path / "optimizer.json" + payload = {"gate": {"max_total_cost": -1}} + + with pytest.raises(ValueError, match="gate.max_total_cost"): + parse_optimizer_config(payload, path=path) + + +def test_validate_inputs_rejects_same_train_val_path(tmp_path: Path): + eval_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "b", "c"]) + config = parse_optimizer_config({"gate": {}}, path=tmp_path / "optimizer.json") + cases = load_eval_cases(eval_path, split="train") + + with pytest.raises(ValueError, match="must be different"): + validate_inputs( + train_path=eval_path, + val_path=eval_path, + optimizer_config_path=tmp_path / "optimizer.json", + train_cases=cases, + validation_cases=cases, + config=config, + ) + + +def test_validate_inputs_rejects_duplicate_case_ids(tmp_path: Path): + train_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "a", "c"]) + val_path = _write_evalset(tmp_path / "val.evalset.json", "validation", ["v1", "v2", "v3"]) + config = parse_optimizer_config({"gate": {}}, path=tmp_path / "optimizer.json") + + with pytest.raises(ValueError, match="duplicate case_id"): + validate_inputs( + train_path=train_path, + val_path=val_path, + optimizer_config_path=tmp_path / "optimizer.json", + train_cases=load_eval_cases(train_path, split="train"), + validation_cases=load_eval_cases(val_path, split="validation"), + config=config, + ) + + +def test_validate_inputs_rejects_missing_protected_case(tmp_path: Path): + train_path = _write_evalset(tmp_path / "train.evalset.json", "train", ["a", "b", "c"]) + val_path = _write_evalset(tmp_path / "val.evalset.json", "validation", ["v1", "v2", "v3"]) + config = parse_optimizer_config( + {"gate": {"protected_case_ids": ["missing"]}}, + path=tmp_path / "optimizer.json", + ) + + with pytest.raises(ValueError, match="gate.protected_case_ids"): + validate_inputs( + train_path=train_path, + val_path=val_path, + optimizer_config_path=tmp_path / "optimizer.json", + train_cases=load_eval_cases(train_path, split="train"), + validation_cases=load_eval_cases(val_path, split="validation"), + config=config, + ) + + +def _write_evalset(path: Path, split: str, ids: list[str]) -> Path: + payload = { + "split": split, + "cases": [ + { + "id": case_id, + "input": "Return JSON", + "expectation": {"type": "json", "required_keys": ["answer"], "expected_values": {"answer": case_id}}, + } + for case_id in ids + ], + } + path.write_text(json.dumps(payload), encoding="utf-8") + return path diff --git a/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py b/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py new file mode 100644 index 00000000..60cbeaf5 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from examples.optimization.eval_optimize_loop.eval_loop.attribution import attribute_failure +from examples.optimization.eval_optimize_loop.eval_loop.attribution import summarize_failures +from examples.optimization.eval_optimize_loop.eval_loop.evaluator import ExampleEvaluator +from examples.optimization.eval_optimize_loop.eval_loop.fake_judge import FakeJudge +from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase + + +def test_failure_attribution_labels_format_violation_and_exact_mismatch(): + assert attribute_failure("json_parse_failure", "bad json")[0] == "format_violation" + assert attribute_failure("exact_answer_mismatch", "wrong answer")[0] == "final_response_mismatch" + + +def test_evaluator_attaches_failure_details_to_failed_cases(): + cases = [ + EvalCase( + case_id="json_case", + split="train", + input="Return JSON.", + expectation={ + "type": "json", + "required_keys": ["answer"], + "expected_values": {"answer": "ok"}, + }, + ), + EvalCase( + case_id="exact_case", + split="train", + input="Answer exactly OK.", + expectation={"type": "exact", "expected": "OK"}, + ), + ] + result = ExampleEvaluator(FakeModel(seed=91), FakeJudge()).evaluate( + prompt_id="baseline", + prompt="baseline prompt", + cases=cases, + split="train", + ) + + by_id = result.by_case_id() + assert by_id["json_case"].failure_category == "format_violation" + assert by_id["json_case"].failure_reason + assert by_id["json_case"].evidence + assert by_id["exact_case"].failure_category == "final_response_mismatch" + assert by_id["exact_case"].failure_reason + assert by_id["exact_case"].evidence + + +def test_failure_attribution_labels_tool_parameter_and_knowledge_failures(): + assert attribute_failure("tool_call_error", "wrong tool")[0] == "tool_call_error" + assert attribute_failure("parameter_error", "wrong arg")[0] == "parameter_error" + assert attribute_failure("knowledge_recall_insufficient", "missing source")[0] == "knowledge_recall_insufficient" + + +def test_fake_judge_scores_tool_and_parameter_expectations(): + judge = FakeJudge() + tool_case = EvalCase( + case_id="tool_case", + split="train", + input="Call tool", + expectation={"type": "tool", "expected_tool": "lookup", "expected_args": {"id": "42"}}, + ) + + assert judge.score(tool_case, '{"tool": "lookup", "args": {"id": "42"}}').passed + wrong_tool = judge.score(tool_case, '{"tool": "search", "args": {"id": "42"}}') + wrong_arg = judge.score(tool_case, '{"tool": "lookup", "args": {"id": "43"}}') + assert wrong_tool.error_code == "tool_call_error" + assert wrong_arg.error_code == "parameter_error" + + +def test_fake_judge_scores_knowledge_expectations(): + case = EvalCase( + case_id="knowledge_case", + split="train", + input="Recall source", + expectation={ + "type": "knowledge", + "required_sources": ["doc-a"], + "must_include_knowledge_terms": ["refund"], + }, + ) + + assert FakeJudge().score(case, "refund policy appears in doc-a").passed + failed = FakeJudge().score(case, "refund policy only") + assert failed.error_code == "knowledge_recall_insufficient" + + +def test_failure_summary_computes_attribution_accuracy(): + result = ExampleEvaluator(FakeModel(seed=91), FakeJudge()).evaluate( + prompt_id="baseline", + prompt="baseline prompt", + split="train", + cases=[ + EvalCase( + case_id="json_labeled", + split="train", + input="Return JSON", + expectation={"type": "json", "expected_values": {"answer": "ok"}}, + expected_failure_category="format_violation", + ) + ], + ) + + summary = summarize_failures([result]) + assert summary["expected_labeled_failures"] == 1 + assert summary["attribution_accuracy"] == 1.0 diff --git a/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py b/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py new file mode 100644 index 00000000..5bd26f9a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_fake_model_generalization.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json + +from examples.optimization.eval_optimize_loop.eval_loop.fake_judge import FakeJudge +from examples.optimization.eval_optimize_loop.eval_loop.fake_model import FakeModel +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalCase + + +def test_fake_model_json_behavior_does_not_depend_on_case_id(): + case = EvalCase( + case_id="hidden_json_case", + split="validation", + input="Return JSON.", + expectation={"type": "json", "expected_values": {"status": "ok"}, "required_keys": ["status"]}, + ) + model = FakeModel(seed=91) + + baseline, _, _ = model.generate("baseline", "plain prompt", case) + safe, _, _ = model.generate("safe", "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED", case) + + assert baseline.startswith("Here is the JSON") + assert json.loads(safe) == {"status": "ok"} + assert FakeJudge().score(case, baseline).passed is False + assert FakeJudge().score(case, safe).passed is True + + +def test_fake_model_protected_exact_baseline_passes_but_overfit_regresses(): + case = EvalCase( + case_id="hidden_protected_case", + split="validation", + input="Answer exactly YES.", + expectation={"type": "exact", "expected": "YES"}, + protected=True, + ) + model = FakeModel(seed=91) + + baseline, _, _ = model.generate("baseline", "plain prompt", case) + overfit, _, _ = model.generate("overfit", "OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON", case) + safe, _, _ = model.generate("safe", "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED", case) + + assert baseline == "YES" + assert overfit == '{"answer": "YES"}' + assert safe == "YES" + + +def test_fake_model_rubric_overfit_forces_json_on_validation_prose(): + case = EvalCase( + case_id="hidden_rubric_case", + split="validation", + input="Explain in prose.", + expectation={ + "type": "rubric", + "must_include": ["cache", "stale data"], + "forbidden": ["{", "}", "json"], + "max_chars": 120, + }, + tags=["prose"], + ) + + output, _, _ = FakeModel(seed=91).generate("overfit", "OPTIMIZER_MARKER: ALWAYS_OUTPUT_JSON", case) + assert output.startswith("{") + judged = FakeJudge().score(case, output) + assert judged.error_code == "forbidden_pattern" + + +def test_fake_model_respects_simulated_outputs_override(): + case = EvalCase( + case_id="override_case", + split="train", + input="Any", + expectation={"type": "exact", "expected": "OK"}, + simulated_outputs={"safe": "CUSTOM"}, + ) + + output, _, _ = FakeModel(seed=91).generate("safe", "OPTIMIZER_MARKER: STRICT_WHEN_REQUESTED", case) + assert output == "CUSTOM" diff --git a/examples/optimization/eval_optimize_loop/tests/test_gate.py b/examples/optimization/eval_optimize_loop/tests/test_gate.py new file mode 100644 index 00000000..fc112caf --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_gate.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +from examples.optimization.eval_optimize_loop.eval_loop.gate import AcceptanceGate +from examples.optimization.eval_optimize_loop.eval_loop.report import compute_case_deltas +from examples.optimization.eval_optimize_loop.eval_loop.schemas import CaseResult +from examples.optimization.eval_optimize_loop.eval_loop.schemas import EvalResult + + +def test_gate_rejects_train_improvement_with_validation_regression(): + baseline_train = _eval("baseline", "train", [("train_a", 0.0), ("train_b", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0), ("train_b", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 1.0), ("val_b", 1.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 1.0), ("val_b", 0.0)]) + + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + decision = AcceptanceGate({"protected_case_ids": []}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + ) + + assert not decision.accepted + assert any("train score improved but validation score regressed" in reason for reason in decision.reasons) + assert decision.overfit_detected + + +def test_gate_rejects_protected_case_regression(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("protected", 1.0), ("val_a", 0.0), ("val_b", 0.0)]) + candidate_val = _eval("candidate", "validation", [("protected", 0.0), ("val_a", 1.0), ("val_b", 1.0)]) + + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + decision = AcceptanceGate({"protected_case_ids": ["protected"]}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + ) + + assert not decision.accepted + assert decision.protected_regressions == ["protected"] + assert any("protected cases regressed" in reason for reason in decision.reasons) + + +def test_gate_accepts_safe_candidate(): + baseline_train = _eval("baseline", "train", [("train_json", 0.0), ("train_exact", 0.0), ("train_ok", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_json", 1.0), ("train_exact", 1.0), ("train_ok", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_json", 0.0), ("val_text", 1.0), ("protected", 1.0)]) + candidate_val = _eval("candidate", "validation", [("val_json", 1.0), ("val_text", 1.0), ("protected", 1.0)]) + + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + decision = AcceptanceGate({"protected_case_ids": ["protected"]}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + ) + + assert decision.accepted + assert decision.protected_regressions == [] + assert decision.new_hard_failures == [] + + +def test_gate_rejects_new_hard_failure_when_not_allowed(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 1.0), ("val_b", 0.0), ("val_c", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 0.0), ("val_b", 1.0), ("val_c", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({"allow_new_hard_fail": False}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + ) + + assert not decision.accepted + assert decision.new_hard_failures == ["val_a"] + assert decision.validation_new_failures == ["val_a"] + + +def test_gate_rejects_excessive_score_drop(): + baseline_train = _eval("baseline", "train", [("train_a", 1.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 1.0), ("val_b", 0.0), ("val_c", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 0.4), ("val_b", 1.0), ("val_c", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({"max_score_drop_per_case": 0.5, "allow_new_hard_fail": True}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + ) + + assert not decision.accepted + assert decision.excessive_score_drops == ["val_a"] + + +def test_gate_rejects_cost_budget(): + baseline_train = _eval("baseline", "train", [("train_a", 0.0)]) + candidate_train = _eval("candidate", "train", [("train_a", 1.0)]) + baseline_val = _eval("baseline", "validation", [("val_a", 0.0)]) + candidate_val = _eval("candidate", "validation", [("val_a", 1.0)]) + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + decision = AcceptanceGate({"max_total_cost": 0.001}).decide( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + deltas=deltas, + cumulative_cost=0.001, + ) + + assert not decision.accepted + assert decision.total_run_cost > 0.001 + assert decision.total_run_cost == decision.cumulative_cost + decision.candidate_cost + + +def test_case_delta_types_are_classified(): + baseline_train = _eval("baseline", "train", [("new_pass", 0.0), ("new_fail", 1.0)]) + candidate_train = _eval("candidate", "train", [("new_pass", 1.0), ("new_fail", 0.0)]) + baseline_val = _eval("baseline", "validation", [("up", 0.2), ("down", 0.8), ("same", 0.5)]) + candidate_val = _eval("candidate", "validation", [("up", 0.5), ("down", 0.5), ("same", 0.5)]) + + deltas = compute_case_deltas( + candidate_id="candidate", + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + ) + + by_case = {delta.case_id: delta.delta_type for delta in deltas} + assert by_case["new_pass"] == "new_pass" + assert by_case["new_fail"] == "new_fail" + assert by_case["up"] == "score_up" + assert by_case["down"] == "score_down" + assert by_case["same"] == "unchanged" + + +def _eval(prompt_id: str, split: str, scores: list[tuple[str, float]]) -> EvalResult: + cases = [ + CaseResult( + case_id=case_id, + split=split, + score=score, + passed=score >= 1.0, + output="", + hard_failed=score <= 0.0, + ) + for case_id, score in scores + ] + return EvalResult( + prompt_id=prompt_id, + split=split, + score=round(sum(case.score for case in cases) / len(cases), 6), + passed=all(case.passed for case in cases), + cost=0.001 * len(cases), + cases=cases, + ) diff --git a/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py b/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py new file mode 100644 index 00000000..318dd460 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_no_sample_case_id_hardcoding.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline + + +BUSINESS_LOGIC_FILES = [ + "eval_loop/fake_model.py", + "eval_loop/fake_judge.py", + "eval_loop/optimizer.py", + "eval_loop/report.py", + "eval_loop/gate.py", +] + + +def test_sample_case_ids_do_not_appear_in_business_logic(): + root = Path("examples/optimization/eval_optimize_loop") + case_ids = set() + for rel in ("data/train.evalset.json", "data/val.evalset.json"): + payload = json.loads((root / rel).read_text(encoding="utf-8")) + case_ids.update(case["id"] for case in payload["cases"]) + + for rel in BUSINESS_LOGIC_FILES: + source = (root / rel).read_text(encoding="utf-8") + leaked = sorted(case_id for case_id in case_ids if case_id in source) + assert leaked == [], f"{rel} contains sample case ids: {leaked}" + + +def test_uuid_style_case_ids_still_drive_expected_behavior(tmp_path: Path): + train_path = tmp_path / "train.evalset.json" + val_path = tmp_path / "val.evalset.json" + optimizer_path = tmp_path / "optimizer.json" + prompt_path = tmp_path / "prompt.txt" + + train_path.write_text( + json.dumps({ + "split": "train", + "cases": [ + _json_case("1d5b548a-39ec-451d-9b01-7d73101b95b1", "train"), + _exact_case("c28596bf-370d-4898-bdf7-e4cc306fa4d3", "train", protected=False), + _rubric_case("3aa39806-878e-4634-9a68-d7f479ab7c9d", "train"), + ], + }), + encoding="utf-8", + ) + val_path.write_text( + json.dumps({ + "split": "validation", + "cases": [ + _json_case("534f045c-06f8-4b96-af55-cb7b6712cc0c", "validation"), + _rubric_case("b6914390-b2bb-4f43-a35b-24b6fe2825cd", "validation"), + _exact_case("70c59d31-adf5-409f-a457-286bcb887f52", "validation", protected=True), + ], + }), + encoding="utf-8", + ) + optimizer_path.write_text( + json.dumps({ + "seed": 91, + "optimizer": {"name": "fake_two_candidate_optimizer"}, + "metrics": {"case_score": "mean"}, + "gate": { + "min_val_score_improvement": 0.01, + "allow_new_hard_fail": False, + "protected_case_ids": ["70c59d31-adf5-409f-a457-286bcb887f52"], + "max_score_drop_per_case": 0.0, + "max_total_cost": 1.0, + }, + }), + encoding="utf-8", + ) + prompt_path.write_text("Baseline prompt", encoding="utf-8") + + report = run_pipeline( + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "out", + mode="fake", + trace=True, + ) + + decisions = {decision.candidate_id: decision for decision in report.gate_decisions} + assert report.selected_candidate == "candidate_002_safe" + assert decisions["candidate_001_overfit"].accepted is False + assert decisions["candidate_001_overfit"].overfit_detected is True + assert decisions["candidate_002_safe"].accepted is True + + +def _json_case(case_id: str, split: str) -> dict: + return { + "id": case_id, + "input": "Return strict JSON.", + "expectation": { + "type": "json", + "required_keys": ["answer"], + "expected_values": {"answer": "ok"}, + "expected_failure_category": "format_violation", + }, + } + + +def _exact_case(case_id: str, split: str, *, protected: bool) -> dict: + payload = { + "id": case_id, + "input": "Answer exactly YES.", + "expectation": { + "type": "exact", + "expected": "YES", + "expected_failure_category": "final_response_mismatch", + }, + "protected": protected, + "tags": ["baseline_pass"] if protected else [], + } + return payload + + +def _rubric_case(case_id: str, split: str) -> dict: + return { + "id": case_id, + "input": "Explain in prose.", + "expectation": { + "type": "rubric", + "must_include": ["cache", "stale data"], + "forbidden": ["{", "}", "json"], + "max_chars": 120, + "expected_failure_category": "format_violation", + }, + "tags": ["prose"], + } diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py new file mode 100644 index 00000000..85247d7e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_fake_mode.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys + +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_OPTIMIZER_CONFIG +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline +from examples.optimization.eval_optimize_loop.eval_loop.report import report_to_json + + +def test_fake_mode_pipeline_generates_json_and_markdown_reports(tmp_path: Path): + output_dir = tmp_path / "run" + report = run_pipeline( + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=output_dir, + fake_model=True, + fake_judge=True, + trace=True, + ) + + json_path = output_dir / "optimization_report.json" + md_path = output_dir / "optimization_report.md" + assert json_path.is_file() + assert md_path.is_file() + assert report.selected_candidate == "candidate_002_safe" + + payload = json.loads(json_path.read_text(encoding="utf-8")) + assert set(payload) >= { + "schema_version", + "run", + "baseline_train", + "baseline_validation", + "baseline", + "candidates", + "per_case_deltas", + "delta", + "failure_attribution_summary", + "gate_decisions", + "selected_candidate", + "audit", + } + assert payload["baseline_train"]["score"] == 0.333333 + assert payload["baseline_validation"]["score"] == 0.666667 + assert payload["selected_candidate"] == "candidate_002_safe" + assert [record["candidate"]["candidate_id"] for record in payload["candidates"]] == [ + "candidate_001_overfit", + "candidate_002_safe", + ] + assert len(payload["per_case_deltas"]) == 12 + assert payload["failure_attribution_summary"]["by_category"]["format_violation"] >= 1 + assert payload["failure_attribution_summary"]["attribution_accuracy"] == 1.0 + assert set(payload["audit"]) >= { + "seed", + "config_hash", + "cost", + "duration_seconds", + "prompt_hash", + "candidate_prompts", + "prompt_diffs", + "input_hashes", + "candidate_prompt_hashes", + "total_run_cost", + } + assert payload["run"]["reproducibility_command"].startswith("python examples/optimization") + assert payload["run"]["run_id"] == "eval_optimize_loop_seed_91" + assert payload["audit"]["total_run_cost"] == payload["audit"]["cost"]["total"] + assert "candidate_001_overfit" in payload["audit"]["prompt_diffs"] + assert "candidate_002_safe" in payload["audit"]["prompt_diffs"] + assert payload["candidates"][0]["candidate"]["prompt_diff"].startswith("--- baseline_system_prompt.txt") + + decisions = {item["candidate_id"]: item for item in payload["gate_decisions"]} + assert not decisions["candidate_001_overfit"]["accepted"] + assert decisions["candidate_002_safe"]["accepted"] + assert decisions["candidate_001_overfit"]["total_run_cost"] > decisions["candidate_001_overfit"]["candidate_cost"] + assert any( + "train score improved but validation score regressed" in reason + for reason in decisions["candidate_001_overfit"]["reasons"] + ) + + markdown = md_path.read_text(encoding="utf-8") + assert "Selected candidate: `candidate_002_safe`." in markdown + assert "candidate_001_overfit (rejected)" in markdown + assert "candidate_002_safe (accepted)" in markdown + assert "Baseline vs Candidate Scores" in markdown + assert "Per-Case Delta" in markdown + assert "Failure Attribution Summary" in markdown + assert "Prompt Diff" in markdown + assert "Reproducibility" in markdown + assert "Cost And Audit" in markdown + + run_dir = output_dir / "runs" / "eval_optimize_loop_seed_91" + assert (run_dir / "config.snapshot.json").is_file() + assert (run_dir / "input_hashes.json").is_file() + assert (run_dir / "candidate_prompts" / "candidate_001_overfit" / "system_prompt.txt").is_file() + assert (run_dir / "case_results" / "candidate_002_safe_validation.json").is_file() + assert (run_dir / "prompt_diffs" / "candidate_002_safe.diff").is_file() + assert (run_dir / "prompt_diffs" / "candidate_002_safe.diff").read_text(encoding="utf-8") == ( + payload["candidates"][1]["candidate"]["prompt_diff"] + ) + + +def test_pipeline_is_deterministic_with_same_seed(tmp_path: Path): + first_dir = tmp_path / "first" + second_dir = tmp_path / "second" + run_pipeline(output_dir=first_dir, fake_model=True, fake_judge=True, trace=True) + run_pipeline(output_dir=second_dir, fake_model=True, fake_judge=True, trace=True) + + assert (first_dir / "optimization_report.json").read_text(encoding="utf-8") == ( + second_dir / "optimization_report.json" + ).read_text(encoding="utf-8") + assert (first_dir / "optimization_report.md").read_text(encoding="utf-8") == ( + second_dir / "optimization_report.md" + ).read_text(encoding="utf-8") + + +def test_pipeline_accepts_mode_fake_without_legacy_flags(tmp_path: Path): + report = run_pipeline(output_dir=tmp_path / "run", mode="fake", trace=True) + assert report.run["mode"] == "fake" + assert report.selected_candidate == "candidate_002_safe" + + +def test_pipeline_selected_candidate_is_null_when_all_candidates_rejected(tmp_path: Path): + config_path = tmp_path / "optimizer.json" + config = json.loads(Path(DEFAULT_OPTIMIZER_CONFIG).read_text(encoding="utf-8")) + config["gate"]["min_val_score_improvement"] = 1.0 + config_path.write_text(json.dumps(config), encoding="utf-8") + + report = run_pipeline( + optimizer_config_path=config_path, + output_dir=tmp_path / "run", + mode="fake", + trace=True, + ) + + assert report.selected_candidate is None + + +def test_cli_mode_fake_and_legacy_fake_flags_both_run(tmp_path: Path): + script = Path("examples/optimization/eval_optimize_loop/run_pipeline.py") + first = tmp_path / "first" + second = tmp_path / "second" + + subprocess.run( + [sys.executable, str(script), "--mode", "fake", "--trace", "--output-dir", str(first)], + check=True, + ) + subprocess.run( + [sys.executable, str(script), "--fake-model", "--fake-judge", "--trace", "--output-dir", str(second)], + check=True, + ) + + assert (first / "optimization_report.json").is_file() + assert (second / "optimization_report.md").is_file() + + +def test_report_to_json_rejects_nan_values(tmp_path: Path): + report = run_pipeline(output_dir=tmp_path / "run", mode="fake", trace=True) + report.audit["bad_float"] = float("nan") + + try: + report_to_json(report) + except ValueError as exc: + assert "Out of range float values are not JSON compliant" in str(exc) + else: + raise AssertionError("report_to_json should reject NaN") + + +def test_fake_report_json_remains_strict_json(tmp_path: Path): + report = run_pipeline(output_dir=tmp_path / "run", mode="fake", trace=True) + payload = report_to_json(report) + + assert "NaN" not in payload + assert "Infinity" not in payload + assert json.loads(payload)["selected_candidate"] == "candidate_002_safe" diff --git a/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py new file mode 100644 index 00000000..de071c97 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py @@ -0,0 +1,892 @@ +from __future__ import annotations + +import builtins +import json +import sys +import types +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.eval_loop.backends import SDKBackend +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_OPTIMIZER_CONFIG +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_PROMPT +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline + + +def test_sdk_backend_requires_call_agent_path(tmp_path: Path): + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt") + + with pytest.raises(ValueError, match="--sdk-call-agent"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_calls_agent_optimizer_and_converts_best_prompt(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + candidates = backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["config_path"].endswith("optimizer.json") + assert calls["update_source"] is False + assert calls["output_dir"].endswith("out") + assert calls["target_prompt"].paths == [("system_prompt", str(prompt_path))] + assert candidates[0].candidate_id == "sdk_best" + assert candidates[0].prompt == "optimized prompt" + assert candidates[0].prompt_diff.startswith("--- baseline_system_prompt.txt") + assert backend.last_result is not None + assert backend.last_result_summary["baseline_pass_rate"] == 0.5 + + +def test_sdk_backend_default_target_prompt_uses_system_prompt_from_prompt_path(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompts={"system_prompt": "optimized system"}) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline system", encoding="utf-8") + + candidates = SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + ).optimize( + baseline_prompt="baseline system", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["target_prompt"].paths == [("system_prompt", str(prompt_path))] + assert candidates[0].prompt == "optimized system" + + +def test_sdk_backend_router_prompt_only_can_succeed(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompts={"router_prompt": "optimized router"}) + router_path = tmp_path / "router.txt" + router_path.write_text("baseline router", encoding="utf-8") + + candidates = SDKBackend( + prompt_path=tmp_path / "unused_system.txt", + call_agent_path="fake_call_agent_module:call_agent", + target_prompt_paths={"router_prompt": router_path}, + ).optimize( + baseline_prompt="unused", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["target_prompt"].paths == [("router_prompt", str(router_path))] + assert candidates[0].prompt == "## router_prompt\n\noptimized router" + + +def test_sdk_backend_skill_prompt_only_can_succeed(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompts={"skill_prompt": "optimized skill"}) + skill_path = tmp_path / "skill.txt" + skill_path.write_text("baseline skill", encoding="utf-8") + + candidates = SDKBackend( + prompt_path=tmp_path / "unused_system.txt", + call_agent_path="fake_call_agent_module:call_agent", + target_prompt_paths={"skill_prompt": skill_path}, + ).optimize( + baseline_prompt="unused", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["target_prompt"].paths == [("skill_prompt", str(skill_path))] + assert candidates[0].prompt == "## skill_prompt\n\noptimized skill" + + +def test_sdk_backend_missing_registered_best_prompt_field_is_clear(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompts={"router_prompt": "optimized router"}) + router_path = tmp_path / "router.txt" + skill_path = tmp_path / "skill.txt" + router_path.write_text("baseline router", encoding="utf-8") + skill_path.write_text("baseline skill", encoding="utf-8") + backend = SDKBackend( + prompt_path=tmp_path / "unused_system.txt", + call_agent_path="fake_call_agent_module:call_agent", + target_prompt_paths={"router_prompt": router_path, "skill_prompt": skill_path}, + ) + + with pytest.raises(ValueError, match="best_prompts.*missing registered target fields.*skill_prompt"): + backend.optimize( + baseline_prompt="unused", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_empty_best_prompts_dict_error_is_clear(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompts={}) + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="best_prompts was empty"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_passes_update_source_true(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + SDKBackend( + prompt_path=prompt_path, + call_agent_path="fake_call_agent_module:call_agent", + update_source=True, + ).optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert calls["update_source"] is True + + +def test_sdk_backend_call_agent_import_failure_names_target(tmp_path: Path): + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="missing.module:call_agent") + + with pytest.raises(ValueError, match="missing.module:call_agent"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_call_agent_must_be_callable(tmp_path: Path, monkeypatch): + call_agent_module = types.ModuleType("fake_call_agent_module") + call_agent_module.call_agent = "not callable" + monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="--sdk-call-agent.*fake_call_agent_module:call_agent"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_sdk_import_failure_is_clear(tmp_path: Path, monkeypatch): + call_agent_module = types.ModuleType("fake_call_agent_module") + + async def call_agent(query: str) -> str: + return query + + call_agent_module.call_agent = call_agent + monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) + + real_import = builtins.__import__ + + def fail_sdk_import(name, globals=None, locals=None, fromlist=(), level=0): + if name == "trpc_agent_sdk.evaluation": + raise ImportError("forced sdk import failure") + return real_import(name, globals, locals, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", fail_sdk_import) + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="AgentOptimizer/TargetPrompt"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +def test_sdk_backend_empty_best_prompt_error_is_clear(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="contained empty registered target fields.*system_prompt"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +@pytest.mark.asyncio +async def test_sdk_backend_sync_optimize_rejects_active_event_loop(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + backend = SDKBackend(prompt_path=tmp_path / "prompt.txt", call_agent_path="fake_call_agent_module:call_agent") + + with pytest.raises(ValueError, match="optimize_async"): + backend.optimize( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + +@pytest.mark.asyncio +async def test_sdk_backend_async_api_works_inside_active_event_loop(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + backend = SDKBackend(prompt_path=prompt_path, call_agent_path="fake_call_agent_module:call_agent") + + candidates = await backend.optimize_async( + baseline_prompt="baseline", + train_path=tmp_path / "train.evalset.json", + val_path=tmp_path / "val.evalset.json", + optimizer_config_path=tmp_path / "optimizer.json", + output_dir=tmp_path / "out", + ) + + assert candidates[0].candidate_id == "sdk_best" + + +def test_run_pipeline_mode_sdk_writes_report_without_fallback(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + run_id="sdk_test_run", + ) + + output_dir = tmp_path / "sdk_run" + payload = (output_dir / "optimization_report.json").read_text(encoding="utf-8") + markdown = (output_dir / "optimization_report.md").read_text(encoding="utf-8") + assert report.run["mode"] == "sdk" + assert report.run["update_source"] is False + assert report.selected_candidate == "sdk_best" + assert report.baseline_validation.score == 0.5 + assert report.candidates[0]["validation_result"].score == 0.75 + assert report.gate_decisions[0].validation_score_delta == 0.25 + assert report.gate_decisions[0].candidate_cost == 0.123 + assert report.gate_decisions[0].gate_status == "partial_applied" + assert report.gate_decisions[0].not_applied_checks == [ + "per_case_delta", + "protected_regression", + "new_hard_failure", + "max_score_drop_per_case", + ] + assert report.audit["duration_seconds"] == 12.3 + assert report.audit["total_run_cost"] == 0.123 + assert report.audit["cost"]["total"] == 0.123 + assert report.audit["sdk_result_summary"]["status"] == "SUCCEEDED" + assert report.audit["sdk_result_summary"]["baseline_metric_breakdown"] == {"exact_match": 0.5} + assert report.audit["sdk_result_summary"]["best_metric_breakdown"] == {"exact_match": 0.75} + assert report.audit["sdk_result_summary"]["metric_thresholds"] == {"exact_match": 0.7} + assert report.audit["sdk_result_summary"]["total_token_usage"] == { + "prompt": 100, + "completion": 25, + "total": 125, + } + assert report.audit["sdk_result_summary"]["rounds"][0]["validation_pass_rate"] == 0.75 + assert report.audit["sdk_result_availability"] == { + "aggregate_validation_result": True, + "full_train_eval_result": False, + "full_per_case_validation_delta": False, + } + assert "train EvalResult compatibility field is unavailable" in report.audit["sdk_score_explanation"] + assert "partial_applied" in payload + assert "sdk_best (partial_applied)" in markdown + assert "not applied checks: per_case_delta" in markdown + assert "SDK mode uses OptimizeResult aggregate validation metrics" in markdown + assert "fake_call_agent_module:call_agent" in report.run["reproducibility_command"] + assert "module:function" not in report.run["reproducibility_command"] + assert (output_dir / "runs" / "sdk_test_run" / "input_hashes.json").is_file() + assert (output_dir / "runs" / "sdk_test_run" / "prompt_diffs" / "sdk_best.diff").is_file() + assert calls["update_source"] is False + assert calls["output_dir"].endswith("sdk_optimizer") + assert report.run["sdk_artifact_dir"].endswith("sdk_optimizer") + + +def test_run_pipeline_mode_sdk_accepts_sdk_shaped_inputs_without_fake_schema(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + train_path = tmp_path / "sdk_train.evalset.json" + val_path = tmp_path / "sdk_val.evalset.json" + optimizer_path = tmp_path / "sdk_optimizer.json" + prompt_path = tmp_path / "system_prompt.txt" + train_path.write_text(json.dumps({"eval_cases": []}), encoding="utf-8") + val_path.write_text(json.dumps({"eval_cases": []}), encoding="utf-8") + optimizer_path.write_text( + json.dumps({"seed": "sdk-owned-seed", "optimize": {"algorithm": {"name": "gepa_reflective"}}}), + encoding="utf-8", + ) + prompt_path.write_text("baseline", encoding="utf-8") + + report = run_pipeline( + mode="sdk", + train_path=train_path, + val_path=val_path, + optimizer_config_path=optimizer_path, + prompt_path=prompt_path, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + assert report.run["mode"] == "sdk" + assert report.run["train_cases"] == 0 + assert report.selected_candidate == "sdk_best" + + +def test_run_pipeline_mode_sdk_default_run_id_uses_sdk_started_at(tmp_path: Path, monkeypatch): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + started_at="2026-07-04T12:34:56+00:00", + ) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + assert report.run["run_id"] == "eval_optimize_loop_sdk_20260704T123456Z" + assert (tmp_path / "sdk_run" / "runs" / report.run["run_id"]).is_dir() + assert "--run-id" not in report.run["reproducibility_command"] + + +def test_run_pipeline_mode_sdk_default_run_id_collision_gets_suffix(tmp_path: Path, monkeypatch): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + started_at="2026-07-04T12:34:56+00:00", + ) + + first = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + second = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + assert first.run["run_id"] == "eval_optimize_loop_sdk_20260704T123456Z" + assert second.run["run_id"] == "eval_optimize_loop_sdk_20260704T123456Z-1" + assert (tmp_path / "sdk_run" / "runs" / first.run["run_id"]).is_dir() + assert (tmp_path / "sdk_run" / "runs" / second.run["run_id"]).is_dir() + + +def test_run_pipeline_mode_sdk_explicit_run_id_stays_stable(tmp_path: Path, monkeypatch): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + first = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + run_id="valid_20260704-1.ok", + ) + second = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + run_id="valid_20260704-1.ok", + ) + + assert first.run["run_id"] == "valid_20260704-1.ok" + assert second.run["run_id"] == "valid_20260704-1.ok" + + +def test_run_pipeline_mode_sdk_uses_default_wrapper_gate_when_sdk_config_has_no_gate( + tmp_path: Path, + monkeypatch, +): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + baseline_pass_rate=0.5, + best_pass_rate=0.505, + pass_rate_improvement=0.005, + total_llm_cost=0.123, + ) + optimizer_path = _write_sdk_optimizer_config(tmp_path) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=optimizer_path, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + decision = report.gate_decisions[0] + assert report.selected_candidate is None + assert decision.accepted is False + assert decision.gate_status == "partial_applied" + assert decision.validation_score_delta == 0.005 + assert any("validation improvement" in reason for reason in decision.reasons) + + +def test_run_pipeline_mode_sdk_custom_gate_rejects_low_aggregate_validation_improvement( + tmp_path: Path, + monkeypatch, +): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + baseline_pass_rate=0.5, + best_pass_rate=0.75, + pass_rate_improvement=0.25, + total_llm_cost=0.123, + ) + gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.3, max_total_cost=1.0) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, + ) + + decision = report.gate_decisions[0] + assert report.selected_candidate is None + assert decision.accepted is False + assert decision.validation_score_delta == 0.25 + assert any("validation improvement" in reason for reason in decision.reasons) + + +def test_run_pipeline_mode_sdk_custom_gate_rejects_cost_over_budget(tmp_path: Path, monkeypatch): + _install_fake_sdk( + monkeypatch, + best_prompt="optimized prompt", + baseline_pass_rate=0.5, + best_pass_rate=0.75, + pass_rate_improvement=0.25, + total_llm_cost=2.0, + ) + gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.01, max_total_cost=0.05) + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, + ) + + decision = report.gate_decisions[0] + assert report.selected_candidate is None + assert decision.accepted is False + assert decision.gate_status == "partial_applied" + assert decision.total_run_cost == 2.0 + assert any("cost" in reason for reason in decision.reasons) + + +@pytest.mark.parametrize( + ("field_name", "field_value"), + [ + ("min_val_score_improvement", True), + ("max_total_cost", float("nan")), + ("max_total_cost", float("inf")), + ], +) +def test_run_pipeline_mode_sdk_rejects_invalid_gate_numbers( + tmp_path: Path, + monkeypatch, + field_name, + field_value, +): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + gate = {"min_val_score_improvement": 0.01, "max_total_cost": 1.0} + gate[field_name] = field_value + gate_path = tmp_path / "bad_gate.json" + gate_path.write_text(json.dumps({"gate": gate}), encoding="utf-8") + + with pytest.raises(ValueError, match=f"--gate-config.*{field_name}"): + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, + ) + + +@pytest.mark.parametrize("run_id", ["../../escape", "a/b", "", ".", "..", "has space", "a\\b"]) +def test_run_pipeline_rejects_invalid_run_id(tmp_path: Path, monkeypatch, run_id): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + with pytest.raises(ValueError, match="--run-id") as exc_info: + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + run_id=run_id, + ) + assert repr(run_id) in str(exc_info.value) + + +@pytest.mark.parametrize( + "field_name", + ["../router", "router/prompt", "router prompt", "router.prompt", "router-prompt", "", " router_prompt"], +) +def test_run_pipeline_mode_sdk_rejects_invalid_target_prompt_field_names( + tmp_path: Path, + monkeypatch, + field_name, +): + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + prompt_path = tmp_path / "prompt.txt" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(ValueError, match="--target-prompt") as exc_info: + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + target_prompts=[f"{field_name}={prompt_path}"], + ) + assert repr(field_name) in str(exc_info.value) + + +@pytest.mark.parametrize( + ("field_name", "field_value"), + [ + ("pass_rate_improvement", float("nan")), + ("total_llm_cost", float("inf")), + ("best_pass_rate", "bad"), + ], +) +def test_run_pipeline_mode_sdk_rejects_non_finite_or_bad_numeric_summary( + tmp_path: Path, + monkeypatch, + field_name, + field_value, +): + kwargs = { + "baseline_pass_rate": 0.5, + "best_pass_rate": 0.75, + "pass_rate_improvement": 0.25, + "total_llm_cost": 0.123, + } + kwargs[field_name] = field_value + _install_fake_sdk(monkeypatch, best_prompt="optimized prompt", **kwargs) + + with pytest.raises(ValueError, match=f"SDK OptimizeResult field {field_name} must be a finite number"): + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + ) + + +def test_run_pipeline_mode_sdk_does_not_pass_wrapper_gate_config_to_agent_optimizer( + tmp_path: Path, + monkeypatch, +): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + optimizer_path = _write_sdk_optimizer_config(tmp_path) + gate_path = _write_gate_config(tmp_path, min_val_score_improvement=0.5, max_total_cost=0.05) + + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=optimizer_path, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + gate_config_path=gate_path, + ) + + assert Path(calls["config_path"]).resolve() == optimizer_path.resolve() + assert "gate" not in json.loads(Path(calls["config_path"]).read_text(encoding="utf-8")) + assert json.loads(gate_path.read_text(encoding="utf-8"))["gate"]["max_total_cost"] == 0.05 + + +def test_run_pipeline_mode_sdk_registers_multiple_target_prompt_paths(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk( + monkeypatch, + best_prompts={ + "system_prompt": "optimized system", + "router_prompt": "optimized router", + "skill_prompt": "optimized skill", + }, + ) + system_path = tmp_path / "system.txt" + router_path = tmp_path / "router.txt" + skill_path = tmp_path / "skill.txt" + system_path.write_text("baseline system", encoding="utf-8") + router_path.write_text("baseline router", encoding="utf-8") + skill_path.write_text("baseline skill", encoding="utf-8") + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=_write_sdk_optimizer_config(tmp_path), + prompt_path=system_path, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + target_prompts=[ + f"system_prompt={system_path}", + f"router_prompt={router_path}", + f"skill_prompt={skill_path}", + ], + gate_config_path=_write_gate_config(tmp_path, min_val_score_improvement=0.01, max_total_cost=1.0), + run_id="sdk_multi_target", + ) + + assert calls["target_prompt"].paths == [ + ("system_prompt", str(system_path)), + ("router_prompt", str(router_path)), + ("skill_prompt", str(skill_path)), + ] + assert report.audit["sdk_result_summary"]["best_prompts"] == { + "system_prompt": "optimized system", + "router_prompt": "optimized router", + "skill_prompt": "optimized skill", + } + assert "router_prompt" in report.candidates[0]["candidate"].prompt_diff + assert set(report.audit["candidate_prompt_hashes_by_field"]["sdk_best"]) == { + "system_prompt", + "router_prompt", + "skill_prompt", + } + run_dir = tmp_path / "sdk_run" / "runs" / "sdk_multi_target" + assert (run_dir / "candidate_prompts" / "sdk_best" / "system_prompt.txt").read_text( + encoding="utf-8" + ) == "optimized system" + assert (run_dir / "candidate_prompts" / "sdk_best" / "router_prompt.txt").read_text( + encoding="utf-8" + ) == "optimized router" + assert (run_dir / "candidate_prompts" / "sdk_best" / "skill_prompt.txt").read_text( + encoding="utf-8" + ) == "optimized skill" + assert (run_dir / "candidate_prompts" / "sdk_best" / "bundle.txt").read_text( + encoding="utf-8" + ) == report.candidates[0]["candidate"].prompt + input_hashes = json.loads((run_dir / "input_hashes.json").read_text(encoding="utf-8")) + assert set(input_hashes["target_prompts"]) == {"system_prompt", "router_prompt", "skill_prompt"} + assert "gate_config" in input_hashes + command = report.run["reproducibility_command"] + assert "--sdk-call-agent fake_call_agent_module:call_agent" in command + assert "--target-prompt" in command + assert f"router_prompt={router_path}" in command + assert "--gate-config" in command + + +def test_run_pipeline_mode_sdk_passes_update_source_true(tmp_path: Path, monkeypatch): + calls = _install_fake_sdk(monkeypatch, best_prompt="optimized prompt") + + report = run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + sdk_call_agent="fake_call_agent_module:call_agent", + update_source=True, + ) + + assert report.run["update_source"] is True + assert calls["update_source"] is True + assert "--update-source" in report.run["reproducibility_command"] + + +def test_run_pipeline_mode_sdk_missing_call_agent_is_not_fake_fallback(tmp_path: Path): + with pytest.raises(ValueError, match="--sdk-call-agent"): + run_pipeline( + mode="sdk", + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + optimizer_config_path=DEFAULT_OPTIMIZER_CONFIG, + prompt_path=DEFAULT_PROMPT, + output_dir=tmp_path / "sdk_run", + ) + + +def _install_fake_sdk( + monkeypatch, + *, + best_prompt: str | None = None, + best_prompts: dict[str, str] | None = None, + status: str = "SUCCEEDED", + baseline_pass_rate: float = 0.5, + best_pass_rate: float = 0.75, + pass_rate_improvement: float = 0.25, + total_llm_cost: float = 0.123, + duration_seconds: float = 12.3, + started_at: str | None = None, +): + calls = {} + + class FakeTargetPrompt: + def __init__(self): + self.paths = [] + + def add_path(self, name, path): + self.paths.append((name, path)) + return self + + class FakeAgentOptimizer: + @staticmethod + async def optimize(**kwargs): + calls.update(kwargs) + result_prompts = best_prompts if best_prompts is not None else { + "system_prompt": "optimized prompt" if best_prompt is None else best_prompt + } + return types.SimpleNamespace( + best_prompts=result_prompts, + status=status, + baseline_pass_rate=baseline_pass_rate, + best_pass_rate=best_pass_rate, + pass_rate_improvement=pass_rate_improvement, + baseline_metric_breakdown={"exact_match": baseline_pass_rate}, + best_metric_breakdown={"exact_match": best_pass_rate}, + metric_thresholds={"exact_match": 0.7}, + total_llm_cost=total_llm_cost, + total_token_usage={"prompt": 100, "completion": 25, "total": 125}, + duration_seconds=duration_seconds, + started_at=started_at, + total_rounds=1, + rounds=[ + types.SimpleNamespace( + validation_pass_rate=best_pass_rate, + accepted=True, + failed_case_ids=["case_a"], + round_llm_cost=total_llm_cost, + budget_used=3, + budget_total=10, + ) + ], + ) + + fake_eval_module = types.ModuleType("trpc_agent_sdk.evaluation") + fake_eval_module.AgentOptimizer = FakeAgentOptimizer + fake_eval_module.TargetPrompt = FakeTargetPrompt + monkeypatch.setitem(sys.modules, "trpc_agent_sdk.evaluation", fake_eval_module) + + call_agent_module = types.ModuleType("fake_call_agent_module") + + async def call_agent(query: str) -> str: + return query + + call_agent_module.call_agent = call_agent + monkeypatch.setitem(sys.modules, "fake_call_agent_module", call_agent_module) + return calls + + +def _write_sdk_optimizer_config(tmp_path: Path) -> Path: + path = tmp_path / "sdk_optimizer.json" + path.write_text( + json.dumps({ + "evaluate": {"metrics": []}, + "optimize": {"algorithm": {"name": "gepa_reflective"}}, + }), + encoding="utf-8", + ) + return path + + +def _write_gate_config( + tmp_path: Path, + *, + min_val_score_improvement: float, + max_total_cost: float, +) -> Path: + path = tmp_path / "wrapper_gate.json" + path.write_text( + json.dumps({ + "gate": { + "min_val_score_improvement": min_val_score_improvement, + "max_total_cost": max_total_cost, + } + }), + encoding="utf-8", + ) + return path diff --git a/pyproject.toml b/pyproject.toml index 6c47d4d5..146e2281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -204,5 +204,5 @@ split_before_logical_operator = true [tool.pytest.ini_options] minversion = "6.0" addopts = "-ra -q" -testpaths = ["tests"] +testpaths = ["tests", "examples/optimization/eval_optimize_loop/tests"] asyncio_mode = "auto"