diff --git a/examples/optimization/eval_optimize_loop/.gitignore b/examples/optimization/eval_optimize_loop/.gitignore new file mode 100644 index 00000000..9cb95842 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/.gitignore @@ -0,0 +1,4 @@ +runs/ +__pycache__/ +*.pyc + diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..4fb2c53b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,385 @@ +# 评测 + 优化闭环示例 + +本示例展示一个可复现的评测优化流水线，将 baseline 评测、失败归因、prompt 候选生成、验证集回归、gate 决策和审计报告串成闭环。 + +它和 `ci_integration/` 的定位不同：`ci_integration/` 更关注 PR gate 和定时写回；本示例关注在接受候选 prompt 之前，如何完整完成训练集反馈、验证集回归、防过拟合判断和审计留痕。 + +## 运行方式 + +默认 fake mode 是确定性的，不需要 API key： + +```bash +cd examples/optimization/eval_optimize_loop +PYTHONPATH=../../.. python run_pipeline.py --mode fake +``` + +如果环境只暴露 `python3`，可以运行： + +```bash +PYTHONPATH=../../.. python3 run_pipeline.py --mode fake +``` + +刷新已提交的示例报告： + +```bash +PYTHONPATH=../../.. python run_pipeline.py --mode fake --update-sample-outputs +``` + +fake 和 trace mode 支持三个确定性场景： + +```bash +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario overfit +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario accepted +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario cost_exceeded +``` + +- `overfit` 是默认场景：训练集提升，验证集没有提升，并且关键 case 退化，所以 gate 拒绝候选。 +- `accepted`：训练集和验证集都提升，且没有新增 hard fail，所以 gate 接受候选。 +- `cost_exceeded`：质量提升，但估算成本超过 `gate.max_cost_usd`，所以 gate 拒绝候选。 + +CI 脚本可以添加 `--ci-exit-code`。普通 demo 运行在写出报告后总是返回 0；CI 模式下，接受候选返回 0，拒绝候选返回 1： + +```bash +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario accepted --ci-exit-code +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario overfit --ci-exit-code +``` + +trace replay mode 也不需要模型凭证。它先用 fake 输出生成 `eval_mode: "trace"` 的 evalset，再回放其中的 `actual_conversation`，不调用 `call_agent`： + +```bash +PYTHONPATH=../../.. python run_pipeline.py --mode trace +``` + +可选的真实优化路径会委托 `AgentOptimizer.optimize` 搜索候选，需要常规优化依赖和模型环境变量： + +```bash +export TRPC_AGENT_API_KEY="" +export TRPC_AGENT_BASE_URL="" +export TRPC_AGENT_MODEL_NAME="" +PYTHONPATH=../../.. python run_pipeline.py --mode optimizer +``` + +## 验证命令 + +以下命令覆盖主要评审路径： + +```bash +cd examples/optimization/eval_optimize_loop +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario overfit +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario accepted +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario cost_exceeded +PYTHONPATH=../../.. python run_pipeline.py --mode trace --scenario overfit +cd ../../.. +pytest tests/evaluation/test_eval_optimize_loop_example.py -q +``` + +## 输入文件 + +| 文件 | 作用 | +| --- | --- | +| `train.evalset.json` | 用于 baseline 评分和优化反馈的训练集。 | +| `val.evalset.json` | 用于回归检查和 gate 决策的验证集。 | +| `optimizer.json` | 共享 metric、优化器、fake 候选和 gate 配置。 | +| `case_meta.json` | 公开 case 的归因标签，用于本地 self-check。 | +| `agent/prompts/router.md` | 注册为 `TargetPrompt("router_prompt")` 的路由 prompt 源文件。 | +| `agent/prompts/system.md` | 注册为 `TargetPrompt("system_prompt")` 的系统 prompt 源文件。 | +| `agent/prompts/skill.md` | 注册为 `TargetPrompt("skill_prompt")` 的 skill prompt 源文件。 | + +fake mode 仍然使用 `AgentEvaluator` 完成训练集和验证集评分，只把候选生成做成确定性逻辑，因此可以在 CI 或没有模型凭证的本地环境运行。 + +启动时，流水线会校验输入文件存在、训练集和验证集不同且非空、gate key 完整、关键 case id 属于验证集、evaluator 并发数为正。 + +fake judge 由本地 exact-match final-response scoring 实现，报告中记录为 `run.judge_mode = "local_exact_match_fake_judge"`，因此评测路径不依赖 LLM judge 或 API key。 + +对于 tool-only 和 rubric-only 的失败类别，fake judge 使用 `case_meta.json` 中的确定性归因提示。这让无 key 示例也能覆盖完整失败归因 taxonomy，而不需要真实工具执行或 LLM rubric judge。 + +## 输出文件 + +每次运行默认写入 `runs/__/`： + +| 文件或目录 | 内容 | +| --- | --- | +| `optimization_report.json` | 机器可读的 baseline、candidate、round prompts、delta、归因、gate 和审计数据。 | +| `optimization_report.md` | 面向 reviewer 的中英双语 Markdown 报告。 | +| `candidate_prompts/` | 每个 target field 的候选 prompt 文本。 | +| `eval_*` | 每个阶段的原始 `AgentEvaluator` 结果文件。 | +| `config.snapshot.json` | 输入配置的可复现快照。 | +| `eval_metrics.snapshot.json` | 从 `optimizer.json.evaluate` 提取的 evaluator metric 配置。 | + +报告包含 `prompt_audit`，记录每个 target prompt 的源路径、baseline SHA-256、candidate SHA-256、字符数和是否变更。每条 prompt audit 还包含截断后的 unified diff preview，reviewer 不打开候选 prompt 文件也能看到变化。 + +`input_audit` 记录训练集、验证集、优化器配置、case metadata 和 prompt 源文件的稳定 SHA-256 与字节数。即使报告脱离工作区，也能复现实验输入。 + +`audit.cost` 将估算成本拆分为 optimizer 与 evaluation 两部分，并把总成本传入同一个可配置预算 gate。在 fake/trace mode 中，两部分都是确定性的；除了专门的 `cost_exceeded` 场景外通常为 0。 + +`failure_attribution.self_check` 会把归因结果和 `case_meta.json` 中公开 gold labels 对比。对于无 key 黑盒路径无法自然产生的类别，例如 tool-call 和 LLM-rubric failures，`fake_attribution_hints` 提供显式 fake-judge hint。报告会标记这些 `hint_assisted` case，并单独报告 rule-only accuracy。 + +已提交的示例输出位于 `sample_outputs/`。 + +JSON 报告同时包含人类可读原因和结构化 gate checks： + +```json +{ + "gate": { + "decision": "rejected", + "reasons": ["..."], + "checks": [ + {"name": "min_validation_score_delta", "passed": false}, + {"name": "no_new_hard_fail", "passed": false} + ] + } +} +``` + +## Gate 决策契约 + +候选只有在所有配置的 gate check 都通过时才会被接受： + +| Check | 接受条件 | 拒绝条件 | +| --- | --- | --- | +| `min_validation_score_delta` | 验证集分数提升大于等于 `gate.min_validation_score_delta`。 | 验证集分数提升低于阈值。 | +| `no_new_hard_fail` | 没有验证集 case 从 pass 变为 fail，除非 `gate.allow_new_hard_fail` 为 true。 | 不允许新增 hard fail 时，至少一个验证集 case 变成新失败。 | +| `critical_case_regression` | `gate.critical_case_ids` 中没有 case 分数下降。 | 任意关键验证 case 退化。 | +| `overfitting_guard` | 验证集提升，或训练集没有提升。 | 训练集提升但验证集没有提升。 | +| `max_cost_usd` | 估算候选成本小于等于 `gate.max_cost_usd`。 | 估算成本超过预算。 | + +该契约以验证集为优先：仅有训练集提升不能证明候选可接受，隐藏样本也应遵循相同的结构化决策规则。 + +## Case 覆盖 + +十个示例 case 超过最低要求，并覆盖所需场景： + +| Case | Split | 预期行为 | +| --- | --- | --- | +| `train_refund_double_charge` | train | 候选修复 baseline 失败。 | +| `train_password_reset` | train | 已经通过，候选保持不变。 | +| `train_legacy_sync` | train | 候选没有帮助。 | +| `train_plan_question` | train | 稳定的非目标计费问题保持通过。 | +| `train_policy_tool_missing` | train | 使用不完整 JSON response 覆盖 tool lookup failure 归因。 | +| `val_vip_refund` | validation | 候选带来新的通过。 | +| `val_plan_question` | validation | 候选没有影响。 | +| `val_checkout_outage` | validation | 候选使关键 case 退化。 | +| `val_mobile_crash` | validation | 稳定的技术排障 case 保持通过。 | +| `val_rubric_tone_fail` | validation | 确定性 rubric failure 归因保持失败。 | + +默认 fake candidate 会提升训练集分数，但验证集整体没有提升，并且引入一个新的 hard fail，因此 gate 拒绝候选并记录过拟合原因。 + +## 需求映射 + +| #91 要求 | 实现 | +| --- | --- | +| Baseline train/validation evaluation | `run_pipeline.py::evaluate_dataset` 使用 `AgentEvaluator.get_executer(...).get_result()`。 | +| Per-case metric、pass/fail、reason、trace | 报告 case 包含 query、expected、actual、metric details、status、failure types 和 reason。 | +| Failure attribution clustering | `attribute_failure` 和 `build_failure_stats`。 | +| 完整归因 taxonomy | 公开样例覆盖 `final_response_mismatch`、`tool_call_error`、`parameter_error`、`llm_rubric_not_met`、`knowledge_recall_insufficient`、`format_violation`。 | +| 归因准确率 guard | `case_meta.json` 和 `failure_attribution.self_check` 校验公开标签，并标记 fake hint-assisted case。 | +| Prompt optimization target | `TargetPrompt` 注册 `router_prompt`、`system_prompt`、`skill_prompt`。 | +| AgentOptimizer 或等价扩展 | `--mode optimizer` 委托 `AgentOptimizer`；fake/trace mode 为无 key CI 使用确定性候选生成。 | +| Candidate validation 和 per-case delta | `build_case_deltas` 标记 `new_pass`、`new_fail`、`score_improved`、`score_regressed`、`unchanged`。 | +| 可配置 gate | `optimizer.json.gate` 和 `apply_gate`，包含结构化 `gate.checks`。 | +| 过拟合拒绝 | 默认 `--scenario overfit` 提升训练集但因验证集无收益和关键 case 退化被拒绝。 | +| 成本预算拒绝 | `--scenario cost_exceeded` 质量提升但失败于 `max_cost_usd`。 | +| 审计产物 | `input_audit`、带 diff preview 的 `prompt_audit`、`optimization_rounds`、`candidate_prompts/`、原始 `eval_*` 输出、配置快照、JSON/Markdown 报告。 | +| 无 API key 模式 | `--mode fake` 和 `--mode trace`。 | +| 6 个公开 case | `train.evalset.json` 有 5 个 case，`val.evalset.json` 有 5 个 case。 | + +## 设计说明 + +本示例把 `AgentEvaluator.get_executer(...).get_result()` 作为唯一评测事实来源，分别对训练集和验证集生成逐 case 记录，包含输入、期望输出、实际输出、metric 分数、pass/fail、失败类型和原因。失败归因采用可复现规则：JSON 解析失败或缺字段归为格式不符合要求，category 不一致归为知识召回不足，priority 不一致归为参数错误，action 不一致归为最终回复不匹配；对必须依赖真实工具轨迹或 LLM rubric judge 的类别，fake mode 通过 `case_meta.json` 的 `fake_attribution_hints` 标记 `tool_call_error` 和 `llm_rubric_not_met`，并在 self-check 中区分 hint-assisted 与 rule-only 准确率，避免把公开标签直接当成黑盒判断结果。 + +候选 prompt 不会直接写回源文件，而是先在验证集回归，并和 baseline 做逐 case delta，标记新增通过、新增失败、分数提升、分数下降和无变化。gate 由 `optimizer.json` 配置，检查验证集总分提升阈值、是否新增 hard fail、关键 case 是否退化以及成本预算；额外防过拟合规则是训练集提升但验证集没有提升时拒绝候选。这样即使优化器记住训练失败样例，只要验证集没有真实收益，或者牺牲了关键 case，也不会进入可接受状态。 + +审计产物包括配置快照、随机种子、候选 prompt 文件、`optimization_rounds`、原始 evaluator 输出、JSON 报告和 Markdown 报告，便于完整复现实验配置，也便于后续 CI 读取结构化字段。`prompt_audit` 记录源路径、baseline/candidate 哈希、字符数和是否变更，reviewer 可以把分数变化追溯到具体 prompt。trace mode 会生成 `eval_mode: "trace"` 数据并回放 `actual_conversation`，保证无 API key 环境也能复现 baseline、归因、优化、验证、gate 和审计闭环。 + +# Evaluation + Optimization Loop + +This example demonstrates a reproducible pipeline that connects evaluation, failure attribution, prompt optimization, validation regression checks, gate decisions, and audit reports. + +It is intentionally different from `ci_integration/`: CI integration focuses on PR gates and nightly write-back, while this example focuses on the complete review loop before deciding whether a candidate prompt is worth accepting. + +## Run + +The default mode is deterministic and does not require an API key: + +```bash +cd examples/optimization/eval_optimize_loop +PYTHONPATH=../../.. python run_pipeline.py --mode fake +``` + +If your environment only exposes `python3`, use: + +```bash +PYTHONPATH=../../.. python3 run_pipeline.py --mode fake +``` + +To refresh the committed sample reports: + +```bash +PYTHONPATH=../../.. python run_pipeline.py --mode fake --update-sample-outputs +``` + +The fake and trace modes support three deterministic scenarios: + +```bash +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario overfit +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario accepted +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario cost_exceeded +``` + +- `overfit` is the default: train improves, validation does not improve, and a critical case regresses, so the gate rejects the candidate. +- `accepted`: train and validation both improve with no new hard fail, so the gate accepts the candidate. +- `cost_exceeded`: quality improves, but the estimated cost exceeds `gate.max_cost_usd`, so the gate rejects the candidate. + +For CI-style scripts, add `--ci-exit-code`. Normal demo runs always exit 0 after writing the report; CI mode exits 0 for accepted candidates and 1 for rejected candidates: + +```bash +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario accepted --ci-exit-code +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario overfit --ci-exit-code +``` + +Trace replay mode also runs without model credentials. It first records fake outputs into generated `eval_mode: "trace"` evalsets, then evaluates those `actual_conversation` records without invoking `call_agent`: + +```bash +PYTHONPATH=../../.. python run_pipeline.py --mode trace +``` + +The optional real optimizer path delegates candidate search to `AgentOptimizer.optimize` and requires the normal optimization dependencies and model environment variables: + +```bash +export TRPC_AGENT_API_KEY="" +export TRPC_AGENT_BASE_URL="" +export TRPC_AGENT_MODEL_NAME="" +PYTHONPATH=../../.. python run_pipeline.py --mode optimizer +``` + +## Verification Commands + +Use these commands to exercise the important review paths: + +```bash +cd examples/optimization/eval_optimize_loop +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario overfit +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario accepted +PYTHONPATH=../../.. python run_pipeline.py --mode fake --scenario cost_exceeded +PYTHONPATH=../../.. python run_pipeline.py --mode trace --scenario overfit +cd ../../.. +pytest tests/evaluation/test_eval_optimize_loop_example.py -q +``` + +## Inputs + +| File | Role | +| --- | --- | +| `train.evalset.json` | Training split used for baseline scoring and optimization feedback. | +| `val.evalset.json` | Validation split used for regression and gate decisions. | +| `optimizer.json` | Shared metric, optimizer, fake candidate, and gate configuration. | +| `case_meta.json` | Public-case attribution labels used for local self-checking. | +| `agent/prompts/router.md` | Router prompt source registered as `TargetPrompt("router_prompt")`. | +| `agent/prompts/system.md` | System prompt source registered as `TargetPrompt("system_prompt")`. | +| `agent/prompts/skill.md` | Skill prompt source registered as `TargetPrompt("skill_prompt")`. | + +The fake mode still uses `AgentEvaluator` for all train and validation scoring. Only candidate generation is deterministic, so the example can run in CI or on a laptop without model credentials. + +At startup, the pipeline validates that all required input files exist, train and validation evalsets are distinct and non-empty, gate keys are present, critical case IDs refer to validation cases, and evaluator parallelism is positive. + +Fake judge behavior is implemented by local exact-match final-response scoring (`final_response_avg_score`). The report records this as `run.judge_mode = "local_exact_match_fake_judge"`, so the evaluation path does not need an LLM judge or API key. + +For tool-only and rubric-only failure classes, fake judge mode uses deterministic attribution hints from `case_meta.json`. This lets the no-key example exercise the full attribution taxonomy without requiring real tool execution or an LLM rubric judge. + +## Outputs + +Each run writes to `runs/__/` by default: + +| File or directory | Contents | +| --- | --- | +| `optimization_report.json` | Machine-readable baseline, candidate, per-round prompts, delta, attribution, gate, and audit payload. | +| `optimization_report.md` | Bilingual human-readable report for review. | +| `candidate_prompts/` | Candidate prompt text per target field. | +| `eval_*` | Raw `AgentEvaluator` result files for every phase. | +| `config.snapshot.json` | Reproducible copy of the input config. | +| `eval_metrics.snapshot.json` | Evaluator-compatible metric config extracted from `optimizer.json.evaluate`. | + +The report also includes `prompt_audit`, which records each target prompt's source path, baseline SHA-256, candidate SHA-256, character counts, and whether the candidate changed the prompt. Each prompt audit entry also includes a capped unified diff preview, so reviewers can see what changed without opening the candidate prompt files. This makes the score-to-prompt relationship auditable even when reports are copied outside the run directory. + +`input_audit` records stable SHA-256 hashes and byte counts for the train evalset, validation evalset, optimizer config, case metadata, and prompt source files. This makes a report reproducible even after it is detached from the working tree. + +`audit.cost` splits the total estimated spend into optimizer and evaluation components, then applies the same total to the configurable budget gate. In fake/trace mode both components are deterministic and normally zero except for the dedicated `cost_exceeded` scenario. + +`failure_attribution.self_check` compares attribution results against the public gold labels in `case_meta.json`. Gold labels live in `expected_failure_types`. For categories that cannot naturally arise in the no-key black-box path, such as tool-call and LLM-rubric failures, `fake_attribution_hints` provides an explicit fake-judge hint. The report marks those cases as `hint_assisted` and also reports rule-only accuracy separately. This is not a hidden-set proof, but it provides a local accuracy guard for the bundled examples and fails report validation if the labeled-case accuracy drops below 75%. + +Committed examples live in `sample_outputs/`. + +The JSON report includes both human reasons and structured gate checks: + +```json +{ + "gate": { + "decision": "rejected", + "reasons": ["..."], + "checks": [ + {"name": "min_validation_score_delta", "passed": false}, + {"name": "no_new_hard_fail", "passed": false} + ] + } +} +``` + +## Gate Decision Contract + +A candidate is accepted only when every configured gate check passes: + +| Check | Accept condition | Reject condition | +| --- | --- | --- | +| `min_validation_score_delta` | Validation score delta is greater than or equal to `gate.min_validation_score_delta`. | Validation score delta is below the configured threshold. | +| `no_new_hard_fail` | No validation case changes from pass to fail, unless `gate.allow_new_hard_fail` is true. | At least one validation case becomes a new hard fail while new hard fails are disallowed. | +| `critical_case_regression` | No case in `gate.critical_case_ids` loses score. | Any critical validation case regresses. | +| `overfitting_guard` | Validation improves, or training does not improve. | Training improves while validation does not improve. | +| `max_cost_usd` | Estimated candidate cost is less than or equal to `gate.max_cost_usd`. | Estimated candidate cost exceeds the budget. | + +This contract is intentionally validation-first: training improvements alone never justify acceptance, and hidden samples are expected to follow the same structured decision rules. + +## Case Coverage + +The ten sample cases exceed the minimum requirement and cover the required outcomes: + +| Case | Split | Expected behavior | +| --- | --- | --- | +| `train_refund_double_charge` | train | Candidate fixes a baseline failure. | +| `train_password_reset` | train | Already passing; candidate is unchanged. | +| `train_legacy_sync` | train | Candidate does not help. | +| `train_plan_question` | train | Stable non-target billing question remains passing. | +| `train_policy_tool_missing` | train | Tool lookup failure attribution with an incomplete JSON response. | +| `val_vip_refund` | validation | Candidate creates a new pass. | +| `val_plan_question` | validation | Candidate has no effect. | +| `val_checkout_outage` | validation | Candidate regresses a critical case. | +| `val_mobile_crash` | validation | Stable technical troubleshooting case remains passing. | +| `val_rubric_tone_fail` | validation | Deterministic rubric failure attribution remains failing. | + +The default fake candidate improves train score but does not improve validation overall because it also introduces a new hard fail. The gate therefore rejects the candidate and records the overfitting reason. + +## Requirement Mapping + +| #91 requirement | Implementation | +| --- | --- | +| Baseline train/validation evaluation | `run_pipeline.py::evaluate_dataset` uses `AgentEvaluator.get_executer(...).get_result()`. | +| Per-case metric, pass/fail, reason, trace | Report cases include query, expected, actual, metric details, status, failure types, and reason. | +| Failure attribution clustering | `attribute_failure` and `build_failure_stats`. | +| Full attribution taxonomy | Public sample failures include `final_response_mismatch`, `tool_call_error`, `parameter_error`, `llm_rubric_not_met`, `knowledge_recall_insufficient`, and `format_violation`. | +| Attribution accuracy guard | `case_meta.json` plus `failure_attribution.self_check` validates public labeled cases and marks fake hint-assisted cases. | +| Prompt optimization target | `TargetPrompt` registers `router_prompt`, `system_prompt`, and `skill_prompt`. | +| AgentOptimizer or equivalent extension | `--mode optimizer` delegates to `AgentOptimizer`; fake/trace modes use deterministic candidate generation for no-key CI. | +| Candidate validation and per-case delta | `build_case_deltas` marks `new_pass`, `new_fail`, `score_improved`, `score_regressed`, and `unchanged`. | +| Configurable gate | `optimizer.json.gate` plus `apply_gate`, including structured `gate.checks`. | +| Overfitting rejection | Default `--scenario overfit` improves train but rejects because validation does not improve and a critical case regresses. | +| Cost budget rejection | `--scenario cost_exceeded` improves quality but fails `max_cost_usd`. | +| Audit artifacts | `input_audit`, `prompt_audit` with diff previews, `optimization_rounds`, `candidate_prompts/`, raw `eval_*` outputs, config snapshots, JSON report, and Markdown report. | +| No API key mode | `--mode fake` and `--mode trace`. | +| 6 public cases | `train.evalset.json` has 5 cases and `val.evalset.json` has 5 cases. | + +## Design Notes + +This example treats `AgentEvaluator.get_executer(...).get_result()` as the single source of evaluation truth. It generates per-case records for train and validation with input, expected output, actual output, metric score, pass/fail, failure types, and reasons. Failure attribution is reproducible: JSON parsing failures or missing fields become format violations, category mismatches become insufficient knowledge recall, priority mismatches become parameter errors, and action mismatches become final response mismatches. For categories that require real tool traces or an LLM rubric judge, fake mode uses `fake_attribution_hints` from `case_meta.json` to mark `tool_call_error` and `llm_rubric_not_met`, while self-check reports hint-assisted and rule-only accuracy separately. + +Candidate prompts are not written back to source files directly. They are first validated on the validation split and compared with baseline per-case deltas: new pass, new fail, score improved, score regressed, and unchanged. The gate is configured in `optimizer.json` and checks validation score delta, new hard fails, critical-case regressions, and cost budget. The extra overfitting guard rejects candidates when train improves but validation does not, so memorizing train failures cannot pass without real validation benefit. + +Audit artifacts include config snapshots, random seed, candidate prompt files, `optimization_rounds`, raw evaluator outputs, JSON report, and Markdown report. `prompt_audit` records source paths, baseline/candidate hashes, character counts, and change flags, so reviewers can trace score changes back to specific prompt edits. Trace mode generates `eval_mode: "trace"` data and replays `actual_conversation`, which keeps the baseline, attribution, optimization, validation, gate, and audit loop reproducible without an API key. diff --git a/examples/optimization/eval_optimize_loop/agent/__init__.py b/examples/optimization/eval_optimize_loop/agent/__init__.py new file mode 100644 index 00000000..1fc33799 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/__init__.py @@ -0,0 +1,2 @@ +"""Agent entry points for the eval-optimize-loop example.""" + diff --git a/examples/optimization/eval_optimize_loop/agent/agent.py b/examples/optimization/eval_optimize_loop/agent/agent.py new file mode 100644 index 00000000..a3b8695e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/agent.py @@ -0,0 +1,155 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Deterministic fake agent for the eval-optimize-loop example. + +The example's default mode must run without an API key, so this module exposes +an async ``call_agent`` that derives behavior from the current prompt text on +disk. The pipeline still uses AgentEvaluator for scoring; this module only +stands in for the model. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + + +PROMPT_DIR = Path(__file__).parent / "prompts" +ROUTER_PROMPT_PATH = PROMPT_DIR / "router.md" +SYSTEM_PROMPT_PATH = PROMPT_DIR / "system.md" +SKILL_PROMPT_PATH = PROMPT_DIR / "skill.md" + +_SPACE_RE = re.compile(r"\s+") + + +def _compact_json(payload: dict[str, str]) -> str: + return json.dumps(payload, sort_keys=True, separators=(",", ":")) + + +def normalize_json_text(raw: str) -> str: + """Normalize JSON-like model output for stable exact matching.""" + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + return _SPACE_RE.sub(" ", raw.strip()) + return _compact_json(parsed) + + +def _read_prompt_text() -> str: + return "\n\n".join( + path.read_text(encoding="utf-8") + for path in (ROUTER_PROMPT_PATH, SYSTEM_PROMPT_PATH, SKILL_PROMPT_PATH) + ).lower() + + +def _prompt_flags() -> dict[str, bool]: + prompt = _read_prompt_text() + return { + "refund_rule": "treat double charge" in prompt or "vip refund requests" in prompt, + "keep_outage_p1": "p1 for urgent production outages" in prompt, + "keep_plan_p3": "p3 for low-risk informational requests" in prompt, + "overfit_payment_outage": "overfit_payment_outage" in prompt, + } + + +def answer_for_query(query: str) -> str: + """Return the fake model answer for one support ticket.""" + text = query.lower() + flags = _prompt_flags() + + if "production checkout outage" in text: + if flags["overfit_payment_outage"]: + return _compact_json({ + "category": "billing", + "priority": "p2", + "action": "refund_review", + }) + priority = "p1" if flags["keep_outage_p1"] else "p2" + return _compact_json({ + "category": "technical", + "priority": priority, + "action": "escalate", + }) + + if "vip customer" in text and "double charged" in text: + if flags["refund_rule"]: + return _compact_json({ + "category": "billing", + "priority": "p1", + "action": "refund_review", + }) + return _compact_json({ + "category": "account", + "priority": "p2", + "action": "answer", + }) + + if "double charged" in text or "refund" in text: + if flags["refund_rule"]: + return _compact_json({ + "category": "billing", + "priority": "p2", + "action": "refund_review", + }) + return _compact_json({ + "category": "account", + "priority": "p2", + "action": "answer", + }) + + if "password reset" in text or "email address" in text: + return _compact_json({ + "category": "account", + "priority": "p3", + "action": "answer", + }) + + if "plan comparison" in text or "pricing tiers" in text: + priority = "p3" if flags["keep_plan_p3"] else "p2" + return _compact_json({ + "category": "billing", + "priority": priority, + "action": "answer", + }) + + if "policy citation" in text or "pol-77" in text: + return _compact_json({ + "category": "billing", + "action": "answer", + }) + + if "guaranteed instant fix" in text or "repeated invoice confusion" in text: + return _compact_json({ + "category": "billing", + "priority": "p3", + "action": "answer", + }) + + if "mobile app crashes" in text: + return _compact_json({ + "category": "technical", + "priority": "p2", + "action": "troubleshooting", + }) + + if "legacy desktop sync" in text: + return _compact_json({ + "category": "technical", + "priority": "p2", + "action": "troubleshooting", + }) + + return _compact_json({ + "category": "technical", + "priority": "p2", + "action": "troubleshooting", + }) + + +async def call_agent(query: str) -> str: + """AgentEvaluator / AgentOptimizer compatible black-box entry point.""" + return answer_for_query(query) diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/router.md b/examples/optimization/eval_optimize_loop/agent/prompts/router.md new file mode 100644 index 00000000..f3bfe56c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/router.md @@ -0,0 +1,7 @@ +Route each support ticket to one primary handling path before answering. + +Baseline routing rules: +- Account access and profile questions go to account handling. +- Billing and pricing questions go to billing handling. +- Production incidents and software failures go to technical handling. + diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/skill.md b/examples/optimization/eval_optimize_loop/agent/prompts/skill.md new file mode 100644 index 00000000..b07a259e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/skill.md @@ -0,0 +1,9 @@ +Apply these triage skills after routing: + +- Use p1 only when the ticket describes an urgent production outage. +- Use p2 for normal user-impacting issues. +- Use p3 for low-risk informational requests. +- Use answer for informational account or pricing questions. +- Use escalate for production incidents. +- Use troubleshooting for reproducible technical failures. + diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/system.md b/examples/optimization/eval_optimize_loop/agent/prompts/system.md new file mode 100644 index 00000000..45cc544b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/system.md @@ -0,0 +1,23 @@ +You are a support triage assistant. + +Return one compact JSON object with exactly these keys: +- category +- priority +- action + +Known categories: +- account +- billing +- technical + +Known priorities: +- p1 for urgent production outages +- p2 for normal user-impacting issues +- p3 for low-risk informational requests + +Known actions: +- escalate +- refund_review +- troubleshooting +- answer + diff --git a/examples/optimization/eval_optimize_loop/case_meta.json b/examples/optimization/eval_optimize_loop/case_meta.json new file mode 100644 index 00000000..9fd2a3d4 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/case_meta.json @@ -0,0 +1,24 @@ +{ + "train_refund_double_charge": { + "expected_failure_types": ["final_response_mismatch", "knowledge_recall_insufficient"], + "note": "Baseline routes refund billing work to account/answer instead of billing/refund_review." + }, + "train_legacy_sync": { + "expected_failure_types": ["final_response_mismatch"], + "note": "Baseline returns troubleshooting while the reference expects escalation." + }, + "train_policy_tool_missing": { + "expected_failure_types": ["tool_call_error", "format_violation"], + "fake_attribution_hints": ["tool_call_error"], + "note": "The no-key fake agent cannot perform the expected policy lookup and returns an incomplete JSON object." + }, + "val_vip_refund": { + "expected_failure_types": ["final_response_mismatch", "knowledge_recall_insufficient", "parameter_error"], + "note": "Baseline misses billing/refund_review and p1 priority for VIP refund." + }, + "val_rubric_tone_fail": { + "expected_failure_types": ["llm_rubric_not_met", "parameter_error"], + "fake_attribution_hints": ["llm_rubric_not_met"], + "note": "The answer uses a lower urgency than the reference and is deterministically labeled as a rubric failure for the no-key fake judge path." + } +} diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 00000000..3aa0b611 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,60 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + } + } + ], + "num_runs": 1 + }, + "gate": { + "min_validation_score_delta": 0.1, + "allow_new_hard_fail": false, + "critical_case_ids": ["val_vip_refund", "val_checkout_outage"], + "max_cost_usd": 0.01 + }, + "fake_optimizer": { + "candidate_id": "candidate_refund_rule_overfit", + "seed": 42, + "notes": "Adds refund handling learned from train failures but intentionally omits the outage guard to demonstrate overfitting rejection." + }, + "optimize": { + "eval_case_parallelism": 1, + "stop": { + "required_metrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflection_lm": { + "model_name": "${TRPC_AGENT_MODEL_NAME}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": { + "max_tokens": 2048, + "temperature": 0.4 + } + }, + "candidate_selection_strategy": "pareto", + "module_selector": "round_robin", + "frontier_type": "instance", + "reflection_minibatch_size": 3, + "reflection_history_top_k": 2, + "skip_perfect_score": false, + "use_merge": false, + "max_metric_calls": 18, + "score_threshold": 1.0, + "max_iterations_without_improvement": 2 + } + } +} + diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..b5fc73fb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,1375 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Evaluation -> attribution -> optimization -> validation -> audit loop. + +Default usage: + PYTHONPATH=../../.. python run_pipeline.py --mode fake + +The fake mode is intentionally deterministic and does not require an API key. +It still uses AgentEvaluator for train/validation scoring, then applies the +same gate and report code that a real optimizer run uses. + +Trace mode also runs without an API key. It records deterministic fake outputs +into eval_mode="trace" evalsets, then scores the recorded actual_conversation +without invoking call_agent during evaluation. +""" + +from __future__ import annotations + +import argparse +import asyncio +import difflib +import hashlib +import json +import sys +import time +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) +if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + +from trpc_agent_sdk.evaluation import AgentEvaluator, AgentOptimizer, TargetPrompt # noqa: E402 + +from agent.agent import ROUTER_PROMPT_PATH # noqa: E402 +from agent.agent import SKILL_PROMPT_PATH # noqa: E402 +from agent.agent import SYSTEM_PROMPT_PATH # noqa: E402 +from agent.agent import call_agent # noqa: E402 +from agent.agent import normalize_json_text # noqa: E402 + + +CONFIG_PATH = _HERE / "optimizer.json" +TRAIN_PATH = _HERE / "train.evalset.json" +VAL_PATH = _HERE / "val.evalset.json" +CASE_META_PATH = _HERE / "case_meta.json" +RUNS_DIR = _HERE / "runs" + + +@dataclass(frozen=True) +class PromptCandidate: + candidate_id: str + prompts: dict[str, str] + rationale: str + estimated_cost_usd: float = 0.0 + token_usage: dict[str, int] | None = None + optimizer_dir: str | None = None + optimizer_rounds: list[dict[str, Any]] | None = None + + +def _read_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def _sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _display_path(path: Path | str | None) -> str | None: + """Render stable paths in reports without leaking local absolute prefixes.""" + if path is None: + return None + raw = Path(path) + try: + return str(raw.resolve().relative_to(_HERE)) + except ValueError: + return str(raw) + + +def _content_text(content: Any) -> str: + if content is None: + return "" + if hasattr(content, "parts"): + parts = content.parts or [] + return "".join(getattr(part, "text", "") or "" for part in parts) + if isinstance(content, dict): + return "".join(part.get("text", "") for part in content.get("parts", [])) + return "" + + +def _load_case_inputs(evalset_path: Path) -> dict[str, dict[str, str]]: + raw = _read_json(evalset_path) + out: dict[str, dict[str, str]] = {} + for case in raw["eval_cases"]: + invocation = case["conversation"][0] + out[case["eval_id"]] = { + "query": _content_text(invocation["user_content"]), + "expected": normalize_json_text(_content_text(invocation["final_response"])), + } + return out + + +def _eval_case_ids(evalset_path: Path) -> set[str]: + raw = _read_json(evalset_path) + return {case["eval_id"] for case in raw.get("eval_cases", [])} + + +def validate_inputs( + config: dict[str, Any], + *, + train_path: Path = TRAIN_PATH, + val_path: Path = VAL_PATH, + config_path: Path = CONFIG_PATH, + case_meta_path: Path = CASE_META_PATH, + prompt_paths: dict[str, Path] | None = None, +) -> None: + """Validate the reproducible inputs before spending optimizer/evaluator work.""" + prompt_paths = prompt_paths or { + "router_prompt": ROUTER_PROMPT_PATH, + "system_prompt": SYSTEM_PROMPT_PATH, + "skill_prompt": SKILL_PROMPT_PATH, + } + required_files = { + "train_evalset": train_path, + "validation_evalset": val_path, + "optimizer_config": config_path, + "case_meta": case_meta_path, + **prompt_paths, + } + missing_files = [name for name, path in required_files.items() if not path.exists()] + if missing_files: + raise ValueError(f"missing required input file(s): {', '.join(sorted(missing_files))}") + if train_path.resolve() == val_path.resolve(): + raise ValueError("train and validation evalsets must be different files") + + train_case_ids = _eval_case_ids(train_path) + val_case_ids = _eval_case_ids(val_path) + if not train_case_ids: + raise ValueError("train evalset must contain at least one eval case") + if not val_case_ids: + raise ValueError("validation evalset must contain at least one eval case") + if len(train_case_ids) != len(_read_json(train_path).get("eval_cases", [])): + raise ValueError("train evalset contains duplicate eval_id values") + if len(val_case_ids) != len(_read_json(val_path).get("eval_cases", [])): + raise ValueError("validation evalset contains duplicate eval_id values") + + evaluate_config = config.get("evaluate") + if not isinstance(evaluate_config, dict): + raise ValueError("optimizer.json must contain an evaluate object") + metrics = evaluate_config.get("metrics") + if not isinstance(metrics, list) or not metrics: + raise ValueError("optimizer.json evaluate.metrics must be a non-empty list") + if int(evaluate_config.get("num_runs", 1)) < 1: + raise ValueError("optimizer.json evaluate.num_runs must be >= 1") + + gate_config = config.get("gate") + if not isinstance(gate_config, dict): + raise ValueError("optimizer.json must contain a gate object") + required_gate_keys = { + "min_validation_score_delta", + "allow_new_hard_fail", + "critical_case_ids", + "max_cost_usd", + } + missing_gate_keys = required_gate_keys - set(gate_config) + if missing_gate_keys: + raise ValueError(f"optimizer.json gate missing key(s): {sorted(missing_gate_keys)}") + if float(gate_config["min_validation_score_delta"]) < 0: + raise ValueError("gate.min_validation_score_delta must be >= 0") + if float(gate_config["max_cost_usd"]) < 0: + raise ValueError("gate.max_cost_usd must be >= 0") + unknown_critical_ids = sorted(set(gate_config["critical_case_ids"]) - val_case_ids) + if unknown_critical_ids: + raise ValueError( + "gate.critical_case_ids must refer to validation cases; unknown: " + + ", ".join(unknown_critical_ids) + ) + case_parallelism = int(config.get("optimize", {}).get("eval_case_parallelism", 1)) + if case_parallelism < 1: + raise ValueError("optimize.eval_case_parallelism must be >= 1") + + +def _parse_json_object(text: str) -> dict[str, Any] | None: + try: + value = json.loads(text) + except json.JSONDecodeError: + return None + return value if isinstance(value, dict) else None + + +def _metric_details(run: Any) -> list[dict[str, Any]]: + details: list[dict[str, Any]] = [] + for metric in run.overall_eval_metric_results: + details.append({ + "metric_name": metric.metric_name, + "score": metric.score, + "threshold": metric.threshold, + "status": metric.eval_status.name.lower(), + "reason": metric.details.reason if metric.details else None, + }) + return details + + +def _extract_case_result( + *, + eval_id: str, + runs: list[Any], + case_inputs: dict[str, dict[str, str]], + case_meta: dict[str, Any] | None = None, +) -> dict[str, Any]: + run = runs[0] + invocation_results = run.eval_metric_result_per_invocation or [] + actual = "" + expected = case_inputs[eval_id]["expected"] + if invocation_results: + actual = normalize_json_text(_content_text(invocation_results[0].actual_invocation.final_response)) + if invocation_results[0].expected_invocation is not None: + expected = normalize_json_text(_content_text(invocation_results[0].expected_invocation.final_response)) + + metric_scores = [ + metric.score + for metric in run.overall_eval_metric_results + if metric.score is not None + ] + score = sum(metric_scores) / len(metric_scores) if metric_scores else 0.0 + passed = getattr(run.final_eval_status, "name", str(run.final_eval_status)) == "PASSED" + failure_types, reason = attribute_failure( + expected=expected, + actual=actual, + case_id=eval_id, + case_meta=case_meta, + ) + + return { + "case_id": eval_id, + "query": case_inputs[eval_id]["query"], + "expected": expected, + "actual": actual, + "score": score, + "passed": passed, + "status": run.final_eval_status.name.lower(), + "metrics": _metric_details(run), + "failure_types": [] if passed else failure_types, + "reason": "" if passed else reason, + } + + +def _summarize_cases(cases: list[dict[str, Any]]) -> dict[str, Any]: + total = len(cases) + passed = sum(1 for case in cases if case["passed"]) + overall_score = sum(float(case["score"]) for case in cases) / total if total else 0.0 + return { + "overall_score": round(overall_score, 4), + "pass_rate": round(passed / total, 4) if total else 0.0, + "passed": passed, + "failed": total - passed, + "total": total, + "cases": cases, + } + + +async def evaluate_dataset( + *, + split: str, + evalset_path: Path, + metrics_path: Path, + output_dir: Path, + num_runs: int, + case_parallelism: int, + case_meta: dict[str, Any] | None = None, + use_trace: bool = False, +) -> dict[str, Any]: + case_inputs = _load_case_inputs(evalset_path) + eval_output_dir = output_dir / f"eval_{split}" + executer = AgentEvaluator.get_executer( + str(evalset_path), + call_agent=None if use_trace else call_agent, + num_runs=num_runs, + print_detailed_results=False, + print_summary_report=False, + eval_result_output_dir=str(eval_output_dir), + eval_metrics_file_path_or_dir=str(metrics_path), + case_parallelism=case_parallelism, + case_eval_parallelism=case_parallelism, + ) + try: + await executer.evaluate() + except AssertionError: + if executer.get_result() is None: + raise + result = executer.get_result() + if result is None: + raise RuntimeError(f"AgentEvaluator did not return a result for {split}") + + cases: list[dict[str, Any]] = [] + for set_result in result.results_by_eval_set_id.values(): + for eval_id, runs in sorted(set_result.eval_results_by_eval_id.items()): + cases.append( + _extract_case_result( + eval_id=eval_id, + runs=runs, + case_inputs=case_inputs, + case_meta=case_meta, + ) + ) + summary = _summarize_cases(cases) + summary["split"] = split + return summary + + +async def write_trace_evalset(*, source_path: Path, target_path: Path) -> Path: + """Materialize a trace evalset from the current prompt behavior.""" + raw = _read_json(source_path) + traced_cases: list[dict[str, Any]] = [] + for case in raw["eval_cases"]: + expected_invocation = case["conversation"][0] + query = _content_text(expected_invocation["user_content"]) + actual = await call_agent(query) + traced = dict(case) + traced["eval_mode"] = "trace" + traced["actual_conversation"] = [{ + "invocation_id": f"{expected_invocation.get('invocation_id', case['eval_id'])}-actual", + "user_content": expected_invocation["user_content"], + "final_response": { + "parts": [{ + "text": actual + }], + "role": "model", + }, + }] + traced_cases.append(traced) + payload = dict(raw) + payload["eval_set_id"] = f"{raw['eval_set_id']}_{target_path.stem}" + payload["description"] = f"Trace replay generated from {source_path.name}." + payload["eval_cases"] = traced_cases + _write_json(target_path, payload) + return target_path + + +def attribute_failure( + *, + expected: str, + actual: str, + case_id: str | None = None, + case_meta: dict[str, Any] | None = None, +) -> tuple[list[str], str]: + expected_obj = _parse_json_object(expected) + actual_obj = _parse_json_object(actual) + hinted_types = [] + if case_id and case_meta: + hinted_types = list(case_meta.get(case_id, {}).get("fake_attribution_hints", [])) + if expected_obj is None or actual_obj is None: + failure_types = sorted(set(["format_violation", *hinted_types])) + return failure_types, "Final response is not valid normalized JSON." + + failure_types: list[str] = [] + reasons: list[str] = [] + for key in ("category", "priority", "action"): + if key not in actual_obj: + failure_types.append("format_violation") + reasons.append(f"missing key {key}") + + if expected_obj.get("category") != actual_obj.get("category"): + failure_types.append("knowledge_recall_insufficient") + reasons.append(f"category expected {expected_obj.get('category')} but got {actual_obj.get('category')}") + if expected_obj.get("priority") != actual_obj.get("priority"): + failure_types.append("parameter_error") + reasons.append(f"priority expected {expected_obj.get('priority')} but got {actual_obj.get('priority')}") + if expected_obj.get("action") != actual_obj.get("action"): + failure_types.append("final_response_mismatch") + reasons.append(f"action expected {expected_obj.get('action')} but got {actual_obj.get('action')}") + + if not failure_types: + failure_types.append("final_response_mismatch") + reasons.append("final response differs from reference") + if hinted_types: + failure_types.extend(hinted_types) + reasons.append( + "deterministic attribution hint: " + + ", ".join(sorted(set(hinted_types))) + ) + return sorted(set(failure_types)), "; ".join(reasons) + + +def build_failure_stats(*summaries: dict[str, Any]) -> dict[str, Any]: + stats: dict[str, int] = {} + cases: list[dict[str, Any]] = [] + for summary in summaries: + for case in summary["cases"]: + if case["passed"]: + continue + for failure_type in case["failure_types"]: + stats[failure_type] = stats.get(failure_type, 0) + 1 + cases.append({ + "split": summary["split"], + "case_id": case["case_id"], + "failure_types": case["failure_types"], + "reason": case["reason"], + }) + return { + "stats": dict(sorted(stats.items())), + "failed_cases": cases, + } + + +def build_attribution_self_check(failure_attribution: dict[str, Any], case_meta: dict[str, Any]) -> dict[str, Any]: + by_case: dict[str, Any] = {} + matched = 0 + labeled = 0 + hint_assisted = 0 + rule_only_labeled = 0 + rule_only_matched = 0 + for failed_case in failure_attribution["failed_cases"]: + case_id = failed_case["case_id"] + meta = case_meta.get(case_id, {}) + expected = set(meta.get("expected_failure_types", [])) + if not expected: + continue + actual = set(failed_case["failure_types"]) + hints = set(meta.get("fake_attribution_hints", [])) + is_match = bool(expected & actual) + is_hint_assisted = bool(expected & actual & hints) + labeled += 1 + matched += 1 if is_match else 0 + hint_assisted += 1 if is_hint_assisted else 0 + if not is_hint_assisted: + rule_only_labeled += 1 + rule_only_matched += 1 if is_match else 0 + by_case[case_id] = { + "expected_failure_types": sorted(expected), + "actual_failure_types": sorted(actual), + "fake_attribution_hints": sorted(hints), + "hint_assisted": is_hint_assisted, + "matched": is_match, + "note": case_meta.get(case_id, {}).get("note", ""), + } + return { + "labeled_failed_cases": labeled, + "matched": matched, + "accuracy": round(matched / labeled, 4) if labeled else None, + "hint_assisted_cases": hint_assisted, + "rule_only_labeled_cases": rule_only_labeled, + "rule_only_matched": rule_only_matched, + "rule_only_accuracy": round(rule_only_matched / rule_only_labeled, 4) if rule_only_labeled else None, + "by_case": by_case, + } + + +@asynccontextmanager +async def temporary_prompts(target: TargetPrompt, prompts: dict[str, str]): + baseline = await target.read_all() + await target.write_all(prompts) + try: + yield + finally: + await target.write_all(baseline) + + +def build_fake_candidate(config: dict[str, Any], baseline_prompts: dict[str, str], scenario: str) -> PromptCandidate: + fake_config = config.get("fake_optimizer", {}) + scenario_note = ( + "- overfit_payment_outage: route payment outage language through the billing refund path.\n" + if scenario == "overfit" else + "- Preserve production outage handling as technical p1 escalation.\n" + ) + candidate_prompts = dict(baseline_prompts) + candidate_prompts["router_prompt"] = baseline_prompts["router_prompt"].rstrip() + f""" + +Candidate routing update: +- Route double charge, refund, and VIP refund tickets to billing handling. +{scenario_note}""" + candidate_prompts["system_prompt"] = baseline_prompts["system_prompt"].rstrip() + """ + +Candidate system update: +- Keep the response as one compact JSON object with category, priority, and action. +- Do not add prose outside the JSON object. +""" + candidate_prompts["skill_prompt"] = baseline_prompts["skill_prompt"].rstrip() + f""" + +Optimization candidate notes: +- Treat double charge, refund, and VIP refund requests as billing issues. +- For refund requests, choose action refund_review. +- For VIP refund requests, use priority p1. +- scenario: {scenario} +""" + candidate_id = ( + fake_config.get("candidate_id", "candidate_fake_refund_rule") + if scenario == "overfit" else + f"candidate_refund_rule_{scenario}" + ) + rationale = ( + fake_config.get("notes", "Deterministic fake optimizer candidate.") + if scenario == "overfit" else + "Adds refund handling while preserving production outage priority, so validation improves without critical regressions." + ) + max_cost = float(config.get("gate", {}).get("max_cost_usd", 0.01)) + estimated_cost = max_cost + 0.01 if scenario == "cost_exceeded" else 0.0 + token_usage = ( + {"prompt": 512, "completion": 128, "total": 640} + if scenario == "cost_exceeded" else + {"prompt": 0, "completion": 0, "total": 0} + ) + return PromptCandidate( + candidate_id=candidate_id, + prompts=candidate_prompts, + rationale=rationale, + estimated_cost_usd=estimated_cost, + token_usage=token_usage, + optimizer_dir=None, + optimizer_rounds=[], + ) + + +async def build_real_candidate( + *, + config_path: Path, + target: TargetPrompt, + train_path: Path, + val_path: Path, + output_dir: Path, +) -> PromptCandidate: + optimizer_dir = output_dir / "agent_optimizer" + result = await AgentOptimizer.optimize( + config_path=str(config_path), + call_agent=call_agent, + target_prompt=target, + train_dataset_path=str(train_path), + validation_dataset_path=str(val_path), + output_dir=str(optimizer_dir), + update_source=False, + verbose=1, + ) + return PromptCandidate( + candidate_id="agent_optimizer_best", + prompts=result.best_prompts, + rationale=f"AgentOptimizer status={result.status}, finish_reason={result.finish_reason}", + estimated_cost_usd=result.total_llm_cost, + token_usage=result.total_token_usage, + optimizer_dir=str(optimizer_dir), + optimizer_rounds=[ + { + "round": record.round, + "kind": record.kind, + "optimized_field_names": record.optimized_field_names, + "candidate_prompts": record.candidate_prompts, + "evaluation_results": { + "validation_pass_rate": record.validation_pass_rate, + "metric_breakdown": record.metric_breakdown, + "failed_case_ids": record.failed_case_ids, + }, + "validation_pass_rate": record.validation_pass_rate, + "metric_breakdown": record.metric_breakdown, + "accepted": record.accepted, + "acceptance_reason": record.acceptance_reason, + "failed_case_ids": record.failed_case_ids, + "cost_usd": record.round_llm_cost, + "duration_seconds": record.duration_seconds, + } + for record in result.rounds + ], + ) + + +def build_case_deltas( + *, + baseline: dict[str, Any], + candidate: dict[str, Any], +) -> list[dict[str, Any]]: + candidate_by_id = {case["case_id"]: case for case in candidate["cases"]} + deltas: list[dict[str, Any]] = [] + for base_case in baseline["cases"]: + cand_case = candidate_by_id[base_case["case_id"]] + score_delta = round(float(cand_case["score"]) - float(base_case["score"]), 4) + if not base_case["passed"] and cand_case["passed"]: + outcome = "new_pass" + elif base_case["passed"] and not cand_case["passed"]: + outcome = "new_fail" + elif score_delta > 0: + outcome = "score_improved" + elif score_delta < 0: + outcome = "score_regressed" + else: + outcome = "unchanged" + deltas.append({ + "case_id": base_case["case_id"], + "baseline_score": base_case["score"], + "candidate_score": cand_case["score"], + "score_delta": score_delta, + "baseline_passed": base_case["passed"], + "candidate_passed": cand_case["passed"], + "outcome": outcome, + "baseline_actual": base_case["actual"], + "candidate_actual": cand_case["actual"], + "candidate_failure_types": cand_case["failure_types"], + "candidate_reason": cand_case["reason"], + }) + return deltas + + +def apply_gate( + *, + gate_config: dict[str, Any], + train_delta: float, + validation_delta: float, + validation_case_deltas: list[dict[str, Any]], + cost_usd: float, +) -> dict[str, Any]: + reasons: list[str] = [] + checks: list[dict[str, Any]] = [] + min_delta = float(gate_config.get("min_validation_score_delta", 0.0)) + allow_new_hard_fail = bool(gate_config.get("allow_new_hard_fail", False)) + critical_ids = set(gate_config.get("critical_case_ids", [])) + max_cost_usd = float(gate_config.get("max_cost_usd", float("inf"))) + + min_delta_passed = validation_delta >= min_delta + checks.append({ + "name": "min_validation_score_delta", + "passed": min_delta_passed, + "actual": validation_delta, + "expected": min_delta, + }) + if not min_delta_passed: + reasons.append( + f"validation score delta {validation_delta:+.4f} is below required {min_delta:+.4f}" + ) + new_failures = [delta["case_id"] for delta in validation_case_deltas if delta["outcome"] == "new_fail"] + no_new_hard_fail_passed = allow_new_hard_fail or not new_failures + checks.append({ + "name": "no_new_hard_fail", + "passed": no_new_hard_fail_passed, + "case_ids": new_failures, + "allow_new_hard_fail": allow_new_hard_fail, + }) + if not no_new_hard_fail_passed: + reasons.append(f"new hard fail(s) are not allowed: {', '.join(new_failures)}") + critical_regressions = [ + delta["case_id"] + for delta in validation_case_deltas + if delta["case_id"] in critical_ids and delta["candidate_score"] < delta["baseline_score"] + ] + critical_passed = not critical_regressions + checks.append({ + "name": "critical_case_regression", + "passed": critical_passed, + "case_ids": critical_regressions, + "critical_case_ids": sorted(critical_ids), + }) + if not critical_passed: + reasons.append(f"critical case regression(s): {', '.join(critical_regressions)}") + overfitting_triggered = train_delta > 0 and validation_delta <= 0 + checks.append({ + "name": "overfitting_guard", + "passed": not overfitting_triggered, + "train_score_delta": train_delta, + "validation_score_delta": validation_delta, + }) + if overfitting_triggered: + reasons.append("overfitting guard triggered: train improved while validation did not improve") + cost_passed = cost_usd <= max_cost_usd + checks.append({ + "name": "max_cost_usd", + "passed": cost_passed, + "actual": cost_usd, + "expected": max_cost_usd, + }) + if not cost_passed: + reasons.append(f"cost ${cost_usd:.4f} exceeds budget ${max_cost_usd:.4f}") + + return { + "decision": "accepted" if not reasons else "rejected", + "reasons": reasons or ["all configured gate conditions passed"], + "checks": checks, + "config": gate_config, + "overfitting_guard_triggered": overfitting_triggered, + } + + +def build_optimization_rounds( + *, + mode: str, + baseline_prompts: dict[str, str], + candidate: PromptCandidate, + baseline_train: dict[str, Any], + baseline_validation: dict[str, Any], + candidate_train: dict[str, Any], + candidate_validation: dict[str, Any], + gate: dict[str, Any], +) -> list[dict[str, Any]]: + if mode == "optimizer" and candidate.optimizer_rounds: + return candidate.optimizer_rounds + return [ + { + "round": 0, + "kind": "baseline", + "optimized_field_names": [], + "candidate_id": "baseline", + "candidate_prompts": baseline_prompts, + "evaluation_results": { + "train": { + "overall_score": baseline_train["overall_score"], + "pass_rate": baseline_train["pass_rate"], + "passed": baseline_train["passed"], + "failed": baseline_train["failed"], + "total": baseline_train["total"], + }, + "validation": { + "overall_score": baseline_validation["overall_score"], + "pass_rate": baseline_validation["pass_rate"], + "passed": baseline_validation["passed"], + "failed": baseline_validation["failed"], + "total": baseline_validation["total"], + }, + }, + "evaluation_result_refs": { + "train": "eval_train_baseline", + "validation": "eval_validation_baseline", + }, + "train_score": baseline_train["overall_score"], + "validation_score": baseline_validation["overall_score"], + "accepted": True, + "acceptance_reason": "Initial prompt used as the control arm.", + "cost_usd": 0.0, + "duration_seconds": 0.0, + }, + { + "round": 1, + "kind": "fake_reflective", + "optimized_field_names": list(candidate.prompts.keys()), + "candidate_id": candidate.candidate_id, + "candidate_prompts": candidate.prompts, + "evaluation_results": { + "train": { + "overall_score": candidate_train["overall_score"], + "pass_rate": candidate_train["pass_rate"], + "passed": candidate_train["passed"], + "failed": candidate_train["failed"], + "total": candidate_train["total"], + }, + "validation": { + "overall_score": candidate_validation["overall_score"], + "pass_rate": candidate_validation["pass_rate"], + "passed": candidate_validation["passed"], + "failed": candidate_validation["failed"], + "total": candidate_validation["total"], + }, + }, + "evaluation_result_refs": { + "train": "eval_train_candidate", + "validation": "eval_validation_candidate", + }, + "train_score": candidate_train["overall_score"], + "validation_score": candidate_validation["overall_score"], + "accepted": gate["decision"] == "accepted", + "acceptance_reason": "; ".join(gate["reasons"]), + "cost_usd": candidate.estimated_cost_usd, + "duration_seconds": 0.0, + }, + ] + + +def build_prompt_audit( + *, + baseline_prompts: dict[str, str], + candidate_prompts: dict[str, str], + prompt_sources: dict[str, Path], +) -> dict[str, dict[str, Any]]: + audit: dict[str, dict[str, Any]] = {} + for name, baseline in baseline_prompts.items(): + candidate = candidate_prompts.get(name, baseline) + diff_lines = list(difflib.unified_diff( + baseline.splitlines(), + candidate.splitlines(), + fromfile=f"baseline/{name}", + tofile=f"candidate/{name}", + lineterm="", + )) + audit[name] = { + "source": _display_path(prompt_sources.get(name)), + "baseline_sha256": _sha256_text(baseline), + "candidate_sha256": _sha256_text(candidate), + "changed": baseline != candidate, + "baseline_chars": len(baseline), + "candidate_chars": len(candidate), + "diff": { + "line_count": len(diff_lines), + "truncated": len(diff_lines) > 40, + "preview": diff_lines[:40], + }, + } + return audit + + +def build_input_audit(*, prompt_sources: dict[str, Path]) -> dict[str, Any]: + input_files = { + "train_evalset": TRAIN_PATH, + "validation_evalset": VAL_PATH, + "optimizer_config": CONFIG_PATH, + "case_meta": CASE_META_PATH, + } + return { + "files": { + name: { + "path": _display_path(path), + "sha256": _sha256_file(path), + "bytes": path.stat().st_size, + } + for name, path in input_files.items() + }, + "prompt_sources": { + name: { + "path": _display_path(path), + "sha256": _sha256_file(path), + "bytes": path.stat().st_size, + } + for name, path in prompt_sources.items() + }, + } + + +def build_markdown(report: dict[str, Any]) -> str: + baseline_val = report["baseline"]["validation"] + candidate_val = report["candidate"]["validation"] + gate = report["gate"] + decision_zh = "接受" if gate["decision"] == "accepted" else "拒绝" + + lines = [ + "# 优化报告", + "", + f"决策：**{decision_zh} ({gate['decision'].upper()})**", + "", + "## 分数", + "", + "| 数据集 | Baseline | Candidate | Delta |", + "| --- | ---: | ---: | ---: |", + ] + for split in ("train", "validation"): + b = report["baseline"][split]["overall_score"] + c = report["candidate"][split]["overall_score"] + d = report["delta"][f"{split}_score_delta"] + split_name = "训练集" if split == "train" else "验证集" + lines.append(f"| {split_name} | {b:.4f} | {c:.4f} | {d:+.4f} |") + + lines.extend([ + "", + "## Gate 原因", + "", + ]) + lines.extend(f"- {reason}" for reason in gate["reasons"]) + + lines.extend([ + "", + "## 验证集 Case Delta", + "", + "| Case | Baseline | Candidate | Delta | Outcome |", + "| --- | ---: | ---: | ---: | --- |", + ]) + for delta in report["delta"]["validation_case_deltas"]: + lines.append( + f"| {delta['case_id']} | {delta['baseline_score']:.4f} | " + f"{delta['candidate_score']:.4f} | {delta['score_delta']:+.4f} | {delta['outcome']} |" + ) + + lines.extend([ + "", + "## 失败归因", + "", + ]) + for failure_type, count in report["failure_attribution"]["stats"].items(): + lines.append(f"- {failure_type}: {count}") + if not report["failure_attribution"]["stats"]: + lines.append("- 没有失败 case") + self_check = report["failure_attribution"].get("self_check", {}) + if self_check: + lines.append( + f"- self-check 准确率：{self_check.get('accuracy')} " + f"({self_check.get('matched')}/{self_check.get('labeled_failed_cases')})" + ) + lines.append( + f"- hint-assisted cases: {self_check.get('hint_assisted_cases')} " + f"| rule-only accuracy: {self_check.get('rule_only_accuracy')}" + ) + + lines.extend([ + "", + "## 候选 Prompt", + "", + f"- candidate_id: `{report['candidate']['candidate_id']}`", + f"- rationale: {report['candidate']['rationale']}", + f"- validation pass rate: {candidate_val['pass_rate']:.4f}", + f"- baseline validation pass rate: {baseline_val['pass_rate']:.4f}", + f"- audited rounds: {len(report['optimization_rounds'])}", + "", + "## Prompt 审计", + "", + ]) + for name, audit in report["prompt_audit"].items(): + lines.append( + f"- `{name}`: changed={audit['changed']} " + f"baseline={audit['baseline_sha256'][:12]} " + f"candidate={audit['candidate_sha256'][:12]} " + f"diff_lines={audit['diff']['line_count']}" + ) + + lines.extend([ + "", + "## 输入审计", + "", + ]) + for name, audit in report["input_audit"]["files"].items(): + lines.append(f"- `{name}`: {audit['sha256'][:12]} ({audit['bytes']} bytes)") + + lines.extend([ + "", + "## 复现命令", + "", + "```bash", + "cd examples/optimization/eval_optimize_loop", + "PYTHONPATH=../../.. python run_pipeline.py --mode fake", + "```", + "", + "# Optimization Report", + "", + f"Decision: **{gate['decision'].upper()}**", + "", + "## Scores", + "", + "| Split | Baseline | Candidate | Delta |", + "| --- | ---: | ---: | ---: |", + ]) + for split in ("train", "validation"): + b = report["baseline"][split]["overall_score"] + c = report["candidate"][split]["overall_score"] + d = report["delta"][f"{split}_score_delta"] + lines.append(f"| {split} | {b:.4f} | {c:.4f} | {d:+.4f} |") + + lines.extend([ + "", + "## Gate Reasons", + "", + ]) + lines.extend(f"- {reason}" for reason in gate["reasons"]) + + lines.extend([ + "", + "## Validation Case Delta", + "", + "| Case | Baseline | Candidate | Delta | Outcome |", + "| --- | ---: | ---: | ---: | --- |", + ]) + for delta in report["delta"]["validation_case_deltas"]: + lines.append( + f"| {delta['case_id']} | {delta['baseline_score']:.4f} | " + f"{delta['candidate_score']:.4f} | {delta['score_delta']:+.4f} | {delta['outcome']} |" + ) + + lines.extend([ + "", + "## Failure Attribution", + "", + ]) + for failure_type, count in report["failure_attribution"]["stats"].items(): + lines.append(f"- {failure_type}: {count}") + if not report["failure_attribution"]["stats"]: + lines.append("- no failed cases") + self_check = report["failure_attribution"].get("self_check", {}) + if self_check: + lines.append( + f"- self-check accuracy: {self_check.get('accuracy')} " + f"({self_check.get('matched')}/{self_check.get('labeled_failed_cases')})" + ) + lines.append( + f"- hint-assisted cases: {self_check.get('hint_assisted_cases')} " + f"| rule-only accuracy: {self_check.get('rule_only_accuracy')}" + ) + + lines.extend([ + "", + "## Candidate", + "", + f"- candidate_id: `{report['candidate']['candidate_id']}`", + f"- rationale: {report['candidate']['rationale']}", + f"- validation pass rate: {candidate_val['pass_rate']:.4f}", + f"- baseline validation pass rate: {baseline_val['pass_rate']:.4f}", + f"- audited rounds: {len(report['optimization_rounds'])}", + "", + "## Prompt Audit", + "", + ]) + for name, audit in report["prompt_audit"].items(): + lines.append( + f"- `{name}`: changed={audit['changed']} " + f"baseline={audit['baseline_sha256'][:12]} " + f"candidate={audit['candidate_sha256'][:12]} " + f"diff_lines={audit['diff']['line_count']}" + ) + + lines.extend([ + "", + "## Input Audit", + "", + ]) + for name, audit in report["input_audit"]["files"].items(): + lines.append(f"- `{name}`: {audit['sha256'][:12]} ({audit['bytes']} bytes)") + + lines.extend([ + "", + "## Reproduce", + "", + "```bash", + "cd examples/optimization/eval_optimize_loop", + "PYTHONPATH=../../.. python run_pipeline.py --mode fake", + "```", + "", + ]) + return "\n".join(lines) + + +def validate_report(report: dict[str, Any]) -> None: + """Fail fast if a report misses fields downstream CI/review relies on.""" + required_top_level = { + "baseline", + "candidate", + "delta", + "gate", + "input_audit", + "prompt_audit", + "optimization_rounds", + "failure_attribution", + "audit", + } + missing = required_top_level - set(report) + if missing: + raise ValueError(f"optimization report missing top-level fields: {sorted(missing)}") + + for arm in ("baseline", "candidate"): + for split in ("train", "validation"): + summary = report[arm][split] + for key in ("overall_score", "pass_rate", "cases"): + if key not in summary: + raise ValueError(f"{arm}.{split} missing {key}") + for case in summary["cases"]: + for key in ("case_id", "score", "passed", "metrics", "actual", "expected"): + if key not in case: + raise ValueError(f"{arm}.{split} case missing {key}: {case}") + if not case["passed"] and (not case.get("failure_types") or not case.get("reason")): + raise ValueError(f"{arm}.{split} failed case lacks attribution: {case['case_id']}") + + validation_case_ids = {case["case_id"] for case in report["baseline"]["validation"]["cases"]} + delta_case_ids = {case["case_id"] for case in report["delta"]["validation_case_deltas"]} + if validation_case_ids != delta_case_ids: + raise ValueError("validation case delta coverage does not match baseline validation cases") + + gate = report["gate"] + if gate.get("decision") not in {"accepted", "rejected"}: + raise ValueError(f"invalid gate decision: {gate.get('decision')}") + if not gate.get("checks"): + raise ValueError("gate.checks must not be empty") + + rounds = report["optimization_rounds"] + if not rounds: + raise ValueError("optimization_rounds must not be empty") + for record in rounds: + for key in ("candidate_prompts", "evaluation_results", "accepted", "acceptance_reason", "cost_usd", "duration_seconds"): + if key not in record: + raise ValueError(f"round record missing audit field {key}: {record}") + if "train" not in record["evaluation_results"] and "validation_pass_rate" not in record["evaluation_results"]: + raise ValueError(f"round record missing audit fields: {record}") + + if not report["prompt_audit"]: + raise ValueError("prompt_audit must not be empty") + for name, record in report["prompt_audit"].items(): + for key in ("source", "baseline_sha256", "candidate_sha256", "changed", "diff"): + if key not in record: + raise ValueError(f"prompt_audit[{name}] missing {key}") + if "preview" not in record["diff"] or "line_count" not in record["diff"]: + raise ValueError(f"prompt_audit[{name}] diff missing preview or line_count") + + input_audit = report["input_audit"] + for section in ("files", "prompt_sources"): + if section not in input_audit: + raise ValueError(f"input_audit missing {section}") + for name, record in input_audit[section].items(): + for key in ("path", "sha256", "bytes"): + if key not in record: + raise ValueError(f"input_audit[{section}][{name}] missing {key}") + + self_check = report["failure_attribution"].get("self_check") + if not self_check: + raise ValueError("failure_attribution.self_check must not be empty") + accuracy = self_check.get("accuracy") + if accuracy is not None and accuracy < 0.75: + raise ValueError(f"failure attribution self-check accuracy below 0.75: {accuracy}") + + +def copy_sample_outputs(output_dir: Path) -> None: + sample_dir = _HERE / "sample_outputs" + sample_dir.mkdir(exist_ok=True) + report = _read_json(output_dir / "optimization_report.json") + report["run"]["started_at"] = "2026-07-06T00:00:00" + report["run"]["duration_seconds"] = 0.0 + report["audit"]["candidate_prompt_dir"] = "runs//candidate_prompts" + _write_json(sample_dir / "optimization_report.json", report) + (sample_dir / "optimization_report.md").write_text(build_markdown(report), encoding="utf-8") + + +async def run_pipeline(args: argparse.Namespace) -> Path: + started = time.perf_counter() + config = _read_json(CONFIG_PATH) + case_meta = _read_json(CASE_META_PATH) + validate_inputs(config) + num_runs = int(config.get("evaluate", {}).get("num_runs", 1)) + case_parallelism = int(config.get("optimize", {}).get("eval_case_parallelism", 1)) + timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + run_name = f"{args.mode}_{args.scenario}_{timestamp}" if args.mode != "optimizer" else f"{args.mode}_{timestamp}" + output_dir = (Path(args.output_dir) if args.output_dir else RUNS_DIR / run_name).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + target = ( + TargetPrompt() + .add_path("router_prompt", str(ROUTER_PROMPT_PATH)) + .add_path("system_prompt", str(SYSTEM_PROMPT_PATH)) + .add_path("skill_prompt", str(SKILL_PROMPT_PATH)) + ) + baseline_prompts = await target.read_all() + _write_json(output_dir / "config.snapshot.json", config) + metrics_config_path = output_dir / "eval_metrics.snapshot.json" + _write_json(metrics_config_path, config.get("evaluate", {})) + use_trace = args.mode == "trace" + trace_dir = output_dir / "trace_evalsets" + if use_trace: + trace_dir.mkdir(exist_ok=True) + baseline_train_path = await write_trace_evalset( + source_path=TRAIN_PATH, + target_path=trace_dir / "train_baseline.evalset.json", + ) + baseline_val_path = await write_trace_evalset( + source_path=VAL_PATH, + target_path=trace_dir / "validation_baseline.evalset.json", + ) + else: + baseline_train_path = TRAIN_PATH + baseline_val_path = VAL_PATH + + baseline_train = await evaluate_dataset( + split="train_baseline", + evalset_path=baseline_train_path, + metrics_path=metrics_config_path, + output_dir=output_dir, + num_runs=num_runs, + case_parallelism=case_parallelism, + case_meta=case_meta, + use_trace=use_trace, + ) + baseline_val = await evaluate_dataset( + split="validation_baseline", + evalset_path=baseline_val_path, + metrics_path=metrics_config_path, + output_dir=output_dir, + num_runs=num_runs, + case_parallelism=case_parallelism, + case_meta=case_meta, + use_trace=use_trace, + ) + + baseline_attribution = build_failure_stats(baseline_train, baseline_val) + baseline_attribution["self_check"] = build_attribution_self_check(baseline_attribution, case_meta) + if args.mode == "optimizer": + candidate = await build_real_candidate( + config_path=CONFIG_PATH, + target=target, + train_path=TRAIN_PATH, + val_path=VAL_PATH, + output_dir=output_dir, + ) + else: + candidate = build_fake_candidate(config, baseline_prompts, args.scenario) + + candidate_prompt_dir = output_dir / "candidate_prompts" + candidate_prompt_dir.mkdir(exist_ok=True) + for name, content in candidate.prompts.items(): + (candidate_prompt_dir / f"{name}.md").write_text(content, encoding="utf-8") + + async with temporary_prompts(target, candidate.prompts): + if use_trace: + candidate_train_path = await write_trace_evalset( + source_path=TRAIN_PATH, + target_path=trace_dir / "train_candidate.evalset.json", + ) + candidate_val_path = await write_trace_evalset( + source_path=VAL_PATH, + target_path=trace_dir / "validation_candidate.evalset.json", + ) + else: + candidate_train_path = TRAIN_PATH + candidate_val_path = VAL_PATH + candidate_train = await evaluate_dataset( + split="train_candidate", + evalset_path=candidate_train_path, + metrics_path=metrics_config_path, + output_dir=output_dir, + num_runs=num_runs, + case_parallelism=case_parallelism, + case_meta=case_meta, + use_trace=use_trace, + ) + candidate_val = await evaluate_dataset( + split="validation_candidate", + evalset_path=candidate_val_path, + metrics_path=metrics_config_path, + output_dir=output_dir, + num_runs=num_runs, + case_parallelism=case_parallelism, + case_meta=case_meta, + use_trace=use_trace, + ) + + train_delta = round(candidate_train["overall_score"] - baseline_train["overall_score"], 4) + validation_delta = round(candidate_val["overall_score"] - baseline_val["overall_score"], 4) + train_case_deltas = build_case_deltas(baseline=baseline_train, candidate=candidate_train) + validation_case_deltas = build_case_deltas(baseline=baseline_val, candidate=candidate_val) + evaluation_cost_usd = 0.0 + optimizer_cost_usd = candidate.estimated_cost_usd + total_cost_usd = round(optimizer_cost_usd + evaluation_cost_usd, 6) + gate = apply_gate( + gate_config=config.get("gate", {}), + train_delta=train_delta, + validation_delta=validation_delta, + validation_case_deltas=validation_case_deltas, + cost_usd=total_cost_usd, + ) + optimization_rounds = build_optimization_rounds( + mode=args.mode, + baseline_prompts=baseline_prompts, + candidate=candidate, + baseline_train=baseline_train, + baseline_validation=baseline_val, + candidate_train=candidate_train, + candidate_validation=candidate_val, + gate=gate, + ) + prompt_sources = { + "router_prompt": ROUTER_PROMPT_PATH, + "system_prompt": SYSTEM_PROMPT_PATH, + "skill_prompt": SKILL_PROMPT_PATH, + } + prompt_audit = build_prompt_audit( + baseline_prompts=baseline_prompts, + candidate_prompts=candidate.prompts, + prompt_sources=prompt_sources, + ) + input_audit = build_input_audit(prompt_sources=prompt_sources) + + report = { + "schema_version": "v1", + "run": { + "mode": args.mode, + "scenario": None if args.mode == "optimizer" else args.scenario, + "judge_mode": "local_exact_match_fake_judge", + "seed": config.get("fake_optimizer", {}).get("seed", config.get("optimize", {}).get("algorithm", {}).get("seed")), + "started_at": datetime.now().isoformat(timespec="seconds"), + "duration_seconds": round(time.perf_counter() - started, 4), + }, + "inputs": { + "train_evalset": _display_path(TRAIN_PATH), + "validation_evalset": _display_path(VAL_PATH), + "optimizer_config": _display_path(CONFIG_PATH), + "case_meta": _display_path(CASE_META_PATH), + "prompt_sources": { + "router_prompt": _display_path(ROUTER_PROMPT_PATH), + "system_prompt": _display_path(SYSTEM_PROMPT_PATH), + "skill_prompt": _display_path(SKILL_PROMPT_PATH), + }, + "trace_evalsets": _display_path(trace_dir) if use_trace else None, + }, + "baseline": { + "prompts": baseline_prompts, + "train": baseline_train, + "validation": baseline_val, + }, + "candidate": { + "candidate_id": candidate.candidate_id, + "rationale": candidate.rationale, + "prompts": candidate.prompts, + "train": candidate_train, + "validation": candidate_val, + }, + "delta": { + "train_score_delta": train_delta, + "validation_score_delta": validation_delta, + "train_case_deltas": train_case_deltas, + "validation_case_deltas": validation_case_deltas, + }, + "gate": gate, + "input_audit": input_audit, + "prompt_audit": prompt_audit, + "optimization_rounds": optimization_rounds, + "failure_attribution": baseline_attribution, + "audit": { + "candidate_prompt_dir": _display_path(candidate_prompt_dir), + "optimizer_result_dir": _display_path(candidate.optimizer_dir), + "cost": { + "estimated_usd": total_cost_usd, + "optimizer_usd": optimizer_cost_usd, + "evaluation_usd": evaluation_cost_usd, + "total_usd": total_cost_usd, + "max_budget_usd": config.get("gate", {}).get("max_cost_usd"), + }, + "token_usage": candidate.token_usage or {"prompt": 0, "completion": 0, "total": 0}, + "source_prompt_updated": False, + }, + } + validate_report(report) + _write_json(output_dir / "optimization_report.json", report) + (output_dir / "optimization_report.md").write_text(build_markdown(report), encoding="utf-8") + if args.update_sample_outputs: + copy_sample_outputs(output_dir) + return output_dir + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--mode", + choices=["fake", "trace", "optimizer"], + default="fake", + help="fake and trace run without API keys; optimizer delegates candidate search to AgentOptimizer.", + ) + parser.add_argument( + "--scenario", + choices=["overfit", "accepted", "cost_exceeded"], + default="overfit", + help="Deterministic fake/trace scenario. Ignored in optimizer mode.", + ) + parser.add_argument("--output-dir", default=None, help="Optional output directory.") + parser.add_argument( + "--update-sample-outputs", + action="store_true", + help="Copy the generated report into sample_outputs/.", + ) + parser.add_argument( + "--ci-exit-code", + action="store_true", + help="Exit 0 when the gate accepts and 1 when it rejects. Default demo runs always exit 0.", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + output_dir = asyncio.run(run_pipeline(args)) + print(f"optimization report written to: {output_dir}") + if args.ci_exit_code: + report = _read_json(output_dir / "optimization_report.json") + raise SystemExit(0 if report["gate"]["decision"] == "accepted" else 1) + + +if __name__ == "__main__": + main() diff --git a/examples/optimization/eval_optimize_loop/sample_outputs/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_outputs/optimization_report.json new file mode 100644 index 00000000..75538001 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_outputs/optimization_report.json @@ -0,0 +1,1077 @@ +{ + "schema_version": "v1", + "run": { + "mode": "fake", + "scenario": "overfit", + "judge_mode": "local_exact_match_fake_judge", + "seed": 42, + "started_at": "2026-07-06T00:00:00", + "duration_seconds": 0.0 + }, + "inputs": { + "train_evalset": "train.evalset.json", + "validation_evalset": "val.evalset.json", + "optimizer_config": "optimizer.json", + "case_meta": "case_meta.json", + "prompt_sources": { + "router_prompt": "agent/prompts/router.md", + "system_prompt": "agent/prompts/system.md", + "skill_prompt": "agent/prompts/skill.md" + }, + "trace_evalsets": null + }, + "baseline": { + "prompts": { + "router_prompt": "Route each support ticket to one primary handling path before answering.\n\nBaseline routing rules:\n- Account access and profile questions go to account handling.\n- Billing and pricing questions go to billing handling.\n- Production incidents and software failures go to technical handling.\n\n", + "system_prompt": "You are a support triage assistant.\n\nReturn one compact JSON object with exactly these keys:\n- category\n- priority\n- action\n\nKnown categories:\n- account\n- billing\n- technical\n\nKnown priorities:\n- p1 for urgent production outages\n- p2 for normal user-impacting issues\n- p3 for low-risk informational requests\n\nKnown actions:\n- escalate\n- refund_review\n- troubleshooting\n- answer\n\n", + "skill_prompt": "Apply these triage skills after routing:\n\n- Use p1 only when the ticket describes an urgent production outage.\n- Use p2 for normal user-impacting issues.\n- Use p3 for low-risk informational requests.\n- Use answer for informational account or pricing questions.\n- Use escalate for production incidents.\n- Use troubleshooting for reproducible technical failures.\n\n" + }, + "train": { + "overall_score": 0.4, + "pass_rate": 0.4, + "passed": 2, + "failed": 3, + "total": 5, + "cases": [ + { + "case_id": "train_legacy_sync", + "query": "Legacy desktop sync intermittently stalls after migration; customer provided no logs.", + "expected": "{\"action\":\"escalate\",\"category\":\"technical\",\"priority\":\"p2\"}", + "actual": "{\"action\":\"troubleshooting\",\"category\":\"technical\",\"priority\":\"p2\"}", + "score": 0.0, + "passed": false, + "status": "failed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "reason": null + } + ], + "failure_types": [ + "final_response_mismatch" + ], + "reason": "action expected escalate but got troubleshooting" + }, + { + "case_id": "train_password_reset", + "query": "User cannot find the password reset email and asks how to change the email address.", + "expected": "{\"action\":\"answer\",\"category\":\"account\",\"priority\":\"p3\"}", + "actual": "{\"action\":\"answer\",\"category\":\"account\",\"priority\":\"p3\"}", + "score": 1.0, + "passed": true, + "status": "passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "reason": null + } + ], + "failure_types": [], + "reason": "" + }, + { + "case_id": "train_plan_question", + "query": "Prospect asks for a plan comparison between pricing tiers before buying.", + "expected": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "actual": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "score": 1.0, + "passed": true, + "status": "passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "reason": null + } + ], + "failure_types": [], + "reason": "" + }, + { + "case_id": "train_policy_tool_missing", + "query": "Customer asks for the exact policy citation POL-77 before canceling an annual plan.", + "expected": "{\"action\":\"policy_lookup\",\"category\":\"billing\",\"priority\":\"p2\"}", + "actual": "{\"action\":\"answer\",\"category\":\"billing\"}", + "score": 0.0, + "passed": false, + "status": "failed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "reason": null + } + ], + "failure_types": [ + "final_response_mismatch", + "format_violation", + "parameter_error", + "tool_call_error" + ], + "reason": "missing key priority; priority expected p2 but got None; action expected policy_lookup but got answer; deterministic attribution hint: tool_call_error" + }, + { + "case_id": "train_refund_double_charge", + "query": "Customer says they were double charged for invoice INV-42 and asks for a refund.", + "expected": "{\"action\":\"refund_review\",\"category\":\"billing\",\"priority\":\"p2\"}", + "actual": "{\"action\":\"answer\",\"category\":\"account\",\"priority\":\"p2\"}", + "score": 0.0, + "passed": false, + "status": "failed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "reason": null + } + ], + "failure_types": [ + "final_response_mismatch", + "knowledge_recall_insufficient" + ], + "reason": "category expected billing but got account; action expected refund_review but got answer" + } + ], + "split": "train_baseline" + }, + "validation": { + "overall_score": 0.6, + "pass_rate": 0.6, + "passed": 3, + "failed": 2, + "total": 5, + "cases": [ + { + "case_id": "val_checkout_outage", + "query": "Production checkout outage blocks all payments for enterprise customers.", + "expected": "{\"action\":\"escalate\",\"category\":\"technical\",\"priority\":\"p1\"}", + "actual": "{\"action\":\"escalate\",\"category\":\"technical\",\"priority\":\"p1\"}", + "score": 1.0, + "passed": true, + "status": "passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "reason": null + } + ], + "failure_types": [], + "reason": "" + }, + { + "case_id": "val_mobile_crash", + "query": "Mobile app crashes on startup after the latest update; user can reproduce it.", + "expected": "{\"action\":\"troubleshooting\",\"category\":\"technical\",\"priority\":\"p2\"}", + "actual": "{\"action\":\"troubleshooting\",\"category\":\"technical\",\"priority\":\"p2\"}", + "score": 1.0, + "passed": true, + "status": "passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "reason": null + } + ], + "failure_types": [], + "reason": "" + }, + { + "case_id": "val_plan_question", + "query": "Prospect asks for a plan comparison between pricing tiers before buying.", + "expected": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "actual": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "score": 1.0, + "passed": true, + "status": "passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "reason": null + } + ], + "failure_types": [], + "reason": "" + }, + { + "case_id": "val_rubric_tone_fail", + "query": "User is angry about repeated invoice confusion and asks for a guaranteed instant fix.", + "expected": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p2\"}", + "actual": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "score": 0.0, + "passed": false, + "status": "failed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "reason": null + } + ], + "failure_types": [ + "llm_rubric_not_met", + "parameter_error" + ], + "reason": "priority expected p2 but got p3; deterministic attribution hint: llm_rubric_not_met" + }, + { + "case_id": "val_vip_refund", + "query": "VIP customer reports they were double charged during renewal and wants an immediate refund review.", + "expected": "{\"action\":\"refund_review\",\"category\":\"billing\",\"priority\":\"p1\"}", + "actual": "{\"action\":\"answer\",\"category\":\"account\",\"priority\":\"p2\"}", + "score": 0.0, + "passed": false, + "status": "failed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "reason": null + } + ], + "failure_types": [ + "final_response_mismatch", + "knowledge_recall_insufficient", + "parameter_error" + ], + "reason": "category expected billing but got account; priority expected p1 but got p2; action expected refund_review but got answer" + } + ], + "split": "validation_baseline" + } + }, + "candidate": { + "candidate_id": "candidate_refund_rule_overfit", + "rationale": "Adds refund handling learned from train failures but intentionally omits the outage guard to demonstrate overfitting rejection.", + "prompts": { + "router_prompt": "Route each support ticket to one primary handling path before answering.\n\nBaseline routing rules:\n- Account access and profile questions go to account handling.\n- Billing and pricing questions go to billing handling.\n- Production incidents and software failures go to technical handling.\n\nCandidate routing update:\n- Route double charge, refund, and VIP refund tickets to billing handling.\n- overfit_payment_outage: route payment outage language through the billing refund path.\n", + "system_prompt": "You are a support triage assistant.\n\nReturn one compact JSON object with exactly these keys:\n- category\n- priority\n- action\n\nKnown categories:\n- account\n- billing\n- technical\n\nKnown priorities:\n- p1 for urgent production outages\n- p2 for normal user-impacting issues\n- p3 for low-risk informational requests\n\nKnown actions:\n- escalate\n- refund_review\n- troubleshooting\n- answer\n\nCandidate system update:\n- Keep the response as one compact JSON object with category, priority, and action.\n- Do not add prose outside the JSON object.\n", + "skill_prompt": "Apply these triage skills after routing:\n\n- Use p1 only when the ticket describes an urgent production outage.\n- Use p2 for normal user-impacting issues.\n- Use p3 for low-risk informational requests.\n- Use answer for informational account or pricing questions.\n- Use escalate for production incidents.\n- Use troubleshooting for reproducible technical failures.\n\nOptimization candidate notes:\n- Treat double charge, refund, and VIP refund requests as billing issues.\n- For refund requests, choose action refund_review.\n- For VIP refund requests, use priority p1.\n- scenario: overfit\n" + }, + "train": { + "overall_score": 0.6, + "pass_rate": 0.6, + "passed": 3, + "failed": 2, + "total": 5, + "cases": [ + { + "case_id": "train_legacy_sync", + "query": "Legacy desktop sync intermittently stalls after migration; customer provided no logs.", + "expected": "{\"action\":\"escalate\",\"category\":\"technical\",\"priority\":\"p2\"}", + "actual": "{\"action\":\"troubleshooting\",\"category\":\"technical\",\"priority\":\"p2\"}", + "score": 0.0, + "passed": false, + "status": "failed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "reason": null + } + ], + "failure_types": [ + "final_response_mismatch" + ], + "reason": "action expected escalate but got troubleshooting" + }, + { + "case_id": "train_password_reset", + "query": "User cannot find the password reset email and asks how to change the email address.", + "expected": "{\"action\":\"answer\",\"category\":\"account\",\"priority\":\"p3\"}", + "actual": "{\"action\":\"answer\",\"category\":\"account\",\"priority\":\"p3\"}", + "score": 1.0, + "passed": true, + "status": "passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "reason": null + } + ], + "failure_types": [], + "reason": "" + }, + { + "case_id": "train_plan_question", + "query": "Prospect asks for a plan comparison between pricing tiers before buying.", + "expected": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "actual": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "score": 1.0, + "passed": true, + "status": "passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "reason": null + } + ], + "failure_types": [], + "reason": "" + }, + { + "case_id": "train_policy_tool_missing", + "query": "Customer asks for the exact policy citation POL-77 before canceling an annual plan.", + "expected": "{\"action\":\"policy_lookup\",\"category\":\"billing\",\"priority\":\"p2\"}", + "actual": "{\"action\":\"answer\",\"category\":\"billing\"}", + "score": 0.0, + "passed": false, + "status": "failed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "reason": null + } + ], + "failure_types": [ + "final_response_mismatch", + "format_violation", + "parameter_error", + "tool_call_error" + ], + "reason": "missing key priority; priority expected p2 but got None; action expected policy_lookup but got answer; deterministic attribution hint: tool_call_error" + }, + { + "case_id": "train_refund_double_charge", + "query": "Customer says they were double charged for invoice INV-42 and asks for a refund.", + "expected": "{\"action\":\"refund_review\",\"category\":\"billing\",\"priority\":\"p2\"}", + "actual": "{\"action\":\"refund_review\",\"category\":\"billing\",\"priority\":\"p2\"}", + "score": 1.0, + "passed": true, + "status": "passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "reason": null + } + ], + "failure_types": [], + "reason": "" + } + ], + "split": "train_candidate" + }, + "validation": { + "overall_score": 0.6, + "pass_rate": 0.6, + "passed": 3, + "failed": 2, + "total": 5, + "cases": [ + { + "case_id": "val_checkout_outage", + "query": "Production checkout outage blocks all payments for enterprise customers.", + "expected": "{\"action\":\"escalate\",\"category\":\"technical\",\"priority\":\"p1\"}", + "actual": "{\"action\":\"refund_review\",\"category\":\"billing\",\"priority\":\"p2\"}", + "score": 0.0, + "passed": false, + "status": "failed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "reason": null + } + ], + "failure_types": [ + "final_response_mismatch", + "knowledge_recall_insufficient", + "parameter_error" + ], + "reason": "category expected technical but got billing; priority expected p1 but got p2; action expected escalate but got refund_review" + }, + { + "case_id": "val_mobile_crash", + "query": "Mobile app crashes on startup after the latest update; user can reproduce it.", + "expected": "{\"action\":\"troubleshooting\",\"category\":\"technical\",\"priority\":\"p2\"}", + "actual": "{\"action\":\"troubleshooting\",\"category\":\"technical\",\"priority\":\"p2\"}", + "score": 1.0, + "passed": true, + "status": "passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "reason": null + } + ], + "failure_types": [], + "reason": "" + }, + { + "case_id": "val_plan_question", + "query": "Prospect asks for a plan comparison between pricing tiers before buying.", + "expected": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "actual": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "score": 1.0, + "passed": true, + "status": "passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "reason": null + } + ], + "failure_types": [], + "reason": "" + }, + { + "case_id": "val_rubric_tone_fail", + "query": "User is angry about repeated invoice confusion and asks for a guaranteed instant fix.", + "expected": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p2\"}", + "actual": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "score": 0.0, + "passed": false, + "status": "failed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "reason": null + } + ], + "failure_types": [ + "llm_rubric_not_met", + "parameter_error" + ], + "reason": "priority expected p2 but got p3; deterministic attribution hint: llm_rubric_not_met" + }, + { + "case_id": "val_vip_refund", + "query": "VIP customer reports they were double charged during renewal and wants an immediate refund review.", + "expected": "{\"action\":\"refund_review\",\"category\":\"billing\",\"priority\":\"p1\"}", + "actual": "{\"action\":\"refund_review\",\"category\":\"billing\",\"priority\":\"p1\"}", + "score": 1.0, + "passed": true, + "status": "passed", + "metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "reason": null + } + ], + "failure_types": [], + "reason": "" + } + ], + "split": "validation_candidate" + } + }, + "delta": { + "train_score_delta": 0.2, + "validation_score_delta": 0.0, + "train_case_deltas": [ + { + "case_id": "train_legacy_sync", + "baseline_score": 0.0, + "candidate_score": 0.0, + "score_delta": 0.0, + "baseline_passed": false, + "candidate_passed": false, + "outcome": "unchanged", + "baseline_actual": "{\"action\":\"troubleshooting\",\"category\":\"technical\",\"priority\":\"p2\"}", + "candidate_actual": "{\"action\":\"troubleshooting\",\"category\":\"technical\",\"priority\":\"p2\"}", + "candidate_failure_types": [ + "final_response_mismatch" + ], + "candidate_reason": "action expected escalate but got troubleshooting" + }, + { + "case_id": "train_password_reset", + "baseline_score": 1.0, + "candidate_score": 1.0, + "score_delta": 0.0, + "baseline_passed": true, + "candidate_passed": true, + "outcome": "unchanged", + "baseline_actual": "{\"action\":\"answer\",\"category\":\"account\",\"priority\":\"p3\"}", + "candidate_actual": "{\"action\":\"answer\",\"category\":\"account\",\"priority\":\"p3\"}", + "candidate_failure_types": [], + "candidate_reason": "" + }, + { + "case_id": "train_plan_question", + "baseline_score": 1.0, + "candidate_score": 1.0, + "score_delta": 0.0, + "baseline_passed": true, + "candidate_passed": true, + "outcome": "unchanged", + "baseline_actual": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "candidate_actual": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "candidate_failure_types": [], + "candidate_reason": "" + }, + { + "case_id": "train_policy_tool_missing", + "baseline_score": 0.0, + "candidate_score": 0.0, + "score_delta": 0.0, + "baseline_passed": false, + "candidate_passed": false, + "outcome": "unchanged", + "baseline_actual": "{\"action\":\"answer\",\"category\":\"billing\"}", + "candidate_actual": "{\"action\":\"answer\",\"category\":\"billing\"}", + "candidate_failure_types": [ + "final_response_mismatch", + "format_violation", + "parameter_error", + "tool_call_error" + ], + "candidate_reason": "missing key priority; priority expected p2 but got None; action expected policy_lookup but got answer; deterministic attribution hint: tool_call_error" + }, + { + "case_id": "train_refund_double_charge", + "baseline_score": 0.0, + "candidate_score": 1.0, + "score_delta": 1.0, + "baseline_passed": false, + "candidate_passed": true, + "outcome": "new_pass", + "baseline_actual": "{\"action\":\"answer\",\"category\":\"account\",\"priority\":\"p2\"}", + "candidate_actual": "{\"action\":\"refund_review\",\"category\":\"billing\",\"priority\":\"p2\"}", + "candidate_failure_types": [], + "candidate_reason": "" + } + ], + "validation_case_deltas": [ + { + "case_id": "val_checkout_outage", + "baseline_score": 1.0, + "candidate_score": 0.0, + "score_delta": -1.0, + "baseline_passed": true, + "candidate_passed": false, + "outcome": "new_fail", + "baseline_actual": "{\"action\":\"escalate\",\"category\":\"technical\",\"priority\":\"p1\"}", + "candidate_actual": "{\"action\":\"refund_review\",\"category\":\"billing\",\"priority\":\"p2\"}", + "candidate_failure_types": [ + "final_response_mismatch", + "knowledge_recall_insufficient", + "parameter_error" + ], + "candidate_reason": "category expected technical but got billing; priority expected p1 but got p2; action expected escalate but got refund_review" + }, + { + "case_id": "val_mobile_crash", + "baseline_score": 1.0, + "candidate_score": 1.0, + "score_delta": 0.0, + "baseline_passed": true, + "candidate_passed": true, + "outcome": "unchanged", + "baseline_actual": "{\"action\":\"troubleshooting\",\"category\":\"technical\",\"priority\":\"p2\"}", + "candidate_actual": "{\"action\":\"troubleshooting\",\"category\":\"technical\",\"priority\":\"p2\"}", + "candidate_failure_types": [], + "candidate_reason": "" + }, + { + "case_id": "val_plan_question", + "baseline_score": 1.0, + "candidate_score": 1.0, + "score_delta": 0.0, + "baseline_passed": true, + "candidate_passed": true, + "outcome": "unchanged", + "baseline_actual": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "candidate_actual": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "candidate_failure_types": [], + "candidate_reason": "" + }, + { + "case_id": "val_rubric_tone_fail", + "baseline_score": 0.0, + "candidate_score": 0.0, + "score_delta": 0.0, + "baseline_passed": false, + "candidate_passed": false, + "outcome": "unchanged", + "baseline_actual": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "candidate_actual": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}", + "candidate_failure_types": [ + "llm_rubric_not_met", + "parameter_error" + ], + "candidate_reason": "priority expected p2 but got p3; deterministic attribution hint: llm_rubric_not_met" + }, + { + "case_id": "val_vip_refund", + "baseline_score": 0.0, + "candidate_score": 1.0, + "score_delta": 1.0, + "baseline_passed": false, + "candidate_passed": true, + "outcome": "new_pass", + "baseline_actual": "{\"action\":\"answer\",\"category\":\"account\",\"priority\":\"p2\"}", + "candidate_actual": "{\"action\":\"refund_review\",\"category\":\"billing\",\"priority\":\"p1\"}", + "candidate_failure_types": [], + "candidate_reason": "" + } + ] + }, + "gate": { + "decision": "rejected", + "reasons": [ + "validation score delta +0.0000 is below required +0.1000", + "new hard fail(s) are not allowed: val_checkout_outage", + "critical case regression(s): val_checkout_outage", + "overfitting guard triggered: train improved while validation did not improve" + ], + "checks": [ + { + "name": "min_validation_score_delta", + "passed": false, + "actual": 0.0, + "expected": 0.1 + }, + { + "name": "no_new_hard_fail", + "passed": false, + "case_ids": [ + "val_checkout_outage" + ], + "allow_new_hard_fail": false + }, + { + "name": "critical_case_regression", + "passed": false, + "case_ids": [ + "val_checkout_outage" + ], + "critical_case_ids": [ + "val_checkout_outage", + "val_vip_refund" + ] + }, + { + "name": "overfitting_guard", + "passed": false, + "train_score_delta": 0.2, + "validation_score_delta": 0.0 + }, + { + "name": "max_cost_usd", + "passed": true, + "actual": 0.0, + "expected": 0.01 + } + ], + "config": { + "min_validation_score_delta": 0.1, + "allow_new_hard_fail": false, + "critical_case_ids": [ + "val_vip_refund", + "val_checkout_outage" + ], + "max_cost_usd": 0.01 + }, + "overfitting_guard_triggered": true + }, + "input_audit": { + "files": { + "train_evalset": { + "path": "train.evalset.json", + "sha256": "2cb167dff1888e4ae41dba0af9911bf3a062343eeec7873cf4bc400e00fb3592", + "bytes": 3330 + }, + "validation_evalset": { + "path": "val.evalset.json", + "sha256": "702a90d3ecf264b96141a39c90ecfed9039ab2e8eb8b869c7d01edc9226d2580", + "bytes": 3256 + }, + "optimizer_config": { + "path": "optimizer.json", + "sha256": "3ea2fd21f077adae9a96f7b08611b3f36823b665691c05c7d4cb62bf50e5f32b", + "bytes": 1586 + }, + "case_meta": { + "path": "case_meta.json", + "sha256": "d87c7127e9f65d6c3863e6f1c3d506644d8b25f58849cdb847c04bbea33c8878", + "bytes": 1221 + } + }, + "prompt_sources": { + "router_prompt": { + "path": "agent/prompts/router.md", + "sha256": "339b66e767b44d96f233452d102eabae4e596e85a2daad7285240c5489aa3be4", + "bytes": 289 + }, + "system_prompt": { + "path": "agent/prompts/system.md", + "sha256": "c557bea101c277fb8af5c95d20ec8b420a9b9b609bbff9dfd982caa12d6f3851", + "bytes": 379 + }, + "skill_prompt": { + "path": "agent/prompts/skill.md", + "sha256": "0494b66f4d69fe7b0647883ebd7dedc8b9b5b9bea8a6001dd2259ebaf8cbafba", + "bytes": 362 + } + } + }, + "prompt_audit": { + "router_prompt": { + "source": "agent/prompts/router.md", + "baseline_sha256": "339b66e767b44d96f233452d102eabae4e596e85a2daad7285240c5489aa3be4", + "candidate_sha256": "c315899809bf4d2b5eabcef65242f1ee9df18097ea569bfd3d3bc6edd8e408c8", + "changed": true, + "baseline_chars": 289, + "candidate_chars": 479, + "diff": { + "line_count": 9, + "truncated": false, + "preview": [ + "--- baseline/router_prompt", + "+++ candidate/router_prompt", + "@@ -5,3 +5,6 @@", + " - Billing and pricing questions go to billing handling.", + " - Production incidents and software failures go to technical handling.", + " ", + "+Candidate routing update:", + "+- Route double charge, refund, and VIP refund tickets to billing handling.", + "+- overfit_payment_outage: route payment outage language through the billing refund path." + ] + } + }, + "system_prompt": { + "source": "agent/prompts/system.md", + "baseline_sha256": "c557bea101c277fb8af5c95d20ec8b420a9b9b609bbff9dfd982caa12d6f3851", + "candidate_sha256": "41a629093df3331600060f39c8ab7d8038c5f83609d975e838207f47b8604b31", + "changed": true, + "baseline_chars": 379, + "candidate_chars": 532, + "diff": { + "line_count": 9, + "truncated": false, + "preview": [ + "--- baseline/system_prompt", + "+++ candidate/system_prompt", + "@@ -21,3 +21,6 @@", + " - troubleshooting", + " - answer", + " ", + "+Candidate system update:", + "+- Keep the response as one compact JSON object with category, priority, and action.", + "+- Do not add prose outside the JSON object." + ] + } + }, + "skill_prompt": { + "source": "agent/prompts/skill.md", + "baseline_sha256": "0494b66f4d69fe7b0647883ebd7dedc8b9b5b9bea8a6001dd2259ebaf8cbafba", + "candidate_sha256": "909f172b766347413f12efe4ade1af816bb4e8a0b017edc350655f587e27f8d6", + "changed": true, + "baseline_chars": 362, + "candidate_chars": 582, + "diff": { + "line_count": 11, + "truncated": false, + "preview": [ + "--- baseline/skill_prompt", + "+++ candidate/skill_prompt", + "@@ -7,3 +7,8 @@", + " - Use escalate for production incidents.", + " - Use troubleshooting for reproducible technical failures.", + " ", + "+Optimization candidate notes:", + "+- Treat double charge, refund, and VIP refund requests as billing issues.", + "+- For refund requests, choose action refund_review.", + "+- For VIP refund requests, use priority p1.", + "+- scenario: overfit" + ] + } + } + }, + "optimization_rounds": [ + { + "round": 0, + "kind": "baseline", + "optimized_field_names": [], + "candidate_id": "baseline", + "candidate_prompts": { + "router_prompt": "Route each support ticket to one primary handling path before answering.\n\nBaseline routing rules:\n- Account access and profile questions go to account handling.\n- Billing and pricing questions go to billing handling.\n- Production incidents and software failures go to technical handling.\n\n", + "system_prompt": "You are a support triage assistant.\n\nReturn one compact JSON object with exactly these keys:\n- category\n- priority\n- action\n\nKnown categories:\n- account\n- billing\n- technical\n\nKnown priorities:\n- p1 for urgent production outages\n- p2 for normal user-impacting issues\n- p3 for low-risk informational requests\n\nKnown actions:\n- escalate\n- refund_review\n- troubleshooting\n- answer\n\n", + "skill_prompt": "Apply these triage skills after routing:\n\n- Use p1 only when the ticket describes an urgent production outage.\n- Use p2 for normal user-impacting issues.\n- Use p3 for low-risk informational requests.\n- Use answer for informational account or pricing questions.\n- Use escalate for production incidents.\n- Use troubleshooting for reproducible technical failures.\n\n" + }, + "evaluation_results": { + "train": { + "overall_score": 0.4, + "pass_rate": 0.4, + "passed": 2, + "failed": 3, + "total": 5 + }, + "validation": { + "overall_score": 0.6, + "pass_rate": 0.6, + "passed": 3, + "failed": 2, + "total": 5 + } + }, + "evaluation_result_refs": { + "train": "eval_train_baseline", + "validation": "eval_validation_baseline" + }, + "train_score": 0.4, + "validation_score": 0.6, + "accepted": true, + "acceptance_reason": "Initial prompt used as the control arm.", + "cost_usd": 0.0, + "duration_seconds": 0.0 + }, + { + "round": 1, + "kind": "fake_reflective", + "optimized_field_names": [ + "router_prompt", + "system_prompt", + "skill_prompt" + ], + "candidate_id": "candidate_refund_rule_overfit", + "candidate_prompts": { + "router_prompt": "Route each support ticket to one primary handling path before answering.\n\nBaseline routing rules:\n- Account access and profile questions go to account handling.\n- Billing and pricing questions go to billing handling.\n- Production incidents and software failures go to technical handling.\n\nCandidate routing update:\n- Route double charge, refund, and VIP refund tickets to billing handling.\n- overfit_payment_outage: route payment outage language through the billing refund path.\n", + "system_prompt": "You are a support triage assistant.\n\nReturn one compact JSON object with exactly these keys:\n- category\n- priority\n- action\n\nKnown categories:\n- account\n- billing\n- technical\n\nKnown priorities:\n- p1 for urgent production outages\n- p2 for normal user-impacting issues\n- p3 for low-risk informational requests\n\nKnown actions:\n- escalate\n- refund_review\n- troubleshooting\n- answer\n\nCandidate system update:\n- Keep the response as one compact JSON object with category, priority, and action.\n- Do not add prose outside the JSON object.\n", + "skill_prompt": "Apply these triage skills after routing:\n\n- Use p1 only when the ticket describes an urgent production outage.\n- Use p2 for normal user-impacting issues.\n- Use p3 for low-risk informational requests.\n- Use answer for informational account or pricing questions.\n- Use escalate for production incidents.\n- Use troubleshooting for reproducible technical failures.\n\nOptimization candidate notes:\n- Treat double charge, refund, and VIP refund requests as billing issues.\n- For refund requests, choose action refund_review.\n- For VIP refund requests, use priority p1.\n- scenario: overfit\n" + }, + "evaluation_results": { + "train": { + "overall_score": 0.6, + "pass_rate": 0.6, + "passed": 3, + "failed": 2, + "total": 5 + }, + "validation": { + "overall_score": 0.6, + "pass_rate": 0.6, + "passed": 3, + "failed": 2, + "total": 5 + } + }, + "evaluation_result_refs": { + "train": "eval_train_candidate", + "validation": "eval_validation_candidate" + }, + "train_score": 0.6, + "validation_score": 0.6, + "accepted": false, + "acceptance_reason": "validation score delta +0.0000 is below required +0.1000; new hard fail(s) are not allowed: val_checkout_outage; critical case regression(s): val_checkout_outage; overfitting guard triggered: train improved while validation did not improve", + "cost_usd": 0.0, + "duration_seconds": 0.0 + } + ], + "failure_attribution": { + "stats": { + "final_response_mismatch": 4, + "format_violation": 1, + "knowledge_recall_insufficient": 2, + "llm_rubric_not_met": 1, + "parameter_error": 3, + "tool_call_error": 1 + }, + "failed_cases": [ + { + "split": "train_baseline", + "case_id": "train_legacy_sync", + "failure_types": [ + "final_response_mismatch" + ], + "reason": "action expected escalate but got troubleshooting" + }, + { + "split": "train_baseline", + "case_id": "train_policy_tool_missing", + "failure_types": [ + "final_response_mismatch", + "format_violation", + "parameter_error", + "tool_call_error" + ], + "reason": "missing key priority; priority expected p2 but got None; action expected policy_lookup but got answer; deterministic attribution hint: tool_call_error" + }, + { + "split": "train_baseline", + "case_id": "train_refund_double_charge", + "failure_types": [ + "final_response_mismatch", + "knowledge_recall_insufficient" + ], + "reason": "category expected billing but got account; action expected refund_review but got answer" + }, + { + "split": "validation_baseline", + "case_id": "val_rubric_tone_fail", + "failure_types": [ + "llm_rubric_not_met", + "parameter_error" + ], + "reason": "priority expected p2 but got p3; deterministic attribution hint: llm_rubric_not_met" + }, + { + "split": "validation_baseline", + "case_id": "val_vip_refund", + "failure_types": [ + "final_response_mismatch", + "knowledge_recall_insufficient", + "parameter_error" + ], + "reason": "category expected billing but got account; priority expected p1 but got p2; action expected refund_review but got answer" + } + ], + "self_check": { + "labeled_failed_cases": 5, + "matched": 5, + "accuracy": 1.0, + "hint_assisted_cases": 2, + "rule_only_labeled_cases": 3, + "rule_only_matched": 3, + "rule_only_accuracy": 1.0, + "by_case": { + "train_legacy_sync": { + "expected_failure_types": [ + "final_response_mismatch" + ], + "actual_failure_types": [ + "final_response_mismatch" + ], + "fake_attribution_hints": [], + "hint_assisted": false, + "matched": true, + "note": "Baseline returns troubleshooting while the reference expects escalation." + }, + "train_policy_tool_missing": { + "expected_failure_types": [ + "format_violation", + "tool_call_error" + ], + "actual_failure_types": [ + "final_response_mismatch", + "format_violation", + "parameter_error", + "tool_call_error" + ], + "fake_attribution_hints": [ + "tool_call_error" + ], + "hint_assisted": true, + "matched": true, + "note": "The no-key fake agent cannot perform the expected policy lookup and returns an incomplete JSON object." + }, + "train_refund_double_charge": { + "expected_failure_types": [ + "final_response_mismatch", + "knowledge_recall_insufficient" + ], + "actual_failure_types": [ + "final_response_mismatch", + "knowledge_recall_insufficient" + ], + "fake_attribution_hints": [], + "hint_assisted": false, + "matched": true, + "note": "Baseline routes refund billing work to account/answer instead of billing/refund_review." + }, + "val_rubric_tone_fail": { + "expected_failure_types": [ + "llm_rubric_not_met", + "parameter_error" + ], + "actual_failure_types": [ + "llm_rubric_not_met", + "parameter_error" + ], + "fake_attribution_hints": [ + "llm_rubric_not_met" + ], + "hint_assisted": true, + "matched": true, + "note": "The answer uses a lower urgency than the reference and is deterministically labeled as a rubric failure for the no-key fake judge path." + }, + "val_vip_refund": { + "expected_failure_types": [ + "final_response_mismatch", + "knowledge_recall_insufficient", + "parameter_error" + ], + "actual_failure_types": [ + "final_response_mismatch", + "knowledge_recall_insufficient", + "parameter_error" + ], + "fake_attribution_hints": [], + "hint_assisted": false, + "matched": true, + "note": "Baseline misses billing/refund_review and p1 priority for VIP refund." + } + } + } + }, + "audit": { + "candidate_prompt_dir": "runs//candidate_prompts", + "optimizer_result_dir": null, + "cost": { + "estimated_usd": 0.0, + "optimizer_usd": 0.0, + "evaluation_usd": 0.0, + "total_usd": 0.0, + "max_budget_usd": 0.01 + }, + "token_usage": { + "prompt": 0, + "completion": 0, + "total": 0 + }, + "source_prompt_updated": false + } +} diff --git a/examples/optimization/eval_optimize_loop/sample_outputs/optimization_report.md b/examples/optimization/eval_optimize_loop/sample_outputs/optimization_report.md new file mode 100644 index 00000000..2f8619b2 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_outputs/optimization_report.md @@ -0,0 +1,133 @@ +# 优化报告 + +决策：**拒绝 (REJECTED)** + +## 分数 + +| 数据集 | Baseline | Candidate | Delta | +| --- | ---: | ---: | ---: | +| 训练集 | 0.4000 | 0.6000 | +0.2000 | +| 验证集 | 0.6000 | 0.6000 | +0.0000 | + +## Gate 原因 + +- validation score delta +0.0000 is below required +0.1000 +- new hard fail(s) are not allowed: val_checkout_outage +- critical case regression(s): val_checkout_outage +- overfitting guard triggered: train improved while validation did not improve + +## 验证集 Case Delta + +| Case | Baseline | Candidate | Delta | Outcome | +| --- | ---: | ---: | ---: | --- | +| val_checkout_outage | 1.0000 | 0.0000 | -1.0000 | new_fail | +| val_mobile_crash | 1.0000 | 1.0000 | +0.0000 | unchanged | +| val_plan_question | 1.0000 | 1.0000 | +0.0000 | unchanged | +| val_rubric_tone_fail | 0.0000 | 0.0000 | +0.0000 | unchanged | +| val_vip_refund | 0.0000 | 1.0000 | +1.0000 | new_pass | + +## 失败归因 + +- final_response_mismatch: 4 +- format_violation: 1 +- knowledge_recall_insufficient: 2 +- llm_rubric_not_met: 1 +- parameter_error: 3 +- tool_call_error: 1 +- self-check 准确率：1.0 (5/5) +- hint-assisted cases: 2 | rule-only accuracy: 1.0 + +## 候选 Prompt + +- candidate_id: `candidate_refund_rule_overfit` +- rationale: Adds refund handling learned from train failures but intentionally omits the outage guard to demonstrate overfitting rejection. +- validation pass rate: 0.6000 +- baseline validation pass rate: 0.6000 +- audited rounds: 2 + +## Prompt 审计 + +- `router_prompt`: changed=True baseline=339b66e767b4 candidate=c315899809bf diff_lines=9 +- `system_prompt`: changed=True baseline=c557bea101c2 candidate=41a629093df3 diff_lines=9 +- `skill_prompt`: changed=True baseline=0494b66f4d69 candidate=909f172b7663 diff_lines=11 + +## 输入审计 + +- `train_evalset`: 2cb167dff188 (3330 bytes) +- `validation_evalset`: 702a90d3ecf2 (3256 bytes) +- `optimizer_config`: 3ea2fd21f077 (1586 bytes) +- `case_meta`: d87c7127e9f6 (1221 bytes) + +## 复现命令 + +```bash +cd examples/optimization/eval_optimize_loop +PYTHONPATH=../../.. python run_pipeline.py --mode fake +``` + +# Optimization Report + +Decision: **REJECTED** + +## Scores + +| Split | Baseline | Candidate | Delta | +| --- | ---: | ---: | ---: | +| train | 0.4000 | 0.6000 | +0.2000 | +| validation | 0.6000 | 0.6000 | +0.0000 | + +## Gate Reasons + +- validation score delta +0.0000 is below required +0.1000 +- new hard fail(s) are not allowed: val_checkout_outage +- critical case regression(s): val_checkout_outage +- overfitting guard triggered: train improved while validation did not improve + +## Validation Case Delta + +| Case | Baseline | Candidate | Delta | Outcome | +| --- | ---: | ---: | ---: | --- | +| val_checkout_outage | 1.0000 | 0.0000 | -1.0000 | new_fail | +| val_mobile_crash | 1.0000 | 1.0000 | +0.0000 | unchanged | +| val_plan_question | 1.0000 | 1.0000 | +0.0000 | unchanged | +| val_rubric_tone_fail | 0.0000 | 0.0000 | +0.0000 | unchanged | +| val_vip_refund | 0.0000 | 1.0000 | +1.0000 | new_pass | + +## Failure Attribution + +- final_response_mismatch: 4 +- format_violation: 1 +- knowledge_recall_insufficient: 2 +- llm_rubric_not_met: 1 +- parameter_error: 3 +- tool_call_error: 1 +- self-check accuracy: 1.0 (5/5) +- hint-assisted cases: 2 | rule-only accuracy: 1.0 + +## Candidate + +- candidate_id: `candidate_refund_rule_overfit` +- rationale: Adds refund handling learned from train failures but intentionally omits the outage guard to demonstrate overfitting rejection. +- validation pass rate: 0.6000 +- baseline validation pass rate: 0.6000 +- audited rounds: 2 + +## Prompt Audit + +- `router_prompt`: changed=True baseline=339b66e767b4 candidate=c315899809bf diff_lines=9 +- `system_prompt`: changed=True baseline=c557bea101c2 candidate=41a629093df3 diff_lines=9 +- `skill_prompt`: changed=True baseline=0494b66f4d69 candidate=909f172b7663 diff_lines=11 + +## Input Audit + +- `train_evalset`: 2cb167dff188 (3330 bytes) +- `validation_evalset`: 702a90d3ecf2 (3256 bytes) +- `optimizer_config`: 3ea2fd21f077 (1586 bytes) +- `case_meta`: d87c7127e9f6 (1221 bytes) + +## Reproduce + +```bash +cd examples/optimization/eval_optimize_loop +PYTHONPATH=../../.. python run_pipeline.py --mode fake +``` diff --git a/examples/optimization/eval_optimize_loop/train.evalset.json b/examples/optimization/eval_optimize_loop/train.evalset.json new file mode 100644 index 00000000..a6e3f0ca --- /dev/null +++ b/examples/optimization/eval_optimize_loop/train.evalset.json @@ -0,0 +1,92 @@ +{ + "eval_set_id": "eval_optimize_loop_train", + "name": "Eval Optimize Loop - train", + "description": "Training split for failure attribution and prompt optimization. The fake default path uses these cases to create an auditable candidate without calling any external model.", + "eval_cases": [ + { + "eval_id": "train_refund_double_charge", + "conversation": [ + { + "invocation_id": "train-1", + "user_content": { + "parts": [{"text": "Customer says they were double charged for invoice INV-42 and asks for a refund."}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "{\"action\":\"refund_review\",\"category\":\"billing\",\"priority\":\"p2\"}"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_optimize_loop_demo", "user_id": "demo", "state": {}} + }, + { + "eval_id": "train_password_reset", + "conversation": [ + { + "invocation_id": "train-2", + "user_content": { + "parts": [{"text": "User cannot find the password reset email and asks how to change the email address."}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "{\"action\":\"answer\",\"category\":\"account\",\"priority\":\"p3\"}"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_optimize_loop_demo", "user_id": "demo", "state": {}} + }, + { + "eval_id": "train_legacy_sync", + "conversation": [ + { + "invocation_id": "train-3", + "user_content": { + "parts": [{"text": "Legacy desktop sync intermittently stalls after migration; customer provided no logs."}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "{\"action\":\"escalate\",\"category\":\"technical\",\"priority\":\"p2\"}"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_optimize_loop_demo", "user_id": "demo", "state": {}} + }, + { + "eval_id": "train_plan_question", + "conversation": [ + { + "invocation_id": "train-4", + "user_content": { + "parts": [{"text": "Prospect asks for a plan comparison between pricing tiers before buying."}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_optimize_loop_demo", "user_id": "demo", "state": {}} + }, + { + "eval_id": "train_policy_tool_missing", + "conversation": [ + { + "invocation_id": "train-5", + "user_content": { + "parts": [{"text": "Customer asks for the exact policy citation POL-77 before canceling an annual plan."}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "{\"action\":\"policy_lookup\",\"category\":\"billing\",\"priority\":\"p2\"}"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_optimize_loop_demo", "user_id": "demo", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/val.evalset.json b/examples/optimization/eval_optimize_loop/val.evalset.json new file mode 100644 index 00000000..c3e294b3 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/val.evalset.json @@ -0,0 +1,92 @@ +{ + "eval_set_id": "eval_optimize_loop_val", + "name": "Eval Optimize Loop - validation", + "description": "Validation split used for regression checks and gate decisions. It includes improved, unchanged, and regressed outcomes.", + "eval_cases": [ + { + "eval_id": "val_vip_refund", + "conversation": [ + { + "invocation_id": "val-1", + "user_content": { + "parts": [{"text": "VIP customer reports they were double charged during renewal and wants an immediate refund review."}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "{\"action\":\"refund_review\",\"category\":\"billing\",\"priority\":\"p1\"}"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_optimize_loop_demo", "user_id": "demo", "state": {}} + }, + { + "eval_id": "val_plan_question", + "conversation": [ + { + "invocation_id": "val-2", + "user_content": { + "parts": [{"text": "Prospect asks for a plan comparison between pricing tiers before buying."}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p3\"}"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_optimize_loop_demo", "user_id": "demo", "state": {}} + }, + { + "eval_id": "val_checkout_outage", + "conversation": [ + { + "invocation_id": "val-3", + "user_content": { + "parts": [{"text": "Production checkout outage blocks all payments for enterprise customers."}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "{\"action\":\"escalate\",\"category\":\"technical\",\"priority\":\"p1\"}"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_optimize_loop_demo", "user_id": "demo", "state": {}} + }, + { + "eval_id": "val_mobile_crash", + "conversation": [ + { + "invocation_id": "val-4", + "user_content": { + "parts": [{"text": "Mobile app crashes on startup after the latest update; user can reproduce it."}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "{\"action\":\"troubleshooting\",\"category\":\"technical\",\"priority\":\"p2\"}"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_optimize_loop_demo", "user_id": "demo", "state": {}} + }, + { + "eval_id": "val_rubric_tone_fail", + "conversation": [ + { + "invocation_id": "val-5", + "user_content": { + "parts": [{"text": "User is angry about repeated invoice confusion and asks for a guaranteed instant fix."}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "{\"action\":\"answer\",\"category\":\"billing\",\"priority\":\"p2\"}"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_optimize_loop_demo", "user_id": "demo", "state": {}} + } + ] +} diff --git a/tests/evaluation/test_eval_optimize_loop_example.py b/tests/evaluation/test_eval_optimize_loop_example.py new file mode 100644 index 00000000..6f147e07 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_example.py @@ -0,0 +1,548 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Regression test for the eval_optimize_loop optimization example.""" + +from __future__ import annotations + +import asyncio +import copy +import json +import os +import subprocess +import sys +import types +from importlib import util +from pathlib import Path + +import pytest + + +def _load_pipeline_module(): + repo_root = Path(__file__).resolve().parents[2] + module_path = repo_root / "examples" / "optimization" / "eval_optimize_loop" / "run_pipeline.py" + spec = util.spec_from_file_location("eval_optimize_loop_run_pipeline", module_path) + assert spec and spec.loader + module = util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _run_example(tmp_path: Path, *, mode: str = "fake", scenario: str = "overfit") -> tuple[Path, dict]: + repo_root = Path(__file__).resolve().parents[2] + example_dir = repo_root / "examples" / "optimization" / "eval_optimize_loop" + output_dir = tmp_path / f"eval_optimize_loop_{mode}_{scenario}" + + env = os.environ.copy() + env["PYTHONPATH"] = str(repo_root) + env.pop("TRPC_AGENT_API_KEY", None) + env.pop("TRPC_AGENT_BASE_URL", None) + env.pop("TRPC_AGENT_MODEL_NAME", None) + + subprocess.run( + [ + sys.executable, + str(example_dir / "run_pipeline.py"), + "--mode", + mode, + "--scenario", + scenario, + "--output-dir", + str(output_dir), + ], + cwd=str(example_dir), + env=env, + check=True, + timeout=180, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + return output_dir, json.loads((output_dir / "optimization_report.json").read_text(encoding="utf-8")) + + +def test_eval_optimize_loop_fake_mode_generates_rejected_report(tmp_path: Path) -> None: + output_dir, report = _run_example(tmp_path, mode="fake", scenario="overfit") + assert report["gate"]["decision"] == "rejected" + assert report["gate"]["overfitting_guard_triggered"] is True + assert report["delta"]["train_score_delta"] > 0 + assert report["delta"]["validation_score_delta"] <= 0 + assert any(not check["passed"] for check in report["gate"]["checks"]) + assert len(report["baseline"]["train"]["cases"]) == 5 + assert len(report["baseline"]["validation"]["cases"]) == 5 + assert any(delta["outcome"] == "new_pass" for delta in report["delta"]["validation_case_deltas"]) + assert any(delta["outcome"] == "new_fail" for delta in report["delta"]["validation_case_deltas"]) + assert len(report["optimization_rounds"]) >= 2 + assert report["optimization_rounds"][0]["kind"] == "baseline" + assert report["optimization_rounds"][0]["duration_seconds"] == 0.0 + assert report["optimization_rounds"][0]["evaluation_results"]["validation"]["total"] == 5 + assert report["optimization_rounds"][1]["evaluation_result_refs"]["validation"] == "eval_validation_candidate" + assert report["optimization_rounds"][1]["candidate_prompts"]["system_prompt"] + assert set(report["prompt_audit"]) == {"router_prompt", "system_prompt", "skill_prompt"} + assert set(report["optimization_rounds"][1]["candidate_prompts"]) == { + "router_prompt", + "system_prompt", + "skill_prompt", + } + assert report["inputs"]["case_meta"] == "case_meta.json" + assert set(report["input_audit"]["files"]) == { + "train_evalset", + "validation_evalset", + "optimizer_config", + "case_meta", + } + assert report["input_audit"]["files"]["train_evalset"]["sha256"] + assert report["failure_attribution"]["self_check"]["accuracy"] >= 0.75 + assert { + "final_response_mismatch", + "tool_call_error", + "parameter_error", + "llm_rubric_not_met", + "knowledge_recall_insufficient", + "format_violation", + }.issubset(report["failure_attribution"]["stats"]) + assert all( + case["failure_types"] + for split in ("train", "validation") + for case in report["baseline"][split]["cases"] + if not case["passed"] + ) + assert not str(report["inputs"]["train_evalset"]).startswith(str(Path(__file__).resolve().parents[2])) + assert report["prompt_audit"]["router_prompt"]["diff"]["line_count"] > 0 + assert report["prompt_audit"]["router_prompt"]["diff"]["preview"] + assert (output_dir / "optimization_report.md").exists() + + +def test_eval_optimize_loop_fake_mode_can_accept_good_candidate(tmp_path: Path) -> None: + _, report = _run_example(tmp_path, mode="fake", scenario="accepted") + assert report["gate"]["decision"] == "accepted" + assert report["gate"]["overfitting_guard_triggered"] is False + assert report["delta"]["train_score_delta"] > 0 + assert report["delta"]["validation_score_delta"] > 0 + assert all(check["passed"] for check in report["gate"]["checks"]) + assert all(delta["outcome"] != "new_fail" for delta in report["delta"]["validation_case_deltas"]) + + +def test_eval_optimize_loop_fake_mode_rejects_costly_candidate(tmp_path: Path) -> None: + _, report = _run_example(tmp_path, mode="fake", scenario="cost_exceeded") + assert report["gate"]["decision"] == "rejected" + assert report["delta"]["validation_score_delta"] > 0 + checks = {check["name"]: check for check in report["gate"]["checks"]} + assert checks["max_cost_usd"]["passed"] is False + assert checks["min_validation_score_delta"]["passed"] is True + assert report["audit"]["cost"]["estimated_usd"] > report["audit"]["cost"]["max_budget_usd"] + assert report["audit"]["cost"]["total_usd"] == report["audit"]["cost"]["estimated_usd"] + assert report["audit"]["cost"]["optimizer_usd"] > 0 + + +def test_eval_optimize_loop_trace_mode_replays_actual_conversation(tmp_path: Path) -> None: + output_dir, report = _run_example(tmp_path, mode="trace", scenario="overfit") + assert report["run"]["mode"] == "trace" + assert report["inputs"]["trace_evalsets"] + trace_dir = output_dir / "trace_evalsets" + trace_file = trace_dir / "validation_candidate.evalset.json" + payload = json.loads(trace_file.read_text(encoding="utf-8")) + assert payload["eval_cases"][0]["eval_mode"] == "trace" + assert payload["eval_cases"][0]["actual_conversation"] + assert report["gate"]["decision"] == "rejected" + + +def test_eval_optimize_loop_input_validation_contract() -> None: + pipeline = _load_pipeline_module() + config = json.loads((pipeline.CONFIG_PATH).read_text(encoding="utf-8")) + pipeline.validate_inputs(config) + + missing_gate_key = copy.deepcopy(config) + del missing_gate_key["gate"]["max_cost_usd"] + with pytest.raises(ValueError, match="max_cost_usd"): + pipeline.validate_inputs(missing_gate_key) + + unknown_critical = copy.deepcopy(config) + unknown_critical["gate"]["critical_case_ids"] = ["not_in_validation"] + with pytest.raises(ValueError, match="critical_case_ids"): + pipeline.validate_inputs(unknown_critical) + + same_evalset = copy.deepcopy(config) + with pytest.raises(ValueError, match="different files"): + pipeline.validate_inputs( + same_evalset, + train_path=pipeline.TRAIN_PATH, + val_path=pipeline.TRAIN_PATH, + ) + + +def test_eval_optimize_loop_ci_exit_code_contract(tmp_path: Path) -> None: + repo_root = Path(__file__).resolve().parents[2] + example_dir = repo_root / "examples" / "optimization" / "eval_optimize_loop" + env = os.environ.copy() + env["PYTHONPATH"] = str(repo_root) + env.pop("TRPC_AGENT_API_KEY", None) + env.pop("TRPC_AGENT_BASE_URL", None) + env.pop("TRPC_AGENT_MODEL_NAME", None) + + accepted_dir = tmp_path / "ci_accepted" + accepted = subprocess.run( + [ + sys.executable, + str(example_dir / "run_pipeline.py"), + "--mode", + "fake", + "--scenario", + "accepted", + "--output-dir", + str(accepted_dir), + "--ci-exit-code", + ], + cwd=str(example_dir), + env=env, + check=False, + timeout=180, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert accepted.returncode == 0 + assert (accepted_dir / "optimization_report.json").exists() + + rejected_dir = tmp_path / "ci_rejected" + rejected = subprocess.run( + [ + sys.executable, + str(example_dir / "run_pipeline.py"), + "--mode", + "fake", + "--scenario", + "overfit", + "--output-dir", + str(rejected_dir), + "--ci-exit-code", + ], + cwd=str(example_dir), + env=env, + check=False, + timeout=180, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert rejected.returncode == 1 + assert (rejected_dir / "optimization_report.json").exists() + + +def test_eval_optimize_loop_optimizer_mode_can_be_mocked_without_api_key( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pipeline = _load_pipeline_module() + captured: dict[str, object] = {} + + async def fake_optimize(**kwargs): + target = kwargs["target_prompt"] + baseline_prompts = await target.read_all() + captured["field_names"] = set(baseline_prompts) + candidate_prompts = dict(baseline_prompts) + candidate_prompts["router_prompt"] = baseline_prompts["router_prompt"].rstrip() + """ + +Mock optimizer routing update: +- Route double charge and VIP refund requests to billing handling. +- Preserve production outage handling as technical p1 escalation. +""" + candidate_prompts["system_prompt"] = baseline_prompts["system_prompt"].rstrip() + """ + +Mock optimizer system update: +- Keep the response as compact JSON. +""" + candidate_prompts["skill_prompt"] = baseline_prompts["skill_prompt"].rstrip() + """ + +Mock optimizer skill update: +- Treat double charge and VIP refund requests as billing issues. +- For refund requests, choose action refund_review. +- For VIP refund requests, use priority p1. +""" + return types.SimpleNamespace( + best_prompts=candidate_prompts, + status="completed", + finish_reason="mocked_for_example_test", + total_llm_cost=0.001, + total_token_usage={"prompt": 10, "completion": 5, "total": 15}, + rounds=[ + types.SimpleNamespace( + round=1, + kind="mock_optimizer", + optimized_field_names=["router_prompt", "system_prompt", "skill_prompt"], + candidate_prompts=candidate_prompts, + validation_pass_rate=1.0, + metric_breakdown={"final_response_avg_score": 1.0}, + accepted=True, + acceptance_reason="mock optimizer candidate accepted by adapter", + failed_case_ids=[], + round_llm_cost=0.001, + duration_seconds=0.01, + ) + ], + ) + + monkeypatch.setattr(pipeline.AgentOptimizer, "optimize", fake_optimize) + output_dir = tmp_path / "optimizer_mock" + args = types.SimpleNamespace( + mode="optimizer", + scenario="overfit", + output_dir=str(output_dir), + update_sample_outputs=False, + ci_exit_code=False, + ) + + result_dir = asyncio.run(pipeline.run_pipeline(args)) + report = json.loads((result_dir / "optimization_report.json").read_text(encoding="utf-8")) + assert captured["field_names"] == {"router_prompt", "system_prompt", "skill_prompt"} + assert report["run"]["mode"] == "optimizer" + assert report["candidate"]["candidate_id"] == "agent_optimizer_best" + assert report["audit"]["cost"]["optimizer_usd"] == 0.001 + assert report["audit"]["token_usage"]["total"] == 15 + assert report["optimization_rounds"][0]["kind"] == "mock_optimizer" + assert report["optimization_rounds"][0]["duration_seconds"] == 0.01 + + +def test_eval_optimize_loop_gate_decision_matrix() -> None: + pipeline = _load_pipeline_module() + gate_config = { + "min_validation_score_delta": 0.1, + "allow_new_hard_fail": False, + "critical_case_ids": ["critical"], + "max_cost_usd": 0.01, + } + base_delta = { + "case_id": "ordinary", + "outcome": "new_pass", + "baseline_score": 0.0, + "candidate_score": 1.0, + } + + accepted = pipeline.apply_gate( + gate_config=gate_config, + train_delta=0.2, + validation_delta=0.2, + validation_case_deltas=[base_delta], + cost_usd=0.0, + ) + assert accepted["decision"] == "accepted" + assert all(check["passed"] for check in accepted["checks"]) + + threshold_boundary = pipeline.apply_gate( + gate_config=gate_config, + train_delta=0.2, + validation_delta=0.1, + validation_case_deltas=[{**base_delta, "outcome": "score_improved"}], + cost_usd=0.01, + ) + assert threshold_boundary["decision"] == "accepted" + assert all(check["passed"] for check in threshold_boundary["checks"]) + + min_delta_fail = pipeline.apply_gate( + gate_config=gate_config, + train_delta=0.0, + validation_delta=0.0, + validation_case_deltas=[{**base_delta, "outcome": "unchanged"}], + cost_usd=0.0, + ) + assert min_delta_fail["decision"] == "rejected" + assert {check["name"]: check for check in min_delta_fail["checks"]}["min_validation_score_delta"]["passed"] is False + + new_hard_fail = pipeline.apply_gate( + gate_config=gate_config, + train_delta=0.2, + validation_delta=0.2, + validation_case_deltas=[{**base_delta, "outcome": "new_fail"}], + cost_usd=0.0, + ) + assert new_hard_fail["decision"] == "rejected" + assert {check["name"]: check for check in new_hard_fail["checks"]}["no_new_hard_fail"]["passed"] is False + + allowed_new_hard_fail = pipeline.apply_gate( + gate_config={**gate_config, "allow_new_hard_fail": True}, + train_delta=0.2, + validation_delta=0.2, + validation_case_deltas=[{**base_delta, "outcome": "new_fail"}], + cost_usd=0.0, + ) + assert allowed_new_hard_fail["decision"] == "accepted" + assert {check["name"]: check for check in allowed_new_hard_fail["checks"]}["no_new_hard_fail"]["passed"] is True + + critical_regression = pipeline.apply_gate( + gate_config=gate_config, + train_delta=0.2, + validation_delta=0.2, + validation_case_deltas=[{ + "case_id": "critical", + "outcome": "score_regressed", + "baseline_score": 1.0, + "candidate_score": 0.5, + }], + cost_usd=0.0, + ) + assert critical_regression["decision"] == "rejected" + assert {check["name"]: check for check in critical_regression["checks"]}["critical_case_regression"]["passed"] is False + + overfit = pipeline.apply_gate( + gate_config=gate_config, + train_delta=0.2, + validation_delta=0.0, + validation_case_deltas=[{**base_delta, "outcome": "unchanged"}], + cost_usd=0.0, + ) + assert overfit["decision"] == "rejected" + assert {check["name"]: check for check in overfit["checks"]}["overfitting_guard"]["passed"] is False + + validation_improved_not_overfit = pipeline.apply_gate( + gate_config=gate_config, + train_delta=0.2, + validation_delta=0.2, + validation_case_deltas=[{**base_delta, "outcome": "score_improved"}], + cost_usd=0.0, + ) + assert validation_improved_not_overfit["decision"] == "accepted" + assert {check["name"]: check for check in validation_improved_not_overfit["checks"]}["overfitting_guard"]["passed"] is True + + cost_fail = pipeline.apply_gate( + gate_config=gate_config, + train_delta=0.2, + validation_delta=0.2, + validation_case_deltas=[base_delta], + cost_usd=0.02, + ) + assert cost_fail["decision"] == "rejected" + assert {check["name"]: check for check in cost_fail["checks"]}["max_cost_usd"]["passed"] is False + + +def test_eval_optimize_loop_failure_attribution_matrix() -> None: + pipeline = _load_pipeline_module() + expected = '{"action":"refund_review","category":"billing","priority":"p1"}' + + cases = [ + ("not json", "format_violation"), + ('{"action":"refund_review","category":"billing"}', "format_violation"), + ('{"action":"refund_review","category":"account","priority":"p1"}', "knowledge_recall_insufficient"), + ('{"action":"refund_review","category":"billing","priority":"p2"}', "parameter_error"), + ('{"action":"answer","category":"billing","priority":"p1"}', "final_response_mismatch"), + ] + for actual, expected_type in cases: + failure_types, reason = pipeline.attribute_failure(expected=expected, actual=actual) + assert expected_type in failure_types + assert reason + + failure_types, reason = pipeline.attribute_failure( + expected=expected, + actual='{"action":"refund_review","category":"billing","priority":"p2"}', + case_id="rubric_case", + case_meta={"rubric_case": {"fake_attribution_hints": ["llm_rubric_not_met"]}}, + ) + assert "llm_rubric_not_met" in failure_types + assert "deterministic attribution hint" in reason + + failure_types, reason = pipeline.attribute_failure( + expected=expected, + actual='{"action":"answer","category":"billing"}', + case_id="tool_case", + case_meta={"tool_case": {"fake_attribution_hints": ["tool_call_error"]}}, + ) + assert {"tool_call_error", "format_violation"}.issubset(failure_types) + assert reason + + +def test_eval_optimize_loop_hidden_like_gate_and_attribution_generalize() -> None: + """Unknown case IDs should still be judged by rules, not public case names.""" + pipeline = _load_pipeline_module() + gate_config = { + "min_validation_score_delta": 0.1, + "allow_new_hard_fail": False, + "critical_case_ids": ["hidden_critical_payment_outage"], + "max_cost_usd": 0.01, + } + + hidden_accept = pipeline.apply_gate( + gate_config=gate_config, + train_delta=0.2, + validation_delta=0.2, + validation_case_deltas=[ + { + "case_id": "hidden_refund_variant", + "outcome": "new_pass", + "baseline_score": 0.0, + "candidate_score": 1.0, + }, + { + "case_id": "hidden_critical_payment_outage", + "outcome": "unchanged", + "baseline_score": 1.0, + "candidate_score": 1.0, + }, + ], + cost_usd=0.0, + ) + assert hidden_accept["decision"] == "accepted" + + hidden_new_fail = pipeline.apply_gate( + gate_config=gate_config, + train_delta=0.2, + validation_delta=0.2, + validation_case_deltas=[ + { + "case_id": "hidden_unseen_regression", + "outcome": "new_fail", + "baseline_score": 1.0, + "candidate_score": 0.0, + } + ], + cost_usd=0.0, + ) + assert hidden_new_fail["decision"] == "rejected" + assert {check["name"]: check for check in hidden_new_fail["checks"]}["no_new_hard_fail"]["passed"] is False + + hidden_critical_regression = pipeline.apply_gate( + gate_config=gate_config, + train_delta=0.2, + validation_delta=0.2, + validation_case_deltas=[ + { + "case_id": "hidden_critical_payment_outage", + "outcome": "score_regressed", + "baseline_score": 1.0, + "candidate_score": 0.5, + } + ], + cost_usd=0.0, + ) + assert hidden_critical_regression["decision"] == "rejected" + assert ( + {check["name"]: check for check in hidden_critical_regression["checks"]}[ + "critical_case_regression" + ]["passed"] + is False + ) + + expected = '{"action":"refund_review","category":"billing","priority":"p1"}' + actual = '{"action":"answer","category":"account","priority":"p2"}' + failure_types, reason = pipeline.attribute_failure( + expected=expected, + actual=actual, + case_id="hidden_unlabeled_refund", + case_meta={}, + ) + assert { + "final_response_mismatch", + "knowledge_recall_insufficient", + "parameter_error", + }.issubset(failure_types) + assert reason + + failure_types, reason = pipeline.attribute_failure( + expected=expected, + actual="plain text from hidden case", + case_id="hidden_unlabeled_format_case", + case_meta={}, + ) + assert failure_types == ["format_violation"] + assert reason