feat(examples): add evaluation optimization closed-loop example

examples/optimization/eval_optimize_loop/.gitignore

@@ -0,0 +1,4 @@
+runs/
+__pycache__/
+*.pyc
+

examples/optimization/eval_optimize_loop/agent/__init__.py

@@ -0,0 +1,2 @@
+"""Agent entry points for the eval-optimize-loop example."""
+

examples/optimization/eval_optimize_loop/agent/agent.py

@@ -0,0 +1,155 @@
+# Tencent is pleased to support the open source community by making tRPC-Agent-Python available.
+#
+# Copyright (C) 2026 Tencent. All rights reserved.
+#
+# tRPC-Agent-Python is licensed under Apache-2.0.
+"""Deterministic fake agent for the eval-optimize-loop example.
+
+The example's default mode must run without an API key, so this module exposes
+an async ``call_agent`` that derives behavior from the current prompt text on
+disk. The pipeline still uses AgentEvaluator for scoring; this module only
+stands in for the model.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+
+PROMPT_DIR = Path(__file__).parent / "prompts"
+ROUTER_PROMPT_PATH = PROMPT_DIR / "router.md"
+SYSTEM_PROMPT_PATH = PROMPT_DIR / "system.md"
+SKILL_PROMPT_PATH = PROMPT_DIR / "skill.md"
+
+_SPACE_RE = re.compile(r"\s+")
+
+
+def _compact_json(payload: dict[str, str]) -> str:
+    return json.dumps(payload, sort_keys=True, separators=(",", ":"))
+
+
+def normalize_json_text(raw: str) -> str:
+    """Normalize JSON-like model output for stable exact matching."""
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError:
+        return _SPACE_RE.sub(" ", raw.strip())
+    return _compact_json(parsed)
+
+
+def _read_prompt_text() -> str:
+    return "\n\n".join(
+        path.read_text(encoding="utf-8")
+        for path in (ROUTER_PROMPT_PATH, SYSTEM_PROMPT_PATH, SKILL_PROMPT_PATH)
+    ).lower()
+
+
+def _prompt_flags() -> dict[str, bool]:
+    prompt = _read_prompt_text()
+    return {
+        "refund_rule": "treat double charge" in prompt or "vip refund requests" in prompt,
+        "keep_outage_p1": "p1 for urgent production outages" in prompt,
+        "keep_plan_p3": "p3 for low-risk informational requests" in prompt,
+        "overfit_payment_outage": "overfit_payment_outage" in prompt,
+    }
+
+
+def answer_for_query(query: str) -> str:
+    """Return the fake model answer for one support ticket."""
+    text = query.lower()
+    flags = _prompt_flags()
+
+    if "production checkout outage" in text:
+        if flags["overfit_payment_outage"]:
+            return _compact_json({
+                "category": "billing",
+                "priority": "p2",
+                "action": "refund_review",
+            })
+        priority = "p1" if flags["keep_outage_p1"] else "p2"
+        return _compact_json({
+            "category": "technical",
+            "priority": priority,
+            "action": "escalate",
+        })
+
+    if "vip customer" in text and "double charged" in text:
+        if flags["refund_rule"]:
+            return _compact_json({
+                "category": "billing",
+                "priority": "p1",
+                "action": "refund_review",
+            })
+        return _compact_json({
+            "category": "account",
+            "priority": "p2",
+            "action": "answer",
+        })
+
+    if "double charged" in text or "refund" in text:
+        if flags["refund_rule"]:
+            return _compact_json({
+                "category": "billing",
+                "priority": "p2",
+                "action": "refund_review",
+            })
+        return _compact_json({
+            "category": "account",
+            "priority": "p2",
+            "action": "answer",
+        })
+
+    if "password reset" in text or "email address" in text:
+        return _compact_json({
+            "category": "account",
+            "priority": "p3",
+            "action": "answer",
+        })
+
+    if "plan comparison" in text or "pricing tiers" in text:
+        priority = "p3" if flags["keep_plan_p3"] else "p2"
+        return _compact_json({
+            "category": "billing",
+            "priority": priority,
+            "action": "answer",
+        })
+
+    if "policy citation" in text or "pol-77" in text:
+        return _compact_json({
+            "category": "billing",
+            "action": "answer",
+        })
+
+    if "guaranteed instant fix" in text or "repeated invoice confusion" in text:
+        return _compact_json({
+            "category": "billing",
+            "priority": "p3",
+            "action": "answer",
+        })
+
+    if "mobile app crashes" in text:
+        return _compact_json({
+            "category": "technical",
+            "priority": "p2",
+            "action": "troubleshooting",
+        })
+
+    if "legacy desktop sync" in text:
+        return _compact_json({
+            "category": "technical",
+            "priority": "p2",
+            "action": "troubleshooting",
+        })
+
+    return _compact_json({
+        "category": "technical",
+        "priority": "p2",
+        "action": "troubleshooting",
+    })
+
+
+async def call_agent(query: str) -> str:
+    """AgentEvaluator / AgentOptimizer compatible black-box entry point."""
+    return answer_for_query(query)

examples/optimization/eval_optimize_loop/agent/prompts/router.md

@@ -0,0 +1,7 @@
+Route each support ticket to one primary handling path before answering.
+
+Baseline routing rules:
+- Account access and profile questions go to account handling.
+- Billing and pricing questions go to billing handling.
+- Production incidents and software failures go to technical handling.
+

examples/optimization/eval_optimize_loop/agent/prompts/skill.md

@@ -0,0 +1,9 @@
+Apply these triage skills after routing:
+
+- Use p1 only when the ticket describes an urgent production outage.
+- Use p2 for normal user-impacting issues.
+- Use p3 for low-risk informational requests.
+- Use answer for informational account or pricing questions.
+- Use escalate for production incidents.
+- Use troubleshooting for reproducible technical failures.
+

examples/optimization/eval_optimize_loop/agent/prompts/system.md

@@ -0,0 +1,23 @@
+You are a support triage assistant.
+
+Return one compact JSON object with exactly these keys:
+- category
+- priority
+- action
+
+Known categories:
+- account
+- billing
+- technical
+
+Known priorities:
+- p1 for urgent production outages
+- p2 for normal user-impacting issues
+- p3 for low-risk informational requests
+
+Known actions:
+- escalate
+- refund_review
+- troubleshooting
+- answer
+

examples/optimization/eval_optimize_loop/case_meta.json

@@ -0,0 +1,24 @@
+{
+  "train_refund_double_charge": {
+    "expected_failure_types": ["final_response_mismatch", "knowledge_recall_insufficient"],
+    "note": "Baseline routes refund billing work to account/answer instead of billing/refund_review."
+  },
+  "train_legacy_sync": {
+    "expected_failure_types": ["final_response_mismatch"],
+    "note": "Baseline returns troubleshooting while the reference expects escalation."
+  },
+  "train_policy_tool_missing": {
+    "expected_failure_types": ["tool_call_error", "format_violation"],
+    "fake_attribution_hints": ["tool_call_error"],
+    "note": "The no-key fake agent cannot perform the expected policy lookup and returns an incomplete JSON object."
+  },
+  "val_vip_refund": {
+    "expected_failure_types": ["final_response_mismatch", "knowledge_recall_insufficient", "parameter_error"],
+    "note": "Baseline misses billing/refund_review and p1 priority for VIP refund."
+  },
+  "val_rubric_tone_fail": {
+    "expected_failure_types": ["llm_rubric_not_met", "parameter_error"],
+    "fake_attribution_hints": ["llm_rubric_not_met"],
+    "note": "The answer uses a lower urgency than the reference and is deterministically labeled as a rubric failure for the no-key fake judge path."
+  }
+}

examples/optimization/eval_optimize_loop/optimizer.json

@@ -0,0 +1,60 @@
+{
+  "evaluate": {
+    "metrics": [
+      {
+        "metric_name": "final_response_avg_score",
+        "threshold": 1.0,
+        "criterion": {
+          "final_response": {
+            "text": {
+              "match": "exact",
+              "case_insensitive": false
+            }
+          }
+        }
+      }
+    ],
+    "num_runs": 1
+  },
+  "gate": {
+    "min_validation_score_delta": 0.1,
+    "allow_new_hard_fail": false,
+    "critical_case_ids": ["val_vip_refund", "val_checkout_outage"],
+    "max_cost_usd": 0.01
+  },
+  "fake_optimizer": {
+    "candidate_id": "candidate_refund_rule_overfit",
+    "seed": 42,
+    "notes": "Adds refund handling learned from train failures but intentionally omits the outage guard to demonstrate overfitting rejection."
+  },
+  "optimize": {
+    "eval_case_parallelism": 1,
+    "stop": {
+      "required_metrics": "all"
+    },
+    "algorithm": {
+      "name": "gepa_reflective",
+      "seed": 42,
+      "reflection_lm": {
+        "model_name": "${TRPC_AGENT_MODEL_NAME}",
+        "base_url": "${TRPC_AGENT_BASE_URL}",
+        "api_key": "${TRPC_AGENT_API_KEY}",
+        "generation_config": {
+          "max_tokens": 2048,
+          "temperature": 0.4
+        }
+      },
+      "candidate_selection_strategy": "pareto",
+      "module_selector": "round_robin",
+      "frontier_type": "instance",
+      "reflection_minibatch_size": 3,
+      "reflection_history_top_k": 2,
+      "skip_perfect_score": false,
+      "use_merge": false,
+      "max_metric_calls": 18,
+      "score_threshold": 1.0,
+      "max_iterations_without_improvement": 2
+    }
+  }
+}
+