diff --git a/session_memory_summary_diff_report.json b/session_memory_summary_diff_report.json new file mode 100644 index 00000000..48ab04d0 --- /dev/null +++ b/session_memory_summary_diff_report.json @@ -0,0 +1,161 @@ +{ + "allowed_diff_count": 0, + "allowed_diffs": [], + "backend_pairs": [ + "inmemory_vs_sqlite" + ], + "backend_statuses": [ + { + "name": "inmemory", + "reason": "", + "status": "ok" + }, + { + "name": "sqlite", + "reason": "", + "status": "ok" + }, + { + "name": "external_sql", + "reason": "TRPC_AGENT_REPLAY_SQL_URL is not set", + "status": "skipped" + }, + { + "name": "redis", + "reason": "TRPC_AGENT_REPLAY_REDIS_URL is not set", + "status": "skipped" + } + ], + "case_count": 14, + "cases": [ + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "single_turn_text", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "multi_turn_append_order", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "tool_call_roundtrip", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "scoped_state_overwrite", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "memory_preference_search", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "memory_multi_session_isolation", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "summary_generation", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "summary_update_overwrite", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "summary_with_event_truncation", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "duplicate_or_error_recovery", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "serialization_order_nested_payload", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "list_sessions_consistency", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "state_temp_key_ignored_but_persistent_key_compared", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "summary_truncation_preserves_recent_context", + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 + } + ], + "diffs": [], + "false_positive_summary": { + "normal_case_count": 14, + "unexpected_diff_count": 0 + }, + "generated_at": "deterministic", + "generated_by": "tests/sessions/test_replay_consistency.py", + "mutation_summary": { + "detected_count": 0, + "mutation_count": 0, + "undetected_mutations": [] + }, + "report_kind": "normal_replay", + "schema_version": 1, + "unallowed_diff_count": 0, + "unallowed_diffs": [], + "unexpected_diff_count": 0 +} \ No newline at end of file diff --git a/tests/sessions/replay_cases/session_memory_summary_replay_cases.jsonl b/tests/sessions/replay_cases/session_memory_summary_replay_cases.jsonl new file mode 100644 index 00000000..54fa6284 --- /dev/null +++ b/tests/sessions/replay_cases/session_memory_summary_replay_cases.jsonl @@ -0,0 +1,14 @@ +{"name":"single_turn_text","description":"Two text events preserve order, author, role, and text.","operations":["create_session","append user text","append assistant text","store_session","search_memory"],"expected_risks":["basic event ordering drift","text/content normalization regression"]} +{"name":"multi_turn_append_order","description":"Three user/assistant turns preserve append order and invocation ids.","operations":["create_session","append six text events","store_session","search_memory"],"expected_risks":["event reorder","invocation_id mismatch","author/text drift"]} +{"name":"tool_call_roundtrip","description":"Weather tool call, tool response, and final assistant text round-trip across backends.","operations":["create_session","append user request","append function_call","append function_response","append final text"],"expected_risks":["tool args drift","tool response drift","tool event visibility loss"]} +{"name":"scoped_state_overwrite","description":"Session, user, and app state overwrites are strict while temp state is not persisted.","operations":["create_session with state","append state deltas","assert no persisted temp state"],"expected_risks":["state overwrite drift","temp state leakage","persistent key loss"]} +{"name":"memory_preference_search","description":"Preference text is stored through store_session and found by memory search.","operations":["create_session","append preference events","store_session","search tea/hiking/vegetarian"],"expected_risks":["memory content loss","memory author drift","search result normalization drift"]} +{"name":"memory_multi_session_isolation","description":"User A memory search must not return overlapping User B memory containing city-museums-b.","operations":["create session A","create session B","store both sessions","search A save_key"],"expected_risks":["save_key isolation failure","cross-user memory leak","query overlap leakage"]} +{"name":"summary_generation","description":"Manual summary creation yields summary text, summary event, and manager metadata.","operations":["create_session","append long conversation","create_session_summary","get_session_summary"],"expected_risks":["missing summary","summary event flag loss","summary session mismatch"]} +{"name":"summary_update_overwrite","description":"A later summary overwrites the cached summary and includes new release checklist context.","operations":["create_session","append first summary window","create summary","append new events","create summary again"],"expected_risks":["stale summary reuse","latest summary drift","summary wrong session"]} +{"name":"summary_with_event_truncation","description":"Summary truncation stores historical events and keeps recent plus post-summary events active.","operations":["create_session","append events","create summary","append post-summary event"],"expected_risks":["historical event loss","summary anchor loss","recent context loss"]} +{"name":"duplicate_or_error_recovery","description":"Duplicate content, retry error metadata, and recovery event are preserved.","operations":["create_session","append duplicate text events","append retry error event","append recovery event","store_session"],"expected_risks":["duplicate drop/reorder","error metadata drift","recovery event loss"]} +{"name":"serialization_order_nested_payload","description":"Nested dict/list tool payloads are canonicalized by order but strict on value changes.","operations":["create_session","append nested function_call","append nested function_response","append final text"],"expected_risks":["serialization order false positive","nested value drift","tool response loss"]} +{"name":"list_sessions_consistency","description":"list_sessions returns normalized id, app, user, and state consistently.","operations":["create_session with state","append state update","list_sessions"],"expected_risks":["list_sessions state omission","backend-specific list shape drift"]} +{"name":"state_temp_key_ignored_but_persistent_key_compared","description":"temp:* state is ignored for snapshots but rejected in raw persisted sessions.","operations":["create_session with persistent state","append temp and persistent deltas","assert raw temp not persisted"],"expected_risks":["temp key persistence","business state false allow","state value drift"]} +{"name":"summary_truncation_preserves_recent_context","description":"Summary truncation preserves historical events and recent rainy-day context before a post-summary append.","operations":["create_session","append context events","create summary","append canal walk"],"expected_risks":["recent context truncation","historical_events missing","summary event ordering drift"]} diff --git a/tests/sessions/replay_consistency/README.md b/tests/sessions/replay_consistency/README.md new file mode 100644 index 00000000..42fb2266 --- /dev/null +++ b/tests/sessions/replay_consistency/README.md @@ -0,0 +1,38 @@ +# Replay Consistency Harness + +This harness verifies that session, memory, and summary behavior replays +consistently across storage backends. Python `ReplayCase` fixtures define the +executable operation DSL: create a session, append deterministic events, create +summaries at fixed points, store the final session to memory, run memory +queries, and normalize the resulting snapshot. The JSONL manifest in +`tests/sessions/replay_cases/session_memory_summary_replay_cases.jsonl` +mirrors the registry for review readability and is checked by tests. + +The default CI matrix is intentionally light: InMemory plus temporary SQLite +files under `tmp_path`. Optional integration backends are only enabled through +environment variables: `TRPC_AGENT_REPLAY_SQL_URL` for an external SQL backend +and `TRPC_AGENT_REPLAY_REDIS_URL` for Redis. When these variables are absent, +the report records the backend as skipped instead of making CI depend on +external services. + +Normalization removes or canonicalizes non-semantic variance: exact timestamps, +summary timestamp values, auto-generated event ids, dict serialization order, +and memory timestamp values. Fixture event ids are preserved so duplicate, +retry, and wrong-id regressions remain visible. Memory search order is sorted +by stable content keys because ranking differs by backend. + +Allowed diffs are deliberately narrow: backend name, raw timestamp values, and +timestamp presence. Event order, author/role/text, tool arguments and results, +state values, memory scope/content, summary text, latest-summary overwrite +semantics, summary session id, summary event flags, and historical events are +strict. Summary text is whitespace-normalized but not semantically relaxed. + +Diff reports include backend statuses, normal replay false-positive counts, +mutation detection summaries, and structured diff entries with case, backend, +session, event, memory, summary, path, left, and right fields. Tests write +runtime reports to `tmp_path`; the repository root JSON is only a deterministic +schema example. The checked-in mutation example lives at +`tests/sessions/replay_consistency/session_memory_summary_mutation_report.json` +and is generated from the same registry that mutates event text, tool +arguments/results, state, memory text, summaries, and retry error/recovery +events on real normalized replay snapshots. diff --git a/tests/sessions/replay_consistency/__init__.py b/tests/sessions/replay_consistency/__init__.py new file mode 100644 index 00000000..b08edec5 --- /dev/null +++ b/tests/sessions/replay_consistency/__init__.py @@ -0,0 +1,35 @@ +"""Replay consistency harness for session, memory, and summary backends.""" + +from .backends import BackendBundle +from .backends import build_backends +from .backends import DeterministicSessionSummarizer +from .cases import EventSpec +from .cases import MemoryQuerySpec +from .cases import ReplayCase +from .cases import replay_cases +from .comparator import DiffEntry +from .comparator import compare_snapshot_pair +from .comparator import recursive_diff +from .mutations import mutate_snapshot +from .mutations import mutations_for_case +from .normalizer import Snapshot +from .normalizer import normalize_snapshot +from .report import write_report + +__all__ = [ + "BackendBundle", + "build_backends", + "DeterministicSessionSummarizer", + "EventSpec", + "MemoryQuerySpec", + "ReplayCase", + "replay_cases", + "DiffEntry", + "compare_snapshot_pair", + "recursive_diff", + "mutate_snapshot", + "mutations_for_case", + "Snapshot", + "normalize_snapshot", + "write_report", +] diff --git a/tests/sessions/replay_consistency/backends.py b/tests/sessions/replay_consistency/backends.py new file mode 100644 index 00000000..8d578174 --- /dev/null +++ b/tests/sessions/replay_consistency/backends.py @@ -0,0 +1,233 @@ +"""Backend construction for replay consistency tests.""" + +from __future__ import annotations + +from collections.abc import Awaitable +from collections.abc import Callable +from dataclasses import dataclass +import os +from pathlib import Path +import re +from typing import Any +from typing import AsyncGenerator + +import pytest + +from trpc_agent_sdk.abc import MemoryServiceConfig +from trpc_agent_sdk.events import Event +from trpc_agent_sdk.memory import InMemoryMemoryService +from trpc_agent_sdk.memory import SqlMemoryService +from trpc_agent_sdk.models import LLMModel +from trpc_agent_sdk.models import LlmRequest +from trpc_agent_sdk.models import LlmResponse +from trpc_agent_sdk.sessions import InMemorySessionService +from trpc_agent_sdk.sessions import Session +from trpc_agent_sdk.sessions import SessionServiceConfig +from trpc_agent_sdk.sessions import SessionSummarizer +from trpc_agent_sdk.sessions import SqlSessionService +from trpc_agent_sdk.sessions import SummarizerSessionManager + + +@dataclass +class BackendBundle: + name: str + session_service: Any + memory_service: Any + close: Callable[[], Awaitable[None]] + + +class FakeModel(LLMModel): + """Concrete model object for summarizer metadata; it is never called.""" + + @classmethod + def supported_models(cls) -> list[str]: + return [r"deterministic-replay-model"] + + async def _generate_async_impl( + self, + request: LlmRequest, + stream: bool = False, + ctx: Any | None = None, + ) -> AsyncGenerator[LlmResponse, None]: + if False: + yield LlmResponse() + + +class DeterministicSessionSummarizer(SessionSummarizer): + """A no-LLM summarizer that still uses SessionSummarizer compression.""" + + async def _compress_session_to_summary( + self, + events: list[Event], + session_id: str, + ctx: Any | None = None, + ) -> str | None: + if not events: + return None + + fragments: list[str] = [] + for event in events: + text = (event.get_text() or "").strip() + if not text: + calls = event.get_function_calls() + responses = event.get_function_responses() + if calls: + text = "tool_call:" + ",".join(call.name or "" for call in calls) + elif responses: + text = "tool_response:" + ",".join(response.name or "" for response in responses) + if text: + normalized_text = re.sub(r"\s+", " ", text).strip() + fragments.append(f"{event.author or 'unknown'}={normalized_text}") + + if not fragments: + return None + + return f"summary({session_id}): {' | '.join(fragments)} | facts={len(events)}-events" + + +def make_session_config(*, store_historical_events: bool = False) -> SessionServiceConfig: + config = SessionServiceConfig(store_historical_events=store_historical_events) + config.clean_ttl_config() + return config + + +def _make_memory_config() -> MemoryServiceConfig: + config = MemoryServiceConfig(enabled=True) + config.clean_ttl_config() + return config + + +def _make_summarizer_manager(keep_recent_count: int = 2) -> SummarizerSessionManager: + model = FakeModel(model_name="deterministic-replay-model") + summarizer = DeterministicSessionSummarizer( + model=model, + check_summarizer_functions=[lambda session: bool(session.events)], + keep_recent_count=keep_recent_count, + ) + return SummarizerSessionManager(model=model, summarizer=summarizer, auto_summarize=True) + + +def _sqlite_url(path: Path) -> str: + return f"sqlite:///{path.as_posix()}" + + +async def _close_services(session_service: Any, memory_service: Any) -> None: + await memory_service.close() + await session_service.close() + + +async def build_backends( + tmp_path: Path, + session_config: SessionServiceConfig | None = None, + *, + keep_recent_count: int = 2, +) -> list[BackendBundle]: + """Build default replay backends. + + SQLite is part of the default matrix. Its initialization must either + succeed, skip for a missing optional dependency, or fail the test. + """ + tmp_path.mkdir(parents=True, exist_ok=True) + base_config = session_config or make_session_config() + memory_config = _make_memory_config() + backends: list[BackendBundle] = [] + + in_memory_session = InMemorySessionService(session_config=base_config.model_copy(deep=True)) + in_memory_session.set_summarizer_manager(_make_summarizer_manager(keep_recent_count), force=True) + in_memory_memory = InMemoryMemoryService(memory_service_config=memory_config.model_copy(deep=True)) + backends.append( + BackendBundle( + name="inmemory", + session_service=in_memory_session, + memory_service=in_memory_memory, + close=lambda s=in_memory_session, m=in_memory_memory: _close_services(s, m), + ) + ) + + sqlite_session = SqlSessionService( + db_url=_sqlite_url(tmp_path / "replay_sessions.sqlite"), + is_async=False, + session_config=base_config.model_copy(deep=True), + ) + sqlite_session.set_summarizer_manager(_make_summarizer_manager(keep_recent_count), force=True) + sqlite_memory = SqlMemoryService( + db_url=_sqlite_url(tmp_path / "replay_memory.sqlite"), + is_async=False, + memory_service_config=memory_config.model_copy(deep=True), + ) + try: + await sqlite_session._sql_storage.create_sql_engine() + await sqlite_memory._sql_storage.create_sql_engine() + except ValueError as exc: + if isinstance(exc.__cause__, ImportError): + pytest.skip(f"SQLite replay backend dependency is unavailable: {exc}") + pytest.fail(f"SQLite replay backend failed to initialize: {exc}") + + backends.append( + BackendBundle( + name="sqlite", + session_service=sqlite_session, + memory_service=sqlite_memory, + close=lambda s=sqlite_session, m=sqlite_memory: _close_services(s, m), + ) + ) + + external_sql_url = os.environ.get("TRPC_AGENT_REPLAY_SQL_URL") + if external_sql_url: + external_sql_session = SqlSessionService( + db_url=external_sql_url, + is_async=False, + session_config=base_config.model_copy(deep=True), + ) + external_sql_session.set_summarizer_manager(_make_summarizer_manager(keep_recent_count), force=True) + external_sql_memory = SqlMemoryService( + db_url=external_sql_url, + is_async=False, + memory_service_config=memory_config.model_copy(deep=True), + ) + try: + await external_sql_session._sql_storage.create_sql_engine() + await external_sql_memory._sql_storage.create_sql_engine() + except ValueError as exc: + if isinstance(exc.__cause__, ImportError): + pytest.skip(f"External SQL replay backend dependency is unavailable: {exc}") + pytest.skip(f"External SQL replay backend failed to initialize: {exc}") + + backends.append( + BackendBundle( + name="external_sql", + session_service=external_sql_session, + memory_service=external_sql_memory, + close=lambda s=external_sql_session, m=external_sql_memory: _close_services(s, m), + ) + ) + + redis_url = os.environ.get("TRPC_AGENT_REPLAY_REDIS_URL") + if redis_url: + try: + from trpc_agent_sdk.memory import RedisMemoryService + from trpc_agent_sdk.sessions import RedisSessionService + except ImportError as exc: + pytest.skip(f"Redis replay backend dependency is unavailable: {exc}") + + redis_session = RedisSessionService( + db_url=redis_url, + is_async=False, + session_config=base_config.model_copy(deep=True), + ) + redis_session.set_summarizer_manager(_make_summarizer_manager(keep_recent_count), force=True) + redis_memory = RedisMemoryService( + db_url=redis_url, + is_async=False, + memory_service_config=memory_config.model_copy(deep=True), + ) + backends.append( + BackendBundle( + name="redis", + session_service=redis_session, + memory_service=redis_memory, + close=lambda s=redis_session, m=redis_memory: _close_services(s, m), + ) + ) + + return backends diff --git a/tests/sessions/replay_consistency/cases.py b/tests/sessions/replay_consistency/cases.py new file mode 100644 index 00000000..fa76403e --- /dev/null +++ b/tests/sessions/replay_consistency/cases.py @@ -0,0 +1,894 @@ +"""Deterministic replay cases for session/memory/summary backends.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class EventSpec: + event_id: str + invocation_id: str + author: str + role: str + text: str | None + function_call: dict | None + function_response: dict | None + state_delta: dict[str, Any] | None + branch: str | None + tag: str | None + filter_key: str | None + partial: bool = False + error_code: str | None = None + error_message: str | None = None + + +@dataclass(frozen=True) +class MemoryQuerySpec: + key: str | None + query: str + limit: int + expected_text_fragments: list[str] + + +@dataclass(frozen=True) +class ReplayCase: + name: str + app_name: str + user_id: str + session_id: str + initial_state: dict[str, Any] + events: list[EventSpec] + memory_queries: list[MemoryQuerySpec] + summary_points: list[int] + description: str + + +def _text_event( + case_name: str, + index: int, + *, + invocation_id: str, + author: str, + role: str, + text: str, + state_delta: dict[str, Any] | None = None, + branch: str | None = None, + tag: str | None = None, + filter_key: str | None = None, + error_code: str | None = None, + error_message: str | None = None, +) -> EventSpec: + return EventSpec( + event_id=f"{case_name}-event-{index:02d}", + invocation_id=invocation_id, + author=author, + role=role, + text=text, + function_call=None, + function_response=None, + state_delta=state_delta, + branch=branch, + tag=tag, + filter_key=filter_key, + error_code=error_code, + error_message=error_message, + ) + + +def replay_cases() -> list[ReplayCase]: + """Return the public replay corpus. Keep names/order stable.""" + return [ + ReplayCase( + name="single_turn_text", + app_name="replay-app", + user_id="user-single", + session_id="session-001", + initial_state={}, + events=[ + _text_event( + "single_turn_text", + 0, + invocation_id="inv-single-1", + author="user", + role="user", + text="Hello replay bot.", + ), + _text_event( + "single_turn_text", + 1, + invocation_id="inv-single-1", + author="assistant", + role="model", + text="Hello! Replay is deterministic.", + ), + ], + memory_queries=[ + MemoryQuerySpec( + key=None, + query="Hello", + limit=10, + expected_text_fragments=["Hello replay bot.", "Replay is deterministic."], + ) + ], + summary_points=[], + description="Two text events preserve order, author, role, and text.", + ), + ReplayCase( + name="multi_turn_append_order", + app_name="replay-app", + user_id="user-order", + session_id="session-002", + initial_state={}, + events=[ + _text_event( + "multi_turn_append_order", + 0, + invocation_id="inv-order-1", + author="user", + role="user", + text="First question about replay order.", + ), + _text_event( + "multi_turn_append_order", + 1, + invocation_id="inv-order-1", + author="assistant", + role="model", + text="First answer keeps the same invocation.", + ), + _text_event( + "multi_turn_append_order", + 2, + invocation_id="inv-order-2", + author="user", + role="user", + text="Second question checks append order.", + ), + _text_event( + "multi_turn_append_order", + 3, + invocation_id="inv-order-2", + author="assistant", + role="model", + text="Second answer follows the second question.", + ), + _text_event( + "multi_turn_append_order", + 4, + invocation_id="inv-order-3", + author="user", + role="user", + text="Third question closes the order test.", + ), + _text_event( + "multi_turn_append_order", + 5, + invocation_id="inv-order-3", + author="assistant", + role="model", + text="Third answer is last in the replay.", + ), + ], + memory_queries=[], + summary_points=[], + description="Three user/assistant turns verify stable append ordering.", + ), + ReplayCase( + name="tool_call_roundtrip", + app_name="replay-app", + user_id="user-tool", + session_id="session-003", + initial_state={}, + events=[ + _text_event( + "tool_call_roundtrip", + 0, + invocation_id="inv-tool-1", + author="user", + role="user", + text="What is the weather in Beijing?", + ), + EventSpec( + event_id="tool_call_roundtrip-event-01", + invocation_id="inv-tool-1", + author="assistant", + role="model", + text=None, + function_call={ + "id": "call-weather-1", + "name": "get_weather", + "args": { + "city": "Beijing", + "unit": "celsius", + }, + }, + function_response=None, + state_delta=None, + branch="weather.main", + tag="tool-call", + filter_key="weather", + ), + EventSpec( + event_id="tool_call_roundtrip-event-02", + invocation_id="inv-tool-1", + author="tool", + role="tool", + text=None, + function_call=None, + function_response={ + "id": "call-weather-1", + "name": "get_weather", + "response": { + "temperature": 25, + "condition": "sunny", + }, + }, + state_delta=None, + branch="weather.main", + tag="tool-response", + filter_key="weather", + ), + _text_event( + "tool_call_roundtrip", + 3, + invocation_id="inv-tool-1", + author="assistant", + role="model", + text="Beijing is sunny and 25 celsius.", + branch="weather.main", + tag="final", + filter_key="weather", + ), + ], + memory_queries=[ + MemoryQuerySpec( + key=None, + query="sunny Beijing", + limit=10, + expected_text_fragments=["Beijing is sunny and 25 celsius."], + ) + ], + summary_points=[], + description="Tool call and function response payloads survive normalization.", + ), + ReplayCase( + name="scoped_state_overwrite", + app_name="replay-app", + user_id="user-state", + session_id="session-004", + initial_state={ + "app:region": "global", + "user:tier": "bronze", + "counter": 0, + }, + events=[ + _text_event( + "scoped_state_overwrite", + 0, + invocation_id="inv-state-1", + author="user", + role="user", + text="Start scoped state test.", + state_delta={ + "counter": 1, + "temp:trace_id": "trace-1", + }, + ), + _text_event( + "scoped_state_overwrite", + 1, + invocation_id="inv-state-1", + author="assistant", + role="model", + text="Region set to north and tier set to silver.", + state_delta={ + "app:region": "north", + "user:tier": "silver", + "counter": 2, + }, + ), + _text_event( + "scoped_state_overwrite", + 2, + invocation_id="inv-state-2", + author="assistant", + role="model", + text="Preference saved and tier promoted to gold.", + state_delta={ + "preference": "quiet", + "user:tier": "gold", + "temp:trace_id": "trace-2", + }, + ), + ], + memory_queries=[], + summary_points=[], + description="App/user/session state overwrites are merged; temp state is not persisted.", + ), + ReplayCase( + name="memory_preference_search", + app_name="replay-app", + user_id="user-memory", + session_id="session-005", + initial_state={}, + events=[ + _text_event( + "memory_preference_search", + 0, + invocation_id="inv-memory-1", + author="user", + role="user", + text="I prefer tea in the morning.", + ), + _text_event( + "memory_preference_search", + 1, + invocation_id="inv-memory-1", + author="assistant", + role="model", + text="I will remember that you prefer tea.", + ), + _text_event( + "memory_preference_search", + 2, + invocation_id="inv-memory-2", + author="user", + role="user", + text="I enjoy hiking on weekends.", + ), + _text_event( + "memory_preference_search", + 3, + invocation_id="inv-memory-2", + author="assistant", + role="model", + text="Hiking preference noted for weekends.", + ), + _text_event( + "memory_preference_search", + 4, + invocation_id="inv-memory-3", + author="user", + role="user", + text="Please suggest vegetarian food.", + ), + _text_event( + "memory_preference_search", + 5, + invocation_id="inv-memory-3", + author="assistant", + role="model", + text="Vegetarian food options will be prioritized.", + ), + ], + memory_queries=[ + MemoryQuerySpec(key=None, query="tea", limit=10, expected_text_fragments=["prefer tea"]), + MemoryQuerySpec(key=None, query="hiking", limit=10, expected_text_fragments=["hiking on weekends"]), + MemoryQuerySpec( + key=None, + query="vegetarian", + limit=10, + expected_text_fragments=["vegetarian food"], + ), + ], + summary_points=[], + description="Memory search returns deterministic preference text for the session save key.", + ), + ReplayCase( + name="memory_multi_session_isolation", + app_name="replay-app", + user_id="user-isolation-a", + session_id="session-006-a", + initial_state={}, + events=[ + _text_event( + "memory_multi_session_isolation", + 0, + invocation_id="inv-isolation-a-1", + author="user", + role="user", + text="User A likes jasmine tea and hiking near lakes.", + ), + _text_event( + "memory_multi_session_isolation", + 1, + invocation_id="inv-isolation-a-1", + author="assistant", + role="model", + text="I will remember jasmine tea and lake hiking for User A.", + ), + ], + memory_queries=[ + MemoryQuerySpec( + key=None, + query="jasmine hiking", + limit=10, + expected_text_fragments=["jasmine tea", "lake hiking"], + ) + ], + summary_points=[], + description="A second user's stored memory must not leak into user A search results.", + ), + ReplayCase( + name="summary_generation", + app_name="replay-app", + user_id="user-summary", + session_id="session-007", + initial_state={}, + events=[ + _text_event( + "summary_generation", + 0, + invocation_id="inv-summary-1", + author="user", + role="user", + text="Help me plan a trip to Shanghai.", + ), + _text_event( + "summary_generation", + 1, + invocation_id="inv-summary-1", + author="assistant", + role="model", + text="Sure! When would you like to go?", + ), + _text_event( + "summary_generation", + 2, + invocation_id="inv-summary-2", + author="user", + role="user", + text="I want museums, vegetarian food, and tea houses.", + ), + _text_event( + "summary_generation", + 3, + invocation_id="inv-summary-2", + author="assistant", + role="model", + text="I will include museums, vegetarian restaurants, and tea houses.", + ), + _text_event( + "summary_generation", + 4, + invocation_id="inv-summary-3", + author="user", + role="user", + text="Budget is mid range and I prefer metro travel.", + ), + _text_event( + "summary_generation", + 5, + invocation_id="inv-summary-3", + author="assistant", + role="model", + text="I will keep it mid range with metro routes.", + ), + _text_event( + "summary_generation", + 6, + invocation_id="inv-summary-4", + author="user", + role="user", + text="Make it a three day plan with one quiet evening.", + ), + _text_event( + "summary_generation", + 7, + invocation_id="inv-summary-4", + author="assistant", + role="model", + text="I will create a three day Shanghai plan with a quiet evening.", + ), + ], + memory_queries=[], + summary_points=[7], + description="Manual summary creation yields summary text, event flag, and manager metadata.", + ), + ReplayCase( + name="summary_update_overwrite", + app_name="replay-app", + user_id="user-summary-update", + session_id="session-008", + initial_state={}, + events=[ + _text_event( + "summary_update_overwrite", + 0, + invocation_id="inv-summary-update-1", + author="user", + role="user", + text="Plan a product launch for replay.", + ), + _text_event( + "summary_update_overwrite", + 1, + invocation_id="inv-summary-update-1", + author="assistant", + role="model", + text="We need goals, audience, and timing.", + ), + _text_event( + "summary_update_overwrite", + 2, + invocation_id="inv-summary-update-2", + author="user", + role="user", + text="Audience is developers and operators.", + ), + _text_event( + "summary_update_overwrite", + 3, + invocation_id="inv-summary-update-2", + author="assistant", + role="model", + text="I will target developers and operators.", + ), + _text_event( + "summary_update_overwrite", + 4, + invocation_id="inv-summary-update-3", + author="user", + role="user", + text="Add a release checklist and owner names.", + ), + _text_event( + "summary_update_overwrite", + 5, + invocation_id="inv-summary-update-3", + author="assistant", + role="model", + text="Checklist and owners are now included.", + ), + ], + memory_queries=[], + summary_points=[3, 5], + description="A later summary overwrites the cached summary for the same session.", + ), + ReplayCase( + name="summary_with_event_truncation", + app_name="replay-app", + user_id="user-summary-truncation", + session_id="session-009", + initial_state={}, + events=[ + _text_event( + "summary_with_event_truncation", + 0, + invocation_id="inv-summary-truncation-1", + author="user", + role="user", + text="Help me plan a trip to Shanghai.", + ), + _text_event( + "summary_with_event_truncation", + 1, + invocation_id="inv-summary-truncation-1", + author="assistant", + role="model", + text="Sure! When would you like to go?", + ), + _text_event( + "summary_with_event_truncation", + 2, + invocation_id="inv-summary-truncation-2", + author="user", + role="user", + text="I prefer spring with museums.", + ), + _text_event( + "summary_with_event_truncation", + 3, + invocation_id="inv-summary-truncation-2", + author="assistant", + role="model", + text="Spring museums are good for Shanghai.", + ), + _text_event( + "summary_with_event_truncation", + 4, + invocation_id="inv-summary-truncation-3", + author="user", + role="user", + text="Keep the last metro rides simple.", + ), + _text_event( + "summary_with_event_truncation", + 5, + invocation_id="inv-summary-truncation-3", + author="assistant", + role="model", + text="I will keep recent metro details active.", + ), + _text_event( + "summary_with_event_truncation", + 6, + invocation_id="inv-summary-truncation-4", + author="user", + role="user", + text="Also add a ferry ride after the summary.", + ), + ], + memory_queries=[], + summary_points=[5], + description="Summary compression keeps historical events and then appends a new event.", + ), + ReplayCase( + name="duplicate_or_error_recovery", + app_name="replay-app", + user_id="user-duplicate", + session_id="session-010", + initial_state={}, + events=[ + _text_event( + "duplicate_or_error_recovery", + 0, + invocation_id="inv-duplicate-1", + author="user", + role="user", + text="Duplicate content check begins.", + ), + _text_event( + "duplicate_or_error_recovery", + 1, + invocation_id="inv-duplicate-1", + author="assistant", + role="model", + text="Same content may repeat.", + ), + _text_event( + "duplicate_or_error_recovery", + 2, + invocation_id="inv-duplicate-2", + author="assistant", + role="model", + text="Same content may repeat.", + ), + _text_event( + "duplicate_or_error_recovery", + 3, + invocation_id="inv-duplicate-3", + author="assistant", + role="model", + text="Transient retry error before recovery.", + tag="retry-error", + filter_key="retry", + error_code="RETRYABLE_BACKEND_ERROR", + error_message="Simulated retry failure before recovery.", + ), + _text_event( + "duplicate_or_error_recovery", + 4, + invocation_id="inv-duplicate-4", + author="assistant", + role="model", + text="Recovery succeeded after retry.", + tag="retry-recovery", + filter_key="retry", + ), + ], + memory_queries=[ + MemoryQuerySpec( + key=None, + query="Same retry recovery", + limit=10, + expected_text_fragments=["Same content may repeat.", "Recovery succeeded after retry."], + ) + ], + summary_points=[], + description=( + "Duplicate content with distinct ids plus an error/retry/recovery event sequence " + "is captured as the backend stores it." + ), + ), + ReplayCase( + name="serialization_order_nested_payload", + app_name="replay-app", + user_id="user-nested-payload", + session_id="session-011", + initial_state={}, + events=[ + _text_event( + "serialization_order_nested_payload", + 0, + invocation_id="inv-nested-1", + author="user", + role="user", + text="Build a nested itinerary payload for Beijing.", + ), + EventSpec( + event_id="serialization_order_nested_payload-event-01", + invocation_id="inv-nested-1", + author="assistant", + role="model", + text=None, + function_call={ + "id": "call-itinerary-1", + "name": "build_itinerary", + "args": { + "city": "Beijing", + "days": [ + {"day": 2, "focus": ["parks", "tea"]}, + {"day": 1, "focus": ["museums", "food"]}, + ], + "preferences": {"transport": "metro", "budget": "mid"}, + }, + }, + function_response=None, + state_delta=None, + branch="nested.main", + tag="tool-call", + filter_key="nested", + ), + EventSpec( + event_id="serialization_order_nested_payload-event-02", + invocation_id="inv-nested-1", + author="tool", + role="tool", + text=None, + function_call=None, + function_response={ + "id": "call-itinerary-1", + "name": "build_itinerary", + "response": { + "temperature": 21, + "condition": "clear", + "plan": { + "city": "Beijing", + "items": [ + {"slot": "morning", "name": "National Museum"}, + {"slot": "afternoon", "name": "Tea house"}, + ], + }, + }, + }, + state_delta=None, + branch="nested.main", + tag="tool-response", + filter_key="nested", + ), + _text_event( + "serialization_order_nested_payload", + 3, + invocation_id="inv-nested-1", + author="assistant", + role="model", + text="The nested itinerary payload is ready.", + branch="nested.main", + tag="final", + filter_key="nested", + ), + ], + memory_queries=[], + summary_points=[], + description=( + "Risk: nested dict/list serialization order should be canonicalized, while " + "tool argument and tool response value drift remains strict." + ), + ), + ReplayCase( + name="list_sessions_consistency", + app_name="replay-app", + user_id="user-list-sessions", + session_id="session-012", + initial_state={"session_label": "list-check", "user:tier": "silver"}, + events=[ + _text_event( + "list_sessions_consistency", + 0, + invocation_id="inv-list-1", + author="user", + role="user", + text="Create a session that can be listed consistently.", + ), + _text_event( + "list_sessions_consistency", + 1, + invocation_id="inv-list-1", + author="assistant", + role="model", + text="The session list should include this normalized state.", + state_delta={"session_label": "list-check-updated"}, + ), + ], + memory_queries=[], + summary_points=[], + description=( + "Risk: list_sessions may omit or reshape id/app/user/state differently across backends." + ), + ), + ReplayCase( + name="state_temp_key_ignored_but_persistent_key_compared", + app_name="replay-app", + user_id="user-state-temp", + session_id="session-013", + initial_state={"user:tier": "gold", "preference": "tea"}, + events=[ + _text_event( + "state_temp_key_ignored_but_persistent_key_compared", + 0, + invocation_id="inv-state-temp-1", + author="user", + role="user", + text="Remember my persistent state but not the trace id.", + state_delta={ + "user:tier": "platinum", + "preference": "oolong tea", + "temp:trace_id": "temp-state-1", + }, + ), + _text_event( + "state_temp_key_ignored_but_persistent_key_compared", + 1, + invocation_id="inv-state-temp-1", + author="assistant", + role="model", + text="Persistent state is updated and temp state is not stored.", + state_delta={"session_counter": 2, "temp:trace_id": "temp-state-2"}, + ), + ], + memory_queries=[], + summary_points=[], + description=( + "Risk: temp:* state must be filtered from persisted raw sessions while business " + "state values remain strictly compared." + ), + ), + ReplayCase( + name="summary_truncation_preserves_recent_context", + app_name="replay-app", + user_id="user-summary-recent", + session_id="session-014", + initial_state={}, + events=[ + _text_event( + "summary_truncation_preserves_recent_context", + 0, + invocation_id="inv-summary-recent-1", + author="user", + role="user", + text="Start a research plan for Hangzhou.", + ), + _text_event( + "summary_truncation_preserves_recent_context", + 1, + invocation_id="inv-summary-recent-1", + author="assistant", + role="model", + text="We will track museums, tea, and quiet walks.", + ), + _text_event( + "summary_truncation_preserves_recent_context", + 2, + invocation_id="inv-summary-recent-2", + author="user", + role="user", + text="Keep the newest context about rainy day options.", + ), + _text_event( + "summary_truncation_preserves_recent_context", + 3, + invocation_id="inv-summary-recent-2", + author="assistant", + role="model", + text="Rainy day options stay in recent context.", + ), + _text_event( + "summary_truncation_preserves_recent_context", + 4, + invocation_id="inv-summary-recent-3", + author="user", + role="user", + text="After summary, add a canal walk.", + ), + ], + memory_queries=[], + summary_points=[3], + description=( + "Risk: truncation should preserve historical events and recent active context " + "after a summary event is inserted." + ), + ), + ] diff --git a/tests/sessions/replay_consistency/comparator.py b/tests/sessions/replay_consistency/comparator.py new file mode 100644 index 00000000..53bee72b --- /dev/null +++ b/tests/sessions/replay_consistency/comparator.py @@ -0,0 +1,170 @@ +"""Recursive snapshot comparator with structured diff entries.""" + +from __future__ import annotations + +from dataclasses import asdict +from dataclasses import dataclass +import re +from typing import Any + +from .normalizer import Snapshot + + +@dataclass +class DiffEntry: + case_name: str + left_backend: str + right_backend: str + session_id: str | None + event_index: int | None + memory_index: int | None + summary_id: str | None + section: str + path: str + left: Any + right: Any + allowed: bool + reason: str + + +_MISSING = object() + + +def _to_dict(value: Any) -> Any: + if isinstance(value, Snapshot): + return asdict(value) + if hasattr(value, "__dataclass_fields__"): + return asdict(value) + return value + + +def _infer_section(path: str) -> str: + if not path: + return "root" + root = path.split(".", 1)[0].split("[", 1)[0] + if root in {"events", "state", "memories", "summary", "historical_events", "list_sessions"}: + return root + return root + + +def _parse_index(path: str, section: str) -> int | None: + match = re.search(rf"{section}\[(\d+)\]", path) + return int(match.group(1)) if match else None + + +def _summary_id(left: dict[str, Any], right: dict[str, Any], section: str) -> str | None: + if section != "summary": + return None + for snapshot in (left, right): + summary = snapshot.get("summary") + if isinstance(summary, dict): + metadata = summary.get("metadata") or {} + session_id = metadata.get("session_id") + if session_id: + return f"summary:{session_id}:latest" + session_id = left.get("session_id") or right.get("session_id") + return f"summary:{session_id}:latest" if session_id else None + + +def _allowed_diff(path: str, left: Any, right: Any) -> tuple[bool, str]: + if path == "backend": + return True, "backend name differs by design" + if path.endswith(".timestamp") or path == "timestamp": + return True, "raw timestamps are backend generated" + if path.endswith(".has_timestamp") and left is True and right is True: + return True, "timestamp presence is normalized" + return False, "" + + +def _display(value: Any) -> Any: + if value is _MISSING: + return "" + return value + + +def _entry(path: str, left_value: Any, right_value: Any, left: dict[str, Any], right: dict[str, Any]) -> DiffEntry: + section = _infer_section(path) + allowed, reason = _allowed_diff(path, left_value, right_value) + event_index = _parse_index(path, "events") + if event_index is None: + event_index = _parse_index(path, "historical_events") + return DiffEntry( + case_name=left.get("case_name") or right.get("case_name") or "", + left_backend=left.get("backend") or "", + right_backend=right.get("backend") or "", + session_id=left.get("session_id") or right.get("session_id"), + event_index=event_index, + memory_index=_parse_index(path, "memories"), + summary_id=_summary_id(left, right, section), + section=section, + path=path, + left=_display(left_value), + right=_display(right_value), + allowed=allowed, + reason=reason, + ) + + +def _join_path(parent: str, key: str) -> str: + return f"{parent}.{key}" if parent else key + + +def _diff_values(current_left: Any, + current_right: Any, + root_left: dict[str, Any], + root_right: dict[str, Any], + path: str) -> list[DiffEntry]: + if isinstance(current_left, dict) and isinstance(current_right, dict): + diffs: list[DiffEntry] = [] + for key in sorted(set(current_left) | set(current_right)): + next_left = current_left.get(key, _MISSING) + next_right = current_right.get(key, _MISSING) + next_path = _join_path(path, str(key)) + if next_left is _MISSING or next_right is _MISSING: + diffs.append(_entry(next_path, next_left, next_right, root_left, root_right)) + else: + diffs.extend(_diff_values(next_left, next_right, root_left, root_right, next_path)) + return diffs + + if isinstance(current_left, list) and isinstance(current_right, list): + diffs = [] + max_len = max(len(current_left), len(current_right)) + for index in range(max_len): + next_path = f"{path}[{index}]" + next_left = current_left[index] if index < len(current_left) else _MISSING + next_right = current_right[index] if index < len(current_right) else _MISSING + if next_left is _MISSING or next_right is _MISSING: + diffs.append(_entry(next_path, next_left, next_right, root_left, root_right)) + else: + diffs.extend(_diff_values(next_left, next_right, root_left, root_right, next_path)) + return diffs + + if current_left != current_right: + return [_entry(path, current_left, current_right, root_left, root_right)] + return [] + + +def recursive_diff(left: Any, right: Any, context: dict[str, Any] | None = None) -> list[DiffEntry]: + left_dict = _to_dict(left) + right_dict = _to_dict(right) + if not isinstance(left_dict, dict) or not isinstance(right_dict, dict): + raise TypeError("recursive_diff expects Snapshot, dataclass, or dict inputs") + + diffs = _diff_values(left_dict, right_dict, left_dict, right_dict, "") + if context: + for diff in diffs: + if context.get("case_name"): + diff.case_name = context["case_name"] + if context.get("left_backend"): + diff.left_backend = context["left_backend"] + if context.get("right_backend"): + diff.right_backend = context["right_backend"] + return diffs + + +def compare_snapshot_pair(left: Snapshot, right: Snapshot) -> list[DiffEntry]: + return recursive_diff(left, right) + + +def unallowed_diffs(diffs: list[DiffEntry]) -> list[DiffEntry]: + return [diff for diff in diffs if not diff.allowed] diff --git a/tests/sessions/replay_consistency/mutations.py b/tests/sessions/replay_consistency/mutations.py new file mode 100644 index 00000000..59cd2156 --- /dev/null +++ b/tests/sessions/replay_consistency/mutations.py @@ -0,0 +1,141 @@ +"""Fault injection helpers for replay snapshot mutation tests.""" + +from __future__ import annotations + +import copy +from typing import Any +from typing import Callable + +from .cases import ReplayCase + + +SYNTHETIC_MUTATIONS = [ + "drop_event", + "reorder_event", + "change_tool_args", + "change_state", + "drop_memory", + "change_memory_text", + "drop_summary", + "overwrite_summary_text", + "wrong_summary_session", + "duplicate_event", +] + +SUMMARY_REQUIRED_MUTATIONS = [ + "drop_summary", + "overwrite_summary_text", + "wrong_summary_session", +] + +MUTATION_SECTION = { + "drop_event": "events", + "alter_event_text": "events", + "reorder_event": "events", + "reorder_events": "events", + "duplicate_event": "events", + "change_tool_args": "events", + "change_tool_response": "events", + "change_error_code": "events", + "drop_recovery_event": "events", + "change_state": "state", + "change_state_value": "state", + "drop_memory": "memories", + "alter_memory_text": "memories", + "change_memory_text": "memories", + "change_memory_text_or_metadata": "memories", + "drop_summary": "summary", + "overwrite_summary_text": "summary", + "overwrite_summary_with_stale_text": "summary", + "wrong_summary_session": "summary", + "summary_wrong_session_id": "summary", +} + + +def mutations_for_case(case: ReplayCase) -> list[str]: + mutations = ["drop_event", "reorder_events", "duplicate_event", "alter_event_text"] + if case.name in {"tool_call_roundtrip", "serialization_order_nested_payload"}: + mutations.extend(["change_tool_args", "change_tool_response"]) + if case.name in {"scoped_state_overwrite", "state_temp_key_ignored_but_persistent_key_compared"}: + mutations.append("change_state_value") + if case.name in {"memory_preference_search", "memory_multi_session_isolation"}: + mutations.extend(["drop_memory", "alter_memory_text"]) + if case.name in { + "summary_generation", + "summary_update_overwrite", + "summary_with_event_truncation", + "summary_truncation_preserves_recent_context", + }: + mutations.extend(["drop_summary", "overwrite_summary_with_stale_text", "summary_wrong_session_id"]) + if case.name == "duplicate_or_error_recovery": + mutations.extend(["change_error_code", "drop_recovery_event"]) + return mutations + + +def _find_mutation_event( + name: str, + snapshot: dict[str, Any], + predicate: Callable[[dict[str, Any]], bool], +) -> dict[str, Any]: + for event in snapshot["events"]: + if predicate(event): + return event + raise AssertionError(f"{name} mutation target was not found") + + +def mutate_snapshot(name: str, snapshot: dict[str, Any]) -> None: + if name == "drop_event": + del snapshot["events"][1] + elif name == "alter_event_text": + event = _find_mutation_event(name, snapshot, lambda item: item.get("text") is not None) + event["text"] = "mutated event text" + elif name in {"reorder_event", "reorder_events"}: + snapshot["events"][0], snapshot["events"][1] = snapshot["events"][1], snapshot["events"][0] + elif name == "change_tool_args": + event = _find_mutation_event(name, snapshot, lambda item: item["function_calls"]) + event["function_calls"][0]["args"]["city"] = "Shanghai" + elif name == "change_tool_response": + event = _find_mutation_event(name, snapshot, lambda item: item["function_responses"]) + response = event["function_responses"][0]["response"] + if "temperature" in response: + response["temperature"] = 30 + else: + response["condition"] = "rainy" + elif name in {"change_state", "change_state_value"}: + if "user:tier" in snapshot["state"]: + snapshot["state"]["user:tier"] = "silver" + elif "preference" in snapshot["state"]: + snapshot["state"]["preference"] = "coffee" + elif snapshot["state"]: + key = sorted(snapshot["state"])[0] + snapshot["state"][key] = "mutated-state" + else: + raise AssertionError(f"{name} mutation target was not found") + elif name == "drop_memory": + del snapshot["memories"][0] + elif name in {"alter_memory_text", "change_memory_text", "change_memory_text_or_metadata"}: + snapshot["memories"][0]["text"] = "I prefer coffee in the morning." + elif name == "drop_summary": + snapshot["summary"] = None + elif name in {"overwrite_summary_text", "overwrite_summary_with_stale_text"}: + if snapshot["summary"] is None: + raise AssertionError(f"{name} mutation target was not found") + snapshot["summary"]["text"] = "summary(session-mutation): stale overwritten summary" + elif name in {"wrong_summary_session", "summary_wrong_session_id"}: + if snapshot["summary"] is None: + raise AssertionError(f"{name} mutation target was not found") + snapshot["summary"]["metadata"]["session_id"] = "wrong-session" + elif name == "duplicate_event": + snapshot["events"].append(copy.deepcopy(snapshot["events"][0])) + elif name == "change_error_code": + event = _find_mutation_event(name, snapshot, lambda item: item.get("error_code")) + event["error_code"] = "WRONG_ERROR_CODE" + elif name == "drop_recovery_event": + for index, event in enumerate(snapshot["events"]): + if event["text"] == "Recovery succeeded after retry.": + del snapshot["events"][index] + break + else: + raise AssertionError("drop_recovery_event mutation target was not found") + else: + raise ValueError(f"Unknown mutation {name}") diff --git a/tests/sessions/replay_consistency/normalizer.py b/tests/sessions/replay_consistency/normalizer.py new file mode 100644 index 00000000..59890781 --- /dev/null +++ b/tests/sessions/replay_consistency/normalizer.py @@ -0,0 +1,218 @@ +"""Snapshot normalization for replay consistency tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +import re +import uuid +from typing import Any + +from trpc_agent_sdk.events import Event +from trpc_agent_sdk.sessions import Session +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import MemoryEntry +from trpc_agent_sdk.types import State + +from .cases import MemoryQuerySpec +from .cases import ReplayCase + + +@dataclass +class Snapshot: + backend: str + case_name: str + session_id: str + app_name: str + user_id: str + state: dict[str, Any] + events: list[dict[str, Any]] + historical_events: list[dict[str, Any]] + memories: list[dict[str, Any]] + summary: dict[str, Any] | None + list_sessions: list[dict[str, Any]] + + +def canonicalize(value: Any) -> Any: + if isinstance(value, dict): + return {key: canonicalize(value[key]) for key in sorted(value)} + if isinstance(value, list): + return [canonicalize(item) for item in value] + if hasattr(value, "model_dump"): + return canonicalize(value.model_dump(exclude_none=True, mode="json")) + return value + + +def normalize_summary_text(text: str | None) -> str | None: + if text is None: + return None + return re.sub(r"\s+", " ", text).strip() + + +def _strip_temp_state(state: dict[str, Any] | None) -> dict[str, Any]: + if not state: + return {} + return canonicalize({key: value for key, value in state.items() if not key.startswith(State.TEMP_PREFIX)}) + + +def _content_text(content: Content | None) -> str | None: + if not content or not content.parts: + return None + text = "".join(part.text for part in content.parts if part.text) + return text or None + + +def _is_uuid_like(value: str) -> bool: + try: + uuid.UUID(value) + except ValueError: + return False + return True + + +def _normalize_event_id(event: Event) -> str: + if event.is_summary_event() or not event.id or _is_uuid_like(event.id): + return "normalized" + return event.id + + +def normalize_event(event: Event, index: int) -> dict[str, Any]: + function_calls = [ + { + "id": call.id, + "name": call.name, + "args": canonicalize(call.args or {}), + } + for call in event.get_function_calls() + ] + function_responses = [ + { + "id": response.id, + "name": response.name, + "response": canonicalize(response.response or {}), + } + for response in event.get_function_responses() + ] + role = event.content.role if event.content else None + state_delta = _strip_temp_state(event.actions.state_delta if event.actions else None) + + return { + "stable_index": index, + "event_id": _normalize_event_id(event), + "invocation_id": event.invocation_id, + "author": event.author, + "role": role, + "text": normalize_summary_text(event.get_text()) if event.get_text() else None, + "function_calls": function_calls, + "function_responses": function_responses, + "state_delta": state_delta, + "branch": event.branch, + "tag": event.tag, + "filter_key": event.filter_key, + "partial": bool(event.partial), + "turn_complete": bool(event.turn_complete), + "error_code": event.error_code, + "error_message": event.error_message, + "model_visible": event.is_model_visible(), + "is_summary_event": event.is_summary_event(), + } + + +def normalize_memory_entry(query_spec: MemoryQuerySpec, key: str, memory: MemoryEntry) -> dict[str, Any]: + return { + "query": query_spec.query, + "key": key, + "author": memory.author, + "text": normalize_summary_text(_content_text(memory.content)), + "has_timestamp": bool(memory.timestamp), + } + + +def normalize_memory_results(memory_records: list[tuple[MemoryQuerySpec, str, list[MemoryEntry]]]) -> list[dict[str, + Any]]: + memories: list[dict[str, Any]] = [] + for query_spec, key, entries in memory_records: + memories.extend(normalize_memory_entry(query_spec, key, entry) for entry in entries) + # Memory search ranking is backend-specific here; content/scope are strict, order is canonicalized. + return sorted( + memories, + key=lambda item: ( + item.get("query") or "", + item.get("author") or "", + item.get("text") or "", + item.get("key") or "", + ), + ) + + +def _normalize_list_session(session: Session) -> dict[str, Any]: + return { + "id": session.id, + "app_name": session.app_name, + "user_id": session.user_id, + "state": _strip_temp_state(session.state), + } + + +async def _normalize_summary(session_service: Any, session: Session) -> dict[str, Any] | None: + summary_text = await session_service.get_session_summary(session) + if summary_text is None: + return None + + summary_events = [event for event in session.events if event.is_summary_event()] + summary_event_text = summary_events[-1].get_text() if summary_events else None + + metadata: dict[str, Any] = { + "session_id": session.id, + "has_summary": True, + "summary_event_count": len(summary_events), + "summary_event_text": normalize_summary_text(summary_event_text), + "compressed_event_count": len(session.events), + "historical_event_count": len(session.historical_events), + } + + manager = getattr(session_service, "summarizer_manager", None) + if manager is not None: + manager_summary = await manager.get_session_summary(session) + if manager_summary is not None: + metadata.update( + { + "manager_session_id": manager_summary.session_id, + "original_event_count": manager_summary.original_event_count, + "manager_compressed_event_count": manager_summary.compressed_event_count, + "has_summary_timestamp": bool(manager_summary.summary_timestamp), + } + ) + + return { + "text": normalize_summary_text(summary_text), + "metadata": canonicalize(metadata), + } + + +async def normalize_snapshot( + *, + backend: str, + case: ReplayCase, + session: Session, + session_service: Any, + memory_records: list[tuple[MemoryQuerySpec, str, list[MemoryEntry]]], +) -> Snapshot: + list_response = await session_service.list_sessions(app_name=case.app_name, user_id=case.user_id) + list_sessions = sorted( + (_normalize_list_session(listed_session) for listed_session in list_response.sessions), + key=lambda item: item["id"], + ) + + return Snapshot( + backend=backend, + case_name=case.name, + session_id=session.id, + app_name=session.app_name, + user_id=session.user_id, + state=_strip_temp_state(session.state), + events=[normalize_event(event, index) for index, event in enumerate(session.events)], + historical_events=[normalize_event(event, index) for index, event in enumerate(session.historical_events)], + memories=normalize_memory_results(memory_records), + summary=await _normalize_summary(session_service, session), + list_sessions=list_sessions, + ) diff --git a/tests/sessions/replay_consistency/report.py b/tests/sessions/replay_consistency/report.py new file mode 100644 index 00000000..536c5191 --- /dev/null +++ b/tests/sessions/replay_consistency/report.py @@ -0,0 +1,141 @@ +"""JSON report writer for replay consistency comparisons.""" + +from __future__ import annotations + +from dataclasses import asdict +import json +import os +from pathlib import Path +from typing import Any + +from .comparator import DiffEntry + + +def _diff_to_dict(diff: DiffEntry) -> dict[str, Any]: + return asdict(diff) + + +def _backend_statuses(backend_pairs: list[str], backend_statuses: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + if backend_statuses is not None: + return backend_statuses + + active_backends = sorted({backend for pair in backend_pairs for backend in pair.split("_vs_")}) + ordered_backends = ["inmemory", "sqlite"] + ordered_backends.extend(backend for backend in active_backends if backend not in ordered_backends) + statuses = [{"name": backend, "status": "ok", "reason": ""} for backend in ordered_backends] + optional_backends = [ + ("external_sql", "TRPC_AGENT_REPLAY_SQL_URL"), + ("redis", "TRPC_AGENT_REPLAY_REDIS_URL"), + ] + for name, env_var in optional_backends: + if name not in active_backends and not os.environ.get(env_var): + statuses.append({"name": name, "status": "skipped", "reason": f"{env_var} is not set"}) + return statuses + + +def _serialize_diffs(diffs: list[DiffEntry], mutation: str | None = None) -> list[dict[str, Any]]: + serialized = [] + for diff in diffs: + item = _diff_to_dict(diff) + if mutation is not None: + item["mutation"] = mutation + serialized.append(item) + return serialized + + +def write_report( + path: Path, + comparison_results: list[dict[str, Any]], + *, + backend_statuses: list[dict[str, Any]] | None = None, + mutation_results: list[dict[str, Any]] | None = None, + generated_at: str = "deterministic", +) -> dict[str, Any]: + backend_pairs = [] + cases: list[dict[str, Any]] = [] + diffs: list[dict[str, Any]] = [] + allowed_diffs: list[dict[str, Any]] = [] + unallowed_diffs: list[dict[str, Any]] = [] + normal_unexpected_count = 0 + + for result in comparison_results: + left_backend = result["left_backend"] + right_backend = result["right_backend"] + pair_name = f"{left_backend}_vs_{right_backend}" + if pair_name not in backend_pairs: + backend_pairs.append(pair_name) + + result_diffs: list[DiffEntry] = result.get("diffs", []) + allowed_count = sum(1 for diff in result_diffs if diff.allowed) + unallowed_count = sum(1 for diff in result_diffs if not diff.allowed) + normal_unexpected_count += unallowed_count + case_name = result["case_name"] + cases.append( + { + "name": case_name, + "backend_pair": pair_name, + "unexpected_diff_count": unallowed_count, + "allowed_diff_count": allowed_count, + "unallowed_diff_count": unallowed_count, + "elapsed_ms": result.get("elapsed_ms", 0), + } + ) + for serialized, diff in zip(_serialize_diffs(result_diffs), result_diffs): + diffs.append(serialized) + if diff.allowed: + allowed_diffs.append(serialized) + else: + unallowed_diffs.append(serialized) + + mutation_results = mutation_results or [] + report_kind = "mutation_replay" if mutation_results else "normal_replay" + undetected_mutations = [] + detected_count = 0 + for result in mutation_results: + result_diffs = result.get("diffs") + if result_diffs is None: + first_diff = result.get("first_diff") + result_diffs = [first_diff] if first_diff is not None else [] + mutation = result["mutation"] + detected = bool(result.get("detected")) + if detected: + detected_count += 1 + else: + undetected_mutations.append({"case_name": result["case_name"], "mutation": mutation}) + + for serialized, diff in zip(_serialize_diffs(result_diffs, mutation), result_diffs): + diffs.append(serialized) + if diff.allowed: + allowed_diffs.append(serialized) + else: + unallowed_diffs.append(serialized) + + report = { + "schema_version": 1, + "report_kind": report_kind, + "generated_at": generated_at, + "generated_by": "tests/sessions/test_replay_consistency.py", + "backend_pairs": backend_pairs, + "backend_statuses": _backend_statuses(backend_pairs, backend_statuses), + "case_count": len(cases), + "cases": cases, + "allowed_diff_count": len(allowed_diffs), + "unallowed_diff_count": len(unallowed_diffs), + "unexpected_diff_count": normal_unexpected_count, + "allowed_diffs": allowed_diffs, + "unallowed_diffs": unallowed_diffs, + "diffs": diffs, + "mutation_summary": { + "mutation_count": len(mutation_results), + "detected_count": detected_count, + "undetected_mutations": undetected_mutations, + }, + "false_positive_summary": { + "normal_case_count": len(comparison_results), + "unexpected_diff_count": normal_unexpected_count, + }, + } + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") + return report diff --git a/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json b/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json new file mode 100644 index 00000000..7a465e12 --- /dev/null +++ b/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json @@ -0,0 +1,3498 @@ +{ + "allowed_diff_count": 0, + "allowed_diffs": [], + "backend_pairs": [], + "backend_statuses": [ + { + "name": "inmemory", + "reason": "", + "status": "ok" + }, + { + "name": "sqlite", + "reason": "", + "status": "ok" + }, + { + "name": "external_sql", + "reason": "TRPC_AGENT_REPLAY_SQL_URL is not set", + "status": "skipped" + }, + { + "name": "redis", + "reason": "TRPC_AGENT_REPLAY_REDIS_URL is not set", + "status": "skipped" + } + ], + "case_count": 0, + "cases": [], + "diffs": [ + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "single_turn_text-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-single-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": {}, + "tag": null, + "text": "Hello! Replay is deterministic.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "single_turn_text-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-single-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Hello replay bot.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 0, + "left": "Hello replay bot.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 6, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[6]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "multi_turn_append_order-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-order-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "First question about replay order.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 0, + "left": "First question about replay order.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "tool", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "tool_call_roundtrip-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-tool-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "What is the weather in Beijing?", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 0, + "left": "What is the weather in Beijing?", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 1, + "left": "Beijing", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_args", + "path": "events[1].function_calls[0].args.city", + "reason": "", + "right": "Shanghai", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 2, + "left": 25, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_response", + "path": "events[2].function_responses[0].response.temperature", + "reason": "", + "right": 30, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 1, + "left": "scoped_state_overwrite-event-01", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].event_id", + "reason": "", + "right": "scoped_state_overwrite-event-02", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 3, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[3]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "scoped_state_overwrite-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-state-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": { + "counter": 1 + }, + "tag": null, + "text": "Start scoped state test.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 0, + "left": "Start scoped state test.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": null, + "left": "gold", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_state_value", + "path": "state.user:tier", + "reason": "", + "right": "silver", + "right_backend": "sqlite", + "section": "state", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 6, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[6]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "memory_preference_search-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-memory-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "I prefer tea in the morning.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 0, + "left": "I prefer tea in the morning.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": null, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "drop_memory", + "path": "memories[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": null, + "left": "Hiking preference noted for weekends.", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "alter_memory_text", + "path": "memories[0].text", + "reason": "", + "right": "I prefer coffee in the morning.", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "memory_multi_session_isolation-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-isolation-a-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": {}, + "tag": null, + "text": "I will remember jasmine tea and lake hiking for User A.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "memory_multi_session_isolation-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-isolation-a-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "User A likes jasmine tea and hiking near lakes.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 0, + "left": "User A likes jasmine tea and hiking near lakes.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": null, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "drop_memory", + "path": "memories[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": null, + "left": "I will remember jasmine tea and lake hiking for User A.", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "alter_memory_text", + "path": "memories[0].text", + "reason": "", + "right": "I prefer coffee in the morning.", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 1, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 3, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[3]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 0, + "left": "Previous conversation summary: summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 3, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 0, + "manager_compressed_event_count": 3, + "manager_session_id": "session-007", + "original_event_count": 8, + "session_id": "session-007", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events" + }, + "text": "summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-007", + "summary_id": "summary:session-007:latest" + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": null, + "left": "summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-007", + "summary_id": "summary:session-007:latest" + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": null, + "left": "session-007", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-007", + "summary_id": "summary:session-007:latest" + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "summary_update_overwrite-event-05", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-summary-update-3", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": {}, + "tag": null, + "text": "Checklist and owners are now included.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 0, + "left": "Previous conversation summary: summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 2, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 0, + "manager_compressed_event_count": 2, + "manager_session_id": "session-008", + "original_event_count": 4, + "session_id": "session-008", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events" + }, + "text": "summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-008", + "summary_id": "summary:session-008:latest" + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": null, + "left": "summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-008", + "summary_id": "summary:session-008:latest" + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": null, + "left": "session-008", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-008", + "summary_id": "summary:session-008:latest" + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 1, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 0, + "left": "Previous conversation summary: summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 4, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 4, + "manager_compressed_event_count": 3, + "manager_session_id": "session-009", + "original_event_count": 6, + "session_id": "session-009", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events" + }, + "text": "summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-009", + "summary_id": "summary:session-009:latest" + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": null, + "left": "summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-009", + "summary_id": "summary:session-009:latest" + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": null, + "left": "session-009", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-009", + "summary_id": "summary:session-009:latest" + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 1, + "left": "duplicate_or_error_recovery-event-01", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].event_id", + "reason": "", + "right": "duplicate_or_error_recovery-event-02", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 5, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[5]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "duplicate_or_error_recovery-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-duplicate-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Duplicate content check begins.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 0, + "left": "Duplicate content check begins.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 3, + "left": "RETRYABLE_BACKEND_ERROR", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_error_code", + "path": "events[3].error_code", + "reason": "", + "right": "WRONG_ERROR_CODE", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 4, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "duplicate_or_error_recovery-event-04", + "filter_key": "retry", + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-duplicate-4", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 4, + "state_delta": {}, + "tag": "retry-recovery", + "text": "Recovery succeeded after retry.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_recovery_event", + "path": "events[4]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "tool", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "serialization_order_nested_payload-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-nested-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Build a nested itinerary payload for Beijing.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 0, + "left": "Build a nested itinerary payload for Beijing.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 1, + "left": "Beijing", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_args", + "path": "events[1].function_calls[0].args.city", + "reason": "", + "right": "Shanghai", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 2, + "left": 21, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_response", + "path": "events[2].function_responses[0].response.temperature", + "reason": "", + "right": 30, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "list_sessions_consistency-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-list-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": { + "session_label": "list-check-updated" + }, + "tag": null, + "text": "The session list should include this normalized state.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "list_sessions_consistency-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-list-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Create a session that can be listed consistently.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 0, + "left": "Create a session that can be listed consistently.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "state_temp_key_ignored_but_persistent_key_compared-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-state-temp-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": { + "session_counter": 2 + }, + "tag": null, + "text": "Persistent state is updated and temp state is not stored.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "state_temp_key_ignored_but_persistent_key_compared-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-state-temp-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": { + "preference": "oolong tea", + "user:tier": "platinum" + }, + "tag": null, + "text": "Remember my persistent state but not the trace id.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 0, + "left": "Remember my persistent state but not the trace id.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": null, + "left": "platinum", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_state_value", + "path": "state.user:tier", + "reason": "", + "right": "silver", + "right_backend": "sqlite", + "section": "state", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 1, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 0, + "left": "Previous conversation summary: summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 4, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 2, + "manager_compressed_event_count": 3, + "manager_session_id": "session-014", + "original_event_count": 4, + "session_id": "session-014", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events" + }, + "text": "summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-014", + "summary_id": "summary:session-014:latest" + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": null, + "left": "summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-014", + "summary_id": "summary:session-014:latest" + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": null, + "left": "session-014", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-014", + "summary_id": "summary:session-014:latest" + } + ], + "false_positive_summary": { + "normal_case_count": 0, + "unexpected_diff_count": 0 + }, + "generated_at": "deterministic", + "generated_by": "tests/sessions/test_replay_consistency.py", + "mutation_summary": { + "detected_count": 80, + "mutation_count": 80, + "undetected_mutations": [] + }, + "report_kind": "mutation_replay", + "schema_version": 1, + "unallowed_diff_count": 80, + "unallowed_diffs": [ + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "single_turn_text-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-single-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": {}, + "tag": null, + "text": "Hello! Replay is deterministic.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "single_turn_text-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-single-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Hello replay bot.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 0, + "left": "Hello replay bot.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 6, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[6]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "multi_turn_append_order-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-order-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "First question about replay order.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 0, + "left": "First question about replay order.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "tool", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "tool_call_roundtrip-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-tool-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "What is the weather in Beijing?", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 0, + "left": "What is the weather in Beijing?", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 1, + "left": "Beijing", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_args", + "path": "events[1].function_calls[0].args.city", + "reason": "", + "right": "Shanghai", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 2, + "left": 25, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_response", + "path": "events[2].function_responses[0].response.temperature", + "reason": "", + "right": 30, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 1, + "left": "scoped_state_overwrite-event-01", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].event_id", + "reason": "", + "right": "scoped_state_overwrite-event-02", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 3, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[3]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "scoped_state_overwrite-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-state-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": { + "counter": 1 + }, + "tag": null, + "text": "Start scoped state test.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 0, + "left": "Start scoped state test.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": null, + "left": "gold", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_state_value", + "path": "state.user:tier", + "reason": "", + "right": "silver", + "right_backend": "sqlite", + "section": "state", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 6, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[6]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "memory_preference_search-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-memory-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "I prefer tea in the morning.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 0, + "left": "I prefer tea in the morning.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": null, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "drop_memory", + "path": "memories[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": null, + "left": "Hiking preference noted for weekends.", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "alter_memory_text", + "path": "memories[0].text", + "reason": "", + "right": "I prefer coffee in the morning.", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "memory_multi_session_isolation-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-isolation-a-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": {}, + "tag": null, + "text": "I will remember jasmine tea and lake hiking for User A.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "memory_multi_session_isolation-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-isolation-a-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "User A likes jasmine tea and hiking near lakes.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 0, + "left": "User A likes jasmine tea and hiking near lakes.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": null, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "drop_memory", + "path": "memories[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": null, + "left": "I will remember jasmine tea and lake hiking for User A.", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "alter_memory_text", + "path": "memories[0].text", + "reason": "", + "right": "I prefer coffee in the morning.", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 1, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 3, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[3]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 0, + "left": "Previous conversation summary: summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 3, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 0, + "manager_compressed_event_count": 3, + "manager_session_id": "session-007", + "original_event_count": 8, + "session_id": "session-007", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events" + }, + "text": "summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-007", + "summary_id": "summary:session-007:latest" + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": null, + "left": "summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-007", + "summary_id": "summary:session-007:latest" + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": null, + "left": "session-007", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-007", + "summary_id": "summary:session-007:latest" + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "summary_update_overwrite-event-05", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-summary-update-3", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": {}, + "tag": null, + "text": "Checklist and owners are now included.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 0, + "left": "Previous conversation summary: summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 2, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 0, + "manager_compressed_event_count": 2, + "manager_session_id": "session-008", + "original_event_count": 4, + "session_id": "session-008", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events" + }, + "text": "summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-008", + "summary_id": "summary:session-008:latest" + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": null, + "left": "summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-008", + "summary_id": "summary:session-008:latest" + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": null, + "left": "session-008", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-008", + "summary_id": "summary:session-008:latest" + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 1, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 0, + "left": "Previous conversation summary: summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 4, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 4, + "manager_compressed_event_count": 3, + "manager_session_id": "session-009", + "original_event_count": 6, + "session_id": "session-009", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events" + }, + "text": "summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-009", + "summary_id": "summary:session-009:latest" + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": null, + "left": "summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-009", + "summary_id": "summary:session-009:latest" + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": null, + "left": "session-009", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-009", + "summary_id": "summary:session-009:latest" + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 1, + "left": "duplicate_or_error_recovery-event-01", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].event_id", + "reason": "", + "right": "duplicate_or_error_recovery-event-02", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 5, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[5]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "duplicate_or_error_recovery-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-duplicate-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Duplicate content check begins.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 0, + "left": "Duplicate content check begins.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 3, + "left": "RETRYABLE_BACKEND_ERROR", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_error_code", + "path": "events[3].error_code", + "reason": "", + "right": "WRONG_ERROR_CODE", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 4, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "duplicate_or_error_recovery-event-04", + "filter_key": "retry", + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-duplicate-4", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 4, + "state_delta": {}, + "tag": "retry-recovery", + "text": "Recovery succeeded after retry.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_recovery_event", + "path": "events[4]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "tool", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "serialization_order_nested_payload-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-nested-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Build a nested itinerary payload for Beijing.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 0, + "left": "Build a nested itinerary payload for Beijing.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 1, + "left": "Beijing", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_args", + "path": "events[1].function_calls[0].args.city", + "reason": "", + "right": "Shanghai", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 2, + "left": 21, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_response", + "path": "events[2].function_responses[0].response.temperature", + "reason": "", + "right": 30, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "list_sessions_consistency-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-list-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": { + "session_label": "list-check-updated" + }, + "tag": null, + "text": "The session list should include this normalized state.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "list_sessions_consistency-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-list-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Create a session that can be listed consistently.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 0, + "left": "Create a session that can be listed consistently.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "state_temp_key_ignored_but_persistent_key_compared-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-state-temp-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": { + "session_counter": 2 + }, + "tag": null, + "text": "Persistent state is updated and temp state is not stored.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "state_temp_key_ignored_but_persistent_key_compared-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-state-temp-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": { + "preference": "oolong tea", + "user:tier": "platinum" + }, + "tag": null, + "text": "Remember my persistent state but not the trace id.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 0, + "left": "Remember my persistent state but not the trace id.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": null, + "left": "platinum", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_state_value", + "path": "state.user:tier", + "reason": "", + "right": "silver", + "right_backend": "sqlite", + "section": "state", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 1, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 0, + "left": "Previous conversation summary: summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 4, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 2, + "manager_compressed_event_count": 3, + "manager_session_id": "session-014", + "original_event_count": 4, + "session_id": "session-014", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events" + }, + "text": "summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-014", + "summary_id": "summary:session-014:latest" + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": null, + "left": "summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-014", + "summary_id": "summary:session-014:latest" + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": null, + "left": "session-014", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-014", + "summary_id": "summary:session-014:latest" + } + ], + "unexpected_diff_count": 0 +} \ No newline at end of file diff --git a/tests/sessions/test_replay_consistency.py b/tests/sessions/test_replay_consistency.py new file mode 100644 index 00000000..c3a1d01f --- /dev/null +++ b/tests/sessions/test_replay_consistency.py @@ -0,0 +1,869 @@ +"""Replay consistency tests for Session / Memory / Summary backends.""" + +from __future__ import annotations + +from dataclasses import asdict +import copy +import json +from pathlib import Path +from typing import Any + +import pytest + +from trpc_agent_sdk.events import Event +from trpc_agent_sdk.sessions import Session +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import EventActions +from trpc_agent_sdk.types import FunctionCall +from trpc_agent_sdk.types import FunctionResponse +from trpc_agent_sdk.types import MemoryEntry +from trpc_agent_sdk.types import Part +from trpc_agent_sdk.types import State + +from .replay_consistency.backends import BackendBundle +from .replay_consistency.backends import build_backends +from .replay_consistency.backends import make_session_config +from .replay_consistency.cases import EventSpec +from .replay_consistency.cases import MemoryQuerySpec +from .replay_consistency.cases import ReplayCase +from .replay_consistency.cases import replay_cases +from .replay_consistency.comparator import DiffEntry +from .replay_consistency.comparator import compare_snapshot_pair +from .replay_consistency.comparator import recursive_diff +from .replay_consistency.comparator import unallowed_diffs +from .replay_consistency.mutations import MUTATION_SECTION +from .replay_consistency.mutations import SYNTHETIC_MUTATIONS as MUTATIONS +from .replay_consistency.mutations import mutate_snapshot +from .replay_consistency.mutations import mutations_for_case +from .replay_consistency.normalizer import Snapshot +from .replay_consistency.normalizer import normalize_snapshot +from .replay_consistency.report import write_report + + +REQUIRED_CASE_NAMES = [ + "single_turn_text", + "multi_turn_append_order", + "tool_call_roundtrip", + "scoped_state_overwrite", + "memory_preference_search", + "memory_multi_session_isolation", + "summary_generation", + "summary_update_overwrite", + "summary_with_event_truncation", + "duplicate_or_error_recovery", +] + +ADDITIONAL_CASE_NAMES = [ + "serialization_order_nested_payload", + "list_sessions_consistency", + "state_temp_key_ignored_but_persistent_key_compared", + "summary_truncation_preserves_recent_context", +] + +FIXED_EVENT_TIMESTAMP_BASE = 2_000_000_000.0 + + +def _event_from_spec(spec: EventSpec, index: int) -> Event: + parts: list[Part] = [] + if spec.text is not None: + parts.append(Part.from_text(text=spec.text)) + if spec.function_call is not None: + parts.append( + Part( + function_call=FunctionCall( + id=spec.function_call.get("id"), + name=spec.function_call["name"], + args=spec.function_call.get("args") or {}, + ) + ) + ) + if spec.function_response is not None: + parts.append( + Part( + function_response=FunctionResponse( + id=spec.function_response.get("id"), + name=spec.function_response["name"], + response=spec.function_response.get("response") or {}, + ) + ) + ) + + return Event( + id=spec.event_id, + invocation_id=spec.invocation_id, + author=spec.author, + content=Content(role=spec.role, parts=parts), + actions=EventActions(state_delta=copy.deepcopy(spec.state_delta or {})), + branch=spec.branch, + tag=spec.tag, + filter_key=spec.filter_key, + partial=spec.partial, + error_code=spec.error_code, + error_message=spec.error_message, + timestamp=FIXED_EVENT_TIMESTAMP_BASE + index, + ) + + +async def _get_required_session(bundle: BackendBundle, case: ReplayCase) -> Session: + stored = await bundle.session_service.get_session( + app_name=case.app_name, + user_id=case.user_id, + session_id=case.session_id, + ) + if stored is None: + pytest.fail(f"{bundle.name} did not return stored session {case.session_id}") + return stored + + +async def _create_required_summary(bundle: BackendBundle, session: Session, case: ReplayCase) -> str: + await bundle.session_service.create_session_summary(session) + summary_text = await bundle.session_service.get_session_summary(session) + if summary_text is None: + pytest.fail(f"{bundle.name} did not create a summary for {case.name}/{session.id}") + if not summary_text.strip(): + pytest.fail(f"{bundle.name} created an empty summary for {case.name}/{session.id}") + return summary_text + + +async def _search_memory_records( + bundle: BackendBundle, + session: Session, + memory_queries: list[MemoryQuerySpec], +) -> list[tuple[MemoryQuerySpec, str, list[MemoryEntry]]]: + records: list[tuple[MemoryQuerySpec, str, list[MemoryEntry]]] = [] + for query_spec in memory_queries: + key = query_spec.key or session.save_key + response = await bundle.memory_service.search_memory(key=key, query=query_spec.query, limit=query_spec.limit) + memory_texts = "\n".join( + "".join(part.text for part in memory.content.parts if part.text) + for memory in response.memories + if memory.content and memory.content.parts + ) + for expected in query_spec.expected_text_fragments: + if expected not in memory_texts: + pytest.fail( + f"{bundle.name} memory query {query_spec.query!r} for {session.id} " + f"did not return expected fragment {expected!r}; got {memory_texts!r}" + ) + records.append((query_spec, key, response.memories)) + return records + + +def _temp_state_keys(state: dict[str, Any] | None) -> list[str]: + if not state: + return [] + return sorted(key for key in state if key.startswith(State.TEMP_PREFIX)) + + +def _assert_no_persisted_temp_state(bundle: BackendBundle, case: ReplayCase, session: Session) -> None: + temp_state_keys = _temp_state_keys(session.state) + if temp_state_keys: + pytest.fail( + f"{bundle.name} persisted temp state on {case.name}/{session.id}: {temp_state_keys}" + ) + + temp_delta_keys: list[tuple[str | None, str]] = [] + for event in [*session.events, *session.historical_events]: + if event.actions: + temp_delta_keys.extend((event.id, key) for key in _temp_state_keys(event.actions.state_delta)) + if temp_delta_keys: + pytest.fail( + f"{bundle.name} persisted temp state_delta keys on {case.name}/{session.id}: {temp_delta_keys}" + ) + + if case.name == "scoped_state_overwrite": + fixture_temp_keys = [ + key + for spec in case.events + for key in _temp_state_keys(spec.state_delta) + ] + if not fixture_temp_keys: + pytest.fail("scoped_state_overwrite no longer exercises temp state filtering") + + +def _assert_summary_snapshot(snapshot: Snapshot, case: ReplayCase) -> None: + if not case.summary_points: + return + if snapshot.summary is None: + pytest.fail(f"{snapshot.backend} summary snapshot is missing for {case.name}") + + summary = snapshot.summary + metadata = summary["metadata"] + if not summary["text"]: + pytest.fail(f"{snapshot.backend} summary text is empty for {case.name}") + if metadata["session_id"] != snapshot.session_id: + pytest.fail(f"{snapshot.backend} summary session mismatch for {case.name}") + if metadata["summary_event_count"] < 1: + pytest.fail(f"{snapshot.backend} summary event missing for {case.name}") + event_text = metadata["summary_event_text"] + if not event_text or not event_text.startswith("Previous conversation summary:"): + pytest.fail(f"{snapshot.backend} summary event text missing prefix for {case.name}") + + summary_events = [event for event in snapshot.events if event["is_summary_event"]] + if not summary_events: + pytest.fail(f"{snapshot.backend} summary event flag missing for {case.name}") + if summary_events[0]["author"] != "system": + pytest.fail(f"{snapshot.backend} summary event author is not system for {case.name}") + + if case.name in {"summary_with_event_truncation", "summary_truncation_preserves_recent_context"}: + if snapshot.events[0]["is_summary_event"] is not True: + pytest.fail(f"{snapshot.backend} truncation case did not keep summary event first") + if metadata["historical_event_count"] == 0 or not snapshot.historical_events: + pytest.fail(f"{snapshot.backend} truncation case did not persist historical events") + expected_last_text = { + "summary_with_event_truncation": "Also add a ferry ride after the summary.", + "summary_truncation_preserves_recent_context": "After summary, add a canal walk.", + }[case.name] + if snapshot.events[-1]["text"] != expected_last_text: + pytest.fail(f"{snapshot.backend} truncation case lost post-summary append") + if case.name == "summary_update_overwrite": + latest_summary_text = summary["text"] + if "Checklist and owners" not in latest_summary_text and "release checklist" not in latest_summary_text: + pytest.fail(f"{snapshot.backend} latest summary did not include post-overwrite content") + + +def _assert_duplicate_error_recovery_snapshot(snapshot: Snapshot) -> None: + if snapshot.case_name != "duplicate_or_error_recovery": + return + error_events = [ + event + for event in snapshot.events + if event["error_code"] == "RETRYABLE_BACKEND_ERROR" + and event["error_message"] == "Simulated retry failure before recovery." + ] + if len(error_events) != 1: + pytest.fail(f"{snapshot.backend} did not preserve the retry error event") + if error_events[0]["event_id"] != "duplicate_or_error_recovery-event-03": + pytest.fail(f"{snapshot.backend} did not preserve the deterministic error event id") + if not any(event["text"] == "Recovery succeeded after retry." for event in snapshot.events): + pytest.fail(f"{snapshot.backend} did not preserve the recovery event") + + +def _assert_fixture_event_ids_preserved(snapshot: Snapshot, case: ReplayCase) -> None: + expected_event_ids = {spec.event_id for spec in case.events} + for event in [*snapshot.events, *snapshot.historical_events]: + if event["is_summary_event"]: + continue + if event["event_id"] not in expected_event_ids: + pytest.fail( + f"{snapshot.backend} did not preserve deterministic event id for " + f"{case.name}: {event['event_id']!r}" + ) + + +async def _run_standard_case(bundle: BackendBundle, case: ReplayCase) -> Snapshot: + summary_texts: list[str] = [] + session = await bundle.session_service.create_session( + app_name=case.app_name, + user_id=case.user_id, + session_id=case.session_id, + state=copy.deepcopy(case.initial_state), + ) + for index, spec in enumerate(case.events): + event = _event_from_spec(spec, index) + await bundle.session_service.append_event(session, event) + if index in case.summary_points: + summary_texts.append(await _create_required_summary(bundle, session, case)) + session = await _get_required_session(bundle, case) + + stored_session = await _get_required_session(bundle, case) + _assert_no_persisted_temp_state(bundle, case, stored_session) + if case.name == "summary_update_overwrite" and len(summary_texts) >= 2 and summary_texts[0] == summary_texts[-1]: + pytest.fail(f"{bundle.name} did not overwrite the cached summary for {case.name}") + + await bundle.memory_service.store_session(stored_session) + memory_records = await _search_memory_records(bundle, stored_session, case.memory_queries) + snapshot = await normalize_snapshot( + backend=bundle.name, + case=case, + session=stored_session, + session_service=bundle.session_service, + memory_records=memory_records, + ) + _assert_summary_snapshot(snapshot, case) + _assert_fixture_event_ids_preserved(snapshot, case) + _assert_duplicate_error_recovery_snapshot(snapshot) + return snapshot + + +async def _run_memory_isolation_case(bundle: BackendBundle, case: ReplayCase) -> Snapshot: + session_a = await bundle.session_service.create_session( + app_name=case.app_name, + user_id=case.user_id, + session_id=case.session_id, + state=copy.deepcopy(case.initial_state), + ) + for index, spec in enumerate(case.events): + await bundle.session_service.append_event(session_a, _event_from_spec(spec, index)) + + session_b = await bundle.session_service.create_session( + app_name=case.app_name, + user_id="user-isolation-b", + session_id="session-006-b", + state={}, + ) + other_specs = [ + EventSpec( + event_id="memory_multi_session_isolation-b-event-00", + invocation_id="inv-isolation-b-1", + author="user", + role="user", + text="User B also likes jasmine tea and hiking, but only with city-museums-b.", + function_call=None, + function_response=None, + state_delta=None, + branch=None, + tag=None, + filter_key=None, + ), + EventSpec( + event_id="memory_multi_session_isolation-b-event-01", + invocation_id="inv-isolation-b-1", + author="assistant", + role="model", + text="I will remember jasmine tea, hiking, and city-museums-b for User B.", + function_call=None, + function_response=None, + state_delta=None, + branch=None, + tag=None, + filter_key=None, + ), + ] + for index, spec in enumerate(other_specs, start=100): + await bundle.session_service.append_event(session_b, _event_from_spec(spec, index)) + + stored_a = await _get_required_session(bundle, case) + stored_b = await bundle.session_service.get_session( + app_name=case.app_name, + user_id="user-isolation-b", + session_id="session-006-b", + ) + if stored_b is None: + pytest.fail(f"{bundle.name} did not return isolation control session") + _assert_no_persisted_temp_state(bundle, case, stored_a) + _assert_no_persisted_temp_state(bundle, case, stored_b) + + await bundle.memory_service.store_session(stored_a) + await bundle.memory_service.store_session(stored_b) + memory_records = await _search_memory_records(bundle, stored_a, case.memory_queries) + leaked_text = "\n".join( + "".join(part.text for part in memory.content.parts if part.text) + for _, _, memories in memory_records + for memory in memories + if memory.content and memory.content.parts + ) + if "city-museums-b" in leaked_text: + pytest.fail(f"{bundle.name} leaked user B memory into user A search: {leaked_text!r}") + + snapshot = await normalize_snapshot( + backend=bundle.name, + case=case, + session=stored_a, + session_service=bundle.session_service, + memory_records=memory_records, + ) + _assert_fixture_event_ids_preserved(snapshot, case) + return snapshot + + +async def run_case(bundle: BackendBundle, case: ReplayCase) -> Snapshot: + try: + if case.name == "memory_multi_session_isolation": + return await _run_memory_isolation_case(bundle, case) + return await _run_standard_case(bundle, case) + finally: + await bundle.close() + + +def _session_config_for_case(case: ReplayCase): + return make_session_config( + store_historical_events=case.name in { + "summary_with_event_truncation", + "summary_truncation_preserves_recent_context", + } + ) + + +def _keep_recent_count_for_case(case: ReplayCase) -> int: + if case.name == "summary_update_overwrite": + return 1 + return 2 + + +async def _run_real_inmemory_snapshot(tmp_path: Path, case: ReplayCase) -> dict[str, Any]: + backends = await build_backends( + tmp_path, + session_config=_session_config_for_case(case), + keep_recent_count=_keep_recent_count_for_case(case), + ) + selected: BackendBundle | None = None + unused: list[BackendBundle] = [] + for backend in backends: + if backend.name == "inmemory": + selected = backend + else: + unused.append(backend) + if selected is None: + for backend in unused: + await backend.close() + pytest.fail("InMemory replay backend was not built") + + for backend in unused: + await backend.close() + return asdict(await run_case(selected, case)) + + +@pytest.mark.asyncio +async def test_replay_consistency_inmemory_vs_sqlite(tmp_path: Path): + cases = replay_cases() + assert len(cases) == len(REQUIRED_CASE_NAMES) + len(ADDITIONAL_CASE_NAMES) + comparison_results: list[dict[str, Any]] = [] + report_path = tmp_path / "session_memory_summary_diff_report.json" + + for case in cases: + case_tmp_path = tmp_path / case.name + backends = await build_backends( + case_tmp_path, + session_config=_session_config_for_case(case), + keep_recent_count=_keep_recent_count_for_case(case), + ) + assert {"inmemory", "sqlite"} <= {backend.name for backend in backends} + + snapshots: dict[str, Snapshot] = {} + for backend in backends: + snapshots[backend.name] = await run_case(backend, case) + + left = snapshots["inmemory"] + for right_name, right in snapshots.items(): + if right_name == "inmemory": + continue + diffs = compare_snapshot_pair(left, right) + comparison_results.append( + { + "case_name": case.name, + "left_backend": left.backend, + "right_backend": right.backend, + "diffs": diffs, + } + ) + unexpected = unallowed_diffs(diffs) + if unexpected: + write_report(report_path, comparison_results) + pytest.fail(f"Replay diff detected for {case.name}: {[asdict(diff) for diff in unexpected]}") + + report = write_report(report_path, comparison_results) + assert report["report_kind"] == "normal_replay" + assert report["false_positive_summary"] == { + "normal_case_count": len(cases), + "unexpected_diff_count": 0, + } + + +def test_replay_case_count_and_names(): + names = [case.name for case in replay_cases()] + assert names[:len(REQUIRED_CASE_NAMES)] == REQUIRED_CASE_NAMES + assert names[len(REQUIRED_CASE_NAMES):] == ADDITIONAL_CASE_NAMES + + +def test_replay_case_manifest_matches_registry(): + manifest_path = Path(__file__).parent / "replay_cases" / "session_memory_summary_replay_cases.jsonl" + manifest_lines = manifest_path.read_text(encoding="utf-8").splitlines() + assert manifest_lines + manifest_cases = [json.loads(line) for line in manifest_lines] + assert [case["name"] for case in manifest_cases] == [case.name for case in replay_cases()] + for case in manifest_cases: + assert case["description"] + assert case["operations"] + assert case["expected_risks"] + + +def test_report_contract(tmp_path: Path): + diff = DiffEntry( + case_name="case", + left_backend="inmemory", + right_backend="sqlite", + session_id="session", + event_index=1, + memory_index=None, + summary_id="summary:session:latest", + section="events", + path="events[1].text", + left="hello", + right="bye", + allowed=False, + reason="", + ) + allowed_diff = DiffEntry( + case_name="case", + left_backend="inmemory", + right_backend="sqlite", + session_id="session", + event_index=None, + memory_index=None, + summary_id=None, + section="backend", + path="backend", + left="inmemory", + right="sqlite", + allowed=True, + reason="backend name differs by design", + ) + path = tmp_path / "report.json" + report = write_report( + path, + [ + { + "case_name": "case", + "left_backend": "inmemory", + "right_backend": "sqlite", + "diffs": [allowed_diff, diff], + } + ], + generated_at="unit-test", + ) + loaded = json.loads(path.read_text(encoding="utf-8")) + assert loaded == report + serialized = loaded["unallowed_diffs"][0] + for field in DiffEntry.__dataclass_fields__: + assert field in serialized + assert loaded["schema_version"] == 1 + assert loaded["report_kind"] == "normal_replay" + assert loaded["generated_at"] == "unit-test" + assert loaded["backend_pairs"] == ["inmemory_vs_sqlite"] + assert loaded["backend_statuses"] + assert loaded["allowed_diff_count"] == 1 + assert loaded["unallowed_diff_count"] == 1 + assert loaded["unexpected_diff_count"] == 1 + assert len(loaded["allowed_diffs"]) == 1 + assert len(loaded["unallowed_diffs"]) == 1 + assert len(loaded["diffs"]) == 2 + assert loaded["cases"][0]["backend_pair"] == "inmemory_vs_sqlite" + assert loaded["cases"][0]["allowed_diff_count"] == 1 + assert loaded["cases"][0]["unexpected_diff_count"] == 1 + assert loaded["cases"][0]["unallowed_diff_count"] == 1 + assert loaded["cases"][0]["elapsed_ms"] == 0 + assert loaded["mutation_summary"] == { + "mutation_count": 0, + "detected_count": 0, + "undetected_mutations": [], + } + assert loaded["false_positive_summary"] == { + "normal_case_count": 1, + "unexpected_diff_count": 1, + } + + +def test_checked_in_reports_are_valid_json(): + report_path = ( + Path(__file__).parent + / "replay_consistency" + / "session_memory_summary_mutation_report.json" + ) + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["schema_version"] == 1 + assert report["report_kind"] == "mutation_replay" + assert report["unexpected_diff_count"] == 0 + assert report["false_positive_summary"] == { + "normal_case_count": 0, + "unexpected_diff_count": 0, + } + mutation_summary = report["mutation_summary"] + assert mutation_summary["mutation_count"] == mutation_summary["detected_count"] + assert mutation_summary["undetected_mutations"] == [] + assert any( + diff.get("mutation") == "summary_wrong_session_id" + and diff["section"] == "summary" + and diff["path"] == "summary.metadata.session_id" + for diff in report["unallowed_diffs"] + ) + assert any( + diff.get("mutation") == "change_tool_response" + and "function_responses" in diff["path"] + for diff in report["unallowed_diffs"] + ) + + +def _assert_real_mutation_diff_context( + *, + case: ReplayCase, + mutation: str, + clean_snapshot: dict[str, Any], + diffs: list[DiffEntry], +) -> list[DiffEntry]: + unexpected = unallowed_diffs(diffs) + assert unexpected, f"{case.name}/{mutation} was not detected" + + for diff in unexpected: + assert diff.case_name == case.name + assert diff.session_id == clean_snapshot["session_id"] + assert diff.section + assert diff.path + assert hasattr(diff, "left") + assert hasattr(diff, "right") + + expected_section = MUTATION_SECTION[mutation] + matching = [diff for diff in unexpected if diff.section == expected_section] + assert matching, f"{case.name}/{mutation} did not report section {expected_section}" + report_diffs = matching + + if mutation == "drop_summary": + report_diffs = [diff for diff in matching if diff.path == "summary"] + assert report_diffs + elif mutation == "overwrite_summary_with_stale_text": + report_diffs = [diff for diff in matching if diff.path == "summary.text"] + assert report_diffs + elif mutation == "summary_wrong_session_id": + report_diffs = [ + diff + for diff in matching + if diff.path == "summary.metadata.session_id" + ] + assert report_diffs + + if mutation == "change_tool_args": + report_diffs = [ + diff + for diff in matching + if diff.event_index is not None and "function_calls" in diff.path + ] + assert report_diffs + elif mutation == "change_tool_response": + report_diffs = [ + diff + for diff in matching + if diff.event_index is not None and "function_responses" in diff.path + ] + assert report_diffs + elif expected_section == "memories": + assert all(diff.section == "memories" for diff in matching) + report_diffs = [ + diff + for diff in matching + if diff.memory_index is not None or diff.path == "memories" + ] + assert report_diffs + elif expected_section == "state": + assert all(diff.section == "state" and diff.path.startswith("state.") for diff in matching) + return report_diffs + + +@pytest.mark.asyncio +async def test_real_replay_snapshot_mutation_detection_reports_precise_paths(tmp_path: Path): + mutation_results: list[dict[str, Any]] = [] + + for case in replay_cases(): + clean = await _run_real_inmemory_snapshot(tmp_path / f"real-mutation-{case.name}", case) + for mutation in mutations_for_case(case): + mutated = copy.deepcopy(clean) + mutated["backend"] = "sqlite" + mutate_snapshot(mutation, mutated) + diffs = recursive_diff(clean, mutated) + unexpected = unallowed_diffs(diffs) + report_diffs = _assert_real_mutation_diff_context( + case=case, + mutation=mutation, + clean_snapshot=clean, + diffs=diffs, + ) + mutation_results.append( + { + "case_name": case.name, + "mutation": mutation, + "detected": bool(unexpected), + "diff_count": len(unexpected), + "diffs": [report_diffs[0]], + } + ) + + report = write_report( + tmp_path / "session_memory_summary_mutation_report.json", + [], + mutation_results=mutation_results, + ) + mutation_summary = report["mutation_summary"] + assert report["report_kind"] == "mutation_replay" + assert report["unexpected_diff_count"] == 0 + assert report["false_positive_summary"] == { + "normal_case_count": 0, + "unexpected_diff_count": 0, + } + assert mutation_summary["mutation_count"] > 0 + assert mutation_summary["mutation_count"] == mutation_summary["detected_count"] + assert mutation_summary["undetected_mutations"] == [] + + +def _clean_mutation_snapshot() -> dict[str, Any]: + snapshot = Snapshot( + backend="inmemory", + case_name="mutation_fixture", + session_id="session-mutation", + app_name="replay-app", + user_id="user-mutation", + state={ + "user:tier": "gold", + "preference": "tea", + }, + events=[ + { + "stable_index": 0, + "event_id": "normalized", + "invocation_id": "inv-mutation-1", + "author": "user", + "role": "user", + "text": "What is the weather in Beijing?", + "function_calls": [], + "function_responses": [], + "state_delta": {}, + "branch": None, + "tag": None, + "filter_key": None, + "partial": False, + "turn_complete": False, + "error_code": None, + "error_message": None, + "model_visible": True, + "is_summary_event": False, + }, + { + "stable_index": 1, + "event_id": "normalized", + "invocation_id": "inv-mutation-1", + "author": "assistant", + "role": "model", + "text": None, + "function_calls": [ + { + "id": "call-weather-1", + "name": "get_weather", + "args": { + "city": "Beijing", + "unit": "celsius", + }, + } + ], + "function_responses": [], + "state_delta": {}, + "branch": "weather.main", + "tag": "tool-call", + "filter_key": "weather", + "partial": False, + "turn_complete": False, + "error_code": None, + "error_message": None, + "model_visible": True, + "is_summary_event": False, + }, + { + "stable_index": 2, + "event_id": "normalized", + "invocation_id": "inv-mutation-1", + "author": "assistant", + "role": "model", + "text": "Beijing is sunny and 25 celsius.", + "function_calls": [], + "function_responses": [], + "state_delta": {}, + "branch": "weather.main", + "tag": "final", + "filter_key": "weather", + "partial": False, + "turn_complete": False, + "error_code": None, + "error_message": None, + "model_visible": True, + "is_summary_event": False, + }, + ], + historical_events=[ + { + "stable_index": 0, + "event_id": "normalized", + "invocation_id": "inv-old", + "author": "user", + "role": "user", + "text": "Old preference was captured.", + "function_calls": [], + "function_responses": [], + "state_delta": {}, + "branch": None, + "tag": None, + "filter_key": None, + "partial": False, + "turn_complete": False, + "error_code": None, + "error_message": None, + "model_visible": True, + "is_summary_event": False, + } + ], + memories=[ + { + "query": "tea", + "key": "replay-app/user-mutation", + "author": "user", + "text": "I prefer tea in the morning.", + "has_timestamp": True, + } + ], + summary={ + "text": "summary(session-mutation): user=Old preference was captured. | facts=1-events", + "metadata": { + "session_id": "session-mutation", + "has_summary": True, + "summary_event_count": 1, + "summary_event_text": ( + "Previous conversation summary: summary(session-mutation): " + "user=Old preference was captured. | facts=1-events" + ), + "compressed_event_count": 3, + "historical_event_count": 1, + "manager_session_id": "session-mutation", + "original_event_count": 4, + "manager_compressed_event_count": 3, + "has_summary_timestamp": True, + }, + }, + list_sessions=[ + { + "id": "session-mutation", + "app_name": "replay-app", + "user_id": "user-mutation", + "state": { + "user:tier": "gold", + "preference": "tea", + }, + } + ], + ) + return asdict(snapshot) + + + + +@pytest.mark.parametrize("mutation", MUTATIONS) +def test_replay_mutation_detection(mutation: str): + clean = _clean_mutation_snapshot() + mutated = copy.deepcopy(clean) + mutated["backend"] = "sqlite" + mutate_snapshot(mutation, mutated) + diffs = recursive_diff(clean, mutated) + assert unallowed_diffs(diffs), f"{mutation} was not detected" + + +@pytest.mark.parametrize( + ("mutation", "expected_path"), + [ + ("drop_summary", "summary"), + ("overwrite_summary_text", "summary.text"), + ("wrong_summary_session", "summary.metadata.session_id"), + ], +) +def test_summary_required_mutations_detected(mutation: str, expected_path: str): + clean = _clean_mutation_snapshot() + mutated = copy.deepcopy(clean) + mutated["backend"] = "sqlite" + mutate_snapshot(mutation, mutated) + diffs = unallowed_diffs(recursive_diff(clean, mutated)) + assert diffs + assert any(diff.section == "summary" and diff.path == expected_path for diff in diffs) diff --git a/tests/sessions/test_replay_consistency_mutations.py b/tests/sessions/test_replay_consistency_mutations.py new file mode 100644 index 00000000..4c5a1824 --- /dev/null +++ b/tests/sessions/test_replay_consistency_mutations.py @@ -0,0 +1,150 @@ +"""End-to-end replay snapshot mutation tests.""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any + +import pytest + +from .replay_consistency.cases import replay_cases +from .replay_consistency.comparator import DiffEntry +from .replay_consistency.comparator import recursive_diff +from .replay_consistency.comparator import unallowed_diffs +from .replay_consistency.mutations import MUTATION_SECTION +from .replay_consistency.mutations import SUMMARY_REQUIRED_MUTATIONS +from .replay_consistency.mutations import mutate_snapshot +from .replay_consistency.mutations import mutations_for_case +from .replay_consistency.report import write_report +from .test_replay_consistency import _run_real_inmemory_snapshot + + +REQUIRED_REAL_MUTATIONS = { + "drop_event", + "reorder_events", + "duplicate_event", + "alter_event_text", + "change_tool_args", + "change_tool_response", + "change_state_value", + "drop_memory", + "alter_memory_text", + "drop_summary", + "overwrite_summary_with_stale_text", + "summary_wrong_session_id", + "change_error_code", + "drop_recovery_event", +} + + +def test_real_mutation_registry_covers_required_mutations(): + assert REQUIRED_REAL_MUTATIONS <= set(MUTATION_SECTION) + seen_mutations = {mutation for case in replay_cases() for mutation in mutations_for_case(case)} + assert REQUIRED_REAL_MUTATIONS <= seen_mutations + + +def _assert_diff_context( + *, + case_name: str, + mutation: str, + clean_snapshot: dict[str, Any], + diffs: list[DiffEntry], +) -> None: + unexpected = unallowed_diffs(diffs) + assert unexpected, f"{case_name}/{mutation} was not detected" + + expected_section = MUTATION_SECTION[mutation] + matching = [diff for diff in unexpected if diff.section == expected_section] + assert matching, f"{case_name}/{mutation} did not report section {expected_section}" + assert all(diff.session_id == clean_snapshot["session_id"] for diff in unexpected) + assert all(diff.path for diff in unexpected) + + if expected_section == "events": + assert any(diff.event_index is not None for diff in matching) + if mutation == "change_tool_args": + assert any("function_calls" in diff.path for diff in matching) + if mutation == "change_tool_response": + assert any("function_responses" in diff.path for diff in matching) + if mutation == "alter_event_text": + assert any(diff.path.endswith(".text") for diff in matching) + if mutation == "change_error_code": + assert any(diff.path.endswith(".error_code") for diff in matching) + if mutation == "drop_recovery_event": + assert any(diff.path.startswith("events[") and diff.right == "" for diff in matching) + elif expected_section == "memories": + assert any(diff.memory_index is not None or diff.section == "memories" for diff in matching) + elif expected_section == "summary": + assert any(diff.summary_id for diff in matching) + if mutation == "drop_summary": + assert any(diff.path == "summary" for diff in matching) + elif mutation in {"overwrite_summary_with_stale_text", "overwrite_summary_text"}: + assert any(diff.path == "summary.text" for diff in matching) + elif mutation in {"summary_wrong_session_id", "wrong_summary_session"}: + assert any("summary.metadata.session_id" in diff.path for diff in matching) + + +@pytest.mark.asyncio +async def test_real_replay_snapshot_mutation_detection_reports_precise_paths(tmp_path: Path): + mutation_results: list[dict[str, Any]] = [] + for case in replay_cases(): + clean = await _run_real_inmemory_snapshot(tmp_path / f"real-mutation-{case.name}", case) + for mutation in mutations_for_case(case): + mutated = copy.deepcopy(clean) + mutated["backend"] = "sqlite" + mutate_snapshot(mutation, mutated) + diffs = unallowed_diffs(recursive_diff(clean, mutated)) + _assert_diff_context( + case_name=case.name, + mutation=mutation, + clean_snapshot=clean, + diffs=diffs, + ) + mutation_results.append( + { + "case_name": case.name, + "mutation": mutation, + "detected": bool(unallowed_diffs(diffs)), + "diff_count": len(diffs), + "first_diff": diffs[0], + "diffs": diffs, + } + ) + + report = write_report( + tmp_path / "session_memory_summary_mutation_report.json", + [], + mutation_results=mutation_results, + ) + assert report["schema_version"] == 1 + assert report["mutation_summary"]["mutation_count"] > 0 + assert report["mutation_summary"]["mutation_count"] == report["mutation_summary"]["detected_count"] + assert report["mutation_summary"]["undetected_mutations"] == [] + assert any( + diff.get("mutation") == "summary_wrong_session_id" + and diff["path"] == "summary.metadata.session_id" + for diff in report["unallowed_diffs"] + ) + assert any( + diff.get("mutation") in {"change_tool_args", "change_tool_response"} + and diff["event_index"] is not None + for diff in report["unallowed_diffs"] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mutation", SUMMARY_REQUIRED_MUTATIONS) +async def test_real_summary_required_mutations_detected(tmp_path: Path, mutation: str): + case = next(case for case in replay_cases() if case.name == "summary_generation") + clean = await _run_real_inmemory_snapshot(tmp_path / f"summary-required-{mutation}", case) + mutated = copy.deepcopy(clean) + mutated["backend"] = "sqlite" + mutate_snapshot(mutation, mutated) + diffs = unallowed_diffs(recursive_diff(clean, mutated)) + _assert_diff_context( + case_name=case.name, + mutation=mutation, + clean_snapshot=clean, + diffs=diffs, + ) + assert all(diff.section == "summary" for diff in unallowed_diffs(diffs) if diff.path.startswith("summary"))