From 9150b1762ff44caa8c672711238335399d7c29f5 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 18:07:39 +0800 Subject: [PATCH 1/9] Add replay consistency harness --- session_memory_summary_diff_report.json | 51 ++ tests/sessions/replay_consistency/README.md | 21 + tests/sessions/replay_consistency/__init__.py | 31 + tests/sessions/replay_consistency/backends.py | 203 ++++++ tests/sessions/replay_consistency/cases.py | 655 ++++++++++++++++++ .../sessions/replay_consistency/comparator.py | 170 +++++ .../sessions/replay_consistency/normalizer.py | 202 ++++++ tests/sessions/replay_consistency/report.py | 54 ++ tests/sessions/test_replay_consistency.py | 591 ++++++++++++++++ 9 files changed, 1978 insertions(+) create mode 100644 session_memory_summary_diff_report.json create mode 100644 tests/sessions/replay_consistency/README.md create mode 100644 tests/sessions/replay_consistency/__init__.py create mode 100644 tests/sessions/replay_consistency/backends.py create mode 100644 tests/sessions/replay_consistency/cases.py create mode 100644 tests/sessions/replay_consistency/comparator.py create mode 100644 tests/sessions/replay_consistency/normalizer.py create mode 100644 tests/sessions/replay_consistency/report.py create mode 100644 tests/sessions/test_replay_consistency.py diff --git a/session_memory_summary_diff_report.json b/session_memory_summary_diff_report.json new file mode 100644 index 00000000..87b2f5b5 --- /dev/null +++ b/session_memory_summary_diff_report.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "generated_by": "tests/sessions/test_replay_consistency.py", + "backend_pairs": [ + "inmemory_vs_sqlite" + ], + "case_count": 10, + "cases": [ + { + "name": "single_turn_text", + "unallowed_diff_count": 0 + }, + { + "name": "multi_turn_append_order", + "unallowed_diff_count": 0 + }, + { + "name": "tool_call_roundtrip", + "unallowed_diff_count": 0 + }, + { + "name": "scoped_state_overwrite", + "unallowed_diff_count": 0 + }, + { + "name": "memory_preference_search", + "unallowed_diff_count": 0 + }, + { + "name": "memory_multi_session_isolation", + "unallowed_diff_count": 0 + }, + { + "name": "summary_generation", + "unallowed_diff_count": 0 + }, + { + "name": "summary_update_overwrite", + "unallowed_diff_count": 0 + }, + { + "name": "summary_with_event_truncation", + "unallowed_diff_count": 0 + }, + { + "name": "duplicate_or_error_recovery", + "unallowed_diff_count": 0 + } + ], + "diffs": [] +} diff --git a/tests/sessions/replay_consistency/README.md b/tests/sessions/replay_consistency/README.md new file mode 100644 index 00000000..dce5044c --- /dev/null +++ b/tests/sessions/replay_consistency/README.md @@ -0,0 +1,21 @@ +# Replay Consistency Harness + +这个 harness 用同一组 deterministic replay cases 驱动多个后端。每个 case 先通过正式 +`SessionServiceABC` 创建 session，再按固定 id、timestamp、invocation_id 写入事件；最后用 +`MemoryServiceABC.store_session(session)` 写入 memory，并通过 `search_memory(key, query, limit)` +读取结果。默认矩阵真实运行 InMemory 和 SQLite；如果设置 +`TRPC_AGENT_REPLAY_REDIS_URL`，Redis 会作为可选后端加入比较。 + +snapshot 会归一化非业务字段：事件 timestamp 不比较精确值，summary timestamp 只比较是否存在， +自动生成的 event id 统一为 normalized，dict 递归按 key 排序，memory timestamp 只保留 +`has_timestamp`。memory 返回顺序不作为语义，因此按 `(query, author, text)` 排序。 + +严格比较的字段包括 event 顺序、author、role、text、tool args、tool response、state、memory +content、summary text、summary session_id、summary overwrite 后的最新值、summary event flag、 +summary event 数量，以及 historical_events 数量和内容。summary 使用 +`DeterministicSessionSummarizer`，不调用真实 LLM，但保留现有 `SessionSummarizer` 的压缩逻辑， +因此 summary event 和 historical_events 仍由产品代码生成。 + +SQLite 后端使用独立临时 SQLite 文件并显式初始化 SQL storage；初始化失败不会静默降级。mutation +tests 会对 clean snapshot 人为制造 drop、reorder、state、memory、summary 等不一致，验证 +recursive diff 能输出定位到 session/event/memory/summary/path 的非允许差异。 diff --git a/tests/sessions/replay_consistency/__init__.py b/tests/sessions/replay_consistency/__init__.py new file mode 100644 index 00000000..7493aa26 --- /dev/null +++ b/tests/sessions/replay_consistency/__init__.py @@ -0,0 +1,31 @@ +"""Replay consistency harness for session, memory, and summary backends.""" + +from .backends import BackendBundle +from .backends import build_backends +from .backends import DeterministicSessionSummarizer +from .cases import EventSpec +from .cases import MemoryQuerySpec +from .cases import ReplayCase +from .cases import replay_cases +from .comparator import DiffEntry +from .comparator import compare_snapshot_pair +from .comparator import recursive_diff +from .normalizer import Snapshot +from .normalizer import normalize_snapshot +from .report import write_report + +__all__ = [ + "BackendBundle", + "build_backends", + "DeterministicSessionSummarizer", + "EventSpec", + "MemoryQuerySpec", + "ReplayCase", + "replay_cases", + "DiffEntry", + "compare_snapshot_pair", + "recursive_diff", + "Snapshot", + "normalize_snapshot", + "write_report", +] diff --git a/tests/sessions/replay_consistency/backends.py b/tests/sessions/replay_consistency/backends.py new file mode 100644 index 00000000..f09fa29e --- /dev/null +++ b/tests/sessions/replay_consistency/backends.py @@ -0,0 +1,203 @@ +"""Backend construction for replay consistency tests.""" + +from __future__ import annotations + +from collections.abc import Awaitable +from collections.abc import Callable +from dataclasses import dataclass +import os +from pathlib import Path +import re +from typing import Any +from typing import AsyncGenerator + +import pytest + +from trpc_agent_sdk.abc import MemoryServiceConfig +from trpc_agent_sdk.events import Event +from trpc_agent_sdk.memory import InMemoryMemoryService +from trpc_agent_sdk.memory import SqlMemoryService +from trpc_agent_sdk.models import LLMModel +from trpc_agent_sdk.models import LlmRequest +from trpc_agent_sdk.models import LlmResponse +from trpc_agent_sdk.sessions import InMemorySessionService +from trpc_agent_sdk.sessions import Session +from trpc_agent_sdk.sessions import SessionServiceConfig +from trpc_agent_sdk.sessions import SessionSummarizer +from trpc_agent_sdk.sessions import SqlSessionService +from trpc_agent_sdk.sessions import SummarizerSessionManager + + +@dataclass +class BackendBundle: + name: str + session_service: Any + memory_service: Any + close: Callable[[], Awaitable[None]] + + +class FakeModel(LLMModel): + """Concrete model object for summarizer metadata; it is never called.""" + + @classmethod + def supported_models(cls) -> list[str]: + return [r"deterministic-replay-model"] + + async def _generate_async_impl( + self, + request: LlmRequest, + stream: bool = False, + ctx: Any | None = None, + ) -> AsyncGenerator[LlmResponse, None]: + if False: + yield LlmResponse() + + +class DeterministicSessionSummarizer(SessionSummarizer): + """A no-LLM summarizer that still uses SessionSummarizer compression.""" + + async def _compress_session_to_summary( + self, + events: list[Event], + session_id: str, + ctx: Any | None = None, + ) -> str | None: + if not events: + return None + + fragments: list[str] = [] + for event in events: + text = event.get_text().strip() + if not text: + calls = event.get_function_calls() + responses = event.get_function_responses() + if calls: + text = "tool_call:" + ",".join(call.name or "" for call in calls) + elif responses: + text = "tool_response:" + ",".join(response.name or "" for response in responses) + if text: + normalized_text = re.sub(r"\s+", " ", text).strip() + fragments.append(f"{event.author or 'unknown'}={normalized_text}") + + if not fragments: + return None + + return f"summary({session_id}): {' | '.join(fragments)} | facts={len(events)}-events" + + +def make_session_config(*, store_historical_events: bool = False) -> SessionServiceConfig: + config = SessionServiceConfig(store_historical_events=store_historical_events) + config.clean_ttl_config() + return config + + +def _make_memory_config() -> MemoryServiceConfig: + config = MemoryServiceConfig(enabled=True) + config.clean_ttl_config() + return config + + +def _make_summarizer_manager(keep_recent_count: int = 2) -> SummarizerSessionManager: + model = FakeModel(model_name="deterministic-replay-model") + summarizer = DeterministicSessionSummarizer( + model=model, + check_summarizer_functions=[lambda session: bool(session.events)], + keep_recent_count=keep_recent_count, + ) + return SummarizerSessionManager(model=model, summarizer=summarizer, auto_summarize=True) + + +def _sqlite_url(path: Path) -> str: + return f"sqlite:///{path.as_posix()}" + + +async def _close_services(session_service: Any, memory_service: Any) -> None: + await memory_service.close() + await session_service.close() + + +async def build_backends( + tmp_path: Path, + session_config: SessionServiceConfig | None = None, + *, + keep_recent_count: int = 2, +) -> list[BackendBundle]: + """Build default replay backends. + + SQLite is part of the default matrix. Its initialization must either + succeed, skip for a missing optional dependency, or fail the test. + """ + tmp_path.mkdir(parents=True, exist_ok=True) + base_config = session_config or make_session_config() + memory_config = _make_memory_config() + backends: list[BackendBundle] = [] + + in_memory_session = InMemorySessionService(session_config=base_config.model_copy(deep=True)) + in_memory_session.set_summarizer_manager(_make_summarizer_manager(keep_recent_count), force=True) + in_memory_memory = InMemoryMemoryService(memory_service_config=memory_config.model_copy(deep=True)) + backends.append( + BackendBundle( + name="inmemory", + session_service=in_memory_session, + memory_service=in_memory_memory, + close=lambda s=in_memory_session, m=in_memory_memory: _close_services(s, m), + ) + ) + + sqlite_session = SqlSessionService( + db_url=_sqlite_url(tmp_path / "replay_sessions.sqlite"), + is_async=False, + session_config=base_config.model_copy(deep=True), + ) + sqlite_session.set_summarizer_manager(_make_summarizer_manager(keep_recent_count), force=True) + sqlite_memory = SqlMemoryService( + db_url=_sqlite_url(tmp_path / "replay_memory.sqlite"), + is_async=False, + memory_service_config=memory_config.model_copy(deep=True), + ) + try: + await sqlite_session._sql_storage.create_sql_engine() + await sqlite_memory._sql_storage.create_sql_engine() + except ValueError as exc: + if isinstance(exc.__cause__, ImportError): + pytest.skip(f"SQLite replay backend dependency is unavailable: {exc}") + pytest.fail(f"SQLite replay backend failed to initialize: {exc}") + + backends.append( + BackendBundle( + name="sqlite", + session_service=sqlite_session, + memory_service=sqlite_memory, + close=lambda s=sqlite_session, m=sqlite_memory: _close_services(s, m), + ) + ) + + redis_url = os.environ.get("TRPC_AGENT_REPLAY_REDIS_URL") + if redis_url: + try: + from trpc_agent_sdk.memory import RedisMemoryService + from trpc_agent_sdk.sessions import RedisSessionService + except ImportError as exc: + pytest.skip(f"Redis replay backend dependency is unavailable: {exc}") + + redis_session = RedisSessionService( + db_url=redis_url, + is_async=False, + session_config=base_config.model_copy(deep=True), + ) + redis_session.set_summarizer_manager(_make_summarizer_manager(keep_recent_count), force=True) + redis_memory = RedisMemoryService( + db_url=redis_url, + is_async=False, + memory_service_config=memory_config.model_copy(deep=True), + ) + backends.append( + BackendBundle( + name="redis", + session_service=redis_session, + memory_service=redis_memory, + close=lambda s=redis_session, m=redis_memory: _close_services(s, m), + ) + ) + + return backends diff --git a/tests/sessions/replay_consistency/cases.py b/tests/sessions/replay_consistency/cases.py new file mode 100644 index 00000000..114a919a --- /dev/null +++ b/tests/sessions/replay_consistency/cases.py @@ -0,0 +1,655 @@ +"""Deterministic replay cases for session/memory/summary backends.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class EventSpec: + event_id: str + invocation_id: str + author: str + role: str + text: str | None + function_call: dict | None + function_response: dict | None + state_delta: dict[str, Any] | None + branch: str | None + tag: str | None + filter_key: str | None + partial: bool = False + + +@dataclass(frozen=True) +class MemoryQuerySpec: + key: str | None + query: str + limit: int + expected_text_fragments: list[str] + + +@dataclass(frozen=True) +class ReplayCase: + name: str + app_name: str + user_id: str + session_id: str + initial_state: dict[str, Any] + events: list[EventSpec] + memory_queries: list[MemoryQuerySpec] + summary_points: list[int] + description: str + + +def _text_event( + case_name: str, + index: int, + *, + invocation_id: str, + author: str, + role: str, + text: str, + state_delta: dict[str, Any] | None = None, + branch: str | None = None, + tag: str | None = None, + filter_key: str | None = None, +) -> EventSpec: + return EventSpec( + event_id=f"{case_name}-event-{index:02d}", + invocation_id=invocation_id, + author=author, + role=role, + text=text, + function_call=None, + function_response=None, + state_delta=state_delta, + branch=branch, + tag=tag, + filter_key=filter_key, + ) + + +def replay_cases() -> list[ReplayCase]: + """Return the public replay corpus. Keep names/order stable.""" + return [ + ReplayCase( + name="single_turn_text", + app_name="replay-app", + user_id="user-single", + session_id="session-001", + initial_state={}, + events=[ + _text_event( + "single_turn_text", + 0, + invocation_id="inv-single-1", + author="user", + role="user", + text="Hello replay bot.", + ), + _text_event( + "single_turn_text", + 1, + invocation_id="inv-single-1", + author="assistant", + role="model", + text="Hello! Replay is deterministic.", + ), + ], + memory_queries=[ + MemoryQuerySpec( + key=None, + query="Hello", + limit=10, + expected_text_fragments=["Hello replay bot.", "Replay is deterministic."], + ) + ], + summary_points=[], + description="Two text events preserve order, author, role, and text.", + ), + ReplayCase( + name="multi_turn_append_order", + app_name="replay-app", + user_id="user-order", + session_id="session-002", + initial_state={}, + events=[ + _text_event( + "multi_turn_append_order", + 0, + invocation_id="inv-order-1", + author="user", + role="user", + text="First question about replay order.", + ), + _text_event( + "multi_turn_append_order", + 1, + invocation_id="inv-order-1", + author="assistant", + role="model", + text="First answer keeps the same invocation.", + ), + _text_event( + "multi_turn_append_order", + 2, + invocation_id="inv-order-2", + author="user", + role="user", + text="Second question checks append order.", + ), + _text_event( + "multi_turn_append_order", + 3, + invocation_id="inv-order-2", + author="assistant", + role="model", + text="Second answer follows the second question.", + ), + _text_event( + "multi_turn_append_order", + 4, + invocation_id="inv-order-3", + author="user", + role="user", + text="Third question closes the order test.", + ), + _text_event( + "multi_turn_append_order", + 5, + invocation_id="inv-order-3", + author="assistant", + role="model", + text="Third answer is last in the replay.", + ), + ], + memory_queries=[], + summary_points=[], + description="Three user/assistant turns verify stable append ordering.", + ), + ReplayCase( + name="tool_call_roundtrip", + app_name="replay-app", + user_id="user-tool", + session_id="session-003", + initial_state={}, + events=[ + _text_event( + "tool_call_roundtrip", + 0, + invocation_id="inv-tool-1", + author="user", + role="user", + text="What is the weather in Beijing?", + ), + EventSpec( + event_id="tool_call_roundtrip-event-01", + invocation_id="inv-tool-1", + author="assistant", + role="model", + text=None, + function_call={ + "id": "call-weather-1", + "name": "get_weather", + "args": { + "city": "Beijing", + "unit": "celsius", + }, + }, + function_response=None, + state_delta=None, + branch="weather.main", + tag="tool-call", + filter_key="weather", + ), + EventSpec( + event_id="tool_call_roundtrip-event-02", + invocation_id="inv-tool-1", + author="tool", + role="tool", + text=None, + function_call=None, + function_response={ + "id": "call-weather-1", + "name": "get_weather", + "response": { + "temperature": 25, + "condition": "sunny", + }, + }, + state_delta=None, + branch="weather.main", + tag="tool-response", + filter_key="weather", + ), + _text_event( + "tool_call_roundtrip", + 3, + invocation_id="inv-tool-1", + author="assistant", + role="model", + text="Beijing is sunny and 25 celsius.", + branch="weather.main", + tag="final", + filter_key="weather", + ), + ], + memory_queries=[ + MemoryQuerySpec( + key=None, + query="sunny Beijing", + limit=10, + expected_text_fragments=["Beijing is sunny and 25 celsius."], + ) + ], + summary_points=[], + description="Tool call and function response payloads survive normalization.", + ), + ReplayCase( + name="scoped_state_overwrite", + app_name="replay-app", + user_id="user-state", + session_id="session-004", + initial_state={ + "app:region": "global", + "user:tier": "bronze", + "counter": 0, + }, + events=[ + _text_event( + "scoped_state_overwrite", + 0, + invocation_id="inv-state-1", + author="user", + role="user", + text="Start scoped state test.", + state_delta={ + "counter": 1, + "temp:trace_id": "trace-1", + }, + ), + _text_event( + "scoped_state_overwrite", + 1, + invocation_id="inv-state-1", + author="assistant", + role="model", + text="Region set to north and tier set to silver.", + state_delta={ + "app:region": "north", + "user:tier": "silver", + "counter": 2, + }, + ), + _text_event( + "scoped_state_overwrite", + 2, + invocation_id="inv-state-2", + author="assistant", + role="model", + text="Preference saved and tier promoted to gold.", + state_delta={ + "preference": "quiet", + "user:tier": "gold", + "temp:trace_id": "trace-2", + }, + ), + ], + memory_queries=[], + summary_points=[], + description="App/user/session state overwrites are merged; temp state is not persisted.", + ), + ReplayCase( + name="memory_preference_search", + app_name="replay-app", + user_id="user-memory", + session_id="session-005", + initial_state={}, + events=[ + _text_event( + "memory_preference_search", + 0, + invocation_id="inv-memory-1", + author="user", + role="user", + text="I prefer tea in the morning.", + ), + _text_event( + "memory_preference_search", + 1, + invocation_id="inv-memory-1", + author="assistant", + role="model", + text="I will remember that you prefer tea.", + ), + _text_event( + "memory_preference_search", + 2, + invocation_id="inv-memory-2", + author="user", + role="user", + text="I enjoy hiking on weekends.", + ), + _text_event( + "memory_preference_search", + 3, + invocation_id="inv-memory-2", + author="assistant", + role="model", + text="Hiking preference noted for weekends.", + ), + _text_event( + "memory_preference_search", + 4, + invocation_id="inv-memory-3", + author="user", + role="user", + text="Please suggest vegetarian food.", + ), + _text_event( + "memory_preference_search", + 5, + invocation_id="inv-memory-3", + author="assistant", + role="model", + text="Vegetarian food options will be prioritized.", + ), + ], + memory_queries=[ + MemoryQuerySpec(key=None, query="tea", limit=10, expected_text_fragments=["prefer tea"]), + MemoryQuerySpec(key=None, query="hiking", limit=10, expected_text_fragments=["hiking on weekends"]), + MemoryQuerySpec( + key=None, + query="vegetarian", + limit=10, + expected_text_fragments=["vegetarian food"], + ), + ], + summary_points=[], + description="Memory search returns deterministic preference text for the session save key.", + ), + ReplayCase( + name="memory_multi_session_isolation", + app_name="replay-app", + user_id="user-isolation-a", + session_id="session-006-a", + initial_state={}, + events=[ + _text_event( + "memory_multi_session_isolation", + 0, + invocation_id="inv-isolation-a-1", + author="user", + role="user", + text="User A likes jasmine tea and hiking near lakes.", + ), + _text_event( + "memory_multi_session_isolation", + 1, + invocation_id="inv-isolation-a-1", + author="assistant", + role="model", + text="I will remember jasmine tea and lake hiking for User A.", + ), + ], + memory_queries=[ + MemoryQuerySpec( + key=None, + query="jasmine hiking", + limit=10, + expected_text_fragments=["jasmine tea", "lake hiking"], + ) + ], + summary_points=[], + description="A second user's stored memory must not leak into user A search results.", + ), + ReplayCase( + name="summary_generation", + app_name="replay-app", + user_id="user-summary", + session_id="session-007", + initial_state={}, + events=[ + _text_event( + "summary_generation", + 0, + invocation_id="inv-summary-1", + author="user", + role="user", + text="Help me plan a trip to Shanghai.", + ), + _text_event( + "summary_generation", + 1, + invocation_id="inv-summary-1", + author="assistant", + role="model", + text="Sure! When would you like to go?", + ), + _text_event( + "summary_generation", + 2, + invocation_id="inv-summary-2", + author="user", + role="user", + text="I want museums, vegetarian food, and tea houses.", + ), + _text_event( + "summary_generation", + 3, + invocation_id="inv-summary-2", + author="assistant", + role="model", + text="I will include museums, vegetarian restaurants, and tea houses.", + ), + _text_event( + "summary_generation", + 4, + invocation_id="inv-summary-3", + author="user", + role="user", + text="Budget is mid range and I prefer metro travel.", + ), + _text_event( + "summary_generation", + 5, + invocation_id="inv-summary-3", + author="assistant", + role="model", + text="I will keep it mid range with metro routes.", + ), + _text_event( + "summary_generation", + 6, + invocation_id="inv-summary-4", + author="user", + role="user", + text="Make it a three day plan with one quiet evening.", + ), + _text_event( + "summary_generation", + 7, + invocation_id="inv-summary-4", + author="assistant", + role="model", + text="I will create a three day Shanghai plan with a quiet evening.", + ), + ], + memory_queries=[], + summary_points=[7], + description="Manual summary creation yields summary text, event flag, and manager metadata.", + ), + ReplayCase( + name="summary_update_overwrite", + app_name="replay-app", + user_id="user-summary-update", + session_id="session-008", + initial_state={}, + events=[ + _text_event( + "summary_update_overwrite", + 0, + invocation_id="inv-summary-update-1", + author="user", + role="user", + text="Plan a product launch for replay.", + ), + _text_event( + "summary_update_overwrite", + 1, + invocation_id="inv-summary-update-1", + author="assistant", + role="model", + text="We need goals, audience, and timing.", + ), + _text_event( + "summary_update_overwrite", + 2, + invocation_id="inv-summary-update-2", + author="user", + role="user", + text="Audience is developers and operators.", + ), + _text_event( + "summary_update_overwrite", + 3, + invocation_id="inv-summary-update-2", + author="assistant", + role="model", + text="I will target developers and operators.", + ), + _text_event( + "summary_update_overwrite", + 4, + invocation_id="inv-summary-update-3", + author="user", + role="user", + text="Add a release checklist and owner names.", + ), + _text_event( + "summary_update_overwrite", + 5, + invocation_id="inv-summary-update-3", + author="assistant", + role="model", + text="Checklist and owners are now included.", + ), + ], + memory_queries=[], + summary_points=[3, 5], + description="A later summary overwrites the cached summary for the same session.", + ), + ReplayCase( + name="summary_with_event_truncation", + app_name="replay-app", + user_id="user-summary-truncation", + session_id="session-009", + initial_state={}, + events=[ + _text_event( + "summary_with_event_truncation", + 0, + invocation_id="inv-summary-truncation-1", + author="user", + role="user", + text="Help me plan a trip to Shanghai.", + ), + _text_event( + "summary_with_event_truncation", + 1, + invocation_id="inv-summary-truncation-1", + author="assistant", + role="model", + text="Sure! When would you like to go?", + ), + _text_event( + "summary_with_event_truncation", + 2, + invocation_id="inv-summary-truncation-2", + author="user", + role="user", + text="I prefer spring with museums.", + ), + _text_event( + "summary_with_event_truncation", + 3, + invocation_id="inv-summary-truncation-2", + author="assistant", + role="model", + text="Spring museums are good for Shanghai.", + ), + _text_event( + "summary_with_event_truncation", + 4, + invocation_id="inv-summary-truncation-3", + author="user", + role="user", + text="Keep the last metro rides simple.", + ), + _text_event( + "summary_with_event_truncation", + 5, + invocation_id="inv-summary-truncation-3", + author="assistant", + role="model", + text="I will keep recent metro details active.", + ), + _text_event( + "summary_with_event_truncation", + 6, + invocation_id="inv-summary-truncation-4", + author="user", + role="user", + text="Also add a ferry ride after the summary.", + ), + ], + memory_queries=[], + summary_points=[5], + description="Summary compression keeps historical events and then appends a new event.", + ), + ReplayCase( + name="duplicate_or_error_recovery", + app_name="replay-app", + user_id="user-duplicate", + session_id="session-010", + initial_state={}, + events=[ + _text_event( + "duplicate_or_error_recovery", + 0, + invocation_id="inv-duplicate-1", + author="user", + role="user", + text="Duplicate content check begins.", + ), + _text_event( + "duplicate_or_error_recovery", + 1, + invocation_id="inv-duplicate-1", + author="assistant", + role="model", + text="Same content may repeat.", + ), + _text_event( + "duplicate_or_error_recovery", + 2, + invocation_id="inv-duplicate-2", + author="assistant", + role="model", + text="Same content may repeat.", + ), + ], + memory_queries=[ + MemoryQuerySpec( + key=None, + query="Same repeat", + limit=10, + expected_text_fragments=["Same content may repeat."], + ) + ], + summary_points=[], + description="Duplicate content with distinct ids is captured as the backend stores it.", + ), + ] diff --git a/tests/sessions/replay_consistency/comparator.py b/tests/sessions/replay_consistency/comparator.py new file mode 100644 index 00000000..53bee72b --- /dev/null +++ b/tests/sessions/replay_consistency/comparator.py @@ -0,0 +1,170 @@ +"""Recursive snapshot comparator with structured diff entries.""" + +from __future__ import annotations + +from dataclasses import asdict +from dataclasses import dataclass +import re +from typing import Any + +from .normalizer import Snapshot + + +@dataclass +class DiffEntry: + case_name: str + left_backend: str + right_backend: str + session_id: str | None + event_index: int | None + memory_index: int | None + summary_id: str | None + section: str + path: str + left: Any + right: Any + allowed: bool + reason: str + + +_MISSING = object() + + +def _to_dict(value: Any) -> Any: + if isinstance(value, Snapshot): + return asdict(value) + if hasattr(value, "__dataclass_fields__"): + return asdict(value) + return value + + +def _infer_section(path: str) -> str: + if not path: + return "root" + root = path.split(".", 1)[0].split("[", 1)[0] + if root in {"events", "state", "memories", "summary", "historical_events", "list_sessions"}: + return root + return root + + +def _parse_index(path: str, section: str) -> int | None: + match = re.search(rf"{section}\[(\d+)\]", path) + return int(match.group(1)) if match else None + + +def _summary_id(left: dict[str, Any], right: dict[str, Any], section: str) -> str | None: + if section != "summary": + return None + for snapshot in (left, right): + summary = snapshot.get("summary") + if isinstance(summary, dict): + metadata = summary.get("metadata") or {} + session_id = metadata.get("session_id") + if session_id: + return f"summary:{session_id}:latest" + session_id = left.get("session_id") or right.get("session_id") + return f"summary:{session_id}:latest" if session_id else None + + +def _allowed_diff(path: str, left: Any, right: Any) -> tuple[bool, str]: + if path == "backend": + return True, "backend name differs by design" + if path.endswith(".timestamp") or path == "timestamp": + return True, "raw timestamps are backend generated" + if path.endswith(".has_timestamp") and left is True and right is True: + return True, "timestamp presence is normalized" + return False, "" + + +def _display(value: Any) -> Any: + if value is _MISSING: + return "" + return value + + +def _entry(path: str, left_value: Any, right_value: Any, left: dict[str, Any], right: dict[str, Any]) -> DiffEntry: + section = _infer_section(path) + allowed, reason = _allowed_diff(path, left_value, right_value) + event_index = _parse_index(path, "events") + if event_index is None: + event_index = _parse_index(path, "historical_events") + return DiffEntry( + case_name=left.get("case_name") or right.get("case_name") or "", + left_backend=left.get("backend") or "", + right_backend=right.get("backend") or "", + session_id=left.get("session_id") or right.get("session_id"), + event_index=event_index, + memory_index=_parse_index(path, "memories"), + summary_id=_summary_id(left, right, section), + section=section, + path=path, + left=_display(left_value), + right=_display(right_value), + allowed=allowed, + reason=reason, + ) + + +def _join_path(parent: str, key: str) -> str: + return f"{parent}.{key}" if parent else key + + +def _diff_values(current_left: Any, + current_right: Any, + root_left: dict[str, Any], + root_right: dict[str, Any], + path: str) -> list[DiffEntry]: + if isinstance(current_left, dict) and isinstance(current_right, dict): + diffs: list[DiffEntry] = [] + for key in sorted(set(current_left) | set(current_right)): + next_left = current_left.get(key, _MISSING) + next_right = current_right.get(key, _MISSING) + next_path = _join_path(path, str(key)) + if next_left is _MISSING or next_right is _MISSING: + diffs.append(_entry(next_path, next_left, next_right, root_left, root_right)) + else: + diffs.extend(_diff_values(next_left, next_right, root_left, root_right, next_path)) + return diffs + + if isinstance(current_left, list) and isinstance(current_right, list): + diffs = [] + max_len = max(len(current_left), len(current_right)) + for index in range(max_len): + next_path = f"{path}[{index}]" + next_left = current_left[index] if index < len(current_left) else _MISSING + next_right = current_right[index] if index < len(current_right) else _MISSING + if next_left is _MISSING or next_right is _MISSING: + diffs.append(_entry(next_path, next_left, next_right, root_left, root_right)) + else: + diffs.extend(_diff_values(next_left, next_right, root_left, root_right, next_path)) + return diffs + + if current_left != current_right: + return [_entry(path, current_left, current_right, root_left, root_right)] + return [] + + +def recursive_diff(left: Any, right: Any, context: dict[str, Any] | None = None) -> list[DiffEntry]: + left_dict = _to_dict(left) + right_dict = _to_dict(right) + if not isinstance(left_dict, dict) or not isinstance(right_dict, dict): + raise TypeError("recursive_diff expects Snapshot, dataclass, or dict inputs") + + diffs = _diff_values(left_dict, right_dict, left_dict, right_dict, "") + if context: + for diff in diffs: + if context.get("case_name"): + diff.case_name = context["case_name"] + if context.get("left_backend"): + diff.left_backend = context["left_backend"] + if context.get("right_backend"): + diff.right_backend = context["right_backend"] + return diffs + + +def compare_snapshot_pair(left: Snapshot, right: Snapshot) -> list[DiffEntry]: + return recursive_diff(left, right) + + +def unallowed_diffs(diffs: list[DiffEntry]) -> list[DiffEntry]: + return [diff for diff in diffs if not diff.allowed] diff --git a/tests/sessions/replay_consistency/normalizer.py b/tests/sessions/replay_consistency/normalizer.py new file mode 100644 index 00000000..eb9028f0 --- /dev/null +++ b/tests/sessions/replay_consistency/normalizer.py @@ -0,0 +1,202 @@ +"""Snapshot normalization for replay consistency tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +import re +from typing import Any + +from trpc_agent_sdk.events import Event +from trpc_agent_sdk.sessions import Session +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import MemoryEntry +from trpc_agent_sdk.types import State + +from .cases import MemoryQuerySpec +from .cases import ReplayCase + + +@dataclass +class Snapshot: + backend: str + case_name: str + session_id: str + app_name: str + user_id: str + state: dict[str, Any] + events: list[dict[str, Any]] + historical_events: list[dict[str, Any]] + memories: list[dict[str, Any]] + summary: dict[str, Any] | None + list_sessions: list[dict[str, Any]] + + +def canonicalize(value: Any) -> Any: + if isinstance(value, dict): + return {key: canonicalize(value[key]) for key in sorted(value)} + if isinstance(value, list): + return [canonicalize(item) for item in value] + if hasattr(value, "model_dump"): + return canonicalize(value.model_dump(exclude_none=True, mode="json")) + return value + + +def normalize_summary_text(text: str | None) -> str | None: + if text is None: + return None + return re.sub(r"\s+", " ", text).strip() + + +def _strip_temp_state(state: dict[str, Any] | None) -> dict[str, Any]: + if not state: + return {} + return canonicalize({key: value for key, value in state.items() if not key.startswith(State.TEMP_PREFIX)}) + + +def _content_text(content: Content | None) -> str | None: + if not content or not content.parts: + return None + text = "".join(part.text for part in content.parts if part.text) + return text or None + + +def normalize_event(event: Event, index: int) -> dict[str, Any]: + function_calls = [ + { + "id": call.id, + "name": call.name, + "args": canonicalize(call.args or {}), + } + for call in event.get_function_calls() + ] + function_responses = [ + { + "id": response.id, + "name": response.name, + "response": canonicalize(response.response or {}), + } + for response in event.get_function_responses() + ] + role = event.content.role if event.content else None + state_delta = _strip_temp_state(event.actions.state_delta if event.actions else None) + + return { + "stable_index": index, + "event_id": "normalized", + "invocation_id": event.invocation_id, + "author": event.author, + "role": role, + "text": normalize_summary_text(event.get_text()) if event.get_text() else None, + "function_calls": function_calls, + "function_responses": function_responses, + "state_delta": state_delta, + "branch": event.branch, + "tag": event.tag, + "filter_key": event.filter_key, + "partial": bool(event.partial), + "turn_complete": bool(event.turn_complete), + "error_code": event.error_code, + "error_message": event.error_message, + "model_visible": event.is_model_visible(), + "is_summary_event": event.is_summary_event(), + } + + +def normalize_memory_entry(query_spec: MemoryQuerySpec, key: str, memory: MemoryEntry) -> dict[str, Any]: + return { + "query": query_spec.query, + "key": key, + "author": memory.author, + "text": normalize_summary_text(_content_text(memory.content)), + "has_timestamp": bool(memory.timestamp), + } + + +def normalize_memory_results(memory_records: list[tuple[MemoryQuerySpec, str, list[MemoryEntry]]]) -> list[dict[str, + Any]]: + memories: list[dict[str, Any]] = [] + for query_spec, key, entries in memory_records: + memories.extend(normalize_memory_entry(query_spec, key, entry) for entry in entries) + return sorted( + memories, + key=lambda item: ( + item.get("query") or "", + item.get("author") or "", + item.get("text") or "", + item.get("key") or "", + ), + ) + + +def _normalize_list_session(session: Session) -> dict[str, Any]: + return { + "id": session.id, + "app_name": session.app_name, + "user_id": session.user_id, + "state": _strip_temp_state(session.state), + } + + +async def _normalize_summary(session_service: Any, session: Session) -> dict[str, Any] | None: + summary_text = await session_service.get_session_summary(session) + if summary_text is None: + return None + + summary_events = [event for event in session.events if event.is_summary_event()] + summary_event_text = summary_events[-1].get_text() if summary_events else None + + metadata: dict[str, Any] = { + "session_id": session.id, + "has_summary": True, + "summary_event_count": len(summary_events), + "summary_event_text": normalize_summary_text(summary_event_text), + "compressed_event_count": len(session.events), + "historical_event_count": len(session.historical_events), + } + + manager = getattr(session_service, "summarizer_manager", None) + if manager is not None: + manager_summary = await manager.get_session_summary(session) + if manager_summary is not None: + metadata.update( + { + "manager_session_id": manager_summary.session_id, + "original_event_count": manager_summary.original_event_count, + "manager_compressed_event_count": manager_summary.compressed_event_count, + "has_summary_timestamp": bool(manager_summary.summary_timestamp), + } + ) + + return { + "text": normalize_summary_text(summary_text), + "metadata": canonicalize(metadata), + } + + +async def normalize_snapshot( + *, + backend: str, + case: ReplayCase, + session: Session, + session_service: Any, + memory_records: list[tuple[MemoryQuerySpec, str, list[MemoryEntry]]], +) -> Snapshot: + list_response = await session_service.list_sessions(app_name=case.app_name, user_id=case.user_id) + list_sessions = sorted( + (_normalize_list_session(listed_session) for listed_session in list_response.sessions), + key=lambda item: item["id"], + ) + + return Snapshot( + backend=backend, + case_name=case.name, + session_id=session.id, + app_name=session.app_name, + user_id=session.user_id, + state=_strip_temp_state(session.state), + events=[normalize_event(event, index) for index, event in enumerate(session.events)], + historical_events=[normalize_event(event, index) for index, event in enumerate(session.historical_events)], + memories=normalize_memory_results(memory_records), + summary=await _normalize_summary(session_service, session), + list_sessions=list_sessions, + ) diff --git a/tests/sessions/replay_consistency/report.py b/tests/sessions/replay_consistency/report.py new file mode 100644 index 00000000..e7b0e746 --- /dev/null +++ b/tests/sessions/replay_consistency/report.py @@ -0,0 +1,54 @@ +"""JSON report writer for replay consistency comparisons.""" + +from __future__ import annotations + +from dataclasses import asdict +import json +from pathlib import Path +from typing import Any + +from .comparator import DiffEntry + + +def _diff_to_dict(diff: DiffEntry) -> dict[str, Any]: + return asdict(diff) + + +def write_report(path: Path, comparison_results: list[dict[str, Any]]) -> dict[str, Any]: + backend_pairs = [] + case_diff_counts: dict[str, int] = {} + diffs: list[dict[str, Any]] = [] + + for result in comparison_results: + left_backend = result["left_backend"] + right_backend = result["right_backend"] + pair_name = f"{left_backend}_vs_{right_backend}" + if pair_name not in backend_pairs: + backend_pairs.append(pair_name) + + result_diffs: list[DiffEntry] = result.get("diffs", []) + unallowed_count = sum(1 for diff in result_diffs if not diff.allowed) + case_name = result["case_name"] + case_diff_counts[case_name] = case_diff_counts.get(case_name, 0) + unallowed_count + diffs.extend(_diff_to_dict(diff) for diff in result_diffs if not diff.allowed) + + cases = [ + { + "name": case_name, + "unallowed_diff_count": unallowed_count, + } + for case_name, unallowed_count in case_diff_counts.items() + ] + + report = { + "schema_version": 1, + "generated_by": "tests/sessions/test_replay_consistency.py", + "backend_pairs": backend_pairs, + "case_count": len(cases), + "cases": cases, + "diffs": diffs, + } + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") + return report diff --git a/tests/sessions/test_replay_consistency.py b/tests/sessions/test_replay_consistency.py new file mode 100644 index 00000000..b66573c0 --- /dev/null +++ b/tests/sessions/test_replay_consistency.py @@ -0,0 +1,591 @@ +"""Replay consistency tests for Session / Memory / Summary backends.""" + +from __future__ import annotations + +from dataclasses import asdict +import copy +import json +from pathlib import Path +from typing import Any + +import pytest + +from trpc_agent_sdk.events import Event +from trpc_agent_sdk.sessions import Session +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import EventActions +from trpc_agent_sdk.types import FunctionCall +from trpc_agent_sdk.types import FunctionResponse +from trpc_agent_sdk.types import MemoryEntry +from trpc_agent_sdk.types import Part + +from .replay_consistency.backends import BackendBundle +from .replay_consistency.backends import build_backends +from .replay_consistency.backends import make_session_config +from .replay_consistency.cases import EventSpec +from .replay_consistency.cases import MemoryQuerySpec +from .replay_consistency.cases import ReplayCase +from .replay_consistency.cases import replay_cases +from .replay_consistency.comparator import DiffEntry +from .replay_consistency.comparator import compare_snapshot_pair +from .replay_consistency.comparator import recursive_diff +from .replay_consistency.comparator import unallowed_diffs +from .replay_consistency.normalizer import Snapshot +from .replay_consistency.normalizer import normalize_snapshot +from .replay_consistency.report import write_report + + +REQUIRED_CASE_NAMES = [ + "single_turn_text", + "multi_turn_append_order", + "tool_call_roundtrip", + "scoped_state_overwrite", + "memory_preference_search", + "memory_multi_session_isolation", + "summary_generation", + "summary_update_overwrite", + "summary_with_event_truncation", + "duplicate_or_error_recovery", +] + +FIXED_EVENT_TIMESTAMP_BASE = 2_000_000_000.0 + + +def _event_from_spec(spec: EventSpec, index: int) -> Event: + parts: list[Part] = [] + if spec.text is not None: + parts.append(Part.from_text(text=spec.text)) + if spec.function_call is not None: + parts.append( + Part( + function_call=FunctionCall( + id=spec.function_call.get("id"), + name=spec.function_call["name"], + args=spec.function_call.get("args") or {}, + ) + ) + ) + if spec.function_response is not None: + parts.append( + Part( + function_response=FunctionResponse( + id=spec.function_response.get("id"), + name=spec.function_response["name"], + response=spec.function_response.get("response") or {}, + ) + ) + ) + + return Event( + id=spec.event_id, + invocation_id=spec.invocation_id, + author=spec.author, + content=Content(role=spec.role, parts=parts), + actions=EventActions(state_delta=copy.deepcopy(spec.state_delta or {})), + branch=spec.branch, + tag=spec.tag, + filter_key=spec.filter_key, + partial=spec.partial, + timestamp=FIXED_EVENT_TIMESTAMP_BASE + index, + ) + + +async def _get_required_session(bundle: BackendBundle, case: ReplayCase) -> Session: + stored = await bundle.session_service.get_session( + app_name=case.app_name, + user_id=case.user_id, + session_id=case.session_id, + ) + if stored is None: + pytest.fail(f"{bundle.name} did not return stored session {case.session_id}") + return stored + + +async def _create_required_summary(bundle: BackendBundle, session: Session, case: ReplayCase) -> str: + await bundle.session_service.create_session_summary(session) + summary_text = await bundle.session_service.get_session_summary(session) + if summary_text is None: + pytest.fail(f"{bundle.name} did not create a summary for {case.name}/{session.id}") + if not summary_text.strip(): + pytest.fail(f"{bundle.name} created an empty summary for {case.name}/{session.id}") + return summary_text + + +async def _search_memory_records( + bundle: BackendBundle, + session: Session, + memory_queries: list[MemoryQuerySpec], +) -> list[tuple[MemoryQuerySpec, str, list[MemoryEntry]]]: + records: list[tuple[MemoryQuerySpec, str, list[MemoryEntry]]] = [] + for query_spec in memory_queries: + key = query_spec.key or session.save_key + response = await bundle.memory_service.search_memory(key=key, query=query_spec.query, limit=query_spec.limit) + memory_texts = "\n".join( + "".join(part.text for part in memory.content.parts if part.text) + for memory in response.memories + if memory.content and memory.content.parts + ) + for expected in query_spec.expected_text_fragments: + if expected not in memory_texts: + pytest.fail( + f"{bundle.name} memory query {query_spec.query!r} for {session.id} " + f"did not return expected fragment {expected!r}; got {memory_texts!r}" + ) + records.append((query_spec, key, response.memories)) + return records + + +def _assert_summary_snapshot(snapshot: Snapshot, case: ReplayCase) -> None: + if not case.summary_points: + return + if snapshot.summary is None: + pytest.fail(f"{snapshot.backend} summary snapshot is missing for {case.name}") + + summary = snapshot.summary + metadata = summary["metadata"] + if not summary["text"]: + pytest.fail(f"{snapshot.backend} summary text is empty for {case.name}") + if metadata["session_id"] != snapshot.session_id: + pytest.fail(f"{snapshot.backend} summary session mismatch for {case.name}") + if metadata["summary_event_count"] < 1: + pytest.fail(f"{snapshot.backend} summary event missing for {case.name}") + event_text = metadata["summary_event_text"] + if not event_text or not event_text.startswith("Previous conversation summary:"): + pytest.fail(f"{snapshot.backend} summary event text missing prefix for {case.name}") + + summary_events = [event for event in snapshot.events if event["is_summary_event"]] + if not summary_events: + pytest.fail(f"{snapshot.backend} summary event flag missing for {case.name}") + if summary_events[0]["author"] != "system": + pytest.fail(f"{snapshot.backend} summary event author is not system for {case.name}") + + if case.name == "summary_with_event_truncation": + if snapshot.events[0]["is_summary_event"] is not True: + pytest.fail(f"{snapshot.backend} truncation case did not keep summary event first") + if metadata["historical_event_count"] == 0 or not snapshot.historical_events: + pytest.fail(f"{snapshot.backend} truncation case did not persist historical events") + if snapshot.events[-1]["text"] != "Also add a ferry ride after the summary.": + pytest.fail(f"{snapshot.backend} truncation case lost post-summary append") + + +async def _run_standard_case(bundle: BackendBundle, case: ReplayCase) -> Snapshot: + summary_texts: list[str] = [] + session = await bundle.session_service.create_session( + app_name=case.app_name, + user_id=case.user_id, + session_id=case.session_id, + state=copy.deepcopy(case.initial_state), + ) + for index, spec in enumerate(case.events): + event = _event_from_spec(spec, index) + await bundle.session_service.append_event(session, event) + if index in case.summary_points: + summary_texts.append(await _create_required_summary(bundle, session, case)) + session = await _get_required_session(bundle, case) + + stored_session = await _get_required_session(bundle, case) + if case.name == "summary_update_overwrite" and len(summary_texts) >= 2 and summary_texts[0] == summary_texts[-1]: + pytest.fail(f"{bundle.name} did not overwrite the cached summary for {case.name}") + + await bundle.memory_service.store_session(stored_session) + memory_records = await _search_memory_records(bundle, stored_session, case.memory_queries) + snapshot = await normalize_snapshot( + backend=bundle.name, + case=case, + session=stored_session, + session_service=bundle.session_service, + memory_records=memory_records, + ) + _assert_summary_snapshot(snapshot, case) + return snapshot + + +async def _run_memory_isolation_case(bundle: BackendBundle, case: ReplayCase) -> Snapshot: + session_a = await bundle.session_service.create_session( + app_name=case.app_name, + user_id=case.user_id, + session_id=case.session_id, + state=copy.deepcopy(case.initial_state), + ) + for index, spec in enumerate(case.events): + await bundle.session_service.append_event(session_a, _event_from_spec(spec, index)) + + session_b = await bundle.session_service.create_session( + app_name=case.app_name, + user_id="user-isolation-b", + session_id="session-006-b", + state={}, + ) + other_specs = [ + EventSpec( + event_id="memory_multi_session_isolation-b-event-00", + invocation_id="inv-isolation-b-1", + author="user", + role="user", + text="User B likes coffee and city museums.", + function_call=None, + function_response=None, + state_delta=None, + branch=None, + tag=None, + filter_key=None, + ), + EventSpec( + event_id="memory_multi_session_isolation-b-event-01", + invocation_id="inv-isolation-b-1", + author="assistant", + role="model", + text="I will remember coffee and city museums for User B.", + function_call=None, + function_response=None, + state_delta=None, + branch=None, + tag=None, + filter_key=None, + ), + ] + for index, spec in enumerate(other_specs, start=100): + await bundle.session_service.append_event(session_b, _event_from_spec(spec, index)) + + stored_a = await _get_required_session(bundle, case) + stored_b = await bundle.session_service.get_session( + app_name=case.app_name, + user_id="user-isolation-b", + session_id="session-006-b", + ) + if stored_b is None: + pytest.fail(f"{bundle.name} did not return isolation control session") + + await bundle.memory_service.store_session(stored_a) + await bundle.memory_service.store_session(stored_b) + memory_records = await _search_memory_records(bundle, stored_a, case.memory_queries) + leaked_text = "\n".join( + "".join(part.text for part in memory.content.parts if part.text) + for _, _, memories in memory_records + for memory in memories + if memory.content and memory.content.parts + ) + if "coffee" in leaked_text or "city museums" in leaked_text: + pytest.fail(f"{bundle.name} leaked user B memory into user A search: {leaked_text!r}") + + return await normalize_snapshot( + backend=bundle.name, + case=case, + session=stored_a, + session_service=bundle.session_service, + memory_records=memory_records, + ) + + +async def run_case(bundle: BackendBundle, case: ReplayCase) -> Snapshot: + try: + if case.name == "memory_multi_session_isolation": + return await _run_memory_isolation_case(bundle, case) + return await _run_standard_case(bundle, case) + finally: + await bundle.close() + + +def _session_config_for_case(case: ReplayCase): + return make_session_config(store_historical_events=case.name == "summary_with_event_truncation") + + +@pytest.mark.asyncio +async def test_replay_consistency_inmemory_vs_sqlite(tmp_path: Path): + cases = replay_cases() + assert len(cases) == 10 + comparison_results: list[dict[str, Any]] = [] + report_path = tmp_path / "session_memory_summary_diff_report.json" + + for case in cases: + case_tmp_path = tmp_path / case.name + backends = await build_backends(case_tmp_path, session_config=_session_config_for_case(case)) + assert {"inmemory", "sqlite"} <= {backend.name for backend in backends} + + snapshots: dict[str, Snapshot] = {} + for backend in backends: + snapshots[backend.name] = await run_case(backend, case) + + left = snapshots["inmemory"] + for right_name, right in snapshots.items(): + if right_name == "inmemory": + continue + diffs = compare_snapshot_pair(left, right) + comparison_results.append( + { + "case_name": case.name, + "left_backend": left.backend, + "right_backend": right.backend, + "diffs": diffs, + } + ) + unexpected = unallowed_diffs(diffs) + if unexpected: + write_report(report_path, comparison_results) + pytest.fail(f"Replay diff detected for {case.name}: {[asdict(diff) for diff in unexpected]}") + + write_report(report_path, comparison_results) + + +def test_replay_case_count_and_names(): + assert [case.name for case in replay_cases()] == REQUIRED_CASE_NAMES + + +def test_report_contract(tmp_path: Path): + diff = DiffEntry( + case_name="case", + left_backend="inmemory", + right_backend="sqlite", + session_id="session", + event_index=1, + memory_index=None, + summary_id="summary:session:latest", + section="events", + path="events[1].text", + left="hello", + right="bye", + allowed=False, + reason="", + ) + path = tmp_path / "report.json" + report = write_report( + path, + [ + { + "case_name": "case", + "left_backend": "inmemory", + "right_backend": "sqlite", + "diffs": [diff], + } + ], + ) + loaded = json.loads(path.read_text(encoding="utf-8")) + assert loaded == report + serialized = loaded["diffs"][0] + for field in DiffEntry.__dataclass_fields__: + assert field in serialized + assert loaded["schema_version"] == 1 + assert loaded["backend_pairs"] == ["inmemory_vs_sqlite"] + assert loaded["cases"][0]["unallowed_diff_count"] == 1 + + +def _clean_mutation_snapshot() -> dict[str, Any]: + snapshot = Snapshot( + backend="inmemory", + case_name="mutation_fixture", + session_id="session-mutation", + app_name="replay-app", + user_id="user-mutation", + state={ + "user:tier": "gold", + "preference": "tea", + }, + events=[ + { + "stable_index": 0, + "event_id": "normalized", + "invocation_id": "inv-mutation-1", + "author": "user", + "role": "user", + "text": "What is the weather in Beijing?", + "function_calls": [], + "function_responses": [], + "state_delta": {}, + "branch": None, + "tag": None, + "filter_key": None, + "partial": False, + "turn_complete": False, + "error_code": None, + "error_message": None, + "model_visible": True, + "is_summary_event": False, + }, + { + "stable_index": 1, + "event_id": "normalized", + "invocation_id": "inv-mutation-1", + "author": "assistant", + "role": "model", + "text": None, + "function_calls": [ + { + "id": "call-weather-1", + "name": "get_weather", + "args": { + "city": "Beijing", + "unit": "celsius", + }, + } + ], + "function_responses": [], + "state_delta": {}, + "branch": "weather.main", + "tag": "tool-call", + "filter_key": "weather", + "partial": False, + "turn_complete": False, + "error_code": None, + "error_message": None, + "model_visible": True, + "is_summary_event": False, + }, + { + "stable_index": 2, + "event_id": "normalized", + "invocation_id": "inv-mutation-1", + "author": "assistant", + "role": "model", + "text": "Beijing is sunny and 25 celsius.", + "function_calls": [], + "function_responses": [], + "state_delta": {}, + "branch": "weather.main", + "tag": "final", + "filter_key": "weather", + "partial": False, + "turn_complete": False, + "error_code": None, + "error_message": None, + "model_visible": True, + "is_summary_event": False, + }, + ], + historical_events=[ + { + "stable_index": 0, + "event_id": "normalized", + "invocation_id": "inv-old", + "author": "user", + "role": "user", + "text": "Old preference was captured.", + "function_calls": [], + "function_responses": [], + "state_delta": {}, + "branch": None, + "tag": None, + "filter_key": None, + "partial": False, + "turn_complete": False, + "error_code": None, + "error_message": None, + "model_visible": True, + "is_summary_event": False, + } + ], + memories=[ + { + "query": "tea", + "key": "replay-app/user-mutation", + "author": "user", + "text": "I prefer tea in the morning.", + "has_timestamp": True, + } + ], + summary={ + "text": "summary(session-mutation): user=Old preference was captured. | facts=1-events", + "metadata": { + "session_id": "session-mutation", + "has_summary": True, + "summary_event_count": 1, + "summary_event_text": ( + "Previous conversation summary: summary(session-mutation): " + "user=Old preference was captured. | facts=1-events" + ), + "compressed_event_count": 3, + "historical_event_count": 1, + "manager_session_id": "session-mutation", + "original_event_count": 4, + "manager_compressed_event_count": 3, + "has_summary_timestamp": True, + }, + }, + list_sessions=[ + { + "id": "session-mutation", + "app_name": "replay-app", + "user_id": "user-mutation", + "state": { + "user:tier": "gold", + "preference": "tea", + }, + } + ], + ) + return asdict(snapshot) + + +def _mutate_snapshot(name: str, snapshot: dict[str, Any]) -> None: + if name == "drop_event": + del snapshot["events"][1] + elif name == "reorder_event": + snapshot["events"][0], snapshot["events"][1] = snapshot["events"][1], snapshot["events"][0] + elif name == "change_tool_args": + snapshot["events"][1]["function_calls"][0]["args"]["city"] = "Shanghai" + elif name == "change_state": + snapshot["state"]["user:tier"] = "silver" + elif name == "drop_memory": + del snapshot["memories"][0] + elif name == "change_memory_text": + snapshot["memories"][0]["text"] = "I prefer coffee in the morning." + elif name == "drop_summary": + snapshot["summary"] = None + elif name == "overwrite_summary_text": + snapshot["summary"]["text"] = "summary(session-mutation): overwritten" + elif name == "wrong_summary_session": + snapshot["summary"]["metadata"]["session_id"] = "wrong-session" + elif name == "duplicate_event": + snapshot["events"].append(copy.deepcopy(snapshot["events"][0])) + else: + raise ValueError(f"Unknown mutation {name}") + + +MUTATIONS = [ + "drop_event", + "reorder_event", + "change_tool_args", + "change_state", + "drop_memory", + "change_memory_text", + "drop_summary", + "overwrite_summary_text", + "wrong_summary_session", + "duplicate_event", +] + + +@pytest.mark.parametrize("mutation", MUTATIONS) +def test_replay_mutation_detection(mutation: str): + clean = _clean_mutation_snapshot() + mutated = copy.deepcopy(clean) + mutated["backend"] = "sqlite" + _mutate_snapshot(mutation, mutated) + diffs = recursive_diff(clean, mutated) + assert unallowed_diffs(diffs), f"{mutation} was not detected" + + +def test_mutation_detection(): + for mutation in MUTATIONS: + clean = _clean_mutation_snapshot() + mutated = copy.deepcopy(clean) + mutated["backend"] = "sqlite" + _mutate_snapshot(mutation, mutated) + assert unallowed_diffs(recursive_diff(clean, mutated)), f"{mutation} was not detected" + + +@pytest.mark.parametrize( + ("mutation", "expected_path"), + [ + ("drop_summary", "summary"), + ("overwrite_summary_text", "summary.text"), + ("wrong_summary_session", "summary.metadata.session_id"), + ], +) +def test_summary_required_mutations_detected(mutation: str, expected_path: str): + clean = _clean_mutation_snapshot() + mutated = copy.deepcopy(clean) + mutated["backend"] = "sqlite" + _mutate_snapshot(mutation, mutated) + diffs = unallowed_diffs(recursive_diff(clean, mutated)) + assert diffs + assert any(diff.section == "summary" and diff.path == expected_path for diff in diffs) From 96014b78364d2fa1619ddbb88619e04f6d8c612f Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 18:50:23 +0800 Subject: [PATCH 2/9] Strengthen replay consistency harness --- session_memory_summary_diff_report.json | 20 ++- tests/sessions/replay_consistency/README.md | 41 +++-- tests/sessions/replay_consistency/backends.py | 2 +- tests/sessions/replay_consistency/cases.py | 37 ++++- .../sessions/replay_consistency/normalizer.py | 17 +- tests/sessions/replay_consistency/report.py | 27 +++- tests/sessions/test_replay_consistency.py | 146 +++++++++++++++++- 7 files changed, 254 insertions(+), 36 deletions(-) diff --git a/session_memory_summary_diff_report.json b/session_memory_summary_diff_report.json index 87b2f5b5..7522ed93 100644 --- a/session_memory_summary_diff_report.json +++ b/session_memory_summary_diff_report.json @@ -1,51 +1,65 @@ { - "schema_version": 1, - "generated_by": "tests/sessions/test_replay_consistency.py", + "allowed_diff_count": 0, + "allowed_diffs": [], "backend_pairs": [ "inmemory_vs_sqlite" ], "case_count": 10, "cases": [ { + "allowed_diff_count": 0, "name": "single_turn_text", "unallowed_diff_count": 0 }, { + "allowed_diff_count": 0, "name": "multi_turn_append_order", "unallowed_diff_count": 0 }, { + "allowed_diff_count": 0, "name": "tool_call_roundtrip", "unallowed_diff_count": 0 }, { + "allowed_diff_count": 0, "name": "scoped_state_overwrite", "unallowed_diff_count": 0 }, { + "allowed_diff_count": 0, "name": "memory_preference_search", "unallowed_diff_count": 0 }, { + "allowed_diff_count": 0, "name": "memory_multi_session_isolation", "unallowed_diff_count": 0 }, { + "allowed_diff_count": 0, "name": "summary_generation", "unallowed_diff_count": 0 }, { + "allowed_diff_count": 0, "name": "summary_update_overwrite", "unallowed_diff_count": 0 }, { + "allowed_diff_count": 0, "name": "summary_with_event_truncation", "unallowed_diff_count": 0 }, { + "allowed_diff_count": 0, "name": "duplicate_or_error_recovery", "unallowed_diff_count": 0 } ], - "diffs": [] + "diffs": [], + "generated_by": "tests/sessions/test_replay_consistency.py", + "schema_version": 1, + "unallowed_diff_count": 0, + "unallowed_diffs": [] } diff --git a/tests/sessions/replay_consistency/README.md b/tests/sessions/replay_consistency/README.md index dce5044c..90a7efc5 100644 --- a/tests/sessions/replay_consistency/README.md +++ b/tests/sessions/replay_consistency/README.md @@ -1,21 +1,30 @@ # Replay Consistency Harness -这个 harness 用同一组 deterministic replay cases 驱动多个后端。每个 case 先通过正式 -`SessionServiceABC` 创建 session，再按固定 id、timestamp、invocation_id 写入事件；最后用 -`MemoryServiceABC.store_session(session)` 写入 memory，并通过 `search_memory(key, query, limit)` -读取结果。默认矩阵真实运行 InMemory 和 SQLite；如果设置 -`TRPC_AGENT_REPLAY_REDIS_URL`，Redis 会作为可选后端加入比较。 +This harness runs the same deterministic replay cases against multiple storage +backends through the public `SessionServiceABC` and `MemoryServiceABC` +interfaces. Each case creates a session, appends fixed-id events with stable +timestamps and invocation ids, stores the resulting session through +`store_session(session)`, and queries memory with `search_memory(key, query, +limit)`. The default matrix runs real InMemory and SQLite services; Redis is +only added when `TRPC_AGENT_REPLAY_REDIS_URL` is set. -snapshot 会归一化非业务字段：事件 timestamp 不比较精确值，summary timestamp 只比较是否存在， -自动生成的 event id 统一为 normalized，dict 递归按 key 排序，memory timestamp 只保留 -`has_timestamp`。memory 返回顺序不作为语义，因此按 `(query, author, text)` 排序。 +Snapshots normalize fields that should not affect replay semantics: raw +timestamps, summary timestamp values, auto-generated event ids, dict key order, +and memory timestamp values. Fixture event ids are preserved so duplicate, +retry, and wrong-id problems remain visible. Memory rows are sorted by +`(query, author, text, key)` because backend search order can differ. -严格比较的字段包括 event 顺序、author、role、text、tool args、tool response、state、memory -content、summary text、summary session_id、summary overwrite 后的最新值、summary event flag、 -summary event 数量，以及 historical_events 数量和内容。summary 使用 -`DeterministicSessionSummarizer`，不调用真实 LLM，但保留现有 `SessionSummarizer` 的压缩逻辑， -因此 summary event 和 historical_events 仍由产品代码生成。 +The comparator strictly checks event order, roles, authors, text, tool call +args, tool responses, persisted state, memory content, summary text, summary +session id, summary overwrite behavior, summary event flags, and +historical_events. `DeterministicSessionSummarizer` avoids real LLM calls while +leaving the production summary compression path responsible for summary events +and historical event storage. SQLite uses temporary database files and explicit +SQL storage initialization; initialization failures are not silently ignored. -SQLite 后端使用独立临时 SQLite 文件并显式初始化 SQL storage；初始化失败不会静默降级。mutation -tests 会对 clean snapshot 人为制造 drop、reorder、state、memory、summary 等不一致，验证 -recursive diff 能输出定位到 session/event/memory/summary/path 的非允许差异。 +Reports include complete diffs plus `allowed_diffs` and `unallowed_diffs` +partitions, with counts per case and globally. Synthetic mutation tests and +real InMemory replay snapshot mutation tests intentionally drop, reorder, +duplicate, and alter event, state, memory, and summary fields to prove the +recursive diff can locate unallowed regressions by session, event, memory, +summary, path, left value, and right value. diff --git a/tests/sessions/replay_consistency/backends.py b/tests/sessions/replay_consistency/backends.py index f09fa29e..a4f3de28 100644 --- a/tests/sessions/replay_consistency/backends.py +++ b/tests/sessions/replay_consistency/backends.py @@ -67,7 +67,7 @@ async def _compress_session_to_summary( fragments: list[str] = [] for event in events: - text = event.get_text().strip() + text = (event.get_text() or "").strip() if not text: calls = event.get_function_calls() responses = event.get_function_responses() diff --git a/tests/sessions/replay_consistency/cases.py b/tests/sessions/replay_consistency/cases.py index 114a919a..478629df 100644 --- a/tests/sessions/replay_consistency/cases.py +++ b/tests/sessions/replay_consistency/cases.py @@ -20,6 +20,8 @@ class EventSpec: tag: str | None filter_key: str | None partial: bool = False + error_code: str | None = None + error_message: str | None = None @dataclass(frozen=True) @@ -55,6 +57,8 @@ def _text_event( branch: str | None = None, tag: str | None = None, filter_key: str | None = None, + error_code: str | None = None, + error_message: str | None = None, ) -> EventSpec: return EventSpec( event_id=f"{case_name}-event-{index:02d}", @@ -68,6 +72,8 @@ def _text_event( branch=branch, tag=tag, filter_key=filter_key, + error_code=error_code, + error_message=error_message, ) @@ -640,16 +646,41 @@ def replay_cases() -> list[ReplayCase]: role="model", text="Same content may repeat.", ), + _text_event( + "duplicate_or_error_recovery", + 3, + invocation_id="inv-duplicate-3", + author="assistant", + role="model", + text="Transient retry error before recovery.", + tag="retry-error", + filter_key="retry", + error_code="RETRYABLE_BACKEND_ERROR", + error_message="Simulated retry failure before recovery.", + ), + _text_event( + "duplicate_or_error_recovery", + 4, + invocation_id="inv-duplicate-4", + author="assistant", + role="model", + text="Recovery succeeded after retry.", + tag="retry-recovery", + filter_key="retry", + ), ], memory_queries=[ MemoryQuerySpec( key=None, - query="Same repeat", + query="Same retry recovery", limit=10, - expected_text_fragments=["Same content may repeat."], + expected_text_fragments=["Same content may repeat.", "Recovery succeeded after retry."], ) ], summary_points=[], - description="Duplicate content with distinct ids is captured as the backend stores it.", + description=( + "Duplicate content with distinct ids plus an error/retry/recovery event sequence " + "is captured as the backend stores it." + ), ), ] diff --git a/tests/sessions/replay_consistency/normalizer.py b/tests/sessions/replay_consistency/normalizer.py index eb9028f0..6a81108a 100644 --- a/tests/sessions/replay_consistency/normalizer.py +++ b/tests/sessions/replay_consistency/normalizer.py @@ -4,6 +4,7 @@ from dataclasses import dataclass import re +import uuid from typing import Any from trpc_agent_sdk.events import Event @@ -60,6 +61,20 @@ def _content_text(content: Content | None) -> str | None: return text or None +def _is_uuid_like(value: str) -> bool: + try: + uuid.UUID(value) + except ValueError: + return False + return True + + +def _normalize_event_id(event: Event) -> str: + if event.is_summary_event() or not event.id or _is_uuid_like(event.id): + return "normalized" + return event.id + + def normalize_event(event: Event, index: int) -> dict[str, Any]: function_calls = [ { @@ -82,7 +97,7 @@ def normalize_event(event: Event, index: int) -> dict[str, Any]: return { "stable_index": index, - "event_id": "normalized", + "event_id": _normalize_event_id(event), "invocation_id": event.invocation_id, "author": event.author, "role": role, diff --git a/tests/sessions/replay_consistency/report.py b/tests/sessions/replay_consistency/report.py index e7b0e746..fb09e496 100644 --- a/tests/sessions/replay_consistency/report.py +++ b/tests/sessions/replay_consistency/report.py @@ -16,8 +16,10 @@ def _diff_to_dict(diff: DiffEntry) -> dict[str, Any]: def write_report(path: Path, comparison_results: list[dict[str, Any]]) -> dict[str, Any]: backend_pairs = [] - case_diff_counts: dict[str, int] = {} + case_diff_counts: dict[str, dict[str, int]] = {} diffs: list[dict[str, Any]] = [] + allowed_diffs: list[dict[str, Any]] = [] + unallowed_diffs: list[dict[str, Any]] = [] for result in comparison_results: left_backend = result["left_backend"] @@ -27,17 +29,26 @@ def write_report(path: Path, comparison_results: list[dict[str, Any]]) -> dict[s backend_pairs.append(pair_name) result_diffs: list[DiffEntry] = result.get("diffs", []) - unallowed_count = sum(1 for diff in result_diffs if not diff.allowed) case_name = result["case_name"] - case_diff_counts[case_name] = case_diff_counts.get(case_name, 0) + unallowed_count - diffs.extend(_diff_to_dict(diff) for diff in result_diffs if not diff.allowed) + if case_name not in case_diff_counts: + case_diff_counts[case_name] = {"allowed": 0, "unallowed": 0} + for diff in result_diffs: + serialized = _diff_to_dict(diff) + diffs.append(serialized) + if diff.allowed: + case_diff_counts[case_name]["allowed"] += 1 + allowed_diffs.append(serialized) + else: + case_diff_counts[case_name]["unallowed"] += 1 + unallowed_diffs.append(serialized) cases = [ { "name": case_name, - "unallowed_diff_count": unallowed_count, + "allowed_diff_count": counts["allowed"], + "unallowed_diff_count": counts["unallowed"], } - for case_name, unallowed_count in case_diff_counts.items() + for case_name, counts in case_diff_counts.items() ] report = { @@ -46,6 +57,10 @@ def write_report(path: Path, comparison_results: list[dict[str, Any]]) -> dict[s "backend_pairs": backend_pairs, "case_count": len(cases), "cases": cases, + "allowed_diff_count": len(allowed_diffs), + "unallowed_diff_count": len(unallowed_diffs), + "allowed_diffs": allowed_diffs, + "unallowed_diffs": unallowed_diffs, "diffs": diffs, } diff --git a/tests/sessions/test_replay_consistency.py b/tests/sessions/test_replay_consistency.py index b66573c0..e6b4a6ba 100644 --- a/tests/sessions/test_replay_consistency.py +++ b/tests/sessions/test_replay_consistency.py @@ -18,6 +18,7 @@ from trpc_agent_sdk.types import FunctionResponse from trpc_agent_sdk.types import MemoryEntry from trpc_agent_sdk.types import Part +from trpc_agent_sdk.types import State from .replay_consistency.backends import BackendBundle from .replay_consistency.backends import build_backends @@ -86,6 +87,8 @@ def _event_from_spec(spec: EventSpec, index: int) -> Event: tag=spec.tag, filter_key=spec.filter_key, partial=spec.partial, + error_code=spec.error_code, + error_message=spec.error_message, timestamp=FIXED_EVENT_TIMESTAMP_BASE + index, ) @@ -135,6 +138,38 @@ async def _search_memory_records( return records +def _temp_state_keys(state: dict[str, Any] | None) -> list[str]: + if not state: + return [] + return sorted(key for key in state if key.startswith(State.TEMP_PREFIX)) + + +def _assert_no_persisted_temp_state(bundle: BackendBundle, case: ReplayCase, session: Session) -> None: + temp_state_keys = _temp_state_keys(session.state) + if temp_state_keys: + pytest.fail( + f"{bundle.name} persisted temp state on {case.name}/{session.id}: {temp_state_keys}" + ) + + temp_delta_keys: list[tuple[str | None, str]] = [] + for event in [*session.events, *session.historical_events]: + if event.actions: + temp_delta_keys.extend((event.id, key) for key in _temp_state_keys(event.actions.state_delta)) + if temp_delta_keys: + pytest.fail( + f"{bundle.name} persisted temp state_delta keys on {case.name}/{session.id}: {temp_delta_keys}" + ) + + if case.name == "scoped_state_overwrite": + fixture_temp_keys = [ + key + for spec in case.events + for key in _temp_state_keys(spec.state_delta) + ] + if not fixture_temp_keys: + pytest.fail("scoped_state_overwrite no longer exercises temp state filtering") + + def _assert_summary_snapshot(snapshot: Snapshot, case: ReplayCase) -> None: if not case.summary_points: return @@ -168,6 +203,35 @@ def _assert_summary_snapshot(snapshot: Snapshot, case: ReplayCase) -> None: pytest.fail(f"{snapshot.backend} truncation case lost post-summary append") +def _assert_duplicate_error_recovery_snapshot(snapshot: Snapshot) -> None: + if snapshot.case_name != "duplicate_or_error_recovery": + return + error_events = [ + event + for event in snapshot.events + if event["error_code"] == "RETRYABLE_BACKEND_ERROR" + and event["error_message"] == "Simulated retry failure before recovery." + ] + if len(error_events) != 1: + pytest.fail(f"{snapshot.backend} did not preserve the retry error event") + if error_events[0]["event_id"] != "duplicate_or_error_recovery-event-03": + pytest.fail(f"{snapshot.backend} did not preserve the deterministic error event id") + if not any(event["text"] == "Recovery succeeded after retry." for event in snapshot.events): + pytest.fail(f"{snapshot.backend} did not preserve the recovery event") + + +def _assert_fixture_event_ids_preserved(snapshot: Snapshot, case: ReplayCase) -> None: + expected_event_ids = {spec.event_id for spec in case.events} + for event in [*snapshot.events, *snapshot.historical_events]: + if event["is_summary_event"]: + continue + if event["event_id"] not in expected_event_ids: + pytest.fail( + f"{snapshot.backend} did not preserve deterministic event id for " + f"{case.name}: {event['event_id']!r}" + ) + + async def _run_standard_case(bundle: BackendBundle, case: ReplayCase) -> Snapshot: summary_texts: list[str] = [] session = await bundle.session_service.create_session( @@ -184,6 +248,7 @@ async def _run_standard_case(bundle: BackendBundle, case: ReplayCase) -> Snapsho session = await _get_required_session(bundle, case) stored_session = await _get_required_session(bundle, case) + _assert_no_persisted_temp_state(bundle, case, stored_session) if case.name == "summary_update_overwrite" and len(summary_texts) >= 2 and summary_texts[0] == summary_texts[-1]: pytest.fail(f"{bundle.name} did not overwrite the cached summary for {case.name}") @@ -197,6 +262,8 @@ async def _run_standard_case(bundle: BackendBundle, case: ReplayCase) -> Snapsho memory_records=memory_records, ) _assert_summary_snapshot(snapshot, case) + _assert_fixture_event_ids_preserved(snapshot, case) + _assert_duplicate_error_recovery_snapshot(snapshot) return snapshot @@ -222,7 +289,7 @@ async def _run_memory_isolation_case(bundle: BackendBundle, case: ReplayCase) -> invocation_id="inv-isolation-b-1", author="user", role="user", - text="User B likes coffee and city museums.", + text="User B also likes jasmine tea and hiking, but only with city-museums-b.", function_call=None, function_response=None, state_delta=None, @@ -235,7 +302,7 @@ async def _run_memory_isolation_case(bundle: BackendBundle, case: ReplayCase) -> invocation_id="inv-isolation-b-1", author="assistant", role="model", - text="I will remember coffee and city museums for User B.", + text="I will remember jasmine tea, hiking, and city-museums-b for User B.", function_call=None, function_response=None, state_delta=None, @@ -255,6 +322,8 @@ async def _run_memory_isolation_case(bundle: BackendBundle, case: ReplayCase) -> ) if stored_b is None: pytest.fail(f"{bundle.name} did not return isolation control session") + _assert_no_persisted_temp_state(bundle, case, stored_a) + _assert_no_persisted_temp_state(bundle, case, stored_b) await bundle.memory_service.store_session(stored_a) await bundle.memory_service.store_session(stored_b) @@ -265,16 +334,18 @@ async def _run_memory_isolation_case(bundle: BackendBundle, case: ReplayCase) -> for memory in memories if memory.content and memory.content.parts ) - if "coffee" in leaked_text or "city museums" in leaked_text: + if "city-museums-b" in leaked_text: pytest.fail(f"{bundle.name} leaked user B memory into user A search: {leaked_text!r}") - return await normalize_snapshot( + snapshot = await normalize_snapshot( backend=bundle.name, case=case, session=stored_a, session_service=bundle.session_service, memory_records=memory_records, ) + _assert_fixture_event_ids_preserved(snapshot, case) + return snapshot async def run_case(bundle: BackendBundle, case: ReplayCase) -> Snapshot: @@ -347,6 +418,21 @@ def test_report_contract(tmp_path: Path): allowed=False, reason="", ) + allowed_diff = DiffEntry( + case_name="case", + left_backend="inmemory", + right_backend="sqlite", + session_id="session", + event_index=None, + memory_index=None, + summary_id=None, + section="backend", + path="backend", + left="inmemory", + right="sqlite", + allowed=True, + reason="backend name differs by design", + ) path = tmp_path / "report.json" report = write_report( path, @@ -355,17 +441,23 @@ def test_report_contract(tmp_path: Path): "case_name": "case", "left_backend": "inmemory", "right_backend": "sqlite", - "diffs": [diff], + "diffs": [allowed_diff, diff], } ], ) loaded = json.loads(path.read_text(encoding="utf-8")) assert loaded == report - serialized = loaded["diffs"][0] + serialized = loaded["unallowed_diffs"][0] for field in DiffEntry.__dataclass_fields__: assert field in serialized assert loaded["schema_version"] == 1 assert loaded["backend_pairs"] == ["inmemory_vs_sqlite"] + assert loaded["allowed_diff_count"] == 1 + assert loaded["unallowed_diff_count"] == 1 + assert len(loaded["allowed_diffs"]) == 1 + assert len(loaded["unallowed_diffs"]) == 1 + assert len(loaded["diffs"]) == 2 + assert loaded["cases"][0]["allowed_diff_count"] == 1 assert loaded["cases"][0]["unallowed_diff_count"] == 1 @@ -554,6 +646,48 @@ def _mutate_snapshot(name: str, snapshot: dict[str, Any]) -> None: ] +def _real_snapshot_mutations_for_case(case: ReplayCase) -> list[str]: + mutations = ["drop_event", "reorder_event", "duplicate_event"] + if case.name == "scoped_state_overwrite": + mutations.append("change_state") + if case.name in {"memory_preference_search", "memory_multi_session_isolation"}: + mutations.extend(["drop_memory", "change_memory_text"]) + if case.name in {"summary_generation", "summary_update_overwrite", "summary_with_event_truncation"}: + mutations.extend(["drop_summary", "overwrite_summary_text", "wrong_summary_session"]) + return mutations + + +async def _run_real_inmemory_snapshot(tmp_path: Path, case: ReplayCase) -> dict[str, Any]: + backends = await build_backends(tmp_path, session_config=_session_config_for_case(case)) + selected: BackendBundle | None = None + unused: list[BackendBundle] = [] + for backend in backends: + if backend.name == "inmemory": + selected = backend + else: + unused.append(backend) + if selected is None: + for backend in unused: + await backend.close() + pytest.fail("InMemory replay backend was not built") + + for backend in unused: + await backend.close() + return asdict(await run_case(selected, case)) + + +@pytest.mark.asyncio +async def test_real_replay_snapshot_mutation_detection(tmp_path: Path): + for case in replay_cases(): + clean = await _run_real_inmemory_snapshot(tmp_path / f"real-mutation-{case.name}", case) + for mutation in _real_snapshot_mutations_for_case(case): + mutated = copy.deepcopy(clean) + mutated["backend"] = "mutated" + _mutate_snapshot(mutation, mutated) + diffs = recursive_diff(clean, mutated) + assert unallowed_diffs(diffs), f"{case.name}/{mutation} was not detected" + + @pytest.mark.parametrize("mutation", MUTATIONS) def test_replay_mutation_detection(mutation: str): clean = _clean_mutation_snapshot() From 4308517c834d6a969644ecadd229750f926215c0 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 19:09:01 +0800 Subject: [PATCH 3/9] Remove duplicate replay mutation test --- tests/sessions/test_replay_consistency.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/sessions/test_replay_consistency.py b/tests/sessions/test_replay_consistency.py index e6b4a6ba..6a59e720 100644 --- a/tests/sessions/test_replay_consistency.py +++ b/tests/sessions/test_replay_consistency.py @@ -698,15 +698,6 @@ def test_replay_mutation_detection(mutation: str): assert unallowed_diffs(diffs), f"{mutation} was not detected" -def test_mutation_detection(): - for mutation in MUTATIONS: - clean = _clean_mutation_snapshot() - mutated = copy.deepcopy(clean) - mutated["backend"] = "sqlite" - _mutate_snapshot(mutation, mutated) - assert unallowed_diffs(recursive_diff(clean, mutated)), f"{mutation} was not detected" - - @pytest.mark.parametrize( ("mutation", "expected_path"), [ From c15c9b92ee7da0557dfb1acdc582d4faffbd96c5 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 20:41:18 +0800 Subject: [PATCH 4/9] Strengthen replay mutation coverage --- tests/sessions/replay_consistency/README.md | 4 +- tests/sessions/test_replay_consistency.py | 53 +++++++++++++++++++-- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/tests/sessions/replay_consistency/README.md b/tests/sessions/replay_consistency/README.md index 90a7efc5..da9e732f 100644 --- a/tests/sessions/replay_consistency/README.md +++ b/tests/sessions/replay_consistency/README.md @@ -27,4 +27,6 @@ partitions, with counts per case and globally. Synthetic mutation tests and real InMemory replay snapshot mutation tests intentionally drop, reorder, duplicate, and alter event, state, memory, and summary fields to prove the recursive diff can locate unallowed regressions by session, event, memory, -summary, path, left value, and right value. +summary, path, left value, and right value. Real replay mutations also cover +tool-call argument and response drift plus retry-error metadata and recovery +event regressions. diff --git a/tests/sessions/test_replay_consistency.py b/tests/sessions/test_replay_consistency.py index 6a59e720..0ed4dd2f 100644 --- a/tests/sessions/test_replay_consistency.py +++ b/tests/sessions/test_replay_consistency.py @@ -201,6 +201,10 @@ def _assert_summary_snapshot(snapshot: Snapshot, case: ReplayCase) -> None: pytest.fail(f"{snapshot.backend} truncation case did not persist historical events") if snapshot.events[-1]["text"] != "Also add a ferry ride after the summary.": pytest.fail(f"{snapshot.backend} truncation case lost post-summary append") + if case.name == "summary_update_overwrite": + latest_summary_text = summary["text"] + if "Checklist and owners" not in latest_summary_text and "release checklist" not in latest_summary_text: + pytest.fail(f"{snapshot.backend} latest summary did not include post-overwrite content") def _assert_duplicate_error_recovery_snapshot(snapshot: Snapshot) -> None: @@ -361,6 +365,12 @@ def _session_config_for_case(case: ReplayCase): return make_session_config(store_historical_events=case.name == "summary_with_event_truncation") +def _keep_recent_count_for_case(case: ReplayCase) -> int: + if case.name == "summary_update_overwrite": + return 1 + return 2 + + @pytest.mark.asyncio async def test_replay_consistency_inmemory_vs_sqlite(tmp_path: Path): cases = replay_cases() @@ -370,7 +380,11 @@ async def test_replay_consistency_inmemory_vs_sqlite(tmp_path: Path): for case in cases: case_tmp_path = tmp_path / case.name - backends = await build_backends(case_tmp_path, session_config=_session_config_for_case(case)) + backends = await build_backends( + case_tmp_path, + session_config=_session_config_for_case(case), + keep_recent_count=_keep_recent_count_for_case(case), + ) assert {"inmemory", "sqlite"} <= {backend.name for backend in backends} snapshots: dict[str, Snapshot] = {} @@ -607,13 +621,28 @@ def _clean_mutation_snapshot() -> dict[str, Any]: return asdict(snapshot) +def _find_mutation_event(name: str, snapshot: dict[str, Any], predicate) -> dict[str, Any]: + for event in snapshot["events"]: + if predicate(event): + return event + raise AssertionError(f"{name} mutation target was not found") + + def _mutate_snapshot(name: str, snapshot: dict[str, Any]) -> None: if name == "drop_event": del snapshot["events"][1] elif name == "reorder_event": snapshot["events"][0], snapshot["events"][1] = snapshot["events"][1], snapshot["events"][0] elif name == "change_tool_args": - snapshot["events"][1]["function_calls"][0]["args"]["city"] = "Shanghai" + event = _find_mutation_event(name, snapshot, lambda item: item["function_calls"]) + event["function_calls"][0]["args"]["city"] = "Shanghai" + elif name == "change_tool_response": + event = _find_mutation_event(name, snapshot, lambda item: item["function_responses"]) + response = event["function_responses"][0]["response"] + if "temperature" in response: + response["temperature"] = 30 + else: + response["condition"] = "rainy" elif name == "change_state": snapshot["state"]["user:tier"] = "silver" elif name == "drop_memory": @@ -628,6 +657,16 @@ def _mutate_snapshot(name: str, snapshot: dict[str, Any]) -> None: snapshot["summary"]["metadata"]["session_id"] = "wrong-session" elif name == "duplicate_event": snapshot["events"].append(copy.deepcopy(snapshot["events"][0])) + elif name == "change_error_code": + event = _find_mutation_event(name, snapshot, lambda item: item.get("error_code")) + event["error_code"] = "WRONG_ERROR_CODE" + elif name == "drop_recovery_event": + for index, event in enumerate(snapshot["events"]): + if event["text"] == "Recovery succeeded after retry.": + del snapshot["events"][index] + break + else: + raise AssertionError("drop_recovery_event mutation target was not found") else: raise ValueError(f"Unknown mutation {name}") @@ -648,17 +687,25 @@ def _mutate_snapshot(name: str, snapshot: dict[str, Any]) -> None: def _real_snapshot_mutations_for_case(case: ReplayCase) -> list[str]: mutations = ["drop_event", "reorder_event", "duplicate_event"] + if case.name == "tool_call_roundtrip": + mutations.extend(["change_tool_args", "change_tool_response"]) if case.name == "scoped_state_overwrite": mutations.append("change_state") if case.name in {"memory_preference_search", "memory_multi_session_isolation"}: mutations.extend(["drop_memory", "change_memory_text"]) if case.name in {"summary_generation", "summary_update_overwrite", "summary_with_event_truncation"}: mutations.extend(["drop_summary", "overwrite_summary_text", "wrong_summary_session"]) + if case.name == "duplicate_or_error_recovery": + mutations.extend(["change_error_code", "drop_recovery_event"]) return mutations async def _run_real_inmemory_snapshot(tmp_path: Path, case: ReplayCase) -> dict[str, Any]: - backends = await build_backends(tmp_path, session_config=_session_config_for_case(case)) + backends = await build_backends( + tmp_path, + session_config=_session_config_for_case(case), + keep_recent_count=_keep_recent_count_for_case(case), + ) selected: BackendBundle | None = None unused: list[BackendBundle] = [] for backend in backends: From 499d98aefc3b39a62f0eda31ecdb37bd60f7ce9f Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 21:26:57 +0800 Subject: [PATCH 5/9] Complete replay consistency harness coverage --- session_memory_summary_diff_report.json | 96 +++++++- .../session_memory_summary_replay_cases.jsonl | 14 ++ tests/sessions/replay_consistency/README.md | 54 ++--- tests/sessions/replay_consistency/__init__.py | 4 + tests/sessions/replay_consistency/backends.py | 30 +++ tests/sessions/replay_consistency/cases.py | 208 ++++++++++++++++++ .../sessions/replay_consistency/mutations.py | 136 ++++++++++++ .../sessions/replay_consistency/normalizer.py | 1 + tests/sessions/replay_consistency/report.py | 95 ++++++-- tests/sessions/test_replay_consistency.py | 197 +++++++---------- .../test_replay_consistency_mutations.py | 95 ++++++++ 11 files changed, 768 insertions(+), 162 deletions(-) create mode 100644 tests/sessions/replay_cases/session_memory_summary_replay_cases.jsonl create mode 100644 tests/sessions/replay_consistency/mutations.py create mode 100644 tests/sessions/test_replay_consistency_mutations.py diff --git a/session_memory_summary_diff_report.json b/session_memory_summary_diff_report.json index 7522ed93..854769f7 100644 --- a/session_memory_summary_diff_report.json +++ b/session_memory_summary_diff_report.json @@ -4,61 +4,155 @@ "backend_pairs": [ "inmemory_vs_sqlite" ], - "case_count": 10, + "backend_statuses": [ + { + "name": "inmemory", + "reason": "", + "status": "ok" + }, + { + "name": "sqlite", + "reason": "", + "status": "ok" + }, + { + "name": "external_sql", + "reason": "TRPC_AGENT_REPLAY_SQL_URL is not set", + "status": "skipped" + }, + { + "name": "redis", + "reason": "TRPC_AGENT_REPLAY_REDIS_URL is not set", + "status": "skipped" + } + ], + "case_count": 14, "cases": [ { "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, "name": "single_turn_text", + "unexpected_diff_count": 0, "unallowed_diff_count": 0 }, { "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, "name": "multi_turn_append_order", + "unexpected_diff_count": 0, "unallowed_diff_count": 0 }, { "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, "name": "tool_call_roundtrip", + "unexpected_diff_count": 0, "unallowed_diff_count": 0 }, { "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, "name": "scoped_state_overwrite", + "unexpected_diff_count": 0, "unallowed_diff_count": 0 }, { "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, "name": "memory_preference_search", + "unexpected_diff_count": 0, "unallowed_diff_count": 0 }, { "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, "name": "memory_multi_session_isolation", + "unexpected_diff_count": 0, "unallowed_diff_count": 0 }, { "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, "name": "summary_generation", + "unexpected_diff_count": 0, "unallowed_diff_count": 0 }, { "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, "name": "summary_update_overwrite", + "unexpected_diff_count": 0, "unallowed_diff_count": 0 }, { "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, "name": "summary_with_event_truncation", + "unexpected_diff_count": 0, "unallowed_diff_count": 0 }, { "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, "name": "duplicate_or_error_recovery", + "unexpected_diff_count": 0, + "unallowed_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "serialization_order_nested_payload", + "unexpected_diff_count": 0, + "unallowed_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "list_sessions_consistency", + "unexpected_diff_count": 0, + "unallowed_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "state_temp_key_ignored_but_persistent_key_compared", + "unexpected_diff_count": 0, + "unallowed_diff_count": 0 + }, + { + "allowed_diff_count": 0, + "backend_pair": "inmemory_vs_sqlite", + "elapsed_ms": 0, + "name": "summary_truncation_preserves_recent_context", + "unexpected_diff_count": 0, "unallowed_diff_count": 0 } ], "diffs": [], + "false_positive_summary": { + "normal_case_count": 14, + "unexpected_diff_count": 0 + }, + "generated_at": "deterministic", "generated_by": "tests/sessions/test_replay_consistency.py", + "mutation_summary": { + "detected_count": 0, + "mutation_count": 0, + "undetected_mutations": [] + }, "schema_version": 1, "unallowed_diff_count": 0, "unallowed_diffs": [] diff --git a/tests/sessions/replay_cases/session_memory_summary_replay_cases.jsonl b/tests/sessions/replay_cases/session_memory_summary_replay_cases.jsonl new file mode 100644 index 00000000..54fa6284 --- /dev/null +++ b/tests/sessions/replay_cases/session_memory_summary_replay_cases.jsonl @@ -0,0 +1,14 @@ +{"name":"single_turn_text","description":"Two text events preserve order, author, role, and text.","operations":["create_session","append user text","append assistant text","store_session","search_memory"],"expected_risks":["basic event ordering drift","text/content normalization regression"]} +{"name":"multi_turn_append_order","description":"Three user/assistant turns preserve append order and invocation ids.","operations":["create_session","append six text events","store_session","search_memory"],"expected_risks":["event reorder","invocation_id mismatch","author/text drift"]} +{"name":"tool_call_roundtrip","description":"Weather tool call, tool response, and final assistant text round-trip across backends.","operations":["create_session","append user request","append function_call","append function_response","append final text"],"expected_risks":["tool args drift","tool response drift","tool event visibility loss"]} +{"name":"scoped_state_overwrite","description":"Session, user, and app state overwrites are strict while temp state is not persisted.","operations":["create_session with state","append state deltas","assert no persisted temp state"],"expected_risks":["state overwrite drift","temp state leakage","persistent key loss"]} +{"name":"memory_preference_search","description":"Preference text is stored through store_session and found by memory search.","operations":["create_session","append preference events","store_session","search tea/hiking/vegetarian"],"expected_risks":["memory content loss","memory author drift","search result normalization drift"]} +{"name":"memory_multi_session_isolation","description":"User A memory search must not return overlapping User B memory containing city-museums-b.","operations":["create session A","create session B","store both sessions","search A save_key"],"expected_risks":["save_key isolation failure","cross-user memory leak","query overlap leakage"]} +{"name":"summary_generation","description":"Manual summary creation yields summary text, summary event, and manager metadata.","operations":["create_session","append long conversation","create_session_summary","get_session_summary"],"expected_risks":["missing summary","summary event flag loss","summary session mismatch"]} +{"name":"summary_update_overwrite","description":"A later summary overwrites the cached summary and includes new release checklist context.","operations":["create_session","append first summary window","create summary","append new events","create summary again"],"expected_risks":["stale summary reuse","latest summary drift","summary wrong session"]} +{"name":"summary_with_event_truncation","description":"Summary truncation stores historical events and keeps recent plus post-summary events active.","operations":["create_session","append events","create summary","append post-summary event"],"expected_risks":["historical event loss","summary anchor loss","recent context loss"]} +{"name":"duplicate_or_error_recovery","description":"Duplicate content, retry error metadata, and recovery event are preserved.","operations":["create_session","append duplicate text events","append retry error event","append recovery event","store_session"],"expected_risks":["duplicate drop/reorder","error metadata drift","recovery event loss"]} +{"name":"serialization_order_nested_payload","description":"Nested dict/list tool payloads are canonicalized by order but strict on value changes.","operations":["create_session","append nested function_call","append nested function_response","append final text"],"expected_risks":["serialization order false positive","nested value drift","tool response loss"]} +{"name":"list_sessions_consistency","description":"list_sessions returns normalized id, app, user, and state consistently.","operations":["create_session with state","append state update","list_sessions"],"expected_risks":["list_sessions state omission","backend-specific list shape drift"]} +{"name":"state_temp_key_ignored_but_persistent_key_compared","description":"temp:* state is ignored for snapshots but rejected in raw persisted sessions.","operations":["create_session with persistent state","append temp and persistent deltas","assert raw temp not persisted"],"expected_risks":["temp key persistence","business state false allow","state value drift"]} +{"name":"summary_truncation_preserves_recent_context","description":"Summary truncation preserves historical events and recent rainy-day context before a post-summary append.","operations":["create_session","append context events","create summary","append canal walk"],"expected_risks":["recent context truncation","historical_events missing","summary event ordering drift"]} diff --git a/tests/sessions/replay_consistency/README.md b/tests/sessions/replay_consistency/README.md index da9e732f..0eb31e9a 100644 --- a/tests/sessions/replay_consistency/README.md +++ b/tests/sessions/replay_consistency/README.md @@ -1,32 +1,34 @@ # Replay Consistency Harness -This harness runs the same deterministic replay cases against multiple storage -backends through the public `SessionServiceABC` and `MemoryServiceABC` -interfaces. Each case creates a session, appends fixed-id events with stable -timestamps and invocation ids, stores the resulting session through -`store_session(session)`, and queries memory with `search_memory(key, query, -limit)`. The default matrix runs real InMemory and SQLite services; Redis is -only added when `TRPC_AGENT_REPLAY_REDIS_URL` is set. +This harness verifies that session, memory, and summary behavior replays +consistently across storage backends. Python `ReplayCase` fixtures define the +executable operation DSL: create a session, append deterministic events, create +summaries at fixed points, store the final session to memory, run memory +queries, and normalize the resulting snapshot. The JSONL manifest in +`tests/sessions/replay_cases/session_memory_summary_replay_cases.jsonl` +mirrors the registry for review readability and is checked by tests. -Snapshots normalize fields that should not affect replay semantics: raw -timestamps, summary timestamp values, auto-generated event ids, dict key order, +The default CI matrix is intentionally light: InMemory plus temporary SQLite +files under `tmp_path`. Optional integration backends are only enabled through +environment variables: `TRPC_AGENT_REPLAY_SQL_URL` for an external SQL backend +and `TRPC_AGENT_REPLAY_REDIS_URL` for Redis. When these variables are absent, +the report records the backend as skipped instead of making CI depend on +external services. + +Normalization removes or canonicalizes non-semantic variance: exact timestamps, +summary timestamp values, auto-generated event ids, dict serialization order, and memory timestamp values. Fixture event ids are preserved so duplicate, -retry, and wrong-id problems remain visible. Memory rows are sorted by -`(query, author, text, key)` because backend search order can differ. +retry, and wrong-id regressions remain visible. Memory search order is sorted +by stable content keys because ranking differs by backend. -The comparator strictly checks event order, roles, authors, text, tool call -args, tool responses, persisted state, memory content, summary text, summary -session id, summary overwrite behavior, summary event flags, and -historical_events. `DeterministicSessionSummarizer` avoids real LLM calls while -leaving the production summary compression path responsible for summary events -and historical event storage. SQLite uses temporary database files and explicit -SQL storage initialization; initialization failures are not silently ignored. +Allowed diffs are deliberately narrow: backend name, raw timestamp values, and +timestamp presence. Event order, author/role/text, tool arguments and results, +state values, memory scope/content, summary text, latest-summary overwrite +semantics, summary session id, summary event flags, and historical events are +strict. Summary text is whitespace-normalized but not semantically relaxed. -Reports include complete diffs plus `allowed_diffs` and `unallowed_diffs` -partitions, with counts per case and globally. Synthetic mutation tests and -real InMemory replay snapshot mutation tests intentionally drop, reorder, -duplicate, and alter event, state, memory, and summary fields to prove the -recursive diff can locate unallowed regressions by session, event, memory, -summary, path, left value, and right value. Real replay mutations also cover -tool-call argument and response drift plus retry-error metadata and recovery -event regressions. +Diff reports include backend statuses, normal replay false-positive counts, +mutation detection summaries, and structured diff entries with case, backend, +session, event, memory, summary, path, left, and right fields. Tests write +runtime reports to `tmp_path`; the repository root JSON is only a deterministic +schema example. diff --git a/tests/sessions/replay_consistency/__init__.py b/tests/sessions/replay_consistency/__init__.py index 7493aa26..b08edec5 100644 --- a/tests/sessions/replay_consistency/__init__.py +++ b/tests/sessions/replay_consistency/__init__.py @@ -10,6 +10,8 @@ from .comparator import DiffEntry from .comparator import compare_snapshot_pair from .comparator import recursive_diff +from .mutations import mutate_snapshot +from .mutations import mutations_for_case from .normalizer import Snapshot from .normalizer import normalize_snapshot from .report import write_report @@ -25,6 +27,8 @@ "DiffEntry", "compare_snapshot_pair", "recursive_diff", + "mutate_snapshot", + "mutations_for_case", "Snapshot", "normalize_snapshot", "write_report", diff --git a/tests/sessions/replay_consistency/backends.py b/tests/sessions/replay_consistency/backends.py index a4f3de28..8d578174 100644 --- a/tests/sessions/replay_consistency/backends.py +++ b/tests/sessions/replay_consistency/backends.py @@ -172,6 +172,36 @@ async def build_backends( ) ) + external_sql_url = os.environ.get("TRPC_AGENT_REPLAY_SQL_URL") + if external_sql_url: + external_sql_session = SqlSessionService( + db_url=external_sql_url, + is_async=False, + session_config=base_config.model_copy(deep=True), + ) + external_sql_session.set_summarizer_manager(_make_summarizer_manager(keep_recent_count), force=True) + external_sql_memory = SqlMemoryService( + db_url=external_sql_url, + is_async=False, + memory_service_config=memory_config.model_copy(deep=True), + ) + try: + await external_sql_session._sql_storage.create_sql_engine() + await external_sql_memory._sql_storage.create_sql_engine() + except ValueError as exc: + if isinstance(exc.__cause__, ImportError): + pytest.skip(f"External SQL replay backend dependency is unavailable: {exc}") + pytest.skip(f"External SQL replay backend failed to initialize: {exc}") + + backends.append( + BackendBundle( + name="external_sql", + session_service=external_sql_session, + memory_service=external_sql_memory, + close=lambda s=external_sql_session, m=external_sql_memory: _close_services(s, m), + ) + ) + redis_url = os.environ.get("TRPC_AGENT_REPLAY_REDIS_URL") if redis_url: try: diff --git a/tests/sessions/replay_consistency/cases.py b/tests/sessions/replay_consistency/cases.py index 478629df..fa76403e 100644 --- a/tests/sessions/replay_consistency/cases.py +++ b/tests/sessions/replay_consistency/cases.py @@ -683,4 +683,212 @@ def replay_cases() -> list[ReplayCase]: "is captured as the backend stores it." ), ), + ReplayCase( + name="serialization_order_nested_payload", + app_name="replay-app", + user_id="user-nested-payload", + session_id="session-011", + initial_state={}, + events=[ + _text_event( + "serialization_order_nested_payload", + 0, + invocation_id="inv-nested-1", + author="user", + role="user", + text="Build a nested itinerary payload for Beijing.", + ), + EventSpec( + event_id="serialization_order_nested_payload-event-01", + invocation_id="inv-nested-1", + author="assistant", + role="model", + text=None, + function_call={ + "id": "call-itinerary-1", + "name": "build_itinerary", + "args": { + "city": "Beijing", + "days": [ + {"day": 2, "focus": ["parks", "tea"]}, + {"day": 1, "focus": ["museums", "food"]}, + ], + "preferences": {"transport": "metro", "budget": "mid"}, + }, + }, + function_response=None, + state_delta=None, + branch="nested.main", + tag="tool-call", + filter_key="nested", + ), + EventSpec( + event_id="serialization_order_nested_payload-event-02", + invocation_id="inv-nested-1", + author="tool", + role="tool", + text=None, + function_call=None, + function_response={ + "id": "call-itinerary-1", + "name": "build_itinerary", + "response": { + "temperature": 21, + "condition": "clear", + "plan": { + "city": "Beijing", + "items": [ + {"slot": "morning", "name": "National Museum"}, + {"slot": "afternoon", "name": "Tea house"}, + ], + }, + }, + }, + state_delta=None, + branch="nested.main", + tag="tool-response", + filter_key="nested", + ), + _text_event( + "serialization_order_nested_payload", + 3, + invocation_id="inv-nested-1", + author="assistant", + role="model", + text="The nested itinerary payload is ready.", + branch="nested.main", + tag="final", + filter_key="nested", + ), + ], + memory_queries=[], + summary_points=[], + description=( + "Risk: nested dict/list serialization order should be canonicalized, while " + "tool argument and tool response value drift remains strict." + ), + ), + ReplayCase( + name="list_sessions_consistency", + app_name="replay-app", + user_id="user-list-sessions", + session_id="session-012", + initial_state={"session_label": "list-check", "user:tier": "silver"}, + events=[ + _text_event( + "list_sessions_consistency", + 0, + invocation_id="inv-list-1", + author="user", + role="user", + text="Create a session that can be listed consistently.", + ), + _text_event( + "list_sessions_consistency", + 1, + invocation_id="inv-list-1", + author="assistant", + role="model", + text="The session list should include this normalized state.", + state_delta={"session_label": "list-check-updated"}, + ), + ], + memory_queries=[], + summary_points=[], + description=( + "Risk: list_sessions may omit or reshape id/app/user/state differently across backends." + ), + ), + ReplayCase( + name="state_temp_key_ignored_but_persistent_key_compared", + app_name="replay-app", + user_id="user-state-temp", + session_id="session-013", + initial_state={"user:tier": "gold", "preference": "tea"}, + events=[ + _text_event( + "state_temp_key_ignored_but_persistent_key_compared", + 0, + invocation_id="inv-state-temp-1", + author="user", + role="user", + text="Remember my persistent state but not the trace id.", + state_delta={ + "user:tier": "platinum", + "preference": "oolong tea", + "temp:trace_id": "temp-state-1", + }, + ), + _text_event( + "state_temp_key_ignored_but_persistent_key_compared", + 1, + invocation_id="inv-state-temp-1", + author="assistant", + role="model", + text="Persistent state is updated and temp state is not stored.", + state_delta={"session_counter": 2, "temp:trace_id": "temp-state-2"}, + ), + ], + memory_queries=[], + summary_points=[], + description=( + "Risk: temp:* state must be filtered from persisted raw sessions while business " + "state values remain strictly compared." + ), + ), + ReplayCase( + name="summary_truncation_preserves_recent_context", + app_name="replay-app", + user_id="user-summary-recent", + session_id="session-014", + initial_state={}, + events=[ + _text_event( + "summary_truncation_preserves_recent_context", + 0, + invocation_id="inv-summary-recent-1", + author="user", + role="user", + text="Start a research plan for Hangzhou.", + ), + _text_event( + "summary_truncation_preserves_recent_context", + 1, + invocation_id="inv-summary-recent-1", + author="assistant", + role="model", + text="We will track museums, tea, and quiet walks.", + ), + _text_event( + "summary_truncation_preserves_recent_context", + 2, + invocation_id="inv-summary-recent-2", + author="user", + role="user", + text="Keep the newest context about rainy day options.", + ), + _text_event( + "summary_truncation_preserves_recent_context", + 3, + invocation_id="inv-summary-recent-2", + author="assistant", + role="model", + text="Rainy day options stay in recent context.", + ), + _text_event( + "summary_truncation_preserves_recent_context", + 4, + invocation_id="inv-summary-recent-3", + author="user", + role="user", + text="After summary, add a canal walk.", + ), + ], + memory_queries=[], + summary_points=[3], + description=( + "Risk: truncation should preserve historical events and recent active context " + "after a summary event is inserted." + ), + ), ] diff --git a/tests/sessions/replay_consistency/mutations.py b/tests/sessions/replay_consistency/mutations.py new file mode 100644 index 00000000..21301e8c --- /dev/null +++ b/tests/sessions/replay_consistency/mutations.py @@ -0,0 +1,136 @@ +"""Fault injection helpers for replay snapshot mutation tests.""" + +from __future__ import annotations + +import copy +from typing import Any +from typing import Callable + +from .cases import ReplayCase + + +SYNTHETIC_MUTATIONS = [ + "drop_event", + "reorder_event", + "change_tool_args", + "change_state", + "drop_memory", + "change_memory_text", + "drop_summary", + "overwrite_summary_text", + "wrong_summary_session", + "duplicate_event", +] + +SUMMARY_REQUIRED_MUTATIONS = [ + "drop_summary", + "overwrite_summary_text", + "wrong_summary_session", +] + +MUTATION_SECTION = { + "drop_event": "events", + "reorder_event": "events", + "reorder_events": "events", + "duplicate_event": "events", + "change_tool_args": "events", + "change_tool_response": "events", + "change_error_code": "events", + "drop_recovery_event": "events", + "change_state": "state", + "change_state_value": "state", + "drop_memory": "memories", + "change_memory_text": "memories", + "change_memory_text_or_metadata": "memories", + "drop_summary": "summary", + "overwrite_summary_text": "summary", + "overwrite_summary_with_stale_text": "summary", + "wrong_summary_session": "summary", + "summary_wrong_session_id": "summary", +} + + +def mutations_for_case(case: ReplayCase) -> list[str]: + mutations = ["drop_event", "reorder_events", "duplicate_event"] + if case.name in {"tool_call_roundtrip", "serialization_order_nested_payload"}: + mutations.extend(["change_tool_args", "change_tool_response"]) + if case.name in {"scoped_state_overwrite", "state_temp_key_ignored_but_persistent_key_compared"}: + mutations.append("change_state_value") + if case.name in {"memory_preference_search", "memory_multi_session_isolation"}: + mutations.extend(["drop_memory", "change_memory_text_or_metadata"]) + if case.name in { + "summary_generation", + "summary_update_overwrite", + "summary_with_event_truncation", + "summary_truncation_preserves_recent_context", + }: + mutations.extend(["drop_summary", "overwrite_summary_with_stale_text", "summary_wrong_session_id"]) + if case.name == "duplicate_or_error_recovery": + mutations.extend(["change_error_code", "drop_recovery_event"]) + return mutations + + +def _find_mutation_event( + name: str, + snapshot: dict[str, Any], + predicate: Callable[[dict[str, Any]], bool], +) -> dict[str, Any]: + for event in snapshot["events"]: + if predicate(event): + return event + raise AssertionError(f"{name} mutation target was not found") + + +def mutate_snapshot(name: str, snapshot: dict[str, Any]) -> None: + if name == "drop_event": + del snapshot["events"][1] + elif name in {"reorder_event", "reorder_events"}: + snapshot["events"][0], snapshot["events"][1] = snapshot["events"][1], snapshot["events"][0] + elif name == "change_tool_args": + event = _find_mutation_event(name, snapshot, lambda item: item["function_calls"]) + event["function_calls"][0]["args"]["city"] = "Shanghai" + elif name == "change_tool_response": + event = _find_mutation_event(name, snapshot, lambda item: item["function_responses"]) + response = event["function_responses"][0]["response"] + if "temperature" in response: + response["temperature"] = 30 + else: + response["condition"] = "rainy" + elif name in {"change_state", "change_state_value"}: + if "user:tier" in snapshot["state"]: + snapshot["state"]["user:tier"] = "silver" + elif "preference" in snapshot["state"]: + snapshot["state"]["preference"] = "coffee" + elif snapshot["state"]: + key = sorted(snapshot["state"])[0] + snapshot["state"][key] = "mutated-state" + else: + raise AssertionError(f"{name} mutation target was not found") + elif name == "drop_memory": + del snapshot["memories"][0] + elif name in {"change_memory_text", "change_memory_text_or_metadata"}: + snapshot["memories"][0]["text"] = "I prefer coffee in the morning." + elif name == "drop_summary": + snapshot["summary"] = None + elif name in {"overwrite_summary_text", "overwrite_summary_with_stale_text"}: + if snapshot["summary"] is None: + raise AssertionError(f"{name} mutation target was not found") + snapshot["summary"]["text"] = "summary(session-mutation): stale overwritten summary" + elif name in {"wrong_summary_session", "summary_wrong_session_id"}: + if snapshot["summary"] is None: + raise AssertionError(f"{name} mutation target was not found") + snapshot["summary"]["metadata"]["session_id"] = "wrong-session" + elif name == "duplicate_event": + snapshot["events"].append(copy.deepcopy(snapshot["events"][0])) + elif name == "change_error_code": + event = _find_mutation_event(name, snapshot, lambda item: item.get("error_code")) + event["error_code"] = "WRONG_ERROR_CODE" + elif name == "drop_recovery_event": + for index, event in enumerate(snapshot["events"]): + if event["text"] == "Recovery succeeded after retry.": + del snapshot["events"][index] + break + else: + raise AssertionError("drop_recovery_event mutation target was not found") + else: + raise ValueError(f"Unknown mutation {name}") diff --git a/tests/sessions/replay_consistency/normalizer.py b/tests/sessions/replay_consistency/normalizer.py index 6a81108a..59890781 100644 --- a/tests/sessions/replay_consistency/normalizer.py +++ b/tests/sessions/replay_consistency/normalizer.py @@ -132,6 +132,7 @@ def normalize_memory_results(memory_records: list[tuple[MemoryQuerySpec, str, li memories: list[dict[str, Any]] = [] for query_spec, key, entries in memory_records: memories.extend(normalize_memory_entry(query_spec, key, entry) for entry in entries) + # Memory search ranking is backend-specific here; content/scope are strict, order is canonicalized. return sorted( memories, key=lambda item: ( diff --git a/tests/sessions/replay_consistency/report.py b/tests/sessions/replay_consistency/report.py index fb09e496..19e53be9 100644 --- a/tests/sessions/replay_consistency/report.py +++ b/tests/sessions/replay_consistency/report.py @@ -4,6 +4,7 @@ from dataclasses import asdict import json +import os from pathlib import Path from typing import Any @@ -14,12 +15,45 @@ def _diff_to_dict(diff: DiffEntry) -> dict[str, Any]: return asdict(diff) -def write_report(path: Path, comparison_results: list[dict[str, Any]]) -> dict[str, Any]: +def _backend_statuses(backend_pairs: list[str], backend_statuses: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + if backend_statuses is not None: + return backend_statuses + + active_backends = sorted({backend for pair in backend_pairs for backend in pair.split("_vs_")}) + statuses = [{"name": backend, "status": "ok", "reason": ""} for backend in active_backends] + optional_backends = [ + ("external_sql", "TRPC_AGENT_REPLAY_SQL_URL"), + ("redis", "TRPC_AGENT_REPLAY_REDIS_URL"), + ] + for name, env_var in optional_backends: + if name not in active_backends and not os.environ.get(env_var): + statuses.append({"name": name, "status": "skipped", "reason": f"{env_var} is not set"}) + return statuses + + +def _serialize_diffs(diffs: list[DiffEntry], mutation: str | None = None) -> list[dict[str, Any]]: + serialized = [] + for diff in diffs: + item = _diff_to_dict(diff) + if mutation is not None: + item["mutation"] = mutation + serialized.append(item) + return serialized + + +def write_report( + path: Path, + comparison_results: list[dict[str, Any]], + *, + backend_statuses: list[dict[str, Any]] | None = None, + mutation_results: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: backend_pairs = [] - case_diff_counts: dict[str, dict[str, int]] = {} + cases: list[dict[str, Any]] = [] diffs: list[dict[str, Any]] = [] allowed_diffs: list[dict[str, Any]] = [] unallowed_diffs: list[dict[str, Any]] = [] + normal_unexpected_count = 0 for result in comparison_results: left_backend = result["left_backend"] @@ -29,32 +63,52 @@ def write_report(path: Path, comparison_results: list[dict[str, Any]]) -> dict[s backend_pairs.append(pair_name) result_diffs: list[DiffEntry] = result.get("diffs", []) + allowed_count = sum(1 for diff in result_diffs if diff.allowed) + unallowed_count = sum(1 for diff in result_diffs if not diff.allowed) + normal_unexpected_count += unallowed_count case_name = result["case_name"] - if case_name not in case_diff_counts: - case_diff_counts[case_name] = {"allowed": 0, "unallowed": 0} - for diff in result_diffs: - serialized = _diff_to_dict(diff) + cases.append( + { + "name": case_name, + "backend_pair": pair_name, + "unexpected_diff_count": unallowed_count, + "allowed_diff_count": allowed_count, + "unallowed_diff_count": unallowed_count, + "elapsed_ms": result.get("elapsed_ms", 0), + } + ) + for serialized, diff in zip(_serialize_diffs(result_diffs), result_diffs): diffs.append(serialized) if diff.allowed: - case_diff_counts[case_name]["allowed"] += 1 allowed_diffs.append(serialized) else: - case_diff_counts[case_name]["unallowed"] += 1 unallowed_diffs.append(serialized) - cases = [ - { - "name": case_name, - "allowed_diff_count": counts["allowed"], - "unallowed_diff_count": counts["unallowed"], - } - for case_name, counts in case_diff_counts.items() - ] + mutation_results = mutation_results or [] + undetected_mutations = [] + detected_count = 0 + for result in mutation_results: + result_diffs = result.get("diffs", []) + mutation = result["mutation"] + detected = bool(result.get("detected")) + if detected: + detected_count += 1 + else: + undetected_mutations.append({"case_name": result["case_name"], "mutation": mutation}) + + for serialized, diff in zip(_serialize_diffs(result_diffs, mutation), result_diffs): + diffs.append(serialized) + if diff.allowed: + allowed_diffs.append(serialized) + else: + unallowed_diffs.append(serialized) report = { "schema_version": 1, + "generated_at": "deterministic", "generated_by": "tests/sessions/test_replay_consistency.py", "backend_pairs": backend_pairs, + "backend_statuses": _backend_statuses(backend_pairs, backend_statuses), "case_count": len(cases), "cases": cases, "allowed_diff_count": len(allowed_diffs), @@ -62,6 +116,15 @@ def write_report(path: Path, comparison_results: list[dict[str, Any]]) -> dict[s "allowed_diffs": allowed_diffs, "unallowed_diffs": unallowed_diffs, "diffs": diffs, + "mutation_summary": { + "mutation_count": len(mutation_results), + "detected_count": detected_count, + "undetected_mutations": undetected_mutations, + }, + "false_positive_summary": { + "normal_case_count": len(comparison_results), + "unexpected_diff_count": normal_unexpected_count, + }, } path.parent.mkdir(parents=True, exist_ok=True) diff --git a/tests/sessions/test_replay_consistency.py b/tests/sessions/test_replay_consistency.py index 0ed4dd2f..1762aaf1 100644 --- a/tests/sessions/test_replay_consistency.py +++ b/tests/sessions/test_replay_consistency.py @@ -31,6 +31,8 @@ from .replay_consistency.comparator import compare_snapshot_pair from .replay_consistency.comparator import recursive_diff from .replay_consistency.comparator import unallowed_diffs +from .replay_consistency.mutations import SYNTHETIC_MUTATIONS as MUTATIONS +from .replay_consistency.mutations import mutate_snapshot from .replay_consistency.normalizer import Snapshot from .replay_consistency.normalizer import normalize_snapshot from .replay_consistency.report import write_report @@ -49,6 +51,13 @@ "duplicate_or_error_recovery", ] +ADDITIONAL_CASE_NAMES = [ + "serialization_order_nested_payload", + "list_sessions_consistency", + "state_temp_key_ignored_but_persistent_key_compared", + "summary_truncation_preserves_recent_context", +] + FIXED_EVENT_TIMESTAMP_BASE = 2_000_000_000.0 @@ -194,12 +203,16 @@ def _assert_summary_snapshot(snapshot: Snapshot, case: ReplayCase) -> None: if summary_events[0]["author"] != "system": pytest.fail(f"{snapshot.backend} summary event author is not system for {case.name}") - if case.name == "summary_with_event_truncation": + if case.name in {"summary_with_event_truncation", "summary_truncation_preserves_recent_context"}: if snapshot.events[0]["is_summary_event"] is not True: pytest.fail(f"{snapshot.backend} truncation case did not keep summary event first") if metadata["historical_event_count"] == 0 or not snapshot.historical_events: pytest.fail(f"{snapshot.backend} truncation case did not persist historical events") - if snapshot.events[-1]["text"] != "Also add a ferry ride after the summary.": + expected_last_text = { + "summary_with_event_truncation": "Also add a ferry ride after the summary.", + "summary_truncation_preserves_recent_context": "After summary, add a canal walk.", + }[case.name] + if snapshot.events[-1]["text"] != expected_last_text: pytest.fail(f"{snapshot.backend} truncation case lost post-summary append") if case.name == "summary_update_overwrite": latest_summary_text = summary["text"] @@ -362,7 +375,12 @@ async def run_case(bundle: BackendBundle, case: ReplayCase) -> Snapshot: def _session_config_for_case(case: ReplayCase): - return make_session_config(store_historical_events=case.name == "summary_with_event_truncation") + return make_session_config( + store_historical_events=case.name in { + "summary_with_event_truncation", + "summary_truncation_preserves_recent_context", + } + ) def _keep_recent_count_for_case(case: ReplayCase) -> int: @@ -371,10 +389,33 @@ def _keep_recent_count_for_case(case: ReplayCase) -> int: return 2 +async def _run_real_inmemory_snapshot(tmp_path: Path, case: ReplayCase) -> dict[str, Any]: + backends = await build_backends( + tmp_path, + session_config=_session_config_for_case(case), + keep_recent_count=_keep_recent_count_for_case(case), + ) + selected: BackendBundle | None = None + unused: list[BackendBundle] = [] + for backend in backends: + if backend.name == "inmemory": + selected = backend + else: + unused.append(backend) + if selected is None: + for backend in unused: + await backend.close() + pytest.fail("InMemory replay backend was not built") + + for backend in unused: + await backend.close() + return asdict(await run_case(selected, case)) + + @pytest.mark.asyncio async def test_replay_consistency_inmemory_vs_sqlite(tmp_path: Path): cases = replay_cases() - assert len(cases) == 10 + assert len(cases) == len(REQUIRED_CASE_NAMES) + len(ADDITIONAL_CASE_NAMES) comparison_results: list[dict[str, Any]] = [] report_path = tmp_path / "session_memory_summary_diff_report.json" @@ -413,7 +454,23 @@ async def test_replay_consistency_inmemory_vs_sqlite(tmp_path: Path): def test_replay_case_count_and_names(): - assert [case.name for case in replay_cases()] == REQUIRED_CASE_NAMES + names = [case.name for case in replay_cases()] + assert names[:len(REQUIRED_CASE_NAMES)] == REQUIRED_CASE_NAMES + assert names[len(REQUIRED_CASE_NAMES):] == ADDITIONAL_CASE_NAMES + + +def test_replay_case_manifest_matches_registry(): + manifest_path = Path(__file__).parent / "replay_cases" / "session_memory_summary_replay_cases.jsonl" + manifest_cases = [ + json.loads(line) + for line in manifest_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + assert [case["name"] for case in manifest_cases] == [case.name for case in replay_cases()] + for case in manifest_cases: + assert case["description"] + assert case["operations"] + assert case["expected_risks"] def test_report_contract(tmp_path: Path): @@ -465,14 +522,28 @@ def test_report_contract(tmp_path: Path): for field in DiffEntry.__dataclass_fields__: assert field in serialized assert loaded["schema_version"] == 1 + assert loaded["generated_at"] == "deterministic" assert loaded["backend_pairs"] == ["inmemory_vs_sqlite"] + assert loaded["backend_statuses"] assert loaded["allowed_diff_count"] == 1 assert loaded["unallowed_diff_count"] == 1 assert len(loaded["allowed_diffs"]) == 1 assert len(loaded["unallowed_diffs"]) == 1 assert len(loaded["diffs"]) == 2 + assert loaded["cases"][0]["backend_pair"] == "inmemory_vs_sqlite" assert loaded["cases"][0]["allowed_diff_count"] == 1 + assert loaded["cases"][0]["unexpected_diff_count"] == 1 assert loaded["cases"][0]["unallowed_diff_count"] == 1 + assert loaded["cases"][0]["elapsed_ms"] == 0 + assert loaded["mutation_summary"] == { + "mutation_count": 0, + "detected_count": 0, + "undetected_mutations": [], + } + assert loaded["false_positive_summary"] == { + "normal_case_count": 1, + "unexpected_diff_count": 1, + } def _clean_mutation_snapshot() -> dict[str, Any]: @@ -621,118 +692,6 @@ def _clean_mutation_snapshot() -> dict[str, Any]: return asdict(snapshot) -def _find_mutation_event(name: str, snapshot: dict[str, Any], predicate) -> dict[str, Any]: - for event in snapshot["events"]: - if predicate(event): - return event - raise AssertionError(f"{name} mutation target was not found") - - -def _mutate_snapshot(name: str, snapshot: dict[str, Any]) -> None: - if name == "drop_event": - del snapshot["events"][1] - elif name == "reorder_event": - snapshot["events"][0], snapshot["events"][1] = snapshot["events"][1], snapshot["events"][0] - elif name == "change_tool_args": - event = _find_mutation_event(name, snapshot, lambda item: item["function_calls"]) - event["function_calls"][0]["args"]["city"] = "Shanghai" - elif name == "change_tool_response": - event = _find_mutation_event(name, snapshot, lambda item: item["function_responses"]) - response = event["function_responses"][0]["response"] - if "temperature" in response: - response["temperature"] = 30 - else: - response["condition"] = "rainy" - elif name == "change_state": - snapshot["state"]["user:tier"] = "silver" - elif name == "drop_memory": - del snapshot["memories"][0] - elif name == "change_memory_text": - snapshot["memories"][0]["text"] = "I prefer coffee in the morning." - elif name == "drop_summary": - snapshot["summary"] = None - elif name == "overwrite_summary_text": - snapshot["summary"]["text"] = "summary(session-mutation): overwritten" - elif name == "wrong_summary_session": - snapshot["summary"]["metadata"]["session_id"] = "wrong-session" - elif name == "duplicate_event": - snapshot["events"].append(copy.deepcopy(snapshot["events"][0])) - elif name == "change_error_code": - event = _find_mutation_event(name, snapshot, lambda item: item.get("error_code")) - event["error_code"] = "WRONG_ERROR_CODE" - elif name == "drop_recovery_event": - for index, event in enumerate(snapshot["events"]): - if event["text"] == "Recovery succeeded after retry.": - del snapshot["events"][index] - break - else: - raise AssertionError("drop_recovery_event mutation target was not found") - else: - raise ValueError(f"Unknown mutation {name}") - - -MUTATIONS = [ - "drop_event", - "reorder_event", - "change_tool_args", - "change_state", - "drop_memory", - "change_memory_text", - "drop_summary", - "overwrite_summary_text", - "wrong_summary_session", - "duplicate_event", -] - - -def _real_snapshot_mutations_for_case(case: ReplayCase) -> list[str]: - mutations = ["drop_event", "reorder_event", "duplicate_event"] - if case.name == "tool_call_roundtrip": - mutations.extend(["change_tool_args", "change_tool_response"]) - if case.name == "scoped_state_overwrite": - mutations.append("change_state") - if case.name in {"memory_preference_search", "memory_multi_session_isolation"}: - mutations.extend(["drop_memory", "change_memory_text"]) - if case.name in {"summary_generation", "summary_update_overwrite", "summary_with_event_truncation"}: - mutations.extend(["drop_summary", "overwrite_summary_text", "wrong_summary_session"]) - if case.name == "duplicate_or_error_recovery": - mutations.extend(["change_error_code", "drop_recovery_event"]) - return mutations - - -async def _run_real_inmemory_snapshot(tmp_path: Path, case: ReplayCase) -> dict[str, Any]: - backends = await build_backends( - tmp_path, - session_config=_session_config_for_case(case), - keep_recent_count=_keep_recent_count_for_case(case), - ) - selected: BackendBundle | None = None - unused: list[BackendBundle] = [] - for backend in backends: - if backend.name == "inmemory": - selected = backend - else: - unused.append(backend) - if selected is None: - for backend in unused: - await backend.close() - pytest.fail("InMemory replay backend was not built") - - for backend in unused: - await backend.close() - return asdict(await run_case(selected, case)) - - -@pytest.mark.asyncio -async def test_real_replay_snapshot_mutation_detection(tmp_path: Path): - for case in replay_cases(): - clean = await _run_real_inmemory_snapshot(tmp_path / f"real-mutation-{case.name}", case) - for mutation in _real_snapshot_mutations_for_case(case): - mutated = copy.deepcopy(clean) - mutated["backend"] = "mutated" - _mutate_snapshot(mutation, mutated) - diffs = recursive_diff(clean, mutated) - assert unallowed_diffs(diffs), f"{case.name}/{mutation} was not detected" @pytest.mark.parametrize("mutation", MUTATIONS) @@ -740,7 +699,7 @@ def test_replay_mutation_detection(mutation: str): clean = _clean_mutation_snapshot() mutated = copy.deepcopy(clean) mutated["backend"] = "sqlite" - _mutate_snapshot(mutation, mutated) + mutate_snapshot(mutation, mutated) diffs = recursive_diff(clean, mutated) assert unallowed_diffs(diffs), f"{mutation} was not detected" @@ -757,7 +716,7 @@ def test_summary_required_mutations_detected(mutation: str, expected_path: str): clean = _clean_mutation_snapshot() mutated = copy.deepcopy(clean) mutated["backend"] = "sqlite" - _mutate_snapshot(mutation, mutated) + mutate_snapshot(mutation, mutated) diffs = unallowed_diffs(recursive_diff(clean, mutated)) assert diffs assert any(diff.section == "summary" and diff.path == expected_path for diff in diffs) diff --git a/tests/sessions/test_replay_consistency_mutations.py b/tests/sessions/test_replay_consistency_mutations.py new file mode 100644 index 00000000..ffc477c6 --- /dev/null +++ b/tests/sessions/test_replay_consistency_mutations.py @@ -0,0 +1,95 @@ +"""End-to-end replay snapshot mutation tests.""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any + +import pytest + +from .replay_consistency.cases import replay_cases +from .replay_consistency.comparator import DiffEntry +from .replay_consistency.comparator import recursive_diff +from .replay_consistency.comparator import unallowed_diffs +from .replay_consistency.mutations import MUTATION_SECTION +from .replay_consistency.mutations import SUMMARY_REQUIRED_MUTATIONS +from .replay_consistency.mutations import mutate_snapshot +from .replay_consistency.mutations import mutations_for_case +from .replay_consistency.report import write_report +from .test_replay_consistency import _run_real_inmemory_snapshot + + +def _assert_diff_context( + *, + case_name: str, + mutation: str, + clean_snapshot: dict[str, Any], + diffs: list[DiffEntry], +) -> None: + unexpected = unallowed_diffs(diffs) + assert unexpected, f"{case_name}/{mutation} was not detected" + + expected_section = MUTATION_SECTION[mutation] + matching = [diff for diff in unexpected if diff.section == expected_section] + assert matching, f"{case_name}/{mutation} did not report section {expected_section}" + assert all(diff.session_id == clean_snapshot["session_id"] for diff in unexpected) + assert all(diff.path for diff in unexpected) + + if expected_section == "events": + assert any(diff.event_index is not None for diff in matching) + elif expected_section == "memories": + assert any(diff.memory_index is not None for diff in matching) + elif expected_section == "summary": + assert any(diff.summary_id for diff in matching) + + +@pytest.mark.asyncio +async def test_real_replay_snapshot_mutation_detection(tmp_path: Path): + mutation_results: list[dict[str, Any]] = [] + for case in replay_cases(): + clean = await _run_real_inmemory_snapshot(tmp_path / f"real-mutation-{case.name}", case) + for mutation in mutations_for_case(case): + mutated = copy.deepcopy(clean) + mutated["backend"] = "mutated" + mutate_snapshot(mutation, mutated) + diffs = recursive_diff(clean, mutated) + _assert_diff_context( + case_name=case.name, + mutation=mutation, + clean_snapshot=clean, + diffs=diffs, + ) + mutation_results.append( + { + "case_name": case.name, + "mutation": mutation, + "detected": bool(unallowed_diffs(diffs)), + "diffs": diffs, + } + ) + + report = write_report( + tmp_path / "session_memory_summary_mutation_diff_report.json", + [], + mutation_results=mutation_results, + ) + assert report["mutation_summary"]["undetected_mutations"] == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mutation", SUMMARY_REQUIRED_MUTATIONS) +async def test_real_summary_required_mutations_detected(tmp_path: Path, mutation: str): + case = next(case for case in replay_cases() if case.name == "summary_generation") + clean = await _run_real_inmemory_snapshot(tmp_path / f"summary-required-{mutation}", case) + mutated = copy.deepcopy(clean) + mutated["backend"] = "mutated" + mutate_snapshot(mutation, mutated) + diffs = recursive_diff(clean, mutated) + _assert_diff_context( + case_name=case.name, + mutation=mutation, + clean_snapshot=clean, + diffs=diffs, + ) + assert all(diff.section == "summary" for diff in unallowed_diffs(diffs) if diff.path.startswith("summary")) From f6d239dd6f13ccf9255c5c8bce600c8aec496706 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 21:40:19 +0800 Subject: [PATCH 6/9] Tighten replay report and mutation assertions --- tests/sessions/replay_consistency/report.py | 4 +++- .../test_replay_consistency_mutations.py | 23 ++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/sessions/replay_consistency/report.py b/tests/sessions/replay_consistency/report.py index 19e53be9..7a3829b7 100644 --- a/tests/sessions/replay_consistency/report.py +++ b/tests/sessions/replay_consistency/report.py @@ -20,7 +20,9 @@ def _backend_statuses(backend_pairs: list[str], backend_statuses: list[dict[str, return backend_statuses active_backends = sorted({backend for pair in backend_pairs for backend in pair.split("_vs_")}) - statuses = [{"name": backend, "status": "ok", "reason": ""} for backend in active_backends] + ordered_backends = ["inmemory", "sqlite"] + ordered_backends.extend(backend for backend in active_backends if backend not in ordered_backends) + statuses = [{"name": backend, "status": "ok", "reason": ""} for backend in ordered_backends] optional_backends = [ ("external_sql", "TRPC_AGENT_REPLAY_SQL_URL"), ("redis", "TRPC_AGENT_REPLAY_REDIS_URL"), diff --git a/tests/sessions/test_replay_consistency_mutations.py b/tests/sessions/test_replay_consistency_mutations.py index ffc477c6..05a60a91 100644 --- a/tests/sessions/test_replay_consistency_mutations.py +++ b/tests/sessions/test_replay_consistency_mutations.py @@ -38,10 +38,20 @@ def _assert_diff_context( if expected_section == "events": assert any(diff.event_index is not None for diff in matching) + if mutation == "change_tool_args": + assert any("function_calls" in diff.path for diff in matching) + if mutation == "change_tool_response": + assert any("function_responses" in diff.path for diff in matching) elif expected_section == "memories": - assert any(diff.memory_index is not None for diff in matching) + assert any(diff.memory_index is not None or diff.section == "memories" for diff in matching) elif expected_section == "summary": assert any(diff.summary_id for diff in matching) + if mutation == "drop_summary": + assert any(diff.path == "summary" for diff in matching) + elif mutation in {"overwrite_summary_with_stale_text", "overwrite_summary_text"}: + assert any(diff.path == "summary.text" for diff in matching) + elif mutation in {"summary_wrong_session_id", "wrong_summary_session"}: + assert any("summary.metadata.session_id" in diff.path for diff in matching) @pytest.mark.asyncio @@ -51,9 +61,9 @@ async def test_real_replay_snapshot_mutation_detection(tmp_path: Path): clean = await _run_real_inmemory_snapshot(tmp_path / f"real-mutation-{case.name}", case) for mutation in mutations_for_case(case): mutated = copy.deepcopy(clean) - mutated["backend"] = "mutated" + mutated["backend"] = "sqlite" mutate_snapshot(mutation, mutated) - diffs = recursive_diff(clean, mutated) + diffs = unallowed_diffs(recursive_diff(clean, mutated)) _assert_diff_context( case_name=case.name, mutation=mutation, @@ -70,10 +80,11 @@ async def test_real_replay_snapshot_mutation_detection(tmp_path: Path): ) report = write_report( - tmp_path / "session_memory_summary_mutation_diff_report.json", + tmp_path / "mutation_report.json", [], mutation_results=mutation_results, ) + assert report["mutation_summary"]["mutation_count"] == report["mutation_summary"]["detected_count"] assert report["mutation_summary"]["undetected_mutations"] == [] @@ -83,9 +94,9 @@ async def test_real_summary_required_mutations_detected(tmp_path: Path, mutation case = next(case for case in replay_cases() if case.name == "summary_generation") clean = await _run_real_inmemory_snapshot(tmp_path / f"summary-required-{mutation}", case) mutated = copy.deepcopy(clean) - mutated["backend"] = "mutated" + mutated["backend"] = "sqlite" mutate_snapshot(mutation, mutated) - diffs = recursive_diff(clean, mutated) + diffs = unallowed_diffs(recursive_diff(clean, mutated)) _assert_diff_context( case_name=case.name, mutation=mutation, From 5557e0061efd0ec073206b992a12888d184c111d Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 21:55:50 +0800 Subject: [PATCH 7/9] Add replay mutation report example --- tests/sessions/replay_consistency/README.md | 3 +- ...ession_memory_summary_mutation_report.json | 3048 +++++++++++++++++ .../test_replay_consistency_mutations.py | 6 +- 3 files changed, 3055 insertions(+), 2 deletions(-) create mode 100644 tests/sessions/replay_consistency/session_memory_summary_mutation_report.json diff --git a/tests/sessions/replay_consistency/README.md b/tests/sessions/replay_consistency/README.md index 0eb31e9a..4ef3502e 100644 --- a/tests/sessions/replay_consistency/README.md +++ b/tests/sessions/replay_consistency/README.md @@ -31,4 +31,5 @@ Diff reports include backend statuses, normal replay false-positive counts, mutation detection summaries, and structured diff entries with case, backend, session, event, memory, summary, path, left, and right fields. Tests write runtime reports to `tmp_path`; the repository root JSON is only a deterministic -schema example. +schema example. The checked-in mutation example lives at +`tests/sessions/replay_consistency/session_memory_summary_mutation_report.json`. diff --git a/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json b/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json new file mode 100644 index 00000000..3eca0587 --- /dev/null +++ b/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json @@ -0,0 +1,3048 @@ +{ + "allowed_diff_count": 0, + "allowed_diffs": [], + "backend_pairs": [], + "backend_statuses": [ + { + "name": "inmemory", + "reason": "", + "status": "ok" + }, + { + "name": "sqlite", + "reason": "", + "status": "ok" + }, + { + "name": "external_sql", + "reason": "TRPC_AGENT_REPLAY_SQL_URL is not set", + "status": "skipped" + }, + { + "name": "redis", + "reason": "TRPC_AGENT_REPLAY_REDIS_URL is not set", + "status": "skipped" + } + ], + "case_count": 0, + "cases": [], + "diffs": [ + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "single_turn_text-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-single-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": {}, + "tag": null, + "text": "Hello! Replay is deterministic.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "single_turn_text-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-single-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Hello replay bot.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 6, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[6]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "multi_turn_append_order-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-order-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "First question about replay order.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "tool", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "tool_call_roundtrip-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-tool-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "What is the weather in Beijing?", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 1, + "left": "Beijing", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_args", + "path": "events[1].function_calls[0].args.city", + "reason": "", + "right": "Shanghai", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 2, + "left": 25, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_response", + "path": "events[2].function_responses[0].response.temperature", + "reason": "", + "right": 30, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 1, + "left": "scoped_state_overwrite-event-01", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].event_id", + "reason": "", + "right": "scoped_state_overwrite-event-02", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 3, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[3]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "scoped_state_overwrite-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-state-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": { + "counter": 1 + }, + "tag": null, + "text": "Start scoped state test.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": null, + "left": "gold", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_state_value", + "path": "state.user:tier", + "reason": "", + "right": "silver", + "right_backend": "sqlite", + "section": "state", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 6, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[6]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "memory_preference_search-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-memory-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "I prefer tea in the morning.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": null, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "drop_memory", + "path": "memories[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": null, + "left": "Hiking preference noted for weekends.", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "change_memory_text_or_metadata", + "path": "memories[0].text", + "reason": "", + "right": "I prefer coffee in the morning.", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "memory_multi_session_isolation-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-isolation-a-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": {}, + "tag": null, + "text": "I will remember jasmine tea and lake hiking for User A.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "memory_multi_session_isolation-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-isolation-a-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "User A likes jasmine tea and hiking near lakes.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": null, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "drop_memory", + "path": "memories[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": null, + "left": "I will remember jasmine tea and lake hiking for User A.", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "change_memory_text_or_metadata", + "path": "memories[0].text", + "reason": "", + "right": "I prefer coffee in the morning.", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 1, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 3, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[3]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 3, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 0, + "manager_compressed_event_count": 3, + "manager_session_id": "session-007", + "original_event_count": 8, + "session_id": "session-007", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events" + }, + "text": "summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-007", + "summary_id": "summary:session-007:latest" + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": null, + "left": "summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-007", + "summary_id": "summary:session-007:latest" + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": null, + "left": "session-007", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-007", + "summary_id": "summary:session-007:latest" + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "summary_update_overwrite-event-05", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-summary-update-3", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": {}, + "tag": null, + "text": "Checklist and owners are now included.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 2, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 0, + "manager_compressed_event_count": 2, + "manager_session_id": "session-008", + "original_event_count": 4, + "session_id": "session-008", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events" + }, + "text": "summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-008", + "summary_id": "summary:session-008:latest" + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": null, + "left": "summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-008", + "summary_id": "summary:session-008:latest" + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": null, + "left": "session-008", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-008", + "summary_id": "summary:session-008:latest" + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 1, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 4, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 4, + "manager_compressed_event_count": 3, + "manager_session_id": "session-009", + "original_event_count": 6, + "session_id": "session-009", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events" + }, + "text": "summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-009", + "summary_id": "summary:session-009:latest" + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": null, + "left": "summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-009", + "summary_id": "summary:session-009:latest" + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": null, + "left": "session-009", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-009", + "summary_id": "summary:session-009:latest" + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 1, + "left": "duplicate_or_error_recovery-event-01", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].event_id", + "reason": "", + "right": "duplicate_or_error_recovery-event-02", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 5, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[5]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "duplicate_or_error_recovery-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-duplicate-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Duplicate content check begins.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 3, + "left": "RETRYABLE_BACKEND_ERROR", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_error_code", + "path": "events[3].error_code", + "reason": "", + "right": "WRONG_ERROR_CODE", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 4, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "duplicate_or_error_recovery-event-04", + "filter_key": "retry", + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-duplicate-4", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 4, + "state_delta": {}, + "tag": "retry-recovery", + "text": "Recovery succeeded after retry.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_recovery_event", + "path": "events[4]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "tool", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "serialization_order_nested_payload-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-nested-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Build a nested itinerary payload for Beijing.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 1, + "left": "Beijing", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_args", + "path": "events[1].function_calls[0].args.city", + "reason": "", + "right": "Shanghai", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 2, + "left": 21, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_response", + "path": "events[2].function_responses[0].response.temperature", + "reason": "", + "right": 30, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "list_sessions_consistency-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-list-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": { + "session_label": "list-check-updated" + }, + "tag": null, + "text": "The session list should include this normalized state.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "list_sessions_consistency-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-list-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Create a session that can be listed consistently.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "state_temp_key_ignored_but_persistent_key_compared-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-state-temp-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": { + "session_counter": 2 + }, + "tag": null, + "text": "Persistent state is updated and temp state is not stored.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "state_temp_key_ignored_but_persistent_key_compared-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-state-temp-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": { + "preference": "oolong tea", + "user:tier": "platinum" + }, + "tag": null, + "text": "Remember my persistent state but not the trace id.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": null, + "left": "platinum", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_state_value", + "path": "state.user:tier", + "reason": "", + "right": "silver", + "right_backend": "sqlite", + "section": "state", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 1, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 4, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 2, + "manager_compressed_event_count": 3, + "manager_session_id": "session-014", + "original_event_count": 4, + "session_id": "session-014", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events" + }, + "text": "summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-014", + "summary_id": "summary:session-014:latest" + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": null, + "left": "summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-014", + "summary_id": "summary:session-014:latest" + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": null, + "left": "session-014", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-014", + "summary_id": "summary:session-014:latest" + } + ], + "false_positive_summary": { + "normal_case_count": 0, + "unexpected_diff_count": 0 + }, + "generated_at": "deterministic", + "generated_by": "tests/sessions/test_replay_consistency.py", + "mutation_summary": { + "detected_count": 66, + "mutation_count": 66, + "undetected_mutations": [] + }, + "schema_version": 1, + "unallowed_diff_count": 66, + "unallowed_diffs": [ + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "single_turn_text-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-single-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": {}, + "tag": null, + "text": "Hello! Replay is deterministic.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "single_turn_text-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-single-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Hello replay bot.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 6, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[6]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "multi_turn_append_order-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-order-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "First question about replay order.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "tool", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "tool_call_roundtrip-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-tool-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "What is the weather in Beijing?", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 1, + "left": "Beijing", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_args", + "path": "events[1].function_calls[0].args.city", + "reason": "", + "right": "Shanghai", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 2, + "left": 25, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_response", + "path": "events[2].function_responses[0].response.temperature", + "reason": "", + "right": 30, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 1, + "left": "scoped_state_overwrite-event-01", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].event_id", + "reason": "", + "right": "scoped_state_overwrite-event-02", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 3, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[3]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "scoped_state_overwrite-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-state-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": { + "counter": 1 + }, + "tag": null, + "text": "Start scoped state test.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": null, + "left": "gold", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_state_value", + "path": "state.user:tier", + "reason": "", + "right": "silver", + "right_backend": "sqlite", + "section": "state", + "session_id": "session-004", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 6, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[6]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "memory_preference_search-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-memory-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "I prefer tea in the morning.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": null, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "drop_memory", + "path": "memories[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": null, + "left": "Hiking preference noted for weekends.", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "change_memory_text_or_metadata", + "path": "memories[0].text", + "reason": "", + "right": "I prefer coffee in the morning.", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-005", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "memory_multi_session_isolation-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-isolation-a-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": {}, + "tag": null, + "text": "I will remember jasmine tea and lake hiking for User A.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "memory_multi_session_isolation-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-isolation-a-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "User A likes jasmine tea and hiking near lakes.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": null, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "drop_memory", + "path": "memories[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": null, + "left": "I will remember jasmine tea and lake hiking for User A.", + "left_backend": "inmemory", + "memory_index": 0, + "mutation": "change_memory_text_or_metadata", + "path": "memories[0].text", + "reason": "", + "right": "I prefer coffee in the morning.", + "right_backend": "sqlite", + "section": "memories", + "session_id": "session-006-a", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 1, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 3, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[3]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 3, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 0, + "manager_compressed_event_count": 3, + "manager_session_id": "session-007", + "original_event_count": 8, + "session_id": "session-007", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events" + }, + "text": "summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-007", + "summary_id": "summary:session-007:latest" + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": null, + "left": "summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-007", + "summary_id": "summary:session-007:latest" + }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": null, + "left": "session-007", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-007", + "summary_id": "summary:session-007:latest" + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "summary_update_overwrite-event-05", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-summary-update-3", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": {}, + "tag": null, + "text": "Checklist and owners are now included.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 2, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 0, + "manager_compressed_event_count": 2, + "manager_session_id": "session-008", + "original_event_count": 4, + "session_id": "session-008", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events" + }, + "text": "summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-008", + "summary_id": "summary:session-008:latest" + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": null, + "left": "summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-008", + "summary_id": "summary:session-008:latest" + }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": null, + "left": "session-008", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-008", + "summary_id": "summary:session-008:latest" + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 1, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 4, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 4, + "manager_compressed_event_count": 3, + "manager_session_id": "session-009", + "original_event_count": 6, + "session_id": "session-009", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events" + }, + "text": "summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-009", + "summary_id": "summary:session-009:latest" + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": null, + "left": "summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-009", + "summary_id": "summary:session-009:latest" + }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": null, + "left": "session-009", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-009", + "summary_id": "summary:session-009:latest" + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 1, + "left": "duplicate_or_error_recovery-event-01", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].event_id", + "reason": "", + "right": "duplicate_or_error_recovery-event-02", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 5, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[5]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "duplicate_or_error_recovery-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-duplicate-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Duplicate content check begins.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 3, + "left": "RETRYABLE_BACKEND_ERROR", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_error_code", + "path": "events[3].error_code", + "reason": "", + "right": "WRONG_ERROR_CODE", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 4, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "duplicate_or_error_recovery-event-04", + "filter_key": "retry", + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-duplicate-4", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 4, + "state_delta": {}, + "tag": "retry-recovery", + "text": "Recovery succeeded after retry.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_recovery_event", + "path": "events[4]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 1, + "left": "assistant", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "tool", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "serialization_order_nested_payload-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-nested-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Build a nested itinerary payload for Beijing.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 1, + "left": "Beijing", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_args", + "path": "events[1].function_calls[0].args.city", + "reason": "", + "right": "Shanghai", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 2, + "left": 21, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_tool_response", + "path": "events[2].function_responses[0].response.temperature", + "reason": "", + "right": 30, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "list_sessions_consistency-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-list-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": { + "session_label": "list-check-updated" + }, + "tag": null, + "text": "The session list should include this normalized state.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "list_sessions_consistency-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-list-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Create a session that can be listed consistently.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 1, + "left": { + "author": "assistant", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "state_temp_key_ignored_but_persistent_key_compared-event-01", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-state-temp-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "model", + "stable_index": 1, + "state_delta": { + "session_counter": 2 + }, + "tag": null, + "text": "Persistent state is updated and temp state is not stored.", + "turn_complete": false + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1]", + "reason": "", + "right": "", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 0, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 2, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[2]", + "reason": "", + "right": { + "author": "user", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "state_temp_key_ignored_but_persistent_key_compared-event-00", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "inv-state-temp-1", + "is_summary_event": false, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": { + "preference": "oolong tea", + "user:tier": "platinum" + }, + "tag": null, + "text": "Remember my persistent state but not the trace id.", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": null, + "left": "platinum", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "change_state_value", + "path": "state.user:tier", + "reason": "", + "right": "silver", + "right_backend": "sqlite", + "section": "state", + "session_id": "session-013", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 1, + "left": "user", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_event", + "path": "events[1].author", + "reason": "", + "right": "assistant", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 0, + "left": "system", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "reorder_events", + "path": "events[0].author", + "reason": "", + "right": "user", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 4, + "left": "", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "duplicate_event", + "path": "events[4]", + "reason": "", + "right": { + "author": "system", + "branch": null, + "error_code": null, + "error_message": null, + "event_id": "normalized", + "filter_key": null, + "function_calls": [], + "function_responses": [], + "invocation_id": "summary", + "is_summary_event": true, + "model_visible": true, + "partial": false, + "role": "user", + "stable_index": 0, + "state_delta": {}, + "tag": null, + "text": "Previous conversation summary: summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events", + "turn_complete": false + }, + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": null, + "left": { + "metadata": { + "compressed_event_count": 4, + "has_summary": true, + "has_summary_timestamp": true, + "historical_event_count": 2, + "manager_compressed_event_count": 3, + "manager_session_id": "session-014", + "original_event_count": 4, + "session_id": "session-014", + "summary_event_count": 1, + "summary_event_text": "Previous conversation summary: summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events" + }, + "text": "summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events" + }, + "left_backend": "inmemory", + "memory_index": null, + "mutation": "drop_summary", + "path": "summary", + "reason": "", + "right": null, + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-014", + "summary_id": "summary:session-014:latest" + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": null, + "left": "summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "overwrite_summary_with_stale_text", + "path": "summary.text", + "reason": "", + "right": "summary(session-mutation): stale overwritten summary", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-014", + "summary_id": "summary:session-014:latest" + }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": null, + "left": "session-014", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "summary_wrong_session_id", + "path": "summary.metadata.session_id", + "reason": "", + "right": "wrong-session", + "right_backend": "sqlite", + "section": "summary", + "session_id": "session-014", + "summary_id": "summary:session-014:latest" + } + ] +} \ No newline at end of file diff --git a/tests/sessions/test_replay_consistency_mutations.py b/tests/sessions/test_replay_consistency_mutations.py index 05a60a91..96035c76 100644 --- a/tests/sessions/test_replay_consistency_mutations.py +++ b/tests/sessions/test_replay_consistency_mutations.py @@ -42,6 +42,10 @@ def _assert_diff_context( assert any("function_calls" in diff.path for diff in matching) if mutation == "change_tool_response": assert any("function_responses" in diff.path for diff in matching) + if mutation == "change_error_code": + assert any(diff.path.endswith(".error_code") for diff in matching) + if mutation == "drop_recovery_event": + assert any(diff.path.startswith("events[") and diff.right == "" for diff in matching) elif expected_section == "memories": assert any(diff.memory_index is not None or diff.section == "memories" for diff in matching) elif expected_section == "summary": @@ -55,7 +59,7 @@ def _assert_diff_context( @pytest.mark.asyncio -async def test_real_replay_snapshot_mutation_detection(tmp_path: Path): +async def test_real_replay_snapshot_mutation_detection_reports_precise_paths(tmp_path: Path): mutation_results: list[dict[str, Any]] = [] for case in replay_cases(): clean = await _run_real_inmemory_snapshot(tmp_path / f"real-mutation-{case.name}", case) From 7569c367b6200ec5ab58bf5bf585b141c724be91 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 22:12:26 +0800 Subject: [PATCH 8/9] Tighten replay mutation registry and reports --- session_memory_summary_diff_report.json | 61 +-- tests/sessions/replay_consistency/README.md | 5 +- .../sessions/replay_consistency/mutations.py | 11 +- tests/sessions/replay_consistency/report.py | 9 +- ...ession_memory_summary_mutation_report.json | 465 +++++++++++++++++- tests/sessions/test_replay_consistency.py | 4 +- .../test_replay_consistency_mutations.py | 42 +- 7 files changed, 551 insertions(+), 46 deletions(-) diff --git a/session_memory_summary_diff_report.json b/session_memory_summary_diff_report.json index 854769f7..5e5c772f 100644 --- a/session_memory_summary_diff_report.json +++ b/session_memory_summary_diff_report.json @@ -33,112 +33,112 @@ "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "single_turn_text", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "multi_turn_append_order", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "tool_call_roundtrip", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "scoped_state_overwrite", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "memory_preference_search", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "memory_multi_session_isolation", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "summary_generation", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "summary_update_overwrite", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "summary_with_event_truncation", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "duplicate_or_error_recovery", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "serialization_order_nested_payload", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "list_sessions_consistency", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "state_temp_key_ignored_but_persistent_key_compared", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 }, { "allowed_diff_count": 0, "backend_pair": "inmemory_vs_sqlite", "elapsed_ms": 0, "name": "summary_truncation_preserves_recent_context", - "unexpected_diff_count": 0, - "unallowed_diff_count": 0 + "unallowed_diff_count": 0, + "unexpected_diff_count": 0 } ], "diffs": [], @@ -155,5 +155,6 @@ }, "schema_version": 1, "unallowed_diff_count": 0, - "unallowed_diffs": [] -} + "unallowed_diffs": [], + "unexpected_diff_count": 0 +} \ No newline at end of file diff --git a/tests/sessions/replay_consistency/README.md b/tests/sessions/replay_consistency/README.md index 4ef3502e..42fb2266 100644 --- a/tests/sessions/replay_consistency/README.md +++ b/tests/sessions/replay_consistency/README.md @@ -32,4 +32,7 @@ mutation detection summaries, and structured diff entries with case, backend, session, event, memory, summary, path, left, and right fields. Tests write runtime reports to `tmp_path`; the repository root JSON is only a deterministic schema example. The checked-in mutation example lives at -`tests/sessions/replay_consistency/session_memory_summary_mutation_report.json`. +`tests/sessions/replay_consistency/session_memory_summary_mutation_report.json` +and is generated from the same registry that mutates event text, tool +arguments/results, state, memory text, summaries, and retry error/recovery +events on real normalized replay snapshots. diff --git a/tests/sessions/replay_consistency/mutations.py b/tests/sessions/replay_consistency/mutations.py index 21301e8c..59cd2156 100644 --- a/tests/sessions/replay_consistency/mutations.py +++ b/tests/sessions/replay_consistency/mutations.py @@ -30,6 +30,7 @@ MUTATION_SECTION = { "drop_event": "events", + "alter_event_text": "events", "reorder_event": "events", "reorder_events": "events", "duplicate_event": "events", @@ -40,6 +41,7 @@ "change_state": "state", "change_state_value": "state", "drop_memory": "memories", + "alter_memory_text": "memories", "change_memory_text": "memories", "change_memory_text_or_metadata": "memories", "drop_summary": "summary", @@ -51,13 +53,13 @@ def mutations_for_case(case: ReplayCase) -> list[str]: - mutations = ["drop_event", "reorder_events", "duplicate_event"] + mutations = ["drop_event", "reorder_events", "duplicate_event", "alter_event_text"] if case.name in {"tool_call_roundtrip", "serialization_order_nested_payload"}: mutations.extend(["change_tool_args", "change_tool_response"]) if case.name in {"scoped_state_overwrite", "state_temp_key_ignored_but_persistent_key_compared"}: mutations.append("change_state_value") if case.name in {"memory_preference_search", "memory_multi_session_isolation"}: - mutations.extend(["drop_memory", "change_memory_text_or_metadata"]) + mutations.extend(["drop_memory", "alter_memory_text"]) if case.name in { "summary_generation", "summary_update_overwrite", @@ -84,6 +86,9 @@ def _find_mutation_event( def mutate_snapshot(name: str, snapshot: dict[str, Any]) -> None: if name == "drop_event": del snapshot["events"][1] + elif name == "alter_event_text": + event = _find_mutation_event(name, snapshot, lambda item: item.get("text") is not None) + event["text"] = "mutated event text" elif name in {"reorder_event", "reorder_events"}: snapshot["events"][0], snapshot["events"][1] = snapshot["events"][1], snapshot["events"][0] elif name == "change_tool_args": @@ -108,7 +113,7 @@ def mutate_snapshot(name: str, snapshot: dict[str, Any]) -> None: raise AssertionError(f"{name} mutation target was not found") elif name == "drop_memory": del snapshot["memories"][0] - elif name in {"change_memory_text", "change_memory_text_or_metadata"}: + elif name in {"alter_memory_text", "change_memory_text", "change_memory_text_or_metadata"}: snapshot["memories"][0]["text"] = "I prefer coffee in the morning." elif name == "drop_summary": snapshot["summary"] = None diff --git a/tests/sessions/replay_consistency/report.py b/tests/sessions/replay_consistency/report.py index 7a3829b7..65d81170 100644 --- a/tests/sessions/replay_consistency/report.py +++ b/tests/sessions/replay_consistency/report.py @@ -49,6 +49,7 @@ def write_report( *, backend_statuses: list[dict[str, Any]] | None = None, mutation_results: list[dict[str, Any]] | None = None, + generated_at: str = "deterministic", ) -> dict[str, Any]: backend_pairs = [] cases: list[dict[str, Any]] = [] @@ -90,7 +91,10 @@ def write_report( undetected_mutations = [] detected_count = 0 for result in mutation_results: - result_diffs = result.get("diffs", []) + result_diffs = result.get("diffs") + if result_diffs is None: + first_diff = result.get("first_diff") + result_diffs = [first_diff] if first_diff is not None else [] mutation = result["mutation"] detected = bool(result.get("detected")) if detected: @@ -107,7 +111,7 @@ def write_report( report = { "schema_version": 1, - "generated_at": "deterministic", + "generated_at": generated_at, "generated_by": "tests/sessions/test_replay_consistency.py", "backend_pairs": backend_pairs, "backend_statuses": _backend_statuses(backend_pairs, backend_statuses), @@ -115,6 +119,7 @@ def write_report( "cases": cases, "allowed_diff_count": len(allowed_diffs), "unallowed_diff_count": len(unallowed_diffs), + "unexpected_diff_count": len(unallowed_diffs), "allowed_diffs": allowed_diffs, "unallowed_diffs": unallowed_diffs, "diffs": diffs, diff --git a/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json b/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json index 3eca0587..7e06d6f0 100644 --- a/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json +++ b/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json @@ -113,6 +113,22 @@ "session_id": "session-001", "summary_id": null }, + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 0, + "left": "Hello replay bot.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, { "allowed": false, "case_name": "multi_turn_append_order", @@ -180,6 +196,22 @@ "session_id": "session-002", "summary_id": null }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 0, + "left": "First question about replay order.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, { "allowed": false, "case_name": "tool_call_roundtrip", @@ -247,6 +279,22 @@ "session_id": "session-003", "summary_id": null }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 0, + "left": "What is the weather in Beijing?", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, { "allowed": false, "case_name": "tool_call_roundtrip", @@ -348,6 +396,22 @@ "session_id": "session-004", "summary_id": null }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 0, + "left": "Start scoped state test.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, { "allowed": false, "case_name": "scoped_state_overwrite", @@ -431,6 +495,22 @@ "session_id": "session-005", "summary_id": null }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 0, + "left": "I prefer tea in the morning.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, { "allowed": false, "case_name": "memory_preference_search", @@ -454,7 +534,7 @@ "left": "Hiking preference noted for weekends.", "left_backend": "inmemory", "memory_index": 0, - "mutation": "change_memory_text_or_metadata", + "mutation": "alter_memory_text", "path": "memories[0].text", "reason": "", "right": "I prefer coffee in the morning.", @@ -549,6 +629,22 @@ "session_id": "session-006-a", "summary_id": null }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 0, + "left": "User A likes jasmine tea and hiking near lakes.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, { "allowed": false, "case_name": "memory_multi_session_isolation", @@ -572,7 +668,7 @@ "left": "I will remember jasmine tea and lake hiking for User A.", "left_backend": "inmemory", "memory_index": 0, - "mutation": "change_memory_text_or_metadata", + "mutation": "alter_memory_text", "path": "memories[0].text", "reason": "", "right": "I prefer coffee in the morning.", @@ -648,6 +744,22 @@ "session_id": "session-007", "summary_id": null }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 0, + "left": "Previous conversation summary: summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, { "allowed": false, "case_name": "summary_generation", @@ -796,6 +908,22 @@ "session_id": "session-008", "summary_id": null }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 0, + "left": "Previous conversation summary: summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, { "allowed": false, "case_name": "summary_update_overwrite", @@ -925,6 +1053,22 @@ "session_id": "session-009", "summary_id": null }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 0, + "left": "Previous conversation summary: summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, { "allowed": false, "case_name": "summary_with_event_truncation", @@ -1054,6 +1198,22 @@ "session_id": "session-010", "summary_id": null }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 0, + "left": "Duplicate content check begins.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, { "allowed": false, "case_name": "duplicate_or_error_recovery", @@ -1172,6 +1332,22 @@ "session_id": "session-011", "summary_id": null }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 0, + "left": "Build a nested itinerary payload for Beijing.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, { "allowed": false, "case_name": "serialization_order_nested_payload", @@ -1292,6 +1468,22 @@ "session_id": "session-012", "summary_id": null }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 0, + "left": "Create a session that can be listed consistently.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, { "allowed": false, "case_name": "state_temp_key_ignored_but_persistent_key_compared", @@ -1383,6 +1575,22 @@ "session_id": "session-013", "summary_id": null }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 0, + "left": "Remember my persistent state but not the trace id.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, { "allowed": false, "case_name": "state_temp_key_ignored_but_persistent_key_compared", @@ -1466,6 +1674,22 @@ "session_id": "session-014", "summary_id": null }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 0, + "left": "Previous conversation summary: summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, { "allowed": false, "case_name": "summary_truncation_preserves_recent_context", @@ -1536,12 +1760,12 @@ "generated_at": "deterministic", "generated_by": "tests/sessions/test_replay_consistency.py", "mutation_summary": { - "detected_count": 66, - "mutation_count": 66, + "detected_count": 80, + "mutation_count": 80, "undetected_mutations": [] }, "schema_version": 1, - "unallowed_diff_count": 66, + "unallowed_diff_count": 80, "unallowed_diffs": [ { "allowed": false, @@ -1629,6 +1853,22 @@ "session_id": "session-001", "summary_id": null }, + { + "allowed": false, + "case_name": "single_turn_text", + "event_index": 0, + "left": "Hello replay bot.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-001", + "summary_id": null + }, { "allowed": false, "case_name": "multi_turn_append_order", @@ -1696,6 +1936,22 @@ "session_id": "session-002", "summary_id": null }, + { + "allowed": false, + "case_name": "multi_turn_append_order", + "event_index": 0, + "left": "First question about replay order.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-002", + "summary_id": null + }, { "allowed": false, "case_name": "tool_call_roundtrip", @@ -1763,6 +2019,22 @@ "session_id": "session-003", "summary_id": null }, + { + "allowed": false, + "case_name": "tool_call_roundtrip", + "event_index": 0, + "left": "What is the weather in Beijing?", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-003", + "summary_id": null + }, { "allowed": false, "case_name": "tool_call_roundtrip", @@ -1864,6 +2136,22 @@ "session_id": "session-004", "summary_id": null }, + { + "allowed": false, + "case_name": "scoped_state_overwrite", + "event_index": 0, + "left": "Start scoped state test.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-004", + "summary_id": null + }, { "allowed": false, "case_name": "scoped_state_overwrite", @@ -1947,6 +2235,22 @@ "session_id": "session-005", "summary_id": null }, + { + "allowed": false, + "case_name": "memory_preference_search", + "event_index": 0, + "left": "I prefer tea in the morning.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-005", + "summary_id": null + }, { "allowed": false, "case_name": "memory_preference_search", @@ -1970,7 +2274,7 @@ "left": "Hiking preference noted for weekends.", "left_backend": "inmemory", "memory_index": 0, - "mutation": "change_memory_text_or_metadata", + "mutation": "alter_memory_text", "path": "memories[0].text", "reason": "", "right": "I prefer coffee in the morning.", @@ -2065,6 +2369,22 @@ "session_id": "session-006-a", "summary_id": null }, + { + "allowed": false, + "case_name": "memory_multi_session_isolation", + "event_index": 0, + "left": "User A likes jasmine tea and hiking near lakes.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-006-a", + "summary_id": null + }, { "allowed": false, "case_name": "memory_multi_session_isolation", @@ -2088,7 +2408,7 @@ "left": "I will remember jasmine tea and lake hiking for User A.", "left_backend": "inmemory", "memory_index": 0, - "mutation": "change_memory_text_or_metadata", + "mutation": "alter_memory_text", "path": "memories[0].text", "reason": "", "right": "I prefer coffee in the morning.", @@ -2164,6 +2484,22 @@ "session_id": "session-007", "summary_id": null }, + { + "allowed": false, + "case_name": "summary_generation", + "event_index": 0, + "left": "Previous conversation summary: summary(session-007): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I want museums, vegetarian food, and tea houses. | assistant=I will include museums, vegetarian restaurants, and tea houses. | user=Budget is mid range and I prefer metro travel. | assistant=I will keep it mid range with metro routes. | facts=6-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-007", + "summary_id": null + }, { "allowed": false, "case_name": "summary_generation", @@ -2312,6 +2648,22 @@ "session_id": "session-008", "summary_id": null }, + { + "allowed": false, + "case_name": "summary_update_overwrite", + "event_index": 0, + "left": "Previous conversation summary: summary(session-008): system=Previous conversation summary: summary(session-008): user=Plan a product launch for replay. | assistant=We need goals, audience, and timing. | user=Audience is developers and operators. | facts=3-events | assistant=I will target developers and operators. | user=Add a release checklist and owner names. | facts=3-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-008", + "summary_id": null + }, { "allowed": false, "case_name": "summary_update_overwrite", @@ -2441,6 +2793,22 @@ "session_id": "session-009", "summary_id": null }, + { + "allowed": false, + "case_name": "summary_with_event_truncation", + "event_index": 0, + "left": "Previous conversation summary: summary(session-009): user=Help me plan a trip to Shanghai. | assistant=Sure! When would you like to go? | user=I prefer spring with museums. | assistant=Spring museums are good for Shanghai. | facts=4-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-009", + "summary_id": null + }, { "allowed": false, "case_name": "summary_with_event_truncation", @@ -2570,6 +2938,22 @@ "session_id": "session-010", "summary_id": null }, + { + "allowed": false, + "case_name": "duplicate_or_error_recovery", + "event_index": 0, + "left": "Duplicate content check begins.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-010", + "summary_id": null + }, { "allowed": false, "case_name": "duplicate_or_error_recovery", @@ -2688,6 +3072,22 @@ "session_id": "session-011", "summary_id": null }, + { + "allowed": false, + "case_name": "serialization_order_nested_payload", + "event_index": 0, + "left": "Build a nested itinerary payload for Beijing.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-011", + "summary_id": null + }, { "allowed": false, "case_name": "serialization_order_nested_payload", @@ -2808,6 +3208,22 @@ "session_id": "session-012", "summary_id": null }, + { + "allowed": false, + "case_name": "list_sessions_consistency", + "event_index": 0, + "left": "Create a session that can be listed consistently.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-012", + "summary_id": null + }, { "allowed": false, "case_name": "state_temp_key_ignored_but_persistent_key_compared", @@ -2899,6 +3315,22 @@ "session_id": "session-013", "summary_id": null }, + { + "allowed": false, + "case_name": "state_temp_key_ignored_but_persistent_key_compared", + "event_index": 0, + "left": "Remember my persistent state but not the trace id.", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-013", + "summary_id": null + }, { "allowed": false, "case_name": "state_temp_key_ignored_but_persistent_key_compared", @@ -2982,6 +3414,22 @@ "session_id": "session-014", "summary_id": null }, + { + "allowed": false, + "case_name": "summary_truncation_preserves_recent_context", + "event_index": 0, + "left": "Previous conversation summary: summary(session-014): user=Start a research plan for Hangzhou. | assistant=We will track museums, tea, and quiet walks. | facts=2-events", + "left_backend": "inmemory", + "memory_index": null, + "mutation": "alter_event_text", + "path": "events[0].text", + "reason": "", + "right": "mutated event text", + "right_backend": "sqlite", + "section": "events", + "session_id": "session-014", + "summary_id": null + }, { "allowed": false, "case_name": "summary_truncation_preserves_recent_context", @@ -3044,5 +3492,6 @@ "session_id": "session-014", "summary_id": "summary:session-014:latest" } - ] + ], + "unexpected_diff_count": 80 } \ No newline at end of file diff --git a/tests/sessions/test_replay_consistency.py b/tests/sessions/test_replay_consistency.py index 1762aaf1..8aaef4c5 100644 --- a/tests/sessions/test_replay_consistency.py +++ b/tests/sessions/test_replay_consistency.py @@ -515,6 +515,7 @@ def test_report_contract(tmp_path: Path): "diffs": [allowed_diff, diff], } ], + generated_at="unit-test", ) loaded = json.loads(path.read_text(encoding="utf-8")) assert loaded == report @@ -522,11 +523,12 @@ def test_report_contract(tmp_path: Path): for field in DiffEntry.__dataclass_fields__: assert field in serialized assert loaded["schema_version"] == 1 - assert loaded["generated_at"] == "deterministic" + assert loaded["generated_at"] == "unit-test" assert loaded["backend_pairs"] == ["inmemory_vs_sqlite"] assert loaded["backend_statuses"] assert loaded["allowed_diff_count"] == 1 assert loaded["unallowed_diff_count"] == 1 + assert loaded["unexpected_diff_count"] == 1 assert len(loaded["allowed_diffs"]) == 1 assert len(loaded["unallowed_diffs"]) == 1 assert len(loaded["diffs"]) == 2 diff --git a/tests/sessions/test_replay_consistency_mutations.py b/tests/sessions/test_replay_consistency_mutations.py index 96035c76..4c5a1824 100644 --- a/tests/sessions/test_replay_consistency_mutations.py +++ b/tests/sessions/test_replay_consistency_mutations.py @@ -20,6 +20,30 @@ from .test_replay_consistency import _run_real_inmemory_snapshot +REQUIRED_REAL_MUTATIONS = { + "drop_event", + "reorder_events", + "duplicate_event", + "alter_event_text", + "change_tool_args", + "change_tool_response", + "change_state_value", + "drop_memory", + "alter_memory_text", + "drop_summary", + "overwrite_summary_with_stale_text", + "summary_wrong_session_id", + "change_error_code", + "drop_recovery_event", +} + + +def test_real_mutation_registry_covers_required_mutations(): + assert REQUIRED_REAL_MUTATIONS <= set(MUTATION_SECTION) + seen_mutations = {mutation for case in replay_cases() for mutation in mutations_for_case(case)} + assert REQUIRED_REAL_MUTATIONS <= seen_mutations + + def _assert_diff_context( *, case_name: str, @@ -42,6 +66,8 @@ def _assert_diff_context( assert any("function_calls" in diff.path for diff in matching) if mutation == "change_tool_response": assert any("function_responses" in diff.path for diff in matching) + if mutation == "alter_event_text": + assert any(diff.path.endswith(".text") for diff in matching) if mutation == "change_error_code": assert any(diff.path.endswith(".error_code") for diff in matching) if mutation == "drop_recovery_event": @@ -79,17 +105,31 @@ async def test_real_replay_snapshot_mutation_detection_reports_precise_paths(tmp "case_name": case.name, "mutation": mutation, "detected": bool(unallowed_diffs(diffs)), + "diff_count": len(diffs), + "first_diff": diffs[0], "diffs": diffs, } ) report = write_report( - tmp_path / "mutation_report.json", + tmp_path / "session_memory_summary_mutation_report.json", [], mutation_results=mutation_results, ) + assert report["schema_version"] == 1 + assert report["mutation_summary"]["mutation_count"] > 0 assert report["mutation_summary"]["mutation_count"] == report["mutation_summary"]["detected_count"] assert report["mutation_summary"]["undetected_mutations"] == [] + assert any( + diff.get("mutation") == "summary_wrong_session_id" + and diff["path"] == "summary.metadata.session_id" + for diff in report["unallowed_diffs"] + ) + assert any( + diff.get("mutation") in {"change_tool_args", "change_tool_response"} + and diff["event_index"] is not None + for diff in report["unallowed_diffs"] + ) @pytest.mark.asyncio From 04cf9c6ca80c53cff0733ab3d5a481359782f5f7 Mon Sep 17 00:00:00 2001 From: yaoyaoshiguonan Date: Sat, 4 Jul 2026 22:40:22 +0800 Subject: [PATCH 9/9] Harden replay mutation evidence --- session_memory_summary_diff_report.json | 1 + tests/sessions/replay_consistency/report.py | 4 +- ...ession_memory_summary_mutation_report.json | 3 +- tests/sessions/test_replay_consistency.py | 157 +++++++++++++++++- 4 files changed, 157 insertions(+), 8 deletions(-) diff --git a/session_memory_summary_diff_report.json b/session_memory_summary_diff_report.json index 5e5c772f..48ab04d0 100644 --- a/session_memory_summary_diff_report.json +++ b/session_memory_summary_diff_report.json @@ -153,6 +153,7 @@ "mutation_count": 0, "undetected_mutations": [] }, + "report_kind": "normal_replay", "schema_version": 1, "unallowed_diff_count": 0, "unallowed_diffs": [], diff --git a/tests/sessions/replay_consistency/report.py b/tests/sessions/replay_consistency/report.py index 65d81170..536c5191 100644 --- a/tests/sessions/replay_consistency/report.py +++ b/tests/sessions/replay_consistency/report.py @@ -88,6 +88,7 @@ def write_report( unallowed_diffs.append(serialized) mutation_results = mutation_results or [] + report_kind = "mutation_replay" if mutation_results else "normal_replay" undetected_mutations = [] detected_count = 0 for result in mutation_results: @@ -111,6 +112,7 @@ def write_report( report = { "schema_version": 1, + "report_kind": report_kind, "generated_at": generated_at, "generated_by": "tests/sessions/test_replay_consistency.py", "backend_pairs": backend_pairs, @@ -119,7 +121,7 @@ def write_report( "cases": cases, "allowed_diff_count": len(allowed_diffs), "unallowed_diff_count": len(unallowed_diffs), - "unexpected_diff_count": len(unallowed_diffs), + "unexpected_diff_count": normal_unexpected_count, "allowed_diffs": allowed_diffs, "unallowed_diffs": unallowed_diffs, "diffs": diffs, diff --git a/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json b/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json index 7e06d6f0..7a465e12 100644 --- a/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json +++ b/tests/sessions/replay_consistency/session_memory_summary_mutation_report.json @@ -1764,6 +1764,7 @@ "mutation_count": 80, "undetected_mutations": [] }, + "report_kind": "mutation_replay", "schema_version": 1, "unallowed_diff_count": 80, "unallowed_diffs": [ @@ -3493,5 +3494,5 @@ "summary_id": "summary:session-014:latest" } ], - "unexpected_diff_count": 80 + "unexpected_diff_count": 0 } \ No newline at end of file diff --git a/tests/sessions/test_replay_consistency.py b/tests/sessions/test_replay_consistency.py index 8aaef4c5..c3a1d01f 100644 --- a/tests/sessions/test_replay_consistency.py +++ b/tests/sessions/test_replay_consistency.py @@ -31,8 +31,10 @@ from .replay_consistency.comparator import compare_snapshot_pair from .replay_consistency.comparator import recursive_diff from .replay_consistency.comparator import unallowed_diffs +from .replay_consistency.mutations import MUTATION_SECTION from .replay_consistency.mutations import SYNTHETIC_MUTATIONS as MUTATIONS from .replay_consistency.mutations import mutate_snapshot +from .replay_consistency.mutations import mutations_for_case from .replay_consistency.normalizer import Snapshot from .replay_consistency.normalizer import normalize_snapshot from .replay_consistency.report import write_report @@ -450,7 +452,12 @@ async def test_replay_consistency_inmemory_vs_sqlite(tmp_path: Path): write_report(report_path, comparison_results) pytest.fail(f"Replay diff detected for {case.name}: {[asdict(diff) for diff in unexpected]}") - write_report(report_path, comparison_results) + report = write_report(report_path, comparison_results) + assert report["report_kind"] == "normal_replay" + assert report["false_positive_summary"] == { + "normal_case_count": len(cases), + "unexpected_diff_count": 0, + } def test_replay_case_count_and_names(): @@ -461,11 +468,9 @@ def test_replay_case_count_and_names(): def test_replay_case_manifest_matches_registry(): manifest_path = Path(__file__).parent / "replay_cases" / "session_memory_summary_replay_cases.jsonl" - manifest_cases = [ - json.loads(line) - for line in manifest_path.read_text(encoding="utf-8").splitlines() - if line.strip() - ] + manifest_lines = manifest_path.read_text(encoding="utf-8").splitlines() + assert manifest_lines + manifest_cases = [json.loads(line) for line in manifest_lines] assert [case["name"] for case in manifest_cases] == [case.name for case in replay_cases()] for case in manifest_cases: assert case["description"] @@ -523,6 +528,7 @@ def test_report_contract(tmp_path: Path): for field in DiffEntry.__dataclass_fields__: assert field in serialized assert loaded["schema_version"] == 1 + assert loaded["report_kind"] == "normal_replay" assert loaded["generated_at"] == "unit-test" assert loaded["backend_pairs"] == ["inmemory_vs_sqlite"] assert loaded["backend_statuses"] @@ -548,6 +554,145 @@ def test_report_contract(tmp_path: Path): } +def test_checked_in_reports_are_valid_json(): + report_path = ( + Path(__file__).parent + / "replay_consistency" + / "session_memory_summary_mutation_report.json" + ) + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["schema_version"] == 1 + assert report["report_kind"] == "mutation_replay" + assert report["unexpected_diff_count"] == 0 + assert report["false_positive_summary"] == { + "normal_case_count": 0, + "unexpected_diff_count": 0, + } + mutation_summary = report["mutation_summary"] + assert mutation_summary["mutation_count"] == mutation_summary["detected_count"] + assert mutation_summary["undetected_mutations"] == [] + assert any( + diff.get("mutation") == "summary_wrong_session_id" + and diff["section"] == "summary" + and diff["path"] == "summary.metadata.session_id" + for diff in report["unallowed_diffs"] + ) + assert any( + diff.get("mutation") == "change_tool_response" + and "function_responses" in diff["path"] + for diff in report["unallowed_diffs"] + ) + + +def _assert_real_mutation_diff_context( + *, + case: ReplayCase, + mutation: str, + clean_snapshot: dict[str, Any], + diffs: list[DiffEntry], +) -> list[DiffEntry]: + unexpected = unallowed_diffs(diffs) + assert unexpected, f"{case.name}/{mutation} was not detected" + + for diff in unexpected: + assert diff.case_name == case.name + assert diff.session_id == clean_snapshot["session_id"] + assert diff.section + assert diff.path + assert hasattr(diff, "left") + assert hasattr(diff, "right") + + expected_section = MUTATION_SECTION[mutation] + matching = [diff for diff in unexpected if diff.section == expected_section] + assert matching, f"{case.name}/{mutation} did not report section {expected_section}" + report_diffs = matching + + if mutation == "drop_summary": + report_diffs = [diff for diff in matching if diff.path == "summary"] + assert report_diffs + elif mutation == "overwrite_summary_with_stale_text": + report_diffs = [diff for diff in matching if diff.path == "summary.text"] + assert report_diffs + elif mutation == "summary_wrong_session_id": + report_diffs = [ + diff + for diff in matching + if diff.path == "summary.metadata.session_id" + ] + assert report_diffs + + if mutation == "change_tool_args": + report_diffs = [ + diff + for diff in matching + if diff.event_index is not None and "function_calls" in diff.path + ] + assert report_diffs + elif mutation == "change_tool_response": + report_diffs = [ + diff + for diff in matching + if diff.event_index is not None and "function_responses" in diff.path + ] + assert report_diffs + elif expected_section == "memories": + assert all(diff.section == "memories" for diff in matching) + report_diffs = [ + diff + for diff in matching + if diff.memory_index is not None or diff.path == "memories" + ] + assert report_diffs + elif expected_section == "state": + assert all(diff.section == "state" and diff.path.startswith("state.") for diff in matching) + return report_diffs + + +@pytest.mark.asyncio +async def test_real_replay_snapshot_mutation_detection_reports_precise_paths(tmp_path: Path): + mutation_results: list[dict[str, Any]] = [] + + for case in replay_cases(): + clean = await _run_real_inmemory_snapshot(tmp_path / f"real-mutation-{case.name}", case) + for mutation in mutations_for_case(case): + mutated = copy.deepcopy(clean) + mutated["backend"] = "sqlite" + mutate_snapshot(mutation, mutated) + diffs = recursive_diff(clean, mutated) + unexpected = unallowed_diffs(diffs) + report_diffs = _assert_real_mutation_diff_context( + case=case, + mutation=mutation, + clean_snapshot=clean, + diffs=diffs, + ) + mutation_results.append( + { + "case_name": case.name, + "mutation": mutation, + "detected": bool(unexpected), + "diff_count": len(unexpected), + "diffs": [report_diffs[0]], + } + ) + + report = write_report( + tmp_path / "session_memory_summary_mutation_report.json", + [], + mutation_results=mutation_results, + ) + mutation_summary = report["mutation_summary"] + assert report["report_kind"] == "mutation_replay" + assert report["unexpected_diff_count"] == 0 + assert report["false_positive_summary"] == { + "normal_case_count": 0, + "unexpected_diff_count": 0, + } + assert mutation_summary["mutation_count"] > 0 + assert mutation_summary["mutation_count"] == mutation_summary["detected_count"] + assert mutation_summary["undetected_mutations"] == [] + + def _clean_mutation_snapshot() -> dict[str, Any]: snapshot = Snapshot( backend="inmemory",