From da5026539d68b134c4513aed5bff5d5008bc5815 Mon Sep 17 00:00:00 2001 From: Reiase Date: Fri, 24 Jul 2026 23:17:52 +0800 Subject: [PATCH] Refactor persisting-capture to replace Vortex references with Lance - Updated all instances of "Vortex" to "Lance" in comments and documentation to reflect the new storage engine. - Adjusted file paths and references in the codebase to align with Lance's structure. - Enhanced README and justfile documentation to clarify testing and integration processes with Lance. - Removed obsolete package dependencies from Cargo.lock related to Vortex. These changes ensure consistency in terminology and improve clarity regarding the use of Lance as the primary storage engine. --- Cargo.lock | 1127 +---------------- README.md | 5 +- .../persisting-capture/src/conversion/mod.rs | 2 +- .../src/conversion/responses_stream.rs | 2 +- .../persisting-capture/src/engine/egress.rs | 2 +- .../persisting-capture/src/engine/prepare.rs | 2 +- .../src/engine/wire/story.rs | 2 +- crates/persisting-capture/src/reconcile.rs | 2 +- .../persisting-capture/src/session/index.rs | 2 +- .../persisting-capture/src/storage/convert.rs | 18 +- .../src/storage/dialogue/draft.rs | 2 +- .../src/storage/dialogue/view.rs | 2 +- .../src/storage/event_row.rs | 4 +- .../src/storage/lifecycle.rs | 2 +- .../src/storage/markdown_policy.rs | 2 +- crates/persisting-capture/src/storage/mod.rs | 2 +- .../src/storage/path_layout.rs | 26 +- .../persisting-capture/src/storage/record.rs | 2 +- .../src/storage/story_coords.rs | 32 +- .../src/storage/subagent_link/extract.rs | 3 +- .../src/storage/subagent_link/tests.rs | 8 +- .../capture/apps/claude/dialogue_extract.rs | 2 +- .../claude/fixtures/agent_tool_stream.sse | 2 +- .../fixtures/subagent_explore_flash.json | 2 +- .../capture/apps/claude/markdown_filter.rs | 2 +- .../tests/capture/apps/claude/mod.rs | 2 +- .../tests/capture/apps/claude/spawn_link.rs | 6 +- crates/persisting-cli/src/capture/mod.rs | 34 +- crates/persisting-cli/src/main.rs | 32 +- .../persisting-cli/src/trajectory_detail.rs | 2 +- .../persisting-cli/src/trajectory_format.rs | 41 +- crates/persisting-compute/Cargo.toml | 2 +- crates/persisting-compute/src/cli.rs | 12 +- crates/persisting-compute/src/lib.rs | 2 +- crates/persisting-compute/src/sink.rs | 2 +- crates/persisting-compute/src/sink_traj.rs | 14 +- crates/persisting-engine/Cargo.toml | 5 +- crates/persisting-engine/src/lib.rs | 2 +- .../src/trajectory/convert.rs | 54 +- .../src/trajectory/expand.rs | 34 +- .../persisting-engine/src/trajectory/judge.rs | 24 +- .../src/trajectory/judge_stats.rs | 4 +- .../src/trajectory/layers/judge_sidecar.rs | 114 +- .../src/trajectory/layers/manifest.rs | 38 +- .../src/trajectory/layers/mod.rs | 4 +- .../persisting-engine/src/trajectory/mod.rs | 130 +- .../src/trajectory/storage.rs | 64 +- .../trajectory/store/{vortex.rs => lance.rs} | 265 ++-- .../src/trajectory/store/mod.rs | 56 +- .../src/trajectory/store/rows.rs | 4 +- crates/persisting-proto/src/messages.rs | 12 +- docs/src/design/architecture.zh.md | 2 +- docs/src/design/capture.md | 88 +- docs/src/design/cli-capture.md | 38 +- docs/src/design/cli-search.md | 2 +- docs/src/design/cli-traj.md | 34 +- docs/src/design/cli.md | 12 +- docs/src/design/compute.md | 10 +- docs/src/design/index.md | 4 +- docs/src/design/index.zh.md | 4 +- .../references/transfer-queue-comparison.md | 2 +- docs/src/design/trajectory-format.md | 52 +- docs/src/design/trajectory.md | 196 +-- docs/src/guide/capture.md | 31 +- docs/src/guide/capture.zh.md | 44 +- docs/src/guide/compute.md | 2 +- docs/src/guide/compute.zh.md | 2 +- docs/src/index.md | 2 +- docs/src/index.zh.md | 2 +- examples/trajectory-tlv/README.md | 2 +- justfile | 282 +---- persisting/store/block.py | 5 +- persisting/store/block_store.py | 4 +- persisting/store/block_store_actor.py | 12 +- persisting/store/local_tensor.py | 20 +- persisting/store/tiered_loop.py | 4 +- scripts/generate_benchmark_data.py | 16 +- scripts/integration/README.md | 44 +- scripts/integration/_common.sh | 4 +- scripts/integration/capture_run_e2e.sh | 36 +- scripts/integration/capture_stress.sh | 6 +- scripts/integration/search_traj_e2e.sh | 189 +++ scripts/integration/traj_e2e.sh | 16 +- scripts/test_suite.sh | 165 +-- 84 files changed, 1259 insertions(+), 2289 deletions(-) rename crates/persisting-engine/src/trajectory/store/{vortex.rs => lance.rs} (68%) create mode 100755 scripts/integration/search_traj_e2e.sh diff --git a/Cargo.lock b/Cargo.lock index 03a768b..cd00efa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,12 +146,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "arcref" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28f6098a1e8ab66ff91324cce8fea6643101882cf7d09c85acdb1485ecf61e29" - [[package]] name = "arrayref" version = "0.3.9" @@ -413,49 +407,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "async-executor" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" -dependencies = [ - "async-task", - "concurrent-queue", - "fastrand", - "futures-lite", - "pin-project-lite", - "slab", -] - -[[package]] -name = "async-fs" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8034a681df4aed8b8edbd7fbe472401ecf009251c8b40556b304567052e294c5" -dependencies = [ - "async-lock", - "blocking", - "futures-lite", -] - -[[package]] -name = "async-io" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" -dependencies = [ - "autocfg", - "cfg-if 1.0.4", - "concurrent-queue", - "futures-io", - "futures-lite", - "parking", - "polling", - "rustix", - "slab", - "windows-sys 0.61.2", -] - [[package]] name = "async-lock" version = "3.4.2" @@ -467,35 +418,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "async-net" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" -dependencies = [ - "async-io", - "blocking", - "futures-lite", -] - -[[package]] -name = "async-process" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" -dependencies = [ - "async-channel", - "async-io", - "async-lock", - "async-signal", - "async-task", - "blocking", - "cfg-if 1.0.4", - "event-listener", - "futures-lite", - "rustix", -] - [[package]] name = "async-recursion" version = "1.1.1" @@ -507,52 +429,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "async-signal" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" -dependencies = [ - "async-io", - "async-lock", - "atomic-waker", - "cfg-if 1.0.4", - "futures-core", - "futures-io", - "rustix", - "signal-hook-registry", - "slab", - "windows-sys 0.61.2", -] - -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "async-task" -version = "4.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" - [[package]] name = "async-trait" version = "0.1.89" @@ -1054,12 +930,6 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" -[[package]] -name = "better_io" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef0a3155e943e341e557863e69a708999c94ede624e37865c8e2a91b94efa78f" - [[package]] name = "bigdecimal" version = "0.4.10" @@ -1092,33 +962,6 @@ dependencies = [ "unty", ] -[[package]] -name = "bindgen" -version = "0.72.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools 0.13.0", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 2.0.117", -] - -[[package]] -name = "bit-vec" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" -dependencies = [ - "serde", -] - [[package]] name = "bitflags" version = "2.11.1" @@ -1199,19 +1042,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "blocking" -version = "1.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" -dependencies = [ - "async-channel", - "async-task", - "futures-io", - "futures-lite", - "piper", -] - [[package]] name = "brotli" version = "8.0.2" @@ -1308,15 +1138,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom 7.1.3", -] - [[package]] name = "cfg-if" version = "0.1.10" @@ -1380,17 +1201,6 @@ dependencies = [ "inout", ] -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "clap" version = "4.5.60" @@ -1545,12 +1355,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" -[[package]] -name = "const_for" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c50fcfdf972929aff202c16b80086aa3cfc6a3a820af714096c58c7c1d0582" - [[package]] name = "const_panic" version = "0.2.15" @@ -1845,18 +1649,6 @@ dependencies = [ "cmov", ] -[[package]] -name = "custom-labels" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4121f4f539659ad975da5bc6702b7d7d6de59ff1bbaec328dfe7dd5058052ef2" -dependencies = [ - "bindgen", - "cc", - "libc", - "pin-project-lite", -] - [[package]] name = "darling" version = "0.23.0" @@ -2672,12 +2464,6 @@ version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" -[[package]] -name = "dtype_dispatch" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab23e69df104e2fd85ee63a533a22d2132ef5975dc6b36f9f3e5a7305e4a8ed7" - [[package]] name = "dunce" version = "1.0.5" @@ -2715,26 +2501,6 @@ dependencies = [ "cfg-if 1.0.4", ] -[[package]] -name = "enum-iterator" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4549325971814bda7a44061bf3fe7e487d447cba01e4220a4b454d630d7a016" -dependencies = [ - "enum-iterator-derive", -] - -[[package]] -name = "enum-iterator-derive" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "685adfa4d6f3d765a26bc5dbc936577de9abf756c1feeb3089b01dd395034842" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -2778,54 +2544,12 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "ext-trait" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d772df1c1a777963712fb68e014235e80863d6a91a85c4e06ba2d16243a310e5" -dependencies = [ - "ext-trait-proc_macros", -] - -[[package]] -name = "ext-trait-proc_macros" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab7934152eaf26aa5aa9f7371408ad5af4c31357073c9e84c3b9d7f11ad639a" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "extension-traits" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a296e5a895621edf9fa8329c83aa1cb69a964643e36cf54d8d7a69b789089537" -dependencies = [ - "ext-trait", -] - [[package]] name = "fast-float2" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" -[[package]] -name = "fastlanes" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414cb755aee48ff7b0907995d2949c68c8c17900970076dff6a808e18e592d71" -dependencies = [ - "arrayref", - "const_for", - "num-traits", - "paste", - "seq-macro", -] - [[package]] name = "fastrand" version = "2.4.1" @@ -2913,15 +2637,6 @@ dependencies = [ "rand 0.9.4", ] -[[package]] -name = "fsst-rs" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bf53d7c403a2b76873d4d66ba7d79c54bde2784cdaba6083f223d6e33270708" -dependencies = [ - "rustc-hash", -] - [[package]] name = "fst" version = "0.4.7" @@ -2985,19 +2700,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" -[[package]] -name = "futures-lite" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "parking", - "pin-project-lite", -] - [[package]] name = "futures-macro" version = "0.3.32" @@ -3326,10 +3028,6 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", - "allocator-api2", -] [[package]] name = "hashbrown" @@ -3358,11 +3056,6 @@ name = "hashbrown" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash 0.2.0", -] [[package]] name = "heapify" @@ -3516,15 +3209,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "humansize" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" -dependencies = [ - "libm", -] - [[package]] name = "humantime" version = "2.3.0" @@ -3837,15 +3521,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "inventory" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" -dependencies = [ - "rustversion", -] - [[package]] name = "io-uring" version = "0.7.12" @@ -4037,7 +3712,7 @@ dependencies = [ "fast-float2", "itoa", "jiff", - "nom 8.0.0", + "nom", "num-traits", "ordered-float", "rand 0.9.4", @@ -4064,16 +3739,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "kanal" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3953adf0cd667798b396c2fa13552d6d9b3269d7dd1154c4c416442d1ff574" -dependencies = [ - "futures-core", - "lock_api", -] - [[package]] name = "konst" version = "0.4.3" @@ -4284,7 +3949,7 @@ dependencies = [ "hex", "rand 0.9.4", "rand_distr", - "rand_xoshiro 0.7.0", + "rand_xoshiro", "random_word", ] @@ -4587,16 +4252,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "lasso" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e14eda50a3494b3bf7b9ce51c52434a761e383d7238ce1dd5dcec2fbc13e9fb" -dependencies = [ - "dashmap", - "hashbrown 0.14.5", -] - [[package]] name = "lazy-regex" version = "3.6.0" @@ -4636,35 +4291,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] -name = "lending-iterator" -version = "0.1.7" +name = "lexical-core" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc07588c853b50689205fb5c00498aa681d89828e0ce8cbd965ebc7a5d8ae260" -dependencies = [ - "extension-traits", - "lending-iterator-proc_macros", - "macro_rules_attribute", - "never-say-never", - "nougat", - "polonius-the-crab", -] - -[[package]] -name = "lending-iterator-proc_macros" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5445dd1c0deb1e97b8a16561d17fc686ca83e8411128fb036e9668a72d51b1d" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "lexical-core" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -4843,22 +4473,6 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" -[[package]] -name = "macro_rules_attribute" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf0c9b980bf4f3a37fd7b1c066941dd1b1d0152ce6ee6e8fe8c49b9f6810d862" -dependencies = [ - "macro_rules_attribute-proc_macro", - "paste", -] - -[[package]] -name = "macro_rules_attribute-proc_macro" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" - [[package]] name = "matchers" version = "0.2.0" @@ -4955,12 +4569,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -5015,28 +4623,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" -[[package]] -name = "multiversion" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edb7f0ff51249dfda9ab96b5823695e15a052dc15074c9dbf3d118afaf2c201" -dependencies = [ - "multiversion-macros", - "target-features", -] - -[[package]] -name = "multiversion-macros" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", - "target-features", -] - [[package]] name = "ndarray" version = "0.16.1" @@ -5052,22 +4638,6 @@ dependencies = [ "rawpointer", ] -[[package]] -name = "never-say-never" -version = "6.6.666" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf5a574dadd7941adeaa71823ecba5e28331b8313fb2e1c6a5c7e5981ea53ad6" - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "nom" version = "8.0.0" @@ -5077,27 +4647,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "nougat" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b57b9ced431322f054fc673f1d3c7fa52d80efd9df74ad2fc759f044742510" -dependencies = [ - "macro_rules_attribute", - "nougat-proc_macros", -] - -[[package]] -name = "nougat-proc_macros" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c84f77a45e99a2f9b492695d99e1c23844619caa5f3e57647cffacad773ca257" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "ntapi" version = "0.4.3" @@ -5309,9 +4858,6 @@ name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" -dependencies = [ - "parking_lot_core", -] [[package]] name = "once_cell_polyfill" @@ -5325,12 +4871,6 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" -[[package]] -name = "oneshot" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe21416a02c693fb9f980befcb230ecc70b0b3d1cc4abf88b9675c4c1457f0c" - [[package]] name = "opendal" version = "0.56.0" @@ -5704,18 +5244,6 @@ dependencies = [ "hmac 0.12.1", ] -[[package]] -name = "pco" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "553ccdc7f6999785559af4998c79712a5ab820e26b68bad9146609c19587ec82" -dependencies = [ - "better_io", - "dtype_dispatch", - "half", - "rand_xoshiro 0.6.0", -] - [[package]] name = "pem" version = "3.0.6" @@ -5876,8 +5404,6 @@ name = "persisting-engine" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array", - "arrow-schema", "async-trait", "futures", "lance", @@ -5894,7 +5420,6 @@ dependencies = [ "tempfile", "tokio", "uuid", - "vortex", ] [[package]] @@ -5969,17 +5494,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "piper" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" -dependencies = [ - "atomic-waker", - "fastrand", - "futures-io", -] - [[package]] name = "pkcs1" version = "0.7.5" @@ -6024,26 +5538,6 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" -[[package]] -name = "polling" -version = "3.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" -dependencies = [ - "cfg-if 1.0.4", - "concurrent-queue", - "hermit-abi", - "pin-project-lite", - "rustix", - "windows-sys 0.61.2", -] - -[[package]] -name = "polonius-the-crab" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a69ee997a6282f8462abf1e0d8c38c965e968799e912b3bed8c9e8a28c2f9f" - [[package]] name = "portable-atomic" version = "1.13.1" @@ -6476,15 +5970,6 @@ dependencies = [ "rand 0.9.4", ] -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "rand_xoshiro" version = "0.7.0" @@ -7458,12 +6943,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" -[[package]] -name = "sketches-ddsketch" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea" - [[package]] name = "slab" version = "0.4.12" @@ -7476,23 +6955,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "smol" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33bd3e260892199c3ccfc487c88b2da2265080acb316cd920da72fdfd7c599f" -dependencies = [ - "async-channel", - "async-executor", - "async-fs", - "async-io", - "async-lock", - "async-net", - "async-process", - "blocking", - "futures-lite", -] - [[package]] name = "snafu" version = "0.9.0" @@ -7742,12 +7204,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" -[[package]] -name = "target-features" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" - [[package]] name = "target-lexicon" version = "0.12.16" @@ -7783,12 +7239,6 @@ dependencies = [ "unicode-width 0.1.14", ] -[[package]] -name = "termtree" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d1330fe7f7f872cd05165130b10602d667b205fd85be09be2814b115d4ced9" - [[package]] name = "thiserror" version = "1.0.69" @@ -8377,562 +7827,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "vortex" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "vortex-alp", - "vortex-array", - "vortex-btrblocks", - "vortex-buffer", - "vortex-bytebool", - "vortex-datetime-parts", - "vortex-decimal-byte-parts", - "vortex-error", - "vortex-fastlanes", - "vortex-file", - "vortex-flatbuffers", - "vortex-fsst", - "vortex-io", - "vortex-ipc", - "vortex-layout", - "vortex-mask", - "vortex-metrics", - "vortex-pco", - "vortex-proto", - "vortex-runend", - "vortex-scan", - "vortex-sequence", - "vortex-session", - "vortex-sparse", - "vortex-utils", - "vortex-zigzag", - "vortex-zstd", -] - -[[package]] -name = "vortex-alp" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "itertools 0.14.0", - "num-traits", - "prost", - "rustc-hash", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-fastlanes", - "vortex-mask", - "vortex-session", - "vortex-utils", -] - -[[package]] -name = "vortex-array" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "arc-swap", - "arcref", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", - "arrow-string", - "async-lock", - "bytes", - "cfg-if 1.0.4", - "enum-iterator", - "flatbuffers", - "futures", - "half", - "humansize", - "inventory", - "itertools 0.14.0", - "jiff", - "multiversion", - "num-traits", - "num_enum", - "parking_lot", - "paste", - "pin-project-lite", - "prost", - "rand 0.10.1", - "rustc-hash", - "simdutf8", - "smallvec", - "static_assertions", - "termtree", - "tracing", - "uuid", - "vortex-array-macros", - "vortex-buffer", - "vortex-error", - "vortex-flatbuffers", - "vortex-mask", - "vortex-proto", - "vortex-session", - "vortex-utils", -] - -[[package]] -name = "vortex-array-macros" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "vortex-btrblocks" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "itertools 0.14.0", - "num-traits", - "pco", - "rand 0.10.1", - "rustc-hash", - "tracing", - "vortex-alp", - "vortex-array", - "vortex-buffer", - "vortex-compressor", - "vortex-datetime-parts", - "vortex-decimal-byte-parts", - "vortex-error", - "vortex-fastlanes", - "vortex-fsst", - "vortex-mask", - "vortex-pco", - "vortex-runend", - "vortex-sequence", - "vortex-sparse", - "vortex-utils", - "vortex-zigzag", - "vortex-zstd", -] - -[[package]] -name = "vortex-buffer" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "arrow-buffer", - "bitvec", - "bytes", - "itertools 0.14.0", - "simdutf8", - "vortex-error", -] - -[[package]] -name = "vortex-bytebool" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "num-traits", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-session", -] - -[[package]] -name = "vortex-compressor" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "itertools 0.14.0", - "num-traits", - "parking_lot", - "rand 0.10.1", - "rustc-hash", - "tracing", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-utils", -] - -[[package]] -name = "vortex-datetime-parts" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "num-traits", - "prost", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-session", -] - -[[package]] -name = "vortex-decimal-byte-parts" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "num-traits", - "prost", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-session", -] - -[[package]] -name = "vortex-error" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "arrow-schema", - "flatbuffers", - "jiff", - "prost", - "tokio", -] - -[[package]] -name = "vortex-fastlanes" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "fastlanes", - "itertools 0.14.0", - "lending-iterator", - "num-traits", - "prost", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-session", -] - -[[package]] -name = "vortex-file" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "async-trait", - "bytes", - "flatbuffers", - "futures", - "itertools 0.14.0", - "kanal", - "moka", - "oneshot 0.2.1", - "parking_lot", - "pin-project-lite", - "tokio", - "tracing", - "vortex-alp", - "vortex-array", - "vortex-btrblocks", - "vortex-buffer", - "vortex-bytebool", - "vortex-datetime-parts", - "vortex-decimal-byte-parts", - "vortex-error", - "vortex-fastlanes", - "vortex-flatbuffers", - "vortex-fsst", - "vortex-io", - "vortex-layout", - "vortex-mask", - "vortex-metrics", - "vortex-pco", - "vortex-runend", - "vortex-scan", - "vortex-sequence", - "vortex-session", - "vortex-sparse", - "vortex-utils", - "vortex-zigzag", - "vortex-zstd", -] - -[[package]] -name = "vortex-flatbuffers" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "flatbuffers", - "vortex-buffer", - "vortex-error", -] - -[[package]] -name = "vortex-fsst" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "fsst-rs", - "prost", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-session", -] - -[[package]] -name = "vortex-io" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "async-fs", - "async-stream", - "async-trait", - "bytes", - "custom-labels", - "futures", - "glob", - "kanal", - "oneshot 0.2.1", - "parking_lot", - "pin-project-lite", - "smol", - "tokio", - "tracing", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-metrics", - "vortex-session", - "vortex-utils", - "wasm-bindgen-futures", -] - -[[package]] -name = "vortex-ipc" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "bytes", - "flatbuffers", - "futures", - "itertools 0.14.0", - "pin-project-lite", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-flatbuffers", - "vortex-session", -] - -[[package]] -name = "vortex-layout" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "arcref", - "arrow-array", - "arrow-schema", - "async-stream", - "async-trait", - "bit-vec", - "flatbuffers", - "futures", - "itertools 0.14.0", - "kanal", - "moka", - "once_cell", - "oneshot 0.2.1", - "parking_lot", - "paste", - "pin-project-lite", - "prost", - "rustc-hash", - "serde", - "serde_json", - "sketches-ddsketch", - "termtree", - "tokio", - "tracing", - "vortex-array", - "vortex-btrblocks", - "vortex-buffer", - "vortex-error", - "vortex-flatbuffers", - "vortex-io", - "vortex-mask", - "vortex-metrics", - "vortex-runend", - "vortex-scan", - "vortex-sequence", - "vortex-session", - "vortex-utils", -] - -[[package]] -name = "vortex-mask" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "itertools 0.14.0", - "vortex-buffer", - "vortex-error", -] - -[[package]] -name = "vortex-metrics" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "parking_lot", - "sketches-ddsketch", -] - -[[package]] -name = "vortex-pco" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "pco", - "prost", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-session", -] - -[[package]] -name = "vortex-proto" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "prost", - "prost-types", -] - -[[package]] -name = "vortex-runend" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "arrow-array", - "itertools 0.14.0", - "num-traits", - "prost", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-session", -] - -[[package]] -name = "vortex-scan" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "async-trait", - "futures", - "roaring", - "tracing", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-session", -] - -[[package]] -name = "vortex-sequence" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "num-traits", - "prost", - "smallvec", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-proto", - "vortex-session", -] - -[[package]] -name = "vortex-session" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "dashmap", - "lasso", - "parking_lot", - "vortex-error", - "vortex-utils", -] - -[[package]] -name = "vortex-sparse" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "itertools 0.14.0", - "num-traits", - "prost", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-session", -] - -[[package]] -name = "vortex-utils" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "dashmap", - "hashbrown 0.17.0", - "vortex-error", -] - -[[package]] -name = "vortex-zigzag" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-session", - "zigzag", -] - -[[package]] -name = "vortex-zstd" -version = "0.1.0" -source = "git+https://github.com/reiase/vortex?branch=feature%2Flayout#1d54301c6fc3692a06993485141e09b83cc29d5a" -dependencies = [ - "itertools 0.14.0", - "prost", - "vortex-array", - "vortex-buffer", - "vortex-error", - "vortex-mask", - "vortex-session", - "zstd", -] - [[package]] name = "vsimd" version = "0.8.0" @@ -9755,7 +8649,7 @@ dependencies = [ "lazy_static", "libc", "more-asserts", - "oneshot 0.1.13", + "oneshot", "pin-project", "rand 0.10.1", "reqwest 0.13.3", @@ -9902,15 +8796,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "zigzag" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70b40401a28d86ce16a330b863b86fd7dbee4d7c940587ab09ab8c019f9e3fdf" -dependencies = [ - "num-traits", -] - [[package]] name = "zmij" version = "1.0.21" diff --git a/README.md b/README.md index f55abcb..23975b7 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Persisting provides a **unified tiered storage** for AI workloads — agent traj ├─────────────────────────────────────────────────────────────────┤ │ Storage Engine │ │ │ -│ Lance (columnar) · Vortex (trajectory) · Numpy (memory) │ +│ Lance (columnar) · Lance (trajectory) · Numpy (memory) │ └─────────────────────────────────────────────────────────────────┘ ``` @@ -53,6 +53,7 @@ Record every LLM call your agent makes: ```bash persisting traj capture -o ./store -c proxy.toml -f md -- claude +# or: -f lance → only events.lance/ dataset; then traj materialize for Markdown ``` Trajectories are stored as `(agent_id, run_id, time)` — the same TTAS model used for KV cache and parameters. @@ -109,7 +110,7 @@ arr = kv["s1", 0, 2, 0:512].tensor() | Capability | Status | Description | |------------|--------|-------------| -| **Agent Trajectory Capture** | ✅ Stable | Proxy + record LLM traffic as Vortex + Markdown | +| **Agent Trajectory Capture** | ✅ Stable | Proxy + record LLM traffic as Lance + Markdown | | **Streaming Queue** | ✅ Stable | Lance-backed append/consume, KV API, samplers | | **Compute Orchestration** | ✅ Stable | `plan()` + `execute()`, local/torchrun | | **Agent Search** | ✅ Stable | Document indexing, IVF-PQ, hybrid search | diff --git a/crates/persisting-capture/src/conversion/mod.rs b/crates/persisting-capture/src/conversion/mod.rs index b73d3bd..822b90d 100644 --- a/crates/persisting-capture/src/conversion/mod.rs +++ b/crates/persisting-capture/src/conversion/mod.rs @@ -171,7 +171,7 @@ impl StreamTranslator { pub fn streaming_capture_snapshot(&self) -> Option { let text = match self { Self::ToMessages(t) => t.accumulated_assistant_text().to_string(), - // Markdown + turn indexing: narrative only; tools stay in Vortex payload. + // Markdown + turn indexing: narrative only; tools stay in Lance payload. Self::ToResponses(t) => t.narrative_for_capture(), }; if text.trim().is_empty() { diff --git a/crates/persisting-capture/src/conversion/responses_stream.rs b/crates/persisting-capture/src/conversion/responses_stream.rs index 57dd238..48eb0e6 100644 --- a/crates/persisting-capture/src/conversion/responses_stream.rs +++ b/crates/persisting-capture/src/conversion/responses_stream.rs @@ -96,7 +96,7 @@ impl CompletionsToResponsesStreamTranslator { /// Visible assistant narrative for live markdown / `assistant_content` (no tool fences). /// - /// Tool calls remain in Vortex via full SSE `body`. Some Codex / reasoning models stream the + /// Tool calls remain in Lance via full SSE `body`. Some Codex / reasoning models stream the /// user-visible answer in `reasoning_content` while `content` stays empty during tool turns. pub fn narrative_for_capture(&self) -> String { if !self.accumulated_text.trim().is_empty() { diff --git a/crates/persisting-capture/src/engine/egress.rs b/crates/persisting-capture/src/engine/egress.rs index 259613b..4c3560e 100644 --- a/crates/persisting-capture/src/engine/egress.rs +++ b/crates/persisting-capture/src/engine/egress.rs @@ -30,7 +30,7 @@ fn rebuild_story_from_records( TurnMachine::replay_records(story_id, agent_id, run_id, records).snapshot() } -/// Rebuild story for one session markdown stem from Vortex records. +/// Rebuild story for one session markdown stem from Lance records. pub fn rebuild_session_story( session_id: &str, root_session: &str, diff --git a/crates/persisting-capture/src/engine/prepare.rs b/crates/persisting-capture/src/engine/prepare.rs index e558e27..4c15d10 100644 --- a/crates/persisting-capture/src/engine/prepare.rs +++ b/crates/persisting-capture/src/engine/prepare.rs @@ -1,6 +1,6 @@ //! Prepare phase — pure capture logic before story-local I/O. //! -//! Runs on the runtime thread; may `ask` the run actor. Vortex/markdown I/O only via [`StoryCommand`]. +//! Runs on the runtime thread; may `ask` the run actor. Lance/markdown I/O only via [`StoryCommand`]. use anyhow::Result; use pulsing_actor::ActorRef; diff --git a/crates/persisting-capture/src/engine/wire/story.rs b/crates/persisting-capture/src/engine/wire/story.rs index 8c9f536..f28a1ab 100644 --- a/crates/persisting-capture/src/engine/wire/story.rs +++ b/crates/persisting-capture/src/engine/wire/story.rs @@ -39,7 +39,7 @@ impl StoryScope { /// first reconstructing a `String`. #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) enum StoryCommand { - /// Append one record to Vortex/sink and sync live markdown. + /// Append one record to Lance/sink and sync live markdown. PersistRecord { scope: StoryScope, /// JSON-encoded [`crate::record::CaptureRecord`]. diff --git a/crates/persisting-capture/src/reconcile.rs b/crates/persisting-capture/src/reconcile.rs index c795d30..3fbde22 100644 --- a/crates/persisting-capture/src/reconcile.rs +++ b/crates/persisting-capture/src/reconcile.rs @@ -1,4 +1,4 @@ -//! Lightweight post-run reconcile: compare live markdown call_ids vs Vortex dialogue records. +//! Lightweight post-run reconcile: compare live markdown call_ids vs Lance dialogue records. use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; diff --git a/crates/persisting-capture/src/session/index.rs b/crates/persisting-capture/src/session/index.rs index c345969..e8b1f59 100644 --- a/crates/persisting-capture/src/session/index.rs +++ b/crates/persisting-capture/src/session/index.rs @@ -1,4 +1,4 @@ -//! Session index on disk for `capture list` without full Vortex replay. +//! Session index on disk for `capture list` without full Lance replay. use std::collections::HashMap; use std::fs; diff --git a/crates/persisting-capture/src/storage/convert.rs b/crates/persisting-capture/src/storage/convert.rs index b9c7150..5a353dc 100644 --- a/crates/persisting-capture/src/storage/convert.rs +++ b/crates/persisting-capture/src/storage/convert.rs @@ -1,8 +1,8 @@ -//! Bidirectional conversion between Vortex raw event log and TLV Markdown. +//! Bidirectional conversion between Lance raw event log and TLV Markdown. //! -//! - **Vortex → Markdown** (`materialize`): full scan, overwrite document. -//! - **Vortex → Markdown** (`stream`): incremental append per batch (capture `-f md`). -//! - **Markdown → Vortex** (`compact`): reconstructs [`CaptureRecord`] rows. +//! - **Lance → Markdown** (`materialize`): full scan, overwrite document. +//! - **Lance → Markdown** (`stream`): incremental append per batch (capture `-f md`). +//! - **Markdown → Lance** (`compact`): reconstructs [`CaptureRecord`] rows. use std::path::Path; @@ -17,7 +17,7 @@ use super::markdown::{ use super::record::{record_to_engine_line, CaptureRecord}; use super::session_client::{resolve_client_meta_for_run_dir, SessionClientMeta}; -/// Outcome of Vortex → Markdown materialization. +/// Outcome of Lance → Markdown materialization. #[derive(Debug, Clone, PartialEq, Eq)] pub struct MaterializeStats { pub source_events: usize, @@ -25,7 +25,7 @@ pub struct MaterializeStats { pub skipped_events: usize, } -/// Outcome of streaming Vortex → Markdown (incremental block append). +/// Outcome of streaming Lance → Markdown (incremental block append). #[derive(Debug, Clone, PartialEq, Eq)] pub struct StreamMaterializeStats { pub events_seen: usize, @@ -33,7 +33,7 @@ pub struct StreamMaterializeStats { pub skipped_events: usize, } -/// Outcome of Markdown → Vortex compaction. +/// Outcome of Markdown → Lance compaction. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CompactStats { pub source_blocks: usize, @@ -131,7 +131,7 @@ fn enrich_record_from_block(mut rec: CaptureRecord, block: &MarkdownBlock) -> Ca rec } -/// Parse a TLV Markdown document into capture records (for Vortex compaction). +/// Parse a TLV Markdown document into capture records (for Lance compaction). pub fn markdown_document_to_capture_records(doc: &str) -> Result> { parse_document(doc)? .iter() @@ -168,7 +168,7 @@ pub fn compact_stats_note( overwrite: bool, ) -> String { format!( - "Compacted Markdown→Vortex: {} block(s) → {} row(s) ({}) at {} → {}", + "Compacted Markdown→Lance: {} block(s) → {} row(s) ({}) at {} → {}", stats.source_blocks, stats.event_rows, if overwrite { "overwrite" } else { "append" }, diff --git a/crates/persisting-capture/src/storage/dialogue/draft.rs b/crates/persisting-capture/src/storage/dialogue/draft.rs index aeba626..345b7f5 100644 --- a/crates/persisting-capture/src/storage/dialogue/draft.rs +++ b/crates/persisting-capture/src/storage/dialogue/draft.rs @@ -6,7 +6,7 @@ use super::skip_markdown_block; use crate::record::CaptureRecord; use crate::storage::markdown::BlockHeader; -/// Build a streaming draft assistant block (markdown view only; not written to Vortex). +/// Build a streaming draft assistant block (markdown view only; not written to Lance). pub fn draft_stream_assistant_block( rec: &CaptureRecord, assistant_content: &str, diff --git a/crates/persisting-capture/src/storage/dialogue/view.rs b/crates/persisting-capture/src/storage/dialogue/view.rs index cdf054b..dda545d 100644 --- a/crates/persisting-capture/src/storage/dialogue/view.rs +++ b/crates/persisting-capture/src/storage/dialogue/view.rs @@ -4,7 +4,7 @@ use super::fields::role_and_body; use super::skip_markdown_block; use crate::record::CaptureRecord; -/// Queryable columns extracted from a capture record (shared by Vortex + markdown block headers). +/// Queryable columns extracted from a capture record (shared by Lance + markdown block headers). #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct CaptureRecordView { pub role: Option, diff --git a/crates/persisting-capture/src/storage/event_row.rs b/crates/persisting-capture/src/storage/event_row.rs index 312914b..1ece0fa 100644 --- a/crates/persisting-capture/src/storage/event_row.rs +++ b/crates/persisting-capture/src/storage/event_row.rs @@ -1,10 +1,10 @@ -//! Vortex event row ↔ [`CaptureRecord`]. +//! Lance event row ↔ [`CaptureRecord`]. use anyhow::{Context, Result}; use super::record::{engine_line_to_record, CaptureRecord}; -/// One row in the Vortex event log (canonical trajectory store). +/// One row in the Lance event log (canonical trajectory store). #[derive(Debug, Clone, PartialEq, Eq)] pub struct EventRow { pub seq: i64, diff --git a/crates/persisting-capture/src/storage/lifecycle.rs b/crates/persisting-capture/src/storage/lifecycle.rs index 46c7a8c..948c15f 100644 --- a/crates/persisting-capture/src/storage/lifecycle.rs +++ b/crates/persisting-capture/src/storage/lifecycle.rs @@ -1,4 +1,4 @@ -//! Session lifecycle events for the raw Vortex event log. +//! Session lifecycle events for the raw Lance event log. use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; diff --git a/crates/persisting-capture/src/storage/markdown_policy.rs b/crates/persisting-capture/src/storage/markdown_policy.rs index ea713fd..6c698ed 100644 --- a/crates/persisting-capture/src/storage/markdown_policy.rs +++ b/crates/persisting-capture/src/storage/markdown_policy.rs @@ -1,4 +1,4 @@ -//! Markdown / Vortex eligibility rules for [`CaptureRecord`] (storage layer only). +//! Markdown / Lance eligibility rules for [`CaptureRecord`] (storage layer only). use super::dialogue_extract::is_subagent_shape_payload; use super::record::CaptureRecord; diff --git a/crates/persisting-capture/src/storage/mod.rs b/crates/persisting-capture/src/storage/mod.rs index 8fd76bd..677fc3e 100644 --- a/crates/persisting-capture/src/storage/mod.rs +++ b/crates/persisting-capture/src/storage/mod.rs @@ -1,4 +1,4 @@ -//! Trajectory storage: Vortex event log, TLV Markdown, session routing, capture sink. +//! Trajectory storage: Lance event log, TLV Markdown, session routing, capture sink. pub mod convert; pub mod dialogue; diff --git a/crates/persisting-capture/src/storage/path_layout.rs b/crates/persisting-capture/src/storage/path_layout.rs index 822b612..4b730f9 100644 --- a/crates/persisting-capture/src/storage/path_layout.rs +++ b/crates/persisting-capture/src/storage/path_layout.rs @@ -86,15 +86,15 @@ fn infer_traj_location_from_path(path_arg: &str) -> Option { fn looks_like_session_dir(dir: &Path) -> bool { dir.is_dir() && (locate_session_markdown(dir).is_some() - || dir.join("events.vortex").is_file() + || dir.join("events.lance").is_dir() || dir.join("_versions").is_dir()) } -/// Capture run layout: `{storage}/{agent}/{run_id}/events.vortex`. -/// Flat layout: `{storage}/{agent}/{session_id}/events.vortex`. +/// Capture run layout: `{storage}/{agent}/{run_id}/events.lance/` (dataset directory). +/// Flat layout: `{storage}/{agent}/{session_id}/events.lance/`. fn infer_root_session_id(session_dir: &Path, session_id: &str) -> Option { let is_run_dir = session_id.starts_with("run-"); - if session_dir.join("events.vortex").is_file() + if session_dir.join("events.lance").is_dir() || (is_run_dir && session_dir.join("_versions").is_dir()) { Some(session_id.to_string()) @@ -222,7 +222,7 @@ pub fn merge_story_location( } } -/// When a capture run uses a header session id (UUID) for Vortex/md but the run folder is `run-*`, +/// When a capture run uses a header session id (UUID) for Lance/md but the run folder is `run-*`, /// remap the story storage key from the primary markdown stem under the run directory. fn remap_capture_run_story_session_id( storage: &str, @@ -392,7 +392,7 @@ pub fn list_story_read_locations( let sessions = list_sessions_under_agent(&stor, &agent)?; if sessions.is_empty() { anyhow::bail!( - "trajectory stats: no sessions under {stor}/{agent}/ (expected run dirs with events.vortex or markdown)" + "trajectory stats: no sessions under {stor}/{agent}/ (expected run dirs with events.lance or markdown)" ); } return Ok(sessions); @@ -402,7 +402,7 @@ pub fn list_story_read_locations( let sessions = list_sessions_under_agent(&storage, &agent)?; if sessions.is_empty() { anyhow::bail!( - "trajectory stats: no sessions under {storage}/{agent}/ (expected run dirs with events.vortex or markdown)" + "trajectory stats: no sessions under {storage}/{agent}/ (expected run dirs with events.lance or markdown)" ); } return Ok(sessions); @@ -546,7 +546,7 @@ mod tests { let _ = std::fs::remove_dir_all(&base); let session = base.join("store").join("agent").join("run-1"); std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("events.vortex"), b"vortex").unwrap(); + std::fs::create_dir_all(session.join("events.lance")).unwrap(); std::fs::write(session.join("run-1.md"), "# test\n").unwrap(); let loc = resolve_traj_read_location( @@ -582,7 +582,7 @@ mod tests { let _ = std::fs::remove_dir_all(&base); let session = base.join("store").join("agent").join("run-1"); std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("events.vortex"), b"vortex").unwrap(); + std::fs::create_dir_all(session.join("events.lance")).unwrap(); std::fs::write(session.join("uuid-story.md"), "# test\n").unwrap(); let loc = resolve_traj_read_location( @@ -647,7 +647,7 @@ mod tests { std::fs::write(store.join(".capture/run_session"), "run-20260101-000000").unwrap(); let session = store.join("agent-a").join("run-20260101-000000"); std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("events.vortex"), b"v").unwrap(); + std::fs::create_dir_all(session.join("events.lance")).unwrap(); let loc = resolve_traj_read_location( "trajectory stats", @@ -673,7 +673,7 @@ mod tests { for run in ["run-001", "run-002"] { let session = agent.join(run); std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("events.vortex"), b"v").unwrap(); + std::fs::create_dir_all(session.join("events.lance")).unwrap(); } let locs = @@ -705,7 +705,7 @@ mod tests { .join("deepseek-proxy") .join("run-with-md"); std::fs::create_dir_all(&run).unwrap(); - std::fs::write(run.join("events.vortex"), b"v").unwrap(); + std::fs::create_dir_all(run.join("events.lance")).unwrap(); std::fs::write(run.join("header-session-uuid.md"), "# story\n").unwrap(); let resolved = resolve_traj_read_location( @@ -732,7 +732,7 @@ mod tests { let agent = base.join("store").join("deepseek-proxy"); let run = agent.join("run-only"); std::fs::create_dir_all(&run).unwrap(); - std::fs::write(run.join("events.vortex"), b"v").unwrap(); + std::fs::create_dir_all(run.join("events.lance")).unwrap(); let locs = list_story_read_locations(agent.to_str().unwrap().into(), None, None, None).unwrap(); diff --git a/crates/persisting-capture/src/storage/record.rs b/crates/persisting-capture/src/storage/record.rs index c6e0b94..583465e 100644 --- a/crates/persisting-capture/src/storage/record.rs +++ b/crates/persisting-capture/src/storage/record.rs @@ -1,4 +1,4 @@ -//! `CaptureRecord` and Vortex append wire encoding (RON lines between engine components). +//! `CaptureRecord` and Lance append wire encoding (RON lines between engine components). use anyhow::Context; use serde::{Deserialize, Serialize}; diff --git a/crates/persisting-capture/src/storage/story_coords.rs b/crates/persisting-capture/src/storage/story_coords.rs index 94bd8af..30197cd 100644 --- a/crates/persisting-capture/src/storage/story_coords.rs +++ b/crates/persisting-capture/src/storage/story_coords.rs @@ -1,7 +1,7 @@ //! Run / Story coordinates for offline trajectory egress (aligned with [`CaptureRoute`]). //! //! - **Run** → `root_session_id` → `{storage}/{agent_id}/{run_id}/` -//! - **Story** → `session_id` → `{session_id}.md` under the run directory; Vortex rows are filtered by session_id +//! - **Story** → `session_id` → `{session_id}.md` under the run directory; Lance rows are filtered by session_id use std::path::{Path, PathBuf}; @@ -51,8 +51,8 @@ impl StoryCoords { ) } - pub fn vortex_event_path(&self) -> Result { - story_vortex_event_path( + pub fn lance_event_path(&self) -> Result { + story_lance_event_path( &self.storage, &self.agent_id, &self.session_id, @@ -103,15 +103,15 @@ pub fn story_run_dir( } } -/// Vortex event log file at `{run}/events.vortex`. -pub fn story_vortex_event_path( +/// Lance event log dataset directory at `{run}/events.lance/`. +pub fn story_lance_event_path( storage: &str, agent_id: &str, session_id: &str, root_session_id: Option<&str>, ) -> Result { let run = story_run_dir(storage, agent_id, session_id, root_session_id)?; - Ok(run.join("events.vortex")) + Ok(run.join("events.lance")) } #[cfg(test)] @@ -119,25 +119,25 @@ mod tests { use super::*; #[test] - fn nested_sessions_share_run_level_vortex_path() { - let root = story_vortex_event_path("/store", "agent", "run-001", Some("run-001")).unwrap(); - let sub = story_vortex_event_path("/store", "agent", "agent-sub", Some("run-001")).unwrap(); + fn nested_sessions_share_run_level_lance_path() { + let root = story_lance_event_path("/store", "agent", "run-001", Some("run-001")).unwrap(); + let sub = story_lance_event_path("/store", "agent", "agent-sub", Some("run-001")).unwrap(); assert_eq!(root, sub); - assert!(root.ends_with("agent/run-001/events.vortex")); + assert!(root.ends_with("agent/run-001/events.lance")); } #[test] - fn flat_session_vortex_path_is_session_scoped() { - let path = story_vortex_event_path("/store", "agent", "sess-flat", None).unwrap(); - assert!(path.ends_with("agent/sess-flat/events.vortex")); + fn flat_session_lance_path_is_session_scoped() { + let path = story_lance_event_path("/store", "agent", "sess-flat", None).unwrap(); + assert!(path.ends_with("agent/sess-flat/events.lance")); } #[test] - fn story_coords_vortex_event_path_matches_helper() { + fn story_coords_lance_event_path_matches_helper() { let coords = StoryCoords::new("/store", "agent", "run-x", Some("run-x".into())); assert_eq!( - coords.vortex_event_path().unwrap(), - story_vortex_event_path("/store", "agent", "run-x", Some("run-x")).unwrap() + coords.lance_event_path().unwrap(), + story_lance_event_path("/store", "agent", "run-x", Some("run-x")).unwrap() ); } } diff --git a/crates/persisting-capture/src/storage/subagent_link/extract.rs b/crates/persisting-capture/src/storage/subagent_link/extract.rs index 467d2f1..cf0464c 100644 --- a/crates/persisting-capture/src/storage/subagent_link/extract.rs +++ b/crates/persisting-capture/src/storage/subagent_link/extract.rs @@ -17,8 +17,9 @@ lazy_static! { Regex::new(r"(?m)(?:^|\n)\s*agentId:\s*([a-zA-Z0-9_-]{4,})").expect("agentId tool result"); static ref TOOL_AGENT_BLOCK: Regex = Regex::new(r"```tool:(?:Agent|Task)\s*\n([\s\S]*?)```").expect("tool agent block"); + /// Design-doc paths in prompts (`*.md` or legacy `*.zh.md` under docs/src/design/). static ref DOC_TARGET: Regex = - Regex::new(r"docs/src/design/[^\s\)\]]+\.zh\.md").expect("doc target path"); + Regex::new(r"docs/src/design/[^\s\)\]]+\.md").expect("doc target path"); static ref TASK_ID_XML: Regex = Regex::new(r"(?i)\s*([a-zA-Z0-9_-]{4,})\s*").expect("task-id xml"); } diff --git a/crates/persisting-capture/src/storage/subagent_link/tests.rs b/crates/persisting-capture/src/storage/subagent_link/tests.rs index 766529d..7a95a58 100644 --- a/crates/persisting-capture/src/storage/subagent_link/tests.rs +++ b/crates/persisting-capture/src/storage/subagent_link/tests.rs @@ -98,7 +98,7 @@ fn extracts_spawn_hints_from_agent_tool_block() { fn match_spawns_by_doc_target_when_subagents_registered_first() { let mut registry = SubagentRegistry::default(); let run_key = "run-20260524-021032"; - let doc = "docs/src/design/cli_capture_command.zh.md"; + let doc = "docs/src/design/cli-capture.md"; let sub_body = json!({ "messages": [{ "role": "user", @@ -118,7 +118,7 @@ fn match_spawns_by_doc_target_when_subagents_registered_first() { ```tool:Agent {{ - "description": "Review cli_capture_command design doc", + "description": "Review cli-capture design doc", "prompt": "Review the Chinese design document at /Users/reiase/workspace/Persisting/{doc}", "subagent_type": "general-purpose" }} @@ -178,13 +178,13 @@ fn match_spawns_by_doc_target_when_subagents_registered_first() { fn backfill_spawn_link_when_subagent_registers_after_assistant() { let mut registry = SubagentRegistry::default(); let run_key = "run-backfill"; - let doc = "docs/src/design/capture_design.zh.md"; + let doc = "docs/src/design/capture.md"; let assistant = format!( r#"Starting. ```tool:Agent {{ - "description": "Review capture_design", + "description": "Review capture design", "prompt": "Review {doc}", "subagent_type": "general-purpose" }} diff --git a/crates/persisting-capture/tests/capture/apps/claude/dialogue_extract.rs b/crates/persisting-capture/tests/capture/apps/claude/dialogue_extract.rs index 523a252..20a3d74 100644 --- a/crates/persisting-capture/tests/capture/apps/claude/dialogue_extract.rs +++ b/crates/persisting-capture/tests/capture/apps/claude/dialogue_extract.rs @@ -49,7 +49,7 @@ fn sse_incremental_parser_emits_tool_agent_on_content_block_stop() { let mut parser = SseStreamBlockParser::default(); let snap = push_sse_tool_snapshot(&mut parser, sse).expect("partial snapshot"); assert!(snap.contains("```tool:Agent")); - assert!(snap.contains("cli_capture_command.zh.md")); + assert!(snap.contains("cli-capture.md")); } #[test] diff --git a/crates/persisting-capture/tests/capture/apps/claude/fixtures/agent_tool_stream.sse b/crates/persisting-capture/tests/capture/apps/claude/fixtures/agent_tool_stream.sse index 3491397..bedb6bb 100644 --- a/crates/persisting-capture/tests/capture/apps/claude/fixtures/agent_tool_stream.sse +++ b/crates/persisting-capture/tests/capture/apps/claude/fixtures/agent_tool_stream.sse @@ -11,7 +11,7 @@ event: content_block_start data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","name":"Agent","id":"toolu_agent_1"}} event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"description\":\"Review capture command design\",\"subagent_type\":\"Explore\",\"prompt\":\"Review docs/src/design/cli_capture_command.zh.md\"}"}} +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"description\":\"Review capture command design\",\"subagent_type\":\"Explore\",\"prompt\":\"Review docs/src/design/cli-capture.md\"}"}} event: content_block_stop data: {"type":"content_block_stop","index":1} diff --git a/crates/persisting-capture/tests/capture/apps/claude/fixtures/subagent_explore_flash.json b/crates/persisting-capture/tests/capture/apps/claude/fixtures/subagent_explore_flash.json index 829983a..18a375a 100644 --- a/crates/persisting-capture/tests/capture/apps/claude/fixtures/subagent_explore_flash.json +++ b/crates/persisting-capture/tests/capture/apps/claude/fixtures/subagent_explore_flash.json @@ -3,7 +3,7 @@ "messages": [ { "role": "user", - "content": "Review the design document at docs/src/design/cli_capture_command.zh.md\n\nPlease review for clarity, consistency, and actionable gaps.\nProvide bullet points in Chinese." + "content": "Review the design document at docs/src/design/cli-capture.md\n\nPlease review for clarity, consistency, and actionable gaps.\nProvide bullet points in Chinese." } ] } diff --git a/crates/persisting-capture/tests/capture/apps/claude/markdown_filter.rs b/crates/persisting-capture/tests/capture/apps/claude/markdown_filter.rs index efc8f61..4284646 100644 --- a/crates/persisting-capture/tests/capture/apps/claude/markdown_filter.rs +++ b/crates/persisting-capture/tests/capture/apps/claude/markdown_filter.rs @@ -77,7 +77,7 @@ fn keeps_subagent_flash_explore_prompt() { serde_json::from_slice(&fixture_body("subagent_explore_flash.json")).unwrap(); let rec = request_record( "deepseek-v4-flash", - "Review docs/src/design/cli_capture_command.zh.md", + "Review docs/src/design/cli-capture.md", Some("ad20bef53147678d4"), Some(&body), ); diff --git a/crates/persisting-capture/tests/capture/apps/claude/mod.rs b/crates/persisting-capture/tests/capture/apps/claude/mod.rs index 22ef7e6..5e55fdc 100644 --- a/crates/persisting-capture/tests/capture/apps/claude/mod.rs +++ b/crates/persisting-capture/tests/capture/apps/claude/mod.rs @@ -1,6 +1,6 @@ //! Claude Code–specific capture regressions (routing, spawn links, markdown filters). //! -//! See `docs/src/design/capture_design.zh.md` §8 and `trajectory_storage.zh.md` §7 for the contract these tests guard. +//! See `docs/src/design/capture.md` §8 and `trajectory.md` §7 for the contract these tests guard. mod dialogue_extract; mod markdown_filter; diff --git a/crates/persisting-capture/tests/capture/apps/claude/spawn_link.rs b/crates/persisting-capture/tests/capture/apps/claude/spawn_link.rs index 3147e1b..c847f86 100644 --- a/crates/persisting-capture/tests/capture/apps/claude/spawn_link.rs +++ b/crates/persisting-capture/tests/capture/apps/claude/spawn_link.rs @@ -20,7 +20,7 @@ fn parses_agent_tool_blocks_into_spawn_hints() { { "description": "Review capture command design", "subagent_type": "Explore", - "prompt": "Review docs/src/design/cli_capture_command.zh.md" + "prompt": "Review docs/src/design/cli-capture.md" } ```"#; let hints = extract_spawn_hints_from_assistant(text); @@ -28,7 +28,7 @@ fn parses_agent_tool_blocks_into_spawn_hints() { assert_eq!(hints[0].subagent_type, "Explore"); assert_eq!( hints[0].doc_target.as_deref(), - Some("docs/src/design/cli_capture_command.zh.md") + Some("docs/src/design/cli-capture.md") ); } @@ -52,7 +52,7 @@ fn matches_spawn_hint_to_registered_subagent_by_doc_target() { serde_json::from_slice(&fixture_body("subagent_explore_flash.json")).unwrap(); registry.register_route(&run_key, &sub_route, "call-sub-first", Some(&body)); let assistant = r#"```tool:Agent -{"description":"Review capture command design","subagent_type":"Explore","prompt":"Review docs/src/design/cli_capture_command.zh.md"} +{"description":"Review capture command design","subagent_type":"Explore","prompt":"Review docs/src/design/cli-capture.md"} ```"#; let mut rec = CaptureRecord { seq: 0, diff --git a/crates/persisting-cli/src/capture/mod.rs b/crates/persisting-cli/src/capture/mod.rs index c4fbe94..147542e 100644 --- a/crates/persisting-cli/src/capture/mod.rs +++ b/crates/persisting-cli/src/capture/mod.rs @@ -26,23 +26,23 @@ pub use record::CaptureRecord; /// Capture trajectory storage format. /// /// - `md`: TLV Markdown only (`{session}.md` live upsert); reconcile replays from Markdown. -/// - `vortex` (alias `bin`): Vortex canonical (`events.vortex`) only; use `traj materialize` for md. +/// - `lance` (alias `bin`): Lance canonical (`events.lance`) only; use `traj materialize` for md. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)] pub enum CaptureFormat { /// Human-readable TLV Markdown only. #[value(name = "md", alias = "markdown")] #[default] Markdown, - /// Vortex event log (canonical); no live Markdown sidecar during capture. - #[value(name = "vortex", alias = "bin")] - Vortex, + /// Lance event log (canonical); no live Markdown sidecar during capture. + #[value(name = "lance", alias = "bin")] + Lance, } impl From for TrajectoryStorageFormat { fn from(v: CaptureFormat) -> Self { match v { CaptureFormat::Markdown => TrajectoryStorageFormat::Markdown, - CaptureFormat::Vortex => TrajectoryStorageFormat::Vortex, + CaptureFormat::Lance => TrajectoryStorageFormat::Lance, } } } @@ -51,7 +51,7 @@ impl CaptureFormat { pub fn as_str(self) -> &'static str { match self { Self::Markdown => "md", - Self::Vortex => "vortex", + Self::Lance => "lance", } } @@ -59,8 +59,8 @@ impl CaptureFormat { matches!(self, Self::Markdown) } - pub fn writes_vortex(self) -> bool { - matches!(self, Self::Vortex) + pub fn writes_lance(self) -> bool { + matches!(self, Self::Lance) } /// Live markdown upsert inside [`CaptureEngine`] (`-f md` only). @@ -71,7 +71,7 @@ impl CaptureFormat { /// Storage layer to replay when comparing live Markdown after a capture run. pub fn reconcile_replay_format(self) -> TrajectoryStorageFormat { match self { - Self::Vortex => TrajectoryStorageFormat::Vortex, + Self::Lance => TrajectoryStorageFormat::Lance, Self::Markdown => TrajectoryStorageFormat::Markdown, } } @@ -83,21 +83,21 @@ mod tests { use clap::ValueEnum; #[test] - fn capture_format_bin_alias_maps_to_vortex() { + fn capture_format_bin_alias_maps_to_lance() { assert_eq!( CaptureFormat::from_str("bin", false).unwrap(), - CaptureFormat::Vortex + CaptureFormat::Lance ); - assert!(CaptureFormat::Vortex.writes_vortex()); + assert!(CaptureFormat::Lance.writes_lance()); } #[test] - fn vortex_does_not_stream_live_markdown() { - assert!(!CaptureFormat::Vortex.stream_markdown_in_engine()); - assert!(!CaptureFormat::Vortex.writes_markdown()); + fn lance_does_not_stream_live_markdown() { + assert!(!CaptureFormat::Lance.stream_markdown_in_engine()); + assert!(!CaptureFormat::Lance.writes_markdown()); assert_eq!( - CaptureFormat::Vortex.reconcile_replay_format(), - TrajectoryStorageFormat::Vortex + CaptureFormat::Lance.reconcile_replay_format(), + TrajectoryStorageFormat::Lance ); } diff --git a/crates/persisting-cli/src/main.rs b/crates/persisting-cli/src/main.rs index 47035a9..ff5d679 100644 --- a/crates/persisting-cli/src/main.rs +++ b/crates/persisting-cli/src/main.rs @@ -321,7 +321,7 @@ struct CaptureRunArgs { /// Log every proxied / captured HTTP request to stderr and `{output_dir}/.capture/debug.log`. #[arg(long)] debug: bool, - /// Storage: `md` (Markdown only) or `vortex` / `bin` (Vortex canonical + live Markdown). + /// Storage: `md` (Markdown only) or `lance` / `bin` (Lance canonical only; no live Markdown). #[arg(long, short = 'f', value_enum, default_value_t = capture::CaptureFormat::Markdown)] format: capture::CaptureFormat, /// Command and arguments to execute (after `--`). @@ -615,19 +615,19 @@ enum TrajectoryCommand { after_long_help = "Differs from `traj replay`, which reads stored events from a session." )] ReplayDeadLetter(CaptureReplayDeadLetterArgs), - /// 批量追加 CaptureRecord 事件（写 Vortex canonical）。 + /// 批量追加 CaptureRecord 事件（写 Lance canonical）。 Add(TrajectoryAddArgs), - /// 截断 Vortex 事件日志（保留前 N 行）；默认在存在 Markdown 层时同步重建 md。 + /// 截断 Lance 事件日志（保留前 N 行）；默认在存在 Markdown 层时同步重建 md。 Truncate(TrajectoryTruncateArgs), - /// 统计规模；`auto` 时报告 Vortex + Markdown 两层。 + /// 统计规模；`auto` 时报告 Lance + Markdown 两层。 Stats(TrajectoryStatsArgs), - /// 按 seq 回放事件（默认读 Vortex）。 + /// 按 seq 回放事件（默认读 Lance）。 Replay(TrajectoryReplayArgs), /// 导出 Story / Run 目录树到目标路径。 Extract(TrajectoryExtractArgs), - /// Vortex → TLV Markdown（有损物化，维护用）。 + /// Lance → TLV Markdown（有损物化，维护用）。 Materialize(TrajectoryMaterializeArgs), - /// LLM-as-judge：读 canonical Vortex，写 `{run}/layers/judge_*.vortex` sidecar。 + /// LLM-as-judge：读 canonical Lance，写 `{run}/layers/judge_*.lance` sidecar。 Judge(TrajectoryJudgeArgs), /// 汇总 judge sidecar 分数（按 session + rubric）。 #[command(name = "judge-stats")] @@ -658,7 +658,7 @@ struct ProxyArgs { /// Foreground proxy only: log proxied HTTP to stderr / `.capture/debug.log`. #[arg(long)] debug: bool, - /// Foreground proxy only: `md` (Markdown only) or `vortex` / `bin` (Vortex + live Markdown). + /// Foreground proxy only: `md` (Markdown only) or `lance` / `bin` (Lance only; no live Markdown). #[arg(long, short = 'f', value_enum)] format: Option, } @@ -720,7 +720,7 @@ struct TrajectoryAddArgs { format: TrajectoryAddFormat, #[arg(long, default_value = "-")] input: String, - /// 写入层：`vortex` / `markdown` / `auto`（`auto` 按已有层探测，默认新建 Vortex）。 + /// 写入层：`lance` / `markdown` / `auto`（`auto` 按已有层探测，默认新建 Lance）。 #[arg(long, value_enum, default_value_t = TrajectoryStorageCli::Auto)] storage_format: TrajectoryStorageCli, } @@ -736,7 +736,7 @@ struct TrajectoryTruncateArgs { session_id: Option, #[arg(long, value_name = "SEG")] root_session_id: Option, - /// 保留按 `seq` 排序的前 N 条 Vortex 行（仅 Vortex 层；需更新 md 请单独 `materialize`）。 + /// 保留按 `seq` 排序的前 N 条 Lance 行（仅 Lance 层；需更新 md 请单独 `materialize`）。 #[arg(long)] keep_rows: usize, } @@ -775,7 +775,7 @@ struct TrajectoryReplayArgs { offset: usize, #[arg(long)] limit: Option, - /// 读取层覆盖（`auto`：有 Vortex 读 Vortex，否则 Markdown；两层并存时默认 Vortex）。 + /// 读取层覆盖（`auto`：有 Lance 读 Lance，否则 Markdown；两层并存时默认 Lance）。 #[arg(long, value_enum, default_value_t = TrajectoryStorageCli::Auto)] storage_format: TrajectoryStorageCli, } @@ -792,7 +792,7 @@ struct TrajectoryStatsArgs { /// 嵌套 subagent session 时指定父 session（路径 `{root}/subagents/{session_id}/`）。 #[arg(long, value_name = "SEG")] root_session_id: Option, - /// 统计层覆盖（`auto`：两层并存时同时报告 Vortex 行数与 Markdown 块数）。 + /// 统计层覆盖（`auto`：两层并存时同时报告 Lance 行数与 Markdown 块数）。 #[arg(long, value_enum, default_value_t = TrajectoryStorageCli::Auto)] storage_format: TrajectoryStorageCli, /// 逐轮一行摘要：用户/模型字符数、TTFT、TPOT。 @@ -1311,7 +1311,7 @@ fn flush_capture_trajectory_batch( session_id: key.session_id.clone(), root_session_id: key.root_session_id.clone(), records_ronl, - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }))?; let raw = engine.invoke_engine_ron(&payload)?; parse_engine_ron_response(&raw)?; @@ -1327,7 +1327,7 @@ fn build_capture_trajectory_sink( std::sync::Arc, TrajectoryAppendWorker, )> { - if !format.writes_vortex() { + if !format.writes_lance() { let sink = std::sync::Arc::new(persisting_capture::sink::SeqOnlySink::new()); return Ok((sink, TrajectoryAppendWorker::noop())); } @@ -2284,7 +2284,7 @@ fn resolve_judge_locations( locs = expand_story_locations_blocking(locs)?; locs = drop_lifecycle_run_partitions(locs); if locs.is_empty() { - anyhow::bail!("no scorable sessions after expanding Vortex partitions"); + anyhow::bail!("no scorable sessions after expanding Lance partitions"); } if args.all { @@ -2455,7 +2455,7 @@ fn run_traj_judge_one( session_id: loc.session_id.clone(), offset: 0, limit: None, - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, root_session_id: loc.root_session_id.clone(), }, )?; diff --git a/crates/persisting-cli/src/trajectory_detail.rs b/crates/persisting-cli/src/trajectory_detail.rs index 7156314..6e233e7 100644 --- a/crates/persisting-cli/src/trajectory_detail.rs +++ b/crates/persisting-cli/src/trajectory_detail.rs @@ -811,7 +811,7 @@ mod tests { } #[test] - fn parses_vortex_capture_record_replay_json() { + fn parses_lance_capture_record_replay_json() { let records = vec![ serde_json::json!({ "seq": 0, diff --git a/crates/persisting-cli/src/trajectory_format.rs b/crates/persisting-cli/src/trajectory_format.rs index d46ef9a..7fb5d36 100644 --- a/crates/persisting-cli/src/trajectory_format.rs +++ b/crates/persisting-cli/src/trajectory_format.rs @@ -11,15 +11,15 @@ use persisting_proto::TrajectoryStorageFormat; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)] pub enum TrajectoryStorageCli { - /// Read/stats: detect layer; append: detect target layer (`auto` → Vortex if empty, else existing layer). + /// Read/stats: detect layer; append: detect target layer (`auto` → Lance if empty, else existing layer). #[default] Auto, - /// Vortex raw event log (canonical). Legacy alias: `bin`. - #[value(name = "vortex", alias = "bin")] - Vortex, + /// Lance raw event log (canonical). Legacy alias: `bin`. + #[value(name = "lance", alias = "bin")] + Lance, /// TLV Markdown session file (read/materialize view). Markdown, - /// Legacy: append → Vortex only; read/stats → same as `auto`. Hidden from help. + /// Legacy: append → Lance only; read/stats → same as `auto`. Hidden from help. #[value(hide = true)] Both, } @@ -28,7 +28,7 @@ impl From for TrajectoryStorageFormat { fn from(v: TrajectoryStorageCli) -> Self { match v { TrajectoryStorageCli::Auto => TrajectoryStorageFormat::Auto, - TrajectoryStorageCli::Vortex => TrajectoryStorageFormat::Vortex, + TrajectoryStorageCli::Lance => TrajectoryStorageFormat::Lance, TrajectoryStorageCli::Markdown => TrajectoryStorageFormat::Markdown, TrajectoryStorageCli::Both => TrajectoryStorageFormat::Both, } @@ -82,13 +82,13 @@ impl TrajectoryFormatManager { } } -/// Numbered session markdown (`0001.md`) or legacy `.tlv.md` → default append target Vortex (parse TLV, write event log). +/// Numbered session markdown (`0001.md`) or legacy `.tlv.md` → default append target Lance (parse TLV, write event log). pub fn infer_storage_format_from_path(input_path: &str) -> Option { if input_path == "-" { return None; } if is_trajectory_markdown_path(Path::new(input_path)) { - return Some(TrajectoryStorageFormat::Vortex); + return Some(TrajectoryStorageFormat::Lance); } let lower = input_path.to_ascii_lowercase(); if lower.ends_with(".jsonl") @@ -96,7 +96,7 @@ pub fn infer_storage_format_from_path(input_path: &str) -> Option, @@ -201,21 +201,21 @@ pub async fn run_compute(args: ComputeArgs) -> Result { .unwrap_or("run") .to_string() }); - let vortex = crate::sink_traj::VortexResultSink::new( + let lance = crate::sink_traj::LanceResultSink::new( traj_storage.display().to_string(), args.traj_agent.clone(), session.clone(), ); if let Ok(ledger) = CheckpointLedger::load(dir).await { - vortex.seed_seen(ledger.skip_ids()); + lance.seed_seen(ledger.skip_ids()); } eprintln!( - "[traj] vortex append → {}/{}/{}", + "[traj] lance append → {}/{}/{}", traj_storage.display(), args.traj_agent, session ); - sinks.push(Box::new(vortex)); + sinks.push(Box::new(lance)); } Some(Arc::new(TeeSink::new(sinks))) } else { diff --git a/crates/persisting-compute/src/lib.rs b/crates/persisting-compute/src/lib.rs index e1be10e..bd9bd44 100644 --- a/crates/persisting-compute/src/lib.rs +++ b/crates/persisting-compute/src/lib.rs @@ -61,5 +61,5 @@ pub use worker::{WorkerActor, WorkerCommand, WorkerReply}; // Sink types used by CLI orchestration (re-exported so `cli` stays thin). pub use sink::{JsonlFileSink, ResultSink, TeeSink}; #[cfg(feature = "traj-sink")] -pub use sink_traj::VortexResultSink; +pub use sink_traj::LanceResultSink; pub use sink_writer::{spawn_sink_writer, SinkSubmitter, SinkWriterHandle}; diff --git a/crates/persisting-compute/src/sink.rs b/crates/persisting-compute/src/sink.rs index 731514f..566bf37 100644 --- a/crates/persisting-compute/src/sink.rs +++ b/crates/persisting-compute/src/sink.rs @@ -3,7 +3,7 @@ //! **Primitive:** [`ResultSink`] · [`persist_terminal`] · [`JsonlFileSink`] (task_id dedup). //! //! - Phase-1 ledger: append-only JSONL under `--sink` (`task_id` for `--resume`). -//! - Optional L1: feature `traj-sink` → [`crate::sink_traj::VortexResultSink`] via Tee. +//! - Optional L1: feature `traj-sink` → [`crate::sink_traj::LanceResultSink`] via Tee. use crate::checkpoint::CheckpointLedger; use crate::task::TaskResult; diff --git a/crates/persisting-compute/src/sink_traj.rs b/crates/persisting-compute/src/sink_traj.rs index 228400f..6dded33 100644 --- a/crates/persisting-compute/src/sink_traj.rs +++ b/crates/persisting-compute/src/sink_traj.rs @@ -1,4 +1,4 @@ -//! Vortex trajectory sink: TaskResult → CaptureRecord → TrajectoryAppend. +//! Lance trajectory sink: TaskResult → CaptureRecord → TrajectoryAppend. //! //! Enabled with feature `traj-sink`. Always Tee with [`JsonlFileSink`] so //! `--resume` keeps using the JSONL task_id ledger. @@ -13,14 +13,14 @@ use std::collections::HashSet; use std::sync::Mutex; /// Append terminal compute results as `compute.result` / `compute.failure` events. -pub struct VortexResultSink { +pub struct LanceResultSink { storage: String, agent_id: String, session_id: String, seen: Mutex>, } -impl VortexResultSink { +impl LanceResultSink { pub fn new( storage: impl Into, agent_id: impl Into, @@ -83,7 +83,7 @@ impl VortexResultSink { let mut seen = self .seen .lock() - .map_err(|_| anyhow::anyhow!("vortex sink lock poisoned"))?; + .map_err(|_| anyhow::anyhow!("lance sink lock poisoned"))?; if !seen.insert(result.task_id.clone()) { return Ok(()); } @@ -97,7 +97,7 @@ impl VortexResultSink { session_id: self.session_id.clone(), root_session_id: None, records_ronl: line, - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }; // bridge is sync (blocks on runtime inside engine); ok from async via spawn_blocking. let resp = @@ -109,7 +109,7 @@ impl VortexResultSink { task_id = %result.task_id, accepted = resp.accepted_records, dataset = %resp.dataset, - "compute result appended to vortex" + "compute result appended to lance" ); Ok(()) }; @@ -127,7 +127,7 @@ impl VortexResultSink { } #[async_trait] -impl ResultSink for VortexResultSink { +impl ResultSink for LanceResultSink { async fn append_ready(&self, result: &TaskResult) -> Result<()> { self.append_record(result).await } diff --git a/crates/persisting-engine/Cargo.toml b/crates/persisting-engine/Cargo.toml index 980bd47..d011d87 100644 --- a/crates/persisting-engine/Cargo.toml +++ b/crates/persisting-engine/Cargo.toml @@ -4,7 +4,7 @@ version.workspace = true edition.workspace = true authors.workspace = true license.workspace = true -description = "Persisting engine: Lance-backed search, Vortex trajectory, and stable C ABI" +description = "Persisting engine: Lance-backed search, Lance trajectory, and stable C ABI" [lib] name = "persisting_engine" @@ -27,9 +27,6 @@ serde_json = "1" tokio = { version = "1", features = ["macros", "rt-multi-thread", "fs"] } async-trait = "0.1" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } -vortex = { git = "https://github.com/reiase/vortex", branch = "feature/layout", default-features = true, features = ["tokio"] } -arrow-array = "58.3.0" -arrow-schema = "58.3.0" [dev-dependencies] persisting-proto = { path = "../persisting-proto" } diff --git a/crates/persisting-engine/src/lib.rs b/crates/persisting-engine/src/lib.rs index 1e25d57..a16dd4e 100644 --- a/crates/persisting-engine/src/lib.rs +++ b/crates/persisting-engine/src/lib.rs @@ -1,4 +1,4 @@ -//! Persisting engine: Lance-backed search, Vortex trajectory, and a stable C ABI for `dlopen`. +//! Persisting engine: Lance-backed search, Lance trajectory, and a stable C ABI for `dlopen`. pub mod abi; pub mod bridge; diff --git a/crates/persisting-engine/src/trajectory/convert.rs b/crates/persisting-engine/src/trajectory/convert.rs index 5f7c4f4..3370dbd 100644 --- a/crates/persisting-engine/src/trajectory/convert.rs +++ b/crates/persisting-engine/src/trajectory/convert.rs @@ -1,4 +1,4 @@ -//! Vortex ↔ TLV Markdown conversion at the trajectory storage layer. +//! Lance ↔ TLV Markdown conversion at the trajectory storage layer. use anyhow::{Context, Result}; use persisting_capture::trajectory_convert::{ @@ -7,12 +7,12 @@ use persisting_capture::trajectory_convert::{ }; use super::store::{ - overwrite_session_lines, MarkdownTrajectoryStore, TrajectorySession, TrajectoryStore, - VortexTrajectoryStore, + overwrite_session_lines, LanceTrajectoryStore, MarkdownTrajectoryStore, TrajectorySession, + TrajectoryStore, }; use persisting_capture::story_coords::story_run_dir; -/// Result of materializing Vortex raw log → TLV Markdown. +/// Result of materializing Lance raw log → TLV Markdown. #[derive(Debug, Clone, PartialEq, Eq)] pub struct MaterializeOutcome { pub markdown_path: String, @@ -20,7 +20,7 @@ pub struct MaterializeOutcome { pub note: String, } -/// Result of compacting TLV Markdown → Vortex raw log. +/// Result of compacting TLV Markdown → Lance raw log. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CompactOutcome { pub event_log_path: String, @@ -31,11 +31,11 @@ pub struct CompactOutcome { async fn load_all_capture_records( session: &TrajectorySession, ) -> Result> { - let vortex = VortexTrajectoryStore; - let outcome = vortex + let lance = LanceTrajectoryStore; + let outcome = lance .replay(session, 0, None) .await - .context("replay Vortex for conversion")?; + .context("replay Lance for conversion")?; outcome .records .iter() @@ -47,15 +47,15 @@ async fn load_all_capture_records( .collect() } -/// **Vortex → Markdown** (lossy): scan raw event log, write dialogue TLV blocks. -pub async fn materialize_vortex_to_markdown( +/// **Lance → Markdown** (lossy): scan raw event log, write dialogue TLV blocks. +pub async fn materialize_lance_to_markdown( session: &TrajectorySession, ) -> Result { - let vortex = VortexTrajectoryStore; - if !vortex.exists(session).await? { + let lance = LanceTrajectoryStore; + if !lance.exists(session).await? { anyhow::bail!( - "Vortex event log missing at {}; materialize requires raw event log", - vortex.display_path(session)? + "Lance event log missing at {}; materialize requires raw event log", + lance.display_path(session)? ); } @@ -72,7 +72,7 @@ pub async fn materialize_vortex_to_markdown( Ok(MaterializeOutcome { markdown_path: md_path.display().to_string(), note: format!( - "Materialized Vortex→Markdown (lossy): {} event(s) → {} block(s), skipped {} internal/non-dialogue event(s) at {}", + "Materialized Lance→Markdown (lossy): {} event(s) → {} block(s), skipped {} internal/non-dialogue event(s) at {}", stats.source_events, stats.markdown_blocks, stats.skipped_events, @@ -82,8 +82,8 @@ pub async fn materialize_vortex_to_markdown( }) } -/// **Markdown → Vortex** (compact): parse TLV blocks into event log rows. -pub async fn compact_markdown_to_vortex( +/// **Markdown → Lance** (compact): parse TLV blocks into event log rows. +pub async fn compact_markdown_to_lance( session: &TrajectorySession, overwrite: bool, ) -> Result { @@ -118,13 +118,13 @@ pub async fn compact_markdown_to_vortex( .collect(); let source_blocks = line_vec.len(); - let vortex = VortexTrajectoryStore; - let event_log_path = vortex.display_path(session)?; + let lance = LanceTrajectoryStore; + let event_log_path = lance.display_path(session)?; let event_rows = if overwrite { overwrite_session_lines(session, &line_vec).await? } else { - VortexTrajectoryStore + LanceTrajectoryStore .append(session, &line_vec) .await? .persisted_units @@ -142,7 +142,7 @@ pub async fn compact_markdown_to_vortex( }) } -/// Session-level stats comparing Vortex row count vs Markdown block count. +/// Session-level stats comparing Lance row count vs Markdown block count. #[derive(Debug, Clone, PartialEq, Eq)] pub struct LayerStats { pub event_rows: usize, @@ -153,11 +153,11 @@ pub struct LayerStats { } pub async fn layer_stats(session: &TrajectorySession) -> Result { - let vortex = VortexTrajectoryStore; + let lance = LanceTrajectoryStore; let markdown = MarkdownTrajectoryStore; - let event_rows = if vortex.exists(session).await? { - vortex.stats(session).await?.row_count + let event_rows = if lance.exists(session).await? { + lance.stats(session).await?.row_count } else { 0 }; @@ -184,10 +184,10 @@ pub async fn layer_stats(session: &TrajectorySession) -> Result { let note = if event_rows > 0 && markdown_blocks > 0 { format!( - "Two-layer storage: Vortex {event_rows} raw event(s) ≥ Markdown {markdown_blocks} dialogue block(s)" + "Two-layer storage: Lance {event_rows} raw event(s) ≥ Markdown {markdown_blocks} dialogue block(s)" ) } else if event_rows > 0 { - format!("Vortex only: {event_rows} raw event(s)") + format!("Lance only: {event_rows} raw event(s)") } else if markdown_blocks > 0 { format!("Markdown only: {markdown_blocks} block(s)") } else { @@ -197,7 +197,7 @@ pub async fn layer_stats(session: &TrajectorySession) -> Result { Ok(LayerStats { event_rows, markdown_blocks, - event_log_path: vortex.display_path(session)?, + event_log_path: lance.display_path(session)?, markdown_path, note, }) diff --git a/crates/persisting-engine/src/trajectory/expand.rs b/crates/persisting-engine/src/trajectory/expand.rs index 2a82db6..8614950 100644 --- a/crates/persisting-engine/src/trajectory/expand.rs +++ b/crates/persisting-engine/src/trajectory/expand.rs @@ -1,22 +1,22 @@ -//! Expand run-bucket locations into per-`session_id` story views (shared Vortex file). +//! Expand run-bucket locations into per-`session_id` story views (shared Lance file). use anyhow::Result; use persisting_capture::StoryCoords; -use super::store::vortex; +use super::store::lance; -fn is_shared_vortex_run_bucket(loc: &StoryCoords) -> bool { +fn is_shared_lance_run_bucket(loc: &StoryCoords) -> bool { loc.root_session_id .as_deref() .is_some_and(|root| root == loc.session_id.as_str()) } /// When a location points at a capture run bucket (`session_id == root_session_id`), -/// read `events.vortex` and emit one coordinate per distinct `session_id` partition. +/// read `events.lance` and emit one coordinate per distinct `session_id` partition. pub async fn expand_story_locations(locations: Vec) -> Result> { let mut expanded = Vec::new(); for loc in locations { - if !is_shared_vortex_run_bucket(&loc) { + if !is_shared_lance_run_bucket(&loc) { expanded.push(loc); continue; } @@ -26,7 +26,7 @@ pub async fn expand_story_locations(locations: Vec) -> Result) -> Result) -> Vec { use std::collections::{HashMap, HashSet}; @@ -109,7 +109,7 @@ pub fn expand_story_locations_blocking(locations: Vec) -> Result String { @@ -133,7 +133,7 @@ mod tests { } #[tokio::test] - async fn expand_run_bucket_into_vortex_partitions() { + async fn expand_run_bucket_into_lance_partitions() { let dir = tempfile::tempdir().unwrap(); let storage = dir.path().join("store"); std::fs::create_dir_all(&storage).unwrap(); @@ -141,10 +141,10 @@ mod tests { let root = "run-expand"; let run = StoryCoords::new(&storage_s, "agent", root, Some(root.into())); - vortex::append(&run, &[note_line(root, "lifecycle")]) + lance::append(&run, &[note_line(root, "lifecycle")]) .await .unwrap(); - vortex::append( + lance::append( &StoryCoords::new(&storage_s, "agent", "story-uuid", Some(root.into())), &[note_line("story-uuid", "dialogue")], ) @@ -164,7 +164,7 @@ mod tests { } /// Regression: capture.run writes `session.started` under run-* while Claude/header - /// traffic uses a UUID `session_id` in the same `events.vortex`. + /// traffic uses a UUID `session_id` in the same `events.lance`. #[tokio::test] async fn claude_run_header_session_stats_regression() { use persisting_capture::engine::Call; @@ -192,7 +192,7 @@ mod tests { started_at: "2026-05-31T03:02:43Z".into(), }; let mut lines = vec![record_to_engine_line(&started).unwrap()]; - vortex::append(&run_coords, &lines).await.unwrap(); + lance::append(&run_coords, &lines).await.unwrap(); lines.clear(); let header_coords = StoryCoords::new(&storage_s, agent, header_session, Some(run.into())); for i in 1..=3 { @@ -220,16 +220,16 @@ mod tests { lines.push(record_to_engine_line(&req).unwrap()); lines.push(record_to_engine_line(&resp).unwrap()); } - vortex::append(&header_coords, &lines).await.unwrap(); + lance::append(&header_coords, &lines).await.unwrap(); - let run_stats = vortex::stats(&run_coords).await.unwrap(); + let run_stats = lance::stats(&run_coords).await.unwrap(); assert_eq!( run_stats.row_count, 1, "run id partition should only see session.started" ); let header_coords = StoryCoords::new(&storage_s, agent, header_session, Some(run.into())); - let header_stats = vortex::stats(&header_coords).await.unwrap(); + let header_stats = lance::stats(&header_coords).await.unwrap(); assert_eq!( header_stats.row_count, 6, "header UUID partition should see all LLM events" @@ -243,7 +243,7 @@ mod tests { for loc in expanded { row_counts.insert( loc.session_id.clone(), - vortex::stats(&loc).await.unwrap().row_count, + lance::stats(&loc).await.unwrap().row_count, ); } assert_eq!(row_counts.get(run), Some(&1)); diff --git a/crates/persisting-engine/src/trajectory/judge.rs b/crates/persisting-engine/src/trajectory/judge.rs index 7dbf8fe..25624c8 100644 --- a/crates/persisting-engine/src/trajectory/judge.rs +++ b/crates/persisting-engine/src/trajectory/judge.rs @@ -1,4 +1,4 @@ -//! Judge egress: read canonical Vortex, write `{run}/layers/judge_*.vortex` sidecars. +//! Judge egress: read canonical Lance, write `{run}/layers/judge_*.lance` sidecars. //! //! Modes (scope × method): //! - **story + llm**: one LLM call scores the full trajectory per rubric dimension @@ -19,7 +19,7 @@ use super::layers::{ read_judge_rows, register_layer, save_manifest, sidecar_path, write_judge_rows, JudgeRow, MANUAL_RATIONALE_PREFIX, STORY_CALL_ID, }; -use super::store::{TrajectoryStore, VortexTrajectoryStore}; +use super::store::{LanceTrajectoryStore, TrajectoryStore}; use super::TrajectorySession; #[derive(Debug, Deserialize)] @@ -57,15 +57,15 @@ pub async fn judge_async(request: TrajectoryJudgeRequest) -> Result = replay .records .iter() @@ -561,7 +561,7 @@ mod tests { session_id: "run".into(), root_session_id: None, records_ronl: lines.join("\n") + "\n", - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); @@ -585,7 +585,7 @@ mod tests { .unwrap(); assert_eq!(out.judged_calls, 1); - assert!(std::path::Path::new(&out.sidecar_path).is_file()); + assert!(std::path::Path::new(&out.sidecar_path).is_dir()); } #[tokio::test] @@ -627,7 +627,7 @@ mod tests { session_id: "run".into(), root_session_id: None, records_ronl: lines.join("\n") + "\n", - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); @@ -703,7 +703,7 @@ mod tests { session_id: "run".into(), root_session_id: None, records_ronl: lines.join("\n") + "\n", - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); @@ -733,7 +733,7 @@ mod tests { .unwrap(); assert_eq!(first.judged_calls, 1); let sidecar = first.sidecar_path.clone(); - assert!(std::path::Path::new(&sidecar).is_file()); + assert!(std::path::Path::new(&sidecar).is_dir()); let second = judge_async(TrajectoryJudgeRequest { storage, @@ -760,7 +760,7 @@ mod tests { .unwrap(); assert_eq!(second.judged_calls, 0); assert_eq!(second.skipped_calls, 1); - assert!(std::path::Path::new(&sidecar).is_file()); + assert!(std::path::Path::new(&sidecar).is_dir()); let rows = read_judge_rows(std::path::Path::new(&sidecar)) .await .unwrap(); diff --git a/crates/persisting-engine/src/trajectory/judge_stats.rs b/crates/persisting-engine/src/trajectory/judge_stats.rs index dc22556..88ec565 100644 --- a/crates/persisting-engine/src/trajectory/judge_stats.rs +++ b/crates/persisting-engine/src/trajectory/judge_stats.rs @@ -139,7 +139,7 @@ async fn load_judge_rows_for_run(key: &RunKey) -> Result> { for entry in std::fs::read_dir(&layers)? { let entry = entry?; let name = entry.file_name().to_string_lossy().to_string(); - if name.starts_with("judge_") && name.ends_with(".vortex") { + if name.starts_with("judge_") && name.ends_with(".lance") { rows.extend(read_judge_rows(&entry.path()).await?); } } @@ -317,7 +317,7 @@ mod tests { session_id: "story-a".into(), root_session_id: Some("run-1".into()), records_ronl: lines.join("\n") + "\n", - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); diff --git a/crates/persisting-engine/src/trajectory/layers/judge_sidecar.rs b/crates/persisting-engine/src/trajectory/layers/judge_sidecar.rs index 09f2d37..683db21 100644 --- a/crates/persisting-engine/src/trajectory/layers/judge_sidecar.rs +++ b/crates/persisting-engine/src/trajectory/layers/judge_sidecar.rs @@ -1,18 +1,15 @@ -//! Call-level judge rows persisted as `{run}/layers/judge_{rubric}.vortex`. +//! Call-level judge rows persisted as `{run}/layers/judge_{rubric}.lance`. use std::path::Path; -use std::sync::{Arc, OnceLock}; +use std::sync::Arc; use anyhow::{Context, Result}; -use arrow_array::cast::AsArray; -use arrow_array::{Array, Int64Array, RecordBatch, StringArray}; -use arrow_schema::{DataType, Field, Schema as ArrowSchema}; -use vortex::array::arrow::{ArrowSessionExt, FromArrowArray}; -use vortex::array::stream::ArrayStreamExt; -use vortex::array::VortexSessionExecute; -use vortex::file::{OpenOptionsSessionExt, WriteOptionsSessionExt}; -use vortex::session::VortexSession; -use vortex::VortexSessionDefault; +use futures::TryStreamExt; +use lance::dataset::{InsertBuilder, WriteMode, WriteParams}; +use lance::deps::arrow_array::{Array, Int64Array, RecordBatch, StringArray}; +use lance::deps::arrow_schema::{DataType, Field, Schema as ArrowSchema}; +use lance::Dataset; +use lance::Error as LanceError; pub const JUDGE_SESSION_ID_COL: &str = "session_id"; pub const JUDGE_CALL_ID_COL: &str = "call_id"; @@ -31,15 +28,6 @@ pub struct JudgeRow { pub rationale: String, } -fn vortex_session() -> &'static VortexSession { - static SESSION: OnceLock = OnceLock::new(); - SESSION.get_or_init(VortexSession::default) -} - -fn vortex_err(e: vortex::error::VortexError) -> anyhow::Error { - anyhow::anyhow!("{e}") -} - pub fn judge_schema() -> Arc { Arc::new(ArrowSchema::new(vec![ Field::new(JUDGE_SESSION_ID_COL, DataType::Utf8, false), @@ -97,13 +85,6 @@ fn rows_from_batch(batch: &RecordBatch) -> Result> { .as_any() .downcast_ref::() .ok_or_else(|| anyhow::anyhow!("expected Utf8 column {name}")) - .map(|_| { - batch - .column(idx) - .as_any() - .downcast_ref::() - .unwrap() - }) }; let session = col(JUDGE_SESSION_ID_COL)?; let call = col(JUDGE_CALL_ID_COL)?; @@ -138,39 +119,36 @@ fn rows_from_batch(batch: &RecordBatch) -> Result> { Ok(rows) } +async fn dataset_exists(uri: &str) -> Result { + match Dataset::open(uri).await { + Ok(_) => Ok(true), + Err(e) if matches!(e, LanceError::DatasetNotFound { .. }) => Ok(false), + Err(e) => Err(anyhow::anyhow!("{:#}", e)), + } +} + async fn read_all_rows(path: &Path) -> Result> { - if !path.is_file() { + let uri = path.to_string_lossy(); + if !dataset_exists(&uri).await? { return Ok(Vec::new()); } - let file = vortex_session() - .open_options() - .open_path(path) + let ds = Dataset::open(uri.as_ref()) .await - .map_err(vortex_err)?; - let array = file + .with_context(|| format!("open judge Lance dataset {}", path.display()))?; + let stream = ds .scan() - .map_err(vortex_err)? - .into_array_stream() - .map_err(vortex_err)? - .read_all() + .try_into_stream() .await - .map_err(vortex_err)?; - if array.len() == 0 { - return Ok(Vec::new()); + .with_context(|| format!("scan judge Lance dataset {}", path.display()))?; + let batches: Vec = stream + .try_collect() + .await + .with_context(|| format!("collect judge Lance batches {}", path.display()))?; + let mut rows = Vec::new(); + for batch in &batches { + rows.extend(rows_from_batch(batch)?); } - let schema = judge_schema(); - let mut ctx = vortex_session().create_execution_ctx(); - let target = Field::new( - "", - DataType::Struct(schema.fields.clone()), - array.dtype().is_nullable(), - ); - let arrow = vortex_session() - .arrow() - .execute_arrow(array, Some(&target), &mut ctx) - .map_err(vortex_err)?; - let batch = RecordBatch::from(arrow.as_struct()); - rows_from_batch(&batch) + Ok(rows) } pub async fn read_judge_rows(path: &Path) -> Result> { @@ -178,34 +156,34 @@ pub async fn read_judge_rows(path: &Path) -> Result> { } pub async fn write_judge_rows(path: &Path, rows: &[JudgeRow]) -> Result<()> { + let uri = path.to_string_lossy().into_owned(); if rows.is_empty() { - if path.is_file() { - tokio::fs::remove_file(path) + if path.exists() { + tokio::fs::remove_dir_all(path) .await .with_context(|| format!("remove empty judge sidecar {}", path.display()))?; } return Ok(()); } let batch = record_batch_from_judge_rows(rows)?; - let array = vortex::array::ArrayRef::from_arrow(batch, false).map_err(vortex_err)?; - let stream = array.to_array_stream(); - let tmp = path.with_extension("vortex.tmp"); if let Some(parent) = path.parent() { tokio::fs::create_dir_all(parent) .await .with_context(|| format!("create_dir_all {}", parent.display()))?; } - let mut file = tokio::fs::File::create(&tmp) - .await - .with_context(|| format!("create {}", tmp.display()))?; - vortex_session() - .write_options() - .write(&mut file, stream) - .await - .map_err(vortex_err)?; - tokio::fs::rename(&tmp, path) + let mode = if dataset_exists(&uri).await? { + WriteMode::Overwrite + } else { + WriteMode::Create + }; + InsertBuilder::new(uri.as_str()) + .with_params(&WriteParams { + mode, + ..Default::default() + }) + .execute(vec![batch]) .await - .with_context(|| format!("rename {} -> {}", tmp.display(), path.display()))?; + .with_context(|| format!("write judge Lance dataset {}", path.display()))?; Ok(()) } diff --git a/crates/persisting-engine/src/trajectory/layers/manifest.rs b/crates/persisting-engine/src/trajectory/layers/manifest.rs index 80fca54..0630adf 100644 --- a/crates/persisting-engine/src/trajectory/layers/manifest.rs +++ b/crates/persisting-engine/src/trajectory/layers/manifest.rs @@ -1,11 +1,42 @@ -//! `layers/manifest.json` — sidecar registry for logical Vortex joins. +//! `layers/manifest.json` — local JSON registry of sidecar datasets for logical joins. +//! +//! Join semantics are application-defined (keys listed in each entry); this is not +//! an external columnar layout API. use anyhow::{Context, Result}; -use vortex::layout::scan::logical::{LayerManifest, LayerManifestEntry}; +use serde::{Deserialize, Serialize}; use super::{join_on_session_call_id, layer_field_name, layer_file_name, manifest_path}; use crate::trajectory::TrajectorySession; +/// One sidecar layer joined onto the primary event log. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct LayerManifestEntry { + /// Top-level logical field name exposed after the join (e.g. `judge_default`). + pub name: String, + /// Path to the layer dataset, relative to the manifest directory. + pub path: String, + /// Column names present in both primary and layer used for the left join. + pub join_on: Vec, +} + +/// Manifest listing all sidecar layers for a logical dataset. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +pub struct LayerManifest { + /// Sidecar layers in stable order. + pub layers: Vec, +} + +impl LayerManifest { + pub fn from_json(json: &str) -> Result { + serde_json::from_str(json).context("parse layers/manifest.json") + } + + pub fn to_json(&self) -> Result { + serde_json::to_string_pretty(self).context("serialize layers/manifest.json") + } +} + pub fn load_manifest(session: &TrajectorySession) -> Result { let path = manifest_path(session)?; if !path.is_file() { @@ -13,7 +44,7 @@ pub fn load_manifest(session: &TrajectorySession) -> Result { } let raw = std::fs::read_to_string(&path) .with_context(|| format!("read manifest {}", path.display()))?; - LayerManifest::from_json(&raw).context("parse layers/manifest.json") + LayerManifest::from_json(&raw) } pub fn save_manifest(session: &TrajectorySession, manifest: &LayerManifest) -> Result<()> { @@ -56,5 +87,6 @@ mod tests { assert_eq!(n1, n2); assert_eq!(m.layers.len(), 1); assert_eq!(m.layers[0].join_on, join_on_session_call_id()); + assert!(m.layers[0].path.ends_with(".lance")); } } diff --git a/crates/persisting-engine/src/trajectory/layers/mod.rs b/crates/persisting-engine/src/trajectory/layers/mod.rs index 50f5947..35712ae 100644 --- a/crates/persisting-engine/src/trajectory/layers/mod.rs +++ b/crates/persisting-engine/src/trajectory/layers/mod.rs @@ -1,4 +1,4 @@ -//! Sidecar Vortex layers under `{run}/layers/` (manifest + joinable files). +//! Sidecar Lance layers under `{run}/layers/` (manifest + joinable datasets). mod judge_sidecar; mod manifest; @@ -48,7 +48,7 @@ pub fn layer_field_name(rubric_id: &str) -> String { } pub fn layer_file_name(rubric_id: &str) -> String { - format!("{}.vortex", layer_field_name(rubric_id)) + format!("{}.lance", layer_field_name(rubric_id)) } fn sanitize_layer_token(raw: &str) -> String { diff --git a/crates/persisting-engine/src/trajectory/mod.rs b/crates/persisting-engine/src/trajectory/mod.rs index 11e4aae..9d7ae72 100644 --- a/crates/persisting-engine/src/trajectory/mod.rs +++ b/crates/persisting-engine/src/trajectory/mod.rs @@ -1,5 +1,5 @@ -//! Trajectory storage: **Vortex** (canonical event log) + **TLV Markdown** (optional materialized view). -//! Append/truncate write **one** layer per `--storage-format`; `materialize` is Vortex → Markdown only. +//! Trajectory storage: **Lance** (canonical event log) + **TLV Markdown** (optional materialized view). +//! Append/truncate write **one** layer per `--storage-format`; `materialize` is Lance → Markdown only. //! //! Path: `{storage}/{agent_id}/{run_id}/` with `{session_id}.md` per logical session. //! @@ -29,7 +29,7 @@ pub mod store; pub use storage::{detect_story_primary_layer, story_stats_note}; pub use convert::{ - compact_markdown_to_vortex, layer_stats, materialize_vortex_to_markdown, CompactOutcome, + compact_markdown_to_lance, layer_stats, materialize_lance_to_markdown, CompactOutcome, LayerStats, MaterializeOutcome, }; pub use expand::{ @@ -43,7 +43,7 @@ pub use path::{ }; pub use persisting_capture::egress::{export_story_bundle, parse_engine_records, ExportOutcome}; pub use persisting_capture::story_coords::{ - story_run_dir as trajectory_run_dir, story_vortex_event_path as trajectory_event_log_path, + story_lance_event_path as trajectory_event_log_path, story_run_dir as trajectory_run_dir, }; /// Flat layout `{storage}/{agent_id}/{session_id}/` (no nested run). @@ -55,9 +55,9 @@ pub fn trajectory_event_log_path_flat( trajectory_event_log_path(storage, agent_id, session_id, None) } pub use store::{ - store_for_append, store_for_read, MarkdownTrajectoryStore, TrajectoryAppendOutcome, - TrajectoryReplayOutcome, TrajectorySession, TrajectoryStatsOutcome, TrajectoryStore, - VortexTrajectoryStore, + store_for_append, store_for_read, LanceTrajectoryStore, MarkdownTrajectoryStore, + TrajectoryAppendOutcome, TrajectoryReplayOutcome, TrajectorySession, TrajectoryStatsOutcome, + TrajectoryStore, }; pub const TRAJECTORY_SEQ_COL: &str = "seq"; @@ -72,7 +72,7 @@ pub const TRAJECTORY_MODEL_COL: &str = "model"; pub const TRAJECTORY_TRACE_ID_COL: &str = "trace_id"; pub const TRAJECTORY_PAYLOAD_JSON_COL: &str = "payload_json"; -/// Vortex trajectory schema v1 columns. +/// Lance trajectory schema v1 columns. pub const TRAJECTORY_V1_COLS: &[&str] = &[ TRAJECTORY_SEQ_COL, TRAJECTORY_TIMESTAMP_COL, @@ -124,7 +124,7 @@ pub async fn materialize_async( &request.session_id, root_session_id, ); - let outcome = materialize_vortex_to_markdown(&session).await?; + let outcome = materialize_lance_to_markdown(&session).await?; Ok(TrajectoryMaterializeResponse { storage: request.storage, agent_id: request.agent_id, @@ -295,9 +295,9 @@ pub async fn truncate_async( root_session_id, ); - let vortex = VortexTrajectoryStore; - let uri = vortex.display_path(&session)?; - let outcome = vortex.replay(&session, 0, None).await?; + let lance = LanceTrajectoryStore; + let uri = lance.display_path(&session)?; + let outcome = lance.replay(&session, 0, None).await?; let total = outcome.records.len(); let keep = request.keep_rows.min(total); let kept: Vec = outcome.records.into_iter().take(keep).collect(); @@ -313,7 +313,7 @@ pub async fn truncate_async( .collect::>>()?; let persisted = store::overwrite_session_lines(&session, &kept_lines).await?; - let note = format!("truncated Vortex: kept {persisted}/{total} row(s) at {uri}"); + let note = format!("truncated Lance: kept {persisted}/{total} row(s) at {uri}"); Ok(TrajectoryTruncateResponse { storage: request.storage, @@ -410,13 +410,13 @@ mod tests { assert!(trajectory_event_log_path("/tmp", "a/b", "s", None).is_err()); assert!(trajectory_event_log_path("/tmp", "..", "s", None).is_err()); let nested = trajectory_event_log_path("/tmp", "agent", "sub-1", Some("root-1")).unwrap(); - assert!(nested.ends_with("agent/root-1/events.vortex")); + assert!(nested.ends_with("agent/root-1/events.lance")); let root = trajectory_event_log_path("/tmp", "agent", "root-1", Some("root-1")).unwrap(); - assert!(root.ends_with("agent/root-1/events.vortex")); + assert!(root.ends_with("agent/root-1/events.lance")); } #[tokio::test] - async fn append_replay_stats_vortex_roundtrip() { + async fn append_replay_stats_lance_roundtrip() { use persisting_capture::record::record_to_engine_line; let dir = tempfile::tempdir().unwrap(); @@ -465,14 +465,14 @@ mod tests { session_id: "sess_1".into(), root_session_id: None, records_ronl: format!("{line1}\n{line2}\n"), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); assert_eq!(append.accepted_records, 2); - assert!(append.note.contains("Vortex v1")); - let vortex_path = trajectory_event_log_path(&storage_s, "agent_a", "sess_1", None).unwrap(); - assert!(vortex_path.is_file(), "expected {}", vortex_path.display()); + assert!(append.note.contains("Lance v1")); + let lance_path = trajectory_event_log_path(&storage_s, "agent_a", "sess_1", None).unwrap(); + assert!(lance_path.is_dir(), "expected {}", lance_path.display()); let replay = replay_async(TrajectoryReplayRequest { storage: storage_s.clone(), @@ -480,7 +480,7 @@ mod tests { session_id: "sess_1".into(), offset: 0, limit: Some(10), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, root_session_id: None, }) .await @@ -491,17 +491,17 @@ mod tests { storage: storage_s.clone(), agent_id: "agent_a".into(), session_id: "sess_1".into(), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, root_session_id: None, }) .await .unwrap(); assert_eq!(st.row_count, 2); - assert!(st.note.contains("Vortex v1")); + assert!(st.note.contains("Lance v1")); } #[tokio::test] - async fn append_replay_stats_nested_vortex_roundtrip() { + async fn append_replay_stats_nested_lance_roundtrip() { use persisting_capture::record::record_to_engine_line; let dir = tempfile::tempdir().unwrap(); @@ -535,7 +535,7 @@ mod tests { session_id: "sub-1".into(), root_session_id: Some("root-1".into()), records_ronl: format!("{}\n{}\n{}\n", mk("a"), mk("b"), mk("c")), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); @@ -547,7 +547,7 @@ mod tests { session_id: "sub-2".into(), root_session_id: Some("root-1".into()), records_ronl: format!("{}\n{}\n", mk("x"), mk("y")), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); @@ -558,7 +558,7 @@ mod tests { session_id: "sub-1".into(), offset: 1, limit: Some(1), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, root_session_id: Some("root-1".into()), }) .await @@ -569,13 +569,13 @@ mod tests { storage: storage_s.clone(), agent_id: "agent_a".into(), session_id: "sub-1".into(), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, root_session_id: Some("root-1".into()), }) .await .unwrap(); assert_eq!(st.row_count, 3); - assert!(st.dataset.contains("events.vortex")); + assert!(st.dataset.contains("events.lance")); let other = replay_async(TrajectoryReplayRequest { storage: storage_s, @@ -583,7 +583,7 @@ mod tests { session_id: "sub-2".into(), offset: 0, limit: None, - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, root_session_id: Some("root-1".into()), }) .await @@ -635,13 +635,13 @@ mod tests { session_id: "s".into(), root_session_id: None, records_ronl: records_ronl.clone(), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); let session = StoryCoords::new(storage_s.clone(), "a", "s", None); - materialize_vortex_to_markdown(&session).await.unwrap(); + materialize_lance_to_markdown(&session).await.unwrap(); let replay = replay_async(TrajectoryReplayRequest { storage: storage_s.clone(), @@ -679,7 +679,7 @@ mod tests { } #[tokio::test] - async fn append_both_storage_format_writes_vortex_only() { + async fn append_both_storage_format_writes_lance_only() { use persisting_capture::record::record_to_engine_line; use persisting_capture::sink::llm_request_record; @@ -708,8 +708,8 @@ mod tests { .await .unwrap(); - let vortex_path = trajectory_event_log_path(&storage_s, "a", "s", None).unwrap(); - assert!(vortex_path.is_file()); + let lance_path = trajectory_event_log_path(&storage_s, "a", "s", None).unwrap(); + assert!(lance_path.is_dir()); let md_path = md::session_markdown_write_path_for_key( &trajectory_run_dir(&storage_s, "a", "s", None).unwrap(), "s", @@ -754,12 +754,12 @@ mod tests { let md_text = std::fs::read_to_string(&md_path).unwrap(); assert!(md_text.contains("new")); - let vortex_path = trajectory_event_log_path(&storage_s, "a", "s", None).unwrap(); - assert!(!vortex_path.is_file()); + let lance_path = trajectory_event_log_path(&storage_s, "a", "s", None).unwrap(); + assert!(!lance_path.exists()); } #[tokio::test] - async fn replay_auto_reads_vortex_when_both_layers_exist() { + async fn replay_auto_reads_lance_when_both_layers_exist() { use persisting_capture::engine::Call; use persisting_capture::record::record_to_engine_line; use persisting_capture::sink::{llm_request_record, llm_response_record}; @@ -778,13 +778,13 @@ mod tests { None, "m", "/v1", - &serde_json::json!({"messages":[{"role":"user","content":"vortex-wins"}]}), + &serde_json::json!({"messages":[{"role":"user","content":"lance-wins"}]}), ); let resp = llm_response_record( Some("s".into()), None, 200, - &serde_json::json!({"choices":[{"message":{"role":"assistant","content":"from-vortex"}}]}), + &serde_json::json!({"choices":[{"message":{"role":"assistant","content":"from-lance"}}]}), false, &call, ); @@ -800,13 +800,13 @@ mod tests { session_id: "s".into(), root_session_id: None, records_ronl, - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); let session = StoryCoords::new(storage_s.clone(), "a", "s", None); - materialize_vortex_to_markdown(&session).await.unwrap(); + materialize_lance_to_markdown(&session).await.unwrap(); let replay = replay_async(TrajectoryReplayRequest { storage: storage_s, @@ -866,13 +866,13 @@ mod tests { session_id: "s".into(), root_session_id: None, records_ronl, - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); let session = StoryCoords::new(storage_s.clone(), "a", "s", None); - materialize_vortex_to_markdown(&session).await.unwrap(); + materialize_lance_to_markdown(&session).await.unwrap(); let st = stats_async(TrajectoryStatsRequest { storage: storage_s, @@ -884,12 +884,12 @@ mod tests { .await .unwrap(); assert_eq!(st.row_count, 2); - assert!(st.note.contains("Story stats via vortex")); + assert!(st.note.contains("Story stats via lance")); assert!(st.note.contains("Markdown 2")); } #[tokio::test] - async fn append_replay_structured_vortex_llm_columns() { + async fn append_replay_structured_lance_llm_columns() { use persisting_capture::engine::Call; use persisting_capture::record::record_to_engine_line; use persisting_capture::sink::{llm_request_record, llm_response_record}; @@ -935,7 +935,7 @@ mod tests { session_id: "s".into(), root_session_id: None, records_ronl, - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); @@ -946,7 +946,7 @@ mod tests { session_id: "s".into(), offset: 0, limit: Some(10), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, root_session_id: None, }) .await @@ -1003,13 +1003,13 @@ mod tests { session_id: "s".into(), root_session_id: None, records_ronl, - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); let session = StoryCoords::new(storage_s.clone(), "a", "s", None); - let mat = materialize_vortex_to_markdown(&session).await.unwrap(); + let mat = materialize_lance_to_markdown(&session).await.unwrap(); assert_eq!(mat.stats.source_events, 2); assert_eq!(mat.stats.markdown_blocks, 2); assert!(std::path::Path::new(&mat.markdown_path).exists()); @@ -1018,7 +1018,7 @@ mod tests { assert_eq!(layers.event_rows, 2); assert_eq!(layers.markdown_blocks, 2); - let compact = compact_markdown_to_vortex(&session, true).await.unwrap(); + let compact = compact_markdown_to_lance(&session, true).await.unwrap(); assert_eq!(compact.stats.source_blocks, 2); assert_eq!(compact.stats.event_rows, 2); } @@ -1062,7 +1062,7 @@ mod tests { session_id: "s".into(), root_session_id: None, records_ronl: note_lines(3), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); @@ -1086,7 +1086,7 @@ mod tests { session_id: "s".into(), offset: 0, limit: None, - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, root_session_id: None, }) .await @@ -1139,13 +1139,13 @@ mod tests { session_id: "s".into(), root_session_id: None, records_ronl, - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); let session = StoryCoords::new(storage_s.clone(), "a", "s", None); - materialize_vortex_to_markdown(&session).await.unwrap(); + materialize_lance_to_markdown(&session).await.unwrap(); let md_path = md::session_markdown_write_path_for_key( &trajectory_run_dir(&storage_s, "a", "s", None).unwrap(), "s", @@ -1169,7 +1169,7 @@ mod tests { session_id: "s".into(), offset: 0, limit: None, - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, root_session_id: None, }) .await @@ -1239,7 +1239,7 @@ mod tests { } #[tokio::test] - async fn append_storage_format_markdown_without_vortex() { + async fn append_storage_format_markdown_without_lance() { use persisting_capture::record::record_to_engine_line; use persisting_capture::sink::llm_request_record; @@ -1269,8 +1269,8 @@ mod tests { assert_eq!(append.accepted_records, 1); assert!(append.note.contains("markdown")); - let vortex_path = trajectory_event_log_path(&storage_s, "a", "s", None).unwrap(); - assert!(!vortex_path.is_file()); + let lance_path = trajectory_event_log_path(&storage_s, "a", "s", None).unwrap(); + assert!(!lance_path.exists()); let md_path = md::session_markdown_write_path_for_key( &trajectory_run_dir(&storage_s, "a", "s", None).unwrap(), @@ -1324,7 +1324,7 @@ mod tests { session_id: "s".into(), root_session_id: None, records_ronl, - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); @@ -1344,9 +1344,9 @@ mod tests { } /// End-to-end regression for `traj stats store/{agent}/` scan: - /// list run buckets → expand vortex partitions → stats each session_id. + /// list run buckets → expand lance partitions → stats each session_id. #[tokio::test] - async fn stats_agent_scan_expands_vortex_session_partitions() { + async fn stats_agent_scan_expands_lance_session_partitions() { use crate::trajectory::expand_story_locations; use persisting_capture::engine::Call; use persisting_capture::lifecycle::{session_started_record, CaptureMode}; @@ -1398,7 +1398,7 @@ mod tests { session_id: run.into(), root_session_id: Some(run.into()), records_ronl: format!("{}\n", record_to_engine_line(&started).unwrap()), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); @@ -1412,7 +1412,7 @@ mod tests { record_to_engine_line(&req).unwrap(), record_to_engine_line(&resp).unwrap(), ), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, }) .await .unwrap(); @@ -1433,7 +1433,7 @@ mod tests { storage: loc.storage.clone(), agent_id: loc.agent_id.clone(), session_id: loc.session_id.clone(), - storage_format: TrajectoryStorageFormat::Vortex, + storage_format: TrajectoryStorageFormat::Lance, root_session_id: loc.root_session_id.clone(), }) .await diff --git a/crates/persisting-engine/src/trajectory/storage.rs b/crates/persisting-engine/src/trajectory/storage.rs index a0acc1b..80ac121 100644 --- a/crates/persisting-engine/src/trajectory/storage.rs +++ b/crates/persisting-engine/src/trajectory/storage.rs @@ -1,10 +1,10 @@ -//! Resolve Vortex vs markdown storage for a session. +//! Resolve Lance vs markdown storage for a session. use persisting_proto::TrajectoryStorageFormat; use super::convert::LayerStats; use super::store::{ - session_vortex_path, MarkdownTrajectoryStore, TrajectoryStore, VortexTrajectoryStore, + session_lance_path, LanceTrajectoryStore, MarkdownTrajectoryStore, TrajectoryStore, }; use persisting_capture::story_coords::StoryCoords; @@ -12,22 +12,22 @@ pub fn detect_story_primary_layer( layers: &LayerStats, session: &StoryCoords, ) -> TrajectoryStorageFormat { - let has_vortex = layers.event_rows > 0; + let has_lance = layers.event_rows > 0; let has_md = layers.markdown_blocks > 0; - match (has_vortex, has_md) { + match (has_lance, has_md) { (false, true) => TrajectoryStorageFormat::Markdown, - (true, false) => TrajectoryStorageFormat::Vortex, + (true, false) => TrajectoryStorageFormat::Lance, (false, false) => TrajectoryStorageFormat::Markdown, (true, true) => { - // Capture run: live TLV markdown is the dialogue story; Vortex is the raw event log. + // Capture run: live TLV markdown is the dialogue story; Lance is the raw event log. if session.root_session_id.is_some() { return TrajectoryStorageFormat::Markdown; } - // Flat layout: materialized / lossy markdown has fewer blocks than raw Vortex. + // Flat layout: materialized / lossy markdown has fewer blocks than raw Lance. if layers.markdown_blocks < layers.event_rows { return TrajectoryStorageFormat::Markdown; } - TrajectoryStorageFormat::Vortex + TrajectoryStorageFormat::Lance } } } @@ -39,11 +39,11 @@ pub fn story_stats_note(layers: &LayerStats, primary: TrajectoryStorageFormat) - }; match (layers.event_rows > 0, layers.markdown_blocks > 0) { (true, true) => format!( - "Story stats via {via} ({primary_count}); Vortex {} raw event(s), Markdown {} dialogue block(s)", + "Story stats via {via} ({primary_count}); Lance {} raw event(s), Markdown {} dialogue block(s)", layers.event_rows, layers.markdown_blocks, via = format_label(primary) ), - (true, false) => format!("Story stats via vortex ({primary_count} raw event(s))"), + (true, false) => format!("Story stats via lance ({primary_count} raw event(s))"), (false, true) => format!("Story stats via markdown ({primary_count} dialogue block(s))"), (false, false) => "Story stats: no trajectory data".to_string(), } @@ -52,8 +52,8 @@ pub fn story_stats_note(layers: &LayerStats, primary: TrajectoryStorageFormat) - async fn resolve_auto_read(session: &StoryCoords) -> anyhow::Result { resolve_auto_read_with( session, - TrajectoryStorageFormat::Vortex, - TrajectoryStorageFormat::Vortex, + TrajectoryStorageFormat::Lance, + TrajectoryStorageFormat::Lance, ) .await } @@ -63,12 +63,12 @@ async fn resolve_auto_read_with( when_empty: TrajectoryStorageFormat, when_both: TrajectoryStorageFormat, ) -> anyhow::Result { - let vortex = VortexTrajectoryStore; + let lance = LanceTrajectoryStore; let md = MarkdownTrajectoryStore; - let has_vortex = vortex.exists(session).await?; + let has_lance = lance.exists(session).await?; let has_md = md.exists(session).await?; - Ok(match (has_vortex, has_md) { - (true, false) => TrajectoryStorageFormat::Vortex, + Ok(match (has_lance, has_md) { + (true, false) => TrajectoryStorageFormat::Lance, (false, true) => TrajectoryStorageFormat::Markdown, (false, false) => when_empty, (true, true) => when_both, @@ -89,7 +89,7 @@ pub async fn resolve_for_read_with_root( root_session_id.map(str::to_string), ); match requested { - TrajectoryStorageFormat::Vortex + TrajectoryStorageFormat::Lance | TrajectoryStorageFormat::Markdown | TrajectoryStorageFormat::Both => Ok(requested), TrajectoryStorageFormat::Auto => resolve_auto_read(&session).await, @@ -110,15 +110,15 @@ pub async fn resolve_for_append( root_session_id.map(str::to_string), ); match requested { - TrajectoryStorageFormat::Vortex => Ok(TrajectoryStorageFormat::Vortex), + TrajectoryStorageFormat::Lance => Ok(TrajectoryStorageFormat::Lance), TrajectoryStorageFormat::Markdown => Ok(TrajectoryStorageFormat::Markdown), - // Legacy alias: append targets Vortex only (same as Vortex). - TrajectoryStorageFormat::Both => Ok(TrajectoryStorageFormat::Vortex), + // Legacy alias: append targets Lance only (same as Lance). + TrajectoryStorageFormat::Both => Ok(TrajectoryStorageFormat::Lance), TrajectoryStorageFormat::Auto => { resolve_auto_read_with( &session, - TrajectoryStorageFormat::Vortex, - TrajectoryStorageFormat::Vortex, + TrajectoryStorageFormat::Lance, + TrajectoryStorageFormat::Lance, ) .await } @@ -128,7 +128,7 @@ pub async fn resolve_for_append( pub fn format_label(fmt: TrajectoryStorageFormat) -> &'static str { match fmt { TrajectoryStorageFormat::Auto => "auto", - TrajectoryStorageFormat::Vortex => "vortex", + TrajectoryStorageFormat::Lance => "lance", TrajectoryStorageFormat::Markdown => "markdown", TrajectoryStorageFormat::Both => "both (legacy)", } @@ -158,7 +158,7 @@ pub fn dataset_display( .unwrap_or_else(|| dir.display().to_string())) } TrajectoryStorageFormat::Both => Ok(dir.display().to_string()), - _ => session_vortex_path(&session).map(|p| p.display().to_string()), + _ => session_lance_path(&session).map(|p| p.display().to_string()), } } @@ -172,7 +172,7 @@ mod tests { let layers = LayerStats { event_rows: 30, markdown_blocks: 30, - event_log_path: "store/a/run/events.vortex".into(), + event_log_path: "store/a/run/events.lance".into(), markdown_path: Some("store/a/run/0001.md".into()), note: String::new(), }; @@ -184,18 +184,18 @@ mod tests { } #[test] - fn detect_primary_flat_vortex_canonical_when_counts_match() { + fn detect_primary_flat_lance_canonical_when_counts_match() { let layers = LayerStats { event_rows: 2, markdown_blocks: 2, - event_log_path: "store/a/s/events.vortex".into(), + event_log_path: "store/a/s/events.lance".into(), markdown_path: Some("store/a/s/s.md".into()), note: String::new(), }; let session = StoryCoords::new("store", "a", "s", None); assert_eq!( detect_story_primary_layer(&layers, &session), - TrajectoryStorageFormat::Vortex + TrajectoryStorageFormat::Lance ); } @@ -204,7 +204,7 @@ mod tests { let layers = LayerStats { event_rows: 10, markdown_blocks: 3, - event_log_path: "store/a/s/events.vortex".into(), + event_log_path: "store/a/s/events.lance".into(), markdown_path: Some("store/a/s/s.md".into()), note: String::new(), }; @@ -216,13 +216,13 @@ mod tests { } #[tokio::test] - async fn resolve_append_auto_on_empty_session_defaults_vortex() { + async fn resolve_append_auto_on_empty_session_defaults_lance() { let storage = tempfile::tempdir().unwrap(); let storage = storage.path().to_string_lossy().into_owned(); let fmt = resolve_for_append(&storage, "a", "s", None, TrajectoryStorageFormat::Auto) .await .unwrap(); - assert_eq!(fmt, TrajectoryStorageFormat::Vortex); + assert_eq!(fmt, TrajectoryStorageFormat::Lance); } #[tokio::test] @@ -250,7 +250,7 @@ mod tests { let both = resolve_for_append(&storage, "a", "s", None, TrajectoryStorageFormat::Both) .await .unwrap(); - assert_eq!(both, TrajectoryStorageFormat::Vortex); + assert_eq!(both, TrajectoryStorageFormat::Lance); } #[tokio::test] diff --git a/crates/persisting-engine/src/trajectory/store/vortex.rs b/crates/persisting-engine/src/trajectory/store/lance.rs similarity index 68% rename from crates/persisting-engine/src/trajectory/store/vortex.rs rename to crates/persisting-engine/src/trajectory/store/lance.rs index 06c15de..6f1f56c 100644 --- a/crates/persisting-engine/src/trajectory/store/vortex.rs +++ b/crates/persisting-engine/src/trajectory/store/lance.rs @@ -1,114 +1,88 @@ -//! Vortex event log backend (canonical trajectory store, schema v1). +//! Lance event log backend (canonical trajectory store, schema v1). //! -//! Capture runs use one run-level table at `{run}/events.vortex`; `session_id` +//! Capture runs use one run-level dataset at `{run}/events.lance`; `session_id` //! filters rows when replaying one story view. -use std::path::{Path, PathBuf}; -use std::sync::OnceLock; +use std::path::Path; +use std::sync::Arc; use anyhow::{Context, Result}; -use arrow_array::cast::AsArray; -use arrow_array::RecordBatch; -use arrow_schema::{DataType, Field, Schema as ArrowSchema}; +use futures::TryStreamExt; +use lance::dataset::{InsertBuilder, WriteMode, WriteParams}; +use lance::deps::arrow_array::RecordBatch; +use lance::Dataset; +use lance::Error as LanceError; use persisting_capture::event_row::EventRow; -use vortex::array::arrow::{ArrowSessionExt, FromArrowArray}; -use vortex::array::stream::ArrayStreamExt; -use vortex::array::VortexSessionExecute; -use vortex::file::OpenOptionsSessionExt; -use vortex::file::WriteOptionsSessionExt; -use vortex::session::VortexSession; -use vortex::VortexSessionDefault; use super::rows::{ reassign_global_seq, record_batch_from_rows, replay_records_from_batch, rows_for_lines, rows_from_batch, schema_columns_note, trajectory_schema, }; use super::{ - session_vortex_path, TrajectoryAppendOutcome, TrajectoryReplayOutcome, TrajectorySession, + session_lance_path, TrajectoryAppendOutcome, TrajectoryReplayOutcome, TrajectorySession, TrajectoryStatsOutcome, }; -fn vortex_session() -> &'static VortexSession { - static SESSION: OnceLock = OnceLock::new(); - SESSION.get_or_init(VortexSession::default) -} - -fn vortex_err(e: vortex::error::VortexError) -> anyhow::Error { - anyhow::anyhow!("{e}") -} - -fn vortex_to_record_batch( - array: vortex::array::ArrayRef, - schema: &ArrowSchema, -) -> Result { - let mut ctx = vortex_session().create_execution_ctx(); - let target = Field::new( - "", - DataType::Struct(schema.fields.clone()), - array.dtype().is_nullable(), - ); - let arrow = vortex_session() - .arrow() - .execute_arrow(array, Some(&target), &mut ctx) - .map_err(vortex_err)?; - Ok(RecordBatch::from(arrow.as_struct())) +async fn dataset_exists(uri: &str) -> Result { + match Dataset::open(uri).await { + Ok(_) => Ok(true), + Err(e) if matches!(e, LanceError::DatasetNotFound { .. }) => Ok(false), + Err(e) => Err(anyhow::anyhow!("{:#}", e)), + } } -async fn read_all_rows(path: &Path) -> Result> { - if !path.is_file() { +async fn read_all_rows(uri: &str) -> Result> { + if !dataset_exists(uri).await? { return Ok(Vec::new()); } - let file = vortex_session() - .open_options() - .open_path(path) + let ds = Dataset::open(uri) .await - .map_err(vortex_err)?; - let array = file + .with_context(|| format!("open trajectory Lance dataset {uri}"))?; + let stream = ds .scan() - .map_err(vortex_err)? - .into_array_stream() - .map_err(vortex_err)? - .read_all() + .try_into_stream() .await - .map_err(vortex_err)?; - if array.len() == 0 { - return Ok(Vec::new()); + .with_context(|| format!("scan trajectory Lance dataset {uri}"))?; + let batches: Vec = stream + .try_collect() + .await + .with_context(|| format!("collect trajectory Lance batches {uri}"))?; + let mut rows = Vec::new(); + for batch in &batches { + rows.extend(rows_from_batch(batch)?); } - let schema = trajectory_schema(); - let batch = vortex_to_record_batch(array, schema.as_ref())?; - rows_from_batch(&batch) + Ok(rows) } -async fn write_all_rows(path: &PathBuf, rows: &[EventRow]) -> Result<()> { +async fn write_all_rows(uri: &str, rows: &[EventRow]) -> Result<()> { if rows.is_empty() { - if path.is_file() { - tokio::fs::remove_file(path) + if Path::new(uri).exists() { + tokio::fs::remove_dir_all(uri) .await - .with_context(|| format!("remove empty vortex file {}", path.display()))?; + .with_context(|| format!("remove empty Lance dataset {uri}"))?; } return Ok(()); } let schema = trajectory_schema(); let batch = record_batch_from_rows(schema, rows)?; - let array = vortex::array::ArrayRef::from_arrow(batch, false).map_err(vortex_err)?; - let stream = array.to_array_stream(); - let tmp = path.with_extension("vortex.tmp"); - if let Some(parent) = path.parent() { + if let Some(parent) = Path::new(uri).parent() { tokio::fs::create_dir_all(parent) .await .with_context(|| format!("create_dir_all {}", parent.display()))?; } - let mut file = tokio::fs::File::create(&tmp) - .await - .with_context(|| format!("create {}", tmp.display()))?; - vortex_session() - .write_options() - .write(&mut file, stream) - .await - .map_err(vortex_err)?; - tokio::fs::rename(&tmp, path) + let mode = if dataset_exists(uri).await? { + WriteMode::Overwrite + } else { + WriteMode::Create + }; + InsertBuilder::new(uri) + .with_params(&WriteParams { + mode, + ..Default::default() + }) + .execute(vec![batch]) .await - .with_context(|| format!("rename {} -> {}", tmp.display(), path.display()))?; + .with_context(|| format!("write trajectory Lance dataset {uri}"))?; Ok(()) } @@ -122,15 +96,16 @@ fn filter_session_rows(rows: Vec, session: &TrajectorySession) -> Vec< } pub fn display_path(session: &TrajectorySession) -> Result { - Ok(session_vortex_path(session)?.to_string_lossy().into_owned()) + Ok(session_lance_path(session)?.to_string_lossy().into_owned()) } pub async fn distinct_session_ids_in_run(run: &TrajectorySession) -> Result> { - let path = session_vortex_path(run)?; - if !path.is_file() { + let path = session_lance_path(run)?; + let uri = path.to_string_lossy().into_owned(); + if !dataset_exists(&uri).await? { return Ok(Vec::new()); } - let rows = read_all_rows(&path).await?; + let rows = read_all_rows(&uri).await?; let mut ids: Vec = rows.into_iter().filter_map(|row| row.session_id).collect(); ids.sort(); ids.dedup(); @@ -138,21 +113,23 @@ pub async fn distinct_session_ids_in_run(run: &TrajectorySession) -> Result Result { - let path = session_vortex_path(session)?; - if !path.is_file() { + let path = session_lance_path(session)?; + let uri = path.to_string_lossy().into_owned(); + if !dataset_exists(&uri).await? { return Ok(false); } - Ok(!filter_session_rows(read_all_rows(&path).await?, session).is_empty()) + Ok(!filter_session_rows(read_all_rows(&uri).await?, session).is_empty()) } pub async fn overwrite_session_lines( session: &TrajectorySession, lines: &[String], ) -> Result { - let path = session_vortex_path(session)?; + let path = session_lance_path(session)?; + let uri = path.to_string_lossy().into_owned(); let replacement_rows = rows_for_lines(&session.session_id, 0, lines)?; let mut merged = Vec::new(); - let existing = read_all_rows(&path).await?; + let existing = read_all_rows(&uri).await?; if !existing.is_empty() { let mut inserted_replacement = false; for row in existing { @@ -172,7 +149,7 @@ pub async fn overwrite_session_lines( merged = replacement_rows; } reassign_global_seq(&mut merged); - write_all_rows(&path, &merged).await?; + write_all_rows(&uri, &merged).await?; Ok(lines.len()) } @@ -180,19 +157,63 @@ pub async fn append( session: &TrajectorySession, lines: &[String], ) -> Result { - let path = session_vortex_path(session)?; + let path = session_lance_path(session)?; + let uri = path.to_string_lossy().into_owned(); let accepted = lines.len(); - let mut rows = read_all_rows(&path).await?; - let base_seq = rows.len() as i64; - rows.extend(rows_for_lines(&session.session_id, base_seq, lines)?); - write_all_rows(&path, &rows).await?; + let base_seq = if dataset_exists(&uri).await? { + Dataset::open(&uri) + .await + .with_context(|| format!("open trajectory Lance dataset {uri}"))? + .count_rows(None) + .await + .context("count trajectory Lance rows")? as i64 + } else { + 0 + }; + let new_rows = rows_for_lines(&session.session_id, base_seq, lines)?; + if new_rows.is_empty() { + return Ok(TrajectoryAppendOutcome { + accepted_lines: 0, + persisted_units: 0, + note: format!( + "Lance v1: 0 row(s) at {uri} (columns: {})", + schema_columns_note() + ), + }); + } + let batch = record_batch_from_rows(trajectory_schema(), &new_rows)?; + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent) + .await + .with_context(|| format!("create_dir_all {}", parent.display()))?; + } + if dataset_exists(&uri).await? { + let ds = Arc::new( + Dataset::open(&uri) + .await + .with_context(|| format!("open trajectory Lance dataset {uri}"))?, + ); + InsertBuilder::new(ds) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![batch]) + .await + .with_context(|| format!("append trajectory Lance dataset {uri}"))?; + } else { + InsertBuilder::new(uri.as_str()) + .execute(vec![batch]) + .await + .with_context(|| format!("create trajectory Lance dataset {uri}"))?; + } Ok(TrajectoryAppendOutcome { accepted_lines: accepted, persisted_units: accepted, note: format!( - "Vortex v1: {} row(s) at {} (columns: {})", + "Lance v1: {} row(s) at {} (columns: {})", accepted, - path.display(), + uri, schema_columns_note() ), }) @@ -203,14 +224,12 @@ pub async fn replay( offset: usize, limit: Option, ) -> Result { - let path = session_vortex_path(session)?; - if !path.is_file() { - anyhow::bail!( - "trajectory Vortex file does not exist at {}", - path.display() - ); + let path = session_lance_path(session)?; + let uri = path.to_string_lossy().into_owned(); + if !dataset_exists(&uri).await? { + anyhow::bail!("trajectory Lance dataset does not exist at {uri}"); } - let mut rows = filter_session_rows(read_all_rows(&path).await?, session); + let mut rows = filter_session_rows(read_all_rows(&uri).await?, session); if offset > 0 { rows = rows.into_iter().skip(offset).collect(); } @@ -223,35 +242,32 @@ pub async fn replay( Ok(TrajectoryReplayOutcome { records, note: format!( - "Replay Vortex v1 at {}: session_id={}, ordered by 'seq', offset={}, limit={:?}.", - path.display(), + "Replay Lance v1 at {uri}: session_id={}, ordered by 'seq', offset={offset}, limit={limit:?}.", session.session_id, - offset, - limit ), }) } pub async fn stats(session: &TrajectorySession) -> Result { - let path = session_vortex_path(session)?; + let path = session_lance_path(session)?; let display = path.to_string_lossy().into_owned(); - if !path.is_file() { + if !dataset_exists(&display).await? { return Ok(TrajectoryStatsOutcome { dataset: display, row_count: 0, manifest_version: None, status: "missing".to_string(), - note: "No Vortex event log at this path yet; use trajectory add first.".to_string(), + note: "No Lance event log at this path yet; use trajectory add first.".to_string(), }); } - let rows = filter_session_rows(read_all_rows(&path).await?, session); + let rows = filter_session_rows(read_all_rows(&display).await?, session); Ok(TrajectoryStatsOutcome { dataset: display.clone(), row_count: rows.len(), manifest_version: None, status: "ok".to_string(), note: format!( - "Vortex v1 [{}]; session_id={}; file={}", + "Lance v1 [{}]; session_id={}; dataset={}", schema_columns_note(), session.session_id, display @@ -301,7 +317,7 @@ mod tests { } #[tokio::test] - async fn staged_commit_leaves_no_tmp_after_successful_write() { + async fn append_creates_lance_dataset() { let dir = tempfile::tempdir().unwrap(); let storage = dir.path().join("store"); std::fs::create_dir_all(&storage).unwrap(); @@ -310,31 +326,24 @@ mod tests { append(&session, &[note_line("one")]).await.unwrap(); - let path = session_vortex_path(&session).unwrap(); - let tmp = path.with_extension("vortex.tmp"); - assert!(path.is_file(), "committed vortex file should exist"); + let path = session_lance_path(&session).unwrap(); assert!( - !tmp.exists(), - "staged tmp must be renamed away after commit" + dataset_exists(&path.to_string_lossy()).await.unwrap(), + "committed Lance dataset should exist" ); } #[tokio::test] - async fn staged_commit_replaces_stale_tmp_on_next_append() { + async fn append_then_append_preserves_rows() { let dir = tempfile::tempdir().unwrap(); let storage = dir.path().join("store"); std::fs::create_dir_all(&storage).unwrap(); let storage_s = storage.to_string_lossy().to_string(); let session = flat_session(&storage_s, "agent", "sess"); - let path = session_vortex_path(&session).unwrap(); - let tmp = path.with_extension("vortex.tmp"); append(&session, &[note_line("first")]).await.unwrap(); - std::fs::write(&tmp, b"stale partial write").unwrap(); - append(&session, &[note_line("second")]).await.unwrap(); - assert!(!tmp.exists()); let replay = replay(&session, 0, None).await.unwrap(); assert_eq!(replay.records.len(), 2); assert_eq!(payload_content(&replay.records[0]), "first"); @@ -342,7 +351,7 @@ mod tests { } #[tokio::test] - async fn session_partition_replay_isolates_stories_in_shared_run_file() { + async fn session_partition_replay_isolates_stories_in_shared_run_dataset() { let dir = tempfile::tempdir().unwrap(); let storage = dir.path().join("store"); std::fs::create_dir_all(&storage).unwrap(); @@ -361,11 +370,11 @@ mod tests { .await .unwrap(); - let vortex_path = session_vortex_path(&main).unwrap(); + let lance_path = session_lance_path(&main).unwrap(); assert_eq!( - session_vortex_path(&sub_a).unwrap(), - vortex_path, - "run-level sessions share one events.vortex" + session_lance_path(&sub_a).unwrap(), + lance_path, + "run-level sessions share one events.lance" ); let main_replay = replay(&main, 0, None).await.unwrap(); @@ -459,7 +468,7 @@ mod tests { .await .unwrap(); - let all_rows = read_all_rows(&session_vortex_path(&main).unwrap()) + let all_rows = read_all_rows(&session_lance_path(&main).unwrap().to_string_lossy()) .await .unwrap(); assert_eq!(all_rows.len(), 3); @@ -473,7 +482,7 @@ mod tests { } #[tokio::test] - async fn append_uses_file_wide_seq_base_across_partitions() { + async fn append_uses_dataset_wide_seq_base_across_partitions() { let dir = tempfile::tempdir().unwrap(); let storage = dir.path().join("store"); std::fs::create_dir_all(&storage).unwrap(); @@ -488,7 +497,7 @@ mod tests { .unwrap(); append(&sub, &[note_line("s1")]).await.unwrap(); - let rows = read_all_rows(&session_vortex_path(&main).unwrap()) + let rows = read_all_rows(&session_lance_path(&main).unwrap().to_string_lossy()) .await .unwrap(); assert_eq!(rows.len(), 3); @@ -499,7 +508,7 @@ mod tests { } #[tokio::test] - async fn large_append_produces_valid_vortex_with_multiple_internal_partitions() { + async fn large_append_produces_valid_lance_dataset() { let dir = tempfile::tempdir().unwrap(); let storage = dir.path().join("store"); std::fs::create_dir_all(&storage).unwrap(); diff --git a/crates/persisting-engine/src/trajectory/store/mod.rs b/crates/persisting-engine/src/trajectory/store/mod.rs index d435290..ab63d7b 100644 --- a/crates/persisting-engine/src/trajectory/store/mod.rs +++ b/crates/persisting-engine/src/trajectory/store/mod.rs @@ -1,23 +1,23 @@ -//! Unified trajectory storage: Vortex canonical + TLV Markdown materialized view. +//! Unified trajectory storage: Lance canonical + TLV Markdown materialized view. +pub(crate) mod lance; mod markdown; pub(crate) mod rows; -pub(crate) mod vortex; -pub(crate) use vortex::overwrite_session_lines; +pub(crate) use lance::overwrite_session_lines; use std::path::PathBuf; use anyhow::Result; use async_trait::async_trait; -use persisting_capture::story_coords::{story_vortex_event_path, StoryCoords}; +use persisting_capture::story_coords::{story_lance_event_path, StoryCoords}; use persisting_proto::TrajectoryStorageFormat; -/// Story coordinates for Vortex/Markdown backends (capture [`StoryCoords`]). +/// Story coordinates for Lance/Markdown backends (capture [`StoryCoords`]). pub type TrajectorySession = StoryCoords; -pub(crate) fn session_vortex_path(session: &StoryCoords) -> Result { - story_vortex_event_path( +pub(crate) fn session_lance_path(session: &StoryCoords) -> Result { + story_lance_event_path( &session.storage, &session.agent_id, &session.session_id, @@ -42,7 +42,7 @@ pub struct TrajectoryReplayOutcome { pub note: String, } -/// Backend-specific stats (`row_count` = Vortex rows or markdown blocks). +/// Backend-specific stats (`row_count` = Lance rows or markdown blocks). #[derive(Debug, Clone, PartialEq, Eq)] pub struct TrajectoryStatsOutcome { pub dataset: String, @@ -52,7 +52,7 @@ pub struct TrajectoryStatsOutcome { pub note: String, } -/// Physical trajectory store: Vortex event log or session markdown file(s). +/// Physical trajectory store: Lance event log or session markdown file(s). #[async_trait] pub trait TrajectoryStore: Send + Sync { fn format(&self) -> TrajectoryStorageFormat; @@ -77,26 +77,26 @@ pub trait TrajectoryStore: Send + Sync { async fn stats(&self, session: &TrajectorySession) -> Result; } -/// Vortex backend: raw event log at `{run}/events.vortex`. +/// Lance backend: raw event log at `{run}/events.lance`. #[derive(Debug, Clone, Copy, Default)] -pub struct VortexTrajectoryStore; +pub struct LanceTrajectoryStore; /// Markdown backend: `0001.md` (TLV blocks) under the session directory. #[derive(Debug, Clone, Copy, Default)] pub struct MarkdownTrajectoryStore; #[async_trait] -impl TrajectoryStore for VortexTrajectoryStore { +impl TrajectoryStore for LanceTrajectoryStore { fn format(&self) -> TrajectoryStorageFormat { - TrajectoryStorageFormat::Vortex + TrajectoryStorageFormat::Lance } fn display_path(&self, session: &TrajectorySession) -> Result { - vortex::display_path(session) + lance::display_path(session) } async fn exists(&self, session: &TrajectorySession) -> Result { - vortex::exists(session).await + lance::exists(session).await } async fn append( @@ -104,7 +104,7 @@ impl TrajectoryStore for VortexTrajectoryStore { session: &TrajectorySession, lines: &[String], ) -> Result { - vortex::append(session, lines).await + lance::append(session, lines).await } async fn replay( @@ -113,11 +113,11 @@ impl TrajectoryStore for VortexTrajectoryStore { offset: usize, limit: Option, ) -> Result { - vortex::replay(session, offset, limit).await + lance::replay(session, offset, limit).await } async fn stats(&self, session: &TrajectorySession) -> Result { - vortex::stats(session).await + lance::stats(session).await } } @@ -161,8 +161,8 @@ impl TrajectoryStore for MarkdownTrajectoryStore { pub fn store_for_append(format: TrajectoryStorageFormat) -> Box { match format { TrajectoryStorageFormat::Markdown => Box::new(MarkdownTrajectoryStore), - TrajectoryStorageFormat::Vortex | TrajectoryStorageFormat::Both => { - Box::new(VortexTrajectoryStore) + TrajectoryStorageFormat::Lance | TrajectoryStorageFormat::Both => { + Box::new(LanceTrajectoryStore) } TrajectoryStorageFormat::Auto => { panic!("resolve_for_append must resolve Auto before store_for_append") @@ -173,9 +173,9 @@ pub fn store_for_append(format: TrajectoryStorageFormat) -> Box Box { match format { - TrajectoryStorageFormat::Vortex + TrajectoryStorageFormat::Lance | TrajectoryStorageFormat::Both - | TrajectoryStorageFormat::Auto => Box::new(VortexTrajectoryStore), + | TrajectoryStorageFormat::Auto => Box::new(LanceTrajectoryStore), TrajectoryStorageFormat::Markdown => Box::new(MarkdownTrajectoryStore), } } @@ -187,8 +187,8 @@ mod tests { #[test] fn append_store_covers_formats() { assert!(matches!( - store_for_append(TrajectoryStorageFormat::Vortex).format(), - TrajectoryStorageFormat::Vortex + store_for_append(TrajectoryStorageFormat::Lance).format(), + TrajectoryStorageFormat::Lance )); assert!(matches!( store_for_append(TrajectoryStorageFormat::Markdown).format(), @@ -196,19 +196,19 @@ mod tests { )); assert!(matches!( store_for_append(TrajectoryStorageFormat::Both).format(), - TrajectoryStorageFormat::Vortex + TrajectoryStorageFormat::Lance )); } #[test] - fn read_store_prefers_vortex_for_both_and_auto() { + fn read_store_prefers_lance_for_both_and_auto() { assert!(matches!( store_for_read(TrajectoryStorageFormat::Both).format(), - TrajectoryStorageFormat::Vortex + TrajectoryStorageFormat::Lance )); assert!(matches!( store_for_read(TrajectoryStorageFormat::Auto).format(), - TrajectoryStorageFormat::Vortex + TrajectoryStorageFormat::Lance )); } } diff --git a/crates/persisting-engine/src/trajectory/store/rows.rs b/crates/persisting-engine/src/trajectory/store/rows.rs index d1b36b9..41b5aa8 100644 --- a/crates/persisting-engine/src/trajectory/store/rows.rs +++ b/crates/persisting-engine/src/trajectory/store/rows.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use anyhow::{Context, Result}; -use arrow_array::{Array, Int64Array, RecordBatch, StringArray}; -use arrow_schema::{DataType, Field, Schema as ArrowSchema}; +use lance::deps::arrow_array::{Array, Int64Array, RecordBatch, StringArray}; +use lance::deps::arrow_schema::{DataType, Field, Schema as ArrowSchema}; use persisting_capture::event_row::{engine_line_to_event_row, event_row_to_replay_json, EventRow}; use crate::trajectory::{ diff --git a/crates/persisting-proto/src/messages.rs b/crates/persisting-proto/src/messages.rs index 09710e7..335aaae 100644 --- a/crates/persisting-proto/src/messages.rs +++ b/crates/persisting-proto/src/messages.rs @@ -279,20 +279,20 @@ pub struct SearchImportLanceResponse { // Trajectory // --------------------------------------------------------------------------- -/// Physical storage layer for a session trajectory (Vortex canonical + TLV Markdown view). +/// Physical storage layer for a session trajectory (Lance canonical + TLV Markdown view). #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum TrajectoryStorageFormat { - /// Read: Vortex if both layers exist else the available layer. Append: always Vortex. + /// Read: Lance if both layers exist else the available layer. Append: always Lance. /// Stats: summarize both layers when present. #[default] Auto, - /// Vortex raw event log (canonical). - Vortex, + /// Lance raw event log (canonical). + Lance, /// TLV Markdown session file (materialized / human-readable view). #[serde(rename = "markdown", alias = "tlv")] Markdown, - /// Legacy alias: append writes [`Vortex`](Self::Vortex) only; read/stats like [`Auto`](Self::Auto). + /// Legacy alias: append writes [`Lance`](Self::Lance) only; read/stats like [`Auto`](Self::Auto). #[serde(alias = "both")] Both, } @@ -388,7 +388,7 @@ pub struct TrajectoryStatsResponse { pub dataset: String, /// Row count in the event log (`0` when file is missing). pub row_count: usize, - /// Reserved for future versioning metadata (`None` for Vortex). + /// Reserved for future versioning metadata (`None` for Lance). pub manifest_version: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub judge: Option, diff --git a/docs/src/design/architecture.zh.md b/docs/src/design/architecture.zh.md index 81f5486..a071023 100644 --- a/docs/src/design/architecture.zh.md +++ b/docs/src/design/architecture.zh.md @@ -58,7 +58,7 @@ Lance Dataset (磁盘) 队列持久化是 Persisting 数据面**已可用**的一层。 -**轨迹存储**（捕获代理、Vortex / Markdown 双视图）已独立落地，见 [轨迹存储模型](trajectory_storage.zh.md)。 +**轨迹存储**（捕获代理、Lance / Markdown 双视图）已独立落地，见 [轨迹存储模型](trajectory.md)。 下一阶段 **Tensor Memory** 将补全核心分布式分层内存能力： diff --git a/docs/src/design/capture.md b/docs/src/design/capture.md index 02b6b3c..d7ef93e 100644 --- a/docs/src/design/capture.md +++ b/docs/src/design/capture.md @@ -69,7 +69,7 @@ Capture 不是通用 API 网关的替代品，而是围绕 **Agent 轨迹（traj - 单次或长期 `traj capture` / `traj proxy` 下，对 **Claude Code / Codex** 的对话采集； - Claude Code 场景的 history replay 去重、subagent 分轨； - Codex 场景的 Responses ↔ Completions 桥接与上下文注入过滤； -- Vortex 全量事件 + Markdown 物化视图的双层存储； +- Lance 全量事件 + Markdown 物化视图的双层存储； - 轻量模型路由、协议桥接（Messages / Completions / Responses 等）。 **不替代** @@ -86,7 +86,7 @@ Agent 客户端 ▼ ┌─────────────────────────────────────┐ │ Persisting Capture（本文） │ ← 代理 + 采集 + 物化 -│ · 事件日志（Vortex） │ +│ · 事件日志（Lance） │ │ · 人读轨迹（Markdown） │ └──────────────┬──────────────────────┘ │ CaptureRecord / 轨迹文件 @@ -106,10 +106,10 @@ Agent 客户端 |------|------| | **观测不阻断** | 用户请求的延迟与成功率优先；采集失败写入 dead letter，**不**因写盘失败而中断 HTTP 响应。 | | **Story 边界** | 协议差异在「进入故事模型」之前消化；故事层只谈谁、第几轮、哪次调用、发生了什么。 | -| **Vortex 为事实源** | 全量事件以结构化记录 append 到 `events.vortex`；Markdown 是**物化视图**，允许有损过滤。 | +| **Lance 为事实源** | 全量事件以结构化记录 append 到 `events.lance`；Markdown 是**物化视图**，允许有损过滤。 | | **写读对称** | 在线维护的「轮次 / 调用」读模型，与离线从事件日志重放的结果一致（通过对账与测试保证）。 | | **单一可见文本语义** | 用户/助手可见正文只有一套提取规则，供轮次索引、Markdown 正文、过滤策略共用。 | -| **单一写入门** | 每条进入 Vortex 的会话事件，经统一的故事线 Actor 路径落盘，避免双写竞态。 | +| **单一写入门** | 每条进入 Lance 的会话事件，经统一的故事线 Actor 路径落盘，避免双写竞态。 | --- @@ -231,22 +231,22 @@ flowchart LR │ ┌─────┴─────┐ ▼ ▼ - Vortex Markdown - (events.vortex) (物化视图) + Lance Markdown + (events.lance/) (物化视图) ``` | 组件 | 职责 | |------|------| | **Proxy** | 唯一 HTTP 入口；在转发前后发射采集事件；流式场景下节流草稿事件。 | -| **采集引擎** | 将事件转为记录；按故事线串行 apply；协调 Vortex 与 Markdown 写入。 | +| **采集引擎** | 将事件转为记录；按故事线串行 apply；协调 Lance 与 Markdown 写入。 | | **Run 协调** | 跨故事线的 spawn 关联、主从路由、记录 enrichment。 | -| **故事线处理** | 每 Story 一个串行执行体：轮次状态、Vortex 序号、Live Markdown、摘要刷新。 | +| **故事线处理** | 每 Story 一个串行执行体：轮次状态、Lance 序号、Live Markdown、摘要刷新。 | | **会话索引** | 轻量 `sessions.json`：列表、token、费用估算、状态；批量刷盘。 | | **对账与 dead letter** | 运行结束三轨校验；失败事件可重放。 | ### 5.2 集成方式（概念） -- **库嵌入**：Rust 工程可挂载 Proxy 与 `CaptureEngine`，自行提供存储 sink（默认对接 Vortex 管线）。 +- **库嵌入**：Rust 工程可挂载 Proxy 与 `CaptureEngine`，自行提供存储 sink（默认对接 Lance 管线）。 - **CLI**：`persisting traj capture` 包装子进程；`persisting traj proxy` 长期监听。 - **配置**：TOML 声明监听地址、模型路由、采集级别、存储根目录；无需改 Agent 源码。 @@ -268,12 +268,12 @@ sequenceDiagram participant Proxy as Capture Proxy participant Engine as 采集引擎 participant Upstream as 上游模型 - participant Vortex as events.vortex + participant Lance as events.lance participant MD as Markdown 视图 Agent->>Proxy: 对话请求 Proxy->>Engine: 请求事件（异步，不阻塞） - Engine->>Vortex: 记录 user 侧事件 + Engine->>Lance: 记录 user 侧事件 Engine->>MD: 追加/更新 user 块 Proxy->>Upstream: 转发（可能协议转换） @@ -287,10 +287,10 @@ sequenceDiagram alt 客户端断开 Proxy->>Engine: 取消事件 - Engine->>Vortex: 仅记录取消（不进 Markdown） + Engine->>Lance: 仅记录取消（不进 Markdown） else 正常结束 Proxy->>Engine: 完成事件 - Engine->>Vortex: 记录完整响应事件 + Engine->>Lance: 记录完整响应事件 Engine->>MD: 定稿 assistant 块 end ``` @@ -298,7 +298,7 @@ sequenceDiagram 要点： 1. **Proxy 不等待**整段采集完成再响应；事件先入 WAL，再进入 per-story 有序队列。 -2. **草稿只更新 Markdown**；完整响应以**一条**事件进入 Vortex，避免 partial 行污染事实源。 +2. **草稿只更新 Markdown**；完整响应以**一条**事件进入 Lance，避免 partial 行污染事实源。 3. 慢客户端通过有界队列对上游施加背压，避免无限缓冲。 ### 6.2 采集事件与记录类型 @@ -307,15 +307,15 @@ sequenceDiagram | 事件 | 典型效果（Dialogue 级别） | |------|---------------------------| -| 请求到达 | Vortex：请求记录；Markdown：user 块 | +| 请求到达 | Lance：请求记录；Markdown：user 块 | | 流式草稿 | 仅 Markdown：assistant 草稿（可原地覆盖） | -| 响应完成 | Vortex：流式/完整响应记录；Markdown：定稿 assistant | -| 调用取消 | 仅 Vortex：取消记录 | -| Spawn 关联 | Vortex + Markdown：关联元数据（不当作可跳过噪音） | +| 响应完成 | Lance：流式/完整响应记录；Markdown：定稿 assistant | +| 调用取消 | 仅 Lance：取消记录 | +| Spawn 关联 | Lance + Markdown：关联元数据（不当作可跳过噪音） | **采集级别**（Summary / Dialogue / Full）控制记录粒度；生产默认 **Dialogue**： -| 级别 | Vortex / Markdown 摘要字段 | `payload.body` | +| 级别 | Lance / Markdown 摘要字段 | `payload.body` | |------|---------------------------|----------------| | `summary` | 仅 model、path、字节数 | ❌ | | `dialogue`（默认） | `user_content` / `assistant_content` 可见对话文本 | ❌ | @@ -330,13 +330,13 @@ sequenceDiagram ```text 助手输出: "你" → "你好" → "你好，我来帮你…" Markdown: [草稿] → [覆盖草稿] → [定稿] -Vortex: — — 一条最终响应事件 +Lance: — — 一条最终响应事件 ``` - 草稿块带明确标记；定稿时按 **call + 角色** 覆盖同一块，避免重复段落。 - 块头 schema 带版本号（`v: 1`），便于将来演进线格式而不改文件后缀。 -详见 [轨迹 Markdown 格式](trajectory_tlv_format.zh.md)。 +详见 [轨迹 Markdown 格式](trajectory-format.md)。 ### 6.4 可见对话提取（含多模态） @@ -369,16 +369,16 @@ Capture 在 **Dialogue** 级别下，从客户端原始 HTTP body（而非 upstr ## 7. 存储与一致性 -> 双层存储、目录约定、materialize/import 路径见 [轨迹存储模型](trajectory_storage.zh.md)。 +> 双层存储、目录约定、materialize/import 路径见 [轨迹存储模型](trajectory.md)。 ### 7.1 双层存储 -| | Vortex（事实源） | Markdown（物化视图） | +| | Lance（事实源） | Markdown（物化视图） | |---|----------------|----------------------| | **读者** | 程序、检索、replay | 人、git、review | | **完整性** | 无损（在采集级别内） | 有损：过滤内部与重复 history | -| **写入** | append 到 `events.vortex` | live upsert 或批量 append / 全量 materialize | -| **关系** | 行数 ≥ 块数（物化只减不增） | 从 Vortex 重建可修复漂移 | +| **写入** | append 到 `events.lance` | live upsert 或批量 append / 全量 materialize | +| **关系** | 行数 ≥ 块数（物化只减不增） | 从 Lance 重建可修复漂移 | ### 7.2 物化过滤（统一策略） @@ -387,7 +387,7 @@ Capture 在 **Dialogue** 级别下，从客户端原始 HTTP body（而非 upstr - 内部 `count_tokens`、影子模型预热； - Claude Code 式 **history replay**（用户消息计数未增加的重发）； - 无可见正文的空记录； -- 纯生命周期、仅-cancel 类记录（保留在 Vortex）。 +- 纯生命周期、仅-cancel 类记录（保留在 Lance）。 Spawn 关联等「对人仍有意义」的事件**不会**被误杀。 @@ -403,7 +403,7 @@ Spawn 关联等「对人仍有意义」的事件**不会**被误杀。 | 轨道 | 含义 | |------|------| | **Markdown** | 物化块中的 call 集合 | -| **Vortex** | 事件日志中应对话出现的 call 集合 | +| **Lance** | 事件日志中应对话出现的 call 集合 | | **Story** | 从事件重放得到的 call 集合 | 三者一致且结构检查通过，才认为「人读视图与事实源对齐」。不一致时应用 materialize 或排查 dead letter，而非直接信任 Markdown。 @@ -413,7 +413,7 @@ Spawn 关联等「对人仍有意义」的事件**不会**被误杀。 | 产物 | 作用 | |------|------| | 事件 WAL | 进程崩溃后重放未确认的采集事件 | -| dead letter | 应用失败或 Vortex 刷盘失败的留存与重放 | +| dead letter | 应用失败或 Lance 刷盘失败的留存与重放 | | 故事快照 | 退出时固化各 Story 的轮次读模型，供摘要与恢复 | --- @@ -437,7 +437,7 @@ Capture 内置**轻量 LLM 网关**，服务于「本地或团队固定上游 + ### 9.1 路由与存储键 -每个 HTTP 请求绑定一条**采集路由**：逻辑 session、磁盘上的 storage 键（决定 `.md` 文件名与 Vortex 事件日志路径）、可选 subagent 标识。 +每个 HTTP 请求绑定一条**采集路由**：逻辑 session、磁盘上的 storage 键（决定 `.md` 文件名与 Lance 事件日志路径）、可选 subagent 标识。 Capture run 下，子 Agent 通常写入 `agent-{id}.md`；主会话写入 `run-{id}.md` 或扁平 session 名。 ### 9.2 文件隔离不变式 @@ -450,16 +450,16 @@ Capture run 下，子 Agent 通常写入 `agent-{id}.md`；主会话写入 `run- 主 Agent 助手消息中的 spawn 提示与子 Agent 首包注册可能**时间错开**。系统用 Run 级注册表做延迟匹配与回填，使主会话在事后仍能看到「调用了哪个子 Agent、轨迹文件在哪」。 -### 9.4 单文件多 `session_id`（Claude run bucket） +### 9.4 单 run dataset 多 `session_id`（Claude run bucket） -一次 `traj capture` 的 run 目录通常只有一个 `events.vortex`，但行内 `session_id` **可能混存多个值**： +一次 `traj capture -f lance` 的 run 目录通常只有一个 `events.lance/` dataset，但行内 `session_id` **可能混存多个值**： | 典型来源 | `session_id` 取值 | |----------|-------------------| | Capture 生命周期 / run 头 | `run-{timestamp}-…`（与目录名一致） | | Claude Code 对话 HTTP | header 注入的 UUID（与 run id 不同） | -因此 **`traj stats` 扫描 agent 目录时**，对 run bucket（`session_id == root_session_id`）会先读 Vortex 中 distinct `session_id`，再**逐分区统计**，避免「第二个 session 显示 0 turns」。实现：`persisting-engine::trajectory::expand_story_locations`。详见 [轨迹存储 §7.1.1](trajectory_storage.zh.md#711-run-bucket-内多-session_id-分区)。 +因此 **`traj stats` 扫描 agent 目录时**，对 run bucket（`session_id == root_session_id`）会先读 Lance 中 distinct `session_id`，再**逐分区统计**，避免「第二个 session 显示 0 turns」。实现：`persisting-engine::trajectory::expand_story_locations`。详见 [轨迹存储 §7.1.1](trajectory.md#711-run-bucket-内多-session_id-分区)。 --- @@ -470,7 +470,7 @@ Capture run 下，子 Agent 通常写入 `agent-{id}.md`；主会话写入 `run- ```text 请求线程 ──► 发事件（写 WAL + 入队）──► 立即继续转发 │ - └──► 后台：有序 apply ──► Vortex / Markdown + └──► 后台：有序 apply ──► Lance / Markdown │ ├─ 成功 → 确认 WAL └─ 失败 → dead letter（不影响 HTTP） @@ -493,7 +493,7 @@ Capture run 下，子 Agent 通常写入 `agent-{id}.md`；主会话写入 `run- | **`traj capture`** | 包装一次 Agent 命令（如 `claude`、`codex`）；注入代理环境变量；结束打印会话摘要 | | **`traj proxy`** | 前台长期代理；多终端共用 | | **`traj proxy start`** | 后台守护进程（spawn `traj proxy`） | -| **仅 Vortex / 补 Markdown** | `-f vortex` 先落盘到 `events.vortex`，事后 `traj materialize` | +| **仅 Lance / 补 Markdown** | `-f lance` 只落盘到 `events.lance/`（capture 期间无 live md），事后 `traj materialize` | | **Dead letter 重放** | `traj replay-dead-letter` | 配置示例（节选）： @@ -520,17 +520,17 @@ api_key_env = "DEEPSEEK_API_KEY" | 方向 | 动机 | |------|------| -| **多模态 sidecar（Phase 1）** | 将 base64 / 生成图落盘到 `{run}/assets/`，Vortex 只存引用；支持 materialize 嵌图与可控 replay | +| **多模态 sidecar（Phase 1）** | 将 base64 / 生成图落盘到 `{run}/assets/`，Lance 只存引用；支持 materialize 嵌图与可控 replay | | **Cursor 实时采集与 import** | 与 Claude Code 对等的注入与 JSONL 导入 | -| Vortex 文件拆分与 compaction | 长 run 下单文件 `events.vortex` 过大时的拆分策略 | +| Lance dataset 拆分与 compaction | 长 run 下 `events.lance/` 过大时的拆分策略 | | WAL 与序号恢复增强 | 降低 crash 后重复 apply 与 seq 冲突风险 | | Markdown 追加日志 + 周期性 compact | 长会话 live upsert 的 IO 与 git diff 友好性 | | 外部定价表 | 摘要费用估算可配置 | | 故事读模型 enrich | 父子 Story、调用元数据与 spawn 完全闭环 | -| Vortex 列布局优化 | 更好利用列存检索，而非大 blob | +| Lance 列布局优化 | 更好利用列存检索，而非大 blob | | 协议面收敛 | 随行业 API 稳定，收缩长期维护的转换矩阵 | -块格式已通过 `v: 1` 预留兼容；详见 [轨迹 Markdown 格式 §2.6](trajectory_tlv_format.zh.md#26-大文件-upsert演进格式不变)。 +块格式已通过 `v: 1` 预留兼容；详见 [轨迹 Markdown 格式 §2.6](trajectory-format.md#26-大文件-upsert演进格式不变)。 --- @@ -538,12 +538,12 @@ api_key_env = "DEEPSEEK_API_KEY" | 文档 | 内容 | |------|------| -| [Capture 快速上手](../guide/capture_quickstart.md) | **上手**：构建 CLI、`traj capture`、查看轨迹、排错 | -| [轨迹存储模型](trajectory_storage.zh.md) | Vortex ↔ Markdown 数据流、materialize、import | -| [轨迹 Markdown 格式](trajectory_tlv_format.zh.md) | 块结构、字段规范、subagent 脚注、golden 示例 | -| [Traj Capture 子命令](cli_capture_command.zh.md) | `traj capture` / `traj proxy` 参数与操作 | -| [Trajectory / `traj` 命令](cli_trajectory_command.zh.md) | Capture 离线 Egress：add、truncate、stats、replay、extract、materialize | -| [CLI 整体架构](cli_architecture.zh.md) | Persisting 命令行体系 | +| [Capture 快速上手](../guide/capture.md) | **上手**：构建 CLI、`traj capture`、查看轨迹、排错 | +| [轨迹存储模型](trajectory.md) | Lance ↔ Markdown 数据流、materialize、import | +| [轨迹 Markdown 格式](trajectory-format.md) | 块结构、字段规范、subagent 脚注、golden 示例 | +| [Traj Capture 子命令](cli-capture.md) | `traj capture` / `traj proxy` 参数与操作 | +| [Trajectory / `traj` 命令](cli-traj.md) | Capture 离线 Egress：add、truncate、stats、replay、extract、materialize | +| [CLI 整体架构](cli.md) | Persisting 命令行体系 | **示例轨迹文件**（由编码器生成的 golden，可供格式对照）： diff --git a/docs/src/design/cli-capture.md b/docs/src/design/cli-capture.md index d8b1861..32d34f5 100644 --- a/docs/src/design/cli-capture.md +++ b/docs/src/design/cli-capture.md @@ -2,7 +2,7 @@ Capture 采集能力已并入 **`persisting traj`**（`trajectory` 全名）：`traj capture`（一次性）、`traj proxy`（长期代理）、`traj import`（事后导入），与 `traj stats` / `replay` / `materialize` 等同属轨迹 CLI。 -> **想先跑起来？** 见 [Capture 快速上手](../guide/capture_quickstart.md)。完整子命令列表见 [Traj 命令](cli_trajectory_command.zh.md)。 +> **想先跑起来？** 见 [Capture 快速上手](../guide/capture.md)。完整子命令列表见 [Traj 命令](cli-traj.md)。 --- @@ -10,13 +10,15 @@ Capture 采集能力已并入 **`persisting traj`**（`trajectory` 全名）：` ``` ┌─────────────────────┐ - LLM SDK / Agent ──│ 内嵌代理 (主路径) │──► Vortex event log + LLM SDK / Agent ──│ 内嵌代理 (主路径) │──► 按 `-f` 分叉 └──────────┬──────────┘ - │ -f md：CaptureEngine live upsert md - ▼ - TLV Markdown（主 run-*.md + sibling agent-*.md） + ┌───────────────────┴───────────────────┐ + │ -f md │ -f lance + ▼ ▼ + TLV Markdown（live upsert） events.lance/（Lance Append） + + reconcile.json （md 经 traj materialize） - IDE JSONL / 历史 OTLP ──► traj import ──► 合并进同一 store + IDE JSONL / 历史 OTLP ──► traj import ──► 合并进同一 store（选定一层） ``` | 路径 | 何时用 | @@ -24,7 +26,7 @@ Capture 采集能力已并入 **`persisting traj`**（`trajectory` 全名）：` | **代理捕获** | `traj capture` / `traj proxy` 实时采集 | | **事后导入** | `traj import` 补录 Claude Code JSONL 等 | -详见 [Capture 架构设计](capture_design.zh.md)。 +详见 [Capture 架构设计](capture.md)。 --- @@ -41,16 +43,16 @@ Capture 采集能力已并入 **`persisting traj`**（`trajectory` 全名）：` | **`traj import`** | IDE / 网关日志事后导入 | | **`traj replay-dead-letter`** | 重放 `.capture/dead_letter.jsonl` | -公共选项：`-o` store 根目录、`-c` 代理 TOML、**`-f md|vortex`**（见下）。 +公共选项：`-o` store 根目录、`-c` 代理 TOML、**`-f md|lance`**（见下）。 -### 格式 `-f md|vortex` +### 格式 `-f md|lance` | 值 | 运行时写入 | Markdown | |----|------------|----------| -| **`md`（默认）** | **仅** CaptureEngine **live upsert** md（不写 Vortex） | -| **`vortex`** | 仅 Vortex 事件日志（`{run}/events.vortex`） | +| **`md`（默认）** | **仅** CaptureEngine **live upsert** md（不写 Lance） | +| **`lance`** | 仅 Lance 事件日志（`{run}/events.lance/` 目录） | -`-f md` 时不自动全量 materialize；run 结束写 `.capture/reconcile.json`（md 与 replay 记录对账）；不一致时用 **`traj materialize`**（需已有 Vortex 层，见 [轨迹存储 §8.3](trajectory_storage.zh.md)）。 +`-f md` 时不自动全量 materialize；run 结束写 `.capture/reconcile.json`（md 与 replay 记录对账）；不一致时用 **`traj materialize`**（需已有 Lance 层，见 [轨迹存储 §8.3](trajectory.md)）。 --- @@ -59,7 +61,7 @@ Capture 采集能力已并入 **`persisting traj`**（`trajectory` 全名）：` 1. 进程内启动代理（不依赖已存在守护进程） 2. 分配 run session，写入 `session.started` 3. 为子进程注入 `HTTP_PROXY`、`OPENAI_BASE_URL` 等 -4. LLM 流量 → Vortex（`-f vortex`）或 CaptureEngine（`-f md`）live upsert Markdown +4. LLM 流量 → Lance（`-f lance`）或 CaptureEngine（`-f md`）live upsert Markdown 5. 子进程退出 → `session.ended` → 代理关闭 同一 `-o` 上若已有 `traj proxy` / `traj proxy start`，`traj capture` 会拒绝。 @@ -74,7 +76,7 @@ Capture 采集能力已并入 **`persisting traj`**（`trajectory` 全名）：` | **`traj proxy start`** | 后台（spawn `traj proxy` 子进程） | | **`traj proxy stop` / `list` / `status`** | 守护进程运维 | -环境变量示例见 [快速上手 §3](../guide/capture_quickstart.md)。**`traj proxy` 不会**为其他终端自动注入环境；需手动 `export` 或使用 `traj capture`。 +环境变量示例见 [快速上手 §3](../guide/capture.md)。**`traj proxy` 不会**为其他终端自动注入环境；需手动 `export` 或使用 `traj capture`。 `list` / `status` / `stop` 可省略 `-o`（最近 `start` 或 `PERSISTING_CAPTURE_STORAGE`）。 @@ -95,8 +97,8 @@ Capture 采集能力已并入 **`persisting traj`**（`trajectory` 全名）：` ## 6. 相关文档 -- [Traj 命令（完整列表）](cli_trajectory_command.zh.md) -- [Capture 架构设计](capture_design.zh.md) -- [轨迹存储模型](trajectory_storage.zh.md) -- [Capture 快速上手](../guide/capture_quickstart.md) +- [Traj 命令（完整列表）](cli-traj.md) +- [Capture 架构设计](capture.md) +- [轨迹存储模型](trajectory.md) +- [Capture 快速上手](../guide/capture.md) - [分步示例](../../examples/capture-walkthrough/README.md) diff --git a/docs/src/design/cli-search.md b/docs/src/design/cli-search.md index 172176f..f317786 100644 --- a/docs/src/design/cli-search.md +++ b/docs/src/design/cli-search.md @@ -27,7 +27,7 @@ persisting |------|----------|------| | `--core-lib ` | `PERSISTING_ENGINE_LIB` | 引擎动态库路径 | -`--core-lib` 的路径解析与加载策略见 [CLI 整体架构设计](cli_architecture.zh.md)。`search` 层本身无额外选项。 +`--core-lib` 的路径解析与加载策略见 [CLI 整体架构设计](cli.md)。`search` 层本身无额外选项。 --- diff --git a/docs/src/design/cli-traj.md b/docs/src/design/cli-traj.md index 073b9a7..fe2070b 100644 --- a/docs/src/design/cli-traj.md +++ b/docs/src/design/cli-traj.md @@ -1,8 +1,8 @@ # `persisting trajectory` / `traj` — 轨迹 CLI -**`traj`** 是轨迹的统一入口：包含 **Ingress**（`traj capture` / `traj proxy` / `traj import`）与 **Egress**（`stats` / `replay` / `materialize` / …）。在 Run/Story 坐标上读写 **Vortex 事实源**（`CaptureRecord`）与 **TLV Markdown 物化视图**，语义与 [Capture 架构](capture_design.zh.md) §4 一致。 +**`traj`** 是轨迹的统一入口：包含 **Ingress**（`traj capture` / `traj proxy` / `traj import`）与 **Egress**（`stats` / `replay` / `materialize` / …）。在 Run/Story 坐标上读写 **Lance 事实源**（`CaptureRecord`）与 **TLV Markdown 物化视图**，语义与 [Capture 架构](capture.md) §4 一致。 -短名：**`traj`**（`trajectory` 全名）。Capture 采集子命令见 [Capture 子命令设计](cli_capture_command.zh.md)。 +短名：**`traj`**（`trajectory` 全名）。Capture 采集子命令见 [Capture 子命令设计](cli-capture.md)。 --- @@ -13,7 +13,7 @@ | **Storage** | `` | 采集根目录 | | **Agent** | `--agent-id` | `{storage}/{agent_id}/` | | **Run** | `--root-session-id` | `{storage}/{agent_id}/{run_id}/` | -| **Story** | `--session-id` | Run 下主 session / subagent stem；Vortex 为 run 级 `events.vortex`（按 `session_id` 过滤），Markdown 为 `{key}.md` | +| **Story** | `--session-id` | Run 下主 session / subagent stem；Lance 为 run 级 `events.lance`（按 `session_id` 过滤），Markdown 为 `{key}.md` | Subagent：`{run}/subagents/{session_id}/`。路径可传 session 目录，CLI 自动推断上述字段。 @@ -36,11 +36,11 @@ Subagent：`{run}/subagents/{session_id}/`。路径可传 session 目录，CLI | 命令 | 说明 | |------|------| | **add** | TOML/JSONL/Markdown → `CaptureRecord` → **写入单一层** | -| **truncate** | 保留 Vortex 前 N 行（按 `seq`） | -| **stats** | 双层摘要；`--detail` 逐轮树；未指定 `--session-id` 时扫描 agent 下全部 run，并对 run bucket **展开** Vortex 内 distinct `session_id` 后逐分区统计（见 [轨迹存储 §7.1.1](trajectory_storage.zh.md#711-run-bucket-内多-session_id-分区)） | +| **truncate** | 保留 Lance 前 N 行（按 `seq`） | +| **stats** | 双层摘要；`--detail` 逐轮树；未指定 `--session-id` 时扫描 agent 下全部 run，并对 run bucket **展开** Lance 内 distinct `session_id` 后逐分区统计（见 [轨迹存储 §7.1.1](trajectory.md#711-run-bucket-内多-session_id-分区)） | | **replay** | 分页输出事件 JSON | | **extract** | 拷贝 Story/Run 目录树 | -| **materialize** | Vortex → Markdown 全量物化 | +| **materialize** | Lance → Markdown 全量物化 | ```text persisting traj capture [OPTIONS] -- @@ -54,33 +54,33 @@ persisting traj extract [OPTIONS] persisting traj materialize [OPTIONS] ``` -实现：`persisting-capture` 的 `storage::{story_coords, path_layout, egress, convert}`；`persisting-engine` 负责 Vortex/Markdown store 与 RPC。 +实现：`persisting-capture` 的 `storage::{story_coords, path_layout, egress, convert}`；`persisting-engine` 负责 Lance/Markdown store 与 RPC。 --- ## 3. 存储层（`--storage-format`） -两种物理层：**Vortex**（canonical）、**Markdown**（视图）。**写命令每次只动一层**；读命令可 `auto` 探测。 +两种物理层：**Lance**（canonical）、**Markdown**（视图）。**写命令每次只动一层**；读命令可 `auto` 探测。 | 命令 | `auto` 写入 | 显式 `--storage-format` | |------|-------------|-------------------------| -| add | 无层 → Vortex；仅 md → Markdown；仅 Vortex → Vortex；两层都有 → Vortex | `vortex` / `markdown` 强制单层 | -| truncate | — | 仅 Vortex（按 `seq` 截断） | -| replay | 有 Vortex 读 Vortex；仅 md 读 md；都有 → Vortex | 可强制 `markdown` | +| add | 无层 → Lance；仅 md → Markdown；仅 Lance → Lance；两层都有 → Lance | `lance` / `markdown` 强制单层 | +| truncate | — | 仅 Lance（按 `seq` 截断） | +| replay | 有 Lance 读 Lance；仅 md 读 md；都有 → Lance | 可强制 `markdown` | | stats | 两层都有 → **同时摘要** | 可强制单层 | -| materialize | — | Vortex → Markdown 全量导出 | +| materialize | — | Lance → Markdown 全量导出 | -两层对齐：先 `add`（Vortex）或采集，再 `materialize`；**不会**在 `add`/`truncate` 时自动双写。 +两层对齐：先 `add`（Lance）或采集，再 `materialize`；**不会**在 `add`/`truncate` 时自动双写。 --- ## 4. 示例 ```bash -# 追加 JSONL 到 Vortex -persisting traj add ./store --agent-id a --session-id s --format jsonl --input batch.jsonl --storage-format vortex +# 追加 JSONL 到 Lance +persisting traj add ./store --agent-id a --session-id s --format jsonl --input batch.jsonl --storage-format lance -# 截断 Vortex（Markdown 需单独 materialize） +# 截断 Lance（Markdown 需单独 materialize） persisting traj truncate ./store --agent-id a --session-id s --keep-rows 100 # 双层统计（只读；省略 --session-id 时扫描 agent 下所有 run / session 分区） @@ -89,7 +89,7 @@ persisting traj stats ./store --agent-id a --detail # 指定单个 header session UUID（Claude 对话分区） persisting traj stats ./store --agent-id a --root-session-id run-1 --session-id 58867536-… -# 从 Vortex 补人读视图 +# 从 Lance 补人读视图 persisting traj materialize ./store --agent-id a --session-id s --root-session-id run-1 ``` diff --git a/docs/src/design/cli.md b/docs/src/design/cli.md index c385307..8ee915b 100644 --- a/docs/src/design/cli.md +++ b/docs/src/design/cli.md @@ -1,6 +1,6 @@ # CLI 整体架构 -`persisting` CLI 是**薄前端**：负责解析用户意图、序列化请求、展示结果；重逻辑（Search 用 Lance、轨迹用 Vortex、检索索引）在**可独立发版的引擎**中运行。 +`persisting` CLI 是**薄前端**：负责解析用户意图、序列化请求、展示结果；重逻辑（Search 用 Lance、轨迹用 Lance、检索索引）在**可独立发版的引擎**中运行。 `search`、`traj`（`trajectory`）等子命令共用此架构。 @@ -26,7 +26,7 @@ CLI（解析 · 组装请求 · 展示结果） │ │ 动态加载 · 窄 ABI · 异步 job ▼ -引擎（Search · Vortex 轨迹 · …） +引擎（Search · Lance 轨迹 · …） │ ▼ 持久化存储 @@ -109,16 +109,16 @@ flowchart LR end subgraph Engine D --> F[分发到 Search / Trajectory / …] - F --> G[Vortex append / materialize Markdown] + F --> G[Lance append / materialize Markdown] end ``` -轨迹存储模型见 [轨迹存储](trajectory_storage.zh.md)。 +轨迹存储模型见 [轨迹存储](trajectory.md)。 --- ## 8. 相关文档 - [`persisting search`](cli_search_command.zh.md) -- [`persisting traj`](cli_trajectory_command.zh.md) — 轨迹统一入口（含 capture / proxy） -- [Capture 子命令](cli_capture_command.zh.md) — `traj capture` / `traj proxy` / `traj import` +- [`persisting traj`](cli-traj.md) — 轨迹统一入口（含 capture / proxy） +- [Capture 子命令](cli-capture.md) — `traj capture` / `traj proxy` / `traj import` diff --git a/docs/src/design/compute.md b/docs/src/design/compute.md index 239b801..63c8064 100644 --- a/docs/src/design/compute.md +++ b/docs/src/design/compute.md @@ -18,7 +18,7 @@ L2 Compute 流式领取 · 有界并发 · sticky 派发 · 取消 / 重 ↓ L3 执行缝 每槽一个长驻 Python host，只调用户 execute ↓ -唯一 sink 控制面 append 结果（JSONL 账本；可选 Tee Vortex） +唯一 sink 控制面 append 结果（JSONL 账本；可选 Tee Lance） ``` | 是 | 不是 | @@ -29,7 +29,7 @@ L3 执行缝 每槽一个长驻 Python host，只调用户 execute | Rust Pulsing 做发现与投递 | Python 侧 Actor 编程模型 | | spawn 外部 `--python` | PyO3 内嵌用户环境 | -与 `search` / `traj` 不同：`compute` **静态链接**进 CLI，直接 async 跑，不经 engine RON ABI。落盘若走 Vortex，再经 sink adapter 写轨迹；编排本身仍在本 crate。 +与 `search` / `traj` 不同：`compute` **静态链接**进 CLI，直接 async 跑，不经 engine RON ABI。落盘若走 Lance，再经 sink adapter 写轨迹；编排本身仍在本 crate。 --- @@ -156,7 +156,7 @@ Worker **不**解释业务；Driver **不**碰 Python。失败 traceback 编进 |------|------| | `--sink DIR` | `ready.ndjson` + `failures.ndjson` + `checkpoint.json` | | 无 `--sink` | 默认仅 stdout NDJSON（开发视图，非耐久） | -| `--traj` | Tee 到 Vortex；**JSONL 仍是 resume 真相** | +| `--traj` | Tee 到 Lance；**JSONL 仍是 resume 真相** | `JsonlFileSink`：打开时从已有文件 **seed** `task_id`；先 reserve 再写盘，失败则 **rollback seen**（seen ⊆ durable）。 异步 `sink_writer`：`join` 汇总 persist 失败并 fail job；失败时 `SkipSet::remove`，便于后续 `--resume` 发现未落盘 id。损坏 / 无 `task_id` 的 JSONL 行：**skip + warn**，不拖垮整本账本。 @@ -231,6 +231,6 @@ Worker **不**解释业务；Driver **不**碰 Python。失败 traceback 编进 ## 9. 相关文档 - [Compute 快速上手](../guide/compute_quickstart.zh.md) -- [CLI 整体架构](cli_architecture.zh.md)（`compute` 为例外静态路径） -- [轨迹存储](trajectory_storage.zh.md)（`--traj` / L1） +- [CLI 整体架构](cli.md)（`compute` 为例外静态路径） +- [轨迹存储](trajectory.md)（`--traj` / L1） - 示例：[`examples/compute/`](../../../examples/compute/) diff --git a/docs/src/design/index.md b/docs/src/design/index.md index 295c92c..43a307e 100644 --- a/docs/src/design/index.md +++ b/docs/src/design/index.md @@ -24,7 +24,7 @@ Persisting's architecture, addressing model, and storage design. | Document | Description | |----------|-------------| | [Capture Architecture](capture.md) | LLM proxy, event model, dual storage (中文) | -| [Trajectory Storage](trajectory.md) | Vortex canonical + Markdown materialization (中文) | +| [Trajectory Storage](trajectory.md) | Lance canonical + Markdown materialization (中文) | | [Trajectory Markdown Format](trajectory-format.md) | TLV block model, frontmatter, live upsert (中文) | | [Traj CLI](cli-traj.md) | `persisting traj` command reference (中文) | | [Capture CLI](cli-capture.md) | `traj capture` / `traj proxy` subcommands (中文) | @@ -66,7 +66,7 @@ Persisting's architecture, addressing model, and storage design. 1. **Lance is the baseline** — All caches and accelerations are built on top of "reads from file." 2. **One foundation, multiple patterns** — Trajectory, Search, KV, Queue share the Lance ecosystem. -3. **Trajectory dual-view** — Vortex (canonical) + Markdown (materialized); live upsert from capture. +3. **Trajectory dual-view** — Lance (canonical dataset) + Markdown (materialized); `-f md` live upserts Markdown, `-f lance` writes Lance only (materialize for md). 4. **Capture is self-contained** — Embedded proxy captures LLM traffic; IDE import is supplementary. 5. **TTAS is internal** — Users see `kv[key].tensor()`, not raw address algebra. 6. **Performance is product** — P99 latency, GPU utilization, capture real-time fidelity. diff --git a/docs/src/design/index.zh.md b/docs/src/design/index.zh.md index c0724a0..3003e5f 100644 --- a/docs/src/design/index.zh.md +++ b/docs/src/design/index.zh.md @@ -25,7 +25,7 @@ Persisting 的架构、寻址模型与存储设计。 | 文档 | 描述 | |------|------| | [Capture 架构设计](capture.md) | LLM 代理、事件模型、双层存储 | -| [轨迹存储模型](trajectory.md) | Vortex canonical + Markdown 物化 | +| [轨迹存储模型](trajectory.md) | Lance canonical + Markdown 物化 | | [轨迹 Markdown 格式](trajectory-format.md) | TLV 块模型、frontmatter、live upsert | | [Traj CLI](cli-traj.md) | `persisting traj` 命令参考 | | [Capture CLI](cli-capture.md) | `traj capture` / `traj proxy` 子命令 | @@ -67,7 +67,7 @@ Persisting 的架构、寻址模型与存储设计。 1. **Lance 是兜底** — 上层缓存与加速都建立在「从文件读」这一基线之上。 2. **一种底座，多种模式** — 轨迹、Search、KV、队列共享 Lance 生态。 -3. **轨迹两层视图** — Vortex canonical + Markdown 物化；live upsert。 +3. **轨迹两层视图** — Lance canonical（`events.lance/`）+ Markdown 物化；`-f md` live upsert，`-f lance` 只写 Lance（md 经 materialize）。 4. **Capture 自给自足** — 内嵌代理即可完整捕获 LLM 流量。 5. **TTAS 对内** — 用户看到 `kv[key].tensor()`，而非原始代数。 6. **性能是产品** — P99 延迟、GPU 利用率、capture 实时性是核心指标。 diff --git a/docs/src/design/references/transfer-queue-comparison.md b/docs/src/design/references/transfer-queue-comparison.md index f10f086..ba5cc6c 100644 --- a/docs/src/design/references/transfer-queue-comparison.md +++ b/docs/src/design/references/transfer-queue-comparison.md @@ -45,7 +45,7 @@ | 项目 | 分数 | 点评 | |------|------|------| -| **Persisting** | 5 | 队列默认 Lance 列存落盘；Agent 轨迹 canonical 为 Vortex（`events.vortex`），队列与 capture 均天然持久化，适合复现、审计与故障恢复。 | +| **Persisting** | 5 | 队列默认 Lance 列存落盘；Agent 轨迹 canonical 为 Lance（`events.lance/` 目录），队列与 capture 均天然持久化，适合复现、审计与故障恢复。 | | **TransferQueue** | 3 | 默认 SimpleStorage 为内存；持久化依赖可选后端（Yuanrong、Mooncake 等）。能力有，但非默认开箱即用，需选型与配置。 | --- diff --git a/docs/src/design/trajectory-format.md b/docs/src/design/trajectory-format.md index 920ff02..9f9147b 100644 --- a/docs/src/design/trajectory-format.md +++ b/docs/src/design/trajectory-format.md @@ -1,8 +1,8 @@ # 轨迹 Markdown 格式 -会话 Markdown 是轨迹的**物化人读视图**：从 Vortex raw event log **materialize** 得到，或在 capture `-f md` 时由 **CaptureEngine live upsert** 增量更新。 +会话 Markdown 是轨迹的**物化人读视图**：从 Lance raw event log **materialize** 得到，或在 capture `-f md` 时由 **CaptureEngine live upsert** 增量更新。 -> **Capture `-f md` / `-f vortex`**：Proxy 内 CaptureEngine 按 `call_id`+`role` **upsert** 块到对应 md（主 session → `run-{run_id}.md`，subagent → `agent-{id}.md`），流式 assistant 块原地 rewrite；每次 dialogue 写入后刷新 **YAML frontmatter 摘要**（turns / tokens / cost）。**两种格式均 live 写 md**；`-f vortex` 额外 append `events.vortex`（canonical）。run 结束写 `reconcile.json`（vortex 模式对比 live md 与 Vortex replay），不一致时可 `traj materialize` 全量重建。 +> **Capture 写入互斥**：`-f md` 时 Proxy 内 CaptureEngine 按 `call_id`+`role` **upsert** 块到对应 md（主 session → `run-{run_id}.md`，subagent → `agent-{id}.md`），流式 assistant 块原地 rewrite，并刷新 **YAML frontmatter**；run 结束写 `reconcile.json`。`-f lance` **只** append `{run}/events.lance/`（Lance dataset 目录），**不** live 写 md；人读视图用 `traj materialize`。 **文件名**（由 `storage_session_id` 决定）： @@ -12,7 +12,7 @@ | Subagent | `agent-{claude_agent_id}.md` | | 扁平 session | `{session_id}.md` | -Markdown 路径解析见 [轨迹存储模型 §7.3](trajectory_storage.zh.md)。 +Markdown 路径解析见 [轨迹存储模型 §7.3](trajectory.md)。 --- @@ -21,9 +21,9 @@ Markdown 路径解析见 [轨迹存储模型 §7.3](trajectory_storage.zh.md)。 | 目标 | 说明 | |------|------| | **可读** | 正文是纯 Markdown，元数据藏在 HTML 注释 | -| **可解析** | 每块带 JSON 元数据，可 compact 回 Vortex | +| **可解析** | 每块带 JSON 元数据，可 compact 回 Lance | | **可 diff** | 适合 git 跟踪 Agent 行为变化 | -| **有损物化** | 相对 Vortex 省略内部流量与 lifecycle；见 [存储模型 §1](trajectory_storage.zh.md) | +| **有损物化** | 相对 Lance 省略内部流量与 lifecycle；见 [存储模型 §1](trajectory.md) | --- @@ -48,7 +48,7 @@ Proxy 路径下 Markdown 按 **`call_id` + `role`** upsert，而非仅末尾 app | `Event::ResponseDraft` | assistant 块 upsert，`draft: true` | | `Event::ResponseComplete` | assistant 块 upsert，移除 draft | -Import 路径仍使用批量 **append**。详见 [轨迹存储 §5.3](trajectory_storage.zh.md)。 +Import 路径仍使用批量 **append**。详见 [轨迹存储 §5.3](trajectory.md)。 ### 2.4 块头 JSON 字段规范 @@ -63,7 +63,7 @@ Import 路径仍使用批量 **append**。详见 [轨迹存储 §5.3](trajectory | `kind` | ✅ | `CaptureRecord.kind`（如 `llm.request`、`llm.response.stream`） | | `role` | ✅ | 与 `{speaker}` 一致（`user` / `assistant` / `tool` / `note` …） | | `source` | 推荐 | 记录来源（如 `persisting-proxy`、`markdown`） | -| `seq` | 推荐 | Vortex 行序号（物化块通常与 canonical 一致） | +| `seq` | 推荐 | Lance 行序号（物化块通常与 canonical 一致） | | `turn` | 可选 | **仅展示用**：`seq / 2 + 1` 启发式；**不得**作为 Story / frontmatter `turns` 的权威轮次（权威值见 `story_user_turn_count` 或 `TurnMachine`） | | `call_id` | live 推荐 | upsert 键之一（与 `role` 配对）；import-only append 可为空 | | `draft` | 可选 | `true` 表示流式 assistant 草稿；finalize 后同键 upsert 覆盖并移除 | @@ -86,7 +86,7 @@ Import 路径仍使用批量 **append**。详见 [轨迹存储 §5.3](trajectory - **编码**：`append_subagent_refs_footer` 在可见文本之后追加。 - **解析 / compact**：`strip_subagent_footer_from_body` 在 `block_to_capture_record` 与可见文本提取前剔除**整行**脚注（行首 trim 后匹配 ``）。 -- **权威元数据**：`subagent_id`、`subagent_trajectories` 等仍以块头 JSON 与 Vortex 为准；脚注供人读与 grep。 +- **权威元数据**：`subagent_id`、`subagent_trajectories` 等仍以块头 JSON 与 Lance 为准；脚注供人读与 grep。 ### 2.6 大文件 upsert（演进，格式不变） @@ -102,7 +102,7 @@ Import 路径仍使用批量 **append**。详见 [轨迹存储 §5.3](trajectory ### 2.7 多模态对话正文（Phase 0） -当请求或响应含图像时，**Markdown 块正文与 Vortex 摘要字段**使用占位符，不内联 base64 像素（避免 git diff 与 frontmatter 膨胀）。 +当请求或响应含图像时，**Markdown 块正文与 Lance 摘要字段**使用占位符，不内联 base64 像素（避免 git diff 与 frontmatter 膨胀）。 | 类型 | 占位符格式 | 说明 | |------|------------|------| @@ -134,7 +134,7 @@ prompt: A gray tabby cat hugging an otter… | `dialogue` | ✅ | ❌ | | `full` | ✅ | 在 `payload.body` 中保留完整 JSON（含 base64），仍不进 Markdown 正文 | -**物化限制**：`traj materialize` 当前**不会**把占位符展开为 ``；完整像素仅在 `full` 级 Vortex `payload.body` 或后续 sidecar Phase 1 中可恢复。详见 [Capture 架构 §6.4](capture_design.zh.md#64-可见对话提取含多模态)。 +**物化限制**：`traj materialize` 当前**不会**把占位符展开为 ``；完整像素仅在 `full` 级 Lance `payload.body` 或后续 sidecar Phase 1 中可恢复。详见 [Capture 架构 §6.4](capture.md#64-可见对话提取含多模态)。 ### 2.8 完整示例 @@ -165,7 +165,7 @@ client: 你好！有什么可以帮你的？ ``` -YAML frontmatter 固定 `format: persisting:1.0`，并含会话级 rollup（见 [轨迹存储 §8.1](trajectory_storage.zh.md)）。 +YAML frontmatter 固定 `format: persisting:1.0`，并含会话级 rollup（见 [轨迹存储 §8.1](trajectory.md)）。 **Golden 示例**（由 `encode_block_with_header` 生成，CI 校验）： @@ -181,17 +181,17 @@ YAML frontmatter 固定 `format: persisting:1.0`，并含会话级 rollup（见 | 场景 | 行为 | |------|------| | Capture `-f md` | **仅** CaptureEngine live upsert 到 run bucket / `agent-*.md`（可 `tail -f`） | -| Capture `-f vortex` | 仅 Vortex（`events.vortex`） | -| `traj materialize` | 全量扫描 Vortex，**重写**指定 session 的 Markdown（修复/补全；traj capture 需 `--root-session-id`） | -| `traj add` + markdown 存储 | append Vortex 后自动 materialize | -| `traj add` + 输入为 `.md` | 解析 TLV → compact/append 到 Vortex | -| `traj replay` | 有 Vortex 时读 Vortex；纯 md session 从块还原 | +| Capture `-f lance` | 仅 Lance（`events.lance/` 目录） | +| `traj materialize` | 全量扫描 Lance，**重写**指定 session 的 Markdown（修复/补全；traj capture 需 `--root-session-id`） | +| `traj add` + markdown 存储 | **只写一层**（`--storage-format markdown`）；需要 Lance 时再单独 `traj compact` / append | +| `traj add` + 输入为 `.md` | 解析 TLV → compact/append 到 Lance | +| `traj replay` | 有 Lance 时读 Lance；纯 md session 从块还原 | --- -## 4. 与 Vortex 的关系 +## 4. 与 Lance 的关系 -| 维度 | Markdown（物化） | Vortex（canonical） | +| 维度 | Markdown（物化） | Lance（canonical） | |------|------------------|---------------------| | 角色 | 人读视图 | 全量 event log | | 单位 | 块 | 行 | @@ -199,25 +199,25 @@ YAML frontmatter 固定 `format: persisting:1.0`，并含会话级 rollup（见 | 数据完整性 | 有损（过滤内部事件） | 无损 | | 典型操作 | 打开、git diff | replay、stats、Search | -双向转换由 `persisting-capture::trajectory_convert` 与引擎 `trajectory::convert` 实现。Subagent 块头字段与 [§2.5](#25-subagent-脚注) 脚注分工见上。详见 [轨迹存储模型](trajectory_storage.zh.md)。 +双向转换由 `persisting-capture::trajectory_convert` 与引擎 `trajectory::convert` 实现。Subagent 块头字段与 [§2.5](#25-subagent-脚注) 脚注分工见上。详见 [轨迹存储模型](trajectory.md)。 --- -## 5. Compact 回 Vortex +## 5. Compact 回 Lance -从 Markdown 导入 Vortex 时： +从 Markdown 导入 Lance 时： - 每块 → 一条 `CaptureRecord`（`llm.request` / `llm.response` 等） - 块头字段额外写入 `payload._tlv`，保留 TLV 元数据 - 完整 HTTP body 若未出现在块中，则由正文近似重建 -因此 **Markdown → Vortex 可压缩存储更多结构化字段**，但无法恢复 materialize 时已丢弃的内部事件。 +因此 **Markdown → Lance 可压缩存储更多结构化字段**，但无法恢复 materialize 时已丢弃的内部事件。 --- ## 6. 相关文档 -- [轨迹存储模型](trajectory_storage.zh.md) -- [Capture 架构设计](capture_design.zh.md) -- [Traj 命令](cli_trajectory_command.zh.md) · [Capture 子命令](cli_capture_command.zh.md) -- [`persisting trajectory` 命令](cli_trajectory_command.zh.md) +- [轨迹存储模型](trajectory.md) +- [Capture 架构设计](capture.md) +- [Traj 命令](cli-traj.md) · [Capture 子命令](cli-capture.md) +- [`persisting trajectory` 命令](cli-traj.md) diff --git a/docs/src/design/trajectory.md b/docs/src/design/trajectory.md index 6fb0205..7dd070a 100644 --- a/docs/src/design/trajectory.md +++ b/docs/src/design/trajectory.md @@ -8,17 +8,17 @@ | 你想了解… | 见 | |-----------|-----| -| Capture 代理的整体架构（概念、Story 边界、采集管线） | [Capture 架构设计](capture_design.zh.md) §4–§6 | -| TLV Markdown 的块格式规范 | [轨迹 Markdown 格式](trajectory_tlv_format.zh.md) | -| `persisting traj` 命令用法 | [Traj 命令](cli_trajectory_command.zh.md) · [Capture 子命令](cli_capture_command.zh.md) | +| Capture 代理的整体架构（概念、Story 边界、采集管线） | [Capture 架构设计](capture.md) §4–§6 | +| TLV Markdown 的块格式规范 | [轨迹 Markdown 格式](trajectory-format.md) | +| `persisting traj` 命令用法 | [Traj 命令](cli-traj.md) · [Capture 子命令](cli-capture.md) | --- ## 1. 概述 -Agent 轨迹在 Persisting 中同时服务两类读者：**机器**（回放、统计、检索）与**人**（阅读、git diff、code review）。为此采用**两层存储**：Vortex 为 canonical raw event log，TLV Markdown 为按需物化的人读视图。 +Agent 轨迹在 Persisting 中同时服务两类读者：**机器**（回放、统计、检索）与**人**（阅读、git diff、code review）。为此采用**两层存储**：Lance 为 canonical raw event log，TLV Markdown 为按需物化的人读视图。 -**与 Story 的关系**：Capture 主路径为 `任意协议 → Story（Call + Event）→ 事件记录 → Vortex/Markdown`。协议差异在 Story 边界前消化；事件记录是存储层统一中间表示。详见 [Capture §4.2 三层词汇表](capture_design.zh.md#42-三层词汇表)。 +**与 Story 的关系**：Capture 主路径为 `任意协议 → Story（Call + Event）→ 事件记录 → Lance/Markdown`。协议差异在 Story 边界前消化；事件记录是存储层统一中间表示。详见 [Capture §4.2 三层词汇表](capture.md#42-三层词汇表)。 ```mermaid graph TD @@ -35,16 +35,16 @@ graph TD CE["CaptureEngine
(事件驱动采集核心)"]:::accent0 AQ["ApplyDispatcher
(per-story_id 有序 apply)"]:::accent0 MP["MarkdownPipeline
(过滤 + live/batch/reconcile)"]:::accent3 - E1["Vortex Append
(canonical)"]:::accent2 - E2["Materialize
(全量 Vortex → Markdown)"]:::accent3 + E1["Lance Append
(canonical)"]:::accent2 + E2["Materialize
(全量 Lance → Markdown)"]:::accent3 E3["Live Upsert
(call_id+role 流式 md)"]:::accent3 E3b["Frontmatter Refresh
(会话 YAML 摘要)"]:::accent3 E4["Stream Append
(import 批量 append md)"]:::accent3 - E5["Compact
(Markdown → Vortex)"]:::accent3 + E5["Compact
(Markdown → Lance)"]:::accent3 end subgraph "存储层" - L1["Vortex Event Log
列存 · 无损 · {run}/events.vortex"]:::accent4 + L1["Lance Event Log
列存 · 无损 · {run}/events.lance/"]:::accent4 L2["TLV Markdown
人读 · 对话块 · 可选物化"]:::accent5 L3["sessions.json
内存索引 · 快速 list"]:::accent5 end @@ -78,28 +78,28 @@ graph TD classDef accent5 fill:#d7ccc8,stroke:#4e342e,color:#000 ``` -**核心 invariant**：`Vortex row_count ≥ Markdown block_count`（materialize 是有损的）。 +**核心 invariant**：`Lance row_count ≥ Markdown block_count`（materialize 是有损的）。 --- ## 2. 两层结构对比 -| 维度 | Vortex (canonical) | TLV Markdown (materialized view) | +| 维度 | Lance (canonical) | TLV Markdown (materialized view) | |------|-------------------|----------------------------------| | **角色** | 全量 event log，系统的 single source of truth | 人类可读对话视图 | | **完整性** | 无损，所有事件全部保留 | 有损，过滤内部流量与 lifecycle | -| **格式** | 列存（`EventRow`），capture run 使用 run 级 `events.vortex` | TLV 块序列（`MarkdownBlock`），纯文本文件 | -| **写入方式** | 每次 LLM 事件 append；import / CLI worker 批量 append | Proxy `-f md` 时 **CaptureEngine live upsert**；import 走批量 append；均可 `materialize` 全量重建 | +| **格式** | 列存（`EventRow`），capture run 使用 run 级 `events.lance/`（Lance dataset **目录**） | TLV 块序列（`MarkdownBlock`），纯文本文件 | +| **写入方式** | `-f lance` / `traj add --storage-format lance` 时 append；import 批量 append | Proxy `-f md` 时 **CaptureEngine live upsert**；均可 `materialize` 全量重建 | | **典型操作** | `replay`、`stats`、Search import、FTS | 直接打开、`git diff`、code review | -| **按 session 分片** | ✅ `events.vortex` 内按 `session_id` 列过滤；同一 run 文件可含**多个** distinct `session_id`（见 [§7.1.1](#711-run-bucket-内多-session_id-分区)） | ✅ `{run_dir}/{storage_session_id}.md` | +| **按 session 分片** | ✅ `events.lance/` 内按 `session_id` 列过滤；同一 run **dataset** 可含**多个** distinct `session_id`（见 [§7.1.1](#711-run-bucket-内多-session_id-分区)） | ✅ `{run_dir}/{storage_session_id}.md` | --- ## 3. 数据流全景 -### 3.1 Capture Proxy（`-f md`）— 事件驱动 + 非阻塞采集 +### 3.1 Capture Proxy（`-f md`）— 事件驱动 + live Markdown -Proxy 侧采集经 [`CaptureEngine`](../../crates/persisting-capture/src/engine/mod.rs) 统一处理；**Proxy 不 await 完整 `apply`**（`spawn_apply` → Event WAL → `ApplyDispatcher`），Vortex/Markdown 写入在后台完成。不再向 Vortex 写入 `llm.response.stream.partial`。 +Proxy 侧采集经 [`CaptureEngine`](../../crates/persisting-capture/src/engine/mod.rs) 统一处理；**Proxy 不 await 完整 `apply`**（`spawn_apply` → Event WAL → `ApplyDispatcher`），Markdown 写入在后台完成。**`-f md` 不写 Lance**。 ```mermaid sequenceDiagram @@ -107,8 +107,6 @@ sequenceDiagram participant Proxy as Capture Proxy participant Queue as ApplyDispatcher participant Engine as CaptureEngine - participant Sink as CallbackSink / Worker - participant Vortex as events.vortex participant MD as TLV Markdown Agent->>Proxy: POST /v1/messages @@ -116,55 +114,75 @@ sequenceDiagram Proxy->>Queue: spawn_apply Event::Request（先写 Event WAL，不阻塞 upstream） Queue->>Engine: 按 story_id 有序 apply Engine->>Engine: prepare + StoryActor ask - Engine->>Sink: append llm.request - Sink->>Vortex: batch flush Engine->>MD: upsert user block + refresh frontmatter loop SSE stream (150ms 节流) Proxy->>Queue: spawn_apply Event::ResponseDraft Queue->>Engine: 有序 apply - Note over Engine,Vortex: draft 仅写 md Engine->>MD: upsert assistant block (draft: true) end alt 客户端断开 Proxy->>Queue: spawn_apply Event::Cancelled Queue->>Engine: 有序 apply - Engine->>Sink: append llm.call.cancelled（仅 Vortex） + Note over Engine,MD: cancelled 可不进 Markdown else 正常结束 Proxy->>Queue: spawn_apply Event::ResponseComplete Queue->>Engine: 有序 apply - Engine->>Sink: append llm.response.stream - Sink->>Vortex: flush Engine->>MD: upsert assistant (final) + refresh frontmatter end ``` 关键设计决策： -1. **Vortex 仍是 canonical**。`Event::ResponseDraft` 只更新 Markdown，不产生 Vortex 行。 -2. **Live Markdown 用 upsert**。`upsert_block_by_call_id` 按 **`call_id` + `role`** 匹配；块区间替换使用 `rewrite_block_range`（**不再** truncate-to-EOF）。 -3. **ApplyDispatcher 保证有序**。同一 `story_id` 内 Request / Draft / Complete 按序 apply，避免跨 spawn 乱序。 -4. **StoryActor `ask`**。生产环境等待 story actor 处理结果；失败写入 `dead_letter.jsonl`（含 `prepared_record_json`）。 -5. **CLI worker 只写 Vortex**。`-f md` 时 Proxy 内 `CaptureEngine(stream_markdown=true)` 负责 live md；worker `stream_markdown=false`；flush 失败 → `trajectory_dead_letter.jsonl`。 -6. **Run 结束对账**。`worker.shutdown()` 后写 `.capture/reconcile.json`（md call_id vs Vortex replay）；不一致时可 `traj materialize` 全量重建。 -7. **Frontmatter 摘要**。每次 dialogue 写入后刷新 YAML（turns / tokens / cost / subagents）；`traj capture` 结束打印 stderr 一行汇总。 +1. **`-f md` 只写 Markdown**。需要 canonical 列存时改用 `-f lance`，或事后 `traj compact` / import。 +2. **Live Markdown 用 upsert**。`upsert_block_by_call_id` 按 **`call_id` + `role`** 匹配；块区间替换使用 `rewrite_block_range`。 +3. **ApplyDispatcher 保证有序**。同一 `story_id` 内 Request / Draft / Complete 按序 apply。 +4. **StoryActor `ask`**。生产环境等待 story actor 处理结果；失败写入 `dead_letter.jsonl`。 +5. **Run 结束对账**。写 `.capture/reconcile.json`（md call_id vs Markdown replay）；不涉及 Lance。 +6. **Frontmatter 摘要**。每次 dialogue 写入后刷新 YAML；`traj capture` 结束打印 stderr 一行汇总。 + +### 3.1b Capture Proxy（`-f lance`）— 仅 canonical dataset + +`-f lance`（别名 `bin`）只 append 到 `{run}/events.lance/`（Lance dataset 目录），**不** live 写 Markdown、**不**写 reconcile。人读视图用 `traj materialize`。 + +```mermaid +sequenceDiagram + participant Agent + participant Proxy as Capture Proxy + participant Queue as ApplyDispatcher + participant Engine as CaptureEngine + participant Sink as CallbackSink / Worker + participant Lance as events.lance/ + + Agent->>Proxy: POST /v1/messages + Proxy->>Queue: spawn_apply Event::Request + Queue->>Engine: 有序 apply + Engine->>Sink: append llm.request + Sink->>Lance: Lance Append flush + Note over Engine,Lance: ResponseDraft 不落盘 + Proxy->>Queue: spawn_apply Event::ResponseComplete + Queue->>Engine: 有序 apply + Engine->>Sink: append llm.response.stream + Sink->>Lance: flush +``` -### 3.2 CLI Import — 批量 Vortex + 可选 append Markdown +### 3.2 CLI Import — 按 `--storage-format` 只写一层 -IDE 日志 / gateway 导入不经 Proxy，仍走 `CaptureRecord` → 批量 Vortex append；`-f md` 时对同批 engine lines 调用 `stream_engine_lines_to_markdown()` **末尾追加**（无 upsert，因无流式 draft）。 +IDE 日志 / gateway 导入不经 Proxy，走 `CaptureRecord` → 选定层 append。默认目标多为 Lance；`-f md` / `--storage-format markdown` 时对 engine lines 调用 `stream_engine_lines_to_markdown()` **末尾追加**（无 upsert）。 ```mermaid sequenceDiagram participant Import as traj import participant Worker as TrajectoryAppendWorker - participant Vortex as events.vortex + participant Lance as events.lance/ participant MD as TLV Markdown Import->>Worker: CaptureRecord (RON line) Worker->>Worker: buffer (≤32 或关键事件 flush) - Worker->>Vortex: TrajectoryAppend (Vortex only) - opt -f md + alt storage-format lance + Worker->>Lance: TrajectoryAppend + else storage-format markdown Worker->>MD: stream_engine_lines_to_markdown (append) end ``` @@ -192,7 +210,7 @@ pub struct CaptureRecord { } ``` -`CaptureRecord` 是系统中**唯一的内部数据交换格式**。数据源（proxy / CLI import）产出 `CaptureRecord`，引擎（Vortex / Markdown / Index）消费它。 +`CaptureRecord` 是系统中**唯一的内部数据交换格式**。数据源（proxy / CLI import）产出 `CaptureRecord`，引擎（Lance / Markdown / Index）消费它。 ### 4.2 EventRow（列存行） @@ -229,7 +247,7 @@ pub struct BlockHeader { } ``` -详见 [轨迹 Markdown 格式](trajectory_tlv_format.zh.md)。 +详见 [轨迹 Markdown 格式](trajectory-format.md)。 --- @@ -243,7 +261,7 @@ graph LR B -->|"2. write_markdown_document"| C["TLV Markdown 文件"] A -->|"3. capture_record_to_event_row"| D["EventRow"] - D -->|"4. Vortex append"| E["events.vortex"] + D -->|"4. Lance append"| E["events.lance/"] C -->|"5. markdown_document_to_capture_records"| A E -->|"6. event_row_to_capture_record"| A @@ -260,14 +278,14 @@ graph LR | **Block → .md 文件（增量 append）** | `stream_engine_lines_to_markdown()` | 末尾追加 | **import** `-f md`（非 Proxy live 路径） | | **Block → .md 文件（live upsert）** | `upsert_block_by_call_id()` | 按 call_id+role 替换或 append | Proxy `-f md` 流式采集 | | **.md 文件 → CaptureRecord** | `markdown_document_to_capture_records()` | 解析 + 重建 | compact 导入 | -| **CaptureRecord → Vortex** | `capture_record_to_event_row()` | 一行一条 | Vortex append | -| **Vortex → CaptureRecord** | `event_row_to_capture_record()` | 反序列化 | replay / stats | +| **CaptureRecord → Lance** | `capture_record_to_event_row()` | 一行一条 | Lance append | +| **Lance → CaptureRecord** | `event_row_to_capture_record()` | 反序列化 | replay / stats | -### 5.2 Materialize（Vortex → Markdown 全量） +### 5.2 Materialize（Lance → Markdown 全量） ```rust pub struct MaterializeStats { - pub source_events: usize, // Vortex 行数 + pub source_events: usize, // Lance 行数 pub markdown_blocks: usize, // Markdown 块数 pub skipped_events: usize, // 被过滤的事件数 } @@ -276,7 +294,7 @@ pub struct MaterializeStats { 流程： ``` -Vortex event log (全量扫描) +Lance event log (全量扫描) │ event_row_to_capture_record() ▼ CaptureRecord[] @@ -304,7 +322,7 @@ Event::ResponseDraft (≤150ms) upsert_block_by_call_id(call_id, role=assistant) ← 流式原地更新 Event::ResponseComplete - │ enrich + append Vortex llm.response.stream + │ enrich + append Lance llm.response.stream │ MarkdownPipeline::try_block() → final assistant ▼ upsert_block_by_call_id(call_id, role=assistant) ← 覆盖 draft，移除 draft 标记 @@ -312,7 +330,7 @@ upsert_block_by_call_id(call_id, role=assistant) ← 覆盖 draft，移除 dra **匹配键**：`call_id` + `role`（user / assistant 各一块，互不覆盖）。同一 call 的 user 与 assistant 可并存。 -**seq 预览**：draft 块通过 `sink.peek_next_seq()` 写入 header，与实际 Vortex append 序号对齐；finalize 后以 stamped record 的 seq 覆盖。 +**seq 预览**：draft 块通过 `sink.peek_next_seq()` 写入 header，与实际 Lance append 序号对齐；finalize 后以 stamped record 的 seq 覆盖。 ### 5.4 Stream Append（Import `-f md` 增量） @@ -336,12 +354,12 @@ pub struct StreamMaterializeStats { 已有 .md 文件 (末尾追加) ``` -### 5.5 Compact（Markdown → Vortex） +### 5.5 Compact（Markdown → Lance） ```rust pub struct CompactStats { pub source_blocks: usize, // Markdown 块数 - pub event_rows: usize, // 生成的 Vortex 行数 + pub event_rows: usize, // 生成的 Lance 行数 } ``` @@ -357,7 +375,7 @@ MarkdownBlock[] CaptureRecord[] │ capture_record_to_event_row() ▼ -Vortex event log (`events.vortex`) +Lance event log (`events.lance`) ``` **`_tlv` 字段**：compact 时为每个 `CaptureRecord` 注入 `payload._tlv`，包含 `role` 和原始 `block_fields`。这保留了 TLV 元数据，使 compact 操作是信息增量的 —— 但 materialize 时丢弃的内部事件（如 lifecycle）无法通过 compact 恢复。 @@ -381,9 +399,9 @@ live / materialize / reconcile **统一**经 `storage/markdown_pipeline.rs` 的 | 无可见文本的空白请求 | `visible_user_text()` 为 None | 无信息量 | | 无可见文本的空白响应 | `visible_assistant_text()` 为 None | 无信息量 | | 流式 partial（仅 md） | payload `stream_partial: true` | 不落 Markdown | -| 流式 draft（仅 md） | header `draft: true` | 不落 Vortex；finalize 后 draft 被覆盖 | +| 流式 draft（仅 md） | header `draft: true` | 不落 Lance；finalize 后 draft 被覆盖 | | 生命周期事件 | `kind` 以 `session.` 开头 | 不在对话中展示 | -| 客户端取消 | `llm.call.cancelled` | 仅 Vortex；不进 Markdown | +| 客户端取消 | `llm.call.cancelled` | 仅 Lance；不进 Markdown | | 主 session flash/haiku 影子请求 | 无 subagent_id + model 含 `flash`/`haiku` + 非 subagent shape payload | Claude Code 的预热探测，与 pro 内容重复 | | `llm.spawn_link` | **不跳过**，始终保留 | 主/子代理关联是重要轨迹信息 | @@ -402,7 +420,7 @@ impl MarkdownPipeline { } ``` -**invariant**：materialize 之后 `Vortex row_count ≥ Markdown block_count`。 +**invariant**：materialize 之后 `Lance row_count ≥ Markdown block_count`。 **turn 字段**：`turn = seq / 2 + 1` 为全局 seq 的派生编号，**不**保证同一 call 的 user/assistant 共享语义轮次；并发 in-flight 时相邻块可能属于不同 call 却共享 turn 值。分析对话轮次时应以 `call_id` 为准。 @@ -415,8 +433,8 @@ impl MarkdownPipeline { ├── .capture/ │ ├── sessions.json ← 全局会话索引 │ ├── dead_letter.jsonl ← Event apply 失败（可 replay） -│ ├── trajectory_dead_letter.jsonl ← Vortex worker flush 失败 -│ ├── reconcile.json ← run 结束 md↔Vortex 对账 +│ ├── trajectory_dead_letter.jsonl ← Lance worker flush 失败 +│ ├── reconcile.json ← run 结束 md↔Lance 对账 │ ├── run_session ← 当前 run_id（纯文本一行） │ ├── run_child.yaml ← 子进程信息 (PID + 命令行) │ └── daemon.env.json ← daemon 环境快照 (API keys) @@ -426,7 +444,7 @@ impl MarkdownPipeline { │ ├── run-{timestamp}-{nanos}.md ← 主 agent Markdown (run bucket) │ ├── agent-{claude_agent_id}.md ← 子 agent Markdown (sibling) │ ├── agent-{...}.md ← 更多子 agent -│ └── events.vortex ← run 级 Vortex 事件日志 +│ └── events.lance/ ← run 级 Lance dataset（目录；仅 `-f lance`） ``` **关键隔离 invariant**： @@ -436,9 +454,13 @@ impl MarkdownPipeline { - 主 md **不内联**子 agent 的完整轨迹，而是通过 `llm.spawn_link` 块引用 sibling 文件 - traj capture 模式下**不写** `session-meta.yaml`；客户端元信息进入 Markdown YAML frontmatter 的 `client:` 段 +#### 7.1.0 Judge sidecar（可选） + +LLM-as-judge 结果写在 `{run}/layers/judge_{rubric}.lance/`，并由同目录 `manifest.json`（本地 JSON，非外部 layout API）登记 join 键 `[session_id, call_id]`。 + #### 7.1.1 Run bucket 内多 `session_id` 分区 -Claude Code 等客户端会在 HTTP header 注入 **header session UUID**，与 capture run 目录名 `run-*` 不同。同一 `events.vortex` 内因此常见： +Claude Code 等客户端会在 HTTP header 注入 **header session UUID**，与 capture run 目录名 `run-*` 不同。同一 `events.lance/` dataset 内因此常见： ```text session_id = run-20260530-120000-123 ← session.started / 部分元数据 @@ -449,7 +471,7 @@ session_id = 58867536-… ← 实际 LLM 对话行 | 操作 | 行为 | |------|------| -| `traj stats `（未指定 `--session-id`） | 列出 run bucket → **展开** Vortex 内 distinct `session_id` → 逐分区统计 | +| `traj stats `（未指定 `--session-id`） | 列出 run bucket → **展开** Lance 内 distinct `session_id` → 逐分区统计 | | `traj stats --session-id ` | 只过滤该 `session_id` 的行 | | `traj replay` / materialize | 按 Story 坐标中的 `session_id` 过滤 | @@ -463,7 +485,7 @@ session_id = 58867536-… ← 实际 LLM 对话行 │ └── {session_id}/ │ ├── {session_id}.md ← Markdown（可选） │ ├── session-meta.yaml ← 客户端进程元信息（serve 模式） -│ └── events.vortex ← Vortex 事件日志（可选） +│ └── events.lance/ ← Lance dataset（可选；`-f lance`） ``` serve 模式下无 `run_session`，每个逻辑 session 独立一个目录。`session-meta.yaml` 记录发起连接的客户端进程信息（peer addr / PID / 命令行）。 @@ -494,20 +516,20 @@ serve 模式下无 `run_session`，每个逻辑 session 独立一个目录。`se | 策略 | `-f` | 行为 | |------|------|------| -| **Markdown only** | `md` | CaptureEngine live upsert 到 `{session}.md`（**不写 Vortex**） | -| **Vortex only** | `vortex` | 仅 append `events.vortex`；人读视图用 `traj materialize` 按需生成 | +| **Markdown only** | `md` | CaptureEngine live upsert 到 `{session}.md`（**不写 Lance**） | +| **Lance only** | `lance` | 仅 append `events.lance`；人读视图用 `traj materialize` 按需生成 | ### Trajectory CLI（`--storage-format`） -两种物理层：**Vortex**（canonical）、**Markdown**（物化视图）。`traj add` **每次只写一层**（由 `--storage-format` 决定）；`materialize` 为 Vortex → Markdown 全量导出（不随 add 自动触发）。 +两种物理层：**Lance**（canonical）、**Markdown**（物化视图）。`traj add` **每次只写一层**（由 `--storage-format` 决定）；`materialize` 为 Lance → Markdown 全量导出（不随 add 自动触发）。 | 值 | 行为 | |----|------| -| **auto** | **写**：无层→Vortex，仅 md→Markdown，仅 Vortex→Vortex，两层都有→Vortex；**读** replay：有 Vortex 读 Vortex，否则 Markdown；stats：两层都有时同时摘要 | -| **vortex** / **markdown** | 强制读/写/统计指定层 | -| **both** | 遗留别名（append→Vortex only；read/stats 同 auto），不推荐 | +| **auto** | **写**：无层→Lance，仅 md→Markdown，仅 Lance→Lance，两层都有→Lance；**读** replay：有 Lance 读 Lance，否则 Markdown；stats：两层都有时同时摘要 | +| **lance** / **markdown** | 强制读/写/统计指定层 | +| **both** | 遗留别名（append→Lance only；read/stats 同 auto），不推荐 | -**读取优先级**：`replay` 在两层并存时默认 Vortex；纯 Markdown session 从块序列还原。`-f md` run 结束后优先查看 `reconcile.json`；不一致且存在 Vortex 层时执行 `traj materialize` 全量对齐。 +**读取优先级**：`replay` 在两层并存时默认 Lance；纯 Markdown session 从块序列还原。`-f md` run 结束后优先查看 `reconcile.json`；不一致且存在 Lance 层时执行 `traj materialize` 全量对齐。 --- @@ -533,14 +555,14 @@ serve 模式下无 `run_session`，每个逻辑 session 独立一个目录。`se ## 8.3 Run 结束 Reconcile -`traj capture -f md` 在 `TrajectoryAppendWorker.shutdown()` **之后**： +`traj capture -f md` 在 worker shutdown **之后**： 1. 扫描 run 目录下所有 `*.md` -2. Engine `replay` 各 session 的 Vortex 记录 -3. 比较 md 与 Vortex 的 **call_id 集合**（及结构性问题如 excessive blank lines） +2. 对 Markdown 层做 replay / 解析，提取 **call_id 集合** +3. 与 live md 结构对照（及 excessive blank lines 等） 4. 写入 `{storage}/.capture/reconcile.json` -失败时 stderr 提示；不阻断子进程 exit code。完整重建仍用 `traj materialize`。 +纯 `-f lance` **不**写 reconcile。若之后又有 Lance 层且需对齐人读视图，用 `traj materialize`。 --- @@ -548,8 +570,8 @@ serve 模式下无 `run_session`，每个逻辑 session 独立一个目录。`se ```mermaid graph LR - V["Vortex
(无损全量)"] -->|"materialize
(有损: 过滤内部事件)"| M["Markdown
(人读视图)"] - M -->|"compact
(信息增量: + _tlv)"| V2["Vortex
(含 _tlv)"] + V["Lance
(无损全量)"] -->|"materialize
(有损: 过滤内部事件)"| M["Markdown
(人读视图)"] + M -->|"compact
(信息增量: + _tlv)"| V2["Lance
(含 _tlv)"] style V fill:#ffccbc,stroke:#d84315,color:#000 style M fill:#c8e6c9,stroke:#2e7d32,color:#000 @@ -558,10 +580,10 @@ graph LR | 操作 | 方向 | 信息变化 | |------|------|---------| -| **materialize** | Vortex → Markdown | **有损**：丢弃 count_tokens / lifecycle / flash影子请求 / 空 turn；**图像仅保留 dialogue 占位符**，不嵌 `` | -| **compact** | Markdown → Vortex | **信息增量**：注入 `payload._tlv`（TLV 元数据）；但无法恢复已丢弃的内部事件与 sidecar 未写入的像素 | +| **materialize** | Lance → Markdown | **有损**：丢弃 count_tokens / lifecycle / flash影子请求 / 空 turn；**图像仅保留 dialogue 占位符**，不嵌 `` | +| **compact** | Markdown → Lance | **信息增量**：注入 `payload._tlv`（TLV 元数据）；但无法恢复已丢弃的内部事件与 sidecar 未写入的像素 | -**结论**：Vortex 是 canonical。Markdown 可以从 Vortex 重建，但反过来不能。永远不要把 Markdown 当作唯一存储。 +**结论**：Lance 是 canonical。Markdown 可以从 Lance 重建，但反过来不能。永远不要把 Markdown 当作唯一存储。 --- @@ -569,11 +591,11 @@ graph LR ### 10.1 CaptureEngine 事件模型 -Proxy 通过 `CaptureEngine::spawn_apply(ctx, event)`（`llm_capture` / `streaming` 直接调用）→ `ApplyDispatcher` → `CaptureEngine::apply` 处理采集；prepare / StoryActor 失败写入 `.capture/dead_letter.jsonl`；Vortex worker flush 失败写入 `.capture/trajectory_dead_letter.jsonl`。 +Proxy 通过 `CaptureEngine::spawn_apply(ctx, event)`（`llm_capture` / `streaming` 直接调用）→ `ApplyDispatcher` → `CaptureEngine::apply` 处理采集；prepare / StoryActor 失败写入 `.capture/dead_letter.jsonl`；Lance worker flush 失败写入 `.capture/trajectory_dead_letter.jsonl`。 事件命名属于 **Story 区**（`engine/story/event.rs`），与 wire 协议无关： -| Event 变体 | Vortex | Markdown | Proxy 调度 | 说明 | +| Event 变体 | Lance | Markdown | Proxy 调度 | 说明 | |------------|-------|----------|------------|------| | `Event::Request` | ✅ `llm.request` | ✅ user upsert + frontmatter | spawn_apply（非阻塞） | 转发 upstream 前触发 | | `Event::ResponseDraft` | ❌ | ✅ assistant upsert (`draft: true`) | spawn_apply | SSE 150ms 节流 | @@ -586,32 +608,32 @@ Proxy (spawn_apply) ApplyDispatcher CaptureEngine ├─ Event::Request ├─ 按 story_id 有序 ───────────→ prepare + StoryActor ask │ │ ├─ md upsert (MarkdownPipeline) ├─ Event::ResponseDraft ├─ 有序 apply ─────────────────→ md only - ├─ Event::ResponseComplete ├─ 有序 apply ─────────────────→ ask → append Vortex + ├─ Event::ResponseComplete ├─ 有序 apply ─────────────────→ ask → append Lance │ │ ├─ md final + frontmatter └─ Event::Cancelled └─ 有序 apply ─────────────────→ ask → append (cancelled) ``` -StoryActor 生产环境使用 **`ask`**（可观测 + 失败 dead letter）；`shutdown` 前先通过 `ApplyDispatcher` barrier drain 已接收事件，再对各 story 发送 `Flush` drain mailbox。`spawn_apply` 在入队前写 `.capture/events.wal.jsonl`，clean shutdown 后 truncate；非 clean shutdown 下次启动 replay 未 ack event。`CallbackSink` → `TrajectoryAppendWorker` 负责 Vortex 批量 flush。 +StoryActor 生产环境使用 **`ask`**（可观测 + 失败 dead letter）；`shutdown` 前先通过 `ApplyDispatcher` barrier drain 已接收事件，再对各 story 发送 `Flush` drain mailbox。`spawn_apply` 在入队前写 `.capture/events.wal.jsonl`，clean shutdown 后 truncate；非 clean shutdown 下次启动 replay 未 ack event。`CallbackSink` → `TrajectoryAppendWorker` 负责 Lance 批量 flush。 ### 10.2 已知限制 | 现象 | 原因 | 缓解 | |------|------|------| -| md 与 Vortex 短暂不一致 | 后台 apply 尚未 drain | shutdown `Flush` + `reconcile.json` | +| md 与 Lance 短暂不一致 | 后台 apply 尚未 drain | shutdown `Flush` + `reconcile.json` | | turn 编号跨 call 重复 | `turn = seq/2+1` 非 per-call | 消费方用 `call_id` 分组 | | apply 队列满 / 采集失败 | 背压 | dead letter + `replay-dead-letter` | | Event WAL 多轮 crash | replay entry 暂未 ack 原 seq；`next_seq` 未从旧 WAL 恢复 | 补 `PendingEntry.seq` + open 扫描 max(seq)+1 | -| Vortex worker flush 失败 | 磁盘 / IO 异常 | `trajectory_dead_letter.jsonl` 保留 RON batch | -| Vortex 单文件过大 | 长 run / 高频事件 | 按 run 或 agent 拆分（规划中） | +| Lance worker flush 失败 | 磁盘 / IO 异常 | `trajectory_dead_letter.jsonl` 保留 RON batch | +| Lance dataset 过大 | 长 run / 高频事件 | 按 run 或 agent 拆分；Overwrite 路径会重写整个 dataset（规划中） | -详见 [Capture 架构设计 §10](capture_design.zh.md#10-可靠性与运行形态)。 +详见 [Capture 架构设计 §10](capture.md#10-可靠性与运行形态)。 --- ## 11. 相关文档 -- [Capture 架构设计](capture_design.zh.md) — 代理路由、协议转换、采集管线 -- [轨迹 Markdown 格式](trajectory_tlv_format.zh.md) — TLV block 完整规范 -- [Capture 架构设计](capture_design.zh.md) — 代理路由、协议转换、子代理隔离 -- [Traj 命令](cli_trajectory_command.zh.md) -- [Traj Capture 子命令](cli_capture_command.zh.md) +- [Capture 架构设计](capture.md) — 代理路由、协议转换、采集管线 +- [轨迹 Markdown 格式](trajectory-format.md) — TLV block 完整规范 +- [Capture 架构设计](capture.md) — 代理路由、协议转换、子代理隔离 +- [Traj 命令](cli-traj.md) +- [Traj Capture 子命令](cli-capture.md) diff --git a/docs/src/guide/capture.md b/docs/src/guide/capture.md index 0df4e0e..eb438f3 100644 --- a/docs/src/guide/capture.md +++ b/docs/src/guide/capture.md @@ -2,7 +2,7 @@ Get **`persisting traj`** running in **5–10 minutes**: use `traj capture` or `traj proxy` to record agent sessions, then `traj stats` / `traj replay` to inspect Markdown trajectories. -> Architecture: [Capture Architecture](../design/capture_design.zh.md) (中文). CLI reference: [Traj command](../design/cli_trajectory_command.zh.md) and [Capture subcommands](../design/cli_capture_command.zh.md). +> Architecture: [Capture Architecture](../design/capture.md) (中文). CLI reference: [Traj command](../design/cli-traj.md) and [Capture subcommands](../design/cli-capture.md). --- @@ -11,9 +11,9 @@ Get **`persisting traj`** running in **5–10 minutes**: use `traj capture` or ` `traj capture`: 1. Starts a local **LLM reverse proxy** (forwards to your upstream model). -2. Writes a **Vortex event log** with `-f vortex` (canonical) or **Markdown only** with `-f md`. -3. **Both formats live-update Markdown** (`tail -f` friendly); `-f vortex` also appends `events.vortex`. -4. Prints a summary when the child exits and writes `.capture/reconcile.json` (`-f vortex` compares live md vs Vortex replay). +2. Writes **one** storage layer: **Markdown only** (`-f md`, default) or **Lance canonical** (`-f lance` → `{run}/events.lance/` dataset directory). +3. With `-f md`, live-upserts dialogue Markdown (`tail -f` friendly) and writes `.capture/reconcile.json` on exit. +4. With `-f lance`, appends structured rows only; generate Markdown later with `traj materialize`. Supported live clients: **Claude Code**, **OpenAI Codex** (via proxy). Cursor is not supported yet. @@ -68,8 +68,8 @@ Codex: replace `claude` with `codex`. Custom agent: `... -- python3 your_agent.p |------|---------| | `-o DIR` | Store root (default `.persisting/capture`) | | `-c FILE` | Proxy TOML (`listen`, `models`, upstream; optional `[network]`) | -| `-f md` | Markdown only (default) | -| `-f vortex` | Vortex canonical + same live Markdown; reconcile md ↔ Vortex | +| `-f md` | Markdown only (default); live upsert + reconcile | +| `-f lance` | Lance only (`events.lance/` directory); use `traj materialize` for md | | `--debug` | Log proxied requests to stderr + `.capture/debug.log` | Optional Harbor-aligned egress control — see [`examples/llm-proxy/allowlist.toml`](../../examples/llm-proxy/allowlist.toml): @@ -192,10 +192,11 @@ store/ └── run-{timestamp}-{nanos}/ ├── run-*.md # main agent dialogue ├── agent-*.md # subagents (if any) - └── events.vortex # Vortex event log + └── events.lance/ # Lance dataset (only with `-f lance`) ``` ```bash +# After `-f md` (or after `traj materialize`): tail -f store/deepseek-proxy/run-*/run-*.md ``` @@ -206,13 +207,13 @@ tail -f store/deepseek-proxy/run-*/run-*.md If you ran `traj proxy start` or set `PERSISTING_CAPTURE_STORAGE`, you can omit `` on `stats`, `replay`, `materialize`, and `truncate`. ```bash -# Omit --session-id to scan all runs under agent/ and expand Vortex session_id partitions +# Omit --session-id to scan all runs under agent/ and expand Lance session_id partitions persisting traj stats ./store/deepseek-proxy/ --detail persisting traj replay ./store --agent-id deepseek-proxy --session-id run-... persisting traj materialize ./store --agent-id ... --root-session-id ... --session-id ... ``` -**Multimodal (screenshots / image generation):** with default `capture_level = dialogue`, images appear as **`[image: …]` / `[image_generated: …]` placeholders** in Markdown and stats—not embedded pixels. Set `capture_level = "full"` in TOML for raw JSON in Vortex. See [trajectory TLV §2.7 (zh)](../design/trajectory_tlv_format.zh.md#27-多模态对话正文phase-0). +**Multimodal (screenshots / image generation):** with default `capture_level = dialogue`, images appear as **`[image: …]` / `[image_generated: …]` placeholders** in Markdown and stats—not embedded pixels. Set `capture_level = "full"` in TOML for raw JSON in Lance. See [trajectory Markdown format §2.7](../design/trajectory-format.md#27-多模态对话正文phase-0). --- @@ -236,7 +237,7 @@ persisting traj import ./store --provider ide --since-days 7 --project "$(pwd)" | `traj replay-dead-letter` | Replay failed capture events | | `traj stats` | Stats; `--detail` per-turn tree | | `traj replay` | Event JSON replay | -| `traj materialize` | Vortex → Markdown rebuild | +| `traj materialize` | Lance → Markdown rebuild | --- @@ -245,16 +246,16 @@ persisting traj import ./store --provider ide --since-days 7 --project "$(pwd)" | Issue | Action | |-------|--------| | Agent cannot reach proxy | `export` env vars from startup banner in a **new** terminal; Codex needs `-c openai_base_url=…` | -| md vs Vortex mismatch | `traj materialize` after checking `.capture/reconcile.json` | +| md vs Lance mismatch | Only when both layers exist: check `.capture/reconcile.json`, then `traj materialize` from Lance | | Failed capture events | `traj replay-dead-letter -o ./store` | | `traj capture` conflicts with daemon | `traj proxy stop` or use another `-o` | -| `stats` shows 0 turns | Scan **`./store/{agent_id}/`** without `--session-id` so CLI expands all Vortex `session_id` partitions; or pass the specific header UUID with `--session-id` | +| `stats` shows 0 turns | Scan **`./store/{agent_id}/`** without `--session-id` so CLI expands all Lance `session_id` partitions; or pass the specific header UUID with `--session-id` | --- ## Next steps -- [Capture architecture](../design/capture_design.zh.md) -- [Trajectory Markdown format](../design/trajectory_tlv_format.zh.md) -- [Traj CLI](../design/cli_trajectory_command.zh.md) +- [Capture architecture](../design/capture.md) +- [Trajectory Markdown format](../design/trajectory-format.md) +- [Traj CLI](../design/cli-traj.md) - [Walkthrough example](../../examples/capture-walkthrough/README.md) diff --git a/docs/src/guide/capture.zh.md b/docs/src/guide/capture.zh.md index 49a1dec..ffef41e 100644 --- a/docs/src/guide/capture.zh.md +++ b/docs/src/guide/capture.zh.md @@ -2,7 +2,7 @@ 本页帮你在 **5–10 分钟内**跑通 `persisting traj`：用 `traj capture` / `traj proxy` 采集 Agent 对话，用 `traj stats` 等查看 Markdown 轨迹。 -> 架构与概念见 [Capture 架构设计](../design/capture_design.zh.md)；命令参数见 [Traj 命令](../design/cli_trajectory_command.zh.md) 与 [Capture 子命令](../design/cli_capture_command.zh.md)。 +> 架构与概念见 [Capture 架构设计](../design/capture.md)；命令参数见 [Traj 命令](../design/cli-traj.md) 与 [Capture 子命令](../design/cli-capture.md)。 --- @@ -11,9 +11,9 @@ 运行 `traj capture` 后，Persisting 会： 1. 在本地启动 **LLM 反向代理**（透明转发到上游模型）； -2. 把每次请求/响应写入 **Vortex 事件日志**（`-f vortex`，canonical）或 **仅 Markdown**（`-f md`）； -3. **两种格式都会 live 更新 Markdown**（人类可读，可 `tail -f`）；`-f vortex` 额外落盘 `events.vortex`； -4. 子进程退出后打印摘要，并写入对账文件 `.capture/reconcile.json`（`-f vortex` 时对比 live md 与 Vortex replay）。 +2. 按 `-f` **只写一层**：默认 **仅 Markdown**（`-f md`），或 **Lance canonical**（`-f lance` → `{run}/events.lance/` 数据集目录）； +3. `-f md` 时 live upsert 对话 Markdown（可 `tail -f`），退出时写 `.capture/reconcile.json`； +4. `-f lance` 时只 append 结构化事件行；需要人读视图时再跑 `traj materialize`。 支持的实时采集客户端：**Claude Code**、**OpenAI Codex**（经代理）。Cursor 当前不支持。 @@ -160,9 +160,9 @@ persisting traj capture -o ./store -c your.toml -f md -- python3 your_agent.py |------|------| | `-o DIR` | 轨迹 store 根目录（默认 `.persisting/capture`） | | `-c FILE` | 代理 TOML（`listen`、`models`、upstream） | -| `-f md` | **仅 Markdown**（默认，live upsert 到 `{session}.md`） | -| `-f vortex` | **Vortex canonical**（`events.vortex`）+ **同样 live Markdown**；对账 md ↔ Vortex | -| `capture_level` | TOML 中 `summary` / `dialogue`（默认）/ `full`；见 [Capture 架构 §6.4](../design/capture_design.zh.md#64-可见对话提取含多模态) | +| `-f md` | **仅 Markdown**（默认，live upsert 到 `{session}.md`，并写 reconcile） | +| `-f lance` | **仅 Lance**（`events.lance/` 目录）；md 用 `traj materialize` 生成 | +| `capture_level` | TOML 中 `summary` / `dialogue`（默认）/ `full`；见 [Capture 架构 §6.4](../design/capture.md#64-可见对话提取含多模态) | | `--debug` | 将代理请求打到 stderr 与 `.capture/debug.log` | | `--` 之后 | 要执行的子命令 | @@ -300,17 +300,17 @@ capture backend 的 `-o`、`-f`、`--debug` 和 daemon actions 都不受 dlcapt store/ ├── .capture/ │ ├── sessions.json # 会话索引 -│ ├── reconcile.json # run 结束 md ↔ Vortex 对账 +│ ├── reconcile.json # `-f md` run 结束对账（纯 `-f lance` 通常无） │ ├── dead_letter.jsonl # 采集失败事件（可 replay） │ └── events.wal.jsonl # 未 ack 事件 WAL（异常退出时） └── {agent_id}/ └── run-20260528-120000-123456789/ - ├── run-20260528-120000-123456789.md # 主 Agent 对话 + ├── run-20260528-120000-123456789.md # 主 Agent 对话（`-f md` 或 materialize 后） ├── agent-{subagent-id}.md # 子 Agent（若有） - └── events.vortex # Vortex 事件日志 + └── events.lance/ # Lance dataset（仅 `-f lance`） ``` -**阅读轨迹**：直接打开 `*.md`，或实时跟踪： +**阅读轨迹**：打开 `*.md`，或实时跟踪（需已有 Markdown 层）： ```bash tail -f store/deepseek-proxy/run-*/run-*.md @@ -325,14 +325,14 @@ Capture **生产**数据；`persisting traj`（别名 `trajectory`）用于 ** 若已 `traj proxy start` 或设置了 `PERSISTING_CAPTURE_STORAGE`，`stats` / `replay` / `materialize` / `truncate` 可 **省略** `` 参数。 ```bash -# 统计（`-f md` 直接读 Markdown；`-f vortex` 可先 materialize） -# 省略 --session-id 时扫描 agent 下全部 run，并展开 Vortex 内多个 session_id 分区 +# 统计（`-f md` 直接读 Markdown；`-f lance` 可先 materialize） +# 省略 --session-id 时扫描 agent 下全部 run，并展开 Lance 内多个 session_id 分区 persisting traj stats ./store/deepseek-proxy/ --detail # 重放为 JSON 行 persisting traj replay ./store --agent-id deepseek-proxy --session-id run-20260528-120000-123456789 -# 从 Vortex 全量重建 Markdown（对账不一致或 md 损坏时） +# 从 Lance 全量重建 Markdown（对账不一致或 md 损坏时） persisting traj materialize ./store \ --agent-id deepseek-proxy \ --root-session-id run-20260528-120000-123456789 \ @@ -341,7 +341,7 @@ persisting traj materialize ./store \ 也可直接传 **run 目录** 或 `*.md` 路径，`traj` 会自动推断 `agent_id` / `session_id`。 -**多模态（截图 / 出图）**：默认 `capture_level = dialogue` 下，Markdown 与 stats 中图像以 **`[image: …]` / `[image_generated: …]` 占位符** 出现，不含像素；需完整 JSON 时在 TOML 设 `capture_level = "full"`。详见 [轨迹 Markdown §2.7](../design/trajectory_tlv_format.zh.md#27-多模态对话正文phase-0)。 +**多模态（截图 / 出图）**：默认 `capture_level = dialogue` 下，Markdown 与 stats 中图像以 **`[image: …]` / `[image_generated: …]` 占位符** 出现，不含像素；需完整 JSON 时在 TOML 设 `capture_level = "full"`。详见 [轨迹 Markdown §2.7](../design/trajectory-format.md#27-多模态对话正文phase-0)。 --- @@ -356,7 +356,7 @@ persisting traj import ./store \ --project "$(pwd)" ``` -`--dry-run` 仅统计；多会话匹配时用 `--session-id` 指定。详见 [Traj import 说明](../design/cli_capture_command.zh.md#5-traj-import)。 +`--dry-run` 仅统计；多会话匹配时用 `--session-id` 指定。详见 [Traj import 说明](../design/cli-capture.md#5-traj-import)。 --- @@ -364,12 +364,14 @@ persisting traj import ./store \ ### 对账不一致 -run 结束后查看 `.capture/reconcile.json`。若 md 与 Vortex 不一致： +`.capture/reconcile.json` 主要出现在 **`-f md`** 采集后。若同时存在 Lance 层且与 md 不一致（例如事后又写了 Lance，或旧数据）： ```bash persisting traj materialize ./store --agent-id --root-session-id --session-id ``` +纯 `-f lance` 采集没有 live Markdown，需要人读视图时直接 `traj materialize`。 + ### 采集失败事件 ```bash @@ -413,13 +415,13 @@ persisting traj capture -o ./store -c your.toml -f md --debug -- claude | `traj replay-dead-letter` | 重放失败采集事件 | | `traj stats` | 统计 / `--detail` 逐轮 | | `traj replay` | 事件 JSON 重放 | -| `traj materialize` | Vortex → Markdown 全量重建 | +| `traj materialize` | Lance → Markdown 全量重建 | --- ## 下一步 -- [Capture 架构设计](../design/capture_design.zh.md) — 产品定位、数据流、设计原则 -- [轨迹 Markdown 格式](../design/trajectory_tlv_format.zh.md) — TLV 块与 frontmatter -- [Trajectory 命令](../design/cli_trajectory_command.zh.md) — `traj` 完整参数 +- [Capture 架构设计](../design/capture.md) — 产品定位、数据流、设计原则 +- [轨迹 Markdown 格式](../design/trajectory-format.md) — TLV 块与 frontmatter +- [Trajectory 命令](../design/cli-traj.md) — `traj` 完整参数 - [分步示例代码](../../examples/capture-walkthrough/README.md) — Mock LLM 与校验脚本 diff --git a/docs/src/guide/compute.md b/docs/src/guide/compute.md index e0e3216..7b86700 100644 --- a/docs/src/guide/compute.md +++ b/docs/src/guide/compute.md @@ -140,7 +140,7 @@ plan() ──streams──► Driver ──dispatches──► Workers (Pyth | `--sink DIR` | Persist results | | `--resume` | Skip completed task IDs | | `--observe` | Show progress | -| `--traj` | Also write results as Vortex trajectory | +| `--traj` | Also write results as Lance trajectory | --- diff --git a/docs/src/guide/compute.zh.md b/docs/src/guide/compute.zh.md index 901ae97..47b1806 100644 --- a/docs/src/guide/compute.zh.md +++ b/docs/src/guide/compute.zh.md @@ -140,7 +140,7 @@ plan() ──流式产生──► Driver ──派发──► Workers (Pyt | `--sink DIR` | 持久化结果 | | `--resume` | 跳过已完成任务 id | | `--observe` | 显示进度 | -| `--traj` | 同时写入 Vortex 轨迹 | +| `--traj` | 同时写入 Lance 轨迹 | --- diff --git a/docs/src/index.md b/docs/src/index.md index c5a0bfa..9c97dad 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -21,7 +21,7 @@ Agent trajectories, model parameters, and KV cache share the same tiered storage --- - Record every LLM call. Stored as `(agent_id, run_id, time)` tensors alongside canonical Vortex event logs and human-readable Markdown. + Record every LLM call. Stored as `(agent_id, run_id, time)` tensors alongside canonical Lance event logs and human-readable Markdown. ```bash persisting traj capture -o ./store -c proxy.toml -- claude diff --git a/docs/src/index.zh.md b/docs/src/index.zh.md index 945c396..28be156 100644 --- a/docs/src/index.zh.md +++ b/docs/src/index.zh.md @@ -21,7 +21,7 @@ Agent 轨迹、模型参数和 KV Cache 共享同一套分层存储 —— **一 --- - 记录每一次 LLM 调用。以 `(agent_id, run_id, time)` 张量存储，附带 canonical Vortex 事件日志和人读 Markdown。 + 记录每一次 LLM 调用。以 `(agent_id, run_id, time)` 张量存储，附带 canonical Lance 事件日志和人读 Markdown。 ```bash persisting traj capture -o ./store -c proxy.toml -- claude diff --git a/examples/trajectory-tlv/README.md b/examples/trajectory-tlv/README.md index 6f2a7a1..cd23f96 100644 --- a/examples/trajectory-tlv/README.md +++ b/examples/trajectory-tlv/README.md @@ -1,6 +1,6 @@ # Session markdown 示例 -[`0001.md`](demo-agent/demo-run-001/0001.md)：每块为 `` + **裸 Markdown 正文**（预览即对话；compact 可还原为 `CaptureRecord` 并写入 Vortex）。 +[`0001.md`](demo-agent/demo-run-001/0001.md)：每块为 `` + **裸 Markdown 正文**（预览即对话；compact 可还原为 `CaptureRecord` 并写入 Lance）。 ```bash ./target/debug/persisting trajectory replay \ diff --git a/justfile b/justfile index 5215756..7f84ec0 100644 --- a/justfile +++ b/justfile @@ -1,9 +1,8 @@ # Persisting — 仓库主任务入口 # 安装：brew install just / cargo install just # -# 测试选单：just test-suite # 交互选择测试 / 回归套件 -# just test-suite list # 列出全部套件 -# just test-suite gate # 直接运行指定套件 +# 集成测试：scripts/integration/*.sh（推荐 ./scripts/test_suite.sh） +# just 仅作薄封装：just smoke / just traj-e2e / just capture-all repo := justfile_directory() docs_dir := repo / "docs" @@ -29,41 +28,62 @@ default: @just --list --unsorted @echo "" @echo "常用：" - @echo " just fmt # 格式化 Rust + Python" - @echo " just lint # clippy + ruff check" - @echo " just fix # 自动 format + ruff --fix" - @echo " just test-suite # 测试 / 回归选单（推荐入口）" - @echo " just test-suite list # 列出全部套件" - @echo " just dev # 日常开发（fmt + clippy + check-quick）" @echo " just gate # 提交前（fmt + lint + test-rust）" - @echo " just ci-lint # CI lint（只检查、不改写）" + @echo " just smoke # search + traj CLI 冒烟" + @echo " just regression # capture Rust + 集成 + smoke" + @echo " ./scripts/test_suite.sh # 集成测试套件（shell）" @echo " just ci # CI 近似全量" @echo " just py-dev # maturin develop（Python 扩展）" @echo " just build-wheel # 打 release wheel → dist/" @echo " just capture-all # 全部 capture 集成" @echo " just docs-serve # 本地文档" -# ── 测试 / 回归统一入口 ─────────────────────────────────────────────────────── +# ── 测试套件导航 ────────────────────────────────────────────────────────────── -# 交互选单或直接运行：just test-suite [dev|gate|traj-e2e|all-regression|…] -# 环境变量：PERSISTING_BUILD_PROFILE=debug SKIP_REBUILD=1 -test-suite name="": +# 列出推荐测试（Rust/门禁用 just；集成用 scripts/test_suite.sh） +[group('test')] +test-list: #!/usr/bin/env bash set -euo pipefail - export PERSISTING_BUILD_PROFILE="${PERSISTING_BUILD_PROFILE:-debug}" - export SKIP_REBUILD="${SKIP_REBUILD:-0}" - bash "{{ test_suite_sh }}" {{ if name != "" { name } else { "menu" } }} - -# 列出全部测试套件（非交互） -test-list: - bash "{{ test_suite_sh }}" list - -# 别名 + cat <<'EOF' + Persisting 测试入口 + + 门禁 / Rust（just） + just gate 提交前：fmt + lint + test-rust + just dev 日常：fmt + clippy + check-quick + just ci CI 近似 + just test-rust / test-engine / capture-test / test-py + + 集成（shell → scripts/integration/） + ./scripts/test_suite.sh list + ./scripts/test_suite.sh smoke + ./scripts/test_suite.sh traj-e2e + ./scripts/test_suite.sh capture-all + ./scripts/test_suite.sh all-integration + + just 薄封装（等价调用上述脚本） + just smoke / just integration / just traj-e2e + just capture-integration / just capture-stress / just capture-run-e2e + just capture-all / just regression + + 环境变量：PERSISTING_BUILD_PROFILE SKIP_BUILD=1 SKIP_REBUILD=1 + EOF + +# 集成套件入口（转发到 scripts/test_suite.sh） +[group('test')] +test-suite name="list": + bash "{{ test_suite_sh }}" {{ name }} + +[group('test')] test-menu: - just test-suite + bash "{{ test_suite_sh }}" list -regression: - SKIP_REBUILD="${SKIP_REBUILD:-0}" just test-suite all-regression +# capture Rust + 全部 shell 集成 + CLI 冒烟（跳过 fmt/clippy） +[group('test')] +regression profile="debug": + just capture-test + just capture-all profile="{{ profile }}" + just smoke profile="{{ profile }}" # ── 构建 ───────────────────────────────────────────────────────────────────── @@ -204,7 +224,9 @@ test-capture-network: test-engine-integration: cargo test -p persisting-engine --test search_integration -_test-engine: +# persisting-engine 库单测 +[group('test')] +test-engine: cargo test -p persisting-engine --lib # Rust + Python @@ -263,202 +285,36 @@ generate-benchmark search_rows="100" traj_rows="50" seed="42" search_out="" traj [[ -n "{{ traj_out }}" ]] && args+=(--traj-out "{{ traj_out }}") python3 "$gen_py" "${args[@]}" -# ── Search + Trajectory CLI 集成 ───────────────────────────────────────────── +# ── Search + Trajectory CLI 集成（scripts/integration/*.sh）────────────────── -# 全链路：generate → search → trajectory；quick=1 为冒烟规模 -integration profile="debug" cli_source="target" cli_bin="" engine_override="" reorder="0" quick="0" search_rows="100000" traj_rows="100000" seed="42" skip_rebuild="0" embed_dim="32" num_partitions="128" nprobes="32" replay_limit="50": (_integration-build-if-target cli_source profile) +# 全链路：generate → search → trajectory（环境变量见 search_traj_e2e.sh） +integration profile="debug": #!/usr/bin/env bash set -euo pipefail - repo="{{ repo }}" - gen_py="{{ gen_py }}" - die() { echo "error: $*" >&2; exit 1; } - command -v python3 >/dev/null || die "need python3" - [[ -f "$gen_py" ]] || die "missing $gen_py" - - cli_source="{{ cli_source }}" - profile="{{ profile }}" - td="$repo/target/$profile" - - if [[ "$cli_source" == "target" ]]; then - export PERSISTING_CLI="$td/persisting" - export PERSISTING_ENGINE_LIB="$td/{{ engine_filename }}" - elif [[ -n "{{ cli_bin }}" ]]; then - export PERSISTING_CLI="{{ cli_bin }}" - if [[ -n "{{ engine_override }}" ]]; then - export PERSISTING_ENGINE_LIB="{{ engine_override }}" - else - unset PERSISTING_ENGINE_LIB || true - fi - else - command -v persisting >/dev/null || die "cli_source=path needs persisting on PATH or cli_bin=..." - export PERSISTING_CLI="$(command -v persisting)" - if [[ -n "{{ engine_override }}" ]]; then - export PERSISTING_ENGINE_LIB="{{ engine_override }}" - else - unset PERSISTING_ENGINE_LIB || true - fi - fi - - [[ -x "$PERSISTING_CLI" ]] || die "not executable: $PERSISTING_CLI" - - reorder="{{ reorder }}" - quick="{{ quick }}" - seed="{{ seed }}" - skip_rebuild="{{ skip_rebuild }}" - embed_dim="{{ embed_dim }}" - replay_limit="{{ replay_limit }}" - - if [[ "$quick" == "1" ]]; then - sr=70; tr=25; np=2; nv=2 - else - sr="{{ search_rows }}"; tr="{{ traj_rows }}" - np="{{ num_partitions }}"; nv="{{ nprobes }}" - fi - - echo "==> CLI: $PERSISTING_CLI" - if [[ -n "${PERSISTING_ENGINE_LIB:-}" ]]; then - echo "==> PERSISTING_ENGINE_LIB=$PERSISTING_ENGINE_LIB" - else - echo "==> PERSISTING_ENGINE_LIB unset (CLI default lookup)" - fi - echo "==> params: cli_source=$cli_source profile=$profile reorder=$reorder quick=$quick rows=$sr/$tr seed=$seed" - - WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/persisting-it.XXXXXX")" - trap 'rm -rf "$WORKDIR"' EXIT - - DATASET="$WORKDIR/search_ds" - STORAGE="$WORKDIR/traj_root" - TRAJ_AGENT="bench_agent" - TRAJ_SESSION="cli_bench_run" - SEARCH_JSONL="$WORKDIR/docs.jsonl" - TRAJ_INPUT="$WORKDIR/traj_records.toml" - - if [[ -t 1 ]]; then - _b=$'\033[1m'; _c=$'\033[36m'; _m=$'\033[35m'; _y=$'\033[33m'; _n=$'\033[0m' - else - _b=; _c=; _m=; _y=; _n= - fi - it_rule() { printf '%b%s%b\n' "$_b$_c" "────────────────────────────────────────────────────────" "$_n"; } - it_sh() { - it_rule; printf '%b$ ' "$_b$_m"; printf '%q ' "$@"; printf '%b\n' "$_n"; it_rule; "$@" - } - it_run_cli_out() { - it_rule; printf '%b$ ' "$_b$_m"; printf '%q' "$PERSISTING_CLI"; printf ' %q' "$@"; printf '%b\n' "$_n" - it_rule; out=$("$PERSISTING_CLI" "$@") - printf '%b[stdout 捕获]%b\n' "$_b$_y" "$_n"; printf '%s\n\n' "$out" - } - timer_start() { export TIMER_START="$(python3 -c "import time; print(time.perf_counter())")"; } - timer_end() { - python3 -c "import os,time,sys; t0=float(os.environ['TIMER_START']); print(f\"[timer] {sys.argv[1]}: {time.perf_counter()-t0:.3f}s\")" "$1" - } - assert_ok() { - grep -qE 'status:\s*"ok"|status\s*=\s*"ok"' <<<"$1" || { echo "--- bad response ---"; echo "$1"; exit 1; } - } - - export RUNNER_T0="$(python3 -c "import time; print(time.perf_counter())")" - - echo "==> generate JSONL (seed=$seed)" - timer_start - it_sh python3 "$gen_py" --seed "$seed" --search-rows "$sr" --traj-rows "$tr" --search-out "$SEARCH_JSONL" --traj-out "$TRAJ_INPUT" - timer_end "generate_benchmark_data.py" - - echo "==> search create ($sr rows)" - timer_start - it_run_cli_out search create "$DATASET" --input "$SEARCH_JSONL" --format jsonl --embedding-dim "$embed_dim" - assert_ok "$out"; timer_end "search create" - - echo "==> search index list (pre-build)" - timer_start - it_run_cli_out search index list "$DATASET" - assert_ok "$out"; timer_end "search index list (pre-build)" - - echo "==> search index build" - timer_start - it_run_cli_out search index build "$DATASET" \ - --vector-column embedding --text-column text --metric cosine \ - --num-partitions "$np" --ivf-max-iters 12 \ - --pq-num-sub-vectors 8 --pq-num-bits 4 --pq-max-iters 12 \ - --pq-sample-rate 4 --pq-kmeans-redos 1 - assert_ok "$out"; timer_end "search index build" - - echo "==> search index list (post-build)" - timer_start - it_run_cli_out search index list "$DATASET" - assert_ok "$out" - grep -q 'persisting_ivf_pq' <<<"$out" || die "missing persisting_ivf_pq" - grep -q 'persisting_fts' <<<"$out" || die "missing persisting_fts" - timer_end "search index list (post-build)" - - echo "==> search query x3" - timer_start - it_run_cli_out search query "$DATASET" "integration alpha gamma" --mode vector --k 5 --embedding-dim "$embed_dim" --nprobes "$nv" - assert_ok "$out" - it_run_cli_out search query "$DATASET" "integration" --mode fts --k 8 --embedding-dim "$embed_dim" - assert_ok "$out" - it_run_cli_out search query "$DATASET" "keyword beta" --mode hybrid --k 5 --embedding-dim "$embed_dim" - assert_ok "$out" - timer_end "search query (3 modes)" - - if [[ "$reorder" == "1" ]]; then - echo "==> search index reorder" - timer_start - it_run_cli_out search index reorder "$DATASET" persisting_ivf_pq --in-place - assert_ok "$out"; timer_end "search index reorder" - fi - - if [[ "$skip_rebuild" != "1" ]]; then - echo "==> search index rebuild" - timer_start - it_run_cli_out search index rebuild "$DATASET" --no-retrain - assert_ok "$out"; timer_end "search index rebuild" - else - echo "==> skip search index rebuild" - fi - - echo "==> trajectory add ($tr rows)" - timer_start - it_run_cli_out trajectory add "$STORAGE" --agent-id "$TRAJ_AGENT" --session-id "$TRAJ_SESSION" --format toml --input "$TRAJ_INPUT" - assert_ok "$out"; timer_end "trajectory add" - - echo "==> trajectory stats" - timer_start - it_run_cli_out trajectory stats "$STORAGE" --agent-id "$TRAJ_AGENT" --session-id "$TRAJ_SESSION" - assert_ok "$out" - grep -Fq "$TRAJ_SESSION" <<<"$out" || die "stats missing session_id" - grep -Fq "$TRAJ_AGENT" <<<"$out" || die "stats missing agent_id" - timer_end "trajectory stats" - - echo "==> trajectory replay" - timer_start - it_run_cli_out trajectory replay "$STORAGE" --agent-id "$TRAJ_AGENT" --session-id "$TRAJ_SESSION" --offset 2 --limit "$replay_limit" - assert_ok "$out" - grep -qE '\[\[records\]\]|^records\s*=' <<<"$out" || die "replay missing TOML records" - timer_end "trajectory replay" - - it_sh python3 -c "import os,time; t0=float(os.environ['RUNNER_T0']); print(f'[timer] TOTAL wall: {time.perf_counter()-t0:.3f}s')" - echo "==> integration OK" - -_integration-build-if-target cli_source profile: - #!/usr/bin/env bash - if [[ "{{ cli_source }}" == "target" ]]; then - cd "{{ repo }}" - if [[ "{{ profile }}" == "release" ]]; then - cargo build -p persisting-cli -p persisting-engine --release - else - cargo build -p persisting-cli -p persisting-engine - fi - fi + export PERSISTING_BUILD_PROFILE="{{ profile }}" + export CLI_SOURCE="${CLI_SOURCE:-target}" + export QUICK="${QUICK:-0}" + [[ "${SKIP_REBUILD:-0}" == "1" || "${SKIP_BUILD:-0}" == "1" ]] && export SKIP_BUILD=1 + echo "==> search/traj integration (profile={{ profile }} QUICK=${QUICK:-0})" + bash "{{ repo }}/scripts/integration/search_traj_e2e.sh" check profile="debug": - just _test-engine + just test-engine just integration profile="{{ profile }}" smoke profile="debug": - just integration profile="{{ profile }}" quick="1" + #!/usr/bin/env bash + set -euo pipefail + export PERSISTING_BUILD_PROFILE="{{ profile }}" + export CLI_SOURCE="${CLI_SOURCE:-target}" + export QUICK=1 + [[ "${SKIP_REBUILD:-0}" == "1" || "${SKIP_BUILD:-0}" == "1" ]] && export SKIP_BUILD=1 + echo "==> search/traj smoke (profile={{ profile }})" + bash "{{ repo }}/scripts/integration/search_traj_e2e.sh" check-quick profile="debug": - just _test-engine - just integration profile="{{ profile }}" quick="1" + just test-engine + just smoke profile="{{ profile }}" # ── Capture 集成（scripts/integration/*.sh，build 由 _common.sh 处理）──────── diff --git a/persisting/store/block.py b/persisting/store/block.py index 11ed19a..19d2c43 100644 --- a/persisting/store/block.py +++ b/persisting/store/block.py @@ -107,10 +107,7 @@ def region_to_blocks( else: return [] - return [ - BlockId(partition_key, k) - for k in range(block_lo, block_hi) - ] + return [BlockId(partition_key, k) for k in range(block_lo, block_hi)] def block_to_region( diff --git a/persisting/store/block_store.py b/persisting/store/block_store.py index 544dc51..6cb0596 100644 --- a/persisting/store/block_store.py +++ b/persisting/store/block_store.py @@ -62,7 +62,9 @@ def __init__( raise ValueError("L1/L3 backing.shape 须与 shape 一致") self._blocks_in_l1: set[BlockId] = set() self._lock = threading.Lock() - self._prefetch_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="blockstore-prefetch") + self._prefetch_executor = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="blockstore-prefetch" + ) self._prefetch_pending: dict[int, tuple[frozenset[BlockId], threading.Event]] = {} self._prefetch_counter = 0 diff --git a/persisting/store/block_store_actor.py b/persisting/store/block_store_actor.py index 3eeb520..8cafb0e 100644 --- a/persisting/store/block_store_actor.py +++ b/persisting/store/block_store_actor.py @@ -53,6 +53,7 @@ def _dims_from_ser(dims_ser: list[tuple[str, str]]) -> tuple[Dimension, ...]: def _ensure_pulsing(): try: import pulsing.core + if not pulsing.core.is_initialized(): raise RuntimeError("Pulsing 未初始化，请先 await pulsing.core.init()") except ImportError as e: @@ -111,6 +112,7 @@ def wait_region(self, region_ser: list[tuple[str, str, Any]]) -> None: # 延迟绑定，避免 import 时未 init pulsing BlockStoreActor = None + def get_block_store_actor_class(): global BlockStoreActor if BlockStoreActor is None: @@ -141,20 +143,26 @@ async def put_async(self, region: Region, value: np.ndarray) -> None: def get(self, region: Region) -> np.ndarray: """同步 get；无运行中 event loop 时可用。""" import asyncio + region_ser = region_serialize(region, self._dims) try: asyncio.get_running_loop() except RuntimeError: return asyncio.run(self._proxy.get_region(region_ser)) - raise RuntimeError("ActorStore.get 不能在已运行的 event loop 内调用，请使用 await store.get_async(region)") + raise RuntimeError( + "ActorStore.get 不能在已运行的 event loop 内调用，请使用 await store.get_async(region)" + ) def put(self, region: Region, value: np.ndarray) -> None: """同步 put；无运行中 event loop 时可用。""" import asyncio + region_ser = region_serialize(region, self._dims) try: asyncio.get_running_loop() except RuntimeError: asyncio.run(self._proxy.put_region(region_ser, value)) return - raise RuntimeError("ActorStore.put 不能在已运行的 event loop 内调用，请使用 await store.put_async(region, value)") + raise RuntimeError( + "ActorStore.put 不能在已运行的 event loop 内调用，请使用 await store.put_async(region, value)" + ) diff --git a/persisting/store/local_tensor.py b/persisting/store/local_tensor.py index dbe260b..6b873e4 100644 --- a/persisting/store/local_tensor.py +++ b/persisting/store/local_tensor.py @@ -70,7 +70,11 @@ def _constraint_to_index( return slice(None) # Point - if hasattr(constraint, "value") and not hasattr(constraint, "lo") and not hasattr(constraint, "values"): + if ( + hasattr(constraint, "value") + and not hasattr(constraint, "lo") + and not hasattr(constraint, "values") + ): v = constraint.value if dim.kind == "int": i = int(v) @@ -196,6 +200,7 @@ class Backing(Protocol): @property def shape(self) -> tuple[int, ...]: ... + """逻辑 shape，与 TensorLayout.shape 一致。""" @property @@ -331,6 +336,7 @@ def __init__( if use_mmap: try: from persisting.core import mmap_reserve + if mmap_reserve is not None: self._region = mmap_reserve(total_bytes) self._inner = None @@ -366,9 +372,7 @@ def write(self, indices: tuple[int | slice | np.ndarray, ...], data: np.ndarray) if self._inner is not None: self._inner.write(indices, data) return - start_byte, nbytes, _ = _indices_to_contiguous_byte_range( - self._shape, self._dtype, indices - ) + start_byte, nbytes, _ = _indices_to_contiguous_byte_range(self._shape, self._dtype, indices) if start_byte is None: raise NotImplementedError("BlockMappedBacking(mmap) 暂仅支持连续索引") self._region.copy_in(start_byte, np.asarray(data, dtype=self._dtype).tobytes()) @@ -437,7 +441,9 @@ def __init__( try: from safetensors import safe_open except ImportError as err: - raise ImportError("SafetensorsBacking 需要安装 safetensors: pip install safetensors") from err + raise ImportError( + "SafetensorsBacking 需要安装 safetensors: pip install safetensors" + ) from err path = Path(path) self._path = path @@ -531,7 +537,9 @@ def numpy(self) -> np.ndarray: """仅当 backing 为 NumpyBacking 时可用；否则 AttributeError。""" if isinstance(self._backing, NumpyBacking): return self._backing.numpy - raise AttributeError("numpy 仅适用于 NumpyBacking，当前 backing 为 %s" % type(self._backing).__name__) + raise AttributeError( + "numpy 仅适用于 NumpyBacking，当前 backing 为 %s" % type(self._backing).__name__ + ) def get(self, region: Region) -> np.ndarray: """按 TTAS Region 读出，返回拷贝。""" diff --git a/persisting/store/tiered_loop.py b/persisting/store/tiered_loop.py index a2792b5..13d9717 100644 --- a/persisting/store/tiered_loop.py +++ b/persisting/store/tiered_loop.py @@ -26,7 +26,9 @@ def __init__(self, fill_blocks: Callable[[list[BlockId]], None]): """ self._fill_blocks = fill_blocks self._reader, self._writer = os.pipe() - self._prefetch_queue: queue.Queue[tuple[list[BlockId], Callable[[], None] | None]] = queue.Queue() + self._prefetch_queue: queue.Queue[tuple[list[BlockId], Callable[[], None] | None]] = ( + queue.Queue() + ) self._loop: asyncio.AbstractEventLoop | None = None self._thread: threading.Thread | None = None self._stop = threading.Event() diff --git a/scripts/generate_benchmark_data.py b/scripts/generate_benchmark_data.py index 61a1f69..2307619 100755 --- a/scripts/generate_benchmark_data.py +++ b/scripts/generate_benchmark_data.py @@ -73,17 +73,23 @@ def main() -> None: n_t = args.traj_rows with args.traj_out.open("w", encoding="utf-8", buffering=1024 * 1024) as f: if args.traj_format == "toml": - f.write("# benchmark trajectory (generated; root `records` via [[records]] tables)\n\n") + f.write("# benchmark trajectory (CaptureRecord-shaped [[records]] for traj add)\n\n") for i in range(n_t): tok = rng.randrange(2**31) f.write("[[records]]\n") - f.write(f"step = {i}\n") - f.write('kind = "tool"\n') - f.write(f"detail = \"mock step {i} rnd={tok}\"\n\n") + f.write(f"seq = {i}\n") + f.write('source = "bench"\n') + f.write('kind = "note"\n') + f.write(f'payload = {{ content = "mock step {i} rnd={tok}" }}\n\n') else: for i in range(n_t): tok = rng.randrange(2**31) - rec = {"step": i, "kind": "tool", "detail": f"mock step {i} rnd={tok}"} + rec = { + "seq": i, + "source": "bench", + "kind": "note", + "payload": {"content": f"mock step {i} rnd={tok}"}, + } f.write(json.dumps(rec) + "\n") print( diff --git a/scripts/integration/README.md b/scripts/integration/README.md index 6ad5e69..66dc451 100644 --- a/scripts/integration/README.md +++ b/scripts/integration/README.md @@ -1,50 +1,60 @@ # CLI 集成测试 +全部实现为 `scripts/integration/*.sh`；`just` 只做薄封装。 + | 脚本 | 说明 | |------|------| +| [`search_traj_e2e.sh`](search_traj_e2e.sh) | **search + traj CLI 全链路**：generate → search → trajectory（`QUICK=1` 冒烟） | | [`traj_e2e.sh`](traj_e2e.sh) | **`traj` 读写链路**：gateway `import` → `stats` → `judge --score` → `stats` / `judge-stats` | | [`capture_integration.sh`](capture_integration.sh) | `persisting traj`：proxy 守护进程、admin/list、mock 代理、`traj import` | -| [`capture_stress.sh`](capture_stress.sh) | 实时写入压测：并发请求、延迟分位、Vortex 行数、append 失败日志 | -| [`capture_run_e2e.sh`](capture_run_e2e.sh) | **`traj capture -f vortex`**（或 `bin`）：mock LLM + 多轮 agent，Vortex drain → materialize → vortex replay | +| [`capture_stress.sh`](capture_stress.sh) | 实时写入压测：并发请求、延迟分位、Lance 行数、append 失败日志 | +| [`capture_run_e2e.sh`](capture_run_e2e.sh) | **`traj capture -f lance`**（或 `bin`）：mock LLM + 多轮 agent，Lance drain → materialize → lance replay | | [`capture_run_agent.py`](capture_run_agent.py) | 在 `traj capture` 子进程内调 `OPENAI_BASE_URL` | | [`mock_llm_api_server.py`](mock_llm_api_server.py) | 可记录请求的 mock LLM API（上游） | +**统一入口：** + ```bash -just capture-integration -# 等价 -cargo build -p persisting-cli -./scripts/integration/capture_integration.sh +./scripts/test_suite.sh list +./scripts/test_suite.sh smoke +./scripts/test_suite.sh traj-e2e +./scripts/test_suite.sh capture-all +./scripts/test_suite.sh all-integration ``` -**`traj` 子命令集成**（import → stats → judge → stats，无需 mock 代理）： +**或 just 薄封装：** ```bash +just smoke +just integration just traj-e2e +just capture-integration +just capture-stress +just capture-run-e2e +just capture-all ``` -**统一测试选单**（交互或指定套件名）： +直接跑脚本： ```bash -just test-suite # 交互选单 -just test-suite list # 列出全部套件 -just test-suite gate # 直接运行 -just regression # 全部 shell + capture Rust 回归 +./scripts/integration/capture_integration.sh +QUICK=1 ./scripts/integration/search_traj_e2e.sh ``` -环境变量：`PERSISTING_CLI`、`PERSISTING_ENGINE_LIB`（Vortex 写入时需要）、`SKIP_BUILD=1`、`PERSISTING_BUILD_PROFILE=release`。 +环境变量：`PERSISTING_CLI`、`PERSISTING_ENGINE_LIB`、`SKIP_BUILD=1`、`PERSISTING_BUILD_PROFILE=release`。 -`capture_run_e2e.sh` 额外变量：`TURNS`、`DRAIN_SEC`、`CAPTURE_FORMAT`（`vortex` 或 legacy 别名 `bin`）。 +`capture_run_e2e.sh` 额外变量：`TURNS`、`DRAIN_SEC`、`CAPTURE_FORMAT`（`lance` 或 legacy 别名 `bin`）。 -压测（需 `persisting-engine`）： +压测： ```bash just capture-stress REQUESTS=200 CONCURRENCY=20 MIN_SUCCESS_RATE=0.99 just capture-stress ``` -`capture_stress.sh` 额外变量：`DRAIN_SEC`、`MIN_ROW_RATIO`、`MAX_P99_MS`；`traj proxy` 日志写入临时目录便于统计 `trajectory append failed`。 +`capture_stress.sh` 额外变量：`DRAIN_SEC`、`MIN_ROW_RATIO`、`MAX_P99_MS`。 -**`traj capture` 全链路**（`-f vortex`：mock 上游 + 多轮 agent，断言 Vortex 行数 + materialize + vortex replay）： +**`traj capture` 全链路：** ```bash just capture-run-e2e diff --git a/scripts/integration/_common.sh b/scripts/integration/_common.sh index 507aa2d..dbcb1dd 100644 --- a/scripts/integration/_common.sh +++ b/scripts/integration/_common.sh @@ -79,7 +79,7 @@ capture_wait_http() { return 1 } -# Poll traj stats (Vortex layer) until row_count >= expected or plateau. +# Poll traj stats (Lance layer) until row_count >= expected or plateau. capture_drain_event_rows() { local storage="$1" agent_id="$2" session_id="$3" expected="$4" drain_sec="$5" local stats_toml="$6" @@ -90,7 +90,7 @@ capture_drain_event_rows() { if "$CLI" traj stats "$storage" \ --agent-id "$agent_id" \ --session-id "$session_id" \ - --storage-format vortex >"$stats_toml" 2>/dev/null; then + --storage-format lance >"$stats_toml" 2>/dev/null; then best="$(capture_parse_toml_row_count "$stats_toml")" if [[ "$best" -ge "$expected" ]]; then echo "$best" diff --git a/scripts/integration/capture_run_e2e.sh b/scripts/integration/capture_run_e2e.sh index 74dbc83..39859b6 100755 --- a/scripts/integration/capture_run_e2e.sh +++ b/scripts/integration/capture_run_e2e.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# E2E: `traj capture -f vortex` + mock LLM API — Vortex drain, materialize, vortex replay. +# E2E: `traj capture -f lance` + mock LLM API — Lance drain, materialize, lance replay. # # Flow: # mock LLM API (upstream) ← capture proxy ← agent (OPENAI_BASE_URL via traj capture) @@ -11,7 +11,7 @@ # Env: # TURNS default 3 # DRAIN_SEC default 60 -# CAPTURE_FORMAT vortex | bin (default vortex) +# CAPTURE_FORMAT lance | bin (default lance) # SKIP_BUILD default 0 set -euo pipefail @@ -25,7 +25,7 @@ AGENT_PY="$REPO_ROOT/scripts/integration/capture_run_agent.py" TURNS="${TURNS:-3}" DRAIN_SEC="${DRAIN_SEC:-60}" -CAPTURE_FORMAT="${CAPTURE_FORMAT:-vortex}" +CAPTURE_FORMAT="${CAPTURE_FORMAT:-lance}" AGENT_ID="capture-run-e2e" pass() { echo " ok: $*"; } @@ -37,8 +37,8 @@ capture_resolve_binaries "${SKIP_BUILD:-0}" 1 command -v python3 >/dev/null || die "need python3" [[ -f "$MOCK_API" && -f "$AGENT_PY" ]] || die "missing scripts" case "$CAPTURE_FORMAT" in - vortex|bin) ;; - *) die "CAPTURE_FORMAT must be vortex or bin (got $CAPTURE_FORMAT)" ;; + lance|bin) ;; + *) die "CAPTURE_FORMAT must be lance or bin (got $CAPTURE_FORMAT)" ;; esac WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/persisting-capture-run-e2e.XXXXXX")" @@ -121,22 +121,22 @@ MOCK_COUNT="$(wc -l <"$MOCK_LOG" | tr -d ' ')" [[ "$MOCK_COUNT" -eq "$TURNS" ]] || die "mock API saw $MOCK_COUNT requests, expected $TURNS" pass "mock API received $TURNS HTTP requests" -VORTEX_PATH="$STORAGE/$AGENT_ID/$ROOT_SESSION/events.vortex" -[[ -f "$VORTEX_PATH" ]] || die "missing Vortex log: $VORTEX_PATH" -pass "events.vortex present" +LANCE_PATH="$STORAGE/$AGENT_ID/$ROOT_SESSION/events.lance" +[[ -d "$LANCE_PATH" ]] || die "missing Lance log: $LANCE_PATH" +pass "events.lance present" -section "vortex-only capture (no live markdown)" +section "lance-only capture (no live markdown)" MD_PATH="$STORAGE/$AGENT_ID/$ROOT_SESSION/${ROOT_SESSION}.md" if [[ -f "$MD_PATH" ]]; then - die "vortex/bin must not write .md during capture (found $MD_PATH); use \`traj materialize\` for md" + die "lance/bin must not write .md during capture (found $MD_PATH); use \`traj materialize\` for md" fi -pass "no live markdown during vortex capture" +pass "no live markdown during lance capture" -section "wait for Vortex drain (up to ${DRAIN_SEC}s)" +section "wait for Lance drain (up to ${DRAIN_SEC}s)" EXPECTED_ROWS=$((TURNS * 2 + 2)) best="$(capture_drain_event_rows "$STORAGE" "$AGENT_ID" "$ROOT_SESSION" "$EXPECTED_ROWS" "$DRAIN_SEC" "$STATS_TOML" || true)" -[[ "${best:-0}" -ge "$EXPECTED_ROWS" ]] || die "Vortex rows ${best:-0} < expected $EXPECTED_ROWS" -pass "Vortex row_count=$best (expected >= $EXPECTED_ROWS)" +[[ "${best:-0}" -ge "$EXPECTED_ROWS" ]] || die "Lance rows ${best:-0} < expected $EXPECTED_ROWS" +pass "Lance row_count=$best (expected >= $EXPECTED_ROWS)" section "traj materialize (idempotent rebuild)" MAT_OUT="$("$CLI" traj materialize "$STORAGE" \ @@ -146,11 +146,11 @@ echo "$MAT_OUT" grep -q 'status = "ok"' <<<"$MAT_OUT" || die "traj materialize failed" pass "traj materialize" -section "trajectory replay (vortex) contains all turns" +section "trajectory replay (lance) contains all turns" "$CLI" traj replay "$STORAGE" \ --agent-id "$AGENT_ID" \ --session-id "$ROOT_SESSION" \ - --storage-format vortex >"$REPLAY_TOML" + --storage-format lance >"$REPLAY_TOML" python3 <= {turns * 2} llm events, got {llm_rows}") -print(f"verified {turns} user/assistant pairs in vortex replay") +print(f"verified {turns} user/assistant pairs in lance replay") PY -pass "vortex replay content matches agent manifest" +pass "lance replay content matches agent manifest" section "materialized markdown content" MD_PATH="$STORAGE/$AGENT_ID/$ROOT_SESSION/${ROOT_SESSION}.md" diff --git a/scripts/integration/capture_stress.sh b/scripts/integration/capture_stress.sh index 8621868..00e1ee7 100755 --- a/scripts/integration/capture_stress.sh +++ b/scripts/integration/capture_stress.sh @@ -76,8 +76,8 @@ MOCK_PID=$! sleep 0.2 kill -0 "$MOCK_PID" || die "mock upstream failed" -# Optional: `-f bin` is a legacy alias for `-f vortex`. -"$CLI" traj proxy -o "$STORAGE" -c "$CONFIG" -f vortex >>"$SERVE_LOG" 2>&1 & +# Optional: `-f bin` is a legacy alias for `-f lance`. +"$CLI" traj proxy -o "$STORAGE" -c "$CONFIG" -f lance >>"$SERVE_LOG" 2>&1 & SERVE_PID=$! sleep 0.3 kill -0 "$SERVE_PID" || { tail -20 "$SERVE_LOG"; die "traj proxy exited early"; } @@ -110,7 +110,7 @@ APPEND_FAILS="$(grep -c 'trajectory append failed' "$SERVE_LOG" 2>/dev/null || t APPEND_FAILS="${APPEND_FAILS:-0}" echo "==> reliability: rows=$best/$EXPECTED_ROWS append_failures=$APPEND_FAILS" -[[ "${best:-0}" -ge 1 ]] || { tail -30 "$SERVE_LOG"; die "no Vortex event rows"; } +[[ "${best:-0}" -ge 1 ]] || { tail -30 "$SERVE_LOG"; die "no Lance event rows"; } python3 </dev/null || die "need python3" +[[ -f "$GEN_PY" ]] || die "missing $GEN_PY" + +# Resolve CLI / engine (reuse capture helpers when building from target) +if [[ "$CLI_SOURCE" == "target" ]]; then + if [[ "${SKIP_BUILD:-0}" != "1" && "$PROFILE" == "release" ]]; then + echo "==> cargo build -p persisting-cli -p persisting-engine --release" + (cd "$REPO_ROOT" && cargo build -p persisting-cli -p persisting-engine --release) + export SKIP_BUILD=1 + fi + capture_resolve_binaries "${SKIP_BUILD:-0}" 1 + export PERSISTING_CLI="$CLI" +elif [[ -n "${PERSISTING_CLI:-}" ]]; then + CLI="$PERSISTING_CLI" + [[ -x "$CLI" ]] || die "not executable: $CLI" +else + command -v persisting >/dev/null || die "CLI_SOURCE=$CLI_SOURCE needs persisting on PATH or PERSISTING_CLI=..." + CLI="$(command -v persisting)" + export PERSISTING_CLI="$CLI" +fi + +echo "==> CLI: $CLI" +if [[ -n "${PERSISTING_ENGINE_LIB:-}" ]]; then + echo "==> PERSISTING_ENGINE_LIB=$PERSISTING_ENGINE_LIB" +else + echo "==> PERSISTING_ENGINE_LIB unset (CLI default lookup)" +fi +echo "==> params: cli_source=$CLI_SOURCE profile=$PROFILE reorder=$REORDER quick=$QUICK rows=$SEARCH_ROWS/$TRAJ_ROWS seed=$SEED" + +WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/persisting-it.XXXXXX")" +trap 'rm -rf "$WORKDIR"' EXIT + +DATASET="$WORKDIR/search_ds" +STORAGE="$WORKDIR/traj_root" +TRAJ_AGENT="bench_agent" +TRAJ_SESSION="cli_bench_run" +SEARCH_JSONL="$WORKDIR/docs.jsonl" +TRAJ_INPUT="$WORKDIR/traj_records.toml" + +if [[ -t 1 ]]; then + _b=$'\033[1m'; _c=$'\033[36m'; _m=$'\033[35m'; _y=$'\033[33m'; _n=$'\033[0m' +else + _b=; _c=; _m=; _y=; _n= +fi +it_rule() { printf '%b%s%b\n' "$_b$_c" "────────────────────────────────────────────────────────" "$_n"; } +it_sh() { + it_rule; printf '%b$ ' "$_b$_m"; printf '%q ' "$@"; printf '%b\n' "$_n"; it_rule; "$@" +} +it_run_cli_out() { + it_rule; printf '%b$ ' "$_b$_m"; printf '%q' "$CLI"; printf ' %q' "$@"; printf '%b\n' "$_n" + it_rule; out=$("$CLI" "$@") + printf '%b[stdout 捕获]%b\n' "$_b$_y" "$_n"; printf '%s\n\n' "$out" +} +timer_start() { export TIMER_START="$(python3 -c "import time; print(time.perf_counter())")"; } +timer_end() { + python3 -c "import os,time,sys; t0=float(os.environ['TIMER_START']); print(f\"[timer] {sys.argv[1]}: {time.perf_counter()-t0:.3f}s\")" "$1" +} +assert_ok() { + grep -qE 'status:\s*"ok"|status\s*=\s*"ok"' <<<"$1" || { echo "--- bad response ---"; echo "$1"; exit 1; } +} + +export RUNNER_T0="$(python3 -c "import time; print(time.perf_counter())")" + +echo "==> generate JSONL (seed=$SEED)" +timer_start +it_sh python3 "$GEN_PY" --seed "$SEED" --search-rows "$SEARCH_ROWS" --traj-rows "$TRAJ_ROWS" \ + --search-out "$SEARCH_JSONL" --traj-out "$TRAJ_INPUT" +timer_end "generate_benchmark_data.py" + +echo "==> search create ($SEARCH_ROWS rows)" +timer_start +it_run_cli_out search create "$DATASET" --input "$SEARCH_JSONL" --format jsonl --embedding-dim "$EMBED_DIM" +assert_ok "$out"; timer_end "search create" + +echo "==> search index list (pre-build)" +timer_start +it_run_cli_out search index list "$DATASET" +assert_ok "$out"; timer_end "search index list (pre-build)" + +echo "==> search index build" +timer_start +it_run_cli_out search index build "$DATASET" \ + --vector-column embedding --text-column text --metric cosine \ + --num-partitions "$NUM_PARTITIONS" --ivf-max-iters 12 \ + --pq-num-sub-vectors 8 --pq-num-bits 4 --pq-max-iters 12 \ + --pq-sample-rate 4 --pq-kmeans-redos 1 +assert_ok "$out"; timer_end "search index build" + +echo "==> search index list (post-build)" +timer_start +it_run_cli_out search index list "$DATASET" +assert_ok "$out" +grep -q 'persisting_ivf_pq' <<<"$out" || die "missing persisting_ivf_pq" +grep -q 'persisting_fts' <<<"$out" || die "missing persisting_fts" +timer_end "search index list (post-build)" + +echo "==> search query x3" +timer_start +it_run_cli_out search query "$DATASET" "integration alpha gamma" --mode vector --k 5 --embedding-dim "$EMBED_DIM" --nprobes "$NPROBES" +assert_ok "$out" +it_run_cli_out search query "$DATASET" "integration" --mode fts --k 8 --embedding-dim "$EMBED_DIM" +assert_ok "$out" +it_run_cli_out search query "$DATASET" "keyword beta" --mode hybrid --k 5 --embedding-dim "$EMBED_DIM" +assert_ok "$out" +timer_end "search query (3 modes)" + +if [[ "$REORDER" == "1" ]]; then + echo "==> search index reorder" + timer_start + it_run_cli_out search index reorder "$DATASET" persisting_ivf_pq --in-place + assert_ok "$out"; timer_end "search index reorder" +fi + +if [[ "$SKIP_REBUILD" != "1" ]]; then + echo "==> search index rebuild" + timer_start + it_run_cli_out search index rebuild "$DATASET" --no-retrain + assert_ok "$out"; timer_end "search index rebuild" +else + echo "==> skip search index rebuild" +fi + +echo "==> trajectory add ($TRAJ_ROWS rows)" +timer_start +it_run_cli_out trajectory add "$STORAGE" --agent-id "$TRAJ_AGENT" --session-id "$TRAJ_SESSION" --format toml --input "$TRAJ_INPUT" +assert_ok "$out"; timer_end "trajectory add" + +echo "==> trajectory stats" +timer_start +it_run_cli_out trajectory stats "$STORAGE" --agent-id "$TRAJ_AGENT" --session-id "$TRAJ_SESSION" +assert_ok "$out" +grep -Fq "$TRAJ_SESSION" <<<"$out" || die "stats missing session_id" +grep -Fq "$TRAJ_AGENT" <<<"$out" || die "stats missing agent_id" +timer_end "trajectory stats" + +echo "==> trajectory replay" +timer_start +it_run_cli_out trajectory replay "$STORAGE" --agent-id "$TRAJ_AGENT" --session-id "$TRAJ_SESSION" --offset 2 --limit "$REPLAY_LIMIT" +assert_ok "$out" +grep -qE '\[\[records\]\]|^records\s*=' <<<"$out" || die "replay missing TOML records" +timer_end "trajectory replay" + +it_sh python3 -c "import os,time; t0=float(os.environ['RUNNER_T0']); print(f'[timer] TOTAL wall: {time.perf_counter()-t0:.3f}s')" +echo "==> search/traj integration OK" diff --git a/scripts/integration/traj_e2e.sh b/scripts/integration/traj_e2e.sh index 1c91f65..e6632cd 100755 --- a/scripts/integration/traj_e2e.sh +++ b/scripts/integration/traj_e2e.sh @@ -67,7 +67,7 @@ echo "==> traj E2E integration" echo " CLI=$CLI" echo " WORKDIR=$WORKDIR" -# --- 1. import (gateway JSONL → Vortex) ------------------------------------ +# --- 1. import (gateway JSONL → Lance) ------------------------------------ section "traj import (gateway JSONL)" IMPORT_OUT="$(run_cli traj import "$STORAGE" \ @@ -79,9 +79,9 @@ grep -qE 'traj-it-agent|traj-it-session|import' <<<"$IMPORT_OUT" \ || { echo "$IMPORT_OUT"; die "import output unexpected"; } pass "traj import wrote session" -[[ -f "$STORAGE/$AGENT_ID/$SESSION_ID/events.vortex" ]] \ - || die "missing events.vortex after import" -pass "events.vortex exists" +[[ -d "$STORAGE/$AGENT_ID/$SESSION_ID/events.lance" ]] \ + || die "missing events.lance after import" +pass "events.lance exists" # --- 2. stats (pre-judge) --------------------------------------------------- section "traj stats (before judge)" @@ -89,7 +89,7 @@ section "traj stats (before judge)" run_cli traj stats "$STORAGE" \ --agent-id "$AGENT_ID" \ --session-id "$SESSION_ID" \ - --storage-format vortex >"$STATS_TOML" + --storage-format lance >"$STATS_TOML" ROW_COUNT="$(parse_toml_field "$STATS_TOML" row_count)" [[ -n "$ROW_COUNT" && "$ROW_COUNT" -gt 0 ]] || die "expected row_count > 0, got '$ROW_COUNT'" pass "stats row_count=$ROW_COUNT" @@ -116,8 +116,8 @@ grep -qE 'judged_calls\s*=\s*[1-9]|judged_calls: [1-9]' <<<"$JUDGE_OUT" \ || { echo "$JUDGE_OUT"; die "judge did not score any units"; } pass "traj judge wrote sidecar" -[[ -f "$STORAGE/$AGENT_ID/$SESSION_ID/layers/judge_default.vortex" ]] \ - || die "missing judge_default.vortex" +[[ -f "$STORAGE/$AGENT_ID/$SESSION_ID/layers/judge_default.lance" ]] \ + || die "missing judge_default.lance" pass "judge sidecar file exists" # --- 4. stats + judge-stats (post-judge) ------------------------------------ @@ -126,7 +126,7 @@ section "traj stats (after judge)" run_cli traj stats "$STORAGE" \ --agent-id "$AGENT_ID" \ --session-id "$SESSION_ID" \ - --storage-format vortex >"$STATS_TOML" + --storage-format lance >"$STATS_TOML" grep -qE 'judgment_count\s*=\s*[1-9]|judgment_count: [1-9]' "$STATS_TOML" \ || die "stats missing judgment_count after judge" pass "stats shows judgment_count after judge" diff --git a/scripts/test_suite.sh b/scripts/test_suite.sh index a8630aa..c4d6a90 100755 --- a/scripts/test_suite.sh +++ b/scripts/test_suite.sh @@ -1,56 +1,53 @@ #!/usr/bin/env bash -# 统一测试 / 回归入口：交互选单或直接指定套件名。 +# 集成测试统一入口（全部是 scripts/integration/*.sh）。 # -# ./scripts/test_suite.sh # 交互菜单（TTY） -# ./scripts/test_suite.sh list # 列出套件 -# ./scripts/test_suite.sh gate # 直接运行 -# ./scripts/test_suite.sh all-regression +# ./scripts/test_suite.sh # 列出套件 +# ./scripts/test_suite.sh list +# ./scripts/test_suite.sh smoke +# ./scripts/test_suite.sh capture-all +# SKIP_BUILD=1 ./scripts/test_suite.sh traj-e2e +# +# just 薄封装：just smoke / just traj-e2e / just capture-all / … set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$REPO_ROOT" +INT="$REPO_ROOT/scripts/integration" PROFILE="${PERSISTING_BUILD_PROFILE:-debug}" -SKIP_REBUILD="${SKIP_REBUILD:-0}" +export PERSISTING_BUILD_PROFILE="$PROFILE" +[[ "${SKIP_REBUILD:-0}" == "1" ]] && export SKIP_BUILD=1 die() { echo "error: $*" >&2; exit 1; } -run_just() { +run_sh() { + local script="$1" + shift || true echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "▶ just $*" + echo "▶ $script${*:+ $*}" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - just "$@" + bash "$INT/$script" "$@" } -# 套件顺序（bash 3.2 兼容，不用关联数组） SUITE_NAMES=( - dev gate ci - rust rust-engine capture-rust search-integration - search-traj-smoke search-traj-full traj-e2e - capture-integration capture-stress capture-run-e2e capture-all - capture-check test-py all-regression + smoke integration + traj-e2e + capture-integration capture-stress capture-run-e2e + capture-all + all-integration ) SUITE_DESCS=( - "日常：fmt + clippy + 引擎单测 + search/traj 冒烟" - "提交前：fmt + clippy + 全 workspace Rust 测试" - "CI 近似：gate + build + capture 集成 + Claude 回归" - "全 workspace cargo test" - "persisting-engine 库单测" - "capture crate + fixtures + Claude 回归" - "engine search_integration 测试" - "search + traj CLI 冒烟（quick=1）" + "search + traj CLI 冒烟（QUICK=1）" "search + traj CLI 全量基准" "traj import → stats → judge → stats" "proxy / import / daemon 集成" "capture 写入压测" - "traj capture -f vortex 全链路" - "全部 capture shell 集成（含 traj-e2e）" - "capture Rust 测试 + capture-all" - "Python pytest" - "Rust capture 测试 + 全部 shell 回归（跳过 fmt/clippy）" + "traj capture -f lance 全链路" + "全部 capture 集成脚本" + "全部集成脚本（smoke + capture-all）" ) suite_desc() { @@ -65,43 +62,48 @@ suite_desc() { } run_suite() { - local name="$1" - if [[ "$SKIP_REBUILD" == "1" ]]; then - export SKIP_BUILD=1 - fi - case "$name" in - dev) run_just dev ;; - gate) run_just gate ;; - ci) run_just ci ;; - rust) run_just test-rust ;; - rust-engine) run_just _test-engine ;; - capture-rust) run_just capture-test ;; - search-integration) run_just test-engine-integration ;; - search-traj-smoke) run_just smoke "$PROFILE" ;; - search-traj-full) run_just integration "$PROFILE" ;; - traj-e2e) run_just traj-e2e "$PROFILE" ;; - capture-integration) run_just capture-integration "$PROFILE" ;; - capture-stress) run_just capture-stress "$PROFILE" ;; - capture-run-e2e) run_just capture-run-e2e "$PROFILE" ;; - capture-all) run_just capture-all "$PROFILE" ;; - capture-check) run_just capture-check "$PROFILE" ;; - test-py) run_just test-py ;; - all-regression) - run_just capture-test - run_just capture-all "$PROFILE" - run_just smoke "$PROFILE" + case "$1" in + smoke) + QUICK=1 run_sh search_traj_e2e.sh + ;; + integration|search-traj-full|search-traj-e2e) + run_sh search_traj_e2e.sh + ;; + traj-e2e) + run_sh traj_e2e.sh + ;; + capture-integration) + run_sh capture_integration.sh + ;; + capture-stress) + run_sh capture_stress.sh + ;; + capture-run-e2e) + run_sh capture_run_e2e.sh + ;; + capture-all) + run_sh traj_e2e.sh + run_sh capture_integration.sh + run_sh capture_stress.sh + run_sh capture_run_e2e.sh + ;; + all-integration|all-regression) + QUICK=1 run_sh search_traj_e2e.sh + run_sh traj_e2e.sh + run_sh capture_integration.sh + run_sh capture_stress.sh + run_sh capture_run_e2e.sh ;; *) - die "unknown suite: $name (try: $0 list)" + die "unknown suite: $1 (try: $0 list)" ;; esac } print_list() { - echo "Persisting 测试 / 回归套件（just test-suite ）：" + echo "Persisting 集成测试（shell → scripts/integration/）：" echo "" - local i=1 - local n desc + local i=1 n desc for n in "${SUITE_NAMES[@]}"; do desc="$(suite_desc "$n")" printf " %2d. %-22s %s\n" "$i" "$n" "$desc" @@ -109,53 +111,16 @@ print_list() { done echo "" echo "示例：" - echo " just test-suite # 交互选单" - echo " just test-suite gate" - echo " just test-suite traj-e2e" - echo " SKIP_REBUILD=1 just test-suite traj-e2e # 跳过 rebuild" - echo " just test-suite all-regression" -} - -interactive_menu() { - echo "Persisting — 测试 / 回归选单" - echo "profile=$PROFILE SKIP_REBUILD=$SKIP_REBUILD" - echo "" - local i=1 - local labels=() - local n desc - for n in "${SUITE_NAMES[@]}"; do - desc="$(suite_desc "$n")" - labels+=("$i) $n — $desc") - i=$((i + 1)) - done - labels+=("0) 退出") - - PS3=$'\n请选择 > ' - select _ in "${labels[@]}"; do - if [[ "${REPLY:-}" == "0" ]]; then - echo "已取消" - exit 0 - fi - if [[ -n "${REPLY:-}" && "$REPLY" -ge 1 && "$REPLY" -le ${#SUITE_NAMES[@]} ]]; then - run_suite "${SUITE_NAMES[$((REPLY - 1))]}" - exit $? - fi - echo "无效选项，请输入 0–${#SUITE_NAMES[@]}" - done + echo " ./scripts/test_suite.sh list" + echo " ./scripts/test_suite.sh smoke" + echo " SKIP_BUILD=1 ./scripts/test_suite.sh traj-e2e" + echo " just smoke / just traj-e2e / just capture-all # 等价薄封装" } main() { - local cmd="${1:-}" + local cmd="${1:-list}" case "$cmd" in - ""|menu) - if [[ -t 0 && -t 1 ]]; then - interactive_menu - else - print_list - die "非交互环境请指定套件：just test-suite " - fi - ;; - list|help|-h|--help) + list|help|-h|--help|"") print_list ;; *)