Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions src/everos/infra/persistence/lancedb/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import contextlib
import datetime as dt

import anyio

from everos.core.observability.logging import get_logger
from everos.core.persistence import MemoryRoot

Expand Down Expand Up @@ -98,9 +100,11 @@ async def migrate_fts_indexes() -> None:
O(N) but only on the first startup after upgrade.
"""
logger = get_logger(__name__)
marker = MemoryRoot.default().lancedb_dir / ".fts_index_version"
marker = anyio.Path(MemoryRoot.default().lancedb_dir / ".fts_index_version")
try:
current = int(marker.read_text().strip()) if marker.exists() else 0
current = (
int((await marker.read_text()).strip()) if await marker.exists() else 0
)
except (ValueError, OSError):
current = 0
if current >= _FTS_INDEX_SCHEMA_VERSION:
Expand All @@ -120,7 +124,7 @@ async def migrate_fts_indexes() -> None:
# so compaction no longer decodes a position List.
with contextlib.suppress(Exception):
await table.optimize(cleanup_older_than=dt.timedelta(seconds=0))
marker.write_text(str(_FTS_INDEX_SCHEMA_VERSION))
await marker.write_text(str(_FTS_INDEX_SCHEMA_VERSION))
logger.info("fts_index_migration_done", version=_FTS_INDEX_SCHEMA_VERSION)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
stem=True
remove_stop_words=True
ascii_folding=True
with_position=False
language="English" (tantivy default)

The app-layer ``JiebaTokenizer`` already handles segmentation +
Expand All @@ -22,6 +23,7 @@
app-layer JiebaTokenizer is the single source of truth for
stop-word filtering (English + Chinese)
- ascii_folding=True → diacritics on Latin chars normalised (café → cafe)
- with_position=False → omit unused phrase-query positions so optimize can compact
- CJK pass-through → no stemming applied to CJK

Tests build a fresh in-memory-ish LanceDB store under ``tmp_path``,
Expand All @@ -33,11 +35,12 @@

from collections.abc import AsyncIterator
from pathlib import Path
from typing import ClassVar
from typing import ClassVar, cast

import lancedb
import pytest
from lancedb import AsyncTable
from lancedb.index import FTS

from everos.core.persistence.lancedb import BaseLanceTable

Expand All @@ -52,6 +55,25 @@ class _FtsSpec(BaseLanceTable):
body: str


class _FtsConfigSpy:
"""Capture index configuration passed through the public schema seam."""

def __init__(self) -> None:
self.created: list[tuple[str, FTS, bool]] = []

async def list_indices(self) -> list[object]:
return []

async def create_index(
self,
column: str,
*,
config: FTS,
replace: bool = False,
) -> None:
self.created.append((column, config, replace))


@pytest.fixture
async def fts_table(tmp_path: Path) -> AsyncIterator[AsyncTable]:
"""Build a fresh tmp LanceDB store + ``_FtsSpec`` table; index gets
Expand All @@ -75,6 +97,19 @@ async def _query_ids(table: AsyncTable, text: str) -> set[str]:
return {r["id"] for r in rows}


async def test_fts_index_disables_positions() -> None:
"""FTS creation omits positions because EverOS does not run phrase queries."""
table = _FtsConfigSpy()

await _FtsSpec.ensure_fts_indexes(cast(AsyncTable, table))

assert len(table.created) == 1
column, config, replace = table.created[0]
assert column == "body"
assert config.with_position is False
assert replace is False


# ── lower_case=True ────────────────────────────────────────────────────


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from pathlib import Path
from typing import ClassVar

import anyio
import pytest

from everos.config import LanceDBSettings
Expand Down Expand Up @@ -687,12 +688,12 @@ async def test_migrate_fts_indexes_runs_once_and_rebuilds(
await _SearchNote.ensure_fts_indexes(table)
assert any("tokens" in (i.columns or []) for i in await table.list_indices())

marker = MemoryRoot.default().lancedb_dir / ".fts_index_version"
assert not marker.exists()
marker = anyio.Path(MemoryRoot.default().lancedb_dir / ".fts_index_version")
assert not await marker.exists()

# First run: migrates + writes the marker, index still present.
await migrate_fts_indexes()
assert marker.read_text().strip() == "2"
assert (await marker.read_text()).strip() == "2"
assert any("tokens" in (i.columns or []) for i in await table.list_indices())

# Marker present → second run is a no-op. Drop the index, re-run,
Expand Down