Initial implementation of RagLite SQLite toolkit

.github/workflows/ci.yml

@@ -0,0 +1,36 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ['3.10', '3.11', '3.12']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[dev]
+      - name: Lint
+        run: |
+          ruff check .
+          black --check .
+      - name: Type check
+        run: mypy raglite_sqlite
+      - name: Test
+        run: pytest --cov=raglite_sqlite

.gitignore

@@ -0,0 +1,17 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+.env
+coverage.xml
+htmlcov/
+.pytest_cache/
+.mypy_cache/
+.cache/
+.DS_Store
+*.db
+*.sqlite
+*.log
+/.ruff_cache
+/.pytest_cache
+/.coverage

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 24.4.2
+    hooks:
+      - id: black
+        args: ["--line-length", "100"]
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.8
+    hooks:
+      - id: ruff
+        args: ["--fix", "--exit-non-zero-on-fix"]
+      - id: ruff-format
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        args: ["--profile", "black", "--line-length", "100"]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 RagLite Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,2 +1,86 @@
-# raglite-sqlite
-Zero-server, zero-docker RAG with SQLite
+# RagLite SQLite
+
+Local-first Retrieval-Augmented Generation (RAG) toolkit built entirely on top of a single SQLite file. RagLite makes it easy to ingest documents, chunk them, embed with pluggable backends, and perform hybrid lexical/vector search—without running servers or Docker images.
+
+## Why RagLite?
+
+- **Zero infrastructure** – everything lives inside one SQLite database (`knowledge.db`).
+- **Deterministic and offline** – default embedding model is local; no network calls unless explicitly configured.
+- **Hybrid retrieval** – combines BM25 via FTS5 with cosine similarity over stored vectors.
+- **Python and CLI** – flexible API plus a friendly Typer-based CLI for scripting.
+- **Extensible** – pluggable parsers, chunkers, embedding backends, and adapters for LangChain / LlamaIndex.
+
+## Installation
+
+```bash
+python -m venv .venv && source .venv/bin/activate  # (Windows: .venv\Scripts\activate)
+pip install -e .
+```
+
+## Quickstart
+
+```bash
+python -m venv .venv && source .venv/bin/activate  # (Windows: .venv\Scripts\activate)
+pip install -e .
+raglite init --db knowledge.db
+raglite index tests/data --db knowledge.db --model sentence-transformers/all-MiniLM-L6-v2
+raglite query "example text" --db knowledge.db -k 8 --hybrid 0.6
+```
+
+## CLI Examples
+
+- Initialize a new database:
+  ```bash
+  raglite init --db knowledge.db
+  ```
+- Index a directory recursively with tags:
+  ```bash
+  raglite index docs --db knowledge.db --tags project,internal --recursive
+  ```
+- Query for relevant chunks:
+  ```bash
+  raglite query "How do I deploy?" --db knowledge.db -k 5
+  ```
+- Inspect statistics:
+  ```bash
+  raglite stats --db knowledge.db
+  ```
+- Export metadata:
+  ```bash
+  raglite export --db knowledge.db --to export.ndjson --include-vectors
+  ```
+
+## Python API
+
+```python
+from raglite_sqlite import RagLite
+from raglite_sqlite.embeddings.sentence_transformers_backend import SentenceTransformersBackend
+
+rag = RagLite("knowledge.db")
+backend = SentenceTransformersBackend()
+rag.index(["tests/data"], tags="demo", embedding_backend=backend)
+results = rag.search("example text", embedding_backend=backend)
+for item in results:
+    print(item["score"], item["snippet"])
+```
+
+## Design Notes
+
+- Uses SQLite with WAL mode and FTS5 for zero-config deployment.
+- Chunker strategies allow fixed token or recursive splitting with overlaps.
+- Embeddings cached via SHA-256 to avoid redundant model calls.
+- Hybrid retrieval fuses normalized BM25 and cosine similarity scores.
+
+## Security & Privacy
+
+RagLite does not perform any network calls by default. Remote embedding backends (such as OpenAI) are opt-in and require explicit configuration via environment variables.
+
+## Roadmap
+
+- Optional REST server for multi-user access.
+- Support for additional document formats and OCR.
+- Pluggable reranking models.
+
+## License
+
+MIT License. See [LICENSE](LICENSE).

pyproject.toml

@@ -0,0 +1,67 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "raglite-sqlite"
+version = "0.1.0"
+description = "Local-first RAG toolkit backed by a single SQLite database"
+readme = "README.md"
+authors = [{ name = "RagLite Contributors" }]
+license = { file = "LICENSE" }
+requires-python = ">=3.10"
+dependencies = [
+  "typer[all]>=0.9.0",
+  "rich>=13.7.0",
+  "pydantic>=2.7.0",
+  "beautifulsoup4>=4.12.0",
+  "python-frontmatter>=1.0.0",
+  "PyPDF2>=3.0.0",
+  "python-docx>=1.0.0",
+  "numpy>=1.24.0",
+  "tqdm>=4.66.0",
+  "rapidfuzz>=3.6.0",
+  "sentence-transformers>=2.5.0",
+  "orjson>=3.9.0",
+  "click-spinner>=0.1.10",
+  "sqlite-fts5",
+]
+
+[project.optional-dependencies]
+openai = ["openai>=1.0.0"]
+dev = [
+  "pytest>=8.1.0",
+  "pytest-cov>=4.1.0",
+  "mypy>=1.10.0",
+  "ruff>=0.4.0",
+  "black>=24.4.0",
+  "build>=1.2.1",
+  "twine>=5.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/example/raglite-sqlite"
+Repository = "https://github.com/example/raglite-sqlite"
+
+[project.scripts]
+raglite = "raglite_sqlite.cli:app"
+
+[tool.black]
+line-length = 100
+target-version = ["py310"]
+
+[tool.ruff]
+line-length = 100
+select = ["E", "F", "I", "B"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["raglite_sqlite"]
+
+[tool.mypy]
+pytorch = false
+python_version = "3.10"
+warn_unused_ignores = true
+warn_redundant_casts = true
+warn_unused_configs = true
+strict = true
+plugins = []

raglite_sqlite/__init__.py

@@ -0,0 +1,5 @@
+"""RagLite SQLite package."""
+
+from .api import RagLite
+
+__all__ = ["RagLite"]

raglite_sqlite/adapters/__init__.py

@@ -0,0 +1,5 @@
+"""Adapters for third-party ecosystems."""
+
+from .langchain import RagLiteRetriever
+
+__all__ = ["RagLiteRetriever"]

raglite_sqlite/adapters/langchain.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List, Optional
+
+try:
+    from langchain.schema import Document
+except Exception:  # pragma: no cover - optional dependency
+    Document = None  # type: ignore[assignment]
+
+from ..api import RagLite
+from ..embeddings.base import EmbeddingBackend
+from ..typing import SearchResult
+
+
+class RagLiteRetriever:
+    """Minimal LangChain retriever wrapper."""
+
+    def __init__(
+        self,
+        rag: RagLite,
+        *,
+        backend: Optional[EmbeddingBackend] = None,
+        k: int = 4,
+    ) -> None:
+        self.rag = rag
+        self.backend = backend
+        self.k = k
+
+    def get_relevant_documents(self, query: str, **kwargs: Any) -> List[Any]:
+        results = self.rag.search(query, k=self.k, embedding_backend=self.backend)
+        if Document is None:
+            raise RuntimeError("LangChain is not installed")
+        docs: List[Any] = []
+        for item in results:
+            metadata: Dict[str, Any] = {
+                "doc_id": item.get("doc_id"),
+                "section": item.get("section"),
+                "source_path": item.get("source_path"),
+                "tags": item.get("tags"),
+            }
+            docs.append(Document(page_content=item.get("text", ""), metadata=metadata))
+        return docs
+
+    async def aget_relevant_documents(self, query: str, **kwargs: Any) -> List[Any]:
+        return self.get_relevant_documents(query, **kwargs)

raglite_sqlite/adapters/llamaindex.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from ..api import RagLite
+from ..embeddings.base import EmbeddingBackend
+
+
+class RagLiteNodeRetriever:
+    """Minimal adapter compatible with LlamaIndex retriever interface."""
+
+    def __init__(self, rag: RagLite, *, backend: Optional[EmbeddingBackend] = None, k: int = 4) -> None:
+        self.rag = rag
+        self.backend = backend
+        self.k = k
+
+    def retrieve(self, query: str, **kwargs: Any) -> List[Dict[str, Any]]:
+        results = self.rag.search(query, k=self.k, embedding_backend=self.backend)
+        nodes: List[Dict[str, Any]] = []
+        for item in results:
+            nodes.append(
+                {
+                    "text": item.get("text", ""),
+                    "id": item.get("chunk_id"),
+                    "metadata": {
+                        "doc_id": item.get("doc_id"),
+                        "section": item.get("section"),
+                        "source_path": item.get("source_path"),
+                        "tags": item.get("tags"),
+                    },
+                }
+            )
+        return nodes