diff --git a/.github/workflows/build-tests.yml b/.github/workflows/build-tests.yml index 5c7e53f..0a05933 100644 --- a/.github/workflows/build-tests.yml +++ b/.github/workflows/build-tests.yml @@ -2,14 +2,15 @@ name: Build Tests on: pull_request: - branches: [dev, master, main] + branches: [dev, master] workflow_dispatch: jobs: build: uses: OpenVoiceOS/gh-automations/.github/workflows/build-tests.yml@dev - secrets: inherit with: python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' install_extras: '' + pre_install_pip: '"hivemind-plugin-manager==0.6.0a1"' test_path: 'tests' + pr_comment: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} diff --git a/.github/workflows/cipher-tests.yml b/.github/workflows/cipher-tests.yml new file mode 100644 index 0000000..5fa6aa3 --- /dev/null +++ b/.github/workflows/cipher-tests.yml @@ -0,0 +1,29 @@ +name: Cipher Tests + +# Dedicated workflow for the [cipher] extra (sqlcipher3 + libsqlcipher-dev). +# The shared build-tests.yml covers the unencrypted path on Python 3.10-3.14; +# this one adds a single Python 3.11 run with sqlcipher installed so the +# encrypted backend is exercised on every PR. + +on: + pull_request: + branches: [dev, master, main] + workflow_dispatch: + +jobs: + test-cipher: + name: "SQLite + SQLCipher" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install sqlcipher system library + run: | + sudo apt-get update + sudo apt-get install -y libsqlcipher-dev + - run: pip install -e ".[cipher]" + # schema-v2 features (migrate, refresh, _check_forward_compat) ship in the 0.6.0a1 prerelease + - run: pip install "hivemind-plugin-manager==0.6.0a1" pytest + - run: pytest tests/test_sqlitedb.py -q -p no:ovoscope diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 1b0b9e3..981f475 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -8,10 +8,11 @@ on: jobs: coverage: uses: OpenVoiceOS/gh-automations/.github/workflows/coverage.yml@dev - secrets: inherit with: python_version: '3.11' coverage_source: 'hivemind_sqlite_database' + pre_install_pip: '"hivemind-plugin-manager==0.6.0a1"' test_path: 'tests/' install_extras: '' min_coverage: 0 + pr_comment: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml index 8757eee..cb7cfb5 100644 --- a/.github/workflows/license_check.yml +++ b/.github/workflows/license_check.yml @@ -8,4 +8,5 @@ on: jobs: license_check: uses: OpenVoiceOS/gh-automations/.github/workflows/license-check.yml@dev - secrets: inherit + with: + pr_comment: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9a6b7a5..926e1ea 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,13 +2,13 @@ name: Lint on: pull_request: - branches: [dev, master, main] + branches: [dev, master] workflow_dispatch: jobs: lint: uses: OpenVoiceOS/gh-automations/.github/workflows/lint.yml@dev - secrets: inherit with: ruff: true pre_commit: false # set true if .pre-commit-config.yaml exists + pr_comment: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} diff --git a/.github/workflows/pip_audit.yml b/.github/workflows/pip_audit.yml index bb3ca4d..d44ece3 100644 --- a/.github/workflows/pip_audit.yml +++ b/.github/workflows/pip_audit.yml @@ -8,4 +8,5 @@ on: jobs: pip_audit: uses: OpenVoiceOS/gh-automations/.github/workflows/pip-audit.yml@dev - secrets: inherit + with: + pr_comment: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} diff --git a/.github/workflows/policy-e2e.yml b/.github/workflows/policy-e2e.yml new file mode 100644 index 0000000..def213d --- /dev/null +++ b/.github/workflows/policy-e2e.yml @@ -0,0 +1,24 @@ +name: Policy Migration E2E + +on: + pull_request: + branches: [dev, master] + workflow_dispatch: + +jobs: + policy_e2e: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - run: pip install -e ".[test]" + # Cross-repo policy stack (published prereleases) + hivescope with db= injection + - run: >- + pip install + "hivescope==0.4.0a2" + "hivemind-core==4.3.0a2" + "hivemind-ovos-agent-plugin==0.2.0a1" + "hivemind-plugin-manager==0.6.0a1" + - run: pytest tests/e2e/test_policy_migration_e2e.py -q -p no:cacheprovider diff --git a/.github/workflows/publish_stable.yml b/.github/workflows/publish_stable.yml index 9ab5b7a..a452be6 100644 --- a/.github/workflows/publish_stable.yml +++ b/.github/workflows/publish_stable.yml @@ -7,7 +7,8 @@ on: jobs: publish_stable: uses: OpenVoiceOS/gh-automations/.github/workflows/publish-stable.yml@dev - secrets: inherit + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} with: branch: 'master' version_file: 'hivemind_sqlite_database/version.py' diff --git a/.github/workflows/release-preview.yml b/.github/workflows/release-preview.yml index 3404ca7..527970a 100644 --- a/.github/workflows/release-preview.yml +++ b/.github/workflows/release-preview.yml @@ -7,8 +7,8 @@ on: jobs: release_preview: + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} uses: OpenVoiceOS/gh-automations/.github/workflows/release-preview.yml@dev - secrets: inherit with: package_name: 'hivemind_sqlite_database' version_file: 'hivemind_sqlite_database/version.py' diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml index 93dfa8c..b1861c3 100644 --- a/.github/workflows/release_workflow.yml +++ b/.github/workflows/release_workflow.yml @@ -4,11 +4,13 @@ on: pull_request: types: [closed] branches: [dev] + workflow_dispatch: jobs: publish_alpha: uses: OpenVoiceOS/gh-automations/.github/workflows/publish-alpha.yml@dev - secrets: inherit + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} with: branch: 'dev' base_branch: 'master' diff --git a/.github/workflows/repo-health.yml b/.github/workflows/repo-health.yml index 96f690d..2a47bce 100644 --- a/.github/workflows/repo-health.yml +++ b/.github/workflows/repo-health.yml @@ -2,12 +2,12 @@ name: Repo Health on: pull_request: - branches: [dev, master, main] + branches: [dev, master] workflow_dispatch: jobs: repo_health: + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} uses: OpenVoiceOS/gh-automations/.github/workflows/repo-health.yml@dev - secrets: inherit with: version_file: 'hivemind_sqlite_database/version.py' diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..e79342f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,51 @@ +# AGENTS.md + +SQLite (and optional SQLCipher-encrypted) `AbstractDB` backend plugin for hivemind-core, storing HiveMind client records (API keys, crypto keys, ACLs). + +## Setup + +```bash +pip install -e . +# encrypted-database support (SQLCipher / AES-256): +pip install -e ".[cipher]" # needs libsqlcipher system lib (apt install libsqlcipher-dev) +``` + +## Test + +```bash +pytest tests/ +``` + +Single suite lives in `tests/test_sqlitedb.py`. The repo's own `tests.yml` runs it with `-p no:ovoscope` and exercises both the plain and the SQLCipher path. Cipher tests need `libsqlcipher-dev` installed. + +## Lint/Typecheck + +Ruff, via the shared `lint.yml` workflow (`ruff: true`, no pre-commit config present). No typecheck configured. + +## Layout + +- `hivemind_sqlite_database/__init__.py` — the entire plugin: `SQLiteDB` dataclass extending `AbstractDB`. Schema creation, `PRAGMA user_version` migration (v1→v2 folds legacy `intent_blacklist`/`skill_blacklist` columns into `Client.metadata`), and CRUD (`add_item`, `search_by_value`, `get_client_by_id`, `__iter__`, `__len__`, `commit`). +- `hivemind_sqlite_database/version.py` — version constants (do not edit). +- `tests/test_sqlitedb.py` — full behavioural suite. + +Entry-point group: `hivemind.database` → `hivemind-sqlite-db-plugin = hivemind_sqlite_database:SQLiteDB`. Discovered by `hivemind-plugin-manager`; selected in hivemind-core config under `database.module`. + +## Conventions + +- Branches: `dev` (work) and `master` (stable). NEVER `main`. +- Never edit `version.py`; gh-automations bumps semver from conventional-commit prefixes (`feat:`/`fix:`/`feat!:`). +- New repos private by default. +- Commit identity: JarbasAi . +- Reference `OpenVoiceOS/gh-automations` reusable workflows at `@dev`. +- No Neon / `neon-*` references. +- No meta-commentary (no history, no dates) in code, docs, commits, or PRs — describe current state only. +- CI is provided by OpenVoiceOS/gh-automations. + +## Gotchas + +- `_VALID_COLUMNS` allowlist gates `search_by_value` keys; SQL column name is interpolated only after passing that frozenset (issue #1 tracks an SQL-security review). +- Legacy `intent_blacklist`/`skill_blacklist`/`message_blacklist` columns remain in the table but are NULLed on write; canonical data lives in `metadata` JSON. `message_blacklist` is purged entirely (not part of `Client`). +- `migrate()` guards on `getattr(AbstractDB, "SCHEMA_VERSION", 1)` so it tolerates older hivemind-plugin-manager that predates the constant — do not assume the attribute exists. +- WAL journal mode is enabled for both plain and encrypted DBs; `check_same_thread=False` with a `threading.Lock` guarding writes. +- Encrypted and plaintext databases are not interchangeable and there is no migration between them; lost passphrase = unrecoverable data. +- `tests.yml` is a repo-local workflow separate from the shared build-tests; both run the same suite. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d004c0..94bef99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,57 +1,56 @@ # Changelog -## [0.2.1a4](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.2.1a4) (2026-04-15) +## [0.4.0a2](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.4.0a2) (2026-06-05) -[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.2.1a3...0.2.1a4) +[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.4.0a1...0.4.0a2) **Merged pull requests:** -- ci: sync publish workflows from shared templates [\#22](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/22) ([JarbasAl](https://github.com/JarbasAl)) +- docs: zero-to-hero README and /docs coverage [\#38](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/38) ([JarbasAl](https://github.com/JarbasAl)) -## [0.2.1a3](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.2.1a3) (2026-04-15) +## [0.4.0a1](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.4.0a1) (2026-06-05) -[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.2.1a2...0.2.1a3) - -## [0.2.1a2](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.2.1a2) (2026-04-15) - -[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.2.1a1...0.2.1a2) +[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.3.0a4...0.4.0a1) **Merged pull requests:** -- Update actions/setup-python action to v6 [\#15](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/15) ([renovate[bot]](https://github.com/apps/renovate)) -- Update actions/checkout action to v6 [\#9](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/9) ([renovate[bot]](https://github.com/apps/renovate)) +- feat\(db\): schema v2 migration — fold legacy blacklist columns into metadata [\#32](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/32) ([JarbasAl](https://github.com/JarbasAl)) -## [0.2.1a1](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.2.1a1) (2026-04-15) +## [0.3.0a4](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.3.0a4) (2026-06-05) -[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.2.0a1...0.2.1a1) +[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.3.0a3...0.3.0a4) **Merged pull requests:** -- fix: move pytest\_plugins to top-level conftest for pytest 9 compatibility [\#18](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/18) ([JarbasAl](https://github.com/JarbasAl)) +- ci: dedupe tests.yml — drop test-plain, rename to cipher-tests.yml [\#34](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/34) ([JarbasAl](https://github.com/JarbasAl)) -## [0.2.0a1](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.2.0a1) (2026-04-15) +## [0.3.0a3](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.3.0a3) (2026-05-20) -[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.0.4a2...0.2.0a1) +[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.3.0a2...0.3.0a3) **Merged pull requests:** -- feat: release 0.1.0 — tests, thread-safety, SQL injection fix [\#14](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/14) ([JarbasAl](https://github.com/JarbasAl)) +- ci: pass PYPI\_TOKEN explicitly, drop secrets:inherit elsewhere [\#33](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/33) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.3.0a2](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.3.0a2) (2026-05-18) + +[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.3.0a1...0.3.0a2) -## [0.0.4a2](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.0.4a2) (2025-12-19) +**Closed issues:** -[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.0.4a1...0.0.4a2) +- security: encrypted db [\#2](https://github.com/JarbasHiveMind/hivemind-sqlite-database/issues/2) **Merged pull requests:** -- chore\(deps\): update actions/setup-python action to v6 [\#12](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/12) ([renovate[bot]](https://github.com/apps/renovate)) +- Preserve client metadata \(supersedes \#29\) [\#30](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/30) ([JarbasAl](https://github.com/JarbasAl)) -## [0.0.4a1](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.0.4a1) (2025-12-18) +## [0.3.0a1](https://github.com/JarbasHiveMind/hivemind-sqlite-database/tree/0.3.0a1) (2026-04-15) -[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.0.3...0.0.4a1) +[Full Changelog](https://github.com/JarbasHiveMind/hivemind-sqlite-database/compare/0.2.1...0.3.0a1) **Merged pull requests:** -- chore: Configure Renovate [\#7](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/7) ([renovate[bot]](https://github.com/apps/renovate)) +- feat: add SQLCipher encryption support \(password kwarg\) [\#24](https://github.com/JarbasHiveMind/hivemind-sqlite-database/pull/24) ([JarbasAl](https://github.com/JarbasAl)) diff --git a/README.md b/README.md index d8a0498..0520255 100644 --- a/README.md +++ b/README.md @@ -1 +1,122 @@ -# HiveMind SQLite Database \ No newline at end of file +# hivemind-sqlite-database + +SQLite database backend for [hivemind-core](https://github.com/JarbasHiveMind/HiveMind-core). + +Implements the [`hivemind-plugin-manager`](https://github.com/JarbasHiveMind/hivemind-plugin-manager) +`AbstractDB` contract on top of Python's built-in `sqlite3` module (or `sqlcipher3` when +encryption is enabled). Client records (API keys, crypto keys, access-control lists) are stored in +a local `.db` file with WAL journal mode for safe multi-reader, single-writer access. + +**This is the default database backend for fresh hivemind-core installations.** + +## Where it fits + +``` +hivemind-core + └── hivemind-plugin-manager (DatabaseFactory loads plugins by entry-point) + └── hivemind-sqlite-database ← this repo + └── sqlite3 (stdlib) or sqlcipher3 (optional encryption) +``` + +The plugin registers under the `hivemind.database` entry-point group as +`hivemind-sqlite-db-plugin`. `hivemind-core` loads it automatically when `server.json` +sets `database.module` to this name. + +## Install + +```bash +pip install hivemind-sqlite-database +``` + +### With encryption support (SQLCipher / AES-256) + +```bash +# Debian/Ubuntu — system library +sudo apt install libsqlcipher0 + +# Python binding +pip install "hivemind-sqlite-database[cipher]" +``` + +> The `sqlcipher3` wheel on PyPI ships its own `libsqlcipher` for x86_64 Linux, +> so the `apt` step may be optional on that platform. On ARM or Alpine you must +> build from source and need the system library. + +## Quickstart + +Add or update the `"database"` block in `~/.config/hivemind-core/server.json`: + +```json +{ + "database": { + "module": "hivemind-sqlite-db-plugin", + "hivemind-sqlite-db-plugin": { + "name": "clients", + "subfolder": "hivemind-core" + } + } +} +``` + +Then start (or restart) hivemind-core: + +```bash +hivemind-core listen +``` + +The database file is created automatically at +`$XDG_DATA_HOME/hivemind-core/clients.db` (typically `~/.local/share/hivemind-core/clients.db`). + +### Optional encryption (SQLCipher) + +```json +{ + "database": { + "module": "hivemind-sqlite-db-plugin", + "hivemind-sqlite-db-plugin": { + "name": "clients", + "subfolder": "hivemind-core", + "password": "your-strong-passphrase" + } + } +} +``` + +> **Warning**: There is no password recovery. If you lose the passphrase the database +> is permanently unrecoverable. Back up the passphrase securely. + +## Configuration reference + +| Key | Default | Description | +|---|---|---| +| `name` | `"clients"` | Base filename (without extension). The database is `.db`. | +| `subfolder` | `"hivemind-core"` | XDG subfolder under `$XDG_DATA_HOME`. | +| `password` | `null` | When set (non-empty string), opens the database with SQLCipher AES-256 encryption. Requires the `[cipher]` extra. | + +The full path resolves to `$XDG_DATA_HOME//.db`, +typically `~/.local/share/hivemind-core/clients.db`. + +## Schema migration + +On first open after an upgrade, `SQLiteDB` runs an automatic schema migration. +Version tracking uses SQLite's built-in `PRAGMA user_version` — no sibling files. + +The v1→v2 migration folds legacy `intent_blacklist` / `skill_blacklist` column +data into each row's `metadata` JSON field and NULLs the legacy columns. +`message_blacklist` is purged outright. + +The migration is idempotent, crash-safe, and transactional — the row rewrites +and the `user_version` bump happen in one transaction. + +To migrate an existing installation to this backend, use hivemind-core's built-in +command: + +```bash +hivemind-core migrate-db +``` + +## Docs + +- [docs/architecture.md](docs/architecture.md) — internals, WAL mode, schema, migration +- [docs/configuration.md](docs/configuration.md) — full configuration reference +- [docs/operations.md](docs/operations.md) — file locations, backup, encryption, authoring a plugin diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..6821185 --- /dev/null +++ b/TODO.md @@ -0,0 +1,17 @@ +# TODO + +## Open issues + +- [ ] #10 Dependency Dashboard +- [ ] #1 Review SQL security + +## Gaps + +- [ ] Two near-duplicate CI surfaces: shared `build-tests.yml` (gh-automations) plus repo-local `tests.yml` run the same suite — consolidate or document why both exist. +- [ ] `opm-check` / `skill-check` not present — acceptable, this is a `hivemind.database` plugin, not an OVOS/OPM plugin or skill, so those checks do not apply. +- [ ] No typecheck (mypy) configured; only ruff lint. +- [ ] Coverage gate is `min_coverage: 0` — effectively disabled despite a 787-line test file; consider raising once stable. + +## Code TODOs + +None found. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..3cd40cc --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,146 @@ +# Architecture + +## Class hierarchy + +``` +hivemind_plugin_manager.database.AbstractDB (abstract) + │ + └─ hivemind_sqlite_database.SQLiteDB + │ + └─ sqlite3.Connection (stdlib) or sqlcipher3.Connection (when password is set) +``` + +`SQLiteDB` is a thin adapter: it maps `AbstractDB`'s CRUD contract to a +single `clients` table in a local SQLite file. + +## On-disk layout + +One file per database under the XDG path (`$XDG_DATA_HOME//`): + +| File | Purpose | +|---|---| +| `.db` | The SQLite database. Contains the `clients` table. | +| `.db-wal` | Write-Ahead Log (WAL) journal. Present while the database is open or has unflushed writes. | +| `.db-shm` | WAL shared memory index. Present alongside the WAL file. | + +WAL journal mode is set on every open (`PRAGMA journal_mode=WAL`). This +allows multiple readers to proceed concurrently with a single writer without +blocking each other. + +## Table schema + +```sql +CREATE TABLE IF NOT EXISTS clients ( + client_id INTEGER PRIMARY KEY, + api_key VARCHAR(255) NOT NULL, + name VARCHAR(255), + description VARCHAR(255), + is_admin BOOLEAN DEFAULT FALSE, + last_seen REAL DEFAULT -1, + intent_blacklist TEXT, -- legacy column, NULLed after v2 migration + skill_blacklist TEXT, -- legacy column, NULLed after v2 migration + message_blacklist TEXT, -- legacy column, NULLed after v2 migration + allowed_types TEXT, + crypto_key VARCHAR(16), + password TEXT, + can_broadcast BOOLEAN DEFAULT TRUE, + can_escalate BOOLEAN DEFAULT TRUE, + can_propagate BOOLEAN DEFAULT TRUE, + metadata TEXT +); +``` + +The legacy `intent_blacklist`, `skill_blacklist`, and `message_blacklist` +columns remain in the schema for backward compatibility with older readers +but are NULLed on every write after the v2 schema migration. The canonical +data for blacklists lives in the JSON `metadata` column. + +`allowed_types` and `metadata` are stored as JSON strings and +deserialised with `json.loads` on read. + +## Schema migration + +`SQLiteDB` tracks its on-disk schema version in `PRAGMA user_version` — a +signed integer slot built into every SQLite file, reserved exactly for +application-level versioning. No sibling files. + +On every `__post_init__`, `_maybe_migrate()` reads `user_version` and +compares it to `AbstractDB.SCHEMA_VERSION` (defaults to `1` if the +attribute is absent on older HPM). If the stored version is lower, it calls +`_migrate_locked()` and bumps `user_version` in the **same transaction**, +so a crash never leaves the DB at "migrated rows but stale sentinel" or vice +versa. + +### v1 → v2 + +For each row: + +- `intent_blacklist` and `skill_blacklist` column values are folded into + the row's `metadata` JSON dict via `setdefault` — explicit `metadata` + values are never clobbered. +- `message_blacklist` is purged outright (top-level column and any + pre-existing `metadata["message_blacklist"]`). +- All three legacy columns are NULLed. + +The operation is idempotent: if a row has NULL legacy columns or +`metadata` already contains the canonical values, the row is unchanged. + +## Thread safety + +`SQLiteDB` connects with `check_same_thread=False` and protects writes +with a `threading.Lock` (`_write_lock`). All `INSERT`, `UPDATE`, and +schema mutations acquire the lock before entering the `with self.conn` +transaction context. Reads (`SELECT`, `PRAGMA table_info`) do not acquire +the lock — SQLite's WAL mode allows concurrent readers. + +For multi-process concurrency (e.g. two `hivemind-core` processes on the +same file), WAL mode provides safe concurrent reads and SQLite's built-in +writer-lock serialises writes. However, `hivemind-core` does not design for +multi-process writes to the same DB — use Redis if you need that. + +## Encryption (SQLCipher) + +When `password` is a non-empty string, `SQLiteDB` opens the file via +`sqlcipher3` instead of the standard `sqlite3` module and issues +`PRAGMA key=''` immediately after opening the connection. + +The encryption is transparent to all methods — `add_item`, `search_by_value`, +and `__iter__` work identically whether the file is encrypted or not. + +An encrypted file is **not** a standard SQLite file. You cannot open it with +the `sqlite3` CLI, DB Browser for SQLite, or any other tool without the key. +Conversely, a plain SQLite file cannot be opened by SQLCipher with a key. +There is no automatic migration between the two modes. + +## Authoring a plugin with the same contract + +To write a different database backend, implement `AbstractDB` from +`hivemind_plugin_manager.database`: + +```python +from dataclasses import dataclass +from hivemind_plugin_manager.database import AbstractDB, Client +from typing import List, Union, Iterable + +@dataclass +class MyDB(AbstractDB): + name: str = "clients" + subfolder: str = "hivemind-core" + + def add_item(self, client: Client) -> bool: ... + def search_by_value(self, key: str, val: Union[str, bool, int, float]) -> List[Client]: ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterable[Client]: ... + def commit(self) -> bool: ... + def migrate(self, from_version: int) -> None: ... +``` + +Register it under the `hivemind.database` entry-point group in +`pyproject.toml`: + +```toml +[project.entry-points."hivemind.database"] +"my-db-plugin" = "my_package:MyDB" +``` + +`DatabaseFactory.create("my-db-plugin")` then discovers and instantiates it. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..5fadef5 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,101 @@ +# Configuration + +`SQLiteDB` accepts three constructor parameters. Defaults match the layout +`hivemind-core` expects out-of-the-box. + +| Parameter | Default | Description | +|---|---|---| +| `name` | `"clients"` | Base filename (without extension). The database file is `.db`. | +| `subfolder` | `"hivemind-core"` | XDG subfolder under `$XDG_DATA_HOME`. | +| `password` | `None` | When set to a non-empty string, enables AES-256 encryption via SQLCipher. Requires `pip install "hivemind-sqlite-database[cipher]"`. | + +All three are passed via the `hivemind-sqlite-db-plugin` block in +`~/.config/hivemind-core/server.json`: + +```json +{ + "database": { + "module": "hivemind-sqlite-db-plugin", + "hivemind-sqlite-db-plugin": { + "name": "clients", + "subfolder": "hivemind-core", + "password": null + } + } +} +``` + +## Paths + +`SQLiteDB` uses `ovos_utils.xdg_utils.xdg_data_home()` for the data root: + +- If `$XDG_DATA_HOME` is set, use it. +- Otherwise, default to `~/.local/share`. + +With the defaults the full path resolves to: + +``` +~/.local/share/hivemind-core/clients.db +``` + +The directory is created on first open — no manual `mkdir` needed. + +### Relocating the database + +Override `$XDG_DATA_HOME` per-process: + +```bash +XDG_DATA_HOME=/srv/hivemind hivemind-core listen +# Resolves to /srv/hivemind/hivemind-core/clients.db +``` + +Or use a symlink inside `~/.local/share/hivemind-core/` pointing to the real +file. + +## Encryption + +Setting a non-empty `password` selects SQLCipher. The system library +and Python binding must be installed first: + +```bash +# Debian/Ubuntu +sudo apt install libsqlcipher0 +pip install "hivemind-sqlite-database[cipher]" +``` + +```json +{ + "database": { + "module": "hivemind-sqlite-db-plugin", + "hivemind-sqlite-db-plugin": { + "name": "clients", + "password": "your-strong-passphrase" + } + } +} +``` + +The password maps directly to SQLCipher's `PRAGMA key`. SQLCipher derives +an AES-256-CBC key from the passphrase via PBKDF2. Any non-empty string is +valid as a passphrase — SQLCipher stretches it internally. + +**Constraints:** + +- A plain (unencrypted) `.db` file cannot be opened with a key. +- An encrypted file cannot be opened without the key. +- There is no built-in re-key or passphrase-change flow. +- There is no password recovery. A lost passphrase means the data is gone. + +## Multiple HiveMind instances on the same host + +Give each instance distinct `name` or `subfolder` values. The plugin uses +WAL journal mode, which is safe for multi-reader single-writer access from +different processes, but `hivemind-core` is not designed for two writers on +the same file simultaneously. + +## Schema version + +`SQLiteDB` tracks the schema version in `PRAGMA user_version` — a built-in +SQLite slot, no sibling files. The current target version is `2`. See +[Architecture → Schema migration](architecture.md#schema-migration) for +what the migration does and how it runs. diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..b50ff80 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,132 @@ +# Operations + +Backup, restore, hand-editing, and recovery for `SQLiteDB` on disk. + +## Locating the files + +By default: + +``` +~/.local/share/hivemind-core/clients.db +~/.local/share/hivemind-core/clients.db-wal (present while DB is open) +~/.local/share/hivemind-core/clients.db-shm (present while WAL file exists) +``` + +Substitute `$XDG_DATA_HOME` if set, and `` / `` if configured +differently. See [Configuration](configuration.md). + +## Backups + +### Plain (unencrypted) + +For a consistent snapshot, use SQLite's online backup API via the CLI: + +```bash +sqlite3 ~/.local/share/hivemind-core/clients.db \ + ".backup /backup/clients.db.$(date +%F)" +``` + +This works even with a live `hivemind-core` process — the WAL-mode backup +is internally consistent regardless of concurrent reads or writes. + +For a quick file copy, stop the HiveMind process first and then copy all +three files: + +```bash +systemctl --user stop hivemind-core +cp ~/.local/share/hivemind-core/clients.db /backup/ +cp ~/.local/share/hivemind-core/clients.db-wal /backup/ 2>/dev/null; true +cp ~/.local/share/hivemind-core/clients.db-shm /backup/ 2>/dev/null; true +systemctl --user start hivemind-core +``` + +### Encrypted (SQLCipher) + +The `.backup` CLI command does not work with encrypted files unless you +compile the SQLCipher CLI. The safest approach is to stop the process and +copy the file: + +```bash +systemctl --user stop hivemind-core +cp ~/.local/share/hivemind-core/clients.db /backup/clients.db.$(date +%F) +systemctl --user start hivemind-core +``` + +Keep the backup and the passphrase together (but stored separately and +securely). + +## Verifying the schema version + +```bash +sqlite3 ~/.local/share/hivemind-core/clients.db "PRAGMA user_version;" +# Output: 2 +``` + +If the output is `0` or `1`, the migration has not yet run. Start +`hivemind-core` once to trigger it, or run: + +```python +from hivemind_sqlite_database import SQLiteDB +db = SQLiteDB() # migration runs automatically on __post_init__ +``` + +## Inspecting the database + +```bash +# List all clients +sqlite3 -header -column ~/.local/share/hivemind-core/clients.db \ + "SELECT client_id, name, is_admin, last_seen FROM clients;" + +# Find a client by API key +sqlite3 ~/.local/share/hivemind-core/clients.db \ + "SELECT * FROM clients WHERE api_key = 'your-key';" +``` + +## Migration to another backend + +```python +from hivemind_sqlite_database import SQLiteDB +from hivemind_plugin_manager import DatabaseFactory + +src = SQLiteDB() +dst = DatabaseFactory.create("hivemind-redis-db-plugin", + config={"host": "127.0.0.1", "port": 6379}) + +for client in src: + dst.add_item(client) +dst.commit() +``` + +Then update `server.json` to set `database.module` to the new plugin name and restart. + +## Authoring a database backend plugin + +To write a different database backend, implement `AbstractDB` from +`hivemind_plugin_manager.database`. The minimum required methods are: + +```python +from dataclasses import dataclass +from hivemind_plugin_manager.database import AbstractDB, Client +from typing import List, Union, Iterable + +@dataclass +class MyDB(AbstractDB): + name: str = "clients" + subfolder: str = "hivemind-core" + + def add_item(self, client: Client) -> bool: ... + def search_by_value(self, key: str, val: Union[str, bool, int, float]) -> List[Client]: ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterable[Client]: ... + def commit(self) -> bool: ... + def migrate(self, from_version: int) -> None: ... +``` + +Register it under `hivemind.database` in `pyproject.toml`: + +```toml +[project.entry-points."hivemind.database"] +"my-db-plugin" = "my_package:MyDB" +``` + +`hivemind-core` will discover it automatically once installed. diff --git a/hivemind_sqlite_database/__init__.py b/hivemind_sqlite_database/__init__.py index 2700b90..f4a66ac 100644 --- a/hivemind_sqlite_database/__init__.py +++ b/hivemind_sqlite_database/__init__.py @@ -2,7 +2,7 @@ import os.path import sqlite3 import threading -from typing import List, Union, Iterable +from typing import List, Optional, Union, Iterable from ovos_utils.log import LOG from ovos_utils.xdg_utils import xdg_data_home @@ -11,11 +11,12 @@ from dataclasses import dataclass + _VALID_COLUMNS = frozenset({ "client_id", "api_key", "name", "description", "is_admin", "last_seen", "intent_blacklist", "skill_blacklist", "message_blacklist", "allowed_types", "crypto_key", "password", - "can_broadcast", "can_escalate", "can_propagate", + "can_broadcast", "can_escalate", "can_propagate", "metadata", }) @@ -24,20 +25,47 @@ class SQLiteDB(AbstractDB): """Database implementation using SQLite.""" name: str = "clients" subfolder: str = "hivemind-core" + password: Optional[str] = None def __post_init__(self): """ Initialize the SQLiteDB connection. + + When *password* is set the database is opened via ``sqlcipher3`` and + encrypted with AES-256 (SQLCipher). The system library + ``libsqlcipher0`` must be installed and ``sqlcipher3`` must be + available (``pip install hivemind-sqlite-database[cipher]``). + + When *password* is ``None`` (default) the standard ``sqlite3`` module + is used and the database file is unencrypted. """ db_path = os.path.join(xdg_data_home(), self.subfolder, self.name + ".db") LOG.debug(f"sqlite database path: {db_path}") os.makedirs(os.path.dirname(db_path), exist_ok=True) - self.conn = sqlite3.connect(db_path, check_same_thread=False) - self.conn.row_factory = sqlite3.Row + if self.password is not None: + if self.password == "": + raise ValueError("password must be non-empty when encryption is enabled") + try: + import sqlcipher3 as _sqlcipher + except ImportError: + raise ImportError( + "sqlcipher3 is required to open an encrypted SQLite database. " + "Install the system library (e.g. 'apt install libsqlcipher-dev') " + "then: pip install hivemind-sqlite-database[cipher]" + ) + self.conn = _sqlcipher.connect(db_path, check_same_thread=False) + self.conn.row_factory = _sqlcipher.Row + escaped_password = self.password.replace("'", "''") + self.conn.execute(f"PRAGMA key='{escaped_password}'") + else: + self.conn = sqlite3.connect(db_path, check_same_thread=False) + self.conn.row_factory = sqlite3.Row + self.conn.execute("PRAGMA journal_mode=WAL") self._write_lock = threading.Lock() self._initialize_database() + self._maybe_migrate() def _initialize_database(self): """Initialize the database schema.""" @@ -60,9 +88,91 @@ def _initialize_database(self): password TEXT, can_broadcast BOOLEAN DEFAULT TRUE, can_escalate BOOLEAN DEFAULT TRUE, - can_propagate BOOLEAN DEFAULT TRUE + can_propagate BOOLEAN DEFAULT TRUE, + metadata TEXT ) """) + columns = { + row["name"] + for row in self.conn.execute("PRAGMA table_info(clients)").fetchall() + } + if "metadata" not in columns: + self.conn.execute("ALTER TABLE clients ADD COLUMN metadata TEXT") + + def _maybe_migrate(self) -> None: + """Run schema migration if the on-disk version is behind ``SCHEMA_VERSION``. + + Persisted version lives in SQLite's ``PRAGMA user_version`` (a + signed integer slot reserved exactly for this use case). + Tolerates older HPM versions that predate ``SCHEMA_VERSION``. + """ + target = getattr(AbstractDB, "SCHEMA_VERSION", 1) + stored = self.conn.execute("PRAGMA user_version").fetchone()[0] + # Tolerate older HPM that predates the forward-compat guard, the + # same way SCHEMA_VERSION is read defensively above. + if hasattr(self, "_check_forward_compat"): + self._check_forward_compat(int(stored)) + if stored < target: + LOG.info("SQLiteDB: migrating schema v%d -> v%d", stored, target) + # Migrate row rewrites and the user_version bump share one + # transaction so a crash never leaves the DB at "migrated rows + # but stale sentinel" or vice versa. + with self._write_lock, self.conn: + self._migrate_locked(from_version=stored) + self.conn.execute(f"PRAGMA user_version = {int(target)}") + + def migrate(self, from_version: int) -> None: + """Migrate on-disk rows to the current ``SCHEMA_VERSION``. + + Idempotent and crash-safe: a partial migration re-run produces the + same final state because the merge is ``setdefault``-style (never + clobbers explicit metadata values) and the legacy columns are + unconditionally NULLed in the same transaction. + + v1 -> v2: fold ``intent_blacklist`` / ``skill_blacklist`` column + values into each row's ``metadata`` JSON dict, then NULL the + legacy columns. ``message_blacklist`` is purged outright (the + field is not part of the ``Client`` data model). The columns + themselves remain in the table (SQLite ``ALTER TABLE ... DROP + COLUMN`` is unreliable on older versions) but are no longer + written by ``add_item``. + """ + if from_version >= 2: + return + with self._write_lock, self.conn: + self._migrate_locked(from_version=from_version) + + def _migrate_locked(self, from_version: int) -> None: + """Inner migration body — assumes the caller already holds + ``_write_lock`` and is inside a ``with self.conn`` transaction. + Lets ``_maybe_migrate`` bundle the version-sentinel bump into the + same transaction as the row rewrites.""" + if from_version >= 2: + return + for row in self.conn.execute( + "SELECT client_id, intent_blacklist, skill_blacklist, " + "message_blacklist, metadata FROM clients" + ).fetchall(): + metadata = self._metadata_from_row(row) or {} + # Drop any pre-existing metadata["message_blacklist"] + # from earlier migration runs that folded it in. + metadata.pop("message_blacklist", None) + for key in ("intent_blacklist", "skill_blacklist"): + raw = row[key] + if not raw: + continue + try: + legacy = json.loads(raw) + except (TypeError, ValueError): + continue + if legacy and key not in metadata: + metadata[key] = list(legacy) + self.conn.execute( + "UPDATE clients SET intent_blacklist = NULL, " + "skill_blacklist = NULL, message_blacklist = NULL, " + "metadata = ? WHERE client_id = ?", + (self._metadata_to_json(metadata), int(row["client_id"])), + ) def add_item(self, client: Client) -> bool: """ @@ -75,23 +185,30 @@ def add_item(self, client: Client) -> bool: True if the addition was successful, False otherwise. """ try: + metadata_json = self._metadata_to_json(client.metadata or {}) with self._write_lock, self.conn: self.conn.execute(""" INSERT OR REPLACE INTO clients ( client_id, api_key, name, description, is_admin, last_seen, intent_blacklist, skill_blacklist, message_blacklist, allowed_types, crypto_key, password, - can_broadcast, can_escalate, can_propagate - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + can_broadcast, can_escalate, can_propagate, metadata + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( client.client_id, client.api_key, client.name, client.description, client.is_admin, client.last_seen, - json.dumps(client.intent_blacklist), - json.dumps(client.skill_blacklist), - json.dumps(client.message_blacklist), + # Legacy OVOS blacklist columns are no longer written — + # the data lives in ``Client.metadata`` (see SCHEMA_VERSION + # v2 migration). Columns kept in the table for back-compat + # with older readers; NULLed on write so the disk stays + # clean. + None, + None, + None, json.dumps(client.allowed_types), client.crypto_key, client.password, - client.can_broadcast, client.can_escalate, client.can_propagate + client.can_broadcast, client.can_escalate, client.can_propagate, + metadata_json, )) return True except sqlite3.Error as e: @@ -121,6 +238,27 @@ def search_by_value(self, key: str, val: Union[str, bool, int, float]) -> List[C LOG.error(f"Failed to search clients in SQLite: {e}") return [] + def get_client_by_id(self, client_id: int) -> Optional[Client]: + """Fetch a single client row by primary key. + + Targeted lookup used by :meth:`refresh` on the admission hot + path — avoids the full ``search_by_value`` fallback. Returns + ``None`` if the row does not exist or on any DB error. + """ + if client_id is None: + return None + try: + cur = self.conn.execute( + "SELECT * FROM clients WHERE client_id = ?", (int(client_id),), + ) + row = cur.fetchone() + if row is None: + return None + return self._row_to_client(row) + except (sqlite3.Error, TypeError, ValueError) as e: + LOG.error(f"Failed to fetch client {client_id} from SQLite: {e}") + return None + def __len__(self) -> int: """Get the number of clients in the database.""" try: @@ -153,7 +291,29 @@ def commit(self) -> bool: @staticmethod def _row_to_client(row: sqlite3.Row) -> Client: - """Convert a database row to a Client instance.""" + """Convert a database row to a Client instance. + + Legacy OVOS blacklist columns are no longer passed as kwargs to + ``Client(...)`` — after the v2 migration they are NULL on disk + and the canonical data lives in ``metadata``. If a row predates + migration (e.g. read by an older plugin version that didn't + migrate, then read again here), the values are folded into + ``metadata`` locally as a defensive fallback. + """ + metadata = SQLiteDB._metadata_from_row(row) or {} + # message_blacklist was removed from the Client data model — drop + # any residual metadata key from earlier migrations. + metadata.pop("message_blacklist", None) + for key in ("intent_blacklist", "skill_blacklist"): + raw = row[key] if key in row.keys() else None + if not raw or key in metadata: + continue + try: + legacy = json.loads(raw) + except (TypeError, ValueError): + continue + if legacy: + metadata[key] = list(legacy) return Client( client_id=int(row["client_id"]), api_key=row["api_key"], @@ -161,13 +321,44 @@ def _row_to_client(row: sqlite3.Row) -> Client: description=row["description"], is_admin=bool(row["is_admin"]), last_seen=row["last_seen"], - intent_blacklist=json.loads(row["intent_blacklist"] or "[]"), - skill_blacklist=json.loads(row["skill_blacklist"] or "[]"), - message_blacklist=json.loads(row["message_blacklist"] or "[]"), allowed_types=json.loads(row["allowed_types"] or "[]"), crypto_key=row["crypto_key"], password=row["password"], can_broadcast=bool(row["can_broadcast"]), can_escalate=bool(row["can_escalate"]), - can_propagate=bool(row["can_propagate"]) + can_propagate=bool(row["can_propagate"]), + metadata=metadata, ) + + @staticmethod + def _metadata_to_json(metadata: object) -> str: + """Serialize ``Client.metadata`` for storage in the ``metadata`` column. + + ``metadata`` is documented as a free-form dict; we use ``default=str`` + so callers can stash convenience values like ``datetime`` or ``UUID`` + without crashing on insert. Note that these come back as strings on + read — the column is opaque JSON, not a typed map. Returns ``"{}"`` + for non-dict input and on any (unexpected) serialisation failure + rather than corrupting the row. + """ + if not isinstance(metadata, dict): + return "{}" + try: + return json.dumps(metadata, default=str) + except (TypeError, ValueError): + return "{}" + + @staticmethod + def _metadata_from_row(row: sqlite3.Row) -> dict: + """Decode the ``metadata`` column into a dict, swallowing garbage. + + Returns ``{}`` for NULL, malformed JSON, or valid-JSON-that-isn't-an-object. + Lets a single bad row not poison iteration over the table. + """ + if "metadata" not in row.keys() or not row["metadata"]: + return {} + try: + metadata = json.loads(row["metadata"]) + except (TypeError, ValueError): + return {} + return metadata if isinstance(metadata, dict) else {} diff --git a/hivemind_sqlite_database/version.py b/hivemind_sqlite_database/version.py index 58d7648..fb4ca12 100644 --- a/hivemind_sqlite_database/version.py +++ b/hivemind_sqlite_database/version.py @@ -1,8 +1,8 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 2 -VERSION_BUILD = 1 -VERSION_ALPHA = 4 +VERSION_MINOR = 4 +VERSION_BUILD = 0 +VERSION_ALPHA = 2 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + (f"a{VERSION_ALPHA}" if VERSION_ALPHA else "") diff --git a/pyproject.toml b/pyproject.toml index 1833c0a..db99161 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,12 +10,15 @@ license = { text = "Apache-2.0" } authors = [{ name = "jarbasAi", email = "jarbasai@mailfence.com" }] requires-python = ">=3.9" dependencies = [ - "hivemind-plugin-manager>=0.1.0,<1.0.0", + "hivemind-plugin-manager>=0.5.0,<1.0.0", "hivemind-bus-client", "ovos-utils", ] +[project.optional-dependencies] +cipher = ["sqlcipher3"] + [project.urls] Homepage = "https://github.com/JarbasHiveMind/hivemind-sqlite-database" diff --git a/tests/e2e/test_policy_migration_e2e.py b/tests/e2e/test_policy_migration_e2e.py new file mode 100644 index 0000000..8511b33 --- /dev/null +++ b/tests/e2e/test_policy_migration_e2e.py @@ -0,0 +1,169 @@ +"""End-to-end: a real SQLiteDB backend, driven through a live hivemind-core +policy chain in a hivescope topology, proving that a blacklist folded into +``Client.metadata`` by the v1→v2 schema migration is injected into the OVOS +session by ``OVOSAgentPolicy``. + +This is the cross-repo seam the unit tests cannot cover on their own: + + legacy skill_blacklist column --(plugin migrate)--> Client.metadata + --(OVOSAgentPolicy)--> session.blacklisted_skills + +Requires the policy stack (hivemind-core policy chain + OVOSAgentPolicy); +skipped when it is not installed. +""" +import importlib.util +import json +import tempfile +import time +from unittest import mock + +import pytest + +# Cross-repo deps absent from the DB plugin's own CI — skip the whole module +# there; it runs in the integration env that pins the policy stack + hivescope. +pytest.importorskip("hivemind_core.policy") +pytest.importorskip("hivemind_ovos_agent_plugin") +pytest.importorskip("hivescope") + +def _hivescope_supports_db_injection() -> bool: + """The master-side real-DB injection needs hivescope's MasterNode.create + to accept a ``db=`` argument (added in hivescope > 0.3.0a1).""" + if importlib.util.find_spec("hivescope") is None: + return False + import inspect + from hivescope.node import MasterNode + return "db" in inspect.signature(MasterNode.create).parameters + + +_HAS_POLICY = ( + importlib.util.find_spec("hivemind_core.policy") is not None + and importlib.util.find_spec("hivemind_ovos_agent_plugin") is not None +) +pytestmark = [ + pytest.mark.skipif( + not _HAS_POLICY, reason="needs hivemind-core policy chain + OVOSAgentPolicy" + ), + pytest.mark.skipif( + not _hivescope_supports_db_injection(), + reason="needs hivescope MasterNode.create(db=...) (> 0.3.0a1)", + ), +] + +from hivemind_bus_client.message import HiveMessage, HiveMessageType # noqa: E402 +from ovos_bus_client.message import Message # noqa: E402 +from hivemind_core.database import ClientDatabase # noqa: E402 +from hivescope.topology import TopologyBuilder # noqa: E402 +from hivescope.assertions import assert_session_blacklists_injected # noqa: E402 + +_SQLITE_PLUGIN = "hivemind-sqlite-db-plugin" + + +class _HivescopeDBAdapter: + """Bridges hivescope's ``register_satellite`` (which passes + ``can_escalate``/``can_propagate``/``can_broadcast`` to ``add_client``) + to the real ``ClientDatabase`` whose ``add_client`` does not take them. + Everything else delegates straight to the backing ClientDatabase, so the + policy chain resolves real clients from the real sqlite plugin. + """ + + def __init__(self, cdb): + object.__setattr__(self, "_cdb", cdb) + + def add_client(self, name, key, password=None, admin=False, + crypto_key=None, allowed_types=None, metadata=None, + intent_blacklist=None, skill_blacklist=None, + message_blacklist=None, can_escalate=True, + can_propagate=True, can_broadcast=True): + return self._cdb.add_client( + name=name, key=key, admin=admin, allowed_types=allowed_types, + crypto_key=crypto_key, password=password, metadata=metadata, + intent_blacklist=intent_blacklist, skill_blacklist=skill_blacklist, + message_blacklist=message_blacklist, + ) + + def __getattr__(self, item): + return getattr(object.__getattribute__(self, "_cdb"), item) + + def __iter__(self): + return iter(self._cdb) + + def __len__(self): + return len(self._cdb) + + def __enter__(self): + return self._cdb.__enter__() + + def __exit__(self, *a): + return self._cdb.__exit__(*a) + + +def _reset_resolution_cache(hm_protocol): + """Invalidate any cached pre-migration user resolution on every live + connection so the policy re-reads the freshly-migrated metadata.""" + for attr in ("clients", "connections", "_clients", "_connections"): + reg = getattr(hm_protocol, attr, None) + if isinstance(reg, dict): + for conn in reg.values(): + if hasattr(conn, "reset_user"): + conn.reset_user() + + +def _seed_legacy_blacklist(db, api_key: str, skills): + """Rewrite a connected client's row into the legacy v1 shape: the + blacklist lives in the top-level ``skill_blacklist`` column and the + on-disk schema version is rolled back to v1, exactly what an operator's + DB looks like before the plugin upgrade runs. + """ + with db._write_lock: + db.conn.execute( + "UPDATE clients SET skill_blacklist = ? WHERE api_key = ?", + (json.dumps(skills), api_key), + ) + db.conn.execute("PRAGMA user_version = 1") + db.conn.commit() + + +def test_migrated_skill_blacklist_reaches_session(): + tmp = tempfile.mkdtemp() + with mock.patch("hivemind_sqlite_database.xdg_data_home", return_value=tmp): + cdb = ClientDatabase(config={ + "module": _SQLITE_PLUGIN, + _SQLITE_PLUGIN: {"name": "clients", "subfolder": "hivemind-core"}, + }) + + b = TopologyBuilder() + m = b.add_master("M0", db=_HivescopeDBAdapter(cdb), require_crypto=False) + s = b.add_satellite( + "S0", upstream=m, is_admin=False, + allowed_types=["recognizer_loop:utterance"], + ) + b.start_all() + try: + key = s.identity.access_key + + # Operator's DB carried a legacy top-level blacklist; the plugin + # migration folds it into metadata on the next open/upgrade. + _seed_legacy_blacklist(cdb.db, key, ["skill-weather"]) + cdb.db.migrate(from_version=1) + _reset_resolution_cache(m.hm_protocol) + + seen = [] + m.agent_protocol.bus.on("recognizer_loop:utterance", seen.append) + s.send(HiveMessage( + HiveMessageType.BUS, + payload=Message("recognizer_loop:utterance", + {"utterances": ["what is the weather"]}), + )) + + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline and not seen: + time.sleep(0.02) + assert seen, "utterance never reached the agent bus" + + assert_session_blacklists_injected( + m, s, + msg_type="recognizer_loop:utterance", + expected_skills=["skill-weather"], + ) + finally: + b.stop_all() diff --git a/tests/test_sqlitedb.py b/tests/test_sqlitedb.py index 1b414bd..183faff 100644 --- a/tests/test_sqlitedb.py +++ b/tests/test_sqlitedb.py @@ -236,9 +236,150 @@ def test_boolean_fields_remain_bool_after_round_trip(self): self.assertIs(type(results[0].can_broadcast), bool) self.assertFalse(results[0].can_broadcast) + def test_metadata_survives_round_trip(self): + db = make_db() + db.add_item( + make_client( + 1, + "k1", + metadata={"owner_id": "owner-123"}, + ) + ) + + results = db.search_by_value("api_key", "k1") + + self.assertEqual(len(results), 1) + self.assertEqual(results[0].metadata, {"owner_id": "owner-123"}) + + def test_metadata_can_be_searched(self): + db = make_db() + db.add_item( + make_client( + 1, + "k1", + metadata={"owner_id": "owner-123"}, + ) + ) + + results = db.search_by_value("metadata", '{"owner_id": "owner-123"}') + + self.assertEqual(len(results), 1) + self.assertEqual(results[0].api_key, "k1") + + def test_initialize_database_migrates_legacy_clients_table(self): + db = object.__new__(SQLiteDB) + db.name = "clients" + db.subfolder = "hivemind-core" + db.conn = sqlite3.connect(":memory:", check_same_thread=False) + db.conn.row_factory = sqlite3.Row + db._write_lock = threading.Lock() + with db.conn: + db.conn.execute(""" + CREATE TABLE clients ( + client_id INTEGER PRIMARY KEY, + api_key VARCHAR(255) NOT NULL, + name VARCHAR(255), + description VARCHAR(255), + is_admin BOOLEAN DEFAULT FALSE, + last_seen REAL DEFAULT -1, + intent_blacklist TEXT, + skill_blacklist TEXT, + message_blacklist TEXT, + allowed_types TEXT, + crypto_key VARCHAR(16), + password TEXT, + can_broadcast BOOLEAN DEFAULT TRUE, + can_escalate BOOLEAN DEFAULT TRUE, + can_propagate BOOLEAN DEFAULT TRUE + ) + """) + db.conn.execute("INSERT INTO clients (client_id, api_key) VALUES (1, 'k1')") + + db._initialize_database() + + columns = { + row["name"] + for row in db.conn.execute("PRAGMA table_info(clients)").fetchall() + } + self.assertIn("metadata", columns) + client = db.search_by_value("api_key", "k1")[0] + self.assertEqual(client.metadata, {}) + + def test_metadata_nested_dict_round_trip(self): + db = make_db() + meta = { + "owner": {"id": "owner-1", "tags": ["a", "b"]}, + "counts": {"x": 1, "y": 2}, + } + db.add_item(make_client(1, "k1", metadata=meta)) + results = db.search_by_value("api_key", "k1") + self.assertEqual(len(results), 1) + self.assertEqual(results[0].metadata, meta) + + def test_metadata_non_ascii_round_trip(self): + db = make_db() + meta = {"name": "Zé Ninguém", "emoji": "🚀", "ru": "Привет"} + db.add_item(make_client(1, "k1", metadata=meta)) + results = db.search_by_value("api_key", "k1") + self.assertEqual(len(results), 1) + self.assertEqual(results[0].metadata, meta) + + def test_metadata_survives_iteration(self): + db = make_db() + db.add_item( + make_client( + 1, + "k1", + metadata={"owner_id": "owner-123"}, + ) + ) + + clients = list(db) + + self.assertEqual(len(clients), 1) + self.assertEqual(clients[0].metadata, {"owner_id": "owner-123"}) + + def test_metadata_defaults_to_empty_dict_when_not_provided(self): + db = make_db() + db.add_item(make_client(1, "k1")) + results = db.search_by_value("api_key", "k1") + self.assertEqual(results[0].metadata, {}) + + def test_metadata_overwritten_on_reinsert_with_same_client_id(self): + db = make_db() + db.add_item(make_client(1, "k1", metadata={"v": 1})) + db.add_item(make_client(1, "k1", metadata={"v": 2, "extra": "x"})) + results = db.search_by_value("api_key", "k1") + self.assertEqual(len(results), 1) + self.assertEqual(results[0].metadata, {"v": 2, "extra": "x"}) + + def test_metadata_to_json_returns_empty_for_non_dict(self): + self.assertEqual(SQLiteDB._metadata_to_json("not a dict"), "{}") + self.assertEqual(SQLiteDB._metadata_to_json(None), "{}") + self.assertEqual(SQLiteDB._metadata_to_json(42), "{}") + + def test_metadata_from_row_returns_empty_for_garbage_or_missing(self): + db = make_db() + # legacy-style row with explicit NULL metadata + db.add_item(make_client(1, "k1")) + with db.conn: + db.conn.execute("UPDATE clients SET metadata = NULL WHERE client_id = 1") + results = db.search_by_value("api_key", "k1") + self.assertEqual(results[0].metadata, {}) + # garbage JSON in the metadata column → coerce to {} + with db.conn: + db.conn.execute("UPDATE clients SET metadata = 'not json{' WHERE client_id = 1") + results = db.search_by_value("api_key", "k1") + self.assertEqual(results[0].metadata, {}) + # valid JSON but not an object → coerce to {} + with db.conn: + db.conn.execute("UPDATE clients SET metadata = '[1,2,3]' WHERE client_id = 1") + results = db.search_by_value("api_key", "k1") + self.assertEqual(results[0].metadata, {}) + def test_full_client_fields_preserved(self): db = make_db() - c = Client( + c = make_client( client_id=42, api_key="full-key", name="test-client", @@ -247,13 +388,13 @@ def test_full_client_fields_preserved(self): last_seen=1234567890.0, intent_blacklist=["a:b"], skill_blacklist=["c:d"], - message_blacklist=["e:f"], allowed_types=["recognizer_loop:utterance"], crypto_key="1234567890123456", password="secret", can_broadcast=True, can_escalate=False, can_propagate=True, + metadata={"owner_id": "owner-123"}, ) db.add_item(c) results = db.search_by_value("api_key", "full-key") @@ -267,6 +408,18 @@ def test_full_client_fields_preserved(self): self.assertEqual(r.crypto_key, "1234567890123456") self.assertEqual(r.password, "secret") self.assertFalse(r.can_escalate) + # After SCHEMA_VERSION=2: legacy skill/intent kwargs auto-migrate + # into ``Client.metadata`` via Client.__init__. message_blacklist + # is gone from the data model — not accepted as a kwarg and not + # carried in metadata. + self.assertEqual(r.metadata, { + "owner_id": "owner-123", + "intent_blacklist": ["a:b"], + "skill_blacklist": ["c:d"], + }) + # Property shims surface skill/intent blacklists at legacy names. + self.assertEqual(r.skill_blacklist, ["c:d"]) + self.assertEqual(r.intent_blacklist, ["a:b"]) class TestSQLiteDBCommit(unittest.TestCase): @@ -302,5 +455,411 @@ def test_replace_item(self): self.assertEqual(db.search_by_value("api_key", "revoked")[0].client_id, 1) +try: + import sqlcipher3 as _sqlcipher3 # noqa: F401 + _SQLCIPHER_AVAILABLE = True +except ImportError: + _SQLCIPHER_AVAILABLE = False + + +@unittest.skipUnless(_SQLCIPHER_AVAILABLE, "sqlcipher3 not installed") +class TestSQLiteDBEncrypted(unittest.TestCase): + """Tests for the SQLCipher-encrypted path. Skipped when sqlcipher3 is absent.""" + + def _make_encrypted_db(self, path: str, password: str = "hunter2") -> SQLiteDB: + import unittest.mock as mock + db = SQLiteDB.__new__(SQLiteDB) + db.name = os.path.splitext(os.path.basename(path))[0] + db.subfolder = "" + db.password = password + with mock.patch( + "hivemind_sqlite_database.xdg_data_home", + return_value=os.path.dirname(path), + ): + db.__post_init__() + return db + + def test_encrypted_file_unreadable_by_stdlib_sqlite3(self): + """A file created with a password must be opaque to plain sqlite3.""" + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f: + path = f.name + os.unlink(path) + try: + db = self._make_encrypted_db(path, password="secret123") + db.add_item(make_client(1, "enc-key")) + db.commit() + # stdlib sqlite3 should not be able to read it + plain_conn = sqlite3.connect(path) + with self.assertRaises(sqlite3.DatabaseError): + plain_conn.execute("SELECT * FROM clients").fetchall() + plain_conn.close() + finally: + if os.path.exists(path): + os.unlink(path) + + def test_encrypted_round_trip(self): + """add_item then search_by_value works through the encryption layer.""" + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f: + path = f.name + os.unlink(path) + try: + db = self._make_encrypted_db(path, password="roundtrip") + db.add_item(make_client(1, "enc-api-key", name="alice")) + db.commit() + # Reopen with same password + db2 = self._make_encrypted_db(path, password="roundtrip") + results = db2.search_by_value("api_key", "enc-api-key") + self.assertEqual(len(results), 1) + self.assertEqual(results[0].name, "alice") + finally: + if os.path.exists(path): + os.unlink(path) + + def test_sqlitedb_password_kwarg_raises_importerror_without_sqlcipher3(self): + """Confirmed separately in test_sqlitedb_no_sqlcipher.py; skip here.""" + pass + + +class TestSQLiteDBMissingCipher(unittest.TestCase): + """Verify ImportError is raised when sqlcipher3 is absent and password is given.""" + + def test_importerror_when_sqlcipher3_missing(self): + import sys + import unittest.mock as mock + + # Simulate sqlcipher3 not being installed + with mock.patch.dict(sys.modules, {"sqlcipher3": None}): + with tempfile.TemporaryDirectory() as tmpdir: + with self.assertRaises(ImportError) as ctx: + db = SQLiteDB.__new__(SQLiteDB) + db.name = "test" + db.subfolder = tmpdir + db.password = "secret" + # Patch xdg_data_home so db_path resolves inside tmpdir + with mock.patch("hivemind_sqlite_database.xdg_data_home", + return_value=tmpdir): + db.__post_init__() + self.assertIn("sqlcipher3", str(ctx.exception)) + + +class TestSQLiteDBMigration(unittest.TestCase): + """v1 -> v2: legacy OVOS blacklist columns folded into metadata.""" + + def _make_v1_db_with_legacy_rows(self) -> SQLiteDB: + """Construct a DB in the v1 shape: legacy columns populated, + ``PRAGMA user_version`` left at 0 (the SQLite default).""" + db = object.__new__(SQLiteDB) + db.name = "clients" + db.subfolder = "hivemind-core" + db.conn = sqlite3.connect(":memory:", check_same_thread=False) + db.conn.row_factory = sqlite3.Row + db._write_lock = threading.Lock() + db._initialize_database() + # Write a row directly with legacy column data — bypass add_item + # which would NULL them. + db.conn.execute( + "INSERT INTO clients (client_id, api_key, intent_blacklist, " + "skill_blacklist, message_blacklist, allowed_types, metadata) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + (7, "legacy-key", + '["i:1"]', '["s:1"]', '["m:1"]', '[]', '{"owner": "u"}'), + ) + db.conn.commit() + return db + + def test_pragma_user_version_starts_at_zero(self): + db = self._make_v1_db_with_legacy_rows() + stored = db.conn.execute("PRAGMA user_version").fetchone()[0] + self.assertEqual(stored, 0) + + def test_migrate_folds_legacy_columns_into_metadata(self): + db = self._make_v1_db_with_legacy_rows() + db.migrate(from_version=1) + row = db.conn.execute( + "SELECT intent_blacklist, skill_blacklist, message_blacklist, " + "metadata FROM clients WHERE client_id = 7" + ).fetchone() + self.assertIsNone(row["intent_blacklist"]) + self.assertIsNone(row["skill_blacklist"]) + self.assertIsNone(row["message_blacklist"]) + import json as _json + meta = _json.loads(row["metadata"]) + self.assertEqual(meta["owner"], "u") + self.assertEqual(meta["intent_blacklist"], ["i:1"]) + self.assertEqual(meta["skill_blacklist"], ["s:1"]) + # message_blacklist is purged outright, NOT folded into metadata. + self.assertNotIn("message_blacklist", meta) + + def test_migrate_purges_residual_metadata_message_blacklist(self): + """An older plugin version may have folded message_blacklist + into metadata before HPM removed the field. The newer migrate() + must purge it on re-run, leaving the disk clean.""" + db = self._make_v1_db_with_legacy_rows() + # Seed an already-half-migrated row: legacy columns NULL, but + # metadata still carries the old key from the prior migration. + db.conn.execute( + "UPDATE clients SET intent_blacklist = NULL, " + "skill_blacklist = NULL, message_blacklist = NULL, " + "metadata = ? WHERE client_id = 7", + ('{"owner": "u", "message_blacklist": ["m:1"]}',), + ) + db.conn.commit() + + db.migrate(from_version=1) + + import json as _json + meta = _json.loads(db.conn.execute( + "SELECT metadata FROM clients WHERE client_id = 7" + ).fetchone()["metadata"]) + self.assertNotIn("message_blacklist", meta) + self.assertEqual(meta["owner"], "u") + + def test_migrate_is_idempotent(self): + db = self._make_v1_db_with_legacy_rows() + db.migrate(from_version=1) + db.migrate(from_version=1) # second run = no-op on already-migrated row + row = db.conn.execute( + "SELECT metadata FROM clients WHERE client_id = 7" + ).fetchone() + import json as _json + meta = _json.loads(row["metadata"]) + self.assertEqual(meta["skill_blacklist"], ["s:1"]) + + def test_migrate_setdefault_does_not_clobber_explicit_metadata(self): + db = self._make_v1_db_with_legacy_rows() + # Explicit metadata.skill_blacklist takes precedence over the + # legacy column. + db.conn.execute( + "UPDATE clients SET metadata = ? WHERE client_id = 7", + ('{"owner": "u", "skill_blacklist": ["explicit"]}',), + ) + db.migrate(from_version=1) + import json as _json + meta = _json.loads(db.conn.execute( + "SELECT metadata FROM clients WHERE client_id = 7" + ).fetchone()["metadata"]) + self.assertEqual(meta["skill_blacklist"], ["explicit"]) + + def test_migrate_skips_when_already_at_target(self): + db = self._make_v1_db_with_legacy_rows() + # Stub: a from_version >= 2 should not touch the row. + db.migrate(from_version=2) + row = db.conn.execute( + "SELECT intent_blacklist FROM clients WHERE client_id = 7" + ).fetchone() + self.assertEqual(row["intent_blacklist"], '["i:1"]') + + def test_maybe_migrate_bumps_user_version(self): + db = self._make_v1_db_with_legacy_rows() + db._maybe_migrate() + stored = db.conn.execute("PRAGMA user_version").fetchone()[0] + self.assertEqual(stored, 2) + # second invocation is a no-op + db._maybe_migrate() + self.assertEqual( + db.conn.execute("PRAGMA user_version").fetchone()[0], 2, + ) + + def test_post_init_runs_migration_on_existing_db(self): + """End-to-end: open a v1 DB file, observe v2 on-disk shape.""" + with tempfile.TemporaryDirectory() as tmp: + db_path = os.path.join(tmp, "hivemind-core", "clients.db") + os.makedirs(os.path.dirname(db_path)) + # Seed a v1-shape DB with legacy column data. + seed = sqlite3.connect(db_path) + seed.execute(""" + CREATE TABLE clients ( + client_id INTEGER PRIMARY KEY, + api_key VARCHAR(255) NOT NULL, + name VARCHAR(255), + description VARCHAR(255), + is_admin BOOLEAN DEFAULT FALSE, + last_seen REAL DEFAULT -1, + intent_blacklist TEXT, + skill_blacklist TEXT, + message_blacklist TEXT, + allowed_types TEXT, + crypto_key VARCHAR(16), + password TEXT, + can_broadcast BOOLEAN DEFAULT TRUE, + can_escalate BOOLEAN DEFAULT TRUE, + can_propagate BOOLEAN DEFAULT TRUE, + metadata TEXT + ) + """) + seed.execute( + "INSERT INTO clients (client_id, api_key, skill_blacklist) " + "VALUES (?, ?, ?)", + (9, "k", '["legacy.skill"]'), + ) + seed.commit() + seed.close() + # Now open it via SQLiteDB — __post_init__ should migrate. + import unittest.mock as mock + with mock.patch("hivemind_sqlite_database.xdg_data_home", + return_value=tmp): + db = SQLiteDB(name="clients", subfolder="hivemind-core") + self.assertEqual( + db.conn.execute("PRAGMA user_version").fetchone()[0], 2, + ) + row = db.conn.execute( + "SELECT skill_blacklist, metadata FROM clients " + "WHERE client_id = 9" + ).fetchone() + self.assertIsNone(row["skill_blacklist"]) + import json as _json + self.assertEqual(_json.loads(row["metadata"])["skill_blacklist"], + ["legacy.skill"]) + + +class TestSQLiteDBEmptyDatabaseMigration(unittest.TestCase): + """A fresh DB (no rows, user_version=0) must still bump to the + current SCHEMA_VERSION on first open. Validates the cotransactional + migrate + sentinel write.""" + + def test_empty_new_db_bumps_user_version(self): + with tempfile.TemporaryDirectory() as tmp: + import unittest.mock as mock + with mock.patch("hivemind_sqlite_database.xdg_data_home", + return_value=tmp): + db = SQLiteDB(name="clients", subfolder="hivemind-core") + stored = db.conn.execute("PRAGMA user_version").fetchone()[0] + from hivemind_plugin_manager.database import AbstractDB + target = getattr(AbstractDB, "SCHEMA_VERSION", 1) + self.assertEqual(stored, target) + # No rows in the clients table — migration must be a no-op + # over rows but the sentinel still moves. + count = db.conn.execute( + "SELECT COUNT(*) FROM clients" + ).fetchone()[0] + self.assertEqual(count, 0) + + +class TestSQLiteDBForwardCompat(unittest.TestCase): + """A DB whose ``user_version`` is newer than this backend supports + must fail loudly with a RuntimeError instead of silently downgrading. + """ + + def test_forward_version_raises_runtime_error(self): + import unittest.mock as mock + with tempfile.TemporaryDirectory() as tmp: + with mock.patch("hivemind_sqlite_database.xdg_data_home", + return_value=tmp): + db = SQLiteDB(name="clients", subfolder="hivemind-core") + db.conn.execute("PRAGMA user_version = 999") + db.conn.commit() + db.conn.close() + with self.assertRaises(RuntimeError) as ctx: + SQLiteDB(name="clients", subfolder="hivemind-core") + self.assertIn("999", str(ctx.exception)) + + +class TestSQLiteDBGetClientByID(unittest.TestCase): + def test_get_client_by_id_returns_row(self): + import unittest.mock as mock + with tempfile.TemporaryDirectory() as tmp: + with mock.patch("hivemind_sqlite_database.xdg_data_home", + return_value=tmp): + db = SQLiteDB(name="clients", subfolder="hivemind-core") + from hivemind_plugin_manager.database import Client + db.add_item(Client(client_id=42, api_key="k", name="alice")) + got = db.get_client_by_id(42) + self.assertIsNotNone(got) + self.assertEqual(got.client_id, 42) + self.assertIsNone(db.get_client_by_id(999)) + + def test_refresh_picks_up_updates(self): + import unittest.mock as mock + with tempfile.TemporaryDirectory() as tmp: + with mock.patch("hivemind_sqlite_database.xdg_data_home", + return_value=tmp): + db = SQLiteDB(name="clients", subfolder="hivemind-core") + from hivemind_plugin_manager.database import Client + db.add_item(Client(client_id=1, api_key="k", name="a", + allowed_types=["x"])) + self.assertEqual(db.refresh(1).allowed_types, ["x"]) + db.add_item(Client(client_id=1, api_key="k", name="a", + allowed_types=["y"])) + self.assertEqual(db.refresh(1).allowed_types, ["y"]) + + +class TestSQLiteDBSchemaV2RoundTrip(unittest.TestCase): + """v2 schema: allowed_types + skill/intent blacklists (in metadata) survive + add→search and add→refresh cycles without loss or mutation.""" + + def test_allowed_types_survives_round_trip(self): + db = make_db() + allowed = ["recognizer_loop:utterance", "speak:b64_audio"] + db.add_item(make_client(1, "k", allowed_types=allowed)) + found = db.search_by_value("api_key", "k") + self.assertEqual(len(found), 1) + self.assertEqual(found[0].allowed_types, allowed) + + def test_skill_blacklist_in_metadata_survives_round_trip(self): + db = make_db() + c = make_client(2, "k2", metadata={"skill_blacklist": ["my.skill"]}) + db.add_item(c) + found = db.search_by_value("api_key", "k2") + self.assertEqual(len(found), 1) + self.assertEqual(found[0].skill_blacklist, ["my.skill"]) + self.assertEqual(found[0].metadata["skill_blacklist"], ["my.skill"]) + + def test_intent_blacklist_in_metadata_survives_round_trip(self): + db = make_db() + c = make_client(3, "k3", metadata={"intent_blacklist": ["my.skill:action"]}) + db.add_item(c) + found = db.search_by_value("api_key", "k3") + self.assertEqual(len(found), 1) + self.assertEqual(found[0].intent_blacklist, ["my.skill:action"]) + self.assertEqual(found[0].metadata["intent_blacklist"], ["my.skill:action"]) + + def test_message_blacklist_not_present_in_stored_record(self): + """message_blacklist must not appear in a freshly-stored record.""" + db = make_db() + db.add_item(make_client(4, "k4")) + row = db.conn.execute( + "SELECT message_blacklist, metadata FROM clients WHERE client_id = 4" + ).fetchone() + self.assertIsNone(row["message_blacklist"]) + import json as _json + meta = _json.loads(row["metadata"] or "{}") + self.assertNotIn("message_blacklist", meta) + + def test_v1_row_reads_cleanly_forward_compat(self): + """A v1 row (legacy columns populated) must deserialize via + _row_to_client without crashing.""" + db = make_db() + db.conn.execute( + "INSERT INTO clients (client_id, api_key, skill_blacklist, " + "intent_blacklist, message_blacklist, allowed_types, metadata) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + (5, "k5", '["old.skill"]', '[]', '["drop.me"]', + '["recognizer_loop:utterance"]', "{}"), + ) + db.conn.commit() + found = db.search_by_value("api_key", "k5") + self.assertEqual(len(found), 1) + self.assertEqual(found[0].api_key, "k5") + self.assertEqual(found[0].allowed_types, ["recognizer_loop:utterance"]) + + def test_refresh_returns_v2_fields(self): + db = make_db() + import unittest.mock as mock + import tempfile + import os + with tempfile.TemporaryDirectory() as tmp: + with mock.patch("hivemind_sqlite_database.xdg_data_home", + return_value=tmp): + filedb = SQLiteDB(name="clients", subfolder="hivemind-core") + allowed = ["recognizer_loop:utterance"] + meta = {"skill_blacklist": ["s:1"], "intent_blacklist": ["i:1"]} + filedb.add_item(make_client(6, "k6", allowed_types=allowed, metadata=meta)) + got = filedb.refresh(6) + self.assertIsNotNone(got) + self.assertEqual(got.allowed_types, allowed) + self.assertEqual(got.skill_blacklist, ["s:1"]) + self.assertEqual(got.intent_blacklist, ["i:1"]) + + if __name__ == "__main__": unittest.main()