From 31d633f99e0a212061aa0c85bf0edcf5cd86c282 Mon Sep 17 00:00:00 2001 From: remote service administrator Date: Thu, 25 Jun 2026 19:02:18 -0400 Subject: [PATCH 01/12] feat(cli): pass chDB config file to local server --- apps/cli/src/commands/server.ts | 27 +++++++++++++++---- apps/cli/src/server/chdb.ts | 3 +++ apps/cli/src/server/serve.ts | 7 ++++- .../content/docs/local-mode/cli-reference.md | 1 + 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/apps/cli/src/commands/server.ts b/apps/cli/src/commands/server.ts index 0135faf31..20442905d 100644 --- a/apps/cli/src/commands/server.ts +++ b/apps/cli/src/commands/server.ts @@ -104,6 +104,12 @@ const dataDirFlag = Flag.optional( ), ) +const chdbConfigFileFlag = Flag.optional( + Flag.string("chdb-config-file").pipe( + Flag.withDescription("Optional ClickHouse config file passed to embedded chDB"), + ), +) + const backgroundFlag = Flag.boolean("background").pipe( Flag.withAlias("d"), Flag.withDescription("Run the server detached (logs to ~/.maple/maple.log); stop with `maple stop`"), @@ -149,7 +155,12 @@ const probeHealth = (addr: string): Effect.Effect => * file; we poll `/health` until it binds, then print a summary and return so the * parent process exits. */ -const startDetached = (port: number, dataDir: string, offline: boolean): Effect.Effect => +const startDetached = ( + port: number, + dataDir: string, + offline: boolean, + chdbConfigFile: string | undefined, +): Effect.Effect => Effect.gen(function* () { const logPath = logFilePath(dataDir) // Rebuild the command explicitly rather than slicing argv: a Bun-compiled @@ -165,6 +176,7 @@ const startDetached = (port: number, dataDir: string, offline: boolean): Effect. String(port), "--data-dir", dataDir, + ...(chdbConfigFile ? ["--chdb-config-file", chdbConfigFile] : []), ...(offline ? ["--offline"] : []), ] @@ -214,6 +226,7 @@ const startDetached = (port: number, dataDir: string, offline: boolean): Effect. export const start = Command.make("start", { port, dataDir: dataDirFlag, + chdbConfigFile: chdbConfigFileFlag, background: backgroundFlag, offline: offlineFlag, reset: resetFlag, @@ -298,7 +311,8 @@ export const start = Command.make("start", { } // Detached: spawn the same command without --background and exit. - if (a.background) return yield* startDetached(a.port, dataDir, a.offline) + if (a.background) + return yield* startDetached(a.port, dataDir, a.offline, Option.getOrUndefined(a.chdbConfigFile)) yield* Effect.sync(() => process.stderr.write( @@ -325,9 +339,12 @@ export const start = Command.make("start", { }), ) - const { port: boundPort } = yield* startServer({ port: a.port, dataDir, assets }).pipe( - Effect.mapError((e) => new ServerError({ message: `failed to start: ${e.message}` })), - ) + const { port: boundPort } = yield* startServer({ + port: a.port, + dataDir, + configFile: Option.getOrUndefined(a.chdbConfigFile), + assets, + }).pipe(Effect.mapError((e) => new ServerError({ message: `failed to start: ${e.message}` }))) started = true // Bootstrap succeeded — stamp the store so a later start over an diff --git a/apps/cli/src/server/chdb.ts b/apps/cli/src/server/chdb.ts index c13613f73..840fc1caf 100644 --- a/apps/cli/src/server/chdb.ts +++ b/apps/cli/src/server/chdb.ts @@ -81,6 +81,8 @@ export interface ChdbOptions { readonly dataDir: string /** Full DDL applied once at open (idempotent `IF NOT EXISTS`). */ readonly schemaSql: string + /** Optional ClickHouse config file passed through to chDB. */ + readonly configFile?: string } /** @@ -117,6 +119,7 @@ export class Chdb { "--async_load_databases=0", "--async_load_system_database=0", `--path=${options.dataDir}`, + ...(options.configFile ? [`--config-file=${options.configFile}`] : []), ] const argBufs = args.map(cstr) const argv = new BigUint64Array(args.length) diff --git a/apps/cli/src/server/serve.ts b/apps/cli/src/server/serve.ts index 05ee77908..5bef6986a 100644 --- a/apps/cli/src/server/serve.ts +++ b/apps/cli/src/server/serve.ts @@ -26,6 +26,7 @@ export interface AssetResolver { export interface ServerOptions { readonly port: number readonly dataDir: string + readonly configFile?: string /** Serves the bundled SPA; omit to disable the UI (API-only). */ readonly assets?: AssetResolver } @@ -335,7 +336,11 @@ export const startServer = ( options: ServerOptions, ): Effect.Effect<{ readonly port: number }, ChdbError, Scope.Scope> => Effect.gen(function* () { - const db = yield* acquireChdb({ dataDir: options.dataDir, schemaSql }) + const db = yield* acquireChdb({ + dataDir: options.dataDir, + schemaSql, + configFile: options.configFile, + }) // A dedicated runtime carrying the OTel tracer for per-request spans: the // Bun.serve handler runs outside Effect, so each request's span effect is // run through this runtime. Disposed on scope close, which flushes any diff --git a/apps/landing/src/content/docs/local-mode/cli-reference.md b/apps/landing/src/content/docs/local-mode/cli-reference.md index fc3655e37..81b3a0406 100644 --- a/apps/landing/src/content/docs/local-mode/cli-reference.md +++ b/apps/landing/src/content/docs/local-mode/cli-reference.md @@ -44,6 +44,7 @@ Start the local ingest + query server (embedded ClickHouse via chDB). | -------------------- | --------------- | ----------------------------------------------------------------------------------- | | `--port ` | `4318` | Port for OTLP/HTTP ingest, the query API, and the bundled UI | | `--data-dir ` | `~/.maple/data` | Embedded ClickHouse data directory | +| `--chdb-config-file ` | | Optional ClickHouse config file passed to embedded chDB | | `--offline` | `false` | Serve the UI bundled in this binary (from `127.0.0.1`) instead of `local.maple.dev` | | `--background`, `-d` | `false` | Run detached (logs to `~/.maple/maple.log`); stop with `maple stop` | | `--reset` | `false` | Wipe the existing store before starting — use after an incompatible upgrade | From 8be7e3ed0cfcdcab5852002480d8d91f45545640 Mon Sep 17 00:00:00 2001 From: remote service administrator Date: Thu, 25 Jun 2026 19:04:48 -0400 Subject: [PATCH 02/12] feat(cli): add local chDB checkpoint command --- apps/cli/src/cli.ts | 3 +- apps/cli/src/commands/server.ts | 23 ++ apps/cli/src/server/chdb.ts | 4 +- apps/cli/src/server/checkpoints.ts | 201 ++++++++++++++++++ .../content/docs/local-mode/cli-reference.md | 23 ++ 5 files changed, 252 insertions(+), 2 deletions(-) create mode 100644 apps/cli/src/server/checkpoints.ts diff --git a/apps/cli/src/cli.ts b/apps/cli/src/cli.ts index 404ca8ca0..88be3904a 100644 --- a/apps/cli/src/cli.ts +++ b/apps/cli/src/cli.ts @@ -9,7 +9,7 @@ import { metrics, query } from "./commands/data" import { timeseries, breakdown, compare } from "./commands/analytics" import { login, logout, whoami } from "./commands/auth" import { use } from "./commands/config" -import { start, stop, reset } from "./commands/server" +import { start, stop, reset, checkpoint } from "./commands/server" import { update } from "./commands/update" // One CLI, two backends. Every query command bottoms out at the shared @@ -46,6 +46,7 @@ export const cli = Command.make("maple").pipe( start, stop, reset, + checkpoint, // Self-update update, // Services diff --git a/apps/cli/src/commands/server.ts b/apps/cli/src/commands/server.ts index 20442905d..434ee57c8 100644 --- a/apps/cli/src/commands/server.ts +++ b/apps/cli/src/commands/server.ts @@ -15,6 +15,7 @@ import { storeMarkerPath, storeOpenMarkerPath, } from "../server/store-version" +import { createCheckpoint } from "../server/checkpoints" import { resolveUiAssets } from "../server/ui-assets" import { amber, bold, cyan, dim, green, underline } from "../lib/style" import { MAPLE_VERSION } from "../version" @@ -460,3 +461,25 @@ export const reset = Command.make("reset", { dataDir: dataDirFlag, yes: yesFlag }), ), ) + +export const checkpoint = Command.make("checkpoint", { dataDir: dataDirFlag, port }).pipe( + Command.withDescription("Create and validate a restorable checkpoint of the local chDB store"), + Command.withHandler( + Effect.fnUntraced(function* (a) { + const dataDir = Option.getOrUndefined(a.dataDir) ?? defaultDataDir() + const result = yield* createCheckpoint({ dataDir, port: a.port }).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) + yield* Effect.sync(() => + process.stdout.write( + `${green("✓")} checkpoint created\n` + + ` ${dim("path")} ${prettyPath(result.path)}\n` + + ` ${dim("traces")} ${result.manifest.validation.traces}\n` + + ` ${dim("logs")} ${result.manifest.validation.logs}\n` + + ` ${dim("metrics")} ${result.manifest.validation.metricsSum}\n` + + ` ${dim("views")} ${result.manifest.validation.materializedViews}\n`, + ), + ) + }), + ), +) diff --git a/apps/cli/src/server/chdb.ts b/apps/cli/src/server/chdb.ts index 840fc1caf..8d8caeafc 100644 --- a/apps/cli/src/server/chdb.ts +++ b/apps/cli/src/server/chdb.ts @@ -83,6 +83,8 @@ export interface ChdbOptions { readonly schemaSql: string /** Optional ClickHouse config file passed through to chDB. */ readonly configFile?: string + /** Apply the Maple schema after connect. Defaults to true. */ + readonly bootstrapSchema?: boolean } /** @@ -135,7 +137,7 @@ export class Chdb { throw new Error(Chdb.#connectFailure(options.dataDir, "chdb_connect produced a NULL connection")) const db = new Chdb(sym, connPtrPtr, conn) - db.#bootstrap(options.schemaSql) + if (options.bootstrapSchema !== false) db.#bootstrap(options.schemaSql) return db } diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts new file mode 100644 index 000000000..ce0f82d83 --- /dev/null +++ b/apps/cli/src/server/checkpoints.ts @@ -0,0 +1,201 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { readdir, rename, rm, stat, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { basename, join, resolve, sep } from "node:path" +import { Effect, Schema } from "effect" +import { Chdb } from "./chdb" +import { SCHEMA_FINGERPRINT } from "./serve" +import schemaSql from "./schema/local-schema.sql" with { type: "text" } +import { CHDB_VERSION, MAPLE_VERSION } from "../version" + +export class CheckpointError extends Schema.TaggedErrorClass()( + "@maple/cli/CheckpointError", + { message: Schema.String }, +) {} + +export interface CheckpointOptions { + readonly dataDir: string + readonly port: number +} + +const checkpointRoot = (dataDir: string): string => join(dataDir, "backups") +const buildingDir = (dataDir: string): string => join(checkpointRoot(dataDir), "building") +const currentDir = (dataDir: string): string => join(checkpointRoot(dataDir), "current") +const previousDir = (dataDir: string): string => join(checkpointRoot(dataDir), "previous") + +const backupSqlPath = (name: string): string => `backups/${name}/backup` + +const xmlEscape = (value: string): string => + value + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'") + +const dataDirWithSlash = (dataDir: string): string => { + const abs = resolve(dataDir) + return abs.endsWith(sep) ? abs : `${abs}${sep}` +} + +const writeBackupConfig = (path: string, sourceDataDir?: string): void => { + const sourceDisk = sourceDataDir + ? ` + + + + ${xmlEscape(dataDirWithSlash(sourceDataDir))} + + + ` + : "" + writeFileSync( + path, + ` + + ${sourceDataDir ? "src" : "default"} + backups + ${sourceDisk} + +`, + ) +} + +const postLocalQuery = async (port: number, sql: string): Promise => { + const response = await fetch(`http://127.0.0.1:${port}/local/query`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ sql }), + }) + if (!response.ok) { + const detail = await response.text().catch(() => "") + throw new Error(`local query failed (${response.status} ${response.statusText})${detail ? `: ${detail}` : ""}`) + } + return response.json() +} + +const readJsonRows = (text: string): ReadonlyArray> => + text + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as Record) + +const countFrom = (rows: ReadonlyArray>): number => { + const row = rows[0] + if (!row) return 0 + const value = row["count()"] ?? row.count + return typeof value === "number" ? value : Number(value ?? 0) +} + +const queryCount = (db: Chdb, sql: string): number => countFrom(readJsonRows(db.query(sql))) + +const dirSize = async (path: string): Promise => { + let total = 0 + const entries = await readdir(path, { withFileTypes: true }) + for (const entry of entries) { + const child = join(path, entry.name) + if (entry.isDirectory()) { + total += await dirSize(child) + } else if (entry.isFile()) { + total += (await stat(child)).size + } + } + return total +} + +interface CheckpointManifest { + readonly mapleVersion: string + readonly chdbVersion: string + readonly schemaFingerprint: string + readonly createdAt: string + readonly sourceDataDir: string + readonly backupPath: string + readonly backupBytes: number + readonly validation: { + readonly validatedAt: string + readonly traces: number + readonly logs: number + readonly metricsSum: number + readonly materializedViews: number + } +} + +const validateBackup = async (dataDir: string): Promise => { + const scratchParent = mkdtempSync(join(tmpdir(), "maple-checkpoint-")) + const scratchData = join(scratchParent, "data") + const scratchConfig = join(scratchParent, "config.xml") + writeBackupConfig(scratchConfig, dataDir) + let db: Chdb | undefined + try { + db = Chdb.open({ + dataDir: scratchData, + schemaSql, + configFile: scratchConfig, + bootstrapSchema: false, + }) + db.exec("CREATE DATABASE IF NOT EXISTS default") + db.exec( + `RESTORE DATABASE default FROM Disk('src', '${backupSqlPath("building")}') ` + + "SETTINGS allow_different_database_def=1", + ) + return { + validatedAt: new Date().toISOString(), + traces: queryCount(db, "SELECT count() FROM traces"), + logs: queryCount(db, "SELECT count() FROM logs"), + metricsSum: queryCount(db, "SELECT count() FROM metrics_sum"), + materializedViews: queryCount( + db, + "SELECT count() FROM system.tables WHERE database = 'default' AND engine = 'MaterializedView'", + ), + } + } finally { + db?.close() + rmSync(scratchParent, { recursive: true, force: true }) + } +} + +const promoteBuilding = async (dataDir: string): Promise => { + await rm(previousDir(dataDir), { recursive: true, force: true }) + try { + await rename(currentDir(dataDir), previousDir(dataDir)) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + } + await rename(buildingDir(dataDir), currentDir(dataDir)) +} + +export const createCheckpoint = ( + options: CheckpointOptions, +): Effect.Effect<{ readonly path: string; readonly manifest: CheckpointManifest }, CheckpointError> => + Effect.tryPromise({ + try: async () => { + const root = checkpointRoot(options.dataDir) + const building = buildingDir(options.dataDir) + const name = basename(building) + if (name !== "building") throw new Error("internal checkpoint path error") + await rm(building, { recursive: true, force: true }) + + await postLocalQuery( + options.port, + `BACKUP DATABASE default TO Disk('default', '${backupSqlPath("building")}')`, + ) + + const validation = await validateBackup(options.dataDir) + const manifest: CheckpointManifest = { + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + createdAt: new Date().toISOString(), + sourceDataDir: resolve(options.dataDir), + backupPath: backupSqlPath("current"), + backupBytes: await dirSize(join(building, "backup")), + validation, + } + await writeFile(join(building, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`) + await promoteBuilding(options.dataDir) + return { path: join(root, "current"), manifest: { ...manifest, backupPath: backupSqlPath("current") } } + }, + catch: (error) => + new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), + }) diff --git a/apps/landing/src/content/docs/local-mode/cli-reference.md b/apps/landing/src/content/docs/local-mode/cli-reference.md index 81b3a0406..3f5bfaa95 100644 --- a/apps/landing/src/content/docs/local-mode/cli-reference.md +++ b/apps/landing/src/content/docs/local-mode/cli-reference.md @@ -198,6 +198,29 @@ maple query "SELECT ServiceName, count() FROM traces GROUP BY ServiceName ORDER > **Local only.** Raw SQL against the multi-tenant cloud warehouse would let a client read other orgs' data, so `maple query` returns a clear error in remote mode. Every other command works in both modes. +### `maple checkpoint` + +Create and validate a restorable checkpoint of the local chDB store. The running +server must have been started with a chDB config that allows ClickHouse backups: + +```xml + + + default + backups + + +``` + +```bash +maple start --chdb-config-file ./chdb-backups.xml +maple checkpoint +``` + +Checkpoints are written under the data directory at +`backups/{building,current,previous}`. `building` is never used for restore; +only a validated checkpoint is promoted to `current`. + ## Analytics ### `maple timeseries` From 19c053064328d85ede4012efcf0483aa5045d5b2 Mon Sep 17 00:00:00 2001 From: remote service administrator Date: Thu, 25 Jun 2026 19:07:18 -0400 Subject: [PATCH 03/12] feat(cli): restore dirty local store from checkpoint --- apps/cli/src/cli.ts | 3 +- apps/cli/src/commands/server.ts | 115 +++++++++++++++--- apps/cli/src/server/checkpoints.ts | 79 +++++++++++- .../content/docs/local-mode/cli-reference.md | 58 +++++---- 4 files changed, 209 insertions(+), 46 deletions(-) diff --git a/apps/cli/src/cli.ts b/apps/cli/src/cli.ts index 88be3904a..8f8aab021 100644 --- a/apps/cli/src/cli.ts +++ b/apps/cli/src/cli.ts @@ -9,7 +9,7 @@ import { metrics, query } from "./commands/data" import { timeseries, breakdown, compare } from "./commands/analytics" import { login, logout, whoami } from "./commands/auth" import { use } from "./commands/config" -import { start, stop, reset, checkpoint } from "./commands/server" +import { start, stop, reset, checkpoint, restore } from "./commands/server" import { update } from "./commands/update" // One CLI, two backends. Every query command bottoms out at the shared @@ -47,6 +47,7 @@ export const cli = Command.make("maple").pipe( stop, reset, checkpoint, + restore, // Self-update update, // Services diff --git a/apps/cli/src/commands/server.ts b/apps/cli/src/commands/server.ts index 434ee57c8..9c3c4cb09 100644 --- a/apps/cli/src/commands/server.ts +++ b/apps/cli/src/commands/server.ts @@ -15,7 +15,7 @@ import { storeMarkerPath, storeOpenMarkerPath, } from "../server/store-version" -import { createCheckpoint } from "../server/checkpoints" +import { createCheckpoint, restoreCheckpoint } from "../server/checkpoints" import { resolveUiAssets } from "../server/ui-assets" import { amber, bold, cyan, dim, green, underline } from "../lib/style" import { MAPLE_VERSION } from "../version" @@ -124,6 +124,11 @@ const resetFlag = Flag.boolean("reset").pipe( Flag.withDefault(false), ) +const onDirtyStoreFlag = Flag.choice("on-dirty-store", ["wipe", "fail", "restore-checkpoint"]).pipe( + Flag.withDescription("Recovery policy when the local chDB store was not cleanly closed"), + Flag.withDefault("wipe" as const), +) + const yesFlag = Flag.boolean("yes").pipe( Flag.withAlias("y"), Flag.withDescription("Skip the confirmation prompt"), @@ -231,6 +236,7 @@ export const start = Command.make("start", { background: backgroundFlag, offline: offlineFlag, reset: resetFlag, + onDirtyStore: onDirtyStoreFlag, }).pipe( Command.withDescription("Start the local ingest + query server (embedded ClickHouse via chDB)"), Command.withHandler( @@ -258,6 +264,52 @@ export const start = Command.make("start", { yield* fs.makeDirectory(dataDir, { recursive: true }) + // A store left "open" (the previous server died without running its close + // finalizer) may be inconsistent — reopening it can crash chDB natively, + // which we cannot catch. Auto-wipe and bootstrap fresh instead of walking + // into the crash. (`--reset` already wiped above, so the marker is gone.) + if (isStoreDirty(dataDir)) { + if (a.onDirtyStore === "fail") { + return yield* new ServerError({ + message: + `the local store at ${prettyPath(dataDir)} was not cleanly closed. ` + + `Run \`${bold("maple restore --yes")}\` to restore from the last checkpoint, ` + + `or \`${bold("maple start --reset")}\` to wipe it.`, + }) + } + if (a.onDirtyStore === "restore-checkpoint") { + yield* Effect.sync(() => + process.stderr.write( + amber( + "⚠ the local store was left inconsistent by an unclean shutdown — " + + "restoring the last checkpoint\n", + ), + ), + ) + const restored = yield* restoreCheckpoint(dataDir).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) + yield* Effect.sync(() => + process.stderr.write( + `${green("✓")} restored checkpoint; quarantined dirty store at ${prettyPath(restored.quarantinePath)}\n`, + ), + ) + } else { + yield* Effect.sync(() => + process.stderr.write( + amber( + "⚠ the local store was left inconsistent by an unclean shutdown — " + + "wiping it and starting fresh (local telemetry data is discarded)\n", + ), + ), + ) + yield* fs.remove(dataDir, { recursive: true, force: true }).pipe(Effect.ignore) + yield* fs.remove(storeMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) + yield* fs.remove(storeOpenMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) + yield* fs.makeDirectory(dataDir, { recursive: true }) + } + } + // Refuse to open a store written by an incompatible chDB build: re-loading // its persisted materialized views crashes the C++ runtime natively // (SIGTRAP), which we cannot catch. Fresh/matching stores pass through. @@ -271,25 +323,6 @@ export const start = Command.make("start", { }) } - // A store left "open" (the previous server died without running its close - // finalizer) may be inconsistent — reopening it can crash chDB natively, - // which we cannot catch. Auto-wipe and bootstrap fresh instead of walking - // into the crash. (`--reset` already wiped above, so the marker is gone.) - if (isStoreDirty(dataDir)) { - yield* Effect.sync(() => - process.stderr.write( - amber( - "⚠ the local store was left inconsistent by an unclean shutdown — " + - "wiping it and starting fresh (local telemetry data is discarded)\n", - ), - ), - ) - yield* fs.remove(dataDir, { recursive: true, force: true }).pipe(Effect.ignore) - yield* fs.remove(storeMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.remove(storeOpenMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.makeDirectory(dataDir, { recursive: true }) - } - // A store bootstrapped from an older bundled schema can't be evolved in // place: `CREATE … IF NOT EXISTS` is a no-op on existing tables, so a // column added to the schema (e.g. ServiceNamespace on trace_list_mv) @@ -483,3 +516,45 @@ export const checkpoint = Command.make("checkpoint", { dataDir: dataDirFlag, por }), ), ) + +export const restore = Command.make("restore", { dataDir: dataDirFlag, yes: yesFlag }).pipe( + Command.withDescription("Restore the local chDB store from the last promoted checkpoint"), + Command.withHandler( + Effect.fnUntraced(function* (a) { + const fs = yield* FileSystem + const dataDir = Option.getOrUndefined(a.dataDir) ?? defaultDataDir() + + const pidOpt = yield* readPid(fs, pidFilePath(dataDir)) + if (Option.isSome(pidOpt) && isProcessAlive(pidOpt.value)) { + return yield* new ServerError({ + message: `maple is running (PID ${pidOpt.value}) — stop it first with \`maple stop\``, + }) + } + + if (!a.yes) { + yield* Effect.sync(() => + process.stderr.write( + `This replaces the local store at ${bold(prettyPath(dataDir))} with the last checkpoint.\n` + + `The existing store is moved aside for quarantine, not deleted.\n` + + `Re-run with ${bold("maple restore --yes")} to confirm.\n`, + ), + ) + return + } + + const result = yield* restoreCheckpoint(dataDir).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) + yield* Effect.sync(() => + process.stderr.write( + `${green("✓")} restored checkpoint\n` + + ` ${dim("quarantine")} ${prettyPath(result.quarantinePath)}\n` + + ` ${dim("traces")} ${result.validation.traces}\n` + + ` ${dim("logs")} ${result.validation.logs}\n` + + ` ${dim("metrics")} ${result.validation.metricsSum}\n` + + ` ${dim("views")} ${result.validation.materializedViews}\n`, + ), + ) + }), + ), +) diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index ce0f82d83..5cb54c15f 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -1,11 +1,12 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs" -import { readdir, rename, rm, stat, writeFile } from "node:fs/promises" +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { cp, readdir, rename, rm, stat, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import { basename, join, resolve, sep } from "node:path" import { Effect, Schema } from "effect" import { Chdb } from "./chdb" import { SCHEMA_FINGERPRINT } from "./serve" import schemaSql from "./schema/local-schema.sql" with { type: "text" } +import { markStoreClosed, storeMarkerJson, storeMarkerPath } from "./store-version" import { CHDB_VERSION, MAPLE_VERSION } from "../version" export class CheckpointError extends Schema.TaggedErrorClass()( @@ -22,6 +23,8 @@ const checkpointRoot = (dataDir: string): string => join(dataDir, "backups") const buildingDir = (dataDir: string): string => join(checkpointRoot(dataDir), "building") const currentDir = (dataDir: string): string => join(checkpointRoot(dataDir), "current") const previousDir = (dataDir: string): string => join(checkpointRoot(dataDir), "previous") +const restoreBuildingDir = (dataDir: string): string => `${dataDir}.restore-building` +const quarantineDir = (dataDir: string): string => `${dataDir}.quarantine-${new Date().toISOString().replace(/[:.]/g, "-")}` const backupSqlPath = (name: string): string => `backups/${name}/backup` @@ -155,6 +158,43 @@ const validateBackup = async (dataDir: string): Promise => { + const scratchParent = mkdtempSync(join(tmpdir(), "maple-restore-")) + const restoreConfig = join(scratchParent, "config.xml") + writeBackupConfig(restoreConfig, sourceDataDir) + let db: Chdb | undefined + try { + db = Chdb.open({ + dataDir: targetDataDir, + schemaSql, + configFile: restoreConfig, + bootstrapSchema: false, + }) + db.exec("CREATE DATABASE IF NOT EXISTS default") + db.exec( + `RESTORE DATABASE default FROM Disk('src', '${backupSqlPath(checkpointName)}') ` + + "SETTINGS allow_different_database_def=1", + ) + return { + validatedAt: new Date().toISOString(), + traces: queryCount(db, "SELECT count() FROM traces"), + logs: queryCount(db, "SELECT count() FROM logs"), + metricsSum: queryCount(db, "SELECT count() FROM metrics_sum"), + materializedViews: queryCount( + db, + "SELECT count() FROM system.tables WHERE database = 'default' AND engine = 'MaterializedView'", + ), + } + } finally { + db?.close() + rmSync(scratchParent, { recursive: true, force: true }) + } +} + const promoteBuilding = async (dataDir: string): Promise => { await rm(previousDir(dataDir), { recursive: true, force: true }) try { @@ -199,3 +239,38 @@ export const createCheckpoint = ( catch: (error) => new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), }) + +export const restoreCheckpoint = ( + dataDir: string, +): Effect.Effect<{ readonly quarantinePath: string; readonly validation: CheckpointManifest["validation"] }, CheckpointError> => + Effect.tryPromise({ + try: async () => { + const sourceBackup = join(currentDir(dataDir), "backup") + if (!existsSync(sourceBackup)) { + throw new Error(`no checkpoint found at ${sourceBackup}`) + } + + const restoreDir = restoreBuildingDir(dataDir) + await rm(restoreDir, { recursive: true, force: true }) + const validation = await restoreIntoScratch(dataDir, restoreDir, "current") + + if (existsSync(join(dataDir, "backups"))) { + await cp(join(dataDir, "backups"), join(restoreDir, "backups"), { + recursive: true, + force: true, + }) + } + + const quarantinePath = quarantineDir(dataDir) + await rename(dataDir, quarantinePath) + await rename(restoreDir, dataDir) + markStoreClosed(dataDir) + writeFileSync( + storeMarkerPath(dataDir), + storeMarkerJson(MAPLE_VERSION, new Date().toISOString(), SCHEMA_FINGERPRINT), + ) + return { quarantinePath, validation } + }, + catch: (error) => + new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), + }) diff --git a/apps/landing/src/content/docs/local-mode/cli-reference.md b/apps/landing/src/content/docs/local-mode/cli-reference.md index 3f5bfaa95..1c325b11b 100644 --- a/apps/landing/src/content/docs/local-mode/cli-reference.md +++ b/apps/landing/src/content/docs/local-mode/cli-reference.md @@ -48,6 +48,7 @@ Start the local ingest + query server (embedded ClickHouse via chDB). | `--offline` | `false` | Serve the UI bundled in this binary (from `127.0.0.1`) instead of `local.maple.dev` | | `--background`, `-d` | `false` | Run detached (logs to `~/.maple/maple.log`); stop with `maple stop` | | `--reset` | `false` | Wipe the existing store before starting — use after an incompatible upgrade | +| `--on-dirty-store ` | `wipe` | Recovery policy when the store was not cleanly closed | ```bash maple start # foreground, UI from local.maple.dev @@ -72,6 +73,40 @@ Delete the local chDB store so the next `maple start` bootstraps fresh. Refuses | `--data-dir ` | `~/.maple/data` | Store to delete | | `--yes`, `-y` | `false` | Skip the confirmation prompt | +### `maple checkpoint` + +Create and validate a restorable checkpoint of the local chDB store. The running +server must have been started with a chDB config that allows ClickHouse backups: + +```xml + + + default + backups + + +``` + +```bash +maple start --chdb-config-file ./chdb-backups.xml +maple checkpoint +``` + +Checkpoints are written under the data directory at +`backups/{building,current,previous}`. `building` is never used for restore; +only a validated checkpoint is promoted to `current`. + +### `maple restore` + +Restore the local chDB store from the last promoted checkpoint. Refuses to run +while a server still owns the store. The existing store is moved aside for +quarantine rather than deleted. + +| Flag | Default | Description | +| ------------------- | --------------- | ---------------------------- | +| `--data-dir ` | `~/.maple/data` | Store to restore | +| `--yes`, `-y` | `false` | Skip the confirmation prompt | + ## Services ### `maple services` @@ -198,29 +233,6 @@ maple query "SELECT ServiceName, count() FROM traces GROUP BY ServiceName ORDER > **Local only.** Raw SQL against the multi-tenant cloud warehouse would let a client read other orgs' data, so `maple query` returns a clear error in remote mode. Every other command works in both modes. -### `maple checkpoint` - -Create and validate a restorable checkpoint of the local chDB store. The running -server must have been started with a chDB config that allows ClickHouse backups: - -```xml - - - default - backups - - -``` - -```bash -maple start --chdb-config-file ./chdb-backups.xml -maple checkpoint -``` - -Checkpoints are written under the data directory at -`backups/{building,current,previous}`. `building` is never used for restore; -only a validated checkpoint is promoted to `current`. - ## Analytics ### `maple timeseries` From 3fa344f34cfabc5e61cfcb278fe6c8bf99aefe81 Mon Sep 17 00:00:00 2001 From: remote service administrator Date: Fri, 26 Jun 2026 11:29:01 -0400 Subject: [PATCH 04/12] fix(cli): harden chDB checkpoint recovery --- apps/cli/src/commands/server.ts | 37 +++++--- apps/cli/src/server/checkpoints.ts | 76 ++++++++++++--- apps/cli/test/checkpoints.test.ts | 143 +++++++++++++++++++++++++++++ 3 files changed, 226 insertions(+), 30 deletions(-) create mode 100644 apps/cli/test/checkpoints.test.ts diff --git a/apps/cli/src/commands/server.ts b/apps/cli/src/commands/server.ts index 9c3c4cb09..2c72573b7 100644 --- a/apps/cli/src/commands/server.ts +++ b/apps/cli/src/commands/server.ts @@ -264,6 +264,19 @@ export const start = Command.make("start", { yield* fs.makeDirectory(dataDir, { recursive: true }) + // Refuse to open a store written by an incompatible chDB build: re-loading + // its persisted materialized views crashes the C++ runtime natively + // (SIGTRAP), which we cannot catch. Fresh/matching stores pass through. + const compat = checkStoreCompatible(dataDir) + if (!compat.compatible) { + return yield* new ServerError({ + message: + `the local store at ${prettyPath(dataDir)} is incompatible with this build's chDB ` + + `(store: ${compat.found}; build: ${compat.current}) — loading it would crash chDB. ` + + `Wipe it with \`${bold("maple reset")}\`, or start fresh via \`${bold("maple start --reset")}\`.`, + }) + } + // A store left "open" (the previous server died without running its close // finalizer) may be inconsistent — reopening it can crash chDB natively, // which we cannot catch. Auto-wipe and bootstrap fresh instead of walking @@ -310,19 +323,6 @@ export const start = Command.make("start", { } } - // Refuse to open a store written by an incompatible chDB build: re-loading - // its persisted materialized views crashes the C++ runtime natively - // (SIGTRAP), which we cannot catch. Fresh/matching stores pass through. - const compat = checkStoreCompatible(dataDir) - if (!compat.compatible) { - return yield* new ServerError({ - message: - `the local store at ${prettyPath(dataDir)} is incompatible with this build's chDB ` + - `(store: ${compat.found}; build: ${compat.current}) — loading it would crash chDB. ` + - `Wipe it with \`${bold("maple reset")}\`, or start fresh via \`${bold("maple start --reset")}\`.`, - }) - } - // A store bootstrapped from an older bundled schema can't be evolved in // place: `CREATE … IF NOT EXISTS` is a no-op on existing tables, so a // column added to the schema (e.g. ServiceNamespace on trace_list_mv) @@ -346,7 +346,12 @@ export const start = Command.make("start", { // Detached: spawn the same command without --background and exit. if (a.background) - return yield* startDetached(a.port, dataDir, a.offline, Option.getOrUndefined(a.chdbConfigFile)) + return yield* startDetached( + a.port, + dataDir, + a.offline, + Option.getOrUndefined(a.chdbConfigFile), + ) yield* Effect.sync(() => process.stderr.write( @@ -378,7 +383,9 @@ export const start = Command.make("start", { dataDir, configFile: Option.getOrUndefined(a.chdbConfigFile), assets, - }).pipe(Effect.mapError((e) => new ServerError({ message: `failed to start: ${e.message}` }))) + }).pipe( + Effect.mapError((e) => new ServerError({ message: `failed to start: ${e.message}` })), + ) started = true // Bootstrap succeeded — stamp the store so a later start over an diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index 5cb54c15f..ab4b683d0 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -1,5 +1,5 @@ import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" -import { cp, readdir, rename, rm, stat, writeFile } from "node:fs/promises" +import { cp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import { basename, join, resolve, sep } from "node:path" import { Effect, Schema } from "effect" @@ -19,12 +19,13 @@ export interface CheckpointOptions { readonly port: number } -const checkpointRoot = (dataDir: string): string => join(dataDir, "backups") -const buildingDir = (dataDir: string): string => join(checkpointRoot(dataDir), "building") -const currentDir = (dataDir: string): string => join(checkpointRoot(dataDir), "current") -const previousDir = (dataDir: string): string => join(checkpointRoot(dataDir), "previous") +export const checkpointRoot = (dataDir: string): string => join(dataDir, "backups") +export const buildingDir = (dataDir: string): string => join(checkpointRoot(dataDir), "building") +export const currentDir = (dataDir: string): string => join(checkpointRoot(dataDir), "current") +export const previousDir = (dataDir: string): string => join(checkpointRoot(dataDir), "previous") const restoreBuildingDir = (dataDir: string): string => `${dataDir}.restore-building` -const quarantineDir = (dataDir: string): string => `${dataDir}.quarantine-${new Date().toISOString().replace(/[:.]/g, "-")}` +const quarantineDir = (dataDir: string): string => + `${dataDir}.quarantine-${new Date().toISOString().replace(/[:.]/g, "-")}` const backupSqlPath = (name: string): string => `backups/${name}/backup` @@ -41,7 +42,7 @@ const dataDirWithSlash = (dataDir: string): string => { return abs.endsWith(sep) ? abs : `${abs}${sep}` } -const writeBackupConfig = (path: string, sourceDataDir?: string): void => { +export const writeBackupConfig = (path: string, sourceDataDir?: string): void => { const sourceDisk = sourceDataDir ? ` @@ -72,7 +73,9 @@ const postLocalQuery = async (port: number, sql: string): Promise => { }) if (!response.ok) { const detail = await response.text().catch(() => "") - throw new Error(`local query failed (${response.status} ${response.statusText})${detail ? `: ${detail}` : ""}`) + throw new Error( + `local query failed (${response.status} ${response.statusText})${detail ? `: ${detail}` : ""}`, + ) } return response.json() } @@ -124,6 +127,28 @@ interface CheckpointManifest { } } +export const readCheckpointManifest = async (dataDir: string): Promise => { + const path = join(currentDir(dataDir), "manifest.json") + let raw: string + try { + raw = await readFile(path, "utf8") + } catch { + throw new Error(`checkpoint manifest not found at ${path}`) + } + const parsed = JSON.parse(raw) as Partial + if (parsed.chdbVersion !== CHDB_VERSION) { + throw new Error( + `checkpoint chDB version mismatch (checkpoint: ${parsed.chdbVersion ?? "unknown"}; build: ${CHDB_VERSION})`, + ) + } + if (parsed.schemaFingerprint !== SCHEMA_FINGERPRINT) { + throw new Error( + `checkpoint schema mismatch (checkpoint: ${parsed.schemaFingerprint ?? "unknown"}; build: ${SCHEMA_FINGERPRINT})`, + ) + } + return parsed as CheckpointManifest +} + const validateBackup = async (dataDir: string): Promise => { const scratchParent = mkdtempSync(join(tmpdir(), "maple-checkpoint-")) const scratchData = join(scratchParent, "data") @@ -195,7 +220,7 @@ const restoreIntoScratch = async ( } } -const promoteBuilding = async (dataDir: string): Promise => { +export const promoteBuilding = async (dataDir: string): Promise => { await rm(previousDir(dataDir), { recursive: true, force: true }) try { await rename(currentDir(dataDir), previousDir(dataDir)) @@ -216,10 +241,24 @@ export const createCheckpoint = ( if (name !== "building") throw new Error("internal checkpoint path error") await rm(building, { recursive: true, force: true }) - await postLocalQuery( - options.port, - `BACKUP DATABASE default TO Disk('default', '${backupSqlPath("building")}')`, - ) + try { + await postLocalQuery( + options.port, + `BACKUP DATABASE default TO Disk('default', '${backupSqlPath("building")}')`, + ) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + if ( + message.includes("backups.allowed_disk") || + message.includes("INVALID_CONFIG_PARAMETER") + ) { + throw new Error( + "checkpoints require the local server to be started with `--chdb-config-file` " + + "pointing at a ClickHouse backups config", + ) + } + throw error + } const validation = await validateBackup(options.dataDir) const manifest: CheckpointManifest = { @@ -234,7 +273,10 @@ export const createCheckpoint = ( } await writeFile(join(building, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`) await promoteBuilding(options.dataDir) - return { path: join(root, "current"), manifest: { ...manifest, backupPath: backupSqlPath("current") } } + return { + path: join(root, "current"), + manifest: { ...manifest, backupPath: backupSqlPath("current") }, + } }, catch: (error) => new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), @@ -242,13 +284,17 @@ export const createCheckpoint = ( export const restoreCheckpoint = ( dataDir: string, -): Effect.Effect<{ readonly quarantinePath: string; readonly validation: CheckpointManifest["validation"] }, CheckpointError> => +): Effect.Effect< + { readonly quarantinePath: string; readonly validation: CheckpointManifest["validation"] }, + CheckpointError +> => Effect.tryPromise({ try: async () => { const sourceBackup = join(currentDir(dataDir), "backup") if (!existsSync(sourceBackup)) { throw new Error(`no checkpoint found at ${sourceBackup}`) } + await readCheckpointManifest(dataDir) const restoreDir = restoreBuildingDir(dataDir) await rm(restoreDir, { recursive: true, force: true }) diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts new file mode 100644 index 000000000..dca5813ae --- /dev/null +++ b/apps/cli/test/checkpoints.test.ts @@ -0,0 +1,143 @@ +import { describe, it } from "@effect/vitest" +import { ok, rejects, strictEqual } from "node:assert" +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { + buildingDir, + currentDir, + previousDir, + promoteBuilding, + readCheckpointManifest, + writeBackupConfig, +} from "../src/server/checkpoints" +import { SCHEMA_FINGERPRINT } from "../src/server/serve" +import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" + +const withDataDir = async (run: (dataDir: string) => Promise | void): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-checkpoint-test-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir, { recursive: true }) + try { + await run(dataDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const writeMarker = (path: string, value: string): void => { + mkdirSync(path, { recursive: true }) + writeFileSync(join(path, "marker.txt"), value) +} + +const readMarker = (path: string): string => readFileSync(join(path, "marker.txt"), "utf8") + +const manifest = (overrides: Record = {}): string => + `${JSON.stringify( + { + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + createdAt: "2026-01-01T00:00:00.000Z", + sourceDataDir: "/tmp/maple-data", + backupPath: "backups/current/backup", + backupBytes: 123, + validation: { + validatedAt: "2026-01-01T00:00:01.000Z", + traces: 1, + logs: 2, + metricsSum: 3, + materializedViews: 33, + }, + ...overrides, + }, + null, + 2, + )}\n` + +describe("writeBackupConfig", () => { + it("writes the runtime backup config for the default disk", async () => { + await withDataDir((dataDir) => { + const configPath = join(dataDir, "config.xml") + writeBackupConfig(configPath) + + const xml = readFileSync(configPath, "utf8") + ok(xml.includes("default")) + ok(xml.includes("backups")) + ok(!xml.includes("")) + }) + }) + + it("writes a restore config with an escaped source disk path", async () => { + await withDataDir((dataDir) => { + const configPath = join(dataDir, "config.xml") + const sourceDataDir = join(dataDir, "source & ") + writeBackupConfig(configPath, sourceDataDir) + + const xml = readFileSync(configPath, "utf8") + ok(xml.includes("src")) + ok(xml.includes("backups")) + ok(xml.includes("")) + ok(xml.includes("source & <store>")) + ok(xml.includes("")) + }) + }) +}) + +describe("promoteBuilding", () => { + it("promotes building to current when no current checkpoint exists", async () => { + await withDataDir(async (dataDir) => { + writeMarker(buildingDir(dataDir), "new") + + await promoteBuilding(dataDir) + + ok(!existsSync(buildingDir(dataDir))) + strictEqual(readMarker(currentDir(dataDir)), "new") + ok(!existsSync(previousDir(dataDir))) + }) + }) + + it("moves current to previous and replaces any older previous checkpoint", async () => { + await withDataDir(async (dataDir) => { + writeMarker(previousDir(dataDir), "old-previous") + writeMarker(currentDir(dataDir), "old-current") + writeMarker(buildingDir(dataDir), "new-current") + + await promoteBuilding(dataDir) + + ok(!existsSync(buildingDir(dataDir))) + strictEqual(readMarker(currentDir(dataDir)), "new-current") + strictEqual(readMarker(previousDir(dataDir)), "old-current") + }) + }) +}) + +describe("readCheckpointManifest", () => { + it("round-trips a compatible checkpoint manifest", async () => { + await withDataDir(async (dataDir) => { + mkdirSync(currentDir(dataDir), { recursive: true }) + writeFileSync(join(currentDir(dataDir), "manifest.json"), manifest()) + + const parsed = await readCheckpointManifest(dataDir) + + strictEqual(parsed.chdbVersion, CHDB_VERSION) + strictEqual(parsed.schemaFingerprint, SCHEMA_FINGERPRINT) + strictEqual(parsed.validation.materializedViews, 33) + }) + }) + + it("rejects a manifest from a different chDB or schema", async () => { + await withDataDir(async (dataDir) => { + mkdirSync(currentDir(dataDir), { recursive: true }) + writeFileSync(join(currentDir(dataDir), "manifest.json"), manifest({ chdbVersion: "v0.0.0" })) + + await rejects(readCheckpointManifest(dataDir), /checkpoint chDB version mismatch/) + + writeFileSync( + join(currentDir(dataDir), "manifest.json"), + manifest({ schemaFingerprint: "old-schema" }), + ) + await rejects(readCheckpointManifest(dataDir), /checkpoint schema mismatch/) + }) + }) +}) From 44c51e84f07d0dbc565abd185e8d44dbda7777bd Mon Sep 17 00:00:00 2001 From: remote service administrator Date: Sat, 27 Jun 2026 12:02:43 -0400 Subject: [PATCH 05/12] fix(cli): make checkpoints immutable and durable --- apps/cli/src/server/checkpoints.ts | 894 ++++++++++++++++++++++----- apps/cli/src/server/durable-files.ts | 112 ++++ apps/cli/test/checkpoints.test.ts | 432 ++++++++++--- 3 files changed, 1189 insertions(+), 249 deletions(-) create mode 100644 apps/cli/src/server/durable-files.ts diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index ab4b683d0..f8fa5b709 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -1,13 +1,27 @@ -import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" -import { cp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises" +import { randomUUID } from "node:crypto" +import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { cp, lstat, mkdir, readFile, readdir, rename, rm, stat } from "node:fs/promises" import { tmpdir } from "node:os" -import { basename, join, resolve, sep } from "node:path" +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path" import { Effect, Schema } from "effect" +import { CHDB_VERSION, MAPLE_VERSION } from "../version" import { Chdb } from "./chdb" +import { + type DurabilityFaults, + durableJson, + durableRename, + ensurePrivateDirectory, + syncDirectory, + syncTree, +} from "./durable-files" import { SCHEMA_FINGERPRINT } from "./serve" import schemaSql from "./schema/local-schema.sql" with { type: "text" } import { markStoreClosed, storeMarkerJson, storeMarkerPath } from "./store-version" -import { CHDB_VERSION, MAPLE_VERSION } from "../version" + +const STATE_FORMAT_VERSION = 1 +const MANIFEST_FORMAT_VERSION = 1 +const OPERATION_FORMAT_VERSION = 1 +const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i export class CheckpointError extends Schema.TaggedErrorClass()( "@maple/cli/CheckpointError", @@ -17,17 +31,153 @@ export class CheckpointError extends Schema.TaggedErrorClass()( export interface CheckpointOptions { readonly dataDir: string readonly port: number + readonly faults?: DurabilityFaults +} + +export interface CheckpointValidation { + readonly validatedAt: string + readonly traces: number + readonly logs: number + readonly metricsSum: number + readonly metricsGauge: number + readonly metricsHistogram: number + readonly metricsExponentialHistogram: number + readonly materializedViews: number +} + +export interface CheckpointManifest { + readonly formatVersion: 1 + readonly checkpointId: string + readonly operationId: string + readonly mapleVersion: string + readonly chdbVersion: string + readonly schemaFingerprint: string + readonly createdAt: string + readonly sourceDataDir: string + readonly backupRelativePath: string + readonly backupBytes: number + readonly validation: CheckpointValidation +} + +export interface CheckpointState { + readonly formatVersion: 1 + readonly revision: string + readonly current: string + readonly previous: string | null + readonly committedAt: string +} + +export interface ResolvedCheckpoint { + readonly checkpointId: string + readonly snapshotDir: string + readonly backupDir: string + readonly backupSqlPath: string + readonly manifest: CheckpointManifest +} + +interface CheckpointOperation { + readonly formatVersion: 1 + readonly operationId: string + readonly checkpointId: string + readonly phase: "intent" | "backup-complete" | "manifest-complete" | "pointer-complete" + readonly startedAt: string +} + +interface MaintenanceOwner { + readonly formatVersion: 1 + readonly operationId: string + readonly pid: number + readonly startedAt: string +} + +export const checkpointRoot = (dataDir: string): string => join(resolve(dataDir), "backups") +export const checkpointStatePath = (dataDir: string): string => join(checkpointRoot(dataDir), "state.json") +export const checkpointSnapshotsRoot = (dataDir: string): string => join(checkpointRoot(dataDir), "snapshots") +export const checkpointOperationsRoot = (dataDir: string): string => + join(checkpointRoot(dataDir), "operations") +export const checkpointPinsRoot = (dataDir: string): string => join(checkpointRoot(dataDir), "pins") +export const checkpointQuarantineRoot = (dataDir: string): string => + join(checkpointRoot(dataDir), "quarantine") +export const checkpointRetiringRoot = (dataDir: string): string => join(checkpointRoot(dataDir), "retiring") +export const checkpointSnapshotDir = (dataDir: string, checkpointId: string): string => + join(checkpointSnapshotsRoot(dataDir), validateId(checkpointId, "checkpoint")) + +const maintenanceLockPath = (dataDir: string): string => `${resolve(dataDir)}.maple-maintenance-lock` +const operationDir = (dataDir: string, operationId: string): string => + join(checkpointOperationsRoot(dataDir), `checkpoint-${validateId(operationId, "operation")}`) +const operationPath = (dataDir: string, operationId: string): string => + join(operationDir(dataDir, operationId), "intent.json") +const snapshotManifestPath = (dataDir: string, checkpointId: string): string => + join(checkpointSnapshotDir(dataDir, checkpointId), "manifest.json") +const snapshotBackupDir = (dataDir: string, checkpointId: string): string => + join(checkpointSnapshotDir(dataDir, checkpointId), "backup") +const snapshotBackupRelativePath = (checkpointId: string): string => + `snapshots/${validateId(checkpointId, "checkpoint")}/backup` +const snapshotBackupSqlPath = (checkpointId: string): string => + `backups/${snapshotBackupRelativePath(checkpointId)}` + +const validateId = (value: string, kind: string): string => { + if (!CHECKPOINT_ID.test(value)) throw new Error(`invalid ${kind} ID: ${value}`) + return value.toLowerCase() +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const requiredString = (record: Record, key: string): string => { + const value = record[key] + if (typeof value !== "string" || value.length === 0) throw new Error(`invalid ${key}`) + return value +} + +const requiredCount = (record: Record, key: string): number => { + const value = record[key] + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`invalid ${key}`) + } + return value } -export const checkpointRoot = (dataDir: string): string => join(dataDir, "backups") -export const buildingDir = (dataDir: string): string => join(checkpointRoot(dataDir), "building") -export const currentDir = (dataDir: string): string => join(checkpointRoot(dataDir), "current") -export const previousDir = (dataDir: string): string => join(checkpointRoot(dataDir), "previous") -const restoreBuildingDir = (dataDir: string): string => `${dataDir}.restore-building` -const quarantineDir = (dataDir: string): string => - `${dataDir}.quarantine-${new Date().toISOString().replace(/[:.]/g, "-")}` +const requiredIso = (record: Record, key: string): string => { + const value = requiredString(record, key) + if (!Number.isFinite(Date.parse(value))) throw new Error(`invalid ${key}`) + return value +} -const backupSqlPath = (name: string): string => `backups/${name}/backup` +const assertContained = (root: string, candidate: string, label: string): string => { + const absoluteRoot = resolve(root) + const absoluteCandidate = resolve(candidate) + const rel = relative(absoluteRoot, absoluteCandidate) + if (rel === "" || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { + throw new Error(`${label} escapes configured root`) + } + return absoluteCandidate +} + +const assertNoSymlink = async (root: string, candidate: string): Promise => { + const absoluteRoot = resolve(root) + const absoluteCandidate = assertContained(absoluteRoot, candidate, "checkpoint path") + try { + if ((await lstat(absoluteRoot)).isSymbolicLink()) { + throw new Error(`refusing symlink checkpoint root: ${absoluteRoot}`) + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + } + const rel = relative(absoluteRoot, absoluteCandidate) + let current = absoluteRoot + for (const part of rel.split(sep)) { + current = join(current, part) + try { + if ((await lstat(current)).isSymbolicLink()) { + throw new Error(`refusing symlink in checkpoint path: ${current}`) + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + return + } + } +} const xmlEscape = (value: string): string => value @@ -62,9 +212,22 @@ export const writeBackupConfig = (path: string, sourceDataDir?: string): void => ${sourceDisk} `, + { mode: 0o600 }, ) } +export class LocalQueryError extends Error { + readonly status: number + readonly detail: string + + constructor(status: number, detail: string) { + super(`local query failed (${status})${detail ? `: ${detail}` : ""}`) + this.name = "LocalQueryError" + this.status = status + this.detail = detail + } +} + const postLocalQuery = async (port: number, sql: string): Promise => { const response = await fetch(`http://127.0.0.1:${port}/local/query`, { method: "POST", @@ -73,13 +236,29 @@ const postLocalQuery = async (port: number, sql: string): Promise => { }) if (!response.ok) { const detail = await response.text().catch(() => "") - throw new Error( - `local query failed (${response.status} ${response.statusText})${detail ? `: ${detail}` : ""}`, - ) + throw new LocalQueryError(response.status, detail) } return response.json() } +export const isMissingBackupConfigurationError = (error: unknown): boolean => { + const detail = + error instanceof LocalQueryError + ? error.detail + : error instanceof Error + ? error.message + : String(error) + const lower = detail.toLowerCase() + const backupSpecific = + lower.includes("backups.allowed_disk") || + lower.includes("backups.allowed_path") || + (lower.includes("backup") && + (lower.includes("not allowed") || + lower.includes("unknown disk") || + lower.includes("allowed disk"))) + return backupSpecific +} + const readJsonRows = (text: string): ReadonlyArray> => text .split("\n") @@ -91,191 +270,554 @@ const countFrom = (rows: ReadonlyArray>): number => { const row = rows[0] if (!row) return 0 const value = row["count()"] ?? row.count - return typeof value === "number" ? value : Number(value ?? 0) + const count = typeof value === "number" ? value : Number(value ?? 0) + if (!Number.isSafeInteger(count) || count < 0) throw new Error(`invalid count result: ${value}`) + return count } const queryCount = (db: Chdb, sql: string): number => countFrom(readJsonRows(db.query(sql))) +const validateRestoredDatabase = (db: Chdb): CheckpointValidation => ({ + validatedAt: new Date().toISOString(), + traces: queryCount(db, "SELECT count() FROM traces"), + logs: queryCount(db, "SELECT count() FROM logs"), + metricsSum: queryCount(db, "SELECT count() FROM metrics_sum"), + metricsGauge: queryCount(db, "SELECT count() FROM metrics_gauge"), + metricsHistogram: queryCount(db, "SELECT count() FROM metrics_histogram"), + metricsExponentialHistogram: queryCount(db, "SELECT count() FROM metrics_exponential_histogram"), + materializedViews: queryCount( + db, + "SELECT count() FROM system.tables WHERE database = 'default' AND engine = 'MaterializedView'", + ), +}) + const dirSize = async (path: string): Promise => { let total = 0 const entries = await readdir(path, { withFileTypes: true }) for (const entry of entries) { const child = join(path, entry.name) - if (entry.isDirectory()) { - total += await dirSize(child) - } else if (entry.isFile()) { - total += (await stat(child)).size - } + if (entry.isSymbolicLink()) throw new Error(`refusing symlink in checkpoint backup: ${child}`) + if (entry.isDirectory()) total += await dirSize(child) + else if (entry.isFile()) total += (await stat(child)).size + else throw new Error(`refusing unsupported checkpoint entry: ${child}`) } return total } -interface CheckpointManifest { - readonly mapleVersion: string - readonly chdbVersion: string - readonly schemaFingerprint: string - readonly createdAt: string - readonly sourceDataDir: string - readonly backupPath: string - readonly backupBytes: number - readonly validation: { - readonly validatedAt: string - readonly traces: number - readonly logs: number - readonly metricsSum: number - readonly materializedViews: number +const parseValidation = (value: unknown): CheckpointValidation => { + if (!isRecord(value)) throw new Error("invalid validation") + return { + validatedAt: requiredIso(value, "validatedAt"), + traces: requiredCount(value, "traces"), + logs: requiredCount(value, "logs"), + metricsSum: requiredCount(value, "metricsSum"), + metricsGauge: requiredCount(value, "metricsGauge"), + metricsHistogram: requiredCount(value, "metricsHistogram"), + metricsExponentialHistogram: requiredCount(value, "metricsExponentialHistogram"), + materializedViews: requiredCount(value, "materializedViews"), } } -export const readCheckpointManifest = async (dataDir: string): Promise => { - const path = join(currentDir(dataDir), "manifest.json") - let raw: string +export const parseCheckpointManifest = ( + value: unknown, + expectedCheckpointId?: string, +): CheckpointManifest => { + if (!isRecord(value) || value.formatVersion !== MANIFEST_FORMAT_VERSION) { + throw new Error("unsupported or malformed checkpoint manifest") + } + const checkpointId = validateId(requiredString(value, "checkpointId"), "checkpoint") + if (expectedCheckpointId && checkpointId !== validateId(expectedCheckpointId, "checkpoint")) { + throw new Error("checkpoint manifest ID does not match its snapshot directory") + } + const operationId = validateId(requiredString(value, "operationId"), "operation") + const sourceDataDir = requiredString(value, "sourceDataDir") + if (!isAbsolute(sourceDataDir)) throw new Error("checkpoint sourceDataDir must be absolute") + const backupRelativePath = requiredString(value, "backupRelativePath") + if (backupRelativePath !== snapshotBackupRelativePath(checkpointId)) { + throw new Error("checkpoint backup path does not match its immutable ID") + } + const manifest: CheckpointManifest = { + formatVersion: 1, + checkpointId, + operationId, + mapleVersion: requiredString(value, "mapleVersion"), + chdbVersion: requiredString(value, "chdbVersion"), + schemaFingerprint: requiredString(value, "schemaFingerprint"), + createdAt: requiredIso(value, "createdAt"), + sourceDataDir, + backupRelativePath, + backupBytes: requiredCount(value, "backupBytes"), + validation: parseValidation(value.validation), + } + if (manifest.chdbVersion !== CHDB_VERSION) { + throw new Error( + `checkpoint chDB version mismatch (checkpoint: ${manifest.chdbVersion}; build: ${CHDB_VERSION})`, + ) + } + if (manifest.schemaFingerprint !== SCHEMA_FINGERPRINT) { + throw new Error( + `checkpoint schema mismatch (checkpoint: ${manifest.schemaFingerprint}; build: ${SCHEMA_FINGERPRINT})`, + ) + } + return manifest +} + +export const parseCheckpointState = (value: unknown): CheckpointState => { + if (!isRecord(value) || value.formatVersion !== STATE_FORMAT_VERSION) { + throw new Error("unsupported or malformed checkpoint state") + } + const current = validateId(requiredString(value, "current"), "current checkpoint") + const previousValue = value.previous + const previous = + previousValue === null + ? null + : validateId( + typeof previousValue === "string" + ? previousValue + : (() => { + throw new Error("invalid previous checkpoint") + })(), + "previous checkpoint", + ) + if (previous === current) throw new Error("checkpoint current and previous IDs must differ") + return { + formatVersion: 1, + revision: validateId(requiredString(value, "revision"), "state revision"), + current, + previous, + committedAt: requiredIso(value, "committedAt"), + } +} + +const readStateFileOptional = async (dataDir: string): Promise => { + try { + return parseCheckpointState(JSON.parse(await readFile(checkpointStatePath(dataDir), "utf8"))) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null + throw error + } +} + +const checkpointLikePaths = async (dataDir: string): Promise => { + const root = checkpointRoot(dataDir) try { - raw = await readFile(path, "utf8") - } catch { - throw new Error(`checkpoint manifest not found at ${path}`) + const entries = await readdir(root) + return entries.filter((entry) => entry !== "state.json").map((entry) => join(root, entry)) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return [] + throw error } - const parsed = JSON.parse(raw) as Partial - if (parsed.chdbVersion !== CHDB_VERSION) { +} + +const assertNoLegacyLayout = (dataDir: string): void => { + const legacy = ["building", "current", "previous"] + .map((name) => join(checkpointRoot(dataDir), name)) + .filter(existsSync) + if (legacy.length > 0) { throw new Error( - `checkpoint chDB version mismatch (checkpoint: ${parsed.chdbVersion ?? "unknown"}; build: ${CHDB_VERSION})`, + `legacy preview checkpoint layout detected; refusing to infer or migrate state. Preserve and move aside after inspection: ${legacy.join(", ")}`, ) } - if (parsed.schemaFingerprint !== SCHEMA_FINGERPRINT) { +} + +export const readCheckpointState = async (dataDir: string): Promise => { + assertNoLegacyLayout(dataDir) + const state = await readStateFileOptional(dataDir) + if (!state) { + const paths = await checkpointLikePaths(dataDir) throw new Error( - `checkpoint schema mismatch (checkpoint: ${parsed.schemaFingerprint ?? "unknown"}; build: ${SCHEMA_FINGERPRINT})`, + paths.length === 0 + ? `checkpoint state not found at ${checkpointStatePath(dataDir)}` + : `checkpoint state missing while checkpoint data exists; refusing to infer selection: ${paths.join(", ")}`, ) } - return parsed as CheckpointManifest + await resolveCheckpoint(dataDir, state.current, state) + if (state.previous) await resolveCheckpoint(dataDir, state.previous, state) + return state } -const validateBackup = async (dataDir: string): Promise => { - const scratchParent = mkdtempSync(join(tmpdir(), "maple-checkpoint-")) - const scratchData = join(scratchParent, "data") - const scratchConfig = join(scratchParent, "config.xml") - writeBackupConfig(scratchConfig, dataDir) +export const resolveCheckpoint = async ( + dataDir: string, + selector: "current" | "previous" | string = "current", + knownState?: CheckpointState, +): Promise => { + const state = knownState ?? (await readCheckpointState(dataDir)) + const checkpointId = + selector === "current" + ? state.current + : selector === "previous" + ? state.previous + : validateId(selector, "checkpoint") + if (!checkpointId) throw new Error("no previous checkpoint is selected") + const snapshotDir = checkpointSnapshotDir(dataDir, checkpointId) + await assertNoSymlink(checkpointSnapshotsRoot(dataDir), snapshotDir) + const manifest = parseCheckpointManifest( + JSON.parse(await readFile(snapshotManifestPath(dataDir, checkpointId), "utf8")), + checkpointId, + ) + const backupDir = snapshotBackupDir(dataDir, checkpointId) + const backupStat = await stat(backupDir).catch(() => null) + if (!backupStat?.isDirectory()) throw new Error(`checkpoint backup is incomplete at ${backupDir}`) + return { + checkpointId, + snapshotDir, + backupDir, + backupSqlPath: snapshotBackupSqlPath(checkpointId), + manifest, + } +} + +export const readCheckpointManifest = async ( + dataDir: string, + selector: "current" | "previous" | string = "current", +): Promise => (await resolveCheckpoint(dataDir, selector)).manifest + +const restoreResolvedInto = async ( + resolvedCheckpoint: ResolvedCheckpoint, + targetDataDir: string, +): Promise<{ readonly db: Chdb; readonly validation: CheckpointValidation }> => { + const scratchParent = dirname(targetDataDir) + const scratchConfig = join(scratchParent, `checkpoint-${randomUUID()}.xml`) + writeBackupConfig(scratchConfig, resolvedCheckpoint.manifest.sourceDataDir) let db: Chdb | undefined try { db = Chdb.open({ - dataDir: scratchData, + dataDir: targetDataDir, schemaSql, configFile: scratchConfig, bootstrapSchema: false, }) db.exec("CREATE DATABASE IF NOT EXISTS default") db.exec( - `RESTORE DATABASE default FROM Disk('src', '${backupSqlPath("building")}') ` + + `RESTORE DATABASE default FROM Disk('src', '${resolvedCheckpoint.backupSqlPath}') ` + "SETTINGS allow_different_database_def=1", ) - return { - validatedAt: new Date().toISOString(), - traces: queryCount(db, "SELECT count() FROM traces"), - logs: queryCount(db, "SELECT count() FROM logs"), - metricsSum: queryCount(db, "SELECT count() FROM metrics_sum"), - materializedViews: queryCount( - db, - "SELECT count() FROM system.tables WHERE database = 'default' AND engine = 'MaterializedView'", - ), - } - } finally { + return { db, validation: validateRestoredDatabase(db) } + } catch (error) { db?.close() - rmSync(scratchParent, { recursive: true, force: true }) + throw error + } finally { + rmSync(scratchConfig, { force: true }) } } -const restoreIntoScratch = async ( - sourceDataDir: string, - targetDataDir: string, - checkpointName: "current" | "building", -): Promise => { - const scratchParent = mkdtempSync(join(tmpdir(), "maple-restore-")) - const restoreConfig = join(scratchParent, "config.xml") - writeBackupConfig(restoreConfig, sourceDataDir) +export const withRestoredCheckpoint = async ( + resolvedCheckpoint: ResolvedCheckpoint, + options: { + readonly scratchRoot?: string + readonly cleanup?: "always" | "never" + }, + use: (restored: { + readonly checkpointId: string + readonly manifest: CheckpointManifest + readonly scratchDataDir: string + readonly db: Chdb + readonly validation: CheckpointValidation + }) => A | Promise, +): Promise => { + const scratchRoot = resolve(options.scratchRoot ?? tmpdir()) + const sourceDataDir = resolve(resolvedCheckpoint.manifest.sourceDataDir) + const sourceRelation = relative(sourceDataDir, scratchRoot) + if ( + scratchRoot === sourceDataDir || + (sourceRelation !== "" && sourceRelation !== ".." && !sourceRelation.startsWith(`..${sep}`)) + ) { + throw new Error("scratch root must not be the live data directory or one of its descendants") + } + await ensurePrivateDirectory(scratchRoot) + const scratchParent = join(scratchRoot, `maple-checkpoint-${randomUUID()}`) + await mkdir(scratchParent, { mode: 0o700 }) + const scratchDataDir = join(scratchParent, "data") let db: Chdb | undefined try { - db = Chdb.open({ - dataDir: targetDataDir, - schemaSql, - configFile: restoreConfig, - bootstrapSchema: false, + const restored = await restoreResolvedInto(resolvedCheckpoint, scratchDataDir) + db = restored.db + return await use({ + checkpointId: resolvedCheckpoint.checkpointId, + manifest: resolvedCheckpoint.manifest, + scratchDataDir, + db, + validation: restored.validation, }) - db.exec("CREATE DATABASE IF NOT EXISTS default") - db.exec( - `RESTORE DATABASE default FROM Disk('src', '${backupSqlPath(checkpointName)}') ` + - "SETTINGS allow_different_database_def=1", - ) - return { - validatedAt: new Date().toISOString(), - traces: queryCount(db, "SELECT count() FROM traces"), - logs: queryCount(db, "SELECT count() FROM logs"), - metricsSum: queryCount(db, "SELECT count() FROM metrics_sum"), - materializedViews: queryCount( - db, - "SELECT count() FROM system.tables WHERE database = 'default' AND engine = 'MaterializedView'", - ), - } } finally { db?.close() - rmSync(scratchParent, { recursive: true, force: true }) + if (options.cleanup !== "never") await rm(scratchParent, { recursive: true, force: true }) } } -export const promoteBuilding = async (dataDir: string): Promise => { - await rm(previousDir(dataDir), { recursive: true, force: true }) +const parseOperation = (value: unknown): CheckpointOperation => { + if (!isRecord(value) || value.formatVersion !== OPERATION_FORMAT_VERSION) { + throw new Error("unsupported or malformed checkpoint operation") + } + const phase = requiredString(value, "phase") + if (!["intent", "backup-complete", "manifest-complete", "pointer-complete"].includes(phase)) { + throw new Error("invalid checkpoint operation phase") + } + return { + formatVersion: 1, + operationId: validateId(requiredString(value, "operationId"), "operation"), + checkpointId: validateId(requiredString(value, "checkpointId"), "checkpoint"), + phase: phase as CheckpointOperation["phase"], + startedAt: requiredIso(value, "startedAt"), + } +} + +const writeOperation = async ( + dataDir: string, + operation: CheckpointOperation, + faults: DurabilityFaults = {}, +): Promise => durableJson(operationPath(dataDir, operation.operationId), operation, faults) + +const processIsAlive = (pid: number): boolean => { try { - await rename(currentDir(dataDir), previousDir(dataDir)) + process.kill(pid, 0) + return true } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + return (error as NodeJS.ErrnoException).code === "EPERM" + } +} + +const acquireMaintenance = async (dataDir: string, operationId: string): Promise<() => Promise> => { + const lockPath = maintenanceLockPath(dataDir) + try { + await mkdir(lockPath, { mode: 0o700 }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error + let owner: MaintenanceOwner + try { + const parsed = JSON.parse(readFileSync(join(lockPath, "owner.json"), "utf8")) as unknown + if (!isRecord(parsed) || parsed.formatVersion !== 1) throw new Error("malformed owner") + owner = { + formatVersion: 1, + operationId: validateId(requiredString(parsed, "operationId"), "operation"), + pid: requiredCount(parsed, "pid"), + startedAt: requiredIso(parsed, "startedAt"), + } + } catch { + throw new Error(`maintenance lock is present but ownership is uncertain: ${lockPath}`) + } + if (processIsAlive(owner.pid)) { + throw new Error(`another Maple maintenance operation is active (PID ${owner.pid})`) + } + const quarantinedLock = `${lockPath}.quarantine-${randomUUID()}` + await durableRename(lockPath, quarantinedLock) + await mkdir(lockPath, { mode: 0o700 }) + } + const owner: MaintenanceOwner = { + formatVersion: 1, + operationId, + pid: process.pid, + startedAt: new Date().toISOString(), + } + await durableJson(join(lockPath, "owner.json"), owner) + return async () => { + const current = JSON.parse(await readFile(join(lockPath, "owner.json"), "utf8")) as unknown + if ( + !isRecord(current) || + requiredString(current, "operationId") !== operationId || + current.pid !== process.pid + ) { + throw new Error(`maintenance lock ownership changed unexpectedly: ${lockPath}`) + } + await rm(lockPath, { recursive: true }) + await syncDirectory(dirname(lockPath)) + } +} + +export const reconcileCheckpointOperations = async (dataDir: string): Promise => { + const state = await readStateFileOptional(dataDir) + let entries + try { + entries = await readdir(checkpointOperationsRoot(dataDir), { withFileTypes: true }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + for (const entry of entries) { + if (!entry.isDirectory() || !entry.name.startsWith("checkpoint-")) { + throw new Error( + `unrecognized checkpoint operation debris: ${join(checkpointOperationsRoot(dataDir), entry.name)}`, + ) + } + const path = join(checkpointOperationsRoot(dataDir), entry.name, "intent.json") + const operation = parseOperation(JSON.parse(await readFile(path, "utf8"))) + const snapshot = checkpointSnapshotDir(dataDir, operation.checkpointId) + const selected = + state?.current === operation.checkpointId || state?.previous === operation.checkpointId + if (operation.phase === "pointer-complete" && selected) { + await rm(operationDir(dataDir, operation.operationId), { recursive: true }) + await syncDirectory(checkpointOperationsRoot(dataDir)) + continue + } + const quarantine = join( + checkpointQuarantineRoot(dataDir), + `operation-${operation.operationId}-${randomUUID()}`, + ) + await ensurePrivateDirectory(checkpointQuarantineRoot(dataDir)) + await ensurePrivateDirectory(quarantine) + if (existsSync(snapshot) && !existsSync(snapshotManifestPath(dataDir, operation.checkpointId))) { + await durableRename(snapshot, join(quarantine, "incomplete-snapshot")) + } + await durableRename(operationDir(dataDir, operation.operationId), join(quarantine, "operation")) } - await rename(buildingDir(dataDir), currentDir(dataDir)) +} + +const hasPins = async (dataDir: string, checkpointId: string): Promise => { + const path = join(checkpointPinsRoot(dataDir), checkpointId) + try { + return (await readdir(path)).length > 0 + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false + throw error + } +} + +export const retireCheckpointIfEligible = async ( + dataDir: string, + checkpointId: string | null, + state: CheckpointState, +): Promise => { + if (!checkpointId || state.current === checkpointId || state.previous === checkpointId) return + if (await hasPins(dataDir, checkpointId)) return + await resolveCheckpoint(dataDir, checkpointId, state) + const retirement = join(checkpointRetiringRoot(dataDir), randomUUID()) + await ensurePrivateDirectory(retirement) + await durableRename(checkpointSnapshotDir(dataDir, checkpointId), join(retirement, checkpointId)) + await rm(retirement, { recursive: true }) + await syncDirectory(checkpointRetiringRoot(dataDir)) } export const createCheckpoint = ( options: CheckpointOptions, -): Effect.Effect<{ readonly path: string; readonly manifest: CheckpointManifest }, CheckpointError> => +): Effect.Effect< + { + readonly checkpointId: string + readonly path: string + readonly state: CheckpointState + readonly manifest: CheckpointManifest + }, + CheckpointError +> => Effect.tryPromise({ try: async () => { - const root = checkpointRoot(options.dataDir) - const building = buildingDir(options.dataDir) - const name = basename(building) - if (name !== "building") throw new Error("internal checkpoint path error") - await rm(building, { recursive: true, force: true }) - + const operationId = randomUUID() + const checkpointId = randomUUID() + const release = await acquireMaintenance(options.dataDir, operationId) try { - await postLocalQuery( - options.port, - `BACKUP DATABASE default TO Disk('default', '${backupSqlPath("building")}')`, - ) - } catch (error) { - const message = error instanceof Error ? error.message : String(error) - if ( - message.includes("backups.allowed_disk") || - message.includes("INVALID_CONFIG_PARAMETER") - ) { + assertCheckpointRootSafe(options.dataDir) + assertNoLegacyLayout(options.dataDir) + await reconcileCheckpointOperations(options.dataDir) + const oldState = await readStateFileOptional(options.dataDir) + if (!oldState && (await checkpointLikePaths(options.dataDir)).length > 0) { throw new Error( - "checkpoints require the local server to be started with `--chdb-config-file` " + - "pointing at a ClickHouse backups config", + "checkpoint state is missing while checkpoint data exists; refusing to infer selection", ) } - throw error - } - - const validation = await validateBackup(options.dataDir) - const manifest: CheckpointManifest = { - mapleVersion: MAPLE_VERSION, - chdbVersion: CHDB_VERSION, - schemaFingerprint: SCHEMA_FINGERPRINT, - createdAt: new Date().toISOString(), - sourceDataDir: resolve(options.dataDir), - backupPath: backupSqlPath("current"), - backupBytes: await dirSize(join(building, "backup")), - validation, - } - await writeFile(join(building, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`) - await promoteBuilding(options.dataDir) - return { - path: join(root, "current"), - manifest: { ...manifest, backupPath: backupSqlPath("current") }, + if (oldState) { + await resolveCheckpoint(options.dataDir, oldState.current, oldState) + if (oldState.previous) { + await resolveCheckpoint(options.dataDir, oldState.previous, oldState) + } + } + for (const path of [ + checkpointRoot(options.dataDir), + checkpointSnapshotsRoot(options.dataDir), + checkpointOperationsRoot(options.dataDir), + checkpointPinsRoot(options.dataDir), + checkpointQuarantineRoot(options.dataDir), + checkpointRetiringRoot(options.dataDir), + ]) { + await ensurePrivateDirectory(path) + } + const startedAt = new Date().toISOString() + let operation: CheckpointOperation = { + formatVersion: 1, + operationId, + checkpointId, + phase: "intent", + startedAt, + } + await writeOperation(options.dataDir, operation, options.faults) + const snapshot = checkpointSnapshotDir(options.dataDir, checkpointId) + await assertNoSymlink(checkpointSnapshotsRoot(options.dataDir), snapshot) + await mkdir(snapshot, { mode: 0o700 }) + try { + await postLocalQuery( + options.port, + `BACKUP DATABASE default TO Disk('default', '${snapshotBackupSqlPath(checkpointId)}')`, + ) + } catch (error) { + if (isMissingBackupConfigurationError(error)) { + throw new Error( + "checkpoints require the local server to be started with `--chdb-config-file` " + + "pointing at a ClickHouse backups config", + { cause: error }, + ) + } + throw error + } + await syncTree(snapshotBackupDir(options.dataDir, checkpointId)) + operation = { ...operation, phase: "backup-complete" } + await writeOperation(options.dataDir, operation, options.faults) + const provisionalManifest: CheckpointManifest = { + formatVersion: 1, + checkpointId, + operationId, + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + createdAt: startedAt, + sourceDataDir: resolve(options.dataDir), + backupRelativePath: snapshotBackupRelativePath(checkpointId), + backupBytes: await dirSize(snapshotBackupDir(options.dataDir, checkpointId)), + validation: { + validatedAt: startedAt, + traces: 0, + logs: 0, + metricsSum: 0, + metricsGauge: 0, + metricsHistogram: 0, + metricsExponentialHistogram: 0, + materializedViews: 0, + }, + } + const provisional: ResolvedCheckpoint = { + checkpointId, + snapshotDir: snapshot, + backupDir: snapshotBackupDir(options.dataDir, checkpointId), + backupSqlPath: snapshotBackupSqlPath(checkpointId), + manifest: provisionalManifest, + } + const validation = await withRestoredCheckpoint( + provisional, + { cleanup: "always" }, + (restored) => restored.validation, + ) + const manifest: CheckpointManifest = { ...provisionalManifest, validation } + await durableJson( + snapshotManifestPath(options.dataDir, checkpointId), + manifest, + options.faults, + ) + await syncDirectory(snapshot) + operation = { ...operation, phase: "manifest-complete" } + await writeOperation(options.dataDir, operation, options.faults) + const state: CheckpointState = { + formatVersion: 1, + revision: operationId, + current: checkpointId, + previous: oldState?.current ?? null, + committedAt: new Date().toISOString(), + } + await durableJson(checkpointStatePath(options.dataDir), state, options.faults) + operation = { ...operation, phase: "pointer-complete" } + await writeOperation(options.dataDir, operation, options.faults) + await retireCheckpointIfEligible(options.dataDir, oldState?.previous ?? null, state) + await rm(operationDir(options.dataDir, operationId), { recursive: true }) + await syncDirectory(checkpointOperationsRoot(options.dataDir)) + return { checkpointId, path: snapshot, state, manifest } + } finally { + await release() } }, catch: (error) => @@ -284,39 +826,67 @@ export const createCheckpoint = ( export const restoreCheckpoint = ( dataDir: string, + selector: "current" | "previous" | string = "current", ): Effect.Effect< - { readonly quarantinePath: string; readonly validation: CheckpointManifest["validation"] }, + { + readonly checkpointId: string + readonly quarantinePath: string + readonly validation: CheckpointValidation + }, CheckpointError > => Effect.tryPromise({ try: async () => { - const sourceBackup = join(currentDir(dataDir), "backup") - if (!existsSync(sourceBackup)) { - throw new Error(`no checkpoint found at ${sourceBackup}`) - } - await readCheckpointManifest(dataDir) - - const restoreDir = restoreBuildingDir(dataDir) - await rm(restoreDir, { recursive: true, force: true }) - const validation = await restoreIntoScratch(dataDir, restoreDir, "current") - - if (existsSync(join(dataDir, "backups"))) { - await cp(join(dataDir, "backups"), join(restoreDir, "backups"), { + const operationId = randomUUID() + const release = await acquireMaintenance(dataDir, operationId) + try { + const resolvedCheckpoint = await resolveCheckpoint(dataDir, selector) + const quarantinePath = `${resolve(dataDir)}.quarantine-${operationId}-${randomUUID()}` + if (existsSync(quarantinePath)) + throw new Error(`quarantine path already exists: ${quarantinePath}`) + const restored = await withRestoredCheckpoint( + resolvedCheckpoint, + { scratchRoot: dirname(resolve(dataDir)), cleanup: "never" }, + ({ scratchDataDir, validation }) => ({ scratchDataDir, validation }), + ) + await cp(checkpointRoot(dataDir), join(restored.scratchDataDir, "backups"), { recursive: true, - force: true, + force: false, + errorOnExist: true, }) + await syncTree(join(restored.scratchDataDir, "backups")) + await rename(resolve(dataDir), quarantinePath) + await syncDirectory(dirname(resolve(dataDir))) + await rename(restored.scratchDataDir, resolve(dataDir)) + await syncDirectory(dirname(resolve(dataDir))) + markStoreClosed(dataDir) + writeFileSync( + storeMarkerPath(dataDir), + storeMarkerJson(MAPLE_VERSION, new Date().toISOString(), SCHEMA_FINGERPRINT), + { mode: 0o600 }, + ) + return { + checkpointId: resolvedCheckpoint.checkpointId, + quarantinePath, + validation: restored.validation, + } + } finally { + await release() } - - const quarantinePath = quarantineDir(dataDir) - await rename(dataDir, quarantinePath) - await rename(restoreDir, dataDir) - markStoreClosed(dataDir) - writeFileSync( - storeMarkerPath(dataDir), - storeMarkerJson(MAPLE_VERSION, new Date().toISOString(), SCHEMA_FINGERPRINT), - ) - return { quarantinePath, validation } }, catch: (error) => new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), }) + +// Test helper: assert generated operation IDs remain unique and valid without +// exposing an override in production command paths. +export const newCheckpointId = (): string => validateId(randomUUID(), "checkpoint") + +// Refuse pre-existing symlink roots even before an operation allocates paths. +export const assertCheckpointRootSafe = (dataDir: string): void => { + const root = checkpointRoot(dataDir) + if (existsSync(root) && lstatSync(root).isSymbolicLink()) { + throw new Error(`refusing symlink checkpoint root: ${root}`) + } + if (basename(root) !== "backups") throw new Error("invalid checkpoint root") +} diff --git a/apps/cli/src/server/durable-files.ts b/apps/cli/src/server/durable-files.ts new file mode 100644 index 000000000..262819cd5 --- /dev/null +++ b/apps/cli/src/server/durable-files.ts @@ -0,0 +1,112 @@ +import { randomUUID } from "node:crypto" +import { constants } from "node:fs" +import { chmod, mkdir, open, readdir, rename, rm } from "node:fs/promises" +import { dirname, join } from "node:path" + +export interface DurabilityFaults { + readonly beforeFileSync?: (path: string) => void | Promise + readonly beforeRename?: (from: string, to: string) => void | Promise + readonly beforeDirectorySync?: (path: string) => void | Promise + readonly beforeRemove?: (path: string) => void | Promise +} + +// APFS and the target Linux filesystems support directory fsync. Keep the +// fallback deliberately narrow for filesystems that explicitly report that +// the operation is unsupported; descriptor/type errors are programming bugs. +const unsupportedDirectorySyncCodes = new Set(["EINVAL", "ENOTSUP", "EOPNOTSUPP"]) + +export const isUnsupportedDirectorySyncError = (error: unknown): boolean => + typeof error === "object" && + error !== null && + "code" in error && + unsupportedDirectorySyncCodes.has(String((error as NodeJS.ErrnoException).code)) + +export const ensurePrivateDirectory = async (path: string): Promise => { + await mkdir(path, { recursive: true, mode: 0o700 }) + await chmod(path, 0o700) +} + +export const syncDirectory = async (path: string, faults: DurabilityFaults = {}): Promise => { + await faults.beforeDirectorySync?.(path) + let handle + try { + handle = await open(path, constants.O_RDONLY) + await handle.sync() + } catch (error) { + if (!isUnsupportedDirectorySyncError(error)) throw error + } finally { + await handle?.close() + } +} + +export const durableWrite = async ( + path: string, + bytes: string | Uint8Array, + faults: DurabilityFaults = {}, +): Promise => { + const parent = dirname(path) + await ensurePrivateDirectory(parent) + const temporary = join(parent, `.${randomUUID()}.tmp`) + const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600) + try { + await handle.writeFile(bytes) + await faults.beforeFileSync?.(path) + await handle.sync() + } catch (error) { + await handle.close().catch(() => undefined) + await rm(temporary, { force: true }).catch(() => undefined) + throw error + } + await handle.close() + try { + await faults.beforeRename?.(temporary, path) + await rename(temporary, path) + await syncDirectory(parent, faults) + } catch (error) { + await rm(temporary, { force: true }).catch(() => undefined) + throw error + } +} + +export const durableJson = async ( + path: string, + value: unknown, + faults: DurabilityFaults = {}, +): Promise => durableWrite(path, `${JSON.stringify(value, null, 2)}\n`, faults) + +export const durableRemove = async (path: string, faults: DurabilityFaults = {}): Promise => { + await faults.beforeRemove?.(path) + await rm(path, { force: true }) + await syncDirectory(dirname(path), faults) +} + +export const durableRename = async ( + from: string, + to: string, + faults: DurabilityFaults = {}, +): Promise => { + await faults.beforeRename?.(from, to) + await rename(from, to) + await syncDirectory(dirname(from), faults) + if (dirname(to) !== dirname(from)) await syncDirectory(dirname(to), faults) +} + +export const syncTree = async (path: string): Promise => { + const entries = await readdir(path, { withFileTypes: true }) + for (const entry of entries) { + const child = join(path, entry.name) + if (entry.isDirectory()) { + await syncTree(child) + } else if (entry.isFile()) { + const handle = await open(child, constants.O_RDONLY) + try { + await handle.sync() + } finally { + await handle.close() + } + } else { + throw new Error(`refusing to sync non-file checkpoint entry at ${child}`) + } + } + await syncDirectory(path) +} diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index dca5813ae..1653fad99 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -1,16 +1,40 @@ import { describe, it } from "@effect/vitest" -import { ok, rejects, strictEqual } from "node:assert" -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { deepStrictEqual, match, ok, rejects, strictEqual } from "node:assert" +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs" import { tmpdir } from "node:os" -import { join } from "node:path" +import { dirname, join } from "node:path" import { - buildingDir, - currentDir, - previousDir, - promoteBuilding, - readCheckpointManifest, + assertCheckpointRootSafe, + checkpointRoot, + checkpointSnapshotDir, + checkpointStatePath, + isMissingBackupConfigurationError, + LocalQueryError, + newCheckpointId, + parseCheckpointManifest, + parseCheckpointState, + readCheckpointState, + reconcileCheckpointOperations, + resolveCheckpoint, + retireCheckpointIfEligible, writeBackupConfig, } from "../src/server/checkpoints" +import { + durableWrite, + isUnsupportedDirectorySyncError, + syncDirectory, + syncTree, +} from "../src/server/durable-files" import { SCHEMA_FINGERPRINT } from "../src/server/serve" import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" @@ -25,119 +49,353 @@ const withDataDir = async (run: (dataDir: string) => Promise | void): Prom } } -const writeMarker = (path: string, value: string): void => { - mkdirSync(path, { recursive: true }) - writeFileSync(join(path, "marker.txt"), value) +const manifest = (checkpointId: string, operationId = newCheckpointId()): Record => ({ + formatVersion: 1, + checkpointId, + operationId, + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + createdAt: "2026-01-01T00:00:00.000Z", + sourceDataDir: "/tmp/maple-data", + backupRelativePath: `snapshots/${checkpointId}/backup`, + backupBytes: 123, + validation: { + validatedAt: "2026-01-01T00:00:01.000Z", + traces: 1, + logs: 2, + metricsSum: 3, + metricsGauge: 4, + metricsHistogram: 5, + metricsExponentialHistogram: 6, + materializedViews: 33, + }, +}) + +const writeSnapshot = (dataDir: string, checkpointId: string): void => { + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + mkdirSync(join(snapshot, "backup"), { recursive: true }) + writeFileSync(join(snapshot, "backup", "data.bin"), "backup") + writeFileSync(join(snapshot, "manifest.json"), `${JSON.stringify(manifest(checkpointId))}\n`) } -const readMarker = (path: string): string => readFileSync(join(path, "marker.txt"), "utf8") - -const manifest = (overrides: Record = {}): string => - `${JSON.stringify( - { - mapleVersion: MAPLE_VERSION, - chdbVersion: CHDB_VERSION, - schemaFingerprint: SCHEMA_FINGERPRINT, - createdAt: "2026-01-01T00:00:00.000Z", - sourceDataDir: "/tmp/maple-data", - backupPath: "backups/current/backup", - backupBytes: 123, - validation: { - validatedAt: "2026-01-01T00:00:01.000Z", - traces: 1, - logs: 2, - metricsSum: 3, - materializedViews: 33, - }, - ...overrides, - }, - null, - 2, - )}\n` +const writeState = ( + dataDir: string, + current: string, + previous: string | null = null, + revision = newCheckpointId(), +): void => { + mkdirSync(checkpointRoot(dataDir), { recursive: true }) + writeFileSync( + checkpointStatePath(dataDir), + `${JSON.stringify({ + formatVersion: 1, + revision, + current, + previous, + committedAt: "2026-01-01T00:00:02.000Z", + })}\n`, + ) +} describe("writeBackupConfig", () => { - it("writes the runtime backup config for the default disk", async () => { + it("writes restrictive runtime and escaped restore configurations", async () => { await withDataDir((dataDir) => { - const configPath = join(dataDir, "config.xml") - writeBackupConfig(configPath) + const runtimePath = join(dataDir, "runtime.xml") + writeBackupConfig(runtimePath) + const runtime = readFileSync(runtimePath, "utf8") + ok(runtime.includes("default")) + ok(runtime.includes("backups")) + strictEqual(lstatSync(runtimePath).mode & 0o777, 0o600) - const xml = readFileSync(configPath, "utf8") - ok(xml.includes("default")) - ok(xml.includes("backups")) - ok(!xml.includes("")) + const restorePath = join(dataDir, "restore.xml") + writeBackupConfig(restorePath, join(dataDir, "source & ")) + const restore = readFileSync(restorePath, "utf8") + ok(restore.includes("src")) + ok(restore.includes("source & <store>")) }) }) +}) + +describe("checkpoint IDs and strict parsers", () => { + it("generates collision-resistant UUIDs", () => { + const ids = new Set(Array.from({ length: 2_000 }, () => newCheckpointId())) + strictEqual(ids.size, 2_000) + for (const id of ids) match(id, /^[0-9a-f-]{36}$/) + }) - it("writes a restore config with an escaped source disk path", async () => { - await withDataDir((dataDir) => { - const configPath = join(dataDir, "config.xml") - const sourceDataDir = join(dataDir, "source & ") - writeBackupConfig(configPath, sourceDataDir) - - const xml = readFileSync(configPath, "utf8") - ok(xml.includes("src")) - ok(xml.includes("backups")) - ok(xml.includes("")) - ok(xml.includes("source & <store>")) - ok(xml.includes("")) - }) + it("accepts a complete manifest and rejects ID, path, compatibility, and count corruption", () => { + const id = newCheckpointId() + strictEqual(parseCheckpointManifest(manifest(id), id).checkpointId, id) + const wrong = newCheckpointId() + ok(wrong !== id) + throwsMessage(() => parseCheckpointManifest(manifest(id), wrong), /does not match/) + throwsMessage( + () => parseCheckpointManifest({ ...manifest(id), backupRelativePath: "../escape" }, id), + /backup path/, + ) + throwsMessage( + () => parseCheckpointManifest({ ...manifest(id), chdbVersion: "v0.0.0" }, id), + /version mismatch/, + ) + throwsMessage( + () => + parseCheckpointManifest({ + ...manifest(id), + validation: { ...(manifest(id).validation as object), logs: -1 }, + }), + /invalid logs/, + ) + }) + + it("accepts versioned current/previous state and rejects malformed selection", () => { + const current = newCheckpointId() + const previous = newCheckpointId() + const revision = newCheckpointId() + deepStrictEqual( + parseCheckpointState({ + formatVersion: 1, + revision, + current, + previous, + committedAt: "2026-01-01T00:00:00.000Z", + }), + { + formatVersion: 1, + revision, + current, + previous, + committedAt: "2026-01-01T00:00:00.000Z", + }, + ) + throwsMessage( + () => + parseCheckpointState({ + formatVersion: 1, + revision, + current, + previous: current, + committedAt: "2026-01-01T00:00:00.000Z", + }), + /must differ/, + ) + throwsMessage( + () => + parseCheckpointState({ + formatVersion: 99, + revision, + current, + previous: null, + committedAt: "2026-01-01T00:00:00.000Z", + }), + /unsupported/, + ) }) }) -describe("promoteBuilding", () => { - it("promotes building to current when no current checkpoint exists", async () => { +describe("checkpoint state resolution", () => { + it("resolves immutable current, previous, and explicit IDs", async () => { await withDataDir(async (dataDir) => { - writeMarker(buildingDir(dataDir), "new") + const current = newCheckpointId() + const previous = newCheckpointId() + writeSnapshot(dataDir, current) + writeSnapshot(dataDir, previous) + writeState(dataDir, current, previous) - await promoteBuilding(dataDir) - - ok(!existsSync(buildingDir(dataDir))) - strictEqual(readMarker(currentDir(dataDir)), "new") - ok(!existsSync(previousDir(dataDir))) + const state = await readCheckpointState(dataDir) + strictEqual(state.current, current) + strictEqual((await resolveCheckpoint(dataDir, "current")).checkpointId, current) + strictEqual((await resolveCheckpoint(dataDir, "previous")).checkpointId, previous) + strictEqual((await resolveCheckpoint(dataDir, previous)).checkpointId, previous) }) }) - it("moves current to previous and replaces any older previous checkpoint", async () => { + it("fails closed for missing/malformed state, incomplete snapshots, and legacy aliases", async () => { await withDataDir(async (dataDir) => { - writeMarker(previousDir(dataDir), "old-previous") - writeMarker(currentDir(dataDir), "old-current") - writeMarker(buildingDir(dataDir), "new-current") + await rejects(readCheckpointState(dataDir), /state not found/) + + mkdirSync(join(checkpointRoot(dataDir), "snapshots"), { recursive: true }) + await rejects(readCheckpointState(dataDir), /state missing while checkpoint data exists/) - await promoteBuilding(dataDir) + writeFileSync(checkpointStatePath(dataDir), "{bad json") + await rejects(readCheckpointState(dataDir), /JSON/) - ok(!existsSync(buildingDir(dataDir))) - strictEqual(readMarker(currentDir(dataDir)), "new-current") - strictEqual(readMarker(previousDir(dataDir)), "old-current") + rmSync(checkpointRoot(dataDir), { recursive: true }) + mkdirSync(join(checkpointRoot(dataDir), "current"), { recursive: true }) + await rejects(readCheckpointState(dataDir), /legacy preview/) + }) + }) + + it("rejects symlink roots and symlinked snapshot paths", async () => { + await withDataDir(async (dataDir) => { + const outside = join(dirname(dataDir), "outside") + mkdirSync(outside) + symlinkSync(outside, checkpointRoot(dataDir)) + throwsMessage(() => assertCheckpointRootSafe(dataDir), /symlink/) }) }) }) -describe("readCheckpointManifest", () => { - it("round-trips a compatible checkpoint manifest", async () => { +describe("checkpoint reconciliation and retention", () => { + it("quarantines only an exactly owned incomplete operation and preserves its bytes", async () => { await withDataDir(async (dataDir) => { - mkdirSync(currentDir(dataDir), { recursive: true }) - writeFileSync(join(currentDir(dataDir), "manifest.json"), manifest()) + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + const operationDir = join(checkpointRoot(dataDir), "operations", `checkpoint-${operationId}`) + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + mkdirSync(join(snapshot, "backup"), { recursive: true }) + writeFileSync(join(snapshot, "backup", "partial.bin"), "partial") + mkdirSync(operationDir, { recursive: true }) + writeFileSync( + join(operationDir, "intent.json"), + `${JSON.stringify({ + formatVersion: 1, + operationId, + checkpointId, + phase: "backup-complete", + startedAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ) - const parsed = await readCheckpointManifest(dataDir) + await reconcileCheckpointOperations(dataDir) - strictEqual(parsed.chdbVersion, CHDB_VERSION) - strictEqual(parsed.schemaFingerprint, SCHEMA_FINGERPRINT) - strictEqual(parsed.validation.materializedViews, 33) + ok(!existsSync(snapshot)) + const quarantineRoot = join(checkpointRoot(dataDir), "quarantine") + const quarantines = readdirSync(quarantineRoot) + strictEqual(quarantines.length, 1) + ok( + existsSync( + join(quarantineRoot, quarantines[0]!, "incomplete-snapshot", "backup", "partial.bin"), + ), + ) + ok(existsSync(join(quarantineRoot, quarantines[0]!, "operation", "intent.json"))) }) }) - it("rejects a manifest from a different chDB or schema", async () => { + it("fails closed and preserves a malformed operation", async () => { await withDataDir(async (dataDir) => { - mkdirSync(currentDir(dataDir), { recursive: true }) - writeFileSync(join(currentDir(dataDir), "manifest.json"), manifest({ chdbVersion: "v0.0.0" })) + const operationDir = join(checkpointRoot(dataDir), "operations", "checkpoint-not-a-uuid") + mkdirSync(operationDir, { recursive: true }) + writeFileSync(join(operationDir, "intent.json"), "{bad json") + await rejects(reconcileCheckpointOperations(dataDir)) + ok(existsSync(join(operationDir, "intent.json"))) + }) + }) - await rejects(readCheckpointManifest(dataDir), /checkpoint chDB version mismatch/) + it("retains current, previous, pinned, and malformed candidates; retires only proven safe", async () => { + await withDataDir(async (dataDir) => { + const current = newCheckpointId() + const previous = newCheckpointId() + const old = newCheckpointId() + writeSnapshot(dataDir, current) + writeSnapshot(dataDir, previous) + writeSnapshot(dataDir, old) + writeState(dataDir, current, previous) + const state = await readCheckpointState(dataDir) - writeFileSync( - join(currentDir(dataDir), "manifest.json"), - manifest({ schemaFingerprint: "old-schema" }), + await retireCheckpointIfEligible(dataDir, current, state) + await retireCheckpointIfEligible(dataDir, previous, state) + ok(existsSync(checkpointSnapshotDir(dataDir, current))) + ok(existsSync(checkpointSnapshotDir(dataDir, previous))) + + const pinDir = join(checkpointRoot(dataDir), "pins", old) + mkdirSync(pinDir, { recursive: true }) + writeFileSync(join(pinDir, "pin.json"), "{}") + await retireCheckpointIfEligible(dataDir, old, state) + ok(existsSync(checkpointSnapshotDir(dataDir, old))) + + rmSync(pinDir, { recursive: true }) + await retireCheckpointIfEligible(dataDir, old, state) + ok(!existsSync(checkpointSnapshotDir(dataDir, old))) + + const malformed = newCheckpointId() + writeSnapshot(dataDir, malformed) + writeFileSync(join(checkpointSnapshotDir(dataDir, malformed), "manifest.json"), "{bad json") + await rejects(retireCheckpointIfEligible(dataDir, malformed, state)) + ok(existsSync(checkpointSnapshotDir(dataDir, malformed))) + }) + }) +}) + +describe("backup configuration classification", () => { + it("classifies only backup-specific errors", () => { + ok( + isMissingBackupConfigurationError( + new LocalQueryError(500, "INVALID_CONFIG_PARAMETER: backups.allowed_disk is not set"), + ), + ) + ok( + isMissingBackupConfigurationError( + new LocalQueryError(500, "Disk default is not allowed for backups"), + ), + ) + ok(!isMissingBackupConfigurationError(new LocalQueryError(500, "INVALID_CONFIG_PARAMETER"))) + ok(!isMissingBackupConfigurationError(new Error("UNKNOWN_TABLE"))) + ok(!isMissingBackupConfigurationError(new Error("connection refused"))) + }) +}) + +describe("durable filesystem primitives", () => { + it("atomically replaces a file and syncs a directory on this platform", async () => { + await withDataDir(async (dataDir) => { + const path = join(dataDir, "state.json") + await durableWrite(path, "old\n") + await durableWrite(path, "new\n") + strictEqual(readFileSync(path, "utf8"), "new\n") + strictEqual(lstatSync(path).mode & 0o777, 0o600) + await syncDirectory(dataDir) + }) + }) + + it("leaves the old destination intact when injected before file sync or rename", async () => { + await withDataDir(async (dataDir) => { + const path = join(dataDir, "state.json") + await durableWrite(path, "old\n") + await rejects( + durableWrite(path, "new\n", { + beforeFileSync: () => { + throw new Error("sync fault") + }, + }), + /sync fault/, ) - await rejects(readCheckpointManifest(dataDir), /checkpoint schema mismatch/) + strictEqual(readFileSync(path, "utf8"), "old\n") + await rejects( + durableWrite(path, "new\n", { + beforeRename: () => { + throw new Error("rename fault") + }, + }), + /rename fault/, + ) + strictEqual(readFileSync(path, "utf8"), "old\n") + strictEqual(readFileSync(path, "utf8"), "old\n", "fault must not partially publish new bytes") + }) + }) + + it("does not treat descriptor/type errors as unsupported directory sync", () => { + ok(isUnsupportedDirectorySyncError({ code: "EINVAL" })) + ok(isUnsupportedDirectorySyncError({ code: "ENOTSUP" })) + ok(!isUnsupportedDirectorySyncError({ code: "EBADF" })) + ok(!isUnsupportedDirectorySyncError({ code: "EISDIR" })) + }) + + it("refuses symlinks while syncing a checkpoint tree", async () => { + await withDataDir(async (dataDir) => { + const outside = join(dirname(dataDir), "outside.txt") + writeFileSync(outside, "outside") + symlinkSync(outside, join(dataDir, "link")) + await rejects(syncTree(dataDir), /non-file checkpoint entry/) + ok(existsSync(outside)) }) }) }) + +const throwsMessage = (run: () => unknown, expected: RegExp): void => { + try { + run() + throw new Error("expected function to throw") + } catch (error) { + match(error instanceof Error ? error.message : String(error), expected) + } +} From 1bd28156f5f88bf601bcee96ec8276d441149158 Mon Sep 17 00:00:00 2001 From: remote service administrator Date: Sat, 27 Jun 2026 16:59:27 -0400 Subject: [PATCH 06/12] fix(cli): reconcile interrupted checkpoint restores --- apps/cli/src/commands/server-args.ts | 28 ++ apps/cli/src/commands/server.ts | 46 ++- apps/cli/src/server/chdb.ts | 22 +- apps/cli/src/server/checkpoints.ts | 294 ++++++++++++++++-- apps/cli/src/server/durable-files.ts | 11 +- apps/cli/src/server/store-version.ts | 15 + apps/cli/test/checkpoints.test.ts | 156 +++++++++- apps/cli/test/native-checkpoint-smoke.sh | 184 +++++++++++ apps/cli/test/server-args.test.ts | 48 +++ .../content/docs/local-mode/cli-reference.md | 78 ++++- scripts/build-local-binary.sh | 4 +- 11 files changed, 821 insertions(+), 65 deletions(-) create mode 100644 apps/cli/src/commands/server-args.ts create mode 100755 apps/cli/test/native-checkpoint-smoke.sh create mode 100644 apps/cli/test/server-args.test.ts diff --git a/apps/cli/src/commands/server-args.ts b/apps/cli/src/commands/server-args.ts new file mode 100644 index 000000000..438ec41c8 --- /dev/null +++ b/apps/cli/src/commands/server-args.ts @@ -0,0 +1,28 @@ +export type DirtyStorePolicy = "wipe" | "fail" | "restore-checkpoint" + +export interface DetachedChildArgs { + readonly entry: string | undefined + readonly port: number + readonly dataDir: string + readonly offline: boolean + readonly chdbConfigFile: string | undefined + readonly onDirtyStore: DirtyStorePolicy +} + +/** Build the foreground child argv without forwarding compiled-Bun virtual + * entrypoints or the background flag that caused the re-exec. */ +export const buildDetachedChildArgs = (options: DetachedChildArgs): string[] => { + const runtimeArgs = options.entry && !options.entry.startsWith("/$bunfs") ? [options.entry] : [] + return [ + ...runtimeArgs, + "start", + "--port", + String(options.port), + "--data-dir", + options.dataDir, + "--on-dirty-store", + options.onDirtyStore, + ...(options.chdbConfigFile ? ["--chdb-config-file", options.chdbConfigFile] : []), + ...(options.offline ? ["--offline"] : []), + ] +} diff --git a/apps/cli/src/commands/server.ts b/apps/cli/src/commands/server.ts index 2c72573b7..36a864de0 100644 --- a/apps/cli/src/commands/server.ts +++ b/apps/cli/src/commands/server.ts @@ -15,10 +15,11 @@ import { storeMarkerPath, storeOpenMarkerPath, } from "../server/store-version" -import { createCheckpoint, restoreCheckpoint } from "../server/checkpoints" +import { createCheckpoint, reconcileCheckpointRecovery, restoreCheckpoint } from "../server/checkpoints" import { resolveUiAssets } from "../server/ui-assets" import { amber, bold, cyan, dim, green, underline } from "../lib/style" import { MAPLE_VERSION } from "../version" +import { buildDetachedChildArgs, type DirtyStorePolicy } from "./server-args" /** A `maple start`/`maple stop` failure. The message is shown to the user and * the process exits non-zero — same role the old `process.exit(1)` paths had, @@ -135,6 +136,12 @@ const yesFlag = Flag.boolean("yes").pipe( Flag.withDefault(false), ) +const checkpointIdFlag = Flag.optional( + Flag.string("checkpoint-id").pipe( + Flag.withDescription("Restore one immutable checkpoint ID instead of the selected current"), + ), +) + const offlineFlag = Flag.boolean("offline").pipe( Flag.withDescription( "Use the UI bundled in this binary (served from 127.0.0.1) instead of local.maple.dev", @@ -166,6 +173,7 @@ const startDetached = ( dataDir: string, offline: boolean, chdbConfigFile: string | undefined, + onDirtyStore: DirtyStorePolicy, ): Effect.Effect => Effect.gen(function* () { const logPath = logFilePath(dataDir) @@ -173,18 +181,14 @@ const startDetached = ( // binary injects a virtual `/$bunfs/...` entrypoint at argv[1] that must // not be forwarded. In dev (`bun run src/bin.ts`) argv[1] is the real // script and Bun needs it; in the compiled binary execPath alone suffices. - const entry = process.argv[1] - const runtimeArgs = entry && !entry.startsWith("/$bunfs") ? [entry] : [] - const childArgs = [ - ...runtimeArgs, - "start", - "--port", - String(port), - "--data-dir", + const childArgs = buildDetachedChildArgs({ + entry: process.argv[1], + port, dataDir, - ...(chdbConfigFile ? ["--chdb-config-file", chdbConfigFile] : []), - ...(offline ? ["--offline"] : []), - ] + offline, + chdbConfigFile, + onDirtyStore, + }) const child = yield* Effect.try({ try: () => { @@ -254,6 +258,12 @@ export const start = Command.make("start", { } if (Option.isSome(existingPid)) yield* fs.remove(pidPath, { force: true }).pipe(Effect.ignore) // stale + // A restore transaction lives beside dataDir and must be reconciled + // before reset, compatibility, dirty-store, or directory creation logic. + yield* reconcileCheckpointRecovery(dataDir).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) + // `--reset`: wipe the store (and its version marker) so we bootstrap fresh. // The escape hatch when an existing store is from an incompatible build. if (a.reset) { @@ -351,6 +361,7 @@ export const start = Command.make("start", { dataDir, a.offline, Option.getOrUndefined(a.chdbConfigFile), + a.onDirtyStore, ) yield* Effect.sync(() => @@ -513,6 +524,7 @@ export const checkpoint = Command.make("checkpoint", { dataDir: dataDirFlag, por yield* Effect.sync(() => process.stdout.write( `${green("✓")} checkpoint created\n` + + ` ${dim("id")} ${result.checkpointId}\n` + ` ${dim("path")} ${prettyPath(result.path)}\n` + ` ${dim("traces")} ${result.manifest.validation.traces}\n` + ` ${dim("logs")} ${result.manifest.validation.logs}\n` + @@ -524,7 +536,11 @@ export const checkpoint = Command.make("checkpoint", { dataDir: dataDirFlag, por ), ) -export const restore = Command.make("restore", { dataDir: dataDirFlag, yes: yesFlag }).pipe( +export const restore = Command.make("restore", { + dataDir: dataDirFlag, + checkpointId: checkpointIdFlag, + yes: yesFlag, +}).pipe( Command.withDescription("Restore the local chDB store from the last promoted checkpoint"), Command.withHandler( Effect.fnUntraced(function* (a) { @@ -549,12 +565,14 @@ export const restore = Command.make("restore", { dataDir: dataDirFlag, yes: yesF return } - const result = yield* restoreCheckpoint(dataDir).pipe( + const checkpointId = Option.getOrUndefined(a.checkpointId) + const result = yield* restoreCheckpoint(dataDir, checkpointId ?? "current").pipe( Effect.mapError((e) => new ServerError({ message: e.message })), ) yield* Effect.sync(() => process.stderr.write( `${green("✓")} restored checkpoint\n` + + ` ${dim("id")} ${result.checkpointId}\n` + ` ${dim("quarantine")} ${prettyPath(result.quarantinePath)}\n` + ` ${dim("traces")} ${result.validation.traces}\n` + ` ${dim("logs")} ${result.validation.logs}\n` + diff --git a/apps/cli/src/server/chdb.ts b/apps/cli/src/server/chdb.ts index 8d8caeafc..b8dea9e23 100644 --- a/apps/cli/src/server/chdb.ts +++ b/apps/cli/src/server/chdb.ts @@ -129,12 +129,22 @@ export class Chdb { argv[i] = BigInt(ptr(b)) }) const connPtrPtr = sym.chdb_connect(args.length, ptr(argv)) - if (!connPtrPtr) throw new Error(Chdb.#connectFailure(options.dataDir, "chdb_connect returned NULL")) + if (!connPtrPtr) { + throw new Error( + Chdb.#connectFailure(options.dataDir, "chdb_connect returned NULL", options.configFile), + ) + } // chdb_connect returns chdb_connection* (a double pointer); chdb_query // wants chdb_connection — dereference once. const conn = read.ptr(connPtrPtr, 0) as Pointer if (!conn) - throw new Error(Chdb.#connectFailure(options.dataDir, "chdb_connect produced a NULL connection")) + throw new Error( + Chdb.#connectFailure( + options.dataDir, + "chdb_connect produced a NULL connection", + options.configFile, + ), + ) const db = new Chdb(sym, connPtrPtr, conn) if (options.bootstrapSchema !== false) db.#bootstrap(options.schemaSql) @@ -146,7 +156,13 @@ export class Chdb { // kill); point the user at the recovery path rather than the raw libchdb // message. A failure over an empty dir is a different problem (missing/broken // libchdb), so keep the generic message there. - static #connectFailure(dataDir: string, raw: string): string { + static #connectFailure(dataDir: string, raw: string, configFile?: string): string { + if (configFile) { + return ( + `${raw} while loading the explicit chDB config at ${configFile}. ` + + "The existing store was not modified; correct or remove the config before retrying." + ) + } if (!storeHasData(dataDir)) return raw return ( `${raw} — the local store at ${dataDir} could not be opened ` + diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index f8fa5b709..348870a42 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto" import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from "node:fs" -import { cp, lstat, mkdir, readFile, readdir, rename, rm, stat } from "node:fs/promises" +import { cp, lstat, mkdir, readFile, readdir, rm, stat } from "node:fs/promises" import { tmpdir } from "node:os" import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path" import { Effect, Schema } from "effect" @@ -9,6 +9,7 @@ import { Chdb } from "./chdb" import { type DurabilityFaults, durableJson, + durableRemove, durableRename, ensurePrivateDirectory, syncDirectory, @@ -16,11 +17,12 @@ import { } from "./durable-files" import { SCHEMA_FINGERPRINT } from "./serve" import schemaSql from "./schema/local-schema.sql" with { type: "text" } -import { markStoreClosed, storeMarkerJson, storeMarkerPath } from "./store-version" +import { markStoreClosedDurable, writeStoreMarkerDurable } from "./store-version" const STATE_FORMAT_VERSION = 1 const MANIFEST_FORMAT_VERSION = 1 const OPERATION_FORMAT_VERSION = 1 +const RESTORE_TRANSACTION_FORMAT_VERSION = 1 const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i export class CheckpointError extends Schema.TaggedErrorClass()( @@ -90,6 +92,16 @@ interface MaintenanceOwner { readonly startedAt: string } +interface RestoreTransaction { + readonly formatVersion: 1 + readonly operationId: string + readonly checkpointId: string + readonly quarantineId: string + readonly phase: "intent" | "restore-ready" | "old-quarantined" | "new-live" | "markers-committed" + readonly createdAt: string + readonly validation: CheckpointValidation | null +} + export const checkpointRoot = (dataDir: string): string => join(resolve(dataDir), "backups") export const checkpointStatePath = (dataDir: string): string => join(checkpointRoot(dataDir), "state.json") export const checkpointSnapshotsRoot = (dataDir: string): string => join(checkpointRoot(dataDir), "snapshots") @@ -103,6 +115,17 @@ export const checkpointSnapshotDir = (dataDir: string, checkpointId: string): st join(checkpointSnapshotsRoot(dataDir), validateId(checkpointId, "checkpoint")) const maintenanceLockPath = (dataDir: string): string => `${resolve(dataDir)}.maple-maintenance-lock` +export const restoreTransactionPath = (dataDir: string): string => + `${resolve(dataDir)}.restore-transaction.json` +export const restoreRootPath = (dataDir: string, operationId: string): string => + `${resolve(dataDir)}.restore-${validateId(operationId, "restore operation")}` +export const restoreDataPath = (dataDir: string, operationId: string): string => + join(restoreRootPath(dataDir, operationId), "data") +export const restoreQuarantinePath = (dataDir: string, operationId: string, quarantineId: string): string => + `${resolve(dataDir)}.quarantine-${validateId(operationId, "restore operation")}-${validateId( + quarantineId, + "quarantine", + )}` const operationDir = (dataDir: string, operationId: string): string => join(checkpointOperationsRoot(dataDir), `checkpoint-${validateId(operationId, "operation")}`) const operationPath = (dataDir: string, operationId: string): string => @@ -401,8 +424,18 @@ const readStateFileOptional = async (dataDir: string): Promise => { const root = checkpointRoot(dataDir) try { - const entries = await readdir(root) - return entries.filter((entry) => entry !== "state.json").map((entry) => join(root, entry)) + const entries = await readdir(root, { withFileTypes: true }) + const unsafe: string[] = [] + for (const entry of entries) { + if (entry.name === "state.json" || entry.name === "quarantine") continue + const path = join(root, entry.name) + if (["snapshots", "operations", "pins", "retiring"].includes(entry.name)) { + if (!entry.isDirectory() || (await readdir(path)).length > 0) unsafe.push(path) + continue + } + unsafe.push(path) + } + return unsafe } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return [] throw error @@ -524,7 +557,10 @@ export const withRestoredCheckpoint = async ( ) { throw new Error("scratch root must not be the live data directory or one of its descendants") } - await ensurePrivateDirectory(scratchRoot) + if (existsSync(scratchRoot) && (await lstat(scratchRoot)).isSymbolicLink()) { + throw new Error(`scratch root must not be a symlink: ${scratchRoot}`) + } + await mkdir(scratchRoot, { recursive: true }) const scratchParent = join(scratchRoot, `maple-checkpoint-${randomUUID()}`) await mkdir(scratchParent, { mode: 0o700 }) const scratchDataDir = join(scratchParent, "data") @@ -824,6 +860,199 @@ export const createCheckpoint = ( new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), }) +const parseRestoreTransaction = (value: unknown): RestoreTransaction => { + if (!isRecord(value) || value.formatVersion !== RESTORE_TRANSACTION_FORMAT_VERSION) { + throw new Error("unsupported or malformed restore transaction") + } + const phase = requiredString(value, "phase") + if (!["intent", "restore-ready", "old-quarantined", "new-live", "markers-committed"].includes(phase)) { + throw new Error("invalid restore transaction phase") + } + return { + formatVersion: 1, + operationId: validateId(requiredString(value, "operationId"), "restore operation"), + checkpointId: validateId(requiredString(value, "checkpointId"), "checkpoint"), + quarantineId: validateId(requiredString(value, "quarantineId"), "quarantine"), + phase: phase as RestoreTransaction["phase"], + createdAt: requiredIso(value, "createdAt"), + validation: value.validation === null ? null : parseValidation(value.validation), + } +} + +const readRestoreTransaction = async (dataDir: string): Promise => { + try { + return parseRestoreTransaction(JSON.parse(await readFile(restoreTransactionPath(dataDir), "utf8"))) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null + throw error + } +} + +const writeRestoreTransaction = async (dataDir: string, transaction: RestoreTransaction) => + durableJson(restoreTransactionPath(dataDir), transaction) + +const restoreReadyPath = (dataDir: string, operationId: string): string => + join(restoreDataPath(dataDir, operationId), ".maple-restore-ready.json") + +const readyIdentityMatches = (dataDir: string, transaction: RestoreTransaction): boolean => { + const candidates = [ + restoreReadyPath(dataDir, transaction.operationId), + join(resolve(dataDir), ".maple-restore-ready.json"), + ] + for (const path of candidates) { + if (!existsSync(path)) continue + try { + const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown + if ( + isRecord(parsed) && + parsed.formatVersion === 1 && + parsed.operationId === transaction.operationId && + parsed.checkpointId === transaction.checkpointId + ) { + return true + } + } catch { + return false + } + } + return false +} + +const finalizeRestoreMarkers = async ( + dataDir: string, + transaction: RestoreTransaction, +): Promise => { + if (!readyIdentityMatches(dataDir, transaction)) { + throw new Error("restored live store identity is missing or does not match the transaction") + } + await writeStoreMarkerDurable(dataDir, MAPLE_VERSION, new Date().toISOString(), SCHEMA_FINGERPRINT) + await markStoreClosedDurable(dataDir) + const committed: RestoreTransaction = { ...transaction, phase: "markers-committed" } + await writeRestoreTransaction(dataDir, committed) + return committed +} + +const completeRestoreTransaction = async ( + dataDir: string, + transaction: RestoreTransaction, +): Promise => { + const readyPath = join(resolve(dataDir), ".maple-restore-ready.json") + if (existsSync(readyPath)) await durableRemove(readyPath) + const root = restoreRootPath(dataDir, transaction.operationId) + if (existsSync(root)) { + await rm(root, { recursive: true }) + await syncDirectory(dirname(root)) + } + await durableRemove(restoreTransactionPath(dataDir)) +} + +const reconcileRestoreTransactionUnlocked = async (dataDir: string): Promise => { + let transaction = await readRestoreTransaction(dataDir) + if (!transaction) { + const parent = dirname(resolve(dataDir)) + const prefix = `${basename(resolve(dataDir))}.restore-` + let names: string[] + try { + names = await readdir(parent) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + const debris = names + .filter( + (name) => + (name.startsWith(prefix) && !name.includes(".quarantine-")) || + name === `${basename(resolve(dataDir))}.restore-transaction.json`, + ) + .map((name) => join(parent, name)) + if (debris.length > 0) { + throw new Error( + `restore-like paths exist without a valid transaction; refusing to infer ownership: ${debris.join(", ")}`, + ) + } + return + } + const live = resolve(dataDir) + const restoreRoot = restoreRootPath(dataDir, transaction.operationId) + const restoreData = restoreDataPath(dataDir, transaction.operationId) + const quarantine = restoreQuarantinePath(dataDir, transaction.operationId, transaction.quarantineId) + const liveExists = existsSync(live) + const restoreExists = existsSync(restoreData) + const quarantineExists = existsSync(quarantine) + + if (transaction.phase === "intent") { + if (!liveExists || quarantineExists) { + throw new Error( + `ambiguous restore intent topology; live=${liveExists} restore=${restoreExists} quarantine=${quarantineExists}`, + ) + } + if (existsSync(restoreRoot)) { + await durableRename(restoreRoot, `${restoreRoot}.quarantine-${randomUUID()}`) + } + await durableRename( + restoreTransactionPath(dataDir), + `${restoreTransactionPath(dataDir)}.quarantine-${randomUUID()}`, + ) + return + } + + if (transaction.phase === "restore-ready" && liveExists && restoreExists && !quarantineExists) { + if (!readyIdentityMatches(dataDir, transaction)) { + throw new Error("restore-ready identity is missing or mismatched") + } + await durableRename(live, quarantine) + transaction = { ...transaction, phase: "old-quarantined" } + await writeRestoreTransaction(dataDir, transaction) + } + + if ( + (transaction.phase === "restore-ready" || transaction.phase === "old-quarantined") && + !existsSync(live) && + existsSync(restoreData) && + existsSync(quarantine) + ) { + await durableRename(restoreData, live) + transaction = { ...transaction, phase: "new-live" } + await writeRestoreTransaction(dataDir, transaction) + } + + if ( + (transaction.phase === "old-quarantined" || transaction.phase === "new-live") && + existsSync(live) && + !existsSync(restoreData) && + existsSync(quarantine) + ) { + transaction = await finalizeRestoreMarkers(dataDir, { + ...transaction, + phase: "new-live", + }) + } + + if (transaction.phase === "markers-committed" && existsSync(live) && existsSync(quarantine)) { + await completeRestoreTransaction(dataDir, transaction) + return + } + + throw new Error( + `restore transaction could not be reconciled safely; phase=${transaction.phase} live=${existsSync(live)} restore=${existsSync(restoreData)} quarantine=${existsSync(quarantine)}`, + ) +} + +export const reconcileCheckpointRecovery = (dataDir: string): Effect.Effect => + Effect.tryPromise({ + try: async () => { + const operationId = randomUUID() + const release = await acquireMaintenance(dataDir, operationId) + try { + await reconcileRestoreTransactionUnlocked(dataDir) + } finally { + await release() + } + }, + catch: (error) => + new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), + }) + export const restoreCheckpoint = ( dataDir: string, selector: "current" | "previous" | string = "current", @@ -838,37 +1067,50 @@ export const restoreCheckpoint = ( Effect.tryPromise({ try: async () => { const operationId = randomUUID() + const quarantineId = randomUUID() const release = await acquireMaintenance(dataDir, operationId) try { + await reconcileRestoreTransactionUnlocked(dataDir) const resolvedCheckpoint = await resolveCheckpoint(dataDir, selector) - const quarantinePath = `${resolve(dataDir)}.quarantine-${operationId}-${randomUUID()}` - if (existsSync(quarantinePath)) - throw new Error(`quarantine path already exists: ${quarantinePath}`) - const restored = await withRestoredCheckpoint( - resolvedCheckpoint, - { scratchRoot: dirname(resolve(dataDir)), cleanup: "never" }, - ({ scratchDataDir, validation }) => ({ scratchDataDir, validation }), - ) - await cp(checkpointRoot(dataDir), join(restored.scratchDataDir, "backups"), { + const restoreRoot = restoreRootPath(dataDir, operationId) + const restoreData = restoreDataPath(dataDir, operationId) + const quarantinePath = restoreQuarantinePath(dataDir, operationId, quarantineId) + if (existsSync(restoreRoot) || existsSync(quarantinePath)) { + throw new Error("restore or quarantine path already exists") + } + let transaction: RestoreTransaction = { + formatVersion: 1, + operationId, + checkpointId: resolvedCheckpoint.checkpointId, + quarantineId, + phase: "intent", + createdAt: new Date().toISOString(), + validation: null, + } + await writeRestoreTransaction(dataDir, transaction) + await mkdir(restoreRoot, { mode: 0o700 }) + const restored = await restoreResolvedInto(resolvedCheckpoint, restoreData) + const validation = restored.validation + restored.db.close() + await cp(checkpointRoot(dataDir), join(restoreData, "backups"), { recursive: true, force: false, errorOnExist: true, }) - await syncTree(join(restored.scratchDataDir, "backups")) - await rename(resolve(dataDir), quarantinePath) - await syncDirectory(dirname(resolve(dataDir))) - await rename(restored.scratchDataDir, resolve(dataDir)) - await syncDirectory(dirname(resolve(dataDir))) - markStoreClosed(dataDir) - writeFileSync( - storeMarkerPath(dataDir), - storeMarkerJson(MAPLE_VERSION, new Date().toISOString(), SCHEMA_FINGERPRINT), - { mode: 0o600 }, - ) + await syncTree(join(restoreData, "backups")) + await durableJson(restoreReadyPath(dataDir, operationId), { + formatVersion: 1, + operationId, + checkpointId: resolvedCheckpoint.checkpointId, + }) + await syncTree(restoreData, { allowSymlinks: true }) + transaction = { ...transaction, phase: "restore-ready", validation } + await writeRestoreTransaction(dataDir, transaction) + await reconcileRestoreTransactionUnlocked(dataDir) return { checkpointId: resolvedCheckpoint.checkpointId, quarantinePath, - validation: restored.validation, + validation, } } finally { await release() diff --git a/apps/cli/src/server/durable-files.ts b/apps/cli/src/server/durable-files.ts index 262819cd5..e83133651 100644 --- a/apps/cli/src/server/durable-files.ts +++ b/apps/cli/src/server/durable-files.ts @@ -91,12 +91,15 @@ export const durableRename = async ( if (dirname(to) !== dirname(from)) await syncDirectory(dirname(to), faults) } -export const syncTree = async (path: string): Promise => { +export const syncTree = async ( + path: string, + options: { readonly allowSymlinks?: boolean } = {}, +): Promise => { const entries = await readdir(path, { withFileTypes: true }) for (const entry of entries) { const child = join(path, entry.name) if (entry.isDirectory()) { - await syncTree(child) + await syncTree(child, options) } else if (entry.isFile()) { const handle = await open(child, constants.O_RDONLY) try { @@ -104,6 +107,10 @@ export const syncTree = async (path: string): Promise => { } finally { await handle.close() } + } else if (entry.isSymbolicLink() && options.allowSymlinks) { + // A closed chDB store contains engine-managed symlinks. Do not + // follow them; syncing this directory below durably records the link. + continue } else { throw new Error(`refusing to sync non-file checkpoint entry at ${child}`) } diff --git a/apps/cli/src/server/store-version.ts b/apps/cli/src/server/store-version.ts index 6d6198d44..1b97ecb95 100644 --- a/apps/cli/src/server/store-version.ts +++ b/apps/cli/src/server/store-version.ts @@ -14,6 +14,7 @@ import { createHash } from "node:crypto" import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs" import { dirname, join } from "node:path" import { CHDB_VERSION } from "../version" +import { durableRemove, durableWrite } from "./durable-files" export interface StoreMarker { /** libchdb version that created the store (e.g. "v26.1.0"); "dev" in dev builds. */ @@ -61,6 +62,12 @@ export const markStoreClosed = (dataDir: string): void => { } } +/** Durably clear the clean-shutdown sentinel for a restore transaction. */ +export const markStoreClosedDurable = async (dataDir: string): Promise => { + const path = storeOpenMarkerPath(dataDir) + if (existsSync(path)) await durableRemove(path) +} + /** True when the store holds data AND was not cleanly closed (the sentinel * survives). Gated on `storeHasData`: a marker over an empty store means a * fresh open that never persisted anything — safe to reopen. */ @@ -89,6 +96,14 @@ export const readMarker = (dataDir: string): StoreMarker | null => { export const storeMarkerJson = (maple: string, now: string, schema: string): string => `${JSON.stringify({ chdb: CHDB_VERSION, maple, createdAt: now, schema } satisfies StoreMarker, null, 2)}\n` +/** Atomically and durably stamp the store version after a live restore swap. */ +export const writeStoreMarkerDurable = async ( + dataDir: string, + maple: string, + now: string, + schema: string, +): Promise => durableWrite(storeMarkerPath(dataDir), storeMarkerJson(maple, now, schema)) + /** * Stable fingerprint of the bundled schema DDL. Comments and whitespace are * stripped first so cosmetic edits (a reworded comment, reindentation) don't diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index 1653fad99..5f5764d1c 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -1,4 +1,5 @@ import { describe, it } from "@effect/vitest" +import { Effect } from "effect" import { deepStrictEqual, match, ok, rejects, strictEqual } from "node:assert" import { existsSync, @@ -7,12 +8,13 @@ import { mkdtempSync, readFileSync, readdirSync, + renameSync, rmSync, symlinkSync, writeFileSync, } from "node:fs" import { tmpdir } from "node:os" -import { dirname, join } from "node:path" +import { basename, dirname, join } from "node:path" import { assertCheckpointRootSafe, checkpointRoot, @@ -24,8 +26,13 @@ import { parseCheckpointManifest, parseCheckpointState, readCheckpointState, + reconcileCheckpointRecovery, reconcileCheckpointOperations, resolveCheckpoint, + restoreDataPath, + restoreQuarantinePath, + restoreRootPath, + restoreTransactionPath, retireCheckpointIfEligible, writeBackupConfig, } from "../src/server/checkpoints" @@ -36,6 +43,7 @@ import { syncTree, } from "../src/server/durable-files" import { SCHEMA_FINGERPRINT } from "../src/server/serve" +import { storeMarkerPath, storeOpenMarkerPath } from "../src/server/store-version" import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" const withDataDir = async (run: (dataDir: string) => Promise | void): Promise => { @@ -98,6 +106,47 @@ const writeState = ( ) } +const restoreValidation = { + validatedAt: "2026-01-01T00:00:01.000Z", + traces: 1, + logs: 2, + metricsSum: 3, + metricsGauge: 4, + metricsHistogram: 5, + metricsExponentialHistogram: 6, + materializedViews: 33, +} + +const writeRestoreTransaction = ( + dataDir: string, + operationId: string, + checkpointId: string, + quarantineId: string, + phase: "intent" | "restore-ready" | "old-quarantined" | "new-live" | "markers-committed", +): void => { + writeFileSync( + restoreTransactionPath(dataDir), + `${JSON.stringify({ + formatVersion: 1, + operationId, + checkpointId, + quarantineId, + phase, + createdAt: "2026-01-01T00:00:00.000Z", + validation: phase === "intent" ? null : restoreValidation, + })}\n`, + ) +} + +const writeRestoreReady = (dataDir: string, operationId: string, checkpointId: string): void => { + const restoreData = restoreDataPath(dataDir, operationId) + mkdirSync(restoreData, { recursive: true }) + writeFileSync( + join(restoreData, ".maple-restore-ready.json"), + `${JSON.stringify({ formatVersion: 1, operationId, checkpointId })}\n`, + ) +} + describe("writeBackupConfig", () => { it("writes restrictive runtime and escaped restore configurations", async () => { await withDataDir((dataDir) => { @@ -214,7 +263,9 @@ describe("checkpoint state resolution", () => { await withDataDir(async (dataDir) => { await rejects(readCheckpointState(dataDir), /state not found/) - mkdirSync(join(checkpointRoot(dataDir), "snapshots"), { recursive: true }) + mkdirSync(join(checkpointRoot(dataDir), "snapshots", newCheckpointId()), { + recursive: true, + }) await rejects(readCheckpointState(dataDir), /state missing while checkpoint data exists/) writeFileSync(checkpointStatePath(dataDir), "{bad json") @@ -317,6 +368,106 @@ describe("checkpoint reconciliation and retention", () => { }) }) +describe("live restore transaction reconciliation", () => { + it("is a no-op when no transaction or restore debris exists", async () => { + await withDataDir(async (dataDir) => { + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + ok(existsSync(dataDir)) + }) + }) + + it("preserves an interrupted pre-ready restore and leaves the old live store selected", async () => { + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + const quarantineId = newCheckpointId() + writeFileSync(join(dataDir, "old-live"), "old") + writeRestoreTransaction(dataDir, operationId, checkpointId, quarantineId, "intent") + mkdirSync(restoreDataPath(dataDir, operationId), { recursive: true }) + writeFileSync(join(restoreDataPath(dataDir, operationId), "partial"), "partial") + + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + + ok(existsSync(join(dataDir, "old-live"))) + ok(!existsSync(restoreRootPath(dataDir, operationId))) + ok(!existsSync(restoreTransactionPath(dataDir))) + const siblingNames = readdirSync(dirname(dataDir)) + ok( + siblingNames.some((name) => + name.startsWith(`${basename(dataDir)}.restore-${operationId}.quarantine-`), + ), + ) + ok( + siblingNames.some((name) => + name.startsWith(`${basename(dataDir)}.restore-transaction.json.quarantine-`), + ), + ) + }) + }) + + it("resumes from restore-ready through quarantine, swap, durable markers, and completion", async () => { + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + const quarantineId = newCheckpointId() + const quarantine = restoreQuarantinePath(dataDir, operationId, quarantineId) + writeFileSync(join(dataDir, "old-live"), "old") + writeFileSync(storeOpenMarkerPath(dataDir), "999\n") + writeRestoreReady(dataDir, operationId, checkpointId) + writeFileSync(join(restoreDataPath(dataDir, operationId), "new-live"), "new") + writeRestoreTransaction(dataDir, operationId, checkpointId, quarantineId, "restore-ready") + + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + + ok(existsSync(join(dataDir, "new-live"))) + ok(existsSync(join(quarantine, "old-live"))) + ok(existsSync(storeMarkerPath(dataDir))) + ok(!existsSync(storeOpenMarkerPath(dataDir))) + ok(!existsSync(restoreTransactionPath(dataDir))) + ok(!existsSync(restoreRootPath(dataDir, operationId))) + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + }) + }) + + it("infers the recorded rename boundary from exact topology and completes idempotently", async () => { + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + const quarantineId = newCheckpointId() + const quarantine = restoreQuarantinePath(dataDir, operationId, quarantineId) + writeFileSync(join(dataDir, "old-live"), "old") + writeRestoreReady(dataDir, operationId, checkpointId) + writeFileSync(join(restoreDataPath(dataDir, operationId), "new-live"), "new") + writeRestoreTransaction(dataDir, operationId, checkpointId, quarantineId, "restore-ready") + renameSync(dataDir, quarantine) + + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + + ok(existsSync(join(dataDir, "new-live"))) + ok(existsSync(join(quarantine, "old-live"))) + ok(!existsSync(restoreTransactionPath(dataDir))) + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + }) + }) + + it("fails closed on malformed or unrecorded restore state without deleting it", async () => { + await withDataDir(async (dataDir) => { + writeFileSync(restoreTransactionPath(dataDir), "{bad json") + await rejects(Effect.runPromise(reconcileCheckpointRecovery(dataDir))) + ok(existsSync(restoreTransactionPath(dataDir))) + rmSync(restoreTransactionPath(dataDir)) + + const debris = `${dataDir}.restore-${newCheckpointId()}` + mkdirSync(debris) + await rejects( + Effect.runPromise(reconcileCheckpointRecovery(dataDir)), + /without a valid transaction/, + ) + ok(existsSync(debris)) + }) + }) +}) + describe("backup configuration classification", () => { it("classifies only backup-specific errors", () => { ok( @@ -386,6 +537,7 @@ describe("durable filesystem primitives", () => { writeFileSync(outside, "outside") symlinkSync(outside, join(dataDir, "link")) await rejects(syncTree(dataDir), /non-file checkpoint entry/) + await syncTree(dataDir, { allowSymlinks: true }) ok(existsSync(outside)) }) }) diff --git a/apps/cli/test/native-checkpoint-smoke.sh b/apps/cli/test/native-checkpoint-smoke.sh new file mode 100755 index 000000000..7360ba0c2 --- /dev/null +++ b/apps/cli/test/native-checkpoint-smoke.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +set -euo pipefail + +BUNDLE_DIR="${1:?usage: native-checkpoint-smoke.sh [port]}" +MAPLE="$BUNDLE_DIR/maple" +PORT="${2:-45231}" +ROOT="$(mktemp -d "${TMPDIR:-/tmp}/maple-native-checkpoint.XXXXXX")" +DATA="$ROOT/data" +CONFIG="$ROOT/backups.xml" +SERVER_PID="" + +cleanup() { + if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + if [[ "${KEEP_ROOT:-0}" == "1" ]]; then + echo "preserved smoke root: $ROOT" >&2 + else + rm -rf "$ROOT" + fi +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +query() { + local sql="$1" + curl --fail-with-body -sS "http://127.0.0.1:$PORT/local/query" \ + -H 'content-type: application/json' \ + --data "$(jq -nc --arg sql "$sql" '{sql:$sql}')" +} + +wait_health() { + for _ in $(seq 1 200); do + if curl -fsS "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then + return + fi + sleep 0.1 + done + fail "server did not become healthy; log follows: $(tail -80 "$ROOT/server.log" 2>/dev/null)" +} + +start_server() { + local policy="${1:-fail}" + "$MAPLE" start \ + --port "$PORT" \ + --data-dir "$DATA" \ + --chdb-config-file "$CONFIG" \ + --on-dirty-store "$policy" \ + --offline >"$ROOT/server.log" 2>&1 & + SERVER_PID=$! + wait_health +} + +stop_server() { + "$MAPLE" stop --data-dir "$DATA" >/dev/null + wait "$SERVER_PID" 2>/dev/null || true + SERVER_PID="" +} + +kill_server() { + kill -9 "$SERVER_PID" + wait "$SERVER_PID" 2>/dev/null || true + SERVER_PID="" +} + +checkpoint() { + if ! "$MAPLE" checkpoint --port "$PORT" --data-dir "$DATA" >"$ROOT/checkpoint.out" 2>&1; then + cat "$ROOT/checkpoint.out" >&2 + return 1 + fi + if [[ ! -f "$DATA/backups/state.json" ]]; then + cat "$ROOT/checkpoint.out" >&2 + fail "checkpoint command returned without publishing state" + fi + jq -r '.current' "$DATA/backups/state.json" +} + +insert_marker() { + local suffix="$1" + # /local/query appends FORMAT JSONEachRow, so INSERT SELECT is used instead + # of VALUES (whose input parser would consume the appended FORMAT token). + query "INSERT INTO logs (OrgId, Timestamp, TimestampTime, TraceId, SpanId, TraceFlags, SeverityText, SeverityNumber, ServiceName, Body) SELECT 'local', now64(9), now(), 'trace-$suffix', 'span-$suffix', 1, 'INFO', 9, 'checkpoint-smoke', 'marker-$suffix'" >/dev/null + query "INSERT INTO traces (OrgId, Timestamp, TraceId, SpanId, ParentSpanId, TraceState, SpanName, SpanKind, ServiceName, StatusCode, StatusMessage) SELECT 'local', now64(9), 'trace-$suffix', 'span-$suffix', '', '', 'marker-$suffix', 'Server', 'checkpoint-smoke', 'Ok', ''" >/dev/null + query "INSERT INTO metrics_sum (OrgId, ServiceName, MetricName, StartTimeUnix, TimeUnix, Value, AggregationTemporality, IsMonotonic) SELECT 'local', 'checkpoint-smoke', 'sum-$suffix', now64(9), now64(9), 1, 2, true" >/dev/null + query "INSERT INTO metrics_gauge (OrgId, ServiceName, MetricName, StartTimeUnix, TimeUnix, Value) SELECT 'local', 'checkpoint-smoke', 'gauge-$suffix', now64(9), now64(9), 1" >/dev/null + query "INSERT INTO metrics_histogram (OrgId, ServiceName, MetricName, StartTimeUnix, TimeUnix, Count, Sum, BucketCounts, ExplicitBounds, AggregationTemporality) SELECT 'local', 'checkpoint-smoke', 'histogram-$suffix', now64(9), now64(9), 1, 1, [1], [1.0], 2" >/dev/null + query "INSERT INTO metrics_exponential_histogram (OrgId, ServiceName, MetricName, StartTimeUnix, TimeUnix, Count, Sum, Scale, ZeroCount, PositiveOffset, PositiveBucketCounts, NegativeOffset, NegativeBucketCounts, AggregationTemporality) SELECT 'local', 'checkpoint-smoke', 'exponential-$suffix', now64(9), now64(9), 1, 1, 0, 0, 0, [1], 0, [], 2" >/dev/null +} + +assert_counts() { + local expected="$1" + for table in logs traces metrics_sum metrics_gauge metrics_histogram metrics_exponential_histogram; do + local actual + actual="$(query "SELECT count() AS count FROM $table WHERE ServiceName = 'checkpoint-smoke'" | + jq -r '.[0].count | tonumber')" + [[ "$actual" == "$expected" ]] || fail "$table count: expected $expected, got $actual" + done +} + +printf '%s\n' \ + '' \ + ' ' \ + ' default' \ + ' backups' \ + ' ' \ + '' >"$CONFIG" +chmod 600 "$CONFIG" + +echo "native smoke root: $ROOT" + +# Prove the real missing-config error is classified narrowly and actionably. +"$MAPLE" start --port "$PORT" --data-dir "$DATA" --on-dirty-store fail --offline \ + >"$ROOT/server.log" 2>&1 & +SERVER_PID=$! +wait_health +set +e +"$MAPLE" checkpoint --port "$PORT" --data-dir "$DATA" >"$ROOT/no-config.out" 2>&1 +no_config_status=$? +set -e +[[ "$no_config_status" -ne 0 ]] || fail "checkpoint without backup config unexpectedly succeeded" +grep -q -- '--chdb-config-file' "$ROOT/no-config.out" || + fail "missing-config failure was not actionable: $(cat "$ROOT/no-config.out")" +stop_server + +start_server +insert_marker A +C1="$(checkpoint)" +[[ "$C1" =~ ^[0-9a-f-]{36}$ ]] || fail "invalid C1 ID: $C1" +insert_marker B +C2="$(checkpoint)" +[[ "$C2" =~ ^[0-9a-f-]{36}$ ]] || fail "invalid C2 ID: $C2" +[[ "$C1" != "$C2" ]] || fail "checkpoint IDs collided" +jq -e --arg c "$C2" --arg p "$C1" \ + '.formatVersion == 1 and .current == $c and .previous == $p' \ + "$DATA/backups/state.json" >/dev/null +stop_server + +"$MAPLE" restore --data-dir "$DATA" --checkpoint-id "$C1" --yes >/dev/null +start_server +assert_counts 1 +stop_server + +"$MAPLE" restore --data-dir "$DATA" --checkpoint-id "$C2" --yes >/dev/null +start_server +assert_counts 2 + +# Foreground dirty-store recovery. +kill_server +start_server restore-checkpoint +assert_counts 2 + +# Detached recovery exercises the exact re-exec argument path. +kill_server +"$MAPLE" start \ + --background \ + --port "$PORT" \ + --data-dir "$DATA" \ + --chdb-config-file "$CONFIG" \ + --on-dirty-store restore-checkpoint \ + --offline >"$ROOT/detached.out" +SERVER_PID="$(cat "$(dirname "$DATA")/maple.pid")" +wait_health +assert_counts 2 + +insert_marker C +C3="$(checkpoint)" +[[ "$C3" =~ ^[0-9a-f-]{36}$ ]] || fail "invalid C3 ID: $C3" +jq -e --arg c "$C3" --arg p "$C2" \ + '.current == $c and .previous == $p' "$DATA/backups/state.json" >/dev/null +[[ ! -e "$DATA/backups/snapshots/$C1" ]] || fail "old previous C1 was not retired" +[[ -d "$DATA/backups/snapshots/$C2" ]] || fail "previous C2 is missing" +[[ -d "$DATA/backups/snapshots/$C3" ]] || fail "current C3 is missing" + +stop_server +start_server fail +assert_counts 3 +stop_server + +echo "PASS native checkpoint smoke: C1=$C1 C2=$C2 C3=$C3" diff --git a/apps/cli/test/server-args.test.ts b/apps/cli/test/server-args.test.ts new file mode 100644 index 000000000..8e6eeeb10 --- /dev/null +++ b/apps/cli/test/server-args.test.ts @@ -0,0 +1,48 @@ +import { describe, it } from "@effect/vitest" +import { deepStrictEqual, strictEqual } from "node:assert" +import { buildDetachedChildArgs, type DirtyStorePolicy } from "../src/commands/server-args" + +describe("buildDetachedChildArgs", () => { + for (const policy of ["wipe", "fail", "restore-checkpoint"] satisfies DirtyStorePolicy[]) { + it(`forwards ${policy} exactly once`, () => { + const args = buildDetachedChildArgs({ + entry: "/repo/apps/cli/src/bin.ts", + port: 4318, + dataDir: "/tmp/maple data", + offline: true, + chdbConfigFile: "/tmp/backup config.xml", + onDirtyStore: policy, + }) + deepStrictEqual(args, [ + "/repo/apps/cli/src/bin.ts", + "start", + "--port", + "4318", + "--data-dir", + "/tmp/maple data", + "--on-dirty-store", + policy, + "--chdb-config-file", + "/tmp/backup config.xml", + "--offline", + ]) + strictEqual(args.filter((arg) => arg === "--on-dirty-store").length, 1) + strictEqual(args.includes("--background"), false) + strictEqual(args.includes("-d"), false) + }) + } + + it("omits the virtual compiled entrypoint and optional flags", () => { + deepStrictEqual( + buildDetachedChildArgs({ + entry: "/$bunfs/root/maple", + port: 4418, + dataDir: "/data", + offline: false, + chdbConfigFile: undefined, + onDirtyStore: "fail", + }), + ["start", "--port", "4418", "--data-dir", "/data", "--on-dirty-store", "fail"], + ) + }) +}) diff --git a/apps/landing/src/content/docs/local-mode/cli-reference.md b/apps/landing/src/content/docs/local-mode/cli-reference.md index 1c325b11b..ebdbc62e5 100644 --- a/apps/landing/src/content/docs/local-mode/cli-reference.md +++ b/apps/landing/src/content/docs/local-mode/cli-reference.md @@ -40,15 +40,15 @@ Local mode only. `maple start` is the long-lived process that owns the embedded Start the local ingest + query server (embedded ClickHouse via chDB). -| Flag | Default | Description | -| -------------------- | --------------- | ----------------------------------------------------------------------------------- | -| `--port ` | `4318` | Port for OTLP/HTTP ingest, the query API, and the bundled UI | -| `--data-dir ` | `~/.maple/data` | Embedded ClickHouse data directory | -| `--chdb-config-file ` | | Optional ClickHouse config file passed to embedded chDB | -| `--offline` | `false` | Serve the UI bundled in this binary (from `127.0.0.1`) instead of `local.maple.dev` | -| `--background`, `-d` | `false` | Run detached (logs to `~/.maple/maple.log`); stop with `maple stop` | -| `--reset` | `false` | Wipe the existing store before starting — use after an incompatible upgrade | -| `--on-dirty-store ` | `wipe` | Recovery policy when the store was not cleanly closed | +| Flag | Default | Description | +| --------------------------------------------------- | --------------- | ----------------------------------------------------------------------------------- | +| `--port ` | `4318` | Port for OTLP/HTTP ingest, the query API, and the bundled UI | +| `--data-dir ` | `~/.maple/data` | Embedded ClickHouse data directory | +| `--chdb-config-file ` | | Optional ClickHouse config file passed to embedded chDB | +| `--offline` | `false` | Serve the UI bundled in this binary (from `127.0.0.1`) instead of `local.maple.dev` | +| `--background`, `-d` | `false` | Run detached (logs to `~/.maple/maple.log`); stop with `maple stop` | +| `--reset` | `false` | Wipe the existing store before starting — use after an incompatible upgrade | +| `--on-dirty-store ` | `wipe` | Recovery policy when the store was not cleanly closed | ```bash maple start # foreground, UI from local.maple.dev @@ -56,6 +56,12 @@ maple start --offline # foreground, bundled UI, no internet needed maple start -d --port 4400 # detached on a custom port ``` +Detached startup forwards the selected `--on-dirty-store` policy unchanged to +the foreground child. Before any reset, compatibility check, dirty-store +decision, or data-directory creation, startup reconciles a recorded checkpoint +restore transaction. Ambiguous or malformed restore state fails closed and +prints the preserved paths. + ### `maple stop` Stop a running `maple start` server (reads the PID file beside the data dir). @@ -92,9 +98,33 @@ maple start --chdb-config-file ./chdb-backups.xml maple checkpoint ``` -Checkpoints are written under the data directory at -`backups/{building,current,previous}`. `building` is never used for restore; -only a validated checkpoint is promoted to `current`. +Every completed checkpoint receives an immutable UUID and is written under: + +```text +/backups/ + state.json + snapshots// + backup/ + manifest.json + operations/ + pins/ + quarantine/ + retiring/ +``` + +`state.json` is the only authority for the selected `current` and `previous` +IDs. Maple writes and syncs a strict versioned manifest only after restoring +the native backup into one sacrificial chDB and validating all six raw telemetry +tables. It then selects the snapshot with a synced atomic state-file +replacement. A third checkpoint retires the old previous snapshot only when it +is complete, compatible, unreferenced, and unpinned; uncertain state is +preserved. + +The earlier unreleased `backups/{building,current,previous}` preview layout is +not inferred or deleted. If it is present without a valid new state pointer, +checkpoint commands fail closed and report the paths for operator inspection. +Missing, malformed, incompatible, incomplete, or symlinked checkpoint state is +also rejected rather than guessed. ### `maple restore` @@ -102,10 +132,26 @@ Restore the local chDB store from the last promoted checkpoint. Refuses to run while a server still owns the store. The existing store is moved aside for quarantine rather than deleted. -| Flag | Default | Description | -| ------------------- | --------------- | ---------------------------- | -| `--data-dir ` | `~/.maple/data` | Store to restore | -| `--yes`, `-y` | `false` | Skip the confirmation prompt | +| Flag | Default | Description | +| ------------------------ | ---------------- | ----------------------------------- | +| `--data-dir ` | `~/.maple/data` | Store to restore | +| `--checkpoint-id ` | selected current | Restore one immutable checkpoint ID | +| `--yes`, `-y` | `false` | Skip the confirmation prompt | + +Restore uses a collision-resistant working path and quarantine, and records a +durable sibling transaction before changing the live directory. Reconciliation +can resume the recorded quarantine, live swap, and marker-update boundaries +idempotently. The displaced live store is never deleted. Unrecorded or +mismatched restore-like paths fail closed without mutation. + +```bash +maple restore --yes +maple restore --checkpoint-id 01234567-89ab-4cde-8fab-0123456789ab --yes +``` + +Checkpoint and restore operations share one maintenance lock. A live owner is +reported as busy; uncertain ownership is preserved and blocks destructive +maintenance. ## Services diff --git a/scripts/build-local-binary.sh b/scripts/build-local-binary.sh index 33811d20a..6453103aa 100755 --- a/scripts/build-local-binary.sh +++ b/scripts/build-local-binary.sh @@ -38,10 +38,10 @@ mkdir -p "$OUT_DIR" # checkout / CI only runs `bun install`, so the dist won't exist and # `bun build --compile` fails to resolve the import. Build it first. echo "==> Building @maple-dev/effect-sdk (telemetry SDK consumed by the CLI)" -bun --filter @maple-dev/effect-sdk build +bun run --filter @maple-dev/effect-sdk build echo "==> Building local-ui SPA" -bun --filter @maple/local-ui build +bun run --filter @maple/local-ui build echo "==> Inlining SPA into ui-embed.gen.ts" restore_stub() { git -C "$REPO_ROOT" checkout -- "$UI_EMBED" 2>/dev/null || true; } From e78435d52e446fdb13ef0734cd058a968c0aca50 Mon Sep 17 00:00:00 2001 From: remote service administrator Date: Sat, 27 Jun 2026 22:07:59 -0400 Subject: [PATCH 07/12] fix(cli): preserve checkpoints across recovery --- apps/cli/src/commands/server.ts | 65 +-- apps/cli/src/server/chdb.ts | 6 +- apps/cli/src/server/checkpoints.ts | 511 +++++++++++++++--- apps/cli/src/server/durable-files.ts | 14 +- apps/cli/test/checkpoints.test.ts | 335 +++++++++++- apps/cli/test/native-checkpoint-smoke.sh | 25 + .../content/docs/local-mode/cli-reference.md | 20 +- 7 files changed, 859 insertions(+), 117 deletions(-) diff --git a/apps/cli/src/commands/server.ts b/apps/cli/src/commands/server.ts index 36a864de0..6f217fca9 100644 --- a/apps/cli/src/commands/server.ts +++ b/apps/cli/src/commands/server.ts @@ -13,9 +13,13 @@ import { isStoreDirty, storeMarkerJson, storeMarkerPath, - storeOpenMarkerPath, } from "../server/store-version" -import { createCheckpoint, reconcileCheckpointRecovery, restoreCheckpoint } from "../server/checkpoints" +import { + createCheckpoint, + reconcileCheckpointRecovery, + resetLiveStorePreservingCheckpoints, + restoreCheckpoint, +} from "../server/checkpoints" import { resolveUiAssets } from "../server/ui-assets" import { amber, bold, cyan, dim, green, underline } from "../lib/style" import { MAPLE_VERSION } from "../version" @@ -120,14 +124,14 @@ const backgroundFlag = Flag.boolean("background").pipe( const resetFlag = Flag.boolean("reset").pipe( Flag.withDescription( - "Wipe the existing store (~/.maple/data) before starting — use after an incompatible upgrade", + "Wipe live chDB data before starting while preserving checkpoints — use after an incompatible upgrade", ), Flag.withDefault(false), ) const onDirtyStoreFlag = Flag.choice("on-dirty-store", ["wipe", "fail", "restore-checkpoint"]).pipe( Flag.withDescription("Recovery policy when the local chDB store was not cleanly closed"), - Flag.withDefault("wipe" as const), + Flag.withDefault("fail" as const), ) const yesFlag = Flag.boolean("yes").pipe( @@ -265,11 +269,11 @@ export const start = Command.make("start", { ) // `--reset`: wipe the store (and its version marker) so we bootstrap fresh. - // The escape hatch when an existing store is from an incompatible build. + // Preserve the checkpoint registry under dataDir/backups. if (a.reset) { - yield* fs.remove(dataDir, { recursive: true, force: true }).pipe(Effect.ignore) - yield* fs.remove(storeMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.remove(storeOpenMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) + yield* resetLiveStorePreservingCheckpoints(dataDir).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) } yield* fs.makeDirectory(dataDir, { recursive: true }) @@ -322,13 +326,13 @@ export const start = Command.make("start", { process.stderr.write( amber( "⚠ the local store was left inconsistent by an unclean shutdown — " + - "wiping it and starting fresh (local telemetry data is discarded)\n", + "explicit wipe selected; discarding live telemetry while preserving checkpoints\n", ), ), ) - yield* fs.remove(dataDir, { recursive: true, force: true }).pipe(Effect.ignore) - yield* fs.remove(storeMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.remove(storeOpenMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) + yield* resetLiveStorePreservingCheckpoints(dataDir).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) yield* fs.makeDirectory(dataDir, { recursive: true }) } } @@ -337,21 +341,15 @@ export const start = Command.make("start", { // place: `CREATE … IF NOT EXISTS` is a no-op on existing tables, so a // column added to the schema (e.g. ServiceNamespace on trace_list_mv) // never lands and facet queries referencing it fail. Rebuild from the - // current schema. Safe to auto-wipe — local telemetry is ephemeral and - // re-ingested — and only triggers once per schema change. + // current schema. Do not silently delete telemetry or checkpoints: + // require an explicit reset, which preserves the checkpoint registry. if (isSchemaStale(dataDir, SCHEMA_FINGERPRINT)) { - yield* Effect.sync(() => - process.stderr.write( - amber( - "⚠ the local store was built from an older schema — " + - "rebuilding it from the current schema (local telemetry data is discarded)\n", - ), - ), - ) - yield* fs.remove(dataDir, { recursive: true, force: true }).pipe(Effect.ignore) - yield* fs.remove(storeMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.remove(storeOpenMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.makeDirectory(dataDir, { recursive: true }) + return yield* new ServerError({ + message: + `the local store at ${prettyPath(dataDir)} was built from an older schema. ` + + `Maple preserved it and its checkpoints; explicitly rebuild live data with ` + + `\`${bold("maple start --reset")}\` or \`${bold("maple reset --yes")}\`.`, + }) } // Detached: spawn the same command without --background and exit. @@ -477,7 +475,7 @@ export const stop = Command.make("stop", { dataDir: dataDirFlag }).pipe( export const reset = Command.make("reset", { dataDir: dataDirFlag, yes: yesFlag }).pipe( Command.withDescription( - "Delete the local chDB store (~/.maple/data) so the next `maple start` bootstraps fresh", + "Delete live chDB data while preserving checkpoints so the next start bootstraps fresh", ), Command.withHandler( Effect.fnUntraced(function* (a) { @@ -496,18 +494,21 @@ export const reset = Command.make("reset", { dataDir: dataDirFlag, yes: yesFlag if (!a.yes) { yield* Effect.sync(() => process.stderr.write( - `This permanently deletes the local store at ${bold(prettyPath(dataDir))}.\n` + + `This permanently deletes live telemetry at ${bold(prettyPath(dataDir))}.\n` + + `The checkpoint registry under its backups directory is preserved.\n` + `Re-run with ${bold("maple reset --yes")} to confirm.\n`, ), ) return } - yield* fs.remove(dataDir, { recursive: true, force: true }).pipe(Effect.ignore) - yield* fs.remove(storeMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.remove(storeOpenMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) + yield* resetLiveStorePreservingCheckpoints(dataDir).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) yield* Effect.sync(() => - process.stderr.write(`${green("✓")} reset — removed ${prettyPath(dataDir)}\n`), + process.stderr.write( + `${green("✓")} reset — cleared live data and preserved checkpoints at ${prettyPath(dataDir)}\n`, + ), ) }), ), diff --git a/apps/cli/src/server/chdb.ts b/apps/cli/src/server/chdb.ts index b8dea9e23..3823c8b2f 100644 --- a/apps/cli/src/server/chdb.ts +++ b/apps/cli/src/server/chdb.ts @@ -114,8 +114,10 @@ export class Chdb { // `recursive_mutex lock failed: Invalid argument` → `ASYNC_LOAD_WAIT_FAILED` // → chdb_connect returns NULL. Most visible after an unclean kill, when more // load/merge work is still in flight on reopen. Waiting serializes load before - // bootstrap and removes the race. (`--async_load_system_database=0` is already - // the default in this build; set for symmetry / future-proofing.) + // bootstrap and substantially narrows the race. One fresh-process native smoke + // still observed a non-reproducible ASYNC_LOAD_WAIT_FAILED before 40 focused + // restart passes; never retry a failed FFI open in this process. + // (`--async_load_system_database=0` is set explicitly for symmetry.) const args = [ "clickhouse", "--async_load_databases=0", diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index 348870a42..b6f9bd46c 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -17,7 +17,12 @@ import { } from "./durable-files" import { SCHEMA_FINGERPRINT } from "./serve" import schemaSql from "./schema/local-schema.sql" with { type: "text" } -import { markStoreClosedDurable, writeStoreMarkerDurable } from "./store-version" +import { + markStoreClosedDurable, + storeMarkerPath, + storeOpenMarkerPath, + writeStoreMarkerDurable, +} from "./store-version" const STATE_FORMAT_VERSION = 1 const MANIFEST_FORMAT_VERSION = 1 @@ -81,7 +86,15 @@ interface CheckpointOperation { readonly formatVersion: 1 readonly operationId: string readonly checkpointId: string - readonly phase: "intent" | "backup-complete" | "manifest-complete" | "pointer-complete" + readonly baseRevision: string | null + readonly baseCurrent: string | null + readonly basePrevious: string | null + readonly phase: + | "intent" + | "backup-complete" + | "manifest-complete" + | "pointer-complete" + | "retention-complete" readonly startedAt: string } @@ -102,6 +115,18 @@ interface RestoreTransaction { readonly validation: CheckpointValidation | null } +export interface RestoreRecoveryFaults { + readonly afterLiveQuarantineRename?: () => void | Promise + readonly afterOldQuarantinedRecord?: () => void | Promise + readonly afterRestoredLiveRename?: () => void | Promise + readonly afterNewLiveRecord?: () => void | Promise + readonly afterStoreMarkerWrite?: () => void | Promise + readonly afterOpenMarkerRemoval?: () => void | Promise + readonly afterMarkersCommittedRecord?: () => void | Promise + readonly afterReadyMarkerRemoval?: () => void | Promise + readonly afterRestoreRootRemoval?: () => void | Promise +} + export const checkpointRoot = (dataDir: string): string => join(resolve(dataDir), "backups") export const checkpointStatePath = (dataDir: string): string => join(checkpointRoot(dataDir), "state.json") export const checkpointSnapshotsRoot = (dataDir: string): string => join(checkpointRoot(dataDir), "snapshots") @@ -202,6 +227,34 @@ const assertNoSymlink = async (root: string, candidate: string): Promise = } } +const assertRealDirectory = async (path: string, label: string): Promise => { + const info = await lstat(path) + if (info.isSymbolicLink() || !info.isDirectory()) { + throw new Error(`${label} must be a real directory: ${path}`) + } +} + +const assertRealFile = async (path: string, label: string): Promise => { + const info = await lstat(path) + if (info.isSymbolicLink() || !info.isFile()) { + throw new Error(`${label} must be a real file: ${path}`) + } +} + +const assertCheckpointInfrastructureSafe = async (dataDir: string): Promise => { + const live = resolve(dataDir) + if (existsSync(live)) await assertRealDirectory(live, "live data directory") + const root = checkpointRoot(dataDir) + if (!existsSync(root)) return + await assertRealDirectory(root, "checkpoint root") + for (const child of ["snapshots", "operations", "pins", "quarantine", "retiring"]) { + const path = join(root, child) + if (existsSync(path)) await assertRealDirectory(path, `checkpoint ${child}`) + } + const statePath = checkpointStatePath(dataDir) + if (existsSync(statePath)) await assertRealFile(statePath, "checkpoint state") +} + const xmlEscape = (value: string): string => value .replace(/&/g, "&") @@ -344,6 +397,7 @@ const parseValidation = (value: unknown): CheckpointValidation => { export const parseCheckpointManifest = ( value: unknown, expectedCheckpointId?: string, + expectedSourceDataDir?: string, ): CheckpointManifest => { if (!isRecord(value) || value.formatVersion !== MANIFEST_FORMAT_VERSION) { throw new Error("unsupported or malformed checkpoint manifest") @@ -355,6 +409,9 @@ export const parseCheckpointManifest = ( const operationId = validateId(requiredString(value, "operationId"), "operation") const sourceDataDir = requiredString(value, "sourceDataDir") if (!isAbsolute(sourceDataDir)) throw new Error("checkpoint sourceDataDir must be absolute") + if (expectedSourceDataDir && resolve(sourceDataDir) !== resolve(expectedSourceDataDir)) { + throw new Error("checkpoint sourceDataDir does not match its configured owner") + } const backupRelativePath = requiredString(value, "backupRelativePath") if (backupRelativePath !== snapshotBackupRelativePath(checkpointId)) { throw new Error("checkpoint backup path does not match its immutable ID") @@ -414,6 +471,7 @@ export const parseCheckpointState = (value: unknown): CheckpointState => { const readStateFileOptional = async (dataDir: string): Promise => { try { + await assertCheckpointInfrastructureSafe(dataDir) return parseCheckpointState(JSON.parse(await readFile(checkpointStatePath(dataDir), "utf8"))) } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null @@ -469,6 +527,33 @@ export const readCheckpointState = async (dataDir: string): Promise => { + await assertCheckpointInfrastructureSafe(dataDir) + const validatedCheckpointId = validateId(checkpointId, "checkpoint") + const snapshotDir = checkpointSnapshotDir(dataDir, validatedCheckpointId) + const snapshotsRoot = checkpointSnapshotsRoot(dataDir) + await assertNoSymlink(checkpointRoot(dataDir), snapshotsRoot) + await assertNoSymlink(snapshotsRoot, snapshotDir) + await assertNoSymlink(snapshotsRoot, snapshotManifestPath(dataDir, validatedCheckpointId)) + await assertNoSymlink(snapshotsRoot, snapshotBackupDir(dataDir, validatedCheckpointId)) + await assertRealDirectory(snapshotDir, "checkpoint snapshot") + await assertRealFile(snapshotManifestPath(dataDir, validatedCheckpointId), "checkpoint manifest") + const manifest = parseCheckpointManifest( + JSON.parse(await readFile(snapshotManifestPath(dataDir, validatedCheckpointId), "utf8")), + validatedCheckpointId, + dataDir, + ) + const backupDir = snapshotBackupDir(dataDir, validatedCheckpointId) + await assertRealDirectory(backupDir, "checkpoint backup") + return { + checkpointId: validatedCheckpointId, + snapshotDir, + backupDir, + backupSqlPath: snapshotBackupSqlPath(validatedCheckpointId), + manifest, + } +} + export const resolveCheckpoint = async ( dataDir: string, selector: "current" | "previous" | string = "current", @@ -482,22 +567,7 @@ export const resolveCheckpoint = async ( ? state.previous : validateId(selector, "checkpoint") if (!checkpointId) throw new Error("no previous checkpoint is selected") - const snapshotDir = checkpointSnapshotDir(dataDir, checkpointId) - await assertNoSymlink(checkpointSnapshotsRoot(dataDir), snapshotDir) - const manifest = parseCheckpointManifest( - JSON.parse(await readFile(snapshotManifestPath(dataDir, checkpointId), "utf8")), - checkpointId, - ) - const backupDir = snapshotBackupDir(dataDir, checkpointId) - const backupStat = await stat(backupDir).catch(() => null) - if (!backupStat?.isDirectory()) throw new Error(`checkpoint backup is incomplete at ${backupDir}`) - return { - checkpointId, - snapshotDir, - backupDir, - backupSqlPath: snapshotBackupSqlPath(checkpointId), - manifest, - } + return resolveCheckpointById(dataDir, checkpointId) } export const readCheckpointManifest = async ( @@ -586,13 +656,45 @@ const parseOperation = (value: unknown): CheckpointOperation => { throw new Error("unsupported or malformed checkpoint operation") } const phase = requiredString(value, "phase") - if (!["intent", "backup-complete", "manifest-complete", "pointer-complete"].includes(phase)) { + if ( + ![ + "intent", + "backup-complete", + "manifest-complete", + "pointer-complete", + "retention-complete", + ].includes(phase) + ) { throw new Error("invalid checkpoint operation phase") } + const baseRevision = + value.baseRevision === null + ? null + : validateId(requiredString(value, "baseRevision"), "base state revision") + const baseCurrent = + value.baseCurrent === null + ? null + : validateId(requiredString(value, "baseCurrent"), "base current checkpoint") + const basePrevious = + value.basePrevious === null + ? null + : validateId(requiredString(value, "basePrevious"), "base previous checkpoint") + if ((baseRevision === null) !== (baseCurrent === null)) { + throw new Error("checkpoint operation has an inconsistent base state") + } + if (baseCurrent === null && basePrevious !== null) { + throw new Error("checkpoint operation cannot have a previous checkpoint without a current checkpoint") + } + if (baseCurrent !== null && baseCurrent === basePrevious) { + throw new Error("checkpoint operation base current and previous must differ") + } return { formatVersion: 1, operationId: validateId(requiredString(value, "operationId"), "operation"), checkpointId: validateId(requiredString(value, "checkpointId"), "checkpoint"), + baseRevision, + baseCurrent, + basePrevious, phase: phase as CheckpointOperation["phase"], startedAt: requiredIso(value, "startedAt"), } @@ -615,13 +717,16 @@ const processIsAlive = (pid: number): boolean => { const acquireMaintenance = async (dataDir: string, operationId: string): Promise<() => Promise> => { const lockPath = maintenanceLockPath(dataDir) + if (existsSync(lockPath)) await assertRealDirectory(lockPath, "maintenance lock") try { await mkdir(lockPath, { mode: 0o700 }) } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error let owner: MaintenanceOwner try { - const parsed = JSON.parse(readFileSync(join(lockPath, "owner.json"), "utf8")) as unknown + const ownerPath = join(lockPath, "owner.json") + await assertRealFile(ownerPath, "maintenance lock owner") + const parsed = JSON.parse(readFileSync(ownerPath, "utf8")) as unknown if (!isRecord(parsed) || parsed.formatVersion !== 1) throw new Error("malformed owner") owner = { formatVersion: 1, @@ -647,7 +752,9 @@ const acquireMaintenance = async (dataDir: string, operationId: string): Promise } await durableJson(join(lockPath, "owner.json"), owner) return async () => { - const current = JSON.parse(await readFile(join(lockPath, "owner.json"), "utf8")) as unknown + const ownerPath = join(lockPath, "owner.json") + await assertRealFile(ownerPath, "maintenance lock owner") + const current = JSON.parse(await readFile(ownerPath, "utf8")) as unknown if ( !isRecord(current) || requiredString(current, "operationId") !== operationId || @@ -661,46 +768,142 @@ const acquireMaintenance = async (dataDir: string, operationId: string): Promise } export const reconcileCheckpointOperations = async (dataDir: string): Promise => { + await assertCheckpointInfrastructureSafe(dataDir) const state = await readStateFileOptional(dataDir) + const operationsRoot = checkpointOperationsRoot(dataDir) + if (existsSync(operationsRoot)) { + await assertNoSymlink(checkpointRoot(dataDir), operationsRoot) + await assertRealDirectory(operationsRoot, "checkpoint operations") + } let entries try { - entries = await readdir(checkpointOperationsRoot(dataDir), { withFileTypes: true }) + entries = await readdir(operationsRoot, { withFileTypes: true }) } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return throw error } - for (const entry of entries) { - if (!entry.isDirectory() || !entry.name.startsWith("checkpoint-")) { - throw new Error( - `unrecognized checkpoint operation debris: ${join(checkpointOperationsRoot(dataDir), entry.name)}`, - ) + if (entries.length === 0) return + if (entries.length !== 1) { + throw new Error( + `multiple checkpoint operations require operator inspection: ${entries + .map((entry) => join(operationsRoot, entry.name)) + .join(", ")}`, + ) + } + const entry = entries[0]! + if (!entry.isDirectory() || !entry.name.startsWith("checkpoint-")) { + throw new Error(`unrecognized checkpoint operation debris: ${join(operationsRoot, entry.name)}`) + } + const entryOperationId = validateId(entry.name.slice("checkpoint-".length), "operation directory") + const entryDir = join(operationsRoot, entry.name) + const intentPath = join(entryDir, "intent.json") + await assertNoSymlink(operationsRoot, entryDir) + await assertNoSymlink(operationsRoot, intentPath) + await assertRealDirectory(entryDir, "checkpoint operation") + await assertRealFile(intentPath, "checkpoint operation intent") + const operation = parseOperation(JSON.parse(await readFile(intentPath, "utf8"))) + if (operation.operationId !== entryOperationId) { + throw new Error( + `checkpoint operation identity mismatch (directory: ${entryOperationId}; intent: ${operation.operationId})`, + ) + } + + const baseMatches = + operation.baseRevision === null + ? state === null + : state !== null && + state.revision === operation.baseRevision && + state.current === operation.baseCurrent && + state.previous === operation.basePrevious + const expectedMatches = + state !== null && + state.revision === operation.operationId && + state.current === operation.checkpointId && + state.previous === operation.baseCurrent + const snapshot = checkpointSnapshotDir(dataDir, operation.checkpointId) + const manifestPath = snapshotManifestPath(dataDir, operation.checkpointId) + + if (existsSync(manifestPath)) { + const resolved = await resolveCheckpointById(dataDir, operation.checkpointId) + if (resolved.manifest.operationId !== operation.operationId) { + throw new Error("checkpoint manifest operation identity does not match its operation") } - const path = join(checkpointOperationsRoot(dataDir), entry.name, "intent.json") - const operation = parseOperation(JSON.parse(await readFile(path, "utf8"))) - const snapshot = checkpointSnapshotDir(dataDir, operation.checkpointId) - const selected = - state?.current === operation.checkpointId || state?.previous === operation.checkpointId - if (operation.phase === "pointer-complete" && selected) { - await rm(operationDir(dataDir, operation.operationId), { recursive: true }) - await syncDirectory(checkpointOperationsRoot(dataDir)) - continue + if (!baseMatches && !expectedMatches) { + throw new Error("checkpoint operation base no longer matches authoritative state") } - const quarantine = join( - checkpointQuarantineRoot(dataDir), - `operation-${operation.operationId}-${randomUUID()}`, - ) - await ensurePrivateDirectory(checkpointQuarantineRoot(dataDir)) - await ensurePrivateDirectory(quarantine) - if (existsSync(snapshot) && !existsSync(snapshotManifestPath(dataDir, operation.checkpointId))) { - await durableRename(snapshot, join(quarantine, "incomplete-snapshot")) + if (baseMatches) { + if (!["backup-complete", "manifest-complete"].includes(operation.phase)) { + throw new Error(`checkpoint operation phase ${operation.phase} cannot publish its snapshot`) + } + if (operation.baseCurrent) await resolveCheckpointById(dataDir, operation.baseCurrent) + if (operation.basePrevious) await resolveCheckpointById(dataDir, operation.basePrevious) + const manifestComplete: CheckpointOperation = { + ...operation, + phase: "manifest-complete", + } + await writeOperation(dataDir, manifestComplete) + const nextState: CheckpointState = { + formatVersion: 1, + revision: operation.operationId, + current: operation.checkpointId, + previous: operation.baseCurrent, + committedAt: new Date().toISOString(), + } + await durableJson(checkpointStatePath(dataDir), nextState) + } + const publishedState = await readCheckpointState(dataDir) + if ( + publishedState.revision !== operation.operationId || + publishedState.current !== operation.checkpointId || + publishedState.previous !== operation.baseCurrent + ) { + throw new Error("checkpoint operation publication did not produce its exact intended state") + } + let retirement: string | null = null + if (operation.phase === "retention-complete") { + const candidate = join(checkpointRetiringRoot(dataDir), `retirement-${operation.operationId}`) + retirement = existsSync(candidate) ? candidate : null + } else { + await writeOperation(dataDir, { ...operation, phase: "pointer-complete" }) + retirement = await retireCheckpointIfEligible( + dataDir, + operation.basePrevious, + publishedState, + operation.operationId, + ) + await writeOperation(dataDir, { ...operation, phase: "retention-complete" }) } - await durableRename(operationDir(dataDir, operation.operationId), join(quarantine, "operation")) + await removeCompletedRetirement(retirement) + await rm(entryDir, { recursive: true }) + await syncDirectory(operationsRoot) + return + } + + if (!baseMatches || !["intent", "backup-complete"].includes(operation.phase)) { + throw new Error("checkpoint operation is missing its manifest in an unsafe state") + } + const quarantineRoot = checkpointQuarantineRoot(dataDir) + if (existsSync(quarantineRoot)) { + await assertNoSymlink(checkpointRoot(dataDir), quarantineRoot) + await assertRealDirectory(quarantineRoot, "checkpoint quarantine") + } + const quarantine = join(quarantineRoot, `operation-${operation.operationId}-${randomUUID()}`) + await ensurePrivateDirectory(quarantineRoot) + await ensurePrivateDirectory(quarantine) + if (existsSync(snapshot)) { + await assertNoSymlink(checkpointSnapshotsRoot(dataDir), snapshot) + await assertRealDirectory(snapshot, "incomplete checkpoint snapshot") + await durableRename(snapshot, join(quarantine, "incomplete-snapshot")) } + await durableRename(entryDir, join(quarantine, "operation")) } const hasPins = async (dataDir: string, checkpointId: string): Promise => { const path = join(checkpointPinsRoot(dataDir), checkpointId) try { + await assertNoSymlink(checkpointRoot(dataDir), checkpointPinsRoot(dataDir)) + await assertNoSymlink(checkpointPinsRoot(dataDir), path) + await assertRealDirectory(path, "checkpoint pin reservation") return (await readdir(path)).length > 0 } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return false @@ -712,15 +915,118 @@ export const retireCheckpointIfEligible = async ( dataDir: string, checkpointId: string | null, state: CheckpointState, -): Promise => { - if (!checkpointId || state.current === checkpointId || state.previous === checkpointId) return - if (await hasPins(dataDir, checkpointId)) return - await resolveCheckpoint(dataDir, checkpointId, state) - const retirement = join(checkpointRetiringRoot(dataDir), randomUUID()) - await ensurePrivateDirectory(retirement) - await durableRename(checkpointSnapshotDir(dataDir, checkpointId), join(retirement, checkpointId)) + retirementId: string = randomUUID(), + faults: DurabilityFaults = {}, +): Promise => { + if (!checkpointId || state.current === checkpointId || state.previous === checkpointId) return null + const validatedCheckpointId = validateId(checkpointId, "checkpoint") + if (await hasPins(dataDir, validatedCheckpointId)) return null + const validatedRetirementId = validateId(retirementId, "retirement") + const retirementRoot = checkpointRetiringRoot(dataDir) + const retirement = join(retirementRoot, `retirement-${validatedRetirementId}`) + const retirementIntent = join(retirement, "intent.json") + const retirementComplete = join(retirement, "complete.json") + const retiredSnapshot = join(retirement, validatedCheckpointId) + if (existsSync(checkpointRetiringRoot(dataDir))) { + await assertNoSymlink(checkpointRoot(dataDir), retirementRoot) + await assertRealDirectory(retirementRoot, "checkpoint retirement root") + } + if (!existsSync(retirement)) { + await resolveCheckpoint(dataDir, validatedCheckpointId, state) + await ensurePrivateDirectory(retirement) + await durableJson(retirementIntent, { + formatVersion: 1, + retirementId: validatedRetirementId, + checkpointId: validatedCheckpointId, + stateRevision: state.revision, + }) + await faults.afterRetirementIntent?.(retirement) + } else { + await assertNoSymlink(retirementRoot, retirement) + await assertRealDirectory(retirement, "checkpoint retirement") + await assertNoSymlink(retirement, retirementIntent) + await assertRealFile(retirementIntent, "checkpoint retirement intent") + const parsed = JSON.parse(await readFile(retirementIntent, "utf8")) as unknown + if ( + !isRecord(parsed) || + parsed.formatVersion !== 1 || + validateId(requiredString(parsed, "retirementId"), "retirement") !== validatedRetirementId || + validateId(requiredString(parsed, "checkpointId"), "checkpoint") !== validatedCheckpointId || + validateId(requiredString(parsed, "stateRevision"), "state revision") !== state.revision + ) { + throw new Error(`checkpoint retirement identity mismatch: ${retirement}`) + } + } + if (existsSync(retirementComplete)) { + await assertNoSymlink(retirement, retirementComplete) + await assertRealFile(retirementComplete, "checkpoint retirement completion") + const complete = JSON.parse(await readFile(retirementComplete, "utf8")) as unknown + if ( + !isRecord(complete) || + complete.formatVersion !== 1 || + validateId(requiredString(complete, "retirementId"), "retirement") !== validatedRetirementId || + validateId(requiredString(complete, "checkpointId"), "checkpoint") !== validatedCheckpointId || + validateId(requiredString(complete, "stateRevision"), "state revision") !== state.revision + ) { + throw new Error(`checkpoint retirement completion identity mismatch: ${retirement}`) + } + if ( + existsSync(checkpointSnapshotDir(dataDir, validatedCheckpointId)) || + existsSync(retiredSnapshot) + ) { + throw new Error("completed checkpoint retirement still has snapshot data") + } + return retirement + } + const source = checkpointSnapshotDir(dataDir, validatedCheckpointId) + const sourceExists = existsSync(source) + const retiredExists = existsSync(retiredSnapshot) + if (sourceExists && retiredExists) { + throw new Error("checkpoint retirement has both source and retired snapshots") + } + if (sourceExists) { + await assertNoSymlink(checkpointSnapshotsRoot(dataDir), source) + await durableRename(source, retiredSnapshot, faults) + await faults.afterRetirementRename?.(retirement) + } + if (existsSync(retiredSnapshot)) { + await rm(retiredSnapshot, { recursive: true }) + await syncDirectory(retirement) + } + await durableJson(retirementComplete, { + formatVersion: 1, + retirementId: validatedRetirementId, + checkpointId: validatedCheckpointId, + stateRevision: state.revision, + }) + return retirement +} + +const removeCompletedRetirement = async (retirement: string | null): Promise => { + if (!retirement || !existsSync(retirement)) return + const intent = join(retirement, "intent.json") + const complete = join(retirement, "complete.json") + await assertNoSymlink(dirname(retirement), retirement) + await assertRealDirectory(retirement, "checkpoint retirement") + const entries = (await readdir(retirement)).sort() + if (entries.length !== 2 || entries[0] !== "complete.json" || entries[1] !== "intent.json") { + throw new Error(`completed checkpoint retirement contains unexpected state: ${retirement}`) + } + await assertNoSymlink(retirement, intent) + await assertNoSymlink(retirement, complete) + await assertRealFile(intent, "checkpoint retirement intent") + await assertRealFile(complete, "checkpoint retirement completion") + const intentValue = JSON.parse(await readFile(intent, "utf8")) as unknown + const completeValue = JSON.parse(await readFile(complete, "utf8")) as unknown + if ( + !isRecord(intentValue) || + !isRecord(completeValue) || + JSON.stringify(intentValue) !== JSON.stringify(completeValue) + ) { + throw new Error(`checkpoint retirement records do not match: ${retirement}`) + } await rm(retirement, { recursive: true }) - await syncDirectory(checkpointRetiringRoot(dataDir)) + await syncDirectory(dirname(retirement)) } export const createCheckpoint = ( @@ -741,6 +1047,7 @@ export const createCheckpoint = ( const release = await acquireMaintenance(options.dataDir, operationId) try { assertCheckpointRootSafe(options.dataDir) + await assertCheckpointInfrastructureSafe(options.dataDir) assertNoLegacyLayout(options.dataDir) await reconcileCheckpointOperations(options.dataDir) const oldState = await readStateFileOptional(options.dataDir) @@ -770,6 +1077,9 @@ export const createCheckpoint = ( formatVersion: 1, operationId, checkpointId, + baseRevision: oldState?.revision ?? null, + baseCurrent: oldState?.current ?? null, + basePrevious: oldState?.previous ?? null, phase: "intent", startedAt, } @@ -848,7 +1158,16 @@ export const createCheckpoint = ( await durableJson(checkpointStatePath(options.dataDir), state, options.faults) operation = { ...operation, phase: "pointer-complete" } await writeOperation(options.dataDir, operation, options.faults) - await retireCheckpointIfEligible(options.dataDir, oldState?.previous ?? null, state) + const retirement = await retireCheckpointIfEligible( + options.dataDir, + oldState?.previous ?? null, + state, + operationId, + options.faults, + ) + operation = { ...operation, phase: "retention-complete" } + await writeOperation(options.dataDir, operation, options.faults) + await removeCompletedRetirement(retirement) await rm(operationDir(options.dataDir, operationId), { recursive: true }) await syncDirectory(checkpointOperationsRoot(options.dataDir)) return { checkpointId, path: snapshot, state, manifest } @@ -881,6 +1200,7 @@ const parseRestoreTransaction = (value: unknown): RestoreTransaction => { const readRestoreTransaction = async (dataDir: string): Promise => { try { + await assertRealFile(restoreTransactionPath(dataDir), "restore transaction") return parseRestoreTransaction(JSON.parse(await readFile(restoreTransactionPath(dataDir), "utf8"))) } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null @@ -902,6 +1222,8 @@ const readyIdentityMatches = (dataDir: string, transaction: RestoreTransaction): for (const path of candidates) { if (!existsSync(path)) continue try { + const info = lstatSync(path) + if (info.isSymbolicLink() || !info.isFile()) return false const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown if ( isRecord(parsed) && @@ -921,32 +1243,45 @@ const readyIdentityMatches = (dataDir: string, transaction: RestoreTransaction): const finalizeRestoreMarkers = async ( dataDir: string, transaction: RestoreTransaction, + faults: RestoreRecoveryFaults = {}, ): Promise => { if (!readyIdentityMatches(dataDir, transaction)) { throw new Error("restored live store identity is missing or does not match the transaction") } await writeStoreMarkerDurable(dataDir, MAPLE_VERSION, new Date().toISOString(), SCHEMA_FINGERPRINT) + await faults.afterStoreMarkerWrite?.() await markStoreClosedDurable(dataDir) + await faults.afterOpenMarkerRemoval?.() const committed: RestoreTransaction = { ...transaction, phase: "markers-committed" } await writeRestoreTransaction(dataDir, committed) + await faults.afterMarkersCommittedRecord?.() return committed } const completeRestoreTransaction = async ( dataDir: string, transaction: RestoreTransaction, + faults: RestoreRecoveryFaults = {}, ): Promise => { const readyPath = join(resolve(dataDir), ".maple-restore-ready.json") if (existsSync(readyPath)) await durableRemove(readyPath) + await faults.afterReadyMarkerRemoval?.() const root = restoreRootPath(dataDir, transaction.operationId) if (existsSync(root)) { await rm(root, { recursive: true }) await syncDirectory(dirname(root)) } + await faults.afterRestoreRootRemoval?.() await durableRemove(restoreTransactionPath(dataDir)) } -const reconcileRestoreTransactionUnlocked = async (dataDir: string): Promise => { +const reconcileRestoreTransactionUnlocked = async ( + dataDir: string, + faults: RestoreRecoveryFaults = {}, +): Promise => { + if (existsSync(resolve(dataDir))) { + await assertRealDirectory(resolve(dataDir), "live data directory") + } let transaction = await readRestoreTransaction(dataDir) if (!transaction) { const parent = dirname(resolve(dataDir)) @@ -979,6 +1314,10 @@ const reconcileRestoreTransactionUnlocked = async (dataDir: string): Promise => +export const reconcileCheckpointRecovery = ( + dataDir: string, + faults: RestoreRecoveryFaults = {}, +): Effect.Effect => + Effect.tryPromise({ + try: async () => { + const operationId = randomUUID() + const release = await acquireMaintenance(dataDir, operationId) + try { + await reconcileRestoreTransactionUnlocked(dataDir, faults) + } finally { + await release() + } + }, + catch: (error) => + new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), + }) + +/** + * Explicitly remove the live chDB store while preserving the checkpoint + * registry below `/backups`. The maintenance lock serializes this + * destructive operation with checkpoint, restore, and archive work. + */ +export const resetLiveStorePreservingCheckpoints = (dataDir: string): Effect.Effect => Effect.tryPromise({ try: async () => { const operationId = randomUUID() const release = await acquireMaintenance(dataDir, operationId) try { await reconcileRestoreTransactionUnlocked(dataDir) + await assertCheckpointInfrastructureSafe(dataDir) + const live = resolve(dataDir) + if (existsSync(live)) { + await assertRealDirectory(live, "live data directory") + const entries = await readdir(live, { withFileTypes: true }) + const removable = entries + .filter((entry) => entry.name !== "backups") + .sort((left, right) => { + const core = (name: string) => (name === "store" || name === "metadata" ? 1 : 0) + return core(left.name) - core(right.name) + }) + for (const entry of removable) { + await rm(join(live, entry.name), { recursive: true, force: true }) + } + await syncDirectory(live) + } + for (const marker of [storeMarkerPath(dataDir), storeOpenMarkerPath(dataDir)]) { + if (existsSync(marker)) await durableRemove(marker) + } } finally { await release() } diff --git a/apps/cli/src/server/durable-files.ts b/apps/cli/src/server/durable-files.ts index e83133651..8a22a189a 100644 --- a/apps/cli/src/server/durable-files.ts +++ b/apps/cli/src/server/durable-files.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto" import { constants } from "node:fs" -import { chmod, mkdir, open, readdir, rename, rm } from "node:fs/promises" +import { chmod, lstat, mkdir, open, readdir, rename, rm } from "node:fs/promises" import { dirname, join } from "node:path" export interface DurabilityFaults { @@ -8,6 +8,8 @@ export interface DurabilityFaults { readonly beforeRename?: (from: string, to: string) => void | Promise readonly beforeDirectorySync?: (path: string) => void | Promise readonly beforeRemove?: (path: string) => void | Promise + readonly afterRetirementIntent?: (path: string) => void | Promise + readonly afterRetirementRename?: (path: string) => void | Promise } // APFS and the target Linux filesystems support directory fsync. Keep the @@ -22,7 +24,17 @@ export const isUnsupportedDirectorySyncError = (error: unknown): boolean => unsupportedDirectorySyncCodes.has(String((error as NodeJS.ErrnoException).code)) export const ensurePrivateDirectory = async (path: string): Promise => { + const before = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null + throw error + }) + if (before?.isSymbolicLink()) throw new Error(`refusing symlink directory: ${path}`) + if (before && !before.isDirectory()) throw new Error(`refusing non-directory path: ${path}`) await mkdir(path, { recursive: true, mode: 0o700 }) + const after = await lstat(path) + if (after.isSymbolicLink() || !after.isDirectory()) { + throw new Error(`private directory is not a real directory: ${path}`) + } await chmod(path, 0o700) } diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index 5f5764d1c..57860eb6f 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -28,6 +28,8 @@ import { readCheckpointState, reconcileCheckpointRecovery, reconcileCheckpointOperations, + resetLiveStorePreservingCheckpoints, + type RestoreRecoveryFaults, resolveCheckpoint, restoreDataPath, restoreQuarantinePath, @@ -57,7 +59,11 @@ const withDataDir = async (run: (dataDir: string) => Promise | void): Prom } } -const manifest = (checkpointId: string, operationId = newCheckpointId()): Record => ({ +const manifest = ( + checkpointId: string, + operationId = newCheckpointId(), + sourceDataDir = "/tmp/maple-data", +): Record => ({ formatVersion: 1, checkpointId, operationId, @@ -65,7 +71,7 @@ const manifest = (checkpointId: string, operationId = newCheckpointId()): Record chdbVersion: CHDB_VERSION, schemaFingerprint: SCHEMA_FINGERPRINT, createdAt: "2026-01-01T00:00:00.000Z", - sourceDataDir: "/tmp/maple-data", + sourceDataDir, backupRelativePath: `snapshots/${checkpointId}/backup`, backupBytes: 123, validation: { @@ -80,11 +86,14 @@ const manifest = (checkpointId: string, operationId = newCheckpointId()): Record }, }) -const writeSnapshot = (dataDir: string, checkpointId: string): void => { +const writeSnapshot = (dataDir: string, checkpointId: string, operationId = newCheckpointId()): void => { const snapshot = checkpointSnapshotDir(dataDir, checkpointId) mkdirSync(join(snapshot, "backup"), { recursive: true }) writeFileSync(join(snapshot, "backup", "data.bin"), "backup") - writeFileSync(join(snapshot, "manifest.json"), `${JSON.stringify(manifest(checkpointId))}\n`) + writeFileSync( + join(snapshot, "manifest.json"), + `${JSON.stringify(manifest(checkpointId, operationId, dataDir))}\n`, + ) } const writeState = ( @@ -106,6 +115,35 @@ const writeState = ( ) } +const writeCheckpointOperation = ( + dataDir: string, + operationId: string, + checkpointId: string, + phase: "intent" | "backup-complete" | "manifest-complete" | "pointer-complete" | "retention-complete", + base: { + readonly revision: string | null + readonly current: string | null + readonly previous: string | null + } = { revision: null, current: null, previous: null }, +): string => { + const dir = join(checkpointRoot(dataDir), "operations", `checkpoint-${operationId}`) + mkdirSync(dir, { recursive: true }) + writeFileSync( + join(dir, "intent.json"), + `${JSON.stringify({ + formatVersion: 1, + operationId, + checkpointId, + baseRevision: base.revision, + baseCurrent: base.current, + basePrevious: base.previous, + phase, + startedAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ) + return dir +} + const restoreValidation = { validatedAt: "2026-01-01T00:00:01.000Z", traces: 1, @@ -176,6 +214,11 @@ describe("checkpoint IDs and strict parsers", () => { it("accepts a complete manifest and rejects ID, path, compatibility, and count corruption", () => { const id = newCheckpointId() strictEqual(parseCheckpointManifest(manifest(id), id).checkpointId, id) + strictEqual(parseCheckpointManifest(manifest(id), id, "/tmp/maple-data").checkpointId, id) + throwsMessage( + () => parseCheckpointManifest(manifest(id), id, "/tmp/different-owner"), + /configured owner/, + ) const wrong = newCheckpointId() ok(wrong !== id) throwsMessage(() => parseCheckpointManifest(manifest(id), wrong), /does not match/) @@ -285,6 +328,28 @@ describe("checkpoint state resolution", () => { throwsMessage(() => assertCheckpointRootSafe(dataDir), /symlink/) }) }) + + it("rejects symlinked manifest and backup leaves outside the registry", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + const outside = join(dirname(dataDir), "outside-checkpoint") + mkdirSync(outside) + writeFileSync( + join(outside, "manifest.json"), + `${JSON.stringify(manifest(checkpointId, newCheckpointId(), dataDir))}\n`, + ) + mkdirSync(join(outside, "backup")) + mkdirSync(snapshot, { recursive: true }) + symlinkSync(join(outside, "manifest.json"), join(snapshot, "manifest.json")) + symlinkSync(join(outside, "backup"), join(snapshot, "backup")) + writeState(dataDir, checkpointId) + + await rejects(readCheckpointState(dataDir), /symlink/) + ok(existsSync(join(outside, "manifest.json"))) + ok(existsSync(join(outside, "backup"))) + }) + }) }) describe("checkpoint reconciliation and retention", () => { @@ -292,21 +357,10 @@ describe("checkpoint reconciliation and retention", () => { await withDataDir(async (dataDir) => { const operationId = newCheckpointId() const checkpointId = newCheckpointId() - const operationDir = join(checkpointRoot(dataDir), "operations", `checkpoint-${operationId}`) const snapshot = checkpointSnapshotDir(dataDir, checkpointId) mkdirSync(join(snapshot, "backup"), { recursive: true }) writeFileSync(join(snapshot, "backup", "partial.bin"), "partial") - mkdirSync(operationDir, { recursive: true }) - writeFileSync( - join(operationDir, "intent.json"), - `${JSON.stringify({ - formatVersion: 1, - operationId, - checkpointId, - phase: "backup-complete", - startedAt: "2026-01-01T00:00:00.000Z", - })}\n`, - ) + writeCheckpointOperation(dataDir, operationId, checkpointId, "backup-complete") await reconcileCheckpointOperations(dataDir) @@ -333,6 +387,149 @@ describe("checkpoint reconciliation and retention", () => { }) }) + it("rejects a symlinked operations root without touching its target", async () => { + await withDataDir(async (dataDir) => { + const outside = join(dirname(dataDir), "outside-operations") + const sentinel = join(outside, "sentinel") + mkdirSync(outside) + writeFileSync(sentinel, "preserve") + mkdirSync(checkpointRoot(dataDir), { recursive: true }) + symlinkSync(outside, join(checkpointRoot(dataDir), "operations")) + + await rejects(reconcileCheckpointOperations(dataDir), /real directory|symlink/) + strictEqual(readFileSync(sentinel, "utf8"), "preserve") + }) + }) + + it("rejects mismatched operation directory and intent identities without mutation", async () => { + await withDataDir(async (dataDir) => { + const directoryId = newCheckpointId() + const intentId = newCheckpointId() + const checkpointId = newCheckpointId() + const directory = writeCheckpointOperation(dataDir, directoryId, checkpointId, "backup-complete") + const intent = JSON.parse(readFileSync(join(directory, "intent.json"), "utf8")) + intent.operationId = intentId + writeFileSync(join(directory, "intent.json"), `${JSON.stringify(intent)}\n`) + + await rejects(reconcileCheckpointOperations(dataDir), /identity mismatch/) + ok(existsSync(join(directory, "intent.json"))) + }) + }) + + it("publishes a completed first checkpoint after an interrupted pointer update", async () => { + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId, operationId) + writeCheckpointOperation(dataDir, operationId, checkpointId, "manifest-complete") + + await reconcileCheckpointOperations(dataDir) + + const state = await readCheckpointState(dataDir) + strictEqual(state.revision, operationId) + strictEqual(state.current, checkpointId) + strictEqual(state.previous, null) + ok(!existsSync(join(checkpointRoot(dataDir), "operations", `checkpoint-${operationId}`))) + }) + }) + + it("resumes a based promotion and retires only the superseded previous checkpoint", async () => { + await withDataDir(async (dataDir) => { + const oldCurrent = newCheckpointId() + const oldPrevious = newCheckpointId() + const baseRevision = newCheckpointId() + writeSnapshot(dataDir, oldCurrent) + writeSnapshot(dataDir, oldPrevious) + writeState(dataDir, oldCurrent, oldPrevious, baseRevision) + + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId, operationId) + writeCheckpointOperation(dataDir, operationId, checkpointId, "manifest-complete", { + revision: baseRevision, + current: oldCurrent, + previous: oldPrevious, + }) + + await reconcileCheckpointOperations(dataDir) + + const state = await readCheckpointState(dataDir) + strictEqual(state.current, checkpointId) + strictEqual(state.previous, oldCurrent) + ok(existsSync(checkpointSnapshotDir(dataDir, oldCurrent))) + ok(!existsSync(checkpointSnapshotDir(dataDir, oldPrevious))) + }) + }) + + it("converges after interruption at retirement intent and snapshot-rename boundaries", async () => { + for (const boundary of ["afterRetirementIntent", "afterRetirementRename"] as const) { + await withDataDir(async (dataDir) => { + const current = newCheckpointId() + const previous = newCheckpointId() + const old = newCheckpointId() + const retirementId = newCheckpointId() + writeSnapshot(dataDir, current) + writeSnapshot(dataDir, previous) + writeSnapshot(dataDir, old) + writeState(dataDir, current, previous) + const state = await readCheckpointState(dataDir) + + await rejects( + retireCheckpointIfEligible(dataDir, old, state, retirementId, { + [boundary]: () => { + throw new Error(`injected ${boundary}`) + }, + }), + /injected/, + ) + const retirement = await retireCheckpointIfEligible(dataDir, old, state, retirementId) + + ok(!existsSync(checkpointSnapshotDir(dataDir, old)), boundary) + ok(retirement !== null && existsSync(join(retirement, "complete.json")), boundary) + ok(existsSync(checkpointSnapshotDir(dataDir, current)), boundary) + ok(existsSync(checkpointSnapshotDir(dataDir, previous)), boundary) + }) + } + }) + + it("cleans an exactly completed retirement after the operation completion record", async () => { + for (const keepRetirementRecord of [true, false]) { + await withDataDir(async (dataDir) => { + const oldCurrent = newCheckpointId() + const oldPrevious = newCheckpointId() + const baseRevision = newCheckpointId() + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, oldCurrent) + writeSnapshot(dataDir, checkpointId, operationId) + writeState(dataDir, checkpointId, oldCurrent, operationId) + writeCheckpointOperation(dataDir, operationId, checkpointId, "retention-complete", { + revision: baseRevision, + current: oldCurrent, + previous: oldPrevious, + }) + const retirement = join(checkpointRoot(dataDir), "retiring", `retirement-${operationId}`) + if (keepRetirementRecord) { + mkdirSync(retirement, { recursive: true }) + const record = { + formatVersion: 1, + retirementId: operationId, + checkpointId: oldPrevious, + stateRevision: operationId, + } + writeFileSync(join(retirement, "intent.json"), `${JSON.stringify(record)}\n`) + writeFileSync(join(retirement, "complete.json"), `${JSON.stringify(record)}\n`) + } + + await reconcileCheckpointOperations(dataDir) + + ok(!existsSync(retirement)) + ok(!existsSync(join(checkpointRoot(dataDir), "operations", `checkpoint-${operationId}`))) + strictEqual((await readCheckpointState(dataDir)).current, checkpointId) + }) + } + }) + it("retains current, previous, pinned, and malformed candidates; retires only proven safe", async () => { await withDataDir(async (dataDir) => { const current = newCheckpointId() @@ -368,6 +565,47 @@ describe("checkpoint reconciliation and retention", () => { }) }) +describe("live-store reset safety", () => { + it("removes live chDB data and sibling markers while preserving the checkpoint registry", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId) + writeState(dataDir, checkpointId) + mkdirSync(join(dataDir, "store"), { recursive: true }) + mkdirSync(join(dataDir, "metadata"), { recursive: true }) + writeFileSync(join(dataDir, "store", "part.bin"), "live") + writeFileSync(join(dataDir, "metadata", "table.sql"), "live") + writeFileSync(join(dataDir, "status"), "live") + writeFileSync(storeMarkerPath(dataDir), "{}") + writeFileSync(storeOpenMarkerPath(dataDir), "999\n") + + await Effect.runPromise(resetLiveStorePreservingCheckpoints(dataDir)) + + strictEqual((await readCheckpointState(dataDir)).current, checkpointId) + ok(existsSync(checkpointSnapshotDir(dataDir, checkpointId))) + ok(!existsSync(join(dataDir, "store"))) + ok(!existsSync(join(dataDir, "metadata"))) + ok(!existsSync(join(dataDir, "status"))) + ok(!existsSync(storeMarkerPath(dataDir))) + ok(!existsSync(storeOpenMarkerPath(dataDir))) + }) + }) + + it("fails closed before deleting live data when checkpoint infrastructure is unsafe", async () => { + await withDataDir(async (dataDir) => { + const outside = join(dirname(dataDir), "outside-pins") + mkdirSync(outside) + mkdirSync(checkpointRoot(dataDir), { recursive: true }) + symlinkSync(outside, join(checkpointRoot(dataDir), "pins")) + mkdirSync(join(dataDir, "store"), { recursive: true }) + writeFileSync(join(dataDir, "store", "preserve.bin"), "live") + + await rejects(Effect.runPromise(resetLiveStorePreservingCheckpoints(dataDir)), /real directory/) + strictEqual(readFileSync(join(dataDir, "store", "preserve.bin"), "utf8"), "live") + }) + }) +}) + describe("live restore transaction reconciliation", () => { it("is a no-op when no transaction or restore debris exists", async () => { await withDataDir(async (dataDir) => { @@ -450,6 +688,48 @@ describe("live restore transaction reconciliation", () => { }) }) + it("converges after interruption at every live-swap, marker, and cleanup boundary", async () => { + const boundaries: ReadonlyArray = [ + "afterLiveQuarantineRename", + "afterOldQuarantinedRecord", + "afterRestoredLiveRename", + "afterNewLiveRecord", + "afterStoreMarkerWrite", + "afterOpenMarkerRemoval", + "afterMarkersCommittedRecord", + "afterReadyMarkerRemoval", + "afterRestoreRootRemoval", + ] + for (const boundary of boundaries) { + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + const quarantineId = newCheckpointId() + const quarantine = restoreQuarantinePath(dataDir, operationId, quarantineId) + writeFileSync(join(dataDir, "old-live"), "old") + writeFileSync(storeOpenMarkerPath(dataDir), "999\n") + writeRestoreReady(dataDir, operationId, checkpointId) + writeFileSync(join(restoreDataPath(dataDir, operationId), "new-live"), "new") + writeRestoreTransaction(dataDir, operationId, checkpointId, quarantineId, "restore-ready") + const faults = { + [boundary]: () => { + throw new Error(`injected ${boundary}`) + }, + } as RestoreRecoveryFaults + + await rejects(Effect.runPromise(reconcileCheckpointRecovery(dataDir, faults)), /injected/) + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + + ok(existsSync(join(dataDir, "new-live")), boundary) + ok(existsSync(join(quarantine, "old-live")), boundary) + ok(existsSync(storeMarkerPath(dataDir)), boundary) + ok(!existsSync(storeOpenMarkerPath(dataDir)), boundary) + ok(!existsSync(restoreTransactionPath(dataDir)), boundary) + ok(!existsSync(restoreRootPath(dataDir, operationId)), boundary) + }) + } + }) + it("fails closed on malformed or unrecorded restore state without deleting it", async () => { await withDataDir(async (dataDir) => { writeFileSync(restoreTransactionPath(dataDir), "{bad json") @@ -466,6 +746,29 @@ describe("live restore transaction reconciliation", () => { ok(existsSync(debris)) }) }) + + it("rejects symlinked transaction and restore topology without touching targets", async () => { + await withDataDir(async (dataDir) => { + const outside = join(dirname(dataDir), "outside-transaction") + writeFileSync(outside, "{}") + symlinkSync(outside, restoreTransactionPath(dataDir)) + await rejects(Effect.runPromise(reconcileCheckpointRecovery(dataDir)), /real file/) + strictEqual(readFileSync(outside, "utf8"), "{}") + }) + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + const quarantineId = newCheckpointId() + const outside = join(dirname(dataDir), "outside-restore") + mkdirSync(outside) + writeFileSync(join(outside, "sentinel"), "preserve") + writeRestoreTransaction(dataDir, operationId, checkpointId, quarantineId, "intent") + symlinkSync(outside, restoreRootPath(dataDir, operationId)) + + await rejects(Effect.runPromise(reconcileCheckpointRecovery(dataDir)), /real directory/) + strictEqual(readFileSync(join(outside, "sentinel"), "utf8"), "preserve") + }) + }) }) describe("backup configuration classification", () => { diff --git a/apps/cli/test/native-checkpoint-smoke.sh b/apps/cli/test/native-checkpoint-smoke.sh index 7360ba0c2..7af07d237 100755 --- a/apps/cli/test/native-checkpoint-smoke.sh +++ b/apps/cli/test/native-checkpoint-smoke.sh @@ -151,6 +151,18 @@ assert_counts 2 # Foreground dirty-store recovery. kill_server +set +e +"$MAPLE" start \ + --port "$PORT" \ + --data-dir "$DATA" \ + --chdb-config-file "$CONFIG" \ + --offline >"$ROOT/default-dirty.out" 2>&1 +default_dirty_status=$? +set -e +[[ "$default_dirty_status" -ne 0 ]] || fail "default dirty-store policy unexpectedly started" +grep -q 'was not cleanly closed' "$ROOT/default-dirty.out" || + fail "default dirty-store failure was not actionable: $(cat "$ROOT/default-dirty.out")" +[[ -f "$DATA/backups/state.json" ]] || fail "default dirty-store failure removed checkpoints" start_server restore-checkpoint assert_counts 2 @@ -181,4 +193,17 @@ start_server fail assert_counts 3 stop_server +"$MAPLE" reset --data-dir "$DATA" --yes >/dev/null +jq -e --arg c "$C3" --arg p "$C2" \ + '.current == $c and .previous == $p' "$DATA/backups/state.json" >/dev/null +[[ -d "$DATA/backups/snapshots/$C2" ]] || fail "reset removed previous checkpoint C2" +[[ -d "$DATA/backups/snapshots/$C3" ]] || fail "reset removed current checkpoint C3" +start_server fail +assert_counts 0 +stop_server +"$MAPLE" restore --data-dir "$DATA" --checkpoint-id "$C3" --yes >/dev/null +start_server fail +assert_counts 3 +stop_server + echo "PASS native checkpoint smoke: C1=$C1 C2=$C2 C3=$C3" diff --git a/apps/landing/src/content/docs/local-mode/cli-reference.md b/apps/landing/src/content/docs/local-mode/cli-reference.md index ebdbc62e5..6eb7c10a0 100644 --- a/apps/landing/src/content/docs/local-mode/cli-reference.md +++ b/apps/landing/src/content/docs/local-mode/cli-reference.md @@ -47,8 +47,8 @@ Start the local ingest + query server (embedded ClickHouse via chDB). | `--chdb-config-file ` | | Optional ClickHouse config file passed to embedded chDB | | `--offline` | `false` | Serve the UI bundled in this binary (from `127.0.0.1`) instead of `local.maple.dev` | | `--background`, `-d` | `false` | Run detached (logs to `~/.maple/maple.log`); stop with `maple stop` | -| `--reset` | `false` | Wipe the existing store before starting — use after an incompatible upgrade | -| `--on-dirty-store ` | `wipe` | Recovery policy when the store was not cleanly closed | +| `--reset` | `false` | Wipe live chDB data while preserving checkpoints | +| `--on-dirty-store ` | `fail` | Recovery policy when the store was not cleanly closed | ```bash maple start # foreground, UI from local.maple.dev @@ -62,6 +62,14 @@ decision, or data-directory creation, startup reconciles a recorded checkpoint restore transaction. Ambiguous or malformed restore state fails closed and prints the preserved paths. +The default dirty-store policy is `fail`, so an unclean shutdown never silently +deletes telemetry. Choose `restore-checkpoint` to recover the selected +checkpoint, or explicitly choose `wipe` to discard only live chDB data. +Checkpoint snapshots, pins, operation evidence, and quarantine state under +`/backups` are preserved by both `--reset` and explicit wipe. +Schema-incompatible stores also fail closed until an operator explicitly +resets live data. + ### `maple stop` Stop a running `maple start` server (reads the PID file beside the data dir). @@ -72,11 +80,13 @@ Stop a running `maple start` server (reads the PID file beside the data dir). ### `maple reset` -Delete the local chDB store so the next `maple start` bootstraps fresh. Refuses to run while a server still owns the store. +Delete live chDB data so the next `maple start` bootstraps fresh. The checkpoint +registry under `/backups` is preserved. Refuses to run while a server +still owns the store. | Flag | Default | Description | | ------------------- | --------------- | ---------------------------- | -| `--data-dir ` | `~/.maple/data` | Store to delete | +| `--data-dir ` | `~/.maple/data` | Store whose live data clears | | `--yes`, `-y` | `false` | Skip the confirmation prompt | ### `maple checkpoint` @@ -411,7 +421,7 @@ The on-disk config at `~/.maple/config.json` stores `apiUrl`, `token`, `orgId`, **`maple is already running (PID …)`.** A server already owns this data dir. Stop it with `maple stop`, or start a second instance on another port and data dir: `maple start --port 4400 --data-dir ~/.maple/data-2`. -**Incompatible store after an upgrade.** If a new binary refuses to open an older store (`the local store … is incompatible`), wipe it with `maple reset`, or start fresh in one step with `maple start --reset`. +**Incompatible store after an upgrade.** If a new binary refuses to open an older store (`the local store … is incompatible`), explicitly clear live data with `maple reset --yes`, or start fresh in one step with `maple start --reset`. Both preserve the checkpoint registry; incompatible checkpoints remain preserved and fail closed until deliberately handled. **Browser asks to "access devices on your local network" (or CORS errors).** The default dashboard at `local.maple.dev` is a public origin reaching your loopback server, which trips Chrome's Private Network Access gate. Run `maple start --offline` to serve the dashboard same-origin from `127.0.0.1` — no prompt, no internet needed. From ecfae1444a1253f9823263c9b0c62ea4f292d2b5 Mon Sep 17 00:00:00 2001 From: remote service administrator Date: Sat, 27 Jun 2026 22:12:06 -0400 Subject: [PATCH 08/12] fix(cli): make retirement cleanup restart-safe --- apps/cli/src/server/checkpoints.ts | 6 ++++-- apps/cli/test/checkpoints.test.ts | 10 ++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index b6f9bd46c..56633e8d3 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -1025,8 +1025,10 @@ const removeCompletedRetirement = async (retirement: string | null): Promise { previous: oldPrevious, }) const retirement = join(checkpointRoot(dataDir), "retiring", `retirement-${operationId}`) + const interruptedCleanup = `${retirement}.cleanup-${newCheckpointId()}` if (keepRetirementRecord) { mkdirSync(retirement, { recursive: true }) const record = { @@ -519,11 +520,20 @@ describe("checkpoint reconciliation and retention", () => { } writeFileSync(join(retirement, "intent.json"), `${JSON.stringify(record)}\n`) writeFileSync(join(retirement, "complete.json"), `${JSON.stringify(record)}\n`) + } else { + mkdirSync(interruptedCleanup, { recursive: true }) + writeFileSync(join(interruptedCleanup, "preserve"), "completed cleanup debris") } await reconcileCheckpointOperations(dataDir) ok(!existsSync(retirement)) + if (!keepRetirementRecord) { + strictEqual( + readFileSync(join(interruptedCleanup, "preserve"), "utf8"), + "completed cleanup debris", + ) + } ok(!existsSync(join(checkpointRoot(dataDir), "operations", `checkpoint-${operationId}`))) strictEqual((await readCheckpointState(dataDir)).current, checkpointId) }) From 3278b595aea5583b1fcba946b55c96a65a1fa0bd Mon Sep 17 00:00:00 2001 From: remote service administrator Date: Sun, 28 Jun 2026 08:09:32 -0400 Subject: [PATCH 09/12] fix(cli): journal destructive store resets --- apps/cli/src/server/checkpoints.ts | 251 +++++++++++++++--- apps/cli/src/server/durable-files.ts | 5 + apps/cli/test/checkpoints.test.ts | 169 +++++++++++- .../content/docs/local-mode/cli-reference.md | 13 +- 4 files changed, 396 insertions(+), 42 deletions(-) diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index 56633e8d3..7eec57622 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -28,7 +28,9 @@ const STATE_FORMAT_VERSION = 1 const MANIFEST_FORMAT_VERSION = 1 const OPERATION_FORMAT_VERSION = 1 const RESTORE_TRANSACTION_FORMAT_VERSION = 1 +const RESET_TRANSACTION_FORMAT_VERSION = 1 const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i +const RESETTABLE_CHDB_ENTRIES = new Set(["data", "metadata", "store", "tmp"]) export class CheckpointError extends Schema.TaggedErrorClass()( "@maple/cli/CheckpointError", @@ -115,6 +117,15 @@ interface RestoreTransaction { readonly validation: CheckpointValidation | null } +interface ResetTransaction { + readonly formatVersion: 1 + readonly operationId: string + readonly dataDir: string + readonly targets: ReadonlyArray + readonly phase: "intent" | "live-cleared" | "markers-cleared" + readonly createdAt: string +} + export interface RestoreRecoveryFaults { readonly afterLiveQuarantineRename?: () => void | Promise readonly afterOldQuarantinedRecord?: () => void | Promise @@ -125,6 +136,13 @@ export interface RestoreRecoveryFaults { readonly afterMarkersCommittedRecord?: () => void | Promise readonly afterReadyMarkerRemoval?: () => void | Promise readonly afterRestoreRootRemoval?: () => void | Promise + readonly afterResetIntent?: () => void | Promise + readonly afterResetEntryRemoval?: (entry: string) => void | Promise + readonly afterResetLiveClearedRecord?: () => void | Promise + readonly afterResetStoreMarkerRemoval?: () => void | Promise + readonly afterResetOpenMarkerRemoval?: () => void | Promise + readonly afterResetMarkersClearedRecord?: () => void | Promise + readonly afterResetTransactionRemoval?: () => void | Promise } export const checkpointRoot = (dataDir: string): string => join(resolve(dataDir), "backups") @@ -142,6 +160,7 @@ export const checkpointSnapshotDir = (dataDir: string, checkpointId: string): st const maintenanceLockPath = (dataDir: string): string => `${resolve(dataDir)}.maple-maintenance-lock` export const restoreTransactionPath = (dataDir: string): string => `${resolve(dataDir)}.restore-transaction.json` +export const resetTransactionPath = (dataDir: string): string => `${resolve(dataDir)}.reset-transaction.json` export const restoreRootPath = (dataDir: string, operationId: string): string => `${resolve(dataDir)}.restore-${validateId(operationId, "restore operation")}` export const restoreDataPath = (dataDir: string, operationId: string): string => @@ -545,6 +564,12 @@ const resolveCheckpointById = async (dataDir: string, checkpointId: string): Pro ) const backupDir = snapshotBackupDir(dataDir, validatedCheckpointId) await assertRealDirectory(backupDir, "checkpoint backup") + const actualBackupBytes = await dirSize(backupDir) + if (actualBackupBytes !== manifest.backupBytes) { + throw new Error( + `checkpoint backup size mismatch (manifest: ${manifest.backupBytes}; actual: ${actualBackupBytes})`, + ) + } return { checkpointId: validatedCheckpointId, snapshotDir, @@ -706,6 +731,26 @@ const writeOperation = async ( faults: DurabilityFaults = {}, ): Promise => durableJson(operationPath(dataDir, operation.operationId), operation, faults) +const preserveCompletedOperation = async ( + dataDir: string, + operationDirPath: string, + operationId: string, + faults: DurabilityFaults = {}, +): Promise => { + const quarantineRoot = checkpointQuarantineRoot(dataDir) + if (existsSync(quarantineRoot)) { + await assertNoSymlink(checkpointRoot(dataDir), quarantineRoot) + await assertRealDirectory(quarantineRoot, "checkpoint quarantine") + } + await ensurePrivateDirectory(quarantineRoot) + const preserved = join( + quarantineRoot, + `completed-operation-${validateId(operationId, "operation")}-${randomUUID()}`, + ) + await durableRename(operationDirPath, preserved, faults) + await faults.afterCompletedOperationPreserved?.(preserved) +} + const processIsAlive = (pid: number): boolean => { try { process.kill(pid, 0) @@ -767,7 +812,10 @@ const acquireMaintenance = async (dataDir: string, operationId: string): Promise } } -export const reconcileCheckpointOperations = async (dataDir: string): Promise => { +export const reconcileCheckpointOperations = async ( + dataDir: string, + faults: DurabilityFaults = {}, +): Promise => { await assertCheckpointInfrastructureSafe(dataDir) const state = await readStateFileOptional(dataDir) const operationsRoot = checkpointOperationsRoot(dataDir) @@ -873,9 +921,8 @@ export const reconcileCheckpointOperations = async (dataDir: string): Promise => { +const removeCompletedRetirement = async ( + retirement: string | null, + faults: DurabilityFaults = {}, +): Promise => { if (!retirement || !existsSync(retirement)) return const intent = join(retirement, "intent.json") const complete = join(retirement, "complete.json") @@ -1026,9 +1078,11 @@ const removeCompletedRetirement = async (retirement: string | null): Promise 0) { throw new Error( @@ -1169,9 +1223,13 @@ export const createCheckpoint = ( ) operation = { ...operation, phase: "retention-complete" } await writeOperation(options.dataDir, operation, options.faults) - await removeCompletedRetirement(retirement) - await rm(operationDir(options.dataDir, operationId), { recursive: true }) - await syncDirectory(checkpointOperationsRoot(options.dataDir)) + await removeCompletedRetirement(retirement, options.faults) + await preserveCompletedOperation( + options.dataDir, + operationDir(options.dataDir, operationId), + operationId, + options.faults, + ) return { checkpointId, path: snapshot, state, manifest } } finally { await release() @@ -1181,6 +1239,147 @@ export const createCheckpoint = ( new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), }) +const parseResetTransaction = (value: unknown, expectedDataDir: string): ResetTransaction => { + if (!isRecord(value) || value.formatVersion !== RESET_TRANSACTION_FORMAT_VERSION) { + throw new Error("unsupported or malformed reset transaction") + } + const phase = requiredString(value, "phase") + if (!["intent", "live-cleared", "markers-cleared"].includes(phase)) { + throw new Error("invalid reset transaction phase") + } + const dataDir = requiredString(value, "dataDir") + if (!isAbsolute(dataDir) || resolve(dataDir) !== resolve(expectedDataDir)) { + throw new Error("reset transaction data directory does not match its configured owner") + } + if (!Array.isArray(value.targets)) throw new Error("invalid reset transaction targets") + const targets = value.targets.map((target) => { + if (typeof target !== "string" || !RESETTABLE_CHDB_ENTRIES.has(target)) { + throw new Error(`unsafe reset transaction target: ${String(target)}`) + } + return target + }) + if (new Set(targets).size !== targets.length || [...targets].sort().join("\0") !== targets.join("\0")) { + throw new Error("reset transaction targets must be unique and sorted") + } + return { + formatVersion: 1, + operationId: validateId(requiredString(value, "operationId"), "reset operation"), + dataDir: resolve(dataDir), + targets, + phase: phase as ResetTransaction["phase"], + createdAt: requiredIso(value, "createdAt"), + } +} + +const readResetTransaction = async (dataDir: string): Promise => { + const path = resetTransactionPath(dataDir) + try { + await assertRealFile(path, "reset transaction") + return parseResetTransaction(JSON.parse(await readFile(path, "utf8")), dataDir) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null + throw error + } +} + +const writeResetTransaction = async (dataDir: string, transaction: ResetTransaction): Promise => + durableJson(resetTransactionPath(dataDir), transaction) + +const reconcileResetTransactionUnlocked = async ( + dataDir: string, + faults: RestoreRecoveryFaults = {}, +): Promise => { + let transaction = await readResetTransaction(dataDir) + if (!transaction) return false + if (existsSync(restoreTransactionPath(dataDir))) { + throw new Error("reset and restore transactions both exist; refusing to choose one") + } + const live = resolve(dataDir) + if (existsSync(live)) await assertRealDirectory(live, "live data directory") + + if (transaction.phase === "intent") { + for (const target of transaction.targets) { + const path = join(live, target) + if (!existsSync(path)) continue + await assertRealDirectory(path, `reset target ${target}`) + await rm(path, { recursive: true }) + await syncDirectory(live) + await faults.afterResetEntryRemoval?.(target) + } + transaction = { ...transaction, phase: "live-cleared" } + await writeResetTransaction(dataDir, transaction) + await faults.afterResetLiveClearedRecord?.() + } + + if (transaction.phase === "live-cleared") { + for (const target of transaction.targets) { + if (existsSync(join(live, target))) { + throw new Error(`reset transaction target reappeared before marker removal: ${target}`) + } + } + const marker = storeMarkerPath(dataDir) + if (existsSync(marker)) await durableRemove(marker) + await faults.afterResetStoreMarkerRemoval?.() + const openMarker = storeOpenMarkerPath(dataDir) + if (existsSync(openMarker)) await durableRemove(openMarker) + await faults.afterResetOpenMarkerRemoval?.() + transaction = { ...transaction, phase: "markers-cleared" } + await writeResetTransaction(dataDir, transaction) + await faults.afterResetMarkersClearedRecord?.() + } + + for (const target of transaction.targets) { + if (existsSync(join(live, target))) { + throw new Error(`reset transaction target reappeared after deletion: ${target}`) + } + } + await durableRemove(resetTransactionPath(dataDir)) + await faults.afterResetTransactionRemoval?.() + return true +} + +const beginResetTransactionUnlocked = async ( + dataDir: string, + operationId: string, + faults: RestoreRecoveryFaults = {}, +): Promise => { + await assertCheckpointInfrastructureSafe(dataDir) + const live = resolve(dataDir) + const targets: string[] = [] + const unknown: string[] = [] + if (existsSync(live)) { + await assertRealDirectory(live, "live data directory") + const entries = await readdir(live, { withFileTypes: true }) + for (const entry of entries) { + if (entry.name === "backups") continue + if (!RESETTABLE_CHDB_ENTRIES.has(entry.name)) { + unknown.push(join(live, entry.name)) + continue + } + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new Error(`reset target is not a real chDB directory: ${join(live, entry.name)}`) + } + targets.push(entry.name) + } + } + if (unknown.length > 0) { + throw new Error( + `unrecognized data-directory entries were preserved; refusing reset: ${unknown.sort().join(", ")}`, + ) + } + const transaction: ResetTransaction = { + formatVersion: 1, + operationId: validateId(operationId, "reset operation"), + dataDir: live, + targets: targets.sort(), + phase: "intent", + createdAt: new Date().toISOString(), + } + await writeResetTransaction(dataDir, transaction) + await faults.afterResetIntent?.() + await reconcileResetTransactionUnlocked(dataDir, faults) +} + const parseRestoreTransaction = (value: unknown): RestoreTransaction => { if (!isRecord(value) || value.formatVersion !== RESTORE_TRANSACTION_FORMAT_VERSION) { throw new Error("unsupported or malformed restore transaction") @@ -1396,7 +1595,8 @@ export const reconcileCheckpointRecovery = ( const operationId = randomUUID() const release = await acquireMaintenance(dataDir, operationId) try { - await reconcileRestoreTransactionUnlocked(dataDir, faults) + const resetReconciled = await reconcileResetTransactionUnlocked(dataDir, faults) + if (!resetReconciled) await reconcileRestoreTransactionUnlocked(dataDir, faults) } finally { await release() } @@ -1410,31 +1610,19 @@ export const reconcileCheckpointRecovery = ( * registry below `/backups`. The maintenance lock serializes this * destructive operation with checkpoint, restore, and archive work. */ -export const resetLiveStorePreservingCheckpoints = (dataDir: string): Effect.Effect => +export const resetLiveStorePreservingCheckpoints = ( + dataDir: string, + faults: RestoreRecoveryFaults = {}, +): Effect.Effect => Effect.tryPromise({ try: async () => { const operationId = randomUUID() const release = await acquireMaintenance(dataDir, operationId) try { - await reconcileRestoreTransactionUnlocked(dataDir) - await assertCheckpointInfrastructureSafe(dataDir) - const live = resolve(dataDir) - if (existsSync(live)) { - await assertRealDirectory(live, "live data directory") - const entries = await readdir(live, { withFileTypes: true }) - const removable = entries - .filter((entry) => entry.name !== "backups") - .sort((left, right) => { - const core = (name: string) => (name === "store" || name === "metadata" ? 1 : 0) - return core(left.name) - core(right.name) - }) - for (const entry of removable) { - await rm(join(live, entry.name), { recursive: true, force: true }) - } - await syncDirectory(live) - } - for (const marker of [storeMarkerPath(dataDir), storeOpenMarkerPath(dataDir)]) { - if (existsSync(marker)) await durableRemove(marker) + const resetReconciled = await reconcileResetTransactionUnlocked(dataDir, faults) + if (!resetReconciled) { + await reconcileRestoreTransactionUnlocked(dataDir) + await beginResetTransactionUnlocked(dataDir, operationId, faults) } } finally { await release() @@ -1461,6 +1649,7 @@ export const restoreCheckpoint = ( const quarantineId = randomUUID() const release = await acquireMaintenance(dataDir, operationId) try { + await reconcileResetTransactionUnlocked(dataDir) await reconcileRestoreTransactionUnlocked(dataDir) const resolvedCheckpoint = await resolveCheckpoint(dataDir, selector) const restoreRoot = restoreRootPath(dataDir, operationId) diff --git a/apps/cli/src/server/durable-files.ts b/apps/cli/src/server/durable-files.ts index 8a22a189a..685fd67de 100644 --- a/apps/cli/src/server/durable-files.ts +++ b/apps/cli/src/server/durable-files.ts @@ -10,6 +10,11 @@ export interface DurabilityFaults { readonly beforeRemove?: (path: string) => void | Promise readonly afterRetirementIntent?: (path: string) => void | Promise readonly afterRetirementRename?: (path: string) => void | Promise + readonly afterRetiredSnapshotRemoval?: (path: string) => void | Promise + readonly afterRetirementComplete?: (path: string) => void | Promise + readonly afterRetirementCleanupRename?: (path: string) => void | Promise + readonly afterRetirementCleanupRemoval?: (path: string) => void | Promise + readonly afterCompletedOperationPreserved?: (path: string) => void | Promise } // APFS and the target Linux filesystems support directory fsync. Keep the diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index 58c2a8383..bcb7e2ff7 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -28,6 +28,7 @@ import { readCheckpointState, reconcileCheckpointRecovery, reconcileCheckpointOperations, + resetTransactionPath, resetLiveStorePreservingCheckpoints, type RestoreRecoveryFaults, resolveCheckpoint, @@ -90,10 +91,9 @@ const writeSnapshot = (dataDir: string, checkpointId: string, operationId = newC const snapshot = checkpointSnapshotDir(dataDir, checkpointId) mkdirSync(join(snapshot, "backup"), { recursive: true }) writeFileSync(join(snapshot, "backup", "data.bin"), "backup") - writeFileSync( - join(snapshot, "manifest.json"), - `${JSON.stringify(manifest(checkpointId, operationId, dataDir))}\n`, - ) + const value = manifest(checkpointId, operationId, dataDir) + value.backupBytes = 6 + writeFileSync(join(snapshot, "manifest.json"), `${JSON.stringify(value)}\n`) } const writeState = ( @@ -350,6 +350,20 @@ describe("checkpoint state resolution", () => { ok(existsSync(join(outside, "backup"))) }) }) + + it("rejects nested symlinks inside an otherwise real checkpoint backup", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + const outside = join(dirname(dataDir), "outside-backup.bin") + writeFileSync(outside, "sensitive") + writeSnapshot(dataDir, checkpointId) + symlinkSync(outside, join(checkpointSnapshotDir(dataDir, checkpointId), "backup", "nested-link")) + writeState(dataDir, checkpointId) + + await rejects(readCheckpointState(dataDir), /symlink/) + strictEqual(readFileSync(outside, "utf8"), "sensitive") + }) + }) }) describe("checkpoint reconciliation and retention", () => { @@ -461,8 +475,13 @@ describe("checkpoint reconciliation and retention", () => { }) }) - it("converges after interruption at retirement intent and snapshot-rename boundaries", async () => { - for (const boundary of ["afterRetirementIntent", "afterRetirementRename"] as const) { + it("converges after every retirement intent, data-removal, and completion boundary", async () => { + for (const boundary of [ + "afterRetirementIntent", + "afterRetirementRename", + "afterRetiredSnapshotRemoval", + "afterRetirementComplete", + ] as const) { await withDataDir(async (dataDir) => { const current = newCheckpointId() const previous = newCheckpointId() @@ -492,6 +511,51 @@ describe("checkpoint reconciliation and retention", () => { } }) + it("converges after every retirement cleanup and completed-operation boundary", async () => { + for (const boundary of [ + "afterRetirementCleanupRename", + "afterRetirementCleanupRemoval", + "afterCompletedOperationPreserved", + ] as const) { + await withDataDir(async (dataDir) => { + const oldCurrent = newCheckpointId() + const oldPrevious = newCheckpointId() + const baseRevision = newCheckpointId() + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, oldCurrent) + writeSnapshot(dataDir, oldPrevious) + writeSnapshot(dataDir, checkpointId, operationId) + writeState(dataDir, checkpointId, oldCurrent, operationId) + writeCheckpointOperation(dataDir, operationId, checkpointId, "pointer-complete", { + revision: baseRevision, + current: oldCurrent, + previous: oldPrevious, + }) + let injected = false + + await rejects( + reconcileCheckpointOperations(dataDir, { + [boundary]: () => { + if (injected) return + injected = true + throw new Error(`injected ${boundary}`) + }, + }), + /injected/, + ) + await reconcileCheckpointOperations(dataDir) + + strictEqual((await readCheckpointState(dataDir)).current, checkpointId) + ok(!existsSync(checkpointSnapshotDir(dataDir, oldPrevious)), boundary) + ok( + !existsSync(join(checkpointRoot(dataDir), "operations", `checkpoint-${operationId}`)), + boundary, + ) + }) + } + }) + it("cleans an exactly completed retirement after the operation completion record", async () => { for (const keepRetirementRecord of [true, false]) { await withDataDir(async (dataDir) => { @@ -583,9 +647,10 @@ describe("live-store reset safety", () => { writeState(dataDir, checkpointId) mkdirSync(join(dataDir, "store"), { recursive: true }) mkdirSync(join(dataDir, "metadata"), { recursive: true }) + mkdirSync(join(dataDir, "tmp"), { recursive: true }) writeFileSync(join(dataDir, "store", "part.bin"), "live") writeFileSync(join(dataDir, "metadata", "table.sql"), "live") - writeFileSync(join(dataDir, "status"), "live") + writeFileSync(join(dataDir, "tmp", "scratch.bin"), "live") writeFileSync(storeMarkerPath(dataDir), "{}") writeFileSync(storeOpenMarkerPath(dataDir), "999\n") @@ -595,9 +660,10 @@ describe("live-store reset safety", () => { ok(existsSync(checkpointSnapshotDir(dataDir, checkpointId))) ok(!existsSync(join(dataDir, "store"))) ok(!existsSync(join(dataDir, "metadata"))) - ok(!existsSync(join(dataDir, "status"))) + ok(!existsSync(join(dataDir, "tmp"))) ok(!existsSync(storeMarkerPath(dataDir))) ok(!existsSync(storeOpenMarkerPath(dataDir))) + ok(!existsSync(resetTransactionPath(dataDir))) }) }) @@ -614,6 +680,93 @@ describe("live-store reset safety", () => { strictEqual(readFileSync(join(dataDir, "store", "preserve.bin"), "utf8"), "live") }) }) + + it("preserves and reports unknown data-directory entries before any deletion", async () => { + await withDataDir(async (dataDir) => { + mkdirSync(join(dataDir, "store"), { recursive: true }) + writeFileSync(join(dataDir, "store", "preserve.bin"), "live") + writeFileSync(join(dataDir, "user-owned.txt"), "preserve") + + await rejects( + Effect.runPromise(resetLiveStorePreservingCheckpoints(dataDir)), + /unrecognized data-directory entries were preserved/, + ) + + strictEqual(readFileSync(join(dataDir, "store", "preserve.bin"), "utf8"), "live") + strictEqual(readFileSync(join(dataDir, "user-owned.txt"), "utf8"), "preserve") + ok(!existsSync(resetTransactionPath(dataDir))) + }) + }) + + it("reconciles interruption at every reset deletion, marker, and journal boundary", async () => { + const boundaries: ReadonlyArray = [ + "afterResetIntent", + "afterResetEntryRemoval", + "afterResetLiveClearedRecord", + "afterResetStoreMarkerRemoval", + "afterResetOpenMarkerRemoval", + "afterResetMarkersClearedRecord", + "afterResetTransactionRemoval", + ] + for (const boundary of boundaries) { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId) + writeState(dataDir, checkpointId) + for (const entry of ["data", "metadata", "store", "tmp"]) { + mkdirSync(join(dataDir, entry), { recursive: true }) + writeFileSync(join(dataDir, entry, "live.bin"), "live") + } + writeFileSync(storeMarkerPath(dataDir), "{}") + writeFileSync(storeOpenMarkerPath(dataDir), "999\n") + let injected = false + const faults = { + [boundary]: () => { + if (injected) return + injected = true + throw new Error(`injected ${boundary}`) + }, + } as RestoreRecoveryFaults + + await rejects( + Effect.runPromise(resetLiveStorePreservingCheckpoints(dataDir, faults)), + /injected/, + ) + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + + for (const entry of ["data", "metadata", "store", "tmp"]) { + ok(!existsSync(join(dataDir, entry)), `${boundary}: ${entry}`) + } + strictEqual((await readCheckpointState(dataDir)).current, checkpointId) + ok(!existsSync(storeMarkerPath(dataDir)), boundary) + ok(!existsSync(storeOpenMarkerPath(dataDir)), boundary) + ok(!existsSync(resetTransactionPath(dataDir)), boundary) + }) + } + }) + + it("rejects malformed or escaping reset journals without mutation", async () => { + await withDataDir(async (dataDir) => { + const outside = join(dirname(dataDir), "outside-reset") + mkdirSync(outside) + writeFileSync(join(outside, "preserve"), "outside") + writeFileSync( + resetTransactionPath(dataDir), + `${JSON.stringify({ + formatVersion: 1, + operationId: newCheckpointId(), + dataDir, + targets: ["../outside-reset"], + phase: "intent", + createdAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ) + + await rejects(Effect.runPromise(reconcileCheckpointRecovery(dataDir)), /unsafe reset/) + strictEqual(readFileSync(join(outside, "preserve"), "utf8"), "outside") + ok(existsSync(resetTransactionPath(dataDir))) + }) + }) }) describe("live restore transaction reconciliation", () => { diff --git a/apps/landing/src/content/docs/local-mode/cli-reference.md b/apps/landing/src/content/docs/local-mode/cli-reference.md index 6eb7c10a0..cf1cd7be5 100644 --- a/apps/landing/src/content/docs/local-mode/cli-reference.md +++ b/apps/landing/src/content/docs/local-mode/cli-reference.md @@ -58,9 +58,9 @@ maple start -d --port 4400 # detached on a custom port Detached startup forwards the selected `--on-dirty-store` policy unchanged to the foreground child. Before any reset, compatibility check, dirty-store -decision, or data-directory creation, startup reconciles a recorded checkpoint -restore transaction. Ambiguous or malformed restore state fails closed and -prints the preserved paths. +decision, or data-directory creation, startup reconciles a recorded reset or +checkpoint-restore transaction. Ambiguous, malformed, or conflicting +transaction state fails closed and prints the preserved paths. The default dirty-store policy is `fail`, so an unclean shutdown never silently deletes telemetry. Choose `restore-checkpoint` to recover the selected @@ -84,6 +84,13 @@ Delete live chDB data so the next `maple start` bootstraps fresh. The checkpoint registry under `/backups` is preserved. Refuses to run while a server still owns the store. +Reset is journaled beside the data directory and removes only the chDB-owned +top-level directories produced by the bundled native build (`data`, `metadata`, +`store`, and `tmp`). If any other entry exists, reset preserves everything and +fails with the unrecognized paths so an operator can inspect them. Startup +finishes an interrupted recorded reset before it evaluates store compatibility +or cleanliness. + | Flag | Default | Description | | ------------------- | --------------- | ---------------------------- | | `--data-dir ` | `~/.maple/data` | Store whose live data clears | From 4fdcbb79edc5b5935793ee830a5f9611d21be168 Mon Sep 17 00:00:00 2001 From: remote service administrator Date: Sat, 11 Jul 2026 23:57:51 -0400 Subject: [PATCH 10/12] fix(cli): require repeatable checkpoint reopen --- .github/workflows/ci.yml | 32 ++++++++- apps/cli/package.json | 2 +- apps/cli/src/bin.ts | 32 +++++++-- apps/cli/src/server/chdb.ts | 36 +++++----- apps/cli/src/server/checkpoints.ts | 88 +++++++++++++++++++++++- apps/cli/test/chdb-args.test.ts | 28 ++++++++ apps/cli/test/checkpoints.test.ts | 9 ++- apps/cli/test/native-checkpoint-smoke.sh | 32 ++++++--- scripts/build-local-binary.sh | 22 +++--- 9 files changed, 229 insertions(+), 52 deletions(-) create mode 100644 apps/cli/test/chdb-args.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 015973e97..4140f1c5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: # shellcheck is preinstalled on the ubuntu-latest runner image. - name: shellcheck - run: shellcheck scripts/install.sh scripts/uninstall.sh scripts/build-local-binary.sh + run: shellcheck scripts/install.sh scripts/uninstall.sh scripts/build-local-binary.sh apps/cli/test/native-checkpoint-smoke.sh # Syntax-check the POSIX installers against the actual /bin/sh they claim. - name: sh -n @@ -54,6 +54,36 @@ jobs: sh -n scripts/install.sh sh -n scripts/uninstall.sh + local-checkpoint-native: + name: Local checkpoint native + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2.4.4 + + - run: bun install --frozen-lockfile + + - name: Build native local bundle + run: scripts/build-local-binary.sh + + - name: Run repeated checkpoint restore and fresh-process reopen smoke + env: + KEEP_ROOT: "1" + TMPDIR: ${{ runner.temp }}/maple-native-checkpoint + run: | + mkdir -p "$TMPDIR" + apps/cli/test/native-checkpoint-smoke.sh "$GITHUB_WORKSPACE/dist" + + - name: Upload checkpoint failure evidence + if: ${{ failure() }} + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 + with: + name: native-checkpoint-failure-${{ github.run_id }} + path: ${{ runner.temp }}/maple-native-checkpoint + if-no-files-found: ignore + service-map-perf: name: Service Map Perf runs-on: ubuntu-latest diff --git a/apps/cli/package.json b/apps/cli/package.json index 8ca0c0e8a..7dc289b0c 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -7,7 +7,7 @@ "type": "module", "scripts": { "start": "bun run src/bin.ts", - "test": "vitest run", + "test": "bun test test/*.test.ts", "typecheck": "tsc --noEmit" }, "dependencies": { diff --git a/apps/cli/src/bin.ts b/apps/cli/src/bin.ts index 5945cbaf9..589fb12e7 100644 --- a/apps/cli/src/bin.ts +++ b/apps/cli/src/bin.ts @@ -9,6 +9,7 @@ import { Mode } from "./core/mode" import { TelemetryLayer } from "./core/telemetry" import { maybeNotifyUpdate } from "./core/update" import { WarehouseExecutorFromMode } from "./core/warehouse" +import { CHECKPOINT_REOPEN_PROBE_ENV, validateCheckpointDataDir } from "./server/checkpoints" import { MAPLE_VERSION } from "./version" // WarehouseExecutorFromMode needs Mode (which needs MapleConfig). provideMerge @@ -31,10 +32,27 @@ const MainLayer = WarehouseExecutorFromMode.pipe( // exporter flushes when its layer scope closes, and only the outermost provide's // scope is the runtime's main scope that `BunRuntime.runMain` closes on exit. // Merging it into MainLayer leaves spans unflushed for short-lived commands. -maybeNotifyUpdate.pipe( - Effect.flatMap(() => Command.run(cli, { version: MAPLE_VERSION })), - Effect.withSpan("maple", { attributes: { "cli.argv": process.argv.slice(2).join(" ") } }), - Effect.provide(MainLayer), - Effect.provide(TelemetryLayer), - BunRuntime.runMain, -) +const checkpointProbeDataDir = process.env[CHECKPOINT_REOPEN_PROBE_ENV] + +if (checkpointProbeDataDir !== undefined) { + // Private re-exec path used by checkpoint restore. It intentionally bypasses + // CLI dispatch, update checks, telemetry, and schema bootstrap: success means + // this new process loaded the persisted restored representation and queried + // its core tables before closing chDB cleanly. + try { + process.stdout.write(`${JSON.stringify(validateCheckpointDataDir(checkpointProbeDataDir))}\n`) + } catch (error) { + process.stderr.write( + `checkpoint reopen probe failed: ${error instanceof Error ? error.message : String(error)}\n`, + ) + process.exitCode = 1 + } +} else { + maybeNotifyUpdate.pipe( + Effect.flatMap(() => Command.run(cli, { version: MAPLE_VERSION })), + Effect.withSpan("maple", { attributes: { "cli.argv": process.argv.slice(2).join(" ") } }), + Effect.provide(MainLayer), + Effect.provide(TelemetryLayer), + BunRuntime.runMain, + ) +} diff --git a/apps/cli/src/server/chdb.ts b/apps/cli/src/server/chdb.ts index 3823c8b2f..3d5c7e215 100644 --- a/apps/cli/src/server/chdb.ts +++ b/apps/cli/src/server/chdb.ts @@ -87,6 +87,21 @@ export interface ChdbOptions { readonly bootstrapSchema?: boolean } +/** Build the embedded ClickHouse argv. Keep table metadata loading serialized: + * chDB v26.1.0 can otherwise fail nondeterministically while its loader resolves + * Maple's materialized-view dependency graph (`recursive_mutex lock failed` / + * `ASYNC_LOAD_WAIT_FAILED`). `async_load_databases=0` waits for loading, but it + * does not make the loader pools single-threaded. */ +export const chdbArgv = (options: Pick): string[] => [ + "clickhouse", + "--async_load_databases=0", + "--async_load_system_database=0", + "--tables_loader_foreground_pool_size=1", + "--tables_loader_background_pool_size=1", + `--path=${options.dataDir}`, + ...(options.configFile ? [`--config-file=${options.configFile}`] : []), +] + /** * A live chDB connection. `query` runs read SQL and returns the raw result * bytes (in whatever `format` was requested — default JSONEachRow). `exec` runs @@ -105,26 +120,7 @@ export class Chdb { static open(options: ChdbOptions): Chdb { const sym = symbols() - // `--async_load_databases=0`: make chdb_connect BLOCK until every persisted - // table has finished loading before it returns. chDB v26.1.0 defaults this to - // true, so connect returns while the existing tables (our ~30 MVs feeding - // MergeTree targets) are still loading on background loader threads — and the - // `#bootstrap` DDL we run immediately after (`CREATE TABLE IF NOT EXISTS …`) - // then races the concurrent load of the same table, tripping a - // `recursive_mutex lock failed: Invalid argument` → `ASYNC_LOAD_WAIT_FAILED` - // → chdb_connect returns NULL. Most visible after an unclean kill, when more - // load/merge work is still in flight on reopen. Waiting serializes load before - // bootstrap and substantially narrows the race. One fresh-process native smoke - // still observed a non-reproducible ASYNC_LOAD_WAIT_FAILED before 40 focused - // restart passes; never retry a failed FFI open in this process. - // (`--async_load_system_database=0` is set explicitly for symmetry.) - const args = [ - "clickhouse", - "--async_load_databases=0", - "--async_load_system_database=0", - `--path=${options.dataDir}`, - ...(options.configFile ? [`--config-file=${options.configFile}`] : []), - ] + const args = chdbArgv(options) const argBufs = args.map(cstr) const argv = new BigUint64Array(args.length) argBufs.forEach((b, i) => { diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index 7eec57622..dc8ecf101 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto" +import { spawnSync } from "node:child_process" import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from "node:fs" import { cp, lstat, mkdir, readFile, readdir, rm, stat } from "node:fs/promises" import { tmpdir } from "node:os" @@ -31,6 +32,7 @@ const RESTORE_TRANSACTION_FORMAT_VERSION = 1 const RESET_TRANSACTION_FORMAT_VERSION = 1 const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i const RESETTABLE_CHDB_ENTRIES = new Set(["data", "metadata", "store", "tmp"]) +export const CHECKPOINT_REOPEN_PROBE_ENV = "MAPLE_INTERNAL_CHECKPOINT_REOPEN_DATA_DIR" export class CheckpointError extends Schema.TaggedErrorClass()( "@maple/cli/CheckpointError", @@ -386,6 +388,34 @@ const validateRestoredDatabase = (db: Chdb): CheckpointValidation => ({ ), }) +/** Open and validate a restored store in the current process. Production + * restore calls this only from the dedicated child-process entrypoint in + * `bin.ts`; keeping it synchronous ensures the child exits only after chDB has + * closed cleanly. */ +export const validateCheckpointDataDir = (dataDir: string): CheckpointValidation => { + const resolvedDataDir = resolve(dataDir) + if (!isAbsolute(dataDir) || resolvedDataDir !== dataDir) { + throw new Error("checkpoint reopen probe requires a normalized absolute data directory") + } + if (!existsSync(resolvedDataDir)) { + throw new Error("checkpoint reopen probe data directory is missing") + } + const info = lstatSync(resolvedDataDir) + if (info.isSymbolicLink() || !info.isDirectory()) { + throw new Error("checkpoint reopen probe data directory is not a real directory") + } + const db = Chdb.open({ + dataDir: resolvedDataDir, + schemaSql, + bootstrapSchema: false, + }) + try { + return validateRestoredDatabase(db) + } finally { + db.close() + } +} + const dirSize = async (path: string): Promise => { let total = 0 const entries = await readdir(path, { withFileTypes: true }) @@ -413,6 +443,55 @@ const parseValidation = (value: unknown): CheckpointValidation => { } } +const validationCountsMatch = (left: CheckpointValidation, right: CheckpointValidation): boolean => + left.traces === right.traces && + left.logs === right.logs && + left.metricsSum === right.metricsSum && + left.metricsGauge === right.metricsGauge && + left.metricsHistogram === right.metricsHistogram && + left.metricsExponentialHistogram === right.metricsExponentialHistogram && + left.materializedViews === right.materializedViews + +/** Re-exec Maple and prove a wholly fresh process can load the restored + * representation. A successful query in the restoring connection is not + * sufficient: chDB's persisted metadata is loaded again only on process start. */ +const validateRestoredDatabaseInFreshProcess = ( + dataDir: string, + expected: CheckpointValidation, +): CheckpointValidation => { + const entry = process.argv[1] + const childArgs = entry && !entry.startsWith("/$bunfs") ? [entry] : [] + const child = spawnSync(process.execPath, childArgs, { + env: { ...process.env, [CHECKPOINT_REOPEN_PROBE_ENV]: resolve(dataDir) }, + encoding: "utf8", + timeout: 30_000, + maxBuffer: 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }) + const stderr = child.stderr.trim() + if (child.error) { + throw new Error(`fresh-process checkpoint reopen probe failed to run: ${child.error.message}`) + } + if (child.status !== 0) { + throw new Error( + `fresh-process checkpoint reopen probe failed${child.signal ? ` (${child.signal})` : ""}` + + `${stderr ? `: ${stderr.slice(-4096)}` : ""}`, + ) + } + let parsed: CheckpointValidation + try { + parsed = parseValidation(JSON.parse(child.stdout.trim()) as unknown) + } catch (error) { + throw new Error( + `fresh-process checkpoint reopen probe returned invalid output: ${error instanceof Error ? error.message : String(error)}`, + ) + } + if (!validationCountsMatch(expected, parsed)) { + throw new Error("fresh-process checkpoint reopen validation does not match the restoring process") + } + return parsed +} + export const parseCheckpointManifest = ( value: unknown, expectedCheckpointId?: string, @@ -1670,8 +1749,15 @@ export const restoreCheckpoint = ( await writeRestoreTransaction(dataDir, transaction) await mkdir(restoreRoot, { mode: 0o700 }) const restored = await restoreResolvedInto(resolvedCheckpoint, restoreData) - const validation = restored.validation + const restoringProcessValidation = restored.validation restored.db.close() + // Do not publish a representation that only the restoring connection can + // read. Re-exec Maple after close and require the persisted metadata to + // load and produce the same counts in a wholly fresh chDB process. + const validation = validateRestoredDatabaseInFreshProcess( + restoreData, + restoringProcessValidation, + ) await cp(checkpointRoot(dataDir), join(restoreData, "backups"), { recursive: true, force: false, diff --git a/apps/cli/test/chdb-args.test.ts b/apps/cli/test/chdb-args.test.ts new file mode 100644 index 000000000..f95f0424c --- /dev/null +++ b/apps/cli/test/chdb-args.test.ts @@ -0,0 +1,28 @@ +import { describe, it } from "@effect/vitest" +import { deepStrictEqual } from "node:assert" +import { chdbArgv } from "../src/server/chdb" + +describe("embedded chDB arguments", () => { + it("waits for metadata and serializes both table-loader pools", () => { + deepStrictEqual(chdbArgv({ dataDir: "/tmp/maple-data" }), [ + "clickhouse", + "--async_load_databases=0", + "--async_load_system_database=0", + "--tables_loader_foreground_pool_size=1", + "--tables_loader_background_pool_size=1", + "--path=/tmp/maple-data", + ]) + }) + + it("keeps an explicit config after the loader safety settings", () => { + deepStrictEqual(chdbArgv({ dataDir: "/tmp/maple-data", configFile: "/tmp/backups.xml" }), [ + "clickhouse", + "--async_load_databases=0", + "--async_load_system_database=0", + "--tables_loader_foreground_pool_size=1", + "--tables_loader_background_pool_size=1", + "--path=/tmp/maple-data", + "--config-file=/tmp/backups.xml", + ]) + }) +}) diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index bcb7e2ff7..f06e1d054 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -1,6 +1,6 @@ import { describe, it } from "@effect/vitest" import { Effect } from "effect" -import { deepStrictEqual, match, ok, rejects, strictEqual } from "node:assert" +import { deepStrictEqual, match, ok, rejects, strictEqual, throws } from "node:assert" import { existsSync, lstatSync, @@ -37,6 +37,7 @@ import { restoreRootPath, restoreTransactionPath, retireCheckpointIfEligible, + validateCheckpointDataDir, writeBackupConfig, } from "../src/server/checkpoints" import { @@ -87,6 +88,12 @@ const manifest = ( }, }) +describe("fresh-process checkpoint reopen probe", () => { + it("rejects a non-normalized or relative data directory before opening chDB", () => { + throws(() => validateCheckpointDataDir("relative/data"), /normalized absolute data directory/) + }) +}) + const writeSnapshot = (dataDir: string, checkpointId: string, operationId = newCheckpointId()): void => { const snapshot = checkpointSnapshotDir(dataDir, checkpointId) mkdirSync(join(snapshot, "backup"), { recursive: true }) diff --git a/apps/cli/test/native-checkpoint-smoke.sh b/apps/cli/test/native-checkpoint-smoke.sh index 7af07d237..d1f7e5af4 100755 --- a/apps/cli/test/native-checkpoint-smoke.sh +++ b/apps/cli/test/native-checkpoint-smoke.sh @@ -102,6 +102,21 @@ assert_counts() { done } +# A restore is not accepted merely because the restoring chDB connection could +# query it. Repeatedly exec wholly fresh Maple processes so persisted metadata +# must load from disk on every cycle. +assert_fresh_reopen_cycles() { + local expected="$1" + local cycles="${2:-3}" + local label="${3:-restored-store}" + for cycle in $(seq 1 "$cycles"); do + start_server fail + assert_counts "$expected" + stop_server + echo "fresh reopen $label: $cycle/$cycles" + done +} + printf '%s\n' \ '' \ ' ' \ @@ -141,12 +156,13 @@ jq -e --arg c "$C2" --arg p "$C1" \ stop_server "$MAPLE" restore --data-dir "$DATA" --checkpoint-id "$C1" --yes >/dev/null -start_server -assert_counts 1 -stop_server +assert_fresh_reopen_cycles 1 3 C1 "$MAPLE" restore --data-dir "$DATA" --checkpoint-id "$C2" --yes >/dev/null -start_server +assert_fresh_reopen_cycles 2 3 C2 + +# Leave one healthy process running for the dirty-store recovery cases below. +start_server fail assert_counts 2 # Foreground dirty-store recovery. @@ -189,9 +205,7 @@ jq -e --arg c "$C3" --arg p "$C2" \ [[ -d "$DATA/backups/snapshots/$C3" ]] || fail "current C3 is missing" stop_server -start_server fail -assert_counts 3 -stop_server +assert_fresh_reopen_cycles 3 5 C3-after-checkpoint "$MAPLE" reset --data-dir "$DATA" --yes >/dev/null jq -e --arg c "$C3" --arg p "$C2" \ @@ -202,8 +216,6 @@ start_server fail assert_counts 0 stop_server "$MAPLE" restore --data-dir "$DATA" --checkpoint-id "$C3" --yes >/dev/null -start_server fail -assert_counts 3 -stop_server +assert_fresh_reopen_cycles 3 3 C3-after-reset echo "PASS native checkpoint smoke: C1=$C1 C2=$C2 C3=$C3" diff --git a/scripts/build-local-binary.sh b/scripts/build-local-binary.sh index 6453103aa..06e79e25f 100755 --- a/scripts/build-local-binary.sh +++ b/scripts/build-local-binary.sh @@ -3,16 +3,17 @@ # executable plus libchdb. No Rust/cargo involved. # # Pipeline: -# 1. Build the lightweight SPA (`apps/local-ui` → its `dist/`). -# 2. Inline that dist into apps/cli/src/server/ui-embed.gen.ts so +# 1. Build the workspace packages consumed through gitignored `dist/` paths. +# 2. Build the lightweight SPA (`apps/local-ui` → its `dist/`). +# 3. Inline that dist into apps/cli/src/server/ui-embed.gen.ts so # `bun build --compile` bakes the SPA into the binary. -# 3. Compile apps/cli (the CLI + the OTLP-ingest/query server) into a single +# 4. Compile apps/cli (the CLI + the OTLP-ingest/query server) into a single # executable with `bun build --compile`. The schema artifacts and SPA are # embedded; the OTLP encoders run in-process; chDB is reached via bun:ffi. -# 4. Download libchdb (v26.1.0, matching what we test against) for the host +# 5. Download libchdb (v26.1.0, matching what we test against) for the host # platform and place it beside the binary. At runtime `maple` dlopens the # sibling libchdb (resolved relative to its own path) — no rpath tricks. -# 5. Restore the committed ui-embed.gen.ts stub so the tree stays clean. +# 6. Restore the committed ui-embed.gen.ts stub so the tree stays clean. # # The distributable is a 2-file bundle: `maple` + `libchdb.so`. Keep them in the # same directory. @@ -33,12 +34,11 @@ MAPLE_BUILD_VERSION="${MAPLE_BUILD_VERSION:-$(git -C "$REPO_ROOT" describe --tag mkdir -p "$OUT_DIR" -# apps/cli imports `@maple-dev/effect-sdk/server`, which is published from its -# built `dist/` (tsdown) — and `lib/effect-sdk/dist` is gitignored. A fresh -# checkout / CI only runs `bun install`, so the dist won't exist and -# `bun build --compile` fails to resolve the import. Build it first. -echo "==> Building @maple-dev/effect-sdk (telemetry SDK consumed by the CLI)" -bun run --filter @maple-dev/effect-sdk build +# Several workspace packages are published from gitignored `dist/` directories. +# A fresh checkout / CI only runs `bun install`, so build the repository's +# canonical prerequisite set before local-ui or the CLI resolves those imports. +echo "==> Building workspace prerequisites" +bun run alchemy:build-deps echo "==> Building local-ui SPA" bun run --filter @maple/local-ui build From d02fbbedafb42fd70b6658bed762340da1318651 Mon Sep 17 00:00:00 2001 From: remote service administrator Date: Sun, 12 Jul 2026 09:33:19 -0400 Subject: [PATCH 11/12] fix(cli): serialize checkpoint restore workers --- apps/cli/src/server/chdb.ts | 8 ++++++-- apps/cli/test/chdb-args.test.ts | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/cli/src/server/chdb.ts b/apps/cli/src/server/chdb.ts index 3d5c7e215..f7209a187 100644 --- a/apps/cli/src/server/chdb.ts +++ b/apps/cli/src/server/chdb.ts @@ -87,17 +87,21 @@ export interface ChdbOptions { readonly bootstrapSchema?: boolean } -/** Build the embedded ClickHouse argv. Keep table metadata loading serialized: +/** Build the embedded ClickHouse argv. Keep table metadata loading and restore + * work serialized: * chDB v26.1.0 can otherwise fail nondeterministically while its loader resolves * Maple's materialized-view dependency graph (`recursive_mutex lock failed` / * `ASYNC_LOAD_WAIT_FAILED`). `async_load_databases=0` waits for loading, but it - * does not make the loader pools single-threaded. */ + * does not make the loader pools single-threaded. RESTORE uses a separate + * 16-thread pool by default and can trip the same invalid recursive-mutex state + * while restoring that dependency graph. */ export const chdbArgv = (options: Pick): string[] => [ "clickhouse", "--async_load_databases=0", "--async_load_system_database=0", "--tables_loader_foreground_pool_size=1", "--tables_loader_background_pool_size=1", + "--restore_threads=1", `--path=${options.dataDir}`, ...(options.configFile ? [`--config-file=${options.configFile}`] : []), ] diff --git a/apps/cli/test/chdb-args.test.ts b/apps/cli/test/chdb-args.test.ts index f95f0424c..ed9fca70b 100644 --- a/apps/cli/test/chdb-args.test.ts +++ b/apps/cli/test/chdb-args.test.ts @@ -3,24 +3,26 @@ import { deepStrictEqual } from "node:assert" import { chdbArgv } from "../src/server/chdb" describe("embedded chDB arguments", () => { - it("waits for metadata and serializes both table-loader pools", () => { + it("waits for metadata and serializes table loading and restore work", () => { deepStrictEqual(chdbArgv({ dataDir: "/tmp/maple-data" }), [ "clickhouse", "--async_load_databases=0", "--async_load_system_database=0", "--tables_loader_foreground_pool_size=1", "--tables_loader_background_pool_size=1", + "--restore_threads=1", "--path=/tmp/maple-data", ]) }) - it("keeps an explicit config after the loader safety settings", () => { + it("keeps an explicit config after the loader and restore safety settings", () => { deepStrictEqual(chdbArgv({ dataDir: "/tmp/maple-data", configFile: "/tmp/backups.xml" }), [ "clickhouse", "--async_load_databases=0", "--async_load_system_database=0", "--tables_loader_foreground_pool_size=1", "--tables_loader_background_pool_size=1", + "--restore_threads=1", "--path=/tmp/maple-data", "--config-file=/tmp/backups.xml", ]) From fb19579eac14dd6a9d315be223a418f0e68d7952 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Tue, 14 Jul 2026 00:26:29 +0200 Subject: [PATCH 12/12] fix(cli): harden checkpoint effects and recovery --- apps/cli/package.json | 2 +- apps/cli/src/bin.ts | 13 +- apps/cli/src/commands/server.ts | 14 +- apps/cli/src/core/update.ts | 2 +- apps/cli/src/server/checkpoints.ts | 1216 ++++++++++++---------- apps/cli/src/server/serve.ts | 4 +- apps/cli/test/checkpoints.test.ts | 124 ++- apps/cli/test/native-checkpoint-smoke.sh | 30 + 8 files changed, 767 insertions(+), 638 deletions(-) diff --git a/apps/cli/package.json b/apps/cli/package.json index 7dc289b0c..a865f4ebb 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -7,7 +7,7 @@ "type": "module", "scripts": { "start": "bun run src/bin.ts", - "test": "bun test test/*.test.ts", + "test": "bun test test/*.test.ts src/server/otlp/encode.test.ts", "typecheck": "tsc --noEmit" }, "dependencies": { diff --git a/apps/cli/src/bin.ts b/apps/cli/src/bin.ts index 589fb12e7..833d39f1d 100644 --- a/apps/cli/src/bin.ts +++ b/apps/cli/src/bin.ts @@ -27,11 +27,11 @@ const MainLayer = WarehouseExecutorFromMode.pipe( // (network is hit at most once per 24h), so the latency cost is negligible. // // `cli.argv` records the sub-command + flags so one root span per invocation -// ties a command to the warehouse queries it runs. TelemetryLayer is provided -// OUTERMOST (after MainLayer), not merged into it: the OTLP tracer's batch -// exporter flushes when its layer scope closes, and only the outermost provide's -// scope is the runtime's main scope that `BunRuntime.runMain` closes on exit. -// Merging it into MainLayer leaves spans unflushed for short-lived commands. +// ties a command to the warehouse queries it runs. A single merged layer keeps +// both layer lifecycles in the runtime's main scope, so telemetry flushes when +// `BunRuntime.runMain` closes that scope for short-lived commands. +const RuntimeLayer = Layer.merge(MainLayer, TelemetryLayer) + const checkpointProbeDataDir = process.env[CHECKPOINT_REOPEN_PROBE_ENV] if (checkpointProbeDataDir !== undefined) { @@ -51,8 +51,7 @@ if (checkpointProbeDataDir !== undefined) { maybeNotifyUpdate.pipe( Effect.flatMap(() => Command.run(cli, { version: MAPLE_VERSION })), Effect.withSpan("maple", { attributes: { "cli.argv": process.argv.slice(2).join(" ") } }), - Effect.provide(MainLayer), - Effect.provide(TelemetryLayer), + Effect.provide(RuntimeLayer), BunRuntime.runMain, ) } diff --git a/apps/cli/src/commands/server.ts b/apps/cli/src/commands/server.ts index 6f217fca9..e0f064f4d 100644 --- a/apps/cli/src/commands/server.ts +++ b/apps/cli/src/commands/server.ts @@ -16,6 +16,7 @@ import { } from "../server/store-version" import { createCheckpoint, + parseCheckpointId, reconcileCheckpointRecovery, resetLiveStorePreservingCheckpoints, restoreCheckpoint, @@ -374,7 +375,7 @@ export const start = Command.make("start", { // `Effect.never`, closing the scope and running finalizers in reverse // registration order: remove PID → stop server → close chDB → print the // stopped notice. - yield* Effect.scoped( + return yield* Effect.scoped( Effect.gen(function* () { // Only announce "stopped" if we actually started. The finalizer is // registered up front so it fires on the SIGINT/SIGTERM shutdown, but @@ -424,7 +425,7 @@ export const start = Command.make("start", { process.stdout.write(startBanner(addr, dataDir, dashboardUrl, a.offline)), ) - yield* Effect.never + return yield* Effect.never }), ) }), @@ -566,8 +567,13 @@ export const restore = Command.make("restore", { return } - const checkpointId = Option.getOrUndefined(a.checkpointId) - const result = yield* restoreCheckpoint(dataDir, checkpointId ?? "current").pipe( + const rawCheckpointId = Option.getOrUndefined(a.checkpointId) + const checkpointId = yield* Effect.try({ + try: () => (rawCheckpointId === undefined ? "current" : parseCheckpointId(rawCheckpointId)), + catch: (error) => + new ServerError({ message: error instanceof Error ? error.message : String(error) }), + }) + const result = yield* restoreCheckpoint(dataDir, checkpointId).pipe( Effect.mapError((e) => new ServerError({ message: e.message })), ) yield* Effect.sync(() => diff --git a/apps/cli/src/core/update.ts b/apps/cli/src/core/update.ts index fd5eabb19..77e1a4364 100644 --- a/apps/cli/src/core/update.ts +++ b/apps/cli/src/core/update.ts @@ -338,4 +338,4 @@ export const maybeNotifyUpdate: Effect.Effect = Effect if (latest && isNewer(MAPLE_VERSION, latest)) { yield* Effect.sync(() => printNotice(MAPLE_VERSION, stripV(latest))) } -}).pipe(Effect.catch(() => Effect.void)) +}) diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index dc8ecf101..54b217d87 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -31,103 +31,186 @@ const OPERATION_FORMAT_VERSION = 1 const RESTORE_TRANSACTION_FORMAT_VERSION = 1 const RESET_TRANSACTION_FORMAT_VERSION = 1 const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i -const RESETTABLE_CHDB_ENTRIES = new Set(["data", "metadata", "store", "tmp"]) +const RESETTABLE_CHDB_ENTRIES = new Set(["data", "metadata", "status", "store", "tmp"]) export const CHECKPOINT_REOPEN_PROBE_ENV = "MAPLE_INTERNAL_CHECKPOINT_REOPEN_DATA_DIR" -export class CheckpointError extends Schema.TaggedErrorClass()( - "@maple/cli/CheckpointError", - { message: Schema.String }, +const CheckpointUuid = Schema.String.check(Schema.isPattern(CHECKPOINT_ID)) + +export const CheckpointId = CheckpointUuid.pipe(Schema.brand("@maple/cli/CheckpointId")) +export type CheckpointId = Schema.Schema.Type + +export const CheckpointOperationId = CheckpointUuid.pipe(Schema.brand("@maple/cli/CheckpointOperationId")) +export type CheckpointOperationId = Schema.Schema.Type + +export const CheckpointQuarantineId = CheckpointUuid.pipe(Schema.brand("@maple/cli/CheckpointQuarantineId")) +export type CheckpointQuarantineId = Schema.Schema.Type + +const IsoDateTime = Schema.String.check( + Schema.makeFilter((value: string) => Number.isFinite(Date.parse(value)), { + description: "Expected an ISO date-time string", + }), +) +const NonNegativeInt = Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)) +const CheckpointValidationSchema = Schema.Struct({ + validatedAt: IsoDateTime, + traces: NonNegativeInt, + logs: NonNegativeInt, + metricsSum: NonNegativeInt, + metricsGauge: NonNegativeInt, + metricsHistogram: NonNegativeInt, + metricsExponentialHistogram: NonNegativeInt, + materializedViews: NonNegativeInt, +}) + +export type CheckpointValidation = Schema.Schema.Type + +const CheckpointManifestSchema = Schema.Struct({ + formatVersion: Schema.Literal(MANIFEST_FORMAT_VERSION), + checkpointId: CheckpointId, + operationId: CheckpointOperationId, + mapleVersion: Schema.String, + chdbVersion: Schema.String, + schemaFingerprint: Schema.String, + createdAt: IsoDateTime, + sourceDataDir: Schema.String, + backupRelativePath: Schema.String, + backupBytes: NonNegativeInt, + validation: CheckpointValidationSchema, +}) + +export type CheckpointManifest = Schema.Schema.Type + +const CheckpointStateSchema = Schema.Struct({ + formatVersion: Schema.Literal(STATE_FORMAT_VERSION), + revision: CheckpointOperationId, + current: CheckpointId, + previous: Schema.NullOr(CheckpointId), + committedAt: IsoDateTime, +}) + +export type CheckpointState = Schema.Schema.Type + +const CheckpointOperationPhase = Schema.Literals([ + "intent", + "backup-complete", + "manifest-complete", + "pointer-complete", + "retention-complete", +]) +const RestoreTransactionPhase = Schema.Literals([ + "intent", + "restore-ready", + "old-quarantined", + "new-live", + "markers-committed", +]) +const ResetTransactionPhase = Schema.Literals(["intent", "live-cleared", "markers-cleared"]) +const ResetTarget = Schema.Literals(["data", "metadata", "status", "store", "tmp"]) + +const CheckpointOperationSchema = Schema.Struct({ + formatVersion: Schema.Literal(OPERATION_FORMAT_VERSION), + operationId: CheckpointOperationId, + checkpointId: CheckpointId, + baseRevision: Schema.NullOr(CheckpointOperationId), + baseCurrent: Schema.NullOr(CheckpointId), + basePrevious: Schema.NullOr(CheckpointId), + phase: CheckpointOperationPhase, + startedAt: IsoDateTime, +}) + +type CheckpointOperation = Schema.Schema.Type + +const MaintenanceOwnerSchema = Schema.Struct({ + formatVersion: Schema.Literal(1), + operationId: CheckpointOperationId, + pid: NonNegativeInt, + startedAt: IsoDateTime, +}) + +type MaintenanceOwner = Schema.Schema.Type + +const RestoreTransactionSchema = Schema.Struct({ + formatVersion: Schema.Literal(RESTORE_TRANSACTION_FORMAT_VERSION), + operationId: CheckpointOperationId, + checkpointId: CheckpointId, + quarantineId: CheckpointQuarantineId, + phase: RestoreTransactionPhase, + createdAt: IsoDateTime, + validation: Schema.NullOr(CheckpointValidationSchema), +}) + +type RestoreTransaction = Schema.Schema.Type + +const ResetTransactionSchema = Schema.Struct({ + formatVersion: Schema.Literal(RESET_TRANSACTION_FORMAT_VERSION), + operationId: CheckpointOperationId, + dataDir: Schema.String, + targets: Schema.Array(ResetTarget), + phase: ResetTransactionPhase, + createdAt: IsoDateTime, +}) + +type ResetTransaction = Schema.Schema.Type + +const RetirementRecordSchema = Schema.Struct({ + formatVersion: Schema.Literal(1), + retirementId: CheckpointOperationId, + checkpointId: CheckpointId, + stateRevision: CheckpointOperationId, +}) + +const RestoreReadySchema = Schema.Struct({ + formatVersion: Schema.Literal(1), + operationId: CheckpointOperationId, + checkpointId: CheckpointId, +}) + +const checkpointErrorFields = { + dataDir: Schema.String, + operationId: CheckpointOperationId, + message: Schema.String, + cause: Schema.String, +} + +export class CheckpointCreateError extends Schema.TaggedErrorClass()( + "@maple/cli/CheckpointCreateError", + { ...checkpointErrorFields, checkpointId: CheckpointId }, +) {} + +export class CheckpointRecoveryError extends Schema.TaggedErrorClass()( + "@maple/cli/CheckpointRecoveryError", + checkpointErrorFields, +) {} + +export class CheckpointResetError extends Schema.TaggedErrorClass()( + "@maple/cli/CheckpointResetError", + checkpointErrorFields, ) {} +export class CheckpointRestoreError extends Schema.TaggedErrorClass()( + "@maple/cli/CheckpointRestoreError", + { ...checkpointErrorFields, selector: Schema.String }, +) {} + +const errorMessage = (error: unknown): string => (error instanceof Error ? error.message : String(error)) + +const errorCause = (error: unknown): string => + error instanceof Error ? (error.stack ?? error.message) : String(error) + export interface CheckpointOptions { readonly dataDir: string readonly port: number readonly faults?: DurabilityFaults } -export interface CheckpointValidation { - readonly validatedAt: string - readonly traces: number - readonly logs: number - readonly metricsSum: number - readonly metricsGauge: number - readonly metricsHistogram: number - readonly metricsExponentialHistogram: number - readonly materializedViews: number -} - -export interface CheckpointManifest { - readonly formatVersion: 1 - readonly checkpointId: string - readonly operationId: string - readonly mapleVersion: string - readonly chdbVersion: string - readonly schemaFingerprint: string - readonly createdAt: string - readonly sourceDataDir: string - readonly backupRelativePath: string - readonly backupBytes: number - readonly validation: CheckpointValidation -} - -export interface CheckpointState { - readonly formatVersion: 1 - readonly revision: string - readonly current: string - readonly previous: string | null - readonly committedAt: string -} - export interface ResolvedCheckpoint { - readonly checkpointId: string + readonly checkpointId: CheckpointId readonly snapshotDir: string readonly backupDir: string readonly backupSqlPath: string readonly manifest: CheckpointManifest } -interface CheckpointOperation { - readonly formatVersion: 1 - readonly operationId: string - readonly checkpointId: string - readonly baseRevision: string | null - readonly baseCurrent: string | null - readonly basePrevious: string | null - readonly phase: - | "intent" - | "backup-complete" - | "manifest-complete" - | "pointer-complete" - | "retention-complete" - readonly startedAt: string -} - -interface MaintenanceOwner { - readonly formatVersion: 1 - readonly operationId: string - readonly pid: number - readonly startedAt: string -} - -interface RestoreTransaction { - readonly formatVersion: 1 - readonly operationId: string - readonly checkpointId: string - readonly quarantineId: string - readonly phase: "intent" | "restore-ready" | "old-quarantined" | "new-live" | "markers-committed" - readonly createdAt: string - readonly validation: CheckpointValidation | null -} - -interface ResetTransaction { - readonly formatVersion: 1 - readonly operationId: string - readonly dataDir: string - readonly targets: ReadonlyArray - readonly phase: "intent" | "live-cleared" | "markers-cleared" - readonly createdAt: string -} - export interface RestoreRecoveryFaults { readonly afterLiveQuarantineRename?: () => void | Promise readonly afterOldQuarantinedRecord?: () => void | Promise @@ -156,63 +239,63 @@ export const checkpointPinsRoot = (dataDir: string): string => join(checkpointRo export const checkpointQuarantineRoot = (dataDir: string): string => join(checkpointRoot(dataDir), "quarantine") export const checkpointRetiringRoot = (dataDir: string): string => join(checkpointRoot(dataDir), "retiring") -export const checkpointSnapshotDir = (dataDir: string, checkpointId: string): string => - join(checkpointSnapshotsRoot(dataDir), validateId(checkpointId, "checkpoint")) + +const decodeCheckpointId = Schema.decodeUnknownSync(CheckpointId) +const decodeCheckpointOperationId = Schema.decodeUnknownSync(CheckpointOperationId) +const decodeCheckpointQuarantineId = Schema.decodeUnknownSync(CheckpointQuarantineId) + +const validateCheckpointId = (value: unknown, kind = "checkpoint"): CheckpointId => { + try { + return decodeCheckpointId(value) + } catch { + throw new Error(`invalid ${kind} ID: ${String(value)}`) + } +} + +const validateOperationId = (value: unknown, kind = "operation"): CheckpointOperationId => { + try { + return decodeCheckpointOperationId(value) + } catch { + throw new Error(`invalid ${kind} ID: ${String(value)}`) + } +} + +const validateQuarantineId = (value: unknown): CheckpointQuarantineId => { + try { + return decodeCheckpointQuarantineId(value) + } catch { + throw new Error(`invalid quarantine ID: ${String(value)}`) + } +} + +export const checkpointSnapshotDir = (dataDir: string, checkpointId: CheckpointId): string => + join(checkpointSnapshotsRoot(dataDir), checkpointId) const maintenanceLockPath = (dataDir: string): string => `${resolve(dataDir)}.maple-maintenance-lock` export const restoreTransactionPath = (dataDir: string): string => `${resolve(dataDir)}.restore-transaction.json` export const resetTransactionPath = (dataDir: string): string => `${resolve(dataDir)}.reset-transaction.json` -export const restoreRootPath = (dataDir: string, operationId: string): string => - `${resolve(dataDir)}.restore-${validateId(operationId, "restore operation")}` -export const restoreDataPath = (dataDir: string, operationId: string): string => +export const restoreRootPath = (dataDir: string, operationId: CheckpointOperationId): string => + `${resolve(dataDir)}.restore-${operationId}` +export const restoreDataPath = (dataDir: string, operationId: CheckpointOperationId): string => join(restoreRootPath(dataDir, operationId), "data") -export const restoreQuarantinePath = (dataDir: string, operationId: string, quarantineId: string): string => - `${resolve(dataDir)}.quarantine-${validateId(operationId, "restore operation")}-${validateId( - quarantineId, - "quarantine", - )}` -const operationDir = (dataDir: string, operationId: string): string => - join(checkpointOperationsRoot(dataDir), `checkpoint-${validateId(operationId, "operation")}`) -const operationPath = (dataDir: string, operationId: string): string => +export const restoreQuarantinePath = ( + dataDir: string, + operationId: CheckpointOperationId, + quarantineId: CheckpointQuarantineId, +): string => `${resolve(dataDir)}.quarantine-${operationId}-${quarantineId}` +const operationDir = (dataDir: string, operationId: CheckpointOperationId): string => + join(checkpointOperationsRoot(dataDir), `checkpoint-${operationId}`) +const operationPath = (dataDir: string, operationId: CheckpointOperationId): string => join(operationDir(dataDir, operationId), "intent.json") -const snapshotManifestPath = (dataDir: string, checkpointId: string): string => +const snapshotManifestPath = (dataDir: string, checkpointId: CheckpointId): string => join(checkpointSnapshotDir(dataDir, checkpointId), "manifest.json") -const snapshotBackupDir = (dataDir: string, checkpointId: string): string => +const snapshotBackupDir = (dataDir: string, checkpointId: CheckpointId): string => join(checkpointSnapshotDir(dataDir, checkpointId), "backup") -const snapshotBackupRelativePath = (checkpointId: string): string => - `snapshots/${validateId(checkpointId, "checkpoint")}/backup` -const snapshotBackupSqlPath = (checkpointId: string): string => +const snapshotBackupRelativePath = (checkpointId: CheckpointId): string => `snapshots/${checkpointId}/backup` +const snapshotBackupSqlPath = (checkpointId: CheckpointId): string => `backups/${snapshotBackupRelativePath(checkpointId)}` -const validateId = (value: string, kind: string): string => { - if (!CHECKPOINT_ID.test(value)) throw new Error(`invalid ${kind} ID: ${value}`) - return value.toLowerCase() -} - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) - -const requiredString = (record: Record, key: string): string => { - const value = record[key] - if (typeof value !== "string" || value.length === 0) throw new Error(`invalid ${key}`) - return value -} - -const requiredCount = (record: Record, key: string): number => { - const value = record[key] - if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { - throw new Error(`invalid ${key}`) - } - return value -} - -const requiredIso = (record: Record, key: string): string => { - const value = requiredString(record, key) - if (!Number.isFinite(Date.parse(value))) throw new Error(`invalid ${key}`) - return value -} - const assertContained = (root: string, candidate: string, label: string): string => { const absoluteRoot = resolve(root) const absoluteCandidate = resolve(candidate) @@ -313,29 +396,65 @@ export const writeBackupConfig = (path: string, sourceDataDir?: string): void => ) } -export class LocalQueryError extends Error { - readonly status: number - readonly detail: string - - constructor(status: number, detail: string) { - super(`local query failed (${status})${detail ? `: ${detail}` : ""}`) - this.name = "LocalQueryError" - this.status = status - this.detail = detail - } -} +export class LocalQueryError extends Schema.TaggedErrorClass()( + "@maple/cli/LocalQueryError", + { + status: NonNegativeInt, + detail: Schema.String, + message: Schema.String, + cause: Schema.String, + }, +) {} -const postLocalQuery = async (port: number, sql: string): Promise => { - const response = await fetch(`http://127.0.0.1:${port}/local/query`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ sql }), +const localQueryError = (status: number, detail: string, cause = detail): LocalQueryError => + new LocalQueryError({ + status, + detail, + message: `local query failed (${status})${detail ? `: ${detail}` : ""}`, + cause, }) - if (!response.ok) { - const detail = await response.text().catch(() => "") - throw new LocalQueryError(response.status, detail) - } - return response.json() + +const postLocalQuery = (port: number, sql: string): Effect.Effect => { + const url = `http://127.0.0.1:${port}/local/query` + return Effect.gen(function* () { + const response = yield* Effect.tryPromise({ + try: (signal) => + fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ sql }), + signal, + }), + catch: (error) => localQueryError(0, errorMessage(error), errorCause(error)), + }) + yield* Effect.annotateCurrentSpan("http.response.status_code", response.status) + if (!response.ok) { + const detail = yield* Effect.tryPromise({ + try: () => response.text(), + catch: (error) => localQueryError(response.status, errorMessage(error), errorCause(error)), + }) + return yield* localQueryError(response.status, detail) + } + return yield* Effect.tryPromise({ + try: () => response.json() as Promise, + catch: (error) => localQueryError(response.status, errorMessage(error), errorCause(error)), + }) + }).pipe( + Effect.timeout("30 seconds"), + Effect.catchTag("TimeoutError", () => + Effect.fail(localQueryError(0, "local checkpoint query timed out after 30 seconds")), + ), + Effect.withSpan("CheckpointService.postLocalQuery", { + kind: "client", + attributes: { + "peer.service": "maple-local", + "http.request.method": "POST", + "server.address": "127.0.0.1", + "server.port": port, + "url.full": url, + }, + }), + ) } export const isMissingBackupConfigurationError = (error: unknown): boolean => { @@ -356,16 +475,25 @@ export const isMissingBackupConfigurationError = (error: unknown): boolean => { return backupSpecific } +const JsonRow = Schema.Record(Schema.String, Schema.Unknown) +const decodeJsonRow = Schema.decodeUnknownSync(JsonRow) + const readJsonRows = (text: string): ReadonlyArray> => text .split("\n") .map((line) => line.trim()) .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as Record) + .map((line) => { + try { + return decodeJsonRow(JSON.parse(line) as unknown) + } catch (error) { + throw new Error(`invalid JSONEachRow result: ${errorMessage(error)}`) + } + }) const countFrom = (rows: ReadonlyArray>): number => { const row = rows[0] - if (!row) return 0 + if (!row) throw new Error("count query returned no rows") const value = row["count()"] ?? row.count const count = typeof value === "number" ? value : Number(value ?? 0) if (!Number.isSafeInteger(count) || count < 0) throw new Error(`invalid count result: ${value}`) @@ -430,16 +558,10 @@ const dirSize = async (path: string): Promise => { } const parseValidation = (value: unknown): CheckpointValidation => { - if (!isRecord(value)) throw new Error("invalid validation") - return { - validatedAt: requiredIso(value, "validatedAt"), - traces: requiredCount(value, "traces"), - logs: requiredCount(value, "logs"), - metricsSum: requiredCount(value, "metricsSum"), - metricsGauge: requiredCount(value, "metricsGauge"), - metricsHistogram: requiredCount(value, "metricsHistogram"), - metricsExponentialHistogram: requiredCount(value, "metricsExponentialHistogram"), - materializedViews: requiredCount(value, "materializedViews"), + try { + return Schema.decodeUnknownSync(CheckpointValidationSchema)(value) + } catch (error) { + throw new Error(`invalid checkpoint validation: ${errorMessage(error)}`) } } @@ -494,39 +616,27 @@ const validateRestoredDatabaseInFreshProcess = ( export const parseCheckpointManifest = ( value: unknown, - expectedCheckpointId?: string, + expectedCheckpointId?: CheckpointId, expectedSourceDataDir?: string, ): CheckpointManifest => { - if (!isRecord(value) || value.formatVersion !== MANIFEST_FORMAT_VERSION) { - throw new Error("unsupported or malformed checkpoint manifest") + let manifest: CheckpointManifest + try { + manifest = Schema.decodeUnknownSync(CheckpointManifestSchema)(value) + } catch (error) { + throw new Error(`unsupported or malformed checkpoint manifest: ${errorMessage(error)}`) } - const checkpointId = validateId(requiredString(value, "checkpointId"), "checkpoint") - if (expectedCheckpointId && checkpointId !== validateId(expectedCheckpointId, "checkpoint")) { + if (expectedCheckpointId && manifest.checkpointId !== expectedCheckpointId) { throw new Error("checkpoint manifest ID does not match its snapshot directory") } - const operationId = validateId(requiredString(value, "operationId"), "operation") - const sourceDataDir = requiredString(value, "sourceDataDir") - if (!isAbsolute(sourceDataDir)) throw new Error("checkpoint sourceDataDir must be absolute") - if (expectedSourceDataDir && resolve(sourceDataDir) !== resolve(expectedSourceDataDir)) { + if (!isAbsolute(manifest.sourceDataDir)) { + throw new Error("checkpoint sourceDataDir must be absolute") + } + if (expectedSourceDataDir && resolve(manifest.sourceDataDir) !== resolve(expectedSourceDataDir)) { throw new Error("checkpoint sourceDataDir does not match its configured owner") } - const backupRelativePath = requiredString(value, "backupRelativePath") - if (backupRelativePath !== snapshotBackupRelativePath(checkpointId)) { + if (manifest.backupRelativePath !== snapshotBackupRelativePath(manifest.checkpointId)) { throw new Error("checkpoint backup path does not match its immutable ID") } - const manifest: CheckpointManifest = { - formatVersion: 1, - checkpointId, - operationId, - mapleVersion: requiredString(value, "mapleVersion"), - chdbVersion: requiredString(value, "chdbVersion"), - schemaFingerprint: requiredString(value, "schemaFingerprint"), - createdAt: requiredIso(value, "createdAt"), - sourceDataDir, - backupRelativePath, - backupBytes: requiredCount(value, "backupBytes"), - validation: parseValidation(value.validation), - } if (manifest.chdbVersion !== CHDB_VERSION) { throw new Error( `checkpoint chDB version mismatch (checkpoint: ${manifest.chdbVersion}; build: ${CHDB_VERSION})`, @@ -541,30 +651,16 @@ export const parseCheckpointManifest = ( } export const parseCheckpointState = (value: unknown): CheckpointState => { - if (!isRecord(value) || value.formatVersion !== STATE_FORMAT_VERSION) { - throw new Error("unsupported or malformed checkpoint state") - } - const current = validateId(requiredString(value, "current"), "current checkpoint") - const previousValue = value.previous - const previous = - previousValue === null - ? null - : validateId( - typeof previousValue === "string" - ? previousValue - : (() => { - throw new Error("invalid previous checkpoint") - })(), - "previous checkpoint", - ) - if (previous === current) throw new Error("checkpoint current and previous IDs must differ") - return { - formatVersion: 1, - revision: validateId(requiredString(value, "revision"), "state revision"), - current, - previous, - committedAt: requiredIso(value, "committedAt"), + let state: CheckpointState + try { + state = Schema.decodeUnknownSync(CheckpointStateSchema)(value) + } catch (error) { + throw new Error(`unsupported or malformed checkpoint state: ${errorMessage(error)}`) + } + if (state.previous === state.current) { + throw new Error("checkpoint current and previous IDs must differ") } + return state } const readStateFileOptional = async (dataDir: string): Promise => { @@ -625,23 +721,25 @@ export const readCheckpointState = async (dataDir: string): Promise => { +const resolveCheckpointById = async ( + dataDir: string, + checkpointId: CheckpointId, +): Promise => { await assertCheckpointInfrastructureSafe(dataDir) - const validatedCheckpointId = validateId(checkpointId, "checkpoint") - const snapshotDir = checkpointSnapshotDir(dataDir, validatedCheckpointId) + const snapshotDir = checkpointSnapshotDir(dataDir, checkpointId) const snapshotsRoot = checkpointSnapshotsRoot(dataDir) await assertNoSymlink(checkpointRoot(dataDir), snapshotsRoot) await assertNoSymlink(snapshotsRoot, snapshotDir) - await assertNoSymlink(snapshotsRoot, snapshotManifestPath(dataDir, validatedCheckpointId)) - await assertNoSymlink(snapshotsRoot, snapshotBackupDir(dataDir, validatedCheckpointId)) + await assertNoSymlink(snapshotsRoot, snapshotManifestPath(dataDir, checkpointId)) + await assertNoSymlink(snapshotsRoot, snapshotBackupDir(dataDir, checkpointId)) await assertRealDirectory(snapshotDir, "checkpoint snapshot") - await assertRealFile(snapshotManifestPath(dataDir, validatedCheckpointId), "checkpoint manifest") + await assertRealFile(snapshotManifestPath(dataDir, checkpointId), "checkpoint manifest") const manifest = parseCheckpointManifest( - JSON.parse(await readFile(snapshotManifestPath(dataDir, validatedCheckpointId), "utf8")), - validatedCheckpointId, + JSON.parse(await readFile(snapshotManifestPath(dataDir, checkpointId), "utf8")), + checkpointId, dataDir, ) - const backupDir = snapshotBackupDir(dataDir, validatedCheckpointId) + const backupDir = snapshotBackupDir(dataDir, checkpointId) await assertRealDirectory(backupDir, "checkpoint backup") const actualBackupBytes = await dirSize(backupDir) if (actualBackupBytes !== manifest.backupBytes) { @@ -650,33 +748,29 @@ const resolveCheckpointById = async (dataDir: string, checkpointId: string): Pro ) } return { - checkpointId: validatedCheckpointId, + checkpointId, snapshotDir, backupDir, - backupSqlPath: snapshotBackupSqlPath(validatedCheckpointId), + backupSqlPath: snapshotBackupSqlPath(checkpointId), manifest, } } export const resolveCheckpoint = async ( dataDir: string, - selector: "current" | "previous" | string = "current", + selector: "current" | "previous" | CheckpointId = "current", knownState?: CheckpointState, ): Promise => { const state = knownState ?? (await readCheckpointState(dataDir)) const checkpointId = - selector === "current" - ? state.current - : selector === "previous" - ? state.previous - : validateId(selector, "checkpoint") + selector === "current" ? state.current : selector === "previous" ? state.previous : selector if (!checkpointId) throw new Error("no previous checkpoint is selected") return resolveCheckpointById(dataDir, checkpointId) } export const readCheckpointManifest = async ( dataDir: string, - selector: "current" | "previous" | string = "current", + selector: "current" | "previous" | CheckpointId = "current", ): Promise => (await resolveCheckpoint(dataDir, selector)).manifest const restoreResolvedInto = async ( @@ -715,7 +809,7 @@ export const withRestoredCheckpoint = async ( readonly cleanup?: "always" | "never" }, use: (restored: { - readonly checkpointId: string + readonly checkpointId: CheckpointId readonly manifest: CheckpointManifest readonly scratchDataDir: string readonly db: Chdb @@ -756,33 +850,8 @@ export const withRestoredCheckpoint = async ( } const parseOperation = (value: unknown): CheckpointOperation => { - if (!isRecord(value) || value.formatVersion !== OPERATION_FORMAT_VERSION) { - throw new Error("unsupported or malformed checkpoint operation") - } - const phase = requiredString(value, "phase") - if ( - ![ - "intent", - "backup-complete", - "manifest-complete", - "pointer-complete", - "retention-complete", - ].includes(phase) - ) { - throw new Error("invalid checkpoint operation phase") - } - const baseRevision = - value.baseRevision === null - ? null - : validateId(requiredString(value, "baseRevision"), "base state revision") - const baseCurrent = - value.baseCurrent === null - ? null - : validateId(requiredString(value, "baseCurrent"), "base current checkpoint") - const basePrevious = - value.basePrevious === null - ? null - : validateId(requiredString(value, "basePrevious"), "base previous checkpoint") + const operation = Schema.decodeUnknownSync(CheckpointOperationSchema)(value) + const { baseRevision, baseCurrent, basePrevious } = operation if ((baseRevision === null) !== (baseCurrent === null)) { throw new Error("checkpoint operation has an inconsistent base state") } @@ -792,16 +861,7 @@ const parseOperation = (value: unknown): CheckpointOperation => { if (baseCurrent !== null && baseCurrent === basePrevious) { throw new Error("checkpoint operation base current and previous must differ") } - return { - formatVersion: 1, - operationId: validateId(requiredString(value, "operationId"), "operation"), - checkpointId: validateId(requiredString(value, "checkpointId"), "checkpoint"), - baseRevision, - baseCurrent, - basePrevious, - phase: phase as CheckpointOperation["phase"], - startedAt: requiredIso(value, "startedAt"), - } + return operation } const writeOperation = async ( @@ -813,7 +873,7 @@ const writeOperation = async ( const preserveCompletedOperation = async ( dataDir: string, operationDirPath: string, - operationId: string, + operationId: CheckpointOperationId, faults: DurabilityFaults = {}, ): Promise => { const quarantineRoot = checkpointQuarantineRoot(dataDir) @@ -822,10 +882,7 @@ const preserveCompletedOperation = async ( await assertRealDirectory(quarantineRoot, "checkpoint quarantine") } await ensurePrivateDirectory(quarantineRoot) - const preserved = join( - quarantineRoot, - `completed-operation-${validateId(operationId, "operation")}-${randomUUID()}`, - ) + const preserved = join(quarantineRoot, `completed-operation-${operationId}-${randomUUID()}`) await durableRename(operationDirPath, preserved, faults) await faults.afterCompletedOperationPreserved?.(preserved) } @@ -839,7 +896,10 @@ const processIsAlive = (pid: number): boolean => { } } -const acquireMaintenance = async (dataDir: string, operationId: string): Promise<() => Promise> => { +const acquireMaintenance = async ( + dataDir: string, + operationId: CheckpointOperationId, +): Promise<() => Promise> => { const lockPath = maintenanceLockPath(dataDir) if (existsSync(lockPath)) await assertRealDirectory(lockPath, "maintenance lock") try { @@ -850,14 +910,9 @@ const acquireMaintenance = async (dataDir: string, operationId: string): Promise try { const ownerPath = join(lockPath, "owner.json") await assertRealFile(ownerPath, "maintenance lock owner") - const parsed = JSON.parse(readFileSync(ownerPath, "utf8")) as unknown - if (!isRecord(parsed) || parsed.formatVersion !== 1) throw new Error("malformed owner") - owner = { - formatVersion: 1, - operationId: validateId(requiredString(parsed, "operationId"), "operation"), - pid: requiredCount(parsed, "pid"), - startedAt: requiredIso(parsed, "startedAt"), - } + owner = Schema.decodeUnknownSync(MaintenanceOwnerSchema)( + JSON.parse(readFileSync(ownerPath, "utf8")), + ) } catch { throw new Error(`maintenance lock is present but ownership is uncertain: ${lockPath}`) } @@ -878,12 +933,10 @@ const acquireMaintenance = async (dataDir: string, operationId: string): Promise return async () => { const ownerPath = join(lockPath, "owner.json") await assertRealFile(ownerPath, "maintenance lock owner") - const current = JSON.parse(await readFile(ownerPath, "utf8")) as unknown - if ( - !isRecord(current) || - requiredString(current, "operationId") !== operationId || - current.pid !== process.pid - ) { + const current = Schema.decodeUnknownSync(MaintenanceOwnerSchema)( + JSON.parse(await readFile(ownerPath, "utf8")), + ) + if (current.operationId !== operationId || current.pid !== process.pid) { throw new Error(`maintenance lock ownership changed unexpectedly: ${lockPath}`) } await rm(lockPath, { recursive: true }) @@ -891,6 +944,20 @@ const acquireMaintenance = async (dataDir: string, operationId: string): Promise } } +const withMaintenance = ( + dataDir: string, + operationId: CheckpointOperationId, + onAcquireError: (error: unknown) => E, + use: () => Effect.Effect, +): Effect.Effect => + Effect.acquireRelease( + Effect.tryPromise({ + try: () => acquireMaintenance(dataDir, operationId), + catch: onAcquireError, + }), + (release) => Effect.promise(release), + ).pipe(Effect.flatMap(use), Effect.scoped, Effect.uninterruptible) + export const reconcileCheckpointOperations = async ( dataDir: string, faults: DurabilityFaults = {}, @@ -921,7 +988,10 @@ export const reconcileCheckpointOperations = async ( if (!entry.isDirectory() || !entry.name.startsWith("checkpoint-")) { throw new Error(`unrecognized checkpoint operation debris: ${join(operationsRoot, entry.name)}`) } - const entryOperationId = validateId(entry.name.slice("checkpoint-".length), "operation directory") + const entryOperationId = validateOperationId( + entry.name.slice("checkpoint-".length), + "operation directory", + ) const entryDir = join(operationsRoot, entry.name) const intentPath = join(entryDir, "intent.json") await assertNoSymlink(operationsRoot, entryDir) @@ -1024,7 +1094,7 @@ export const reconcileCheckpointOperations = async ( await durableRename(entryDir, join(quarantine, "operation")) } -const hasPins = async (dataDir: string, checkpointId: string): Promise => { +const hasPins = async (dataDir: string, checkpointId: CheckpointId): Promise => { const path = join(checkpointPinsRoot(dataDir), checkpointId) try { await assertNoSymlink(checkpointRoot(dataDir), checkpointPinsRoot(dataDir)) @@ -1039,31 +1109,29 @@ const hasPins = async (dataDir: string, checkpointId: string): Promise export const retireCheckpointIfEligible = async ( dataDir: string, - checkpointId: string | null, + checkpointId: CheckpointId | null, state: CheckpointState, - retirementId: string = randomUUID(), + retirementId: CheckpointOperationId = newCheckpointOperationId(), faults: DurabilityFaults = {}, ): Promise => { if (!checkpointId || state.current === checkpointId || state.previous === checkpointId) return null - const validatedCheckpointId = validateId(checkpointId, "checkpoint") - if (await hasPins(dataDir, validatedCheckpointId)) return null - const validatedRetirementId = validateId(retirementId, "retirement") + if (await hasPins(dataDir, checkpointId)) return null const retirementRoot = checkpointRetiringRoot(dataDir) - const retirement = join(retirementRoot, `retirement-${validatedRetirementId}`) + const retirement = join(retirementRoot, `retirement-${retirementId}`) const retirementIntent = join(retirement, "intent.json") const retirementComplete = join(retirement, "complete.json") - const retiredSnapshot = join(retirement, validatedCheckpointId) + const retiredSnapshot = join(retirement, checkpointId) if (existsSync(checkpointRetiringRoot(dataDir))) { await assertNoSymlink(checkpointRoot(dataDir), retirementRoot) await assertRealDirectory(retirementRoot, "checkpoint retirement root") } if (!existsSync(retirement)) { - await resolveCheckpoint(dataDir, validatedCheckpointId, state) + await resolveCheckpoint(dataDir, checkpointId, state) await ensurePrivateDirectory(retirement) await durableJson(retirementIntent, { formatVersion: 1, - retirementId: validatedRetirementId, - checkpointId: validatedCheckpointId, + retirementId, + checkpointId, stateRevision: state.revision, }) await faults.afterRetirementIntent?.(retirement) @@ -1072,13 +1140,13 @@ export const retireCheckpointIfEligible = async ( await assertRealDirectory(retirement, "checkpoint retirement") await assertNoSymlink(retirement, retirementIntent) await assertRealFile(retirementIntent, "checkpoint retirement intent") - const parsed = JSON.parse(await readFile(retirementIntent, "utf8")) as unknown + const parsed = Schema.decodeUnknownSync(RetirementRecordSchema)( + JSON.parse(await readFile(retirementIntent, "utf8")), + ) if ( - !isRecord(parsed) || - parsed.formatVersion !== 1 || - validateId(requiredString(parsed, "retirementId"), "retirement") !== validatedRetirementId || - validateId(requiredString(parsed, "checkpointId"), "checkpoint") !== validatedCheckpointId || - validateId(requiredString(parsed, "stateRevision"), "state revision") !== state.revision + parsed.retirementId !== retirementId || + parsed.checkpointId !== checkpointId || + parsed.stateRevision !== state.revision ) { throw new Error(`checkpoint retirement identity mismatch: ${retirement}`) } @@ -1086,25 +1154,22 @@ export const retireCheckpointIfEligible = async ( if (existsSync(retirementComplete)) { await assertNoSymlink(retirement, retirementComplete) await assertRealFile(retirementComplete, "checkpoint retirement completion") - const complete = JSON.parse(await readFile(retirementComplete, "utf8")) as unknown + const complete = Schema.decodeUnknownSync(RetirementRecordSchema)( + JSON.parse(await readFile(retirementComplete, "utf8")), + ) if ( - !isRecord(complete) || - complete.formatVersion !== 1 || - validateId(requiredString(complete, "retirementId"), "retirement") !== validatedRetirementId || - validateId(requiredString(complete, "checkpointId"), "checkpoint") !== validatedCheckpointId || - validateId(requiredString(complete, "stateRevision"), "state revision") !== state.revision + complete.retirementId !== retirementId || + complete.checkpointId !== checkpointId || + complete.stateRevision !== state.revision ) { throw new Error(`checkpoint retirement completion identity mismatch: ${retirement}`) } - if ( - existsSync(checkpointSnapshotDir(dataDir, validatedCheckpointId)) || - existsSync(retiredSnapshot) - ) { + if (existsSync(checkpointSnapshotDir(dataDir, checkpointId)) || existsSync(retiredSnapshot)) { throw new Error("completed checkpoint retirement still has snapshot data") } return retirement } - const source = checkpointSnapshotDir(dataDir, validatedCheckpointId) + const source = checkpointSnapshotDir(dataDir, checkpointId) const sourceExists = existsSync(source) const retiredExists = existsSync(retiredSnapshot) if (sourceExists && retiredExists) { @@ -1122,8 +1187,8 @@ export const retireCheckpointIfEligible = async ( } await durableJson(retirementComplete, { formatVersion: 1, - retirementId: validatedRetirementId, - checkpointId: validatedCheckpointId, + retirementId, + checkpointId, stateRevision: state.revision, }) await faults.afterRetirementComplete?.(retirement) @@ -1147,13 +1212,13 @@ const removeCompletedRetirement = async ( await assertNoSymlink(retirement, complete) await assertRealFile(intent, "checkpoint retirement intent") await assertRealFile(complete, "checkpoint retirement completion") - const intentValue = JSON.parse(await readFile(intent, "utf8")) as unknown - const completeValue = JSON.parse(await readFile(complete, "utf8")) as unknown - if ( - !isRecord(intentValue) || - !isRecord(completeValue) || - JSON.stringify(intentValue) !== JSON.stringify(completeValue) - ) { + const intentValue = Schema.decodeUnknownSync(RetirementRecordSchema)( + JSON.parse(await readFile(intent, "utf8")), + ) + const completeValue = Schema.decodeUnknownSync(RetirementRecordSchema)( + JSON.parse(await readFile(complete, "utf8")), + ) + if (JSON.stringify(intentValue) !== JSON.stringify(completeValue)) { throw new Error(`checkpoint retirement records do not match: ${retirement}`) } const cleanup = `${retirement}.cleanup-${randomUUID()}` @@ -1164,190 +1229,180 @@ const removeCompletedRetirement = async ( await faults.afterRetirementCleanupRemoval?.(cleanup) } -export const createCheckpoint = ( - options: CheckpointOptions, -): Effect.Effect< - { - readonly checkpointId: string - readonly path: string - readonly state: CheckpointState - readonly manifest: CheckpointManifest - }, - CheckpointError -> => - Effect.tryPromise({ - try: async () => { - const operationId = randomUUID() - const checkpointId = randomUUID() - const release = await acquireMaintenance(options.dataDir, operationId) - try { - assertCheckpointRootSafe(options.dataDir) - await assertCheckpointInfrastructureSafe(options.dataDir) - assertNoLegacyLayout(options.dataDir) - await reconcileCheckpointOperations(options.dataDir, options.faults) - const oldState = await readStateFileOptional(options.dataDir) - if (!oldState && (await checkpointLikePaths(options.dataDir)).length > 0) { - throw new Error( - "checkpoint state is missing while checkpoint data exists; refusing to infer selection", - ) - } - if (oldState) { - await resolveCheckpoint(options.dataDir, oldState.current, oldState) - if (oldState.previous) { - await resolveCheckpoint(options.dataDir, oldState.previous, oldState) - } - } - for (const path of [ - checkpointRoot(options.dataDir), - checkpointSnapshotsRoot(options.dataDir), - checkpointOperationsRoot(options.dataDir), - checkpointPinsRoot(options.dataDir), - checkpointQuarantineRoot(options.dataDir), - checkpointRetiringRoot(options.dataDir), - ]) { - await ensurePrivateDirectory(path) - } - const startedAt = new Date().toISOString() - let operation: CheckpointOperation = { - formatVersion: 1, - operationId, - checkpointId, - baseRevision: oldState?.revision ?? null, - baseCurrent: oldState?.current ?? null, - basePrevious: oldState?.previous ?? null, - phase: "intent", - startedAt, - } - await writeOperation(options.dataDir, operation, options.faults) - const snapshot = checkpointSnapshotDir(options.dataDir, checkpointId) - await assertNoSymlink(checkpointSnapshotsRoot(options.dataDir), snapshot) - await mkdir(snapshot, { mode: 0o700 }) - try { - await postLocalQuery( - options.port, - `BACKUP DATABASE default TO Disk('default', '${snapshotBackupSqlPath(checkpointId)}')`, - ) - } catch (error) { - if (isMissingBackupConfigurationError(error)) { +export const createCheckpoint = Effect.fn("CheckpointService.create")(function* (options: CheckpointOptions) { + const operationId = newCheckpointOperationId() + const checkpointId = newCheckpointId() + const createError = (error: unknown): CheckpointCreateError => + new CheckpointCreateError({ + dataDir: resolve(options.dataDir), + operationId, + checkpointId, + message: errorMessage(error), + cause: errorCause(error), + }) + yield* Effect.annotateCurrentSpan({ + "maple.checkpoint.operation_id": operationId, + "maple.checkpoint.id": checkpointId, + }) + return yield* withMaintenance(options.dataDir, operationId, createError, () => + Effect.gen(function* () { + const prepared = yield* Effect.tryPromise({ + try: async () => { + assertCheckpointRootSafe(options.dataDir) + await assertCheckpointInfrastructureSafe(options.dataDir) + assertNoLegacyLayout(options.dataDir) + await reconcileCheckpointOperations(options.dataDir, options.faults) + const oldState = await readStateFileOptional(options.dataDir) + if (!oldState && (await checkpointLikePaths(options.dataDir)).length > 0) { throw new Error( - "checkpoints require the local server to be started with `--chdb-config-file` " + - "pointing at a ClickHouse backups config", - { cause: error }, + "checkpoint state is missing while checkpoint data exists; refusing to infer selection", ) } - throw error - } - await syncTree(snapshotBackupDir(options.dataDir, checkpointId)) - operation = { ...operation, phase: "backup-complete" } - await writeOperation(options.dataDir, operation, options.faults) - const provisionalManifest: CheckpointManifest = { - formatVersion: 1, - checkpointId, - operationId, - mapleVersion: MAPLE_VERSION, - chdbVersion: CHDB_VERSION, - schemaFingerprint: SCHEMA_FINGERPRINT, - createdAt: startedAt, - sourceDataDir: resolve(options.dataDir), - backupRelativePath: snapshotBackupRelativePath(checkpointId), - backupBytes: await dirSize(snapshotBackupDir(options.dataDir, checkpointId)), - validation: { - validatedAt: startedAt, - traces: 0, - logs: 0, - metricsSum: 0, - metricsGauge: 0, - metricsHistogram: 0, - metricsExponentialHistogram: 0, - materializedViews: 0, - }, - } - const provisional: ResolvedCheckpoint = { - checkpointId, - snapshotDir: snapshot, - backupDir: snapshotBackupDir(options.dataDir, checkpointId), - backupSqlPath: snapshotBackupSqlPath(checkpointId), - manifest: provisionalManifest, - } - const validation = await withRestoredCheckpoint( - provisional, - { cleanup: "always" }, - (restored) => restored.validation, - ) - const manifest: CheckpointManifest = { ...provisionalManifest, validation } - await durableJson( - snapshotManifestPath(options.dataDir, checkpointId), - manifest, - options.faults, - ) - await syncDirectory(snapshot) - operation = { ...operation, phase: "manifest-complete" } - await writeOperation(options.dataDir, operation, options.faults) - const state: CheckpointState = { - formatVersion: 1, - revision: operationId, - current: checkpointId, - previous: oldState?.current ?? null, - committedAt: new Date().toISOString(), - } - await durableJson(checkpointStatePath(options.dataDir), state, options.faults) - operation = { ...operation, phase: "pointer-complete" } - await writeOperation(options.dataDir, operation, options.faults) - const retirement = await retireCheckpointIfEligible( - options.dataDir, - oldState?.previous ?? null, - state, - operationId, - options.faults, - ) - operation = { ...operation, phase: "retention-complete" } - await writeOperation(options.dataDir, operation, options.faults) - await removeCompletedRetirement(retirement, options.faults) - await preserveCompletedOperation( - options.dataDir, - operationDir(options.dataDir, operationId), - operationId, - options.faults, - ) - return { checkpointId, path: snapshot, state, manifest } - } finally { - await release() - } - }, - catch: (error) => - new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), - }) + if (oldState) { + await resolveCheckpoint(options.dataDir, oldState.current, oldState) + if (oldState.previous) { + await resolveCheckpoint(options.dataDir, oldState.previous, oldState) + } + } + for (const path of [ + checkpointRoot(options.dataDir), + checkpointSnapshotsRoot(options.dataDir), + checkpointOperationsRoot(options.dataDir), + checkpointPinsRoot(options.dataDir), + checkpointQuarantineRoot(options.dataDir), + checkpointRetiringRoot(options.dataDir), + ]) { + await ensurePrivateDirectory(path) + } + const startedAt = new Date().toISOString() + let operation: CheckpointOperation = { + formatVersion: 1, + operationId, + checkpointId, + baseRevision: oldState?.revision ?? null, + baseCurrent: oldState?.current ?? null, + basePrevious: oldState?.previous ?? null, + phase: "intent", + startedAt, + } + await writeOperation(options.dataDir, operation, options.faults) + const snapshot = checkpointSnapshotDir(options.dataDir, checkpointId) + await assertNoSymlink(checkpointSnapshotsRoot(options.dataDir), snapshot) + await mkdir(snapshot, { mode: 0o700 }) + return { oldState, operation, snapshot, startedAt } + }, + catch: createError, + }) + yield* postLocalQuery( + options.port, + `BACKUP DATABASE default TO Disk('default', '${snapshotBackupSqlPath(checkpointId)}')`, + ).pipe( + Effect.mapError((error) => + createError( + isMissingBackupConfigurationError(error) + ? new Error( + "checkpoints require the local server to be started with `--chdb-config-file` " + + "pointing at a ClickHouse backups config", + { cause: error }, + ) + : error, + ), + ), + ) + return yield* Effect.tryPromise({ + try: async () => { + const { oldState, snapshot, startedAt } = prepared + let { operation } = prepared + await syncTree(snapshotBackupDir(options.dataDir, checkpointId)) + operation = { ...operation, phase: "backup-complete" } + await writeOperation(options.dataDir, operation, options.faults) + const provisionalManifest: CheckpointManifest = { + formatVersion: 1, + checkpointId, + operationId, + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + createdAt: startedAt, + sourceDataDir: resolve(options.dataDir), + backupRelativePath: snapshotBackupRelativePath(checkpointId), + backupBytes: await dirSize(snapshotBackupDir(options.dataDir, checkpointId)), + validation: { + validatedAt: startedAt, + traces: 0, + logs: 0, + metricsSum: 0, + metricsGauge: 0, + metricsHistogram: 0, + metricsExponentialHistogram: 0, + materializedViews: 0, + }, + } + const provisional: ResolvedCheckpoint = { + checkpointId, + snapshotDir: snapshot, + backupDir: snapshotBackupDir(options.dataDir, checkpointId), + backupSqlPath: snapshotBackupSqlPath(checkpointId), + manifest: provisionalManifest, + } + const validation = await withRestoredCheckpoint( + provisional, + { cleanup: "always" }, + (restored) => restored.validation, + ) + const manifest: CheckpointManifest = { ...provisionalManifest, validation } + await durableJson( + snapshotManifestPath(options.dataDir, checkpointId), + manifest, + options.faults, + ) + await syncDirectory(snapshot) + operation = { ...operation, phase: "manifest-complete" } + await writeOperation(options.dataDir, operation, options.faults) + const state: CheckpointState = { + formatVersion: 1, + revision: operationId, + current: checkpointId, + previous: oldState?.current ?? null, + committedAt: new Date().toISOString(), + } + await durableJson(checkpointStatePath(options.dataDir), state, options.faults) + operation = { ...operation, phase: "pointer-complete" } + await writeOperation(options.dataDir, operation, options.faults) + const retirement = await retireCheckpointIfEligible( + options.dataDir, + oldState?.previous ?? null, + state, + operationId, + options.faults, + ) + operation = { ...operation, phase: "retention-complete" } + await writeOperation(options.dataDir, operation, options.faults) + await removeCompletedRetirement(retirement, options.faults) + await preserveCompletedOperation( + options.dataDir, + operationDir(options.dataDir, operationId), + operationId, + options.faults, + ) + return { checkpointId, path: snapshot, state, manifest } + }, + catch: createError, + }) + }), + ) +}) const parseResetTransaction = (value: unknown, expectedDataDir: string): ResetTransaction => { - if (!isRecord(value) || value.formatVersion !== RESET_TRANSACTION_FORMAT_VERSION) { - throw new Error("unsupported or malformed reset transaction") - } - const phase = requiredString(value, "phase") - if (!["intent", "live-cleared", "markers-cleared"].includes(phase)) { - throw new Error("invalid reset transaction phase") - } - const dataDir = requiredString(value, "dataDir") + const transaction = Schema.decodeUnknownSync(ResetTransactionSchema)(value) + const { dataDir, targets } = transaction if (!isAbsolute(dataDir) || resolve(dataDir) !== resolve(expectedDataDir)) { throw new Error("reset transaction data directory does not match its configured owner") } - if (!Array.isArray(value.targets)) throw new Error("invalid reset transaction targets") - const targets = value.targets.map((target) => { - if (typeof target !== "string" || !RESETTABLE_CHDB_ENTRIES.has(target)) { - throw new Error(`unsafe reset transaction target: ${String(target)}`) - } - return target - }) if (new Set(targets).size !== targets.length || [...targets].sort().join("\0") !== targets.join("\0")) { throw new Error("reset transaction targets must be unique and sorted") } - return { - formatVersion: 1, - operationId: validateId(requiredString(value, "operationId"), "reset operation"), - dataDir: resolve(dataDir), - targets, - phase: phase as ResetTransaction["phase"], - createdAt: requiredIso(value, "createdAt"), - } + return transaction } const readResetTransaction = async (dataDir: string): Promise => { @@ -1364,6 +1419,17 @@ const readResetTransaction = async (dataDir: string): Promise => durableJson(resetTransactionPath(dataDir), transaction) +const assertResetTarget = async ( + path: string, + target: Schema.Schema.Type, +): Promise => { + if (target === "status") { + await assertRealFile(path, "reset target status") + } else { + await assertRealDirectory(path, `reset target ${target}`) + } +} + const reconcileResetTransactionUnlocked = async ( dataDir: string, faults: RestoreRecoveryFaults = {}, @@ -1380,8 +1446,8 @@ const reconcileResetTransactionUnlocked = async ( for (const target of transaction.targets) { const path = join(live, target) if (!existsSync(path)) continue - await assertRealDirectory(path, `reset target ${target}`) - await rm(path, { recursive: true }) + await assertResetTarget(path, target) + await rm(path, { recursive: target !== "status" }) await syncDirectory(live) await faults.afterResetEntryRemoval?.(target) } @@ -1419,12 +1485,12 @@ const reconcileResetTransactionUnlocked = async ( const beginResetTransactionUnlocked = async ( dataDir: string, - operationId: string, + operationId: CheckpointOperationId, faults: RestoreRecoveryFaults = {}, ): Promise => { await assertCheckpointInfrastructureSafe(dataDir) const live = resolve(dataDir) - const targets: string[] = [] + const targets: Array> = [] const unknown: string[] = [] if (existsSync(live)) { await assertRealDirectory(live, "live data directory") @@ -1435,10 +1501,11 @@ const beginResetTransactionUnlocked = async ( unknown.push(join(live, entry.name)) continue } - if (!entry.isDirectory() || entry.isSymbolicLink()) { - throw new Error(`reset target is not a real chDB directory: ${join(live, entry.name)}`) + const validType = entry.name === "status" ? entry.isFile() : entry.isDirectory() + if (!validType || entry.isSymbolicLink()) { + throw new Error(`reset target has an unsafe chDB entry type: ${join(live, entry.name)}`) } - targets.push(entry.name) + targets.push(Schema.decodeUnknownSync(ResetTarget)(entry.name)) } } if (unknown.length > 0) { @@ -1448,7 +1515,7 @@ const beginResetTransactionUnlocked = async ( } const transaction: ResetTransaction = { formatVersion: 1, - operationId: validateId(operationId, "reset operation"), + operationId, dataDir: live, targets: targets.sort(), phase: "intent", @@ -1460,22 +1527,7 @@ const beginResetTransactionUnlocked = async ( } const parseRestoreTransaction = (value: unknown): RestoreTransaction => { - if (!isRecord(value) || value.formatVersion !== RESTORE_TRANSACTION_FORMAT_VERSION) { - throw new Error("unsupported or malformed restore transaction") - } - const phase = requiredString(value, "phase") - if (!["intent", "restore-ready", "old-quarantined", "new-live", "markers-committed"].includes(phase)) { - throw new Error("invalid restore transaction phase") - } - return { - formatVersion: 1, - operationId: validateId(requiredString(value, "operationId"), "restore operation"), - checkpointId: validateId(requiredString(value, "checkpointId"), "checkpoint"), - quarantineId: validateId(requiredString(value, "quarantineId"), "quarantine"), - phase: phase as RestoreTransaction["phase"], - createdAt: requiredIso(value, "createdAt"), - validation: value.validation === null ? null : parseValidation(value.validation), - } + return Schema.decodeUnknownSync(RestoreTransactionSchema)(value) } const readRestoreTransaction = async (dataDir: string): Promise => { @@ -1491,7 +1543,7 @@ const readRestoreTransaction = async (dataDir: string): Promise durableJson(restoreTransactionPath(dataDir), transaction) -const restoreReadyPath = (dataDir: string, operationId: string): string => +const restoreReadyPath = (dataDir: string, operationId: CheckpointOperationId): string => join(restoreDataPath(dataDir, operationId), ".maple-restore-ready.json") const readyIdentityMatches = (dataDir: string, transaction: RestoreTransaction): boolean => { @@ -1504,10 +1556,10 @@ const readyIdentityMatches = (dataDir: string, transaction: RestoreTransaction): try { const info = lstatSync(path) if (info.isSymbolicLink() || !info.isFile()) return false - const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown + const parsed = Schema.decodeUnknownSync(RestoreReadySchema)( + JSON.parse(readFileSync(path, "utf8")), + ) if ( - isRecord(parsed) && - parsed.formatVersion === 1 && parsed.operationId === transaction.operationId && parsed.checkpointId === transaction.checkpointId ) { @@ -1665,69 +1717,83 @@ const reconcileRestoreTransactionUnlocked = async ( ) } -export const reconcileCheckpointRecovery = ( +export const reconcileCheckpointRecovery = Effect.fn("CheckpointService.reconcileRecovery")(function* ( dataDir: string, faults: RestoreRecoveryFaults = {}, -): Effect.Effect => - Effect.tryPromise({ - try: async () => { - const operationId = randomUUID() - const release = await acquireMaintenance(dataDir, operationId) - try { +) { + const operationId = newCheckpointOperationId() + const recoveryError = (error: unknown): CheckpointRecoveryError => + new CheckpointRecoveryError({ + dataDir: resolve(dataDir), + operationId, + message: errorMessage(error), + cause: errorCause(error), + }) + yield* Effect.annotateCurrentSpan("maple.checkpoint.operation_id", operationId) + return yield* withMaintenance(dataDir, operationId, recoveryError, () => + Effect.tryPromise({ + try: async () => { const resetReconciled = await reconcileResetTransactionUnlocked(dataDir, faults) if (!resetReconciled) await reconcileRestoreTransactionUnlocked(dataDir, faults) - } finally { - await release() - } - }, - catch: (error) => - new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), - }) + }, + catch: recoveryError, + }), + ) +}) /** * Explicitly remove the live chDB store while preserving the checkpoint * registry below `/backups`. The maintenance lock serializes this * destructive operation with checkpoint, restore, and archive work. */ -export const resetLiveStorePreservingCheckpoints = ( +export const resetLiveStorePreservingCheckpoints = Effect.fn("CheckpointService.reset")(function* ( dataDir: string, faults: RestoreRecoveryFaults = {}, -): Effect.Effect => - Effect.tryPromise({ - try: async () => { - const operationId = randomUUID() - const release = await acquireMaintenance(dataDir, operationId) - try { +) { + const operationId = newCheckpointOperationId() + const resetError = (error: unknown): CheckpointResetError => + new CheckpointResetError({ + dataDir: resolve(dataDir), + operationId, + message: errorMessage(error), + cause: errorCause(error), + }) + yield* Effect.annotateCurrentSpan("maple.checkpoint.operation_id", operationId) + return yield* withMaintenance(dataDir, operationId, resetError, () => + Effect.tryPromise({ + try: async () => { const resetReconciled = await reconcileResetTransactionUnlocked(dataDir, faults) if (!resetReconciled) { await reconcileRestoreTransactionUnlocked(dataDir) await beginResetTransactionUnlocked(dataDir, operationId, faults) } - } finally { - await release() - } - }, - catch: (error) => - new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), - }) + }, + catch: resetError, + }), + ) +}) -export const restoreCheckpoint = ( +export const restoreCheckpoint = Effect.fn("CheckpointService.restore")(function* ( dataDir: string, - selector: "current" | "previous" | string = "current", -): Effect.Effect< - { - readonly checkpointId: string - readonly quarantinePath: string - readonly validation: CheckpointValidation - }, - CheckpointError -> => - Effect.tryPromise({ - try: async () => { - const operationId = randomUUID() - const quarantineId = randomUUID() - const release = await acquireMaintenance(dataDir, operationId) - try { + selector: "current" | "previous" | CheckpointId = "current", +) { + const operationId = newCheckpointOperationId() + const quarantineId = newCheckpointQuarantineId() + const restoreError = (error: unknown): CheckpointRestoreError => + new CheckpointRestoreError({ + dataDir: resolve(dataDir), + operationId, + selector, + message: errorMessage(error), + cause: errorCause(error), + }) + yield* Effect.annotateCurrentSpan({ + "maple.checkpoint.operation_id": operationId, + "maple.checkpoint.selector": selector, + }) + return yield* withMaintenance(dataDir, operationId, restoreError, () => + Effect.tryPromise({ + try: async () => { await reconcileResetTransactionUnlocked(dataDir) await reconcileRestoreTransactionUnlocked(dataDir) const resolvedCheckpoint = await resolveCheckpoint(dataDir, selector) @@ -1758,6 +1824,9 @@ export const restoreCheckpoint = ( restoreData, restoringProcessValidation, ) + if (!validationCountsMatch(resolvedCheckpoint.manifest.validation, validation)) { + throw new Error("restored checkpoint counts do not match its signed manifest") + } await cp(checkpointRoot(dataDir), join(restoreData, "backups"), { recursive: true, force: false, @@ -1778,17 +1847,18 @@ export const restoreCheckpoint = ( quarantinePath, validation, } - } finally { - await release() - } - }, - catch: (error) => - new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), - }) + }, + catch: restoreError, + }), + ) +}) // Test helper: assert generated operation IDs remain unique and valid without // exposing an override in production command paths. -export const newCheckpointId = (): string => validateId(randomUUID(), "checkpoint") +export const newCheckpointId = (): CheckpointId => validateCheckpointId(randomUUID()) +export const newCheckpointOperationId = (): CheckpointOperationId => validateOperationId(randomUUID()) +export const newCheckpointQuarantineId = (): CheckpointQuarantineId => validateQuarantineId(randomUUID()) +export const parseCheckpointId = (value: unknown): CheckpointId => validateCheckpointId(value) // Refuse pre-existing symlink roots even before an operation allocates paths. export const assertCheckpointRootSafe = (dataDir: string): void => { diff --git a/apps/cli/src/server/serve.ts b/apps/cli/src/server/serve.ts index 5bef6986a..73545097f 100644 --- a/apps/cli/src/server/serve.ts +++ b/apps/cli/src/server/serve.ts @@ -247,7 +247,7 @@ const failIfError = (response: Response): Effect.Effect response.clone().text())).trim() const message = body.length > 0 ? body : `HTTP ${response.status}` yield* Effect.annotateCurrentSpan({ "error.type": `HTTP ${response.status}` }) - return yield* Effect.fail(new IngestRejected({ response, status: response.status, message })) + return yield* new IngestRejected({ response, status: response.status, message }) } return response }) @@ -358,7 +358,7 @@ export const startServer = ( fetch: makeFetch(db, options, runSpan), }), ), - (s) => Effect.sync(() => s.stop(true)), + (s) => Effect.promise(() => s.stop(true)), ) return { port: server.port ?? options.port } }) diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index f06e1d054..d06451b9c 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -1,5 +1,5 @@ import { describe, it } from "@effect/vitest" -import { Effect } from "effect" +import { Effect, Exit, Option } from "effect" import { deepStrictEqual, match, ok, rejects, strictEqual, throws } from "node:assert" import { existsSync, @@ -17,12 +17,17 @@ import { tmpdir } from "node:os" import { basename, dirname, join } from "node:path" import { assertCheckpointRootSafe, + type CheckpointId, + type CheckpointOperationId, + type CheckpointQuarantineId, checkpointRoot, checkpointSnapshotDir, checkpointStatePath, isMissingBackupConfigurationError, LocalQueryError, newCheckpointId, + newCheckpointOperationId, + newCheckpointQuarantineId, parseCheckpointManifest, parseCheckpointState, readCheckpointState, @@ -62,8 +67,8 @@ const withDataDir = async (run: (dataDir: string) => Promise | void): Prom } const manifest = ( - checkpointId: string, - operationId = newCheckpointId(), + checkpointId: CheckpointId, + operationId = newCheckpointOperationId(), sourceDataDir = "/tmp/maple-data", ): Record => ({ formatVersion: 1, @@ -94,7 +99,11 @@ describe("fresh-process checkpoint reopen probe", () => { }) }) -const writeSnapshot = (dataDir: string, checkpointId: string, operationId = newCheckpointId()): void => { +const writeSnapshot = ( + dataDir: string, + checkpointId: CheckpointId, + operationId = newCheckpointOperationId(), +): void => { const snapshot = checkpointSnapshotDir(dataDir, checkpointId) mkdirSync(join(snapshot, "backup"), { recursive: true }) writeFileSync(join(snapshot, "backup", "data.bin"), "backup") @@ -105,9 +114,9 @@ const writeSnapshot = (dataDir: string, checkpointId: string, operationId = newC const writeState = ( dataDir: string, - current: string, - previous: string | null = null, - revision = newCheckpointId(), + current: CheckpointId, + previous: CheckpointId | null = null, + revision = newCheckpointOperationId(), ): void => { mkdirSync(checkpointRoot(dataDir), { recursive: true }) writeFileSync( @@ -124,13 +133,13 @@ const writeState = ( const writeCheckpointOperation = ( dataDir: string, - operationId: string, - checkpointId: string, + operationId: CheckpointOperationId, + checkpointId: CheckpointId, phase: "intent" | "backup-complete" | "manifest-complete" | "pointer-complete" | "retention-complete", base: { - readonly revision: string | null - readonly current: string | null - readonly previous: string | null + readonly revision: CheckpointOperationId | null + readonly current: CheckpointId | null + readonly previous: CheckpointId | null } = { revision: null, current: null, previous: null }, ): string => { const dir = join(checkpointRoot(dataDir), "operations", `checkpoint-${operationId}`) @@ -164,9 +173,9 @@ const restoreValidation = { const writeRestoreTransaction = ( dataDir: string, - operationId: string, - checkpointId: string, - quarantineId: string, + operationId: CheckpointOperationId, + checkpointId: CheckpointId, + quarantineId: CheckpointQuarantineId, phase: "intent" | "restore-ready" | "old-quarantined" | "new-live" | "markers-committed", ): void => { writeFileSync( @@ -183,7 +192,11 @@ const writeRestoreTransaction = ( ) } -const writeRestoreReady = (dataDir: string, operationId: string, checkpointId: string): void => { +const writeRestoreReady = ( + dataDir: string, + operationId: CheckpointOperationId, + checkpointId: CheckpointId, +): void => { const restoreData = restoreDataPath(dataDir, operationId) mkdirSync(restoreData, { recursive: true }) writeFileSync( @@ -243,14 +256,14 @@ describe("checkpoint IDs and strict parsers", () => { ...manifest(id), validation: { ...(manifest(id).validation as object), logs: -1 }, }), - /invalid logs/, + /validation.*logs|greater than or equal to 0/s, ) }) it("accepts versioned current/previous state and rejects malformed selection", () => { const current = newCheckpointId() const previous = newCheckpointId() - const revision = newCheckpointId() + const revision = newCheckpointOperationId() deepStrictEqual( parseCheckpointState({ formatVersion: 1, @@ -376,7 +389,7 @@ describe("checkpoint state resolution", () => { describe("checkpoint reconciliation and retention", () => { it("quarantines only an exactly owned incomplete operation and preserves its bytes", async () => { await withDataDir(async (dataDir) => { - const operationId = newCheckpointId() + const operationId = newCheckpointOperationId() const checkpointId = newCheckpointId() const snapshot = checkpointSnapshotDir(dataDir, checkpointId) mkdirSync(join(snapshot, "backup"), { recursive: true }) @@ -424,8 +437,8 @@ describe("checkpoint reconciliation and retention", () => { it("rejects mismatched operation directory and intent identities without mutation", async () => { await withDataDir(async (dataDir) => { - const directoryId = newCheckpointId() - const intentId = newCheckpointId() + const directoryId = newCheckpointOperationId() + const intentId = newCheckpointOperationId() const checkpointId = newCheckpointId() const directory = writeCheckpointOperation(dataDir, directoryId, checkpointId, "backup-complete") const intent = JSON.parse(readFileSync(join(directory, "intent.json"), "utf8")) @@ -439,7 +452,7 @@ describe("checkpoint reconciliation and retention", () => { it("publishes a completed first checkpoint after an interrupted pointer update", async () => { await withDataDir(async (dataDir) => { - const operationId = newCheckpointId() + const operationId = newCheckpointOperationId() const checkpointId = newCheckpointId() writeSnapshot(dataDir, checkpointId, operationId) writeCheckpointOperation(dataDir, operationId, checkpointId, "manifest-complete") @@ -458,12 +471,12 @@ describe("checkpoint reconciliation and retention", () => { await withDataDir(async (dataDir) => { const oldCurrent = newCheckpointId() const oldPrevious = newCheckpointId() - const baseRevision = newCheckpointId() + const baseRevision = newCheckpointOperationId() writeSnapshot(dataDir, oldCurrent) writeSnapshot(dataDir, oldPrevious) writeState(dataDir, oldCurrent, oldPrevious, baseRevision) - const operationId = newCheckpointId() + const operationId = newCheckpointOperationId() const checkpointId = newCheckpointId() writeSnapshot(dataDir, checkpointId, operationId) writeCheckpointOperation(dataDir, operationId, checkpointId, "manifest-complete", { @@ -493,7 +506,7 @@ describe("checkpoint reconciliation and retention", () => { const current = newCheckpointId() const previous = newCheckpointId() const old = newCheckpointId() - const retirementId = newCheckpointId() + const retirementId = newCheckpointOperationId() writeSnapshot(dataDir, current) writeSnapshot(dataDir, previous) writeSnapshot(dataDir, old) @@ -527,8 +540,8 @@ describe("checkpoint reconciliation and retention", () => { await withDataDir(async (dataDir) => { const oldCurrent = newCheckpointId() const oldPrevious = newCheckpointId() - const baseRevision = newCheckpointId() - const operationId = newCheckpointId() + const baseRevision = newCheckpointOperationId() + const operationId = newCheckpointOperationId() const checkpointId = newCheckpointId() writeSnapshot(dataDir, oldCurrent) writeSnapshot(dataDir, oldPrevious) @@ -568,8 +581,8 @@ describe("checkpoint reconciliation and retention", () => { await withDataDir(async (dataDir) => { const oldCurrent = newCheckpointId() const oldPrevious = newCheckpointId() - const baseRevision = newCheckpointId() - const operationId = newCheckpointId() + const baseRevision = newCheckpointOperationId() + const operationId = newCheckpointOperationId() const checkpointId = newCheckpointId() writeSnapshot(dataDir, oldCurrent) writeSnapshot(dataDir, checkpointId, operationId) @@ -657,6 +670,7 @@ describe("live-store reset safety", () => { mkdirSync(join(dataDir, "tmp"), { recursive: true }) writeFileSync(join(dataDir, "store", "part.bin"), "live") writeFileSync(join(dataDir, "metadata", "table.sql"), "live") + writeFileSync(join(dataDir, "status"), "live") writeFileSync(join(dataDir, "tmp", "scratch.bin"), "live") writeFileSync(storeMarkerPath(dataDir), "{}") writeFileSync(storeOpenMarkerPath(dataDir), "999\n") @@ -667,6 +681,7 @@ describe("live-store reset safety", () => { ok(existsSync(checkpointSnapshotDir(dataDir, checkpointId))) ok(!existsSync(join(dataDir, "store"))) ok(!existsSync(join(dataDir, "metadata"))) + ok(!existsSync(join(dataDir, "status"))) ok(!existsSync(join(dataDir, "tmp"))) ok(!existsSync(storeMarkerPath(dataDir))) ok(!existsSync(storeOpenMarkerPath(dataDir))) @@ -724,6 +739,7 @@ describe("live-store reset safety", () => { mkdirSync(join(dataDir, entry), { recursive: true }) writeFileSync(join(dataDir, entry, "live.bin"), "live") } + writeFileSync(join(dataDir, "status"), "live") writeFileSync(storeMarkerPath(dataDir), "{}") writeFileSync(storeOpenMarkerPath(dataDir), "999\n") let injected = false @@ -741,7 +757,7 @@ describe("live-store reset safety", () => { ) await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) - for (const entry of ["data", "metadata", "store", "tmp"]) { + for (const entry of ["data", "metadata", "status", "store", "tmp"]) { ok(!existsSync(join(dataDir, entry)), `${boundary}: ${entry}`) } strictEqual((await readCheckpointState(dataDir)).current, checkpointId) @@ -761,7 +777,7 @@ describe("live-store reset safety", () => { resetTransactionPath(dataDir), `${JSON.stringify({ formatVersion: 1, - operationId: newCheckpointId(), + operationId: newCheckpointOperationId(), dataDir, targets: ["../outside-reset"], phase: "intent", @@ -769,7 +785,12 @@ describe("live-store reset safety", () => { })}\n`, ) - await rejects(Effect.runPromise(reconcileCheckpointRecovery(dataDir)), /unsafe reset/) + const exit = await Effect.runPromise(Effect.exit(reconcileCheckpointRecovery(dataDir))) + ok(Exit.isFailure(exit)) + const failure = Option.getOrUndefined(Exit.findErrorOption(exit)) + ok(failure) + strictEqual(failure._tag, "@maple/cli/CheckpointRecoveryError") + match(failure.message, /targets|outside-reset/) strictEqual(readFileSync(join(outside, "preserve"), "utf8"), "outside") ok(existsSync(resetTransactionPath(dataDir))) }) @@ -786,9 +807,9 @@ describe("live restore transaction reconciliation", () => { it("preserves an interrupted pre-ready restore and leaves the old live store selected", async () => { await withDataDir(async (dataDir) => { - const operationId = newCheckpointId() + const operationId = newCheckpointOperationId() const checkpointId = newCheckpointId() - const quarantineId = newCheckpointId() + const quarantineId = newCheckpointQuarantineId() writeFileSync(join(dataDir, "old-live"), "old") writeRestoreTransaction(dataDir, operationId, checkpointId, quarantineId, "intent") mkdirSync(restoreDataPath(dataDir, operationId), { recursive: true }) @@ -815,9 +836,9 @@ describe("live restore transaction reconciliation", () => { it("resumes from restore-ready through quarantine, swap, durable markers, and completion", async () => { await withDataDir(async (dataDir) => { - const operationId = newCheckpointId() + const operationId = newCheckpointOperationId() const checkpointId = newCheckpointId() - const quarantineId = newCheckpointId() + const quarantineId = newCheckpointQuarantineId() const quarantine = restoreQuarantinePath(dataDir, operationId, quarantineId) writeFileSync(join(dataDir, "old-live"), "old") writeFileSync(storeOpenMarkerPath(dataDir), "999\n") @@ -839,9 +860,9 @@ describe("live restore transaction reconciliation", () => { it("infers the recorded rename boundary from exact topology and completes idempotently", async () => { await withDataDir(async (dataDir) => { - const operationId = newCheckpointId() + const operationId = newCheckpointOperationId() const checkpointId = newCheckpointId() - const quarantineId = newCheckpointId() + const quarantineId = newCheckpointQuarantineId() const quarantine = restoreQuarantinePath(dataDir, operationId, quarantineId) writeFileSync(join(dataDir, "old-live"), "old") writeRestoreReady(dataDir, operationId, checkpointId) @@ -872,9 +893,9 @@ describe("live restore transaction reconciliation", () => { ] for (const boundary of boundaries) { await withDataDir(async (dataDir) => { - const operationId = newCheckpointId() + const operationId = newCheckpointOperationId() const checkpointId = newCheckpointId() - const quarantineId = newCheckpointId() + const quarantineId = newCheckpointQuarantineId() const quarantine = restoreQuarantinePath(dataDir, operationId, quarantineId) writeFileSync(join(dataDir, "old-live"), "old") writeFileSync(storeOpenMarkerPath(dataDir), "999\n") @@ -907,7 +928,7 @@ describe("live restore transaction reconciliation", () => { ok(existsSync(restoreTransactionPath(dataDir))) rmSync(restoreTransactionPath(dataDir)) - const debris = `${dataDir}.restore-${newCheckpointId()}` + const debris = `${dataDir}.restore-${newCheckpointOperationId()}` mkdirSync(debris) await rejects( Effect.runPromise(reconcileCheckpointRecovery(dataDir)), @@ -926,9 +947,9 @@ describe("live restore transaction reconciliation", () => { strictEqual(readFileSync(outside, "utf8"), "{}") }) await withDataDir(async (dataDir) => { - const operationId = newCheckpointId() + const operationId = newCheckpointOperationId() const checkpointId = newCheckpointId() - const quarantineId = newCheckpointId() + const quarantineId = newCheckpointQuarantineId() const outside = join(dirname(dataDir), "outside-restore") mkdirSync(outside) writeFileSync(join(outside, "sentinel"), "preserve") @@ -943,17 +964,20 @@ describe("live restore transaction reconciliation", () => { describe("backup configuration classification", () => { it("classifies only backup-specific errors", () => { + const queryError = (detail: string) => + new LocalQueryError({ + status: 500, + detail, + message: `local query failed (500): ${detail}`, + cause: detail, + }) ok( isMissingBackupConfigurationError( - new LocalQueryError(500, "INVALID_CONFIG_PARAMETER: backups.allowed_disk is not set"), - ), - ) - ok( - isMissingBackupConfigurationError( - new LocalQueryError(500, "Disk default is not allowed for backups"), + queryError("INVALID_CONFIG_PARAMETER: backups.allowed_disk is not set"), ), ) - ok(!isMissingBackupConfigurationError(new LocalQueryError(500, "INVALID_CONFIG_PARAMETER"))) + ok(isMissingBackupConfigurationError(queryError("Disk default is not allowed for backups"))) + ok(!isMissingBackupConfigurationError(queryError("INVALID_CONFIG_PARAMETER"))) ok(!isMissingBackupConfigurationError(new Error("UNKNOWN_TABLE"))) ok(!isMissingBackupConfigurationError(new Error("connection refused"))) }) diff --git a/apps/cli/test/native-checkpoint-smoke.sh b/apps/cli/test/native-checkpoint-smoke.sh index d1f7e5af4..2d5f363da 100755 --- a/apps/cli/test/native-checkpoint-smoke.sh +++ b/apps/cli/test/native-checkpoint-smoke.sh @@ -153,6 +153,23 @@ C2="$(checkpoint)" jq -e --arg c "$C2" --arg p "$C1" \ '.formatVersion == 1 and .current == $c and .previous == $p' \ "$DATA/backups/state.json" >/dev/null + +# Restore must reject before allocating a transaction or mutating the live +# process when the server PID is active. +state_before_running_restore="$(cat "$DATA/backups/state.json")" +set +e +"$MAPLE" restore --data-dir "$DATA" --checkpoint-id "$C1" --yes \ + >"$ROOT/restore-while-running.out" 2>&1 +running_restore_status=$? +set -e +[[ "$running_restore_status" -ne 0 ]] || fail "restore while server was running unexpectedly succeeded" +grep -q 'maple is running' "$ROOT/restore-while-running.out" || + fail "restore while running was not actionable: $(cat "$ROOT/restore-while-running.out")" +[[ "$(cat "$DATA/backups/state.json")" == "$state_before_running_restore" ]] || + fail "restore while running changed checkpoint state" +[[ ! -e "${DATA}.restore-transaction.json" ]] || + fail "restore while running allocated a restore transaction" +assert_counts 2 stop_server "$MAPLE" restore --data-dir "$DATA" --checkpoint-id "$C1" --yes >/dev/null @@ -179,6 +196,19 @@ set -e grep -q 'was not cleanly closed' "$ROOT/default-dirty.out" || fail "default dirty-store failure was not actionable: $(cat "$ROOT/default-dirty.out")" [[ -f "$DATA/backups/state.json" ]] || fail "default dirty-store failure removed checkpoints" + +# Explicit dirty-store wipe clears live telemetry while preserving the exact +# checkpoint registry. Restore C2 afterwards so the remaining recovery paths +# continue from a known two-row store. +start_server wipe +assert_counts 0 +stop_server +jq -e --arg c "$C2" --arg p "$C1" \ + '.current == $c and .previous == $p' "$DATA/backups/state.json" >/dev/null +"$MAPLE" restore --data-dir "$DATA" --checkpoint-id "$C2" --yes >/dev/null +start_server fail +assert_counts 2 +kill_server start_server restore-checkpoint assert_counts 2