diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 015973e97..7a742890e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,11 @@ jobs: # shellcheck is preinstalled on the ubuntu-latest runner image. - name: shellcheck - run: shellcheck scripts/install.sh scripts/uninstall.sh scripts/build-local-binary.sh + run: >- + shellcheck scripts/install.sh scripts/uninstall.sh scripts/build-local-binary.sh + apps/cli/test/native-checkpoint-smoke.sh + apps/cli/test/native-archive-smoke.sh + apps/cli/test/native-archive-crash-recovery-probe.sh # Syntax-check the POSIX installers against the actual /bin/sh they claim. - name: sh -n @@ -54,6 +58,86 @@ jobs: sh -n scripts/install.sh sh -n scripts/uninstall.sh + local-checkpoint-native: + name: Local checkpoint native + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2.4.4 + + - run: bun install --frozen-lockfile + + - name: Build native local bundle + run: scripts/build-local-binary.sh + + - name: Run repeated checkpoint restore and fresh-process reopen smoke + env: + KEEP_ROOT: "1" + TMPDIR: ${{ runner.temp }}/maple-native-checkpoint + run: | + mkdir -p "$TMPDIR" + apps/cli/test/native-checkpoint-smoke.sh "$GITHUB_WORKSPACE/dist" + + - name: Upload checkpoint failure evidence + if: ${{ failure() }} + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 + with: + name: native-checkpoint-failure-${{ github.run_id }} + path: ${{ runner.temp }}/maple-native-checkpoint + if-no-files-found: ignore + + local-archive-native: + name: Local archive native + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2.4.4 + + - run: bun install --frozen-lockfile + + - name: Install pinned DuckDB CLI + run: | + curl -fsSL https://github.com/duckdb/duckdb/releases/download/v1.5.3/duckdb_cli-linux-amd64.gz \ + -o "$RUNNER_TEMP/duckdb.gz" + echo "f05f3b448a9a1bc6e7ac27ff14dfe67bf5761b153c2002723365a456618ef35b $RUNNER_TEMP/duckdb.gz" \ + | sha256sum --check --strict + gunzip "$RUNNER_TEMP/duckdb.gz" + chmod +x "$RUNNER_TEMP/duckdb" + echo "$RUNNER_TEMP" >> "$GITHUB_PATH" + + - name: Build native local bundle + run: scripts/build-local-binary.sh + + - name: Run archive lifecycle smoke + env: + KEEP_ROOT: "1" + TMPDIR: ${{ runner.temp }}/maple-native-archive + run: | + mkdir -p "$TMPDIR" + apps/cli/test/native-archive-smoke.sh "$GITHUB_WORKSPACE/dist" + + - name: Run archive crash-recovery boundaries + env: + KEEP_ROOT: "1" + TMPDIR: ${{ runner.temp }}/maple-native-archive-recovery + run: | + mkdir -p "$TMPDIR" + apps/cli/test/native-archive-crash-recovery-probe.sh "$GITHUB_WORKSPACE/dist" + + - name: Upload archive failure evidence + if: ${{ failure() }} + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 + with: + name: native-archive-failure-${{ github.run_id }} + path: | + ${{ runner.temp }}/maple-native-archive + ${{ runner.temp }}/maple-native-archive-recovery + if-no-files-found: ignore + service-map-perf: name: Service Map Perf runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index a15ba75ba..191bcb60b 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,5 @@ playwright/.cache/ # local-only Drizzle Studio config pointing at miniflare D1 drizzle.config.local.ts +.18bd7*.bun-build +.[0-9a-f]*.bun-build diff --git a/apps/cli/package.json b/apps/cli/package.json index 8ca0c0e8a..7dc289b0c 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -7,7 +7,7 @@ "type": "module", "scripts": { "start": "bun run src/bin.ts", - "test": "vitest run", + "test": "bun test test/*.test.ts", "typecheck": "tsc --noEmit" }, "dependencies": { diff --git a/apps/cli/src/bin.ts b/apps/cli/src/bin.ts index 5945cbaf9..6a6270e21 100644 --- a/apps/cli/src/bin.ts +++ b/apps/cli/src/bin.ts @@ -9,6 +9,8 @@ import { Mode } from "./core/mode" import { TelemetryLayer } from "./core/telemetry" import { maybeNotifyUpdate } from "./core/update" import { WarehouseExecutorFromMode } from "./core/warehouse" +import { archiveErrorMessage } from "./server/archives/errors" +import { CHECKPOINT_REOPEN_PROBE_ENV, validateCheckpointDataDir } from "./server/checkpoints" import { MAPLE_VERSION } from "./version" // WarehouseExecutorFromMode needs Mode (which needs MapleConfig). provideMerge @@ -31,10 +33,33 @@ const MainLayer = WarehouseExecutorFromMode.pipe( // exporter flushes when its layer scope closes, and only the outermost provide's // scope is the runtime's main scope that `BunRuntime.runMain` closes on exit. // Merging it into MainLayer leaves spans unflushed for short-lived commands. -maybeNotifyUpdate.pipe( - Effect.flatMap(() => Command.run(cli, { version: MAPLE_VERSION })), - Effect.withSpan("maple", { attributes: { "cli.argv": process.argv.slice(2).join(" ") } }), - Effect.provide(MainLayer), - Effect.provide(TelemetryLayer), - BunRuntime.runMain, -) +const checkpointProbeDataDir = process.env[CHECKPOINT_REOPEN_PROBE_ENV] + +if (checkpointProbeDataDir !== undefined) { + // Private re-exec path used by checkpoint restore. It intentionally bypasses + // CLI dispatch, update checks, telemetry, and schema bootstrap: success means + // this new process loaded the persisted restored representation and queried + // its core tables before closing chDB cleanly. + try { + process.stdout.write(`${JSON.stringify(validateCheckpointDataDir(checkpointProbeDataDir))}\n`) + } catch (error) { + process.stderr.write( + `checkpoint reopen probe failed: ${error instanceof Error ? error.message : String(error)}\n`, + ) + process.exitCode = 1 + } +} else { + maybeNotifyUpdate.pipe( + Effect.flatMap(() => Command.run(cli, { version: MAPLE_VERSION })), + Effect.withSpan("maple", { attributes: { "cli.argv": process.argv.slice(2).join(" ") } }), + Effect.catchTag("@maple/cli/ArchiveError", (error) => + Effect.sync(() => { + process.stderr.write(archiveErrorMessage(error)) + process.exitCode = 1 + }), + ), + Effect.provide(MainLayer), + Effect.provide(TelemetryLayer), + BunRuntime.runMain, + ) +} diff --git a/apps/cli/src/cli.ts b/apps/cli/src/cli.ts index 404ca8ca0..21ce946d5 100644 --- a/apps/cli/src/cli.ts +++ b/apps/cli/src/cli.ts @@ -9,7 +9,8 @@ import { metrics, query } from "./commands/data" import { timeseries, breakdown, compare } from "./commands/analytics" import { login, logout, whoami } from "./commands/auth" import { use } from "./commands/config" -import { start, stop, reset } from "./commands/server" +import { start, stop, reset, checkpoint, restore } from "./commands/server" +import { archive } from "./commands/archive" import { update } from "./commands/update" // One CLI, two backends. Every query command bottoms out at the shared @@ -46,6 +47,10 @@ export const cli = Command.make("maple").pipe( start, stop, reset, + checkpoint, + restore, + // Parquet archives (local mode) + archive, // Self-update update, // Services diff --git a/apps/cli/src/commands/archive.ts b/apps/cli/src/commands/archive.ts new file mode 100644 index 000000000..40a9d9388 --- /dev/null +++ b/apps/cli/src/commands/archive.ts @@ -0,0 +1,1736 @@ +import { Effect, Option, Schema } from "effect" +import * as Command from "effect/unstable/cli/Command" +import * as Flag from "effect/unstable/cli/Flag" +import * as Argument from "effect/unstable/cli/Argument" +import { spawn } from "node:child_process" +import { randomUUID } from "node:crypto" +import { homedir } from "node:os" +import { join, resolve } from "node:path" +import { createArchiveGeneration, runArchiveReconciliation } from "../server/archives/generation" +import { + listActiveGenerations, + activeParquetPaths, + rebuildCatalog, + verifyActiveGenerations, +} from "../server/archives/listing" +import { runArchiveGc } from "../server/archives/gc" +import { + resolveArchiveTuning, + loadTuningConfig, + TUNING_CONFIG_FORMAT_VERSION, + type ArchiveTuningOverrides, + type TuningConfigIdentity, + type LoadedTuningConfig, +} from "../server/archives/config" +import { ARCHIVE_SIGNALS, isArchiveSignalName, type ArchiveSignalName } from "../server/archives/signals" +import { validateRangeDate } from "../server/archives/paths" +import { + type CalibrationBudget, + type CalibrationCandidate, + type CandidateMetrics, + type CandidateResult, + CANDIDATE_MATRIX, + captureEnvironment, + meetsCeilings, + recommendationToTuning, + selectCandidates, + writeCalibrationConfig, + type CalibrationRecommendation, + HELD_OUT_TOLERANCES, + isSameCalibrationCandidate, + heldOutSampleRows, + compareHeldOutPerSignal, + validateCalibrationBudget, +} from "../server/archives/calibrate" +import { + preflightCalibrationFreeSpace, + reconcileCalibration, + writeCalibrationRecord, + directoryTreeBytes, + archiveVolumeIdentity, + derivedSampleDir, + derivedScratchSubdir, + calibrationPinPurpose, + assertCalibrationSession, + cleanupCalibrationSample, +} from "../server/archives/calibration-recovery" +import { + acquireCheckpointPin, + resolveCheckpoint, + withMaintenanceLock, + withRestoredCheckpoint, +} from "../server/checkpoints" +import { + captureSourceSchema, + exportShardPlans, + planCalibrationShards, + measureShardBytes, + type ExportSettings, +} from "../server/archives/export" +import { archiveSignal } from "../server/archives/signals" +import { ensurePrivateDirectory } from "../server/archives/paths" +import { CHDB_VERSION, MAPLE_VERSION } from "../version" +import { SCHEMA_FINGERPRINT } from "../server/serve" +import { amber, bold, dim, green, red } from "../lib/style" +import { + collectChildOutputAfterClose, + createTimeReport, + parsePeakRss, + timeArgv, +} from "../server/archives/timed-process" +import { ArchiveError } from "../server/archives/errors" + +const defaultDataDir = (): string => join(homedir(), ".maple", "data") +const defaultArchiveDir = (): string => join(homedir(), ".maple", "archive") +const defaultScratchRoot = (): string => join(homedir(), ".maple", "scratch") + +const prettyPath = (p: string): string => { + const home = homedir() + return p.startsWith(home) ? `~${p.slice(home.length)}` : p +} + +const dataDirFlag = Flag.optional( + Flag.string("data-dir").pipe( + Flag.withDescription("Embedded ClickHouse data directory (default: ~/.maple/data)"), + ), +) + +const archiveDirFlag = Flag.optional( + Flag.string("archive-dir").pipe( + Flag.withDescription("Archive root directory for Parquet generations (default: ~/.maple/archive)"), + ), +) + +const scratchRootFlag = Flag.optional( + Flag.string("scratch-root").pipe( + Flag.withDescription("Root for restored-checkpoint scratch instances (default: ~/.maple/scratch)"), + ), +) + +const checkpointIdFlag = Flag.optional( + Flag.string("checkpoint-id").pipe( + Flag.withDescription("Archive from one immutable checkpoint ID instead of the selected current"), + ), +) + +const dryRunFlag = Flag.boolean("dry-run").pipe( + Flag.withDescription("Report the exact planned actions without modifying any archive state"), + Flag.withDefault(false), +) + +const keepFlag = Flag.integer("keep").pipe( + Flag.withDescription( + "Newest superseded generations to retain per signal/range (default 1; 0 reclaims all superseded)", + ), + Flag.withDefault(1), +) + +const memoryBudgetFlag = Flag.integer("memory-budget").pipe( + Flag.withDescription("Maximum peak RSS in bytes allowed for any calibration candidate"), + Flag.withDefault(512 * 1024 * 1024), +) + +const timeBudgetFlag = Flag.integer("time-budget").pipe( + Flag.withDescription("Maximum wall-clock milliseconds for the full calibration matrix"), + Flag.withDefault(60_000), +) + +const sampleRowsFlag = Flag.integer("sample-rows").pipe( + Flag.withDescription("Rows to sample per calibration candidate"), + Flag.withDefault(10_000), +) + +const writeConfigFlag = Flag.optional( + Flag.string("write-config").pipe( + Flag.withDescription("Write the generated tuning configuration to this path"), + ), +) + +const configFlag = Flag.optional( + Flag.string("config").pipe( + Flag.withDescription( + "Load tuning overrides from a versioned calibration config document (see: archive calibrate --write-config)", + ), + ), +) + +const maxCandidateWallMsFlag = Flag.integer("max-candidate-wall-ms").pipe( + Flag.withDescription("Maximum wall-clock milliseconds for a single calibration candidate run"), + Flag.withDefault(30_000), +) + +const minThroughputFlag = Flag.integer("min-throughput").pipe( + Flag.withDescription("Minimum logical write throughput in bytes/sec required of a candidate"), + Flag.withDefault(0), +) + +const maxTempDiskFlag = Flag.integer("max-temp-disk").pipe( + Flag.withDescription("Maximum peak temporary disk (restored scratch + sample output) in bytes"), + Flag.withDefault(2 * 1024 * 1024 * 1024), +) + +const freeSpaceReserveFlag = Flag.integer("free-space-reserve").pipe( + Flag.withDescription("Minimum free-space reserve on the archive volume in bytes before calibrating"), + Flag.withDefault(512 * 1024 * 1024), +) + +const safetyMarginFlag = Flag.integer("safety-margin-milli").pipe( + Flag.withDescription( + "Safety margin in thousandths applied inside each ceiling (e.g. 1100 = 1.1x, reserving 10% headroom)", + ), + Flag.withDefault(1100), +) + +const writerThreadsFlag = Flag.integer("writer-threads").pipe( + Flag.withDescription("Parquet writer thread count for a calibration run"), + Flag.withDefault(1), +) + +const rowGroupRowsFlag = Flag.integer("row-group-rows").pipe( + Flag.withDescription("Parquet row-group row count for a calibration run"), + Flag.withDefault(10_000), +) + +const maxShardRowsFlag = Flag.integer("max-shard-rows").pipe( + Flag.withDescription("Maximum rows per shard for a calibration run"), + Flag.withDefault(500_000), +) + +const maxShardBytesFlag = Flag.integer("max-shard-bytes").pipe( + Flag.withDescription("Maximum estimated bytes per shard for a calibration run"), + Flag.withDefault(256 * 1024 * 1024), +) + +const rangeDateArgument = Argument.string("range-date").pipe( + Argument.withDescription("UTC day to seal as YYYY-MM-DD"), +) + +const signalArgument = Argument.string("signal").pipe( + Argument.withDescription(`One of: ${ARCHIVE_SIGNALS.map((s) => s.name).join(", ")}`), +) + +const outputFlag = Flag.choice("output", ["summary", "paths", "json"]).pipe( + Flag.withDescription( + "Output format: summary (default), paths (machine-readable active Parquet paths), or json", + ), + Flag.withDefault("summary" as const), +) + +/** Build tuning overrides from parsed flags. */ +const tuningOverrides = (dataDir: string, archiveDir: string, scratchRoot: string): ArchiveTuningOverrides => + ({ archiveDir, scratchRoot, dataDir }) as ArchiveTuningOverrides + +/** Resolve the archive and scratch roots from flags, falling back to defaults. */ +const resolveRoots = ( + dataDirOpt: Option.Option, + archiveDirOpt: Option.Option, + scratchRootOpt: Option.Option, +): { dataDir: string; archiveDir: string; scratchRoot: string } => ({ + dataDir: resolve(Option.getOrUndefined(dataDirOpt) ?? defaultDataDir()), + archiveDir: resolve(Option.getOrUndefined(archiveDirOpt) ?? defaultArchiveDir()), + scratchRoot: resolve(Option.getOrUndefined(scratchRootOpt) ?? defaultScratchRoot()), +}) + +export const archiveCreate = Command.make("create", { + dataDir: dataDirFlag, + archiveDir: archiveDirFlag, + scratchRoot: scratchRootFlag, + checkpointId: checkpointIdFlag, + config: configFlag, + rangeDate: rangeDateArgument, + signal: signalArgument, +}).pipe( + Command.withDescription( + "Seal one UTC day of one signal into a validated Parquet archive generation from a checkpoint", + ), + Command.withHandler( + Effect.fnUntraced(function* (a) { + if (!isArchiveSignalName(a.signal)) { + return yield* new ArchiveError({ + message: `unknown signal '${a.signal}'; expected one of ${ARCHIVE_SIGNALS.map((s) => s.name).join(", ")}`, + }) + } + let rangeDate: string + try { + rangeDate = validateRangeDate(a.rangeDate) + } catch (error) { + return yield* new ArchiveError({ + message: error instanceof Error ? error.message : String(error), + }) + } + const { dataDir, archiveDir, scratchRoot } = resolveRoots(a.dataDir, a.archiveDir, a.scratchRoot) + const checkpointId = Option.getOrUndefined(a.checkpointId) + // Resolve tuning. Precedence: explicit CLI tuning flags > config-file + // effective values > defaults. A --config document is loaded from one fd + // (SHA-256-bound) and its effective values become the override base; the + // config identity is recorded in the manifest so the generation is + // reproducible. archive create does not yet expose per-knob CLI flags, + // so config-file values override defaults directly; conflicting root + // overrides (archiveDir/scratchRoot in config) are not applied — roots + // always come from the CLI/defaults. + const configPath = Option.getOrUndefined(a.config) + let tuning + let tuningConfigIdentity: TuningConfigIdentity | null = null + let loadedTuningConfig: LoadedTuningConfig | null = null + try { + if (configPath) { + const loaded = loadTuningConfig(configPath) + loadedTuningConfig = loaded + tuningConfigIdentity = loaded.identity + tuning = resolveArchiveTuning({ ...loaded.overrides, archiveDir, scratchRoot }) + } else { + tuning = resolveArchiveTuning(tuningOverrides(dataDir, archiveDir, scratchRoot)) + } + } catch (error) { + return yield* new ArchiveError({ + message: error instanceof Error ? error.message : String(error), + }) + } + yield* Effect.sync(() => + process.stderr.write( + `${amber("⟳")} archiving ${bold(a.signal)} for ${bold(rangeDate)} ` + + `from ${prettyPath(dataDir)}` + + (tuningConfigIdentity ? ` (config ${tuningConfigIdentity.configName})` : "") + + `\n`, + ), + ) + const result = yield* Effect.tryPromise({ + try: () => + createArchiveGeneration( + dataDir, + archiveDir, + a.signal, + rangeDate, + tuning, + checkpointId ?? "current", + {}, + loadedTuningConfig, + ), + catch: (error) => + new ArchiveError({ message: error instanceof Error ? error.message : String(error) }), + }) + yield* Effect.sync(() => + process.stdout.write( + `${green("✓")} archive generation sealed\n` + + ` ${dim("signal")} ${result.signal}\n` + + ` ${dim("range")} ${result.rangeStart}\n` + + ` ${dim("generation")} ${result.generationId}\n` + + ` ${dim("shards")} ${result.shardCount}\n` + + ` ${dim("rows")} ${result.archivedRowCount}\n` + + ` ${dim("archive-dir")} ${prettyPath(archiveDir)}\n` + + ` ${dim("scratch-root")} ${prettyPath(scratchRoot)}\n` + + ` ${dim("effective")} t=${tuning.writerThreads} rg=${tuning.rowGroupRows} ` + + `msr=${tuning.maxShardRows} msb=${tuning.maxShardBytes} ` + + `tc=${tuning.targetChunkBytes} reserve=${tuning.minFreeSpaceReserve}\n` + + (tuningConfigIdentity + ? ` ${dim("config")} ${tuningConfigIdentity.configName} (${tuningConfigIdentity.sha256})\n` + : "") + + (result.superseded ? ` ${dim("superseded")} ${result.superseded}\n` : ""), + ), + ) + }), + ), +) + +const signalFlag = Flag.optional( + Flag.string("signal").pipe( + Flag.withDescription(`One of: ${ARCHIVE_SIGNALS.map((s) => s.name).join(", ")}`), + ), +) + +export const archiveList = Command.make("list", { + archiveDir: archiveDirFlag, + output: outputFlag, + signal: signalFlag, +}).pipe( + Command.withDescription( + "List active archive metadata and Parquet shard paths without hashing shard contents", + ), + Command.withHandler( + Effect.fnUntraced(function* (a) { + const archiveDir = Option.getOrUndefined(a.archiveDir) ?? defaultArchiveDir() + if (a.output === "paths") { + const signalOpt = Option.getOrUndefined(a.signal) + if (!signalOpt || !isArchiveSignalName(signalOpt)) { + return yield* new ArchiveError({ + message: `--output paths requires a signal argument; expected one of ${ARCHIVE_SIGNALS.map((s) => s.name).join(", ")}`, + }) + } + const paths = activeParquetPaths(archiveDir, signalOpt) + yield* Effect.sync(() => process.stdout.write(`${paths.map((p) => `"${p}"`).join(",")}\n`)) + return + } + const listing = listActiveGenerations(archiveDir) + if (a.output === "json") { + yield* Effect.sync(() => process.stdout.write(`${JSON.stringify(listing, null, 2)}\n`)) + return + } + if (listing.active.length === 0) { + yield* Effect.sync(() => + process.stderr.write(`No active archive generations in ${prettyPath(archiveDir)}\n`), + ) + return + } + const lines = listing.active.map( + (summary) => + ` ${dim(summary.signal.padEnd(34))} ${summary.rangeStart} ` + + `${summary.archivedRowCount.toString().padStart(10)} rows ` + + `${summary.shardCount} shards ${summary.generationId.slice(0, 8)}`, + ) + yield* Effect.sync(() => + process.stdout.write( + `${green("✓")} ${listing.active.length} active generation(s) in ${prettyPath(archiveDir)} ` + + `(metadata only; run 'maple archive verify' for SHA-256)\n${lines.join("\n")}\n`, + ), + ) + }), + ), +) + +export const archiveVerify = Command.make("verify", { + archiveDir: archiveDirFlag, + signal: signalFlag, +}).pipe( + Command.withDescription("Stream and SHA-256 verify active archive shards with bounded memory"), + Command.withHandler( + Effect.fnUntraced(function* (a) { + const archiveDir = Option.getOrUndefined(a.archiveDir) ?? defaultArchiveDir() + const signalOpt = Option.getOrUndefined(a.signal) + if (signalOpt !== undefined && !isArchiveSignalName(signalOpt)) { + return yield* new ArchiveError({ + message: `unknown signal '${signalOpt}'; expected one of ${ARCHIVE_SIGNALS.map((s) => s.name).join(", ")}`, + }) + } + const result = yield* Effect.tryPromise({ + try: () => verifyActiveGenerations(archiveDir, signalOpt), + catch: (error) => + new ArchiveError({ message: error instanceof Error ? error.message : String(error) }), + }) + yield* Effect.sync(() => + process.stdout.write( + `${green("✓")} verified ${result.shardCount} active shard(s) across ` + + `${result.generationCount} generation(s) (${formatBytes(result.verifiedBytes)})\n`, + ), + ) + }), + ), +) + +export const archiveRebuild = Command.make("rebuild", { + archiveDir: archiveDirFlag, + signal: signalArgument, +}).pipe( + Command.withDescription("Rebuild a signal's catalog.jsonl from authoritative generation manifests"), + Command.withHandler( + Effect.fnUntraced(function* (a) { + if (!isArchiveSignalName(a.signal)) { + return yield* new ArchiveError({ + message: `unknown signal '${a.signal}'; expected one of ${ARCHIVE_SIGNALS.map((s) => s.name).join(", ")}`, + }) + } + const archiveDir = Option.getOrUndefined(a.archiveDir) ?? defaultArchiveDir() + const signalName: ArchiveSignalName = a.signal + const entries = yield* Effect.tryPromise({ + try: () => rebuildCatalog(archiveDir, signalName), + catch: (error) => + new ArchiveError({ message: error instanceof Error ? error.message : String(error) }), + }) + yield* Effect.sync(() => + process.stdout.write( + `${green("✓")} rebuilt ${a.signal} catalog with ${entries.length} generation(s)\n`, + ), + ) + }), + ), +) + +export const archiveReconcile = Command.make("reconcile", { + dataDir: dataDirFlag, + archiveDir: archiveDirFlag, + scratchRoot: scratchRootFlag, + dryRun: dryRunFlag, +}).pipe( + Command.withDescription( + "Reconcile an interrupted archive create or gc operation to its intended state without a fresh export", + ), + Command.withHandler( + Effect.fnUntraced(function* (a) { + const { dataDir, archiveDir, scratchRoot } = resolveRoots(a.dataDir, a.archiveDir, a.scratchRoot) + // Both dry-run and apply go through the locked runArchiveReconciliation + // entry point (blocker 2): dry-run returns the plan without mutating; + // apply acquires the maintenance lock, migrates any v2 intent, then + // reconciles — never racing create/GC planning or pointer/catalog repair. + const decision = yield* Effect.tryPromise({ + try: () => runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: a.dryRun }), + catch: (error) => + new ArchiveError({ message: error instanceof Error ? error.message : String(error) }), + }) + const renderDecision = (d: typeof decision): string => { + if (d.kind === "NoOp") return `${green("✓")} reconcile: no active operation\n` + if (d.kind === "FailClosed") return `${red("!")} FAIL CLOSED: ${d.reason}\n` + const id = "operationId" in d ? d.operationId : "" + const mig = "migrationRequired" in d && d.migrationRequired ? " (migrate v2)" : "" + return `${green("✓")} reconcile ${d.kind}: ${id}${mig}\n` + } + if (a.dryRun) { + if (decision.kind === "FailClosed") { + return yield* new ArchiveError({ message: renderDecision(decision).trim() }) + } + yield* Effect.sync(() => + process.stdout.write( + `${amber("◌")} dry-run reconcile\n${renderDecision(decision)} ${dim("archive")} ${prettyPath(archiveDir)}\n ${dim("note")} no archive state is modified\n`, + ), + ) + return + } + if (decision.kind === "FailClosed") { + return yield* new ArchiveError({ message: renderDecision(decision).trim() }) + } + yield* Effect.sync(() => + process.stderr.write( + `${amber("⟳")} reconciling interrupted archive operation in ${prettyPath(archiveDir)}\n`, + ), + ) + yield* Effect.sync(() => process.stdout.write(renderDecision(decision))) + }), + ), +) + +export const archiveGc = Command.make("gc", { + dataDir: dataDirFlag, + archiveDir: archiveDirFlag, + scratchRoot: scratchRootFlag, + keep: keepFlag, + dryRun: dryRunFlag, +}).pipe( + Command.withDescription( + "Reclaim superseded archive generations, retaining the newest N per signal/range (default 1)", + ), + Command.withHandler( + Effect.fnUntraced(function* (a) { + if (!Number.isSafeInteger(a.keep) || a.keep < 0) { + return yield* new ArchiveError({ + message: `invalid --keep value: ${a.keep} (must be a non-negative integer)`, + }) + } + const { dataDir, archiveDir, scratchRoot } = resolveRoots(a.dataDir, a.archiveDir, a.scratchRoot) + const result = yield* Effect.tryPromise({ + try: () => runArchiveGc({ dataDir, archiveDir, scratchRoot, keep: a.keep, dryRun: a.dryRun }), + catch: (error) => + new ArchiveError({ message: error instanceof Error ? error.message : String(error) }), + }) + const { plan } = result + if (a.dryRun) { + yield* Effect.sync(() => + process.stdout.write( + `${amber("◌")} dry-run gc: would delete ${plan.deleteSet.length} generation(s), ` + + `reclaim ${formatBytes(plan.reclaimableBytes)}\n` + + ` ${dim("keep")} ${plan.keep} newest superseded per range\n` + + (plan.deleteSet.length === 0 + ? ` ${dim("note")} nothing to reclaim\n` + : plan.deleteSet + .map( + (c) => + ` ${dim("delete")} ${c.signal}/${c.rangeStart}/${c.generationId} (${formatBytes(c.bytes)})`, + ) + .join("\n") + "\n") + + (plan.excludedSignals.length + plan.excludedRanges.length === 0 + ? "" + : `${red("!")} ${plan.excludedSignals.length + plan.excludedRanges.length} range(s)/signal(s) excluded (over-retained)\n`), + ), + ) + return + } + yield* Effect.sync(() => + process.stdout.write( + `${green("✓")} gc complete: deleted ${result.deleted.length} generation(s), ` + + `reclaimed ${formatBytes(plan.reclaimableBytes)}\n` + + ` ${dim("kept")} ${plan.keep} newest superseded per range\n`, + ), + ) + }), + ), +) + +const formatBytes = (bytes: number): string => { + if (bytes < 1024) return `${bytes} B` + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KiB` + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MiB` + return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GiB` +} + +const pauseAtSessionPhaseFlag = Flag.optional( + Flag.string("pause-at-session-phase").pipe( + Flag.withDescription("TEST ONLY: pause after durable-writing the parent session phase"), + ), +) +const sessionMarkerDirFlag = Flag.optional( + Flag.string("session-marker-dir").pipe( + Flag.withDescription("TEST ONLY: marker directory for parent-session pause"), + ), +) + +type CalibrationSelection = NonNullable + +/** Require a successful calibration recommendation through Effect's typed + * error channel. Keeping this check outside Effect.sync prevents an expected + * calibration failure from becoming an unhandled fiber defect. */ +export const requireCalibrationSelection = ( + recommendation: Pick, +): Effect.Effect => + recommendation.selected === null + ? Effect.fail( + new ArchiveError({ + message: `${red("!")} calibration did not produce a recommendation: ${recommendation.note}`, + }), + ) + : Effect.succeed(recommendation.selected) + +export const archiveCalibrate = Command.make("calibrate", { + dataDir: dataDirFlag, + archiveDir: archiveDirFlag, + scratchRoot: scratchRootFlag, + checkpointId: checkpointIdFlag, + rangeDate: rangeDateArgument, + memoryBudget: memoryBudgetFlag, + timeBudget: timeBudgetFlag, + sampleRows: sampleRowsFlag, + maxCandidateWallMs: maxCandidateWallMsFlag, + minThroughput: minThroughputFlag, + maxTempDisk: maxTempDiskFlag, + freeSpaceReserve: freeSpaceReserveFlag, + safetyMarginMilli: safetyMarginFlag, + writeConfig: writeConfigFlag, + pauseAtSessionPhase: pauseAtSessionPhaseFlag, + sessionMarkerDir: sessionMarkerDirFlag, +}).pipe( + Command.withDescription( + "Calibrate archive tuning by running a candidate matrix against a pinned checkpoint across all six signals", + ), + Command.withHandler( + Effect.fnUntraced(function* (a) { + let rangeDate: string + try { + rangeDate = validateRangeDate(a.rangeDate) + } catch (error) { + return yield* new ArchiveError({ + message: error instanceof Error ? error.message : String(error), + }) + } + let budget: CalibrationBudget + try { + budget = validateCalibrationBudget({ + memoryBudget: a.memoryBudget, + timeBudget: a.timeBudget, + sampleRows: a.sampleRows, + maxCandidateWallMs: a.maxCandidateWallMs, + minThroughputBytesPerSec: a.minThroughput, + maxTempDiskBytes: a.maxTempDisk, + freeSpaceReserve: a.freeSpaceReserve, + safetyMargin: a.safetyMarginMilli / 1000, + }) + } catch (error) { + return yield* new ArchiveError({ + message: error instanceof Error ? error.message : String(error), + }) + } + const { dataDir, archiveDir, scratchRoot } = resolveRoots(a.dataDir, a.archiveDir, a.scratchRoot) + const checkpointId = Option.getOrUndefined(a.checkpointId) ?? "current" + yield* Effect.sync(() => + process.stderr.write( + `${amber("⟳")} calibrating all six signals for ${bold(rangeDate)} ` + + `(memory ${a.memoryBudget}B, time ${a.timeBudget}ms, sample ${a.sampleRows} rows, ` + + `margin ${budget.safetyMargin.toFixed(3)}x)\n`, + ), + ) + // Run the candidate matrix across all six signals. Each candidate x + // signal combination is a fresh calibrate-run child spawned under + // /usr/bin/time so peak RSS is measured externally. A per-child watchdog + // enforces the candidate wall deadline and temp-disk ceiling DURING the + // run (SIGKILL on overrun -> candidate marked failed). + const rec = yield* Effect.tryPromise({ + try: () => + runCalibrationMatrix( + process.execPath, + dataDir, + checkpointId, + rangeDate, + scratchRoot, + archiveDir, + budget, + { + pauseAtPhase: Option.getOrUndefined(a.pauseAtSessionPhase), + markerDir: Option.getOrUndefined(a.sessionMarkerDir), + }, + ), + catch: (error) => + new ArchiveError({ message: error instanceof Error ? error.message : String(error) }), + }) + if ( + Option.getOrUndefined(a.pauseAtSessionPhase) === "post-session-release" && + Option.getOrUndefined(a.sessionMarkerDir) + ) { + const markerDir = Option.getOrUndefined(a.sessionMarkerDir)! + yield* Effect.tryPromise({ + try: async () => { + const { mkdirSync, writeFileSync } = await import("node:fs") + mkdirSync(markerDir, { recursive: true }) + writeFileSync( + join(markerDir, "paused"), + `post-session-release\n${process.pid}\n${new Date().toISOString()}\n`, + ) + await new Promise(() => { + /* deterministic SIGKILL seam after reconcile, before config/no-config publication */ + }) + }, + catch: (error) => + new ArchiveError({ + message: error instanceof Error ? error.message : String(error), + }), + }) + } + yield* Effect.sync(() => { + for (const r of rec.results) { + const status = r.ok && r.metrics ? `${r.metrics.peakRssBytes}B RSS` : `FAIL: ${r.error}` + process.stderr.write( + ` ${dim(`${r.signal} t=${r.candidate.writerThreads} rg=${r.candidate.rowGroupRows}`)} ${status}\n`, + ) + } + }) + const selected = yield* requireCalibrationSelection(rec) + const tuning = recommendationToTuning(rec, archiveDir, scratchRoot) + const writePath = Option.getOrUndefined(a.writeConfig) + if (writePath) { + yield* Effect.try({ + try: () => writeCalibrationConfig(writePath, rec, tuning), + catch: (error) => + new ArchiveError({ message: error instanceof Error ? error.message : String(error) }), + }) + yield* Effect.sync(() => + process.stdout.write( + `${green("✓")} calibration ${rec.confidence} confidence; config written to ${writePath}\n` + + ` ${dim("selected")} t=${selected.candidate.writerThreads} ` + + `rg=${selected.candidate.rowGroupRows} rss=${selected.worstCase.peakRssBytes}B\n` + + ` ${dim("margin")} ${budget.safetyMargin.toFixed(3)}x applied inside each ceiling\n`, + ), + ) + } else { + yield* Effect.sync(() => + process.stdout.write( + `${green("✓")} calibration ${rec.confidence} confidence\n` + + ` ${dim("selected")} t=${selected.candidate.writerThreads} ` + + `rg=${selected.candidate.rowGroupRows} rss=${selected.worstCase.peakRssBytes}B\n` + + ` ${dim("note")} pass --write-config to apply\n`, + ), + ) + } + }), + ), +) + +/** + * The metrics line printed by calibrate-run children and parsed by the parent. + * `exportWallMs` is the wall time of the calibrated export section only (not + * process-launch-to-exit), so write throughput is export-throughput, not + * end-to-end. + */ +const NonNegativeFinite = Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)) +const NonNegativeSafeInteger = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) + +const ChildMetricsSchema = Schema.Struct({ + logicalBytes: NonNegativeFinite, + physicalBytes: NonNegativeFinite, + peakTempDiskBytes: NonNegativeFinite, + peakRssBytes: NonNegativeFinite, + exportWallMs: NonNegativeFinite, + rowCount: NonNegativeSafeInteger, + sample: Schema.Struct({ + checkpointId: Schema.String, + checkpointManifestFingerprint: Schema.String, + rangeDate: Schema.String, + role: Schema.Literals(["training", "held-out"]), + startRow: NonNegativeSafeInteger, + requestedRows: NonNegativeSafeInteger, + rowCount: NonNegativeSafeInteger, + }), +}) + +type ChildMetrics = typeof ChildMetricsSchema.Type + +export interface ExpectedChildSampleScope { + readonly checkpointId: string + readonly checkpointManifestFingerprint: string + readonly rangeDate: string + readonly role: "training" | "held-out" + readonly startRow: number + readonly requestedRows: number +} + +/** Decode the child protocol exactly and bind its authoritative sample scope + * to the request that the parent actually sent. */ +export const decodeChildMetrics = (input: unknown, expected: ExpectedChildSampleScope): ChildMetrics => { + const raw = Schema.decodeUnknownSync(ChildMetricsSchema, { onExcessProperty: "error" })(input) + const sample = raw.sample + if ( + sample.checkpointId !== expected.checkpointId || + sample.checkpointManifestFingerprint !== expected.checkpointManifestFingerprint || + sample.rangeDate !== expected.rangeDate || + sample.role !== expected.role || + sample.startRow !== expected.startRow || + sample.requestedRows !== expected.requestedRows || + sample.rowCount !== raw.rowCount + ) { + throw new Error("calibrate-run emitted an inconsistent sample scope") + } + return raw +} + +/** + * Run one calibrate-run child under /usr/bin/time in a DEDICATED PROCESS GROUP + * so peak RSS is measured externally and the watchdog can kill the entire + * group (Maple descendant included). The watchdog uses the MINIMUM of the + * remaining total budget and the per-candidate wallMs. During the run, the + * parent POLLS the exact derived scratch/sample paths for temp-disk usage and + * kills the group on overrun (fail-loud: read/symlink/special-file errors fail + * the candidate). Peak RSS is FAIL-CLOSED: unparseable /usr/bin/time output + * fails the candidate (no completion-RSS fallback). + */ +const runCandidateChild = ( + bundlePath: string, + dataDir: string, + checkpointId: string, + checkpointManifestFingerprint: string, + rangeDate: string, + signal: string, + scratchRoot: string, + archiveDir: string, + candidate: CalibrationCandidate, + budget: CalibrationBudget, + operationId: string, + startRow: number, + sampleRows: number, + matrixStart: number, +): Promise => { + return new Promise((resolvePromise) => { + // Bun creates nonblocking stdio pipes for spawned children. GNU/BSD `time` + // writes a large multi-line report on exit, and that report can fail with + // EAGAIN when directed at the inherited stderr pipe. Write it to an + // independent temporary file instead; stderr remains available for real + // worker diagnostics and the report is removed after this one child closes. + let timeReport: ReturnType + try { + timeReport = createTimeReport() + } catch (error) { + resolvePromise({ + candidate, + signal, + metrics: null, + ok: false, + error: `failed to create time-report directory: ${error instanceof Error ? error.message : String(error)}`, + }) + return + } + const args = [ + "archive", + "calibrate-run", + signal, + rangeDate, + "--data-dir", + dataDir, + "--archive-dir", + archiveDir, + "--scratch-root", + scratchRoot, + "--checkpoint-id", + checkpointId, + "--checkpoint-fingerprint", + checkpointManifestFingerprint, + "--operation-id", + operationId, + "--start-row", + String(startRow), + "--sample-rows", + String(sampleRows), + "--max-temp-disk", + String(budget.maxTempDiskBytes), + "--free-space-reserve", + String(budget.freeSpaceReserve), + "--writer-threads", + String(candidate.writerThreads), + "--row-group-rows", + String(candidate.rowGroupRows), + "--max-shard-rows", + String(candidate.maxShardRows), + "--max-shard-bytes", + String(candidate.maxShardBytes), + ] + // Spawn under /usr/bin/time in its own process group so the watchdog can + // kill the whole group (Maple descendant included), not just /usr/bin/time. + const child = spawn("/usr/bin/time", [...timeArgv(), "-o", timeReport.path, bundlePath, ...args], { + stdio: ["ignore", "pipe", "pipe"], + detached: true, + }) + const childOutput = collectChildOutputAfterClose(child) + const pgid = child.pid ?? 0 + let killedByWatchdog = false + let killReason = "" + let settled = false + const finish = (result: CandidateResult) => { + if (settled) return + settled = true + resolvePromise(result) + } + // Watchdog deadline = min(remaining total budget, per-candidate wallMs). + const remaining = budget.timeBudget - (Date.now() - matrixStart) + const deadline = Math.max(1000, Math.min(budget.maxCandidateWallMs, remaining)) + // The exact derived paths the parent polls for temp-disk enforcement. + const pollScratch = resolve(scratchRoot, `calibrate-${operationId}`) + const pollSample = resolve(archiveDir, "calibration", "samples", operationId) + const killGroup = (reason: string) => { + killedByWatchdog = true + killReason = reason + try { + process.kill(-pgid, "SIGKILL") + } catch { + try { + child.kill("SIGKILL") + } catch { + // best-effort + } + } + } + const watchdog = setTimeout(() => killGroup(`exceeded ${deadline}ms wall deadline`), deadline) + // Poll temp-disk every 500ms during the run; kill on overrun. Read/symlink/ + // special-file errors fail-loud (kill the candidate). + const diskPoll = setInterval(async () => { + try { + const sz = (await directoryTreeBytes(pollScratch)) + (await directoryTreeBytes(pollSample)) + if (sz * budget.safetyMargin > budget.maxTempDiskBytes) { + clearInterval(diskPoll) + killGroup(`exceeded ${budget.maxTempDiskBytes}B temp-disk ceiling (saw ${sz}B)`) + } + } catch (error) { + clearInterval(diskPoll) + killGroup( + `temp-disk poll read error (fail-loud): ${error instanceof Error ? error.message : String(error)}`, + ) + } + }, 500) + child.on("error", (error) => { + clearTimeout(watchdog) + clearInterval(diskPoll) + timeReport.remove() + finish({ candidate, signal, metrics: null, ok: false, error: error.message }) + }) + // `exit` fires before stdio has necessarily drained. Wait for `close` so + // the next candidate cannot start while this worker still owns its pipes, + // and so failure reports include the complete worker diagnostics. + void childOutput.then(({ code, stdout, stderr }) => { + if (settled) return + clearTimeout(watchdog) + clearInterval(diskPoll) + const timeOutput = timeReport.readAndRemove() + if (killedByWatchdog) { + finish({ + candidate, + signal, + metrics: null, + ok: false, + error: `candidate killed by watchdog: ${killReason}`, + }) + return + } + // A nonzero exit means the child failed (export error OR cleanup + // failure). The child emits its metrics JSON only after successful + // cleanup; a JSON line present with a nonzero exit still means the + // owned resources may not have been released. Treat nonzero as failure. + if (code !== 0) { + const fullDiagnostic = `${stderr}\n${stdout}\n${timeOutput.report}` + const diagnostic = + fullDiagnostic.length <= 1600 + ? fullDiagnostic + : `${fullDiagnostic.slice(0, 800)}\n… diagnostics truncated …\n${fullDiagnostic.slice(-800)}` + finish({ + candidate, + signal, + metrics: null, + ok: false, + error: `calibrate-run exited ${code} (cleanup or export failure): ${diagnostic}${timeOutput.error ? `\n${timeOutput.error}` : ""}`, + }) + return + } + // Peak RSS: FAIL-CLOSED. Unparseable /usr/bin/time output fails the + // candidate (no completion-RSS fallback). + const peakRssBytes = + timeOutput.error === undefined ? parsePeakRss(timeOutput.report, process.platform) : null + if (peakRssBytes === null) { + finish({ + candidate, + signal, + metrics: null, + ok: false, + error: timeOutput.error + ? `${timeOutput.error} (fail-closed)` + : `failed to parse peak RSS from /usr/bin/time report (fail-closed)`, + }) + return + } + try { + const lines = stdout.trim().split("\n") + const parsed: unknown = JSON.parse(lines[lines.length - 1]!) + const raw = decodeChildMetrics(parsed, { + checkpointId, + checkpointManifestFingerprint, + rangeDate, + role: startRow === 0 ? "training" : "held-out", + startRow, + requestedRows: sampleRows, + }) + const logicalBytes = raw.logicalBytes + const physicalBytes = raw.physicalBytes + const compressionRatio = logicalBytes > 0 ? physicalBytes / logicalBytes : 0 + // Write throughput from the EXPORT section wall time, not process-launch-to-exit. + const writeThroughputBytesPerSec = + raw.exportWallMs > 0 ? logicalBytes / (raw.exportWallMs / 1000) : 0 + const metrics: CandidateMetrics = { + logicalBytes, + physicalBytes, + compressionRatio, + writeThroughputBytesPerSec, + peakTempDiskBytes: raw.peakTempDiskBytes, + peakRssBytes, + wallMs: raw.exportWallMs, + rowCount: raw.rowCount, + } + const sample = raw.sample + finish({ candidate, signal, metrics, ok: true, sample }) + } catch (error) { + finish({ + candidate, + signal, + metrics: null, + ok: false, + error: `failed to parse calibrate-run output: ${error instanceof Error ? error.message : String(error)}`, + }) + } + }) + }) +} + +/** + * Run the full calibration matrix across all six signals, select the best + * candidate by worst-case metrics, and validate it on a DISJOINT held-out + * sample (row window [sampleRows, 2*sampleRows), not the training window + * [0, sampleRows)). Requires complete six-signal evidence for eligibility and + * held-out. Confidence "high" ⟺ a config is emitted; "low" ⟺ selected null + * (small/unrepresentative data or insufficient disjoint held-out). + */ +const runCalibrationMatrix = async ( + bundlePath: string, + dataDir: string, + checkpointSelector: string, + rangeDate: string, + scratchRoot: string, + archiveDir: string, + budget: CalibrationBudget, + faults: { pauseAtPhase?: string; markerDir?: string } = {}, +): Promise => { + if (!Number.isSafeInteger(budget.freeSpaceReserve) || budget.freeSpaceReserve <= 0) { + throw new Error("calibration free-space reserve must be a positive integer") + } + const operationId = randomUUID() + const pinId = randomUUID() + const pinPurpose = calibrationPinPurpose(operationId) + const scratchSubdir = derivedScratchSubdir(operationId) + const sampleDir = derivedSampleDir(archiveDir, operationId) + const roots = { dataDir, archiveDir, scratchRoot } + const maybePauseSession = async (phase: string): Promise => { + if (faults.pauseAtPhase !== phase || !faults.markerDir) return + const { mkdirSync, writeFileSync } = await import("node:fs") + mkdirSync(faults.markerDir, { recursive: true }) + writeFileSync( + join(faults.markerDir, "paused"), + `${phase}\n${process.pid}\n${new Date().toISOString()}\n`, + ) + await new Promise(() => { + /* deterministic SIGKILL seam */ + }) + } + const session = await withMaintenanceLock(dataDir, operationId, async () => { + await reconcileCalibration(archiveDir, roots) + const resolved = await resolveCheckpoint(dataDir, checkpointSelector) + const manifestFingerprint = `${resolved.manifest.checkpointId}:${resolved.manifest.createdAt}:${resolved.manifest.backupBytes}` + await writeCalibrationRecord(archiveDir, { + phase: "intent", + operationId, + pinId, + pinPurpose, + pinPath: null, + checkpointId: resolved.checkpointId, + checkpointManifestFingerprint: manifestFingerprint, + boundRoots: roots, + ownedPaths: { scratchSubdir, sampleDir }, + }) + await maybePauseSession("intent") + const pinPath = await acquireCheckpointPin(dataDir, resolved.checkpointId, pinPurpose, pinId) + await writeCalibrationRecord(archiveDir, { + phase: "pin-acquired", + operationId, + pinId, + pinPurpose, + pinPath, + checkpointId: resolved.checkpointId, + checkpointManifestFingerprint: manifestFingerprint, + boundRoots: roots, + ownedPaths: { scratchSubdir, sampleDir }, + }) + await maybePauseSession("pin-acquired") + return { checkpointId: resolved.checkpointId, manifestFingerprint } + }) + try { + return await runBoundCalibrationMatrix( + bundlePath, + dataDir, + session.checkpointId, + session.manifestFingerprint, + operationId, + rangeDate, + scratchRoot, + archiveDir, + budget, + ) + } finally { + await withMaintenanceLock(dataDir, operationId, () => reconcileCalibration(archiveDir, roots)) + } +} + +const runBoundCalibrationMatrix = async ( + bundlePath: string, + dataDir: string, + checkpointId: string, + checkpointManifestFingerprint: string, + operationId: string, + rangeDate: string, + scratchRoot: string, + archiveDir: string, + budget: CalibrationBudget, +): Promise => { + const volId = await archiveVolumeIdentity(archiveDir) + const environment = captureEnvironment(MAPLE_VERSION, CHDB_VERSION, SCHEMA_FINGERPRINT, archiveDir, volId) + const allResults: CandidateResult[] = [] + const perSignal = new Map() + const matrixStart = Date.now() + for (const signal of ARCHIVE_SIGNALS) { + for (const candidate of CANDIDATE_MATRIX) { + if (Date.now() - matrixStart > budget.timeBudget) break + const result = await runCandidateChild( + bundlePath, + dataDir, + checkpointId, + checkpointManifestFingerprint, + rangeDate, + signal.name, + scratchRoot, + archiveDir, + candidate, + budget, + operationId, + 0, + budget.sampleRows, + matrixStart, + ) + allResults.push(result) + const list = perSignal.get(candidate) ?? [] + list.push(result) + perSignal.set(candidate, list) + } + if (Date.now() - matrixStart > budget.timeBudget) break + } + // Select eligible candidates requiring EXACTLY six signals each. + const requiredSignals = ARCHIVE_SIGNALS.map((s) => s.name) + const eligible = selectCandidates(perSignal, budget, requiredSignals) + let selected: { candidate: CalibrationCandidate; worstCase: CandidateMetrics } | null = null + let selectedHeldOut: CalibrationRecommendation["heldOut"] = null + const heldOutAttempts: CalibrationRecommendation["heldOutAttempts"][number][] = [] + let note: string + if (eligible.length === 0) { + note = + `no candidate met the declared goals across all six signals ` + + `(memory ${budget.memoryBudget}B, candidate ${budget.maxCandidateWallMs}ms, ` + + `throughput ${budget.minThroughputBytesPerSec}B/s, temp disk ${budget.maxTempDiskBytes}B) ` + + `with margin ${budget.safetyMargin.toFixed(3)}x; no configuration emitted` + } else { + // Held-out validation on a DISJOINT row window: startRow=sampleRows so the + // held-out sample is rows [sampleRows, 2*sampleRows) — not overlapping the + // training window [0, sampleRows). A candidate that fails held-out is + // REJECTED; try the next eligible. If none pass, no config. + for (const cand of eligible) { + const heldOutResults: CandidateResult[] = [] + for (const signal of ARCHIVE_SIGNALS) { + if (Date.now() - matrixStart > budget.timeBudget) break + // Held-out: a STRICTLY LARGER, disjoint window. Training covered + // ordered rows [0, sampleRows); held-out covers + // [sampleRows, sampleRows + heldOutRows) where heldOutRows is a + // fixed multiple of the training size (plan-required larger sample). + const result = await runCandidateChild( + bundlePath, + dataDir, + checkpointId, + checkpointManifestFingerprint, + rangeDate, + signal.name, + scratchRoot, + archiveDir, + cand.candidate, + budget, + operationId, + budget.sampleRows, + heldOutSampleRows(budget.sampleRows), + matrixStart, + ) + heldOutResults.push(result) + } + // Require complete six-signal held-out evidence: every result within + // ceilings AND observing exactly heldOutSampleRows rows (a larger + // request is not a larger observed sample). + const heldOutComplete = + heldOutResults.length === requiredSignals.length && + heldOutResults.every( + (r) => + meetsCeilings(r, budget) && + r.metrics?.rowCount === heldOutSampleRows(budget.sampleRows), + ) + if (heldOutComplete) { + const heldWorst = selectCandidates( + new Map([[cand.candidate, heldOutResults]]), + budget, + requiredSignals, + )[0]!.worstCase + // PER-SIGNAL, like-for-like hybrid comparison: each signal's held-out + // result is paired with the same candidate's TRAINING result for that + // signal, and wallMs/physicalBytes are scaled by THAT signal's own + // heldOut/training logical-byte ratio. Aggregate extrema never decide + // acceptance; heldWorst is a descriptive summary only. + const perSignal = compareHeldOutPerSignal( + allResults, + heldOutResults, + requiredSignals, + cand.candidate, + HELD_OUT_TOLERANCES, + ) + if (perSignal === null) { + // Unpairable or non-positive logical bytes: treat as incomplete. + heldOutAttempts.push({ + candidate: cand.candidate, + results: heldOutResults, + worstCase: null, + signalComparisons: [], + passed: false, + }) + continue + } + heldOutAttempts.push({ + candidate: cand.candidate, + results: heldOutResults, + worstCase: heldWorst, + signalComparisons: perSignal.signalComparisons, + passed: perSignal.passed, + }) + if (!perSignal.passed) continue + selected = cand + selectedHeldOut = { + results: heldOutResults, + worstCase: heldWorst, + signalComparisons: perSignal.signalComparisons, + passed: true, + tolerances: HELD_OUT_TOLERANCES, + } + note = + `selected the lowest-worst-case-peak-RSS candidate that met every ceiling ` + + `on the disjoint held-out window across all six signals (per-signal comparison)` + break + } + heldOutAttempts.push({ + candidate: cand.candidate, + results: heldOutResults, + worstCase: null, + // Incomplete/over-budget/short-window attempt: no comparisons ran. + signalComparisons: [], + passed: false, + }) + } + if (selected === null) { + note = + `every eligible candidate failed held-out validation (disjoint window) ` + + `or the data was insufficient for a complete six-signal held-out split; ` + + `no configuration emitted` + } + } + // Confidence "high" ⟺ selected !== null ⟺ a config is emitted. "low" means + // small/unrepresentative data OR no disjoint held-out — always paired with + // selected null and no config. Per-signal representative check (not a + // cross-candidate sum that repetition could inflate): every signal's + // training rowCount must reach at least the sampleRows target for the data + // to be representative. + const perSignalRepresentative = (() => { + if (selected === null) return true // no false-high; selected null → low anyway + const bySignal = new Map() + for (const r of allResults) { + if (isSameCalibrationCandidate(r.candidate, selected.candidate) && r.ok && r.metrics) { + bySignal.set(r.signal, Math.max(bySignal.get(r.signal) ?? 0, r.metrics.rowCount)) + } + } + return requiredSignals.every((s) => bySignal.get(s) === budget.sampleRows) + })() + const confidence: "high" | "low" = selected !== null && perSignalRepresentative ? "high" : "low" + if (confidence === "low" && selected !== null) { + // Downgrade to no-config: low confidence ⟺ selected null. + note = `selected candidate's per-signal data is unrepresentative (below the ${budget.sampleRows}-row target); no configuration emitted` + selected = null + selectedHeldOut = null + } + return { + formatVersion: TUNING_CONFIG_FORMAT_VERSION, + checkpoint: { checkpointId, manifestFingerprint: checkpointManifestFingerprint }, + selected, + heldOut: selectedHeldOut, + heldOutAttempts, + results: allResults, + budget, + environment, + confidence, + measuredAt: new Date().toISOString(), + note: note!, + } +} + +/** + * Internal calibration worker. The PARENT generates the operation id and passes + * it via `--operation-id`, along with `--start-row` (for the disjoint held-out + * window) and the operator's `--max-temp-disk` / `--free-space-reserve`. The + * child reconciles any prior interrupted run INSIDE the maintenance lock, + * records ownership derived from the operation id, restores a pinned checkpoint + * into owned scratch, exports a deterministic EXACT window of rows through the + * REAL shared writer, measures real metrics, and cleans up via the SAME + * authoritative reconciler (no duplicate removal logic). + */ +const operationIdFlag = Flag.optional( + Flag.string("operation-id").pipe( + Flag.withDescription("Calibration operation id (parent-generated); derives owned paths"), + ), +) +const checkpointFingerprintFlag = Flag.optional( + Flag.string("checkpoint-fingerprint").pipe( + Flag.withDescription("Exact parent-session checkpoint manifest fingerprint"), + ), +) +const startRowFlag = Flag.integer("start-row").pipe( + Flag.withDescription("Start row offset for the calibration window (0=training, sampleRows=held-out)"), + Flag.withDefault(0), +) +const maxTempDiskCalibFlag = Flag.integer("max-temp-disk").pipe( + Flag.withDescription("Maximum peak temporary disk in bytes (operator-supplied ceiling)"), + Flag.withDefault(2 * 1024 * 1024 * 1024), +) +const freeSpaceReserveCalibFlag = Flag.integer("free-space-reserve").pipe( + Flag.withDescription("Minimum free-space reserve on the archive volume in bytes"), + Flag.withDefault(512 * 1024 * 1024), +) +// TEST SEAM (not for operator use): when set, the child writes a `paused` marker +// into the marker dir AFTER durable-writing the recovery record at the named +// phase, then blocks forever. The SIGKILL crash probe waits for the marker, +// asserts the durable state exists, then kills the process group. This makes the +// crash boundary deterministic and authoritative (C1). +const pauseAtPhaseFlag = Flag.optional( + Flag.string("pause-at-phase").pipe( + Flag.withDescription("TEST ONLY: pause (block) after durable-writing the record at this phase"), + ), +) +const markerDirFlag = Flag.optional( + Flag.string("marker-dir").pipe(Flag.withDescription("TEST ONLY: directory for the pause marker file")), +) + +export const archiveCalibrateRun = Command.make("calibrate-run", { + signal: signalArgument, + rangeDate: rangeDateArgument, + dataDir: dataDirFlag, + archiveDir: archiveDirFlag, + scratchRoot: scratchRootFlag, + checkpointId: checkpointIdFlag, + checkpointFingerprint: checkpointFingerprintFlag, + operationId: operationIdFlag, + startRow: startRowFlag, + sampleRows: sampleRowsFlag, + maxTempDisk: maxTempDiskCalibFlag, + freeSpaceReserve: freeSpaceReserveCalibFlag, + writerThreads: writerThreadsFlag, + rowGroupRows: rowGroupRowsFlag, + maxShardRows: maxShardRowsFlag, + maxShardBytes: maxShardBytesFlag, + pauseAtPhase: pauseAtPhaseFlag, + markerDir: markerDirFlag, +}).pipe( + Command.withDescription( + "Internal: export a calibration sample through the real writer and print metrics JSON", + ), + Command.withHandler( + Effect.fnUntraced(function* (a) { + if (!isArchiveSignalName(a.signal)) { + return yield* new ArchiveError({ message: `unknown signal '${a.signal}'` }) + } + let rangeDate: string + try { + rangeDate = validateRangeDate(a.rangeDate) + } catch (error) { + return yield* new ArchiveError({ + message: error instanceof Error ? error.message : String(error), + }) + } + const { dataDir, archiveDir, scratchRoot } = resolveRoots(a.dataDir, a.archiveDir, a.scratchRoot) + const checkpointSelector = Option.getOrUndefined(a.checkpointId) ?? "current" + yield* Effect.tryPromise({ + try: () => + runCalibrateSample(a, dataDir, archiveDir, scratchRoot, checkpointSelector, rangeDate), + catch: (error) => + new ArchiveError({ message: error instanceof Error ? error.message : String(error) }), + }) + }), + ), +) + +/** + * Open or close the parent calibration session that owns the single source + * checkpoint pin and the durable recovery record. The matrix runner does this + * inline; this command makes the lifecycle explicit so a single child sample + * (or a SIGKILL probe) can run against an already-open session and so an + * operator can inspect/retire a wedged session. `open` resolves the checkpoint, + * reconciles any prior interrupted session, acquires the pin, and durably + * records `pin-acquired`; it prints the operation id, checkpoint id, and + * manifest fingerprint a child must bind to. `close` runs the authoritative + * reconciler (releasing the pin and clearing the record). + */ +const sessionActionFlag = Flag.optional( + Flag.string("action").pipe( + Flag.withDescription("open: acquire the session pin + record; close: reconcile + release"), + ), +) +export const archiveCalibrateSession = Command.make("calibrate-session", { + dataDir: dataDirFlag, + archiveDir: archiveDirFlag, + scratchRoot: scratchRootFlag, + checkpointId: checkpointIdFlag, + action: sessionActionFlag, + pauseAtSessionPhase: pauseAtSessionPhaseFlag, + sessionMarkerDir: sessionMarkerDirFlag, +}).pipe( + Command.withDescription( + "Internal: open or close the parent calibration session that owns the source pin", + ), + Command.withHandler( + Effect.fnUntraced(function* (a) { + const { dataDir, archiveDir, scratchRoot } = resolveRoots(a.dataDir, a.archiveDir, a.scratchRoot) + const action = Option.getOrUndefined(a.action) ?? "open" + const checkpointSelector = Option.getOrUndefined(a.checkpointId) ?? "current" + const roots = { dataDir, archiveDir, scratchRoot } + if (action === "close") { + yield* Effect.tryPromise({ + try: () => + withMaintenanceLock(dataDir, randomUUID(), () => + reconcileCalibration(archiveDir, roots), + ), + catch: (error) => + new ArchiveError({ message: error instanceof Error ? error.message : String(error) }), + }) + process.stdout.write(`${green("✓")} calibration session closed\n`) + return + } + if (action !== "open") { + return yield* new ArchiveError({ message: `unknown calibrate-session action '${action}'` }) + } + const operationId = randomUUID() + const pinId = randomUUID() + const pinPurpose = calibrationPinPurpose(operationId) + const scratchSubdir = derivedScratchSubdir(operationId) + const sampleDir = derivedSampleDir(archiveDir, operationId) + const result = yield* Effect.tryPromise({ + try: () => + withMaintenanceLock(dataDir, operationId, async () => { + await reconcileCalibration(archiveDir, roots) + const resolved = await resolveCheckpoint(dataDir, checkpointSelector) + const manifestFingerprint = `${resolved.manifest.checkpointId}:${resolved.manifest.createdAt}:${resolved.manifest.backupBytes}` + await writeCalibrationRecord(archiveDir, { + phase: "intent", + operationId, + pinId, + pinPurpose, + pinPath: null, + checkpointId: resolved.checkpointId, + checkpointManifestFingerprint: manifestFingerprint, + boundRoots: roots, + ownedPaths: { scratchSubdir, sampleDir }, + }) + // TEST seam: a SIGKILL here leaves a durable intent record with + // no pin, reproducing the intent-retention wedge. + const pausePhase: string | undefined = Option.getOrUndefined(a.pauseAtSessionPhase) + const markerDir: string | undefined = Option.getOrUndefined(a.sessionMarkerDir) + if (pausePhase === "intent" && markerDir) { + const { mkdirSync, writeFileSync } = await import("node:fs") + mkdirSync(markerDir, { recursive: true }) + writeFileSync( + join(markerDir, "paused"), + `intent\n${process.pid}\n${new Date().toISOString()}\n`, + ) + await new Promise(() => { + /* deterministic SIGKILL seam */ + }) + } + const pinPath = await acquireCheckpointPin( + dataDir, + resolved.checkpointId, + pinPurpose, + pinId, + ) + await writeCalibrationRecord(archiveDir, { + phase: "pin-acquired", + operationId, + pinId, + pinPurpose, + pinPath, + checkpointId: resolved.checkpointId, + checkpointManifestFingerprint: manifestFingerprint, + boundRoots: roots, + ownedPaths: { scratchSubdir, sampleDir }, + }) + return { checkpointId: resolved.checkpointId, manifestFingerprint, pinPath } + }), + catch: (error) => + new ArchiveError({ message: error instanceof Error ? error.message : String(error) }), + }) + // Machine-readable: a child binds to operation-id + checkpoint-id + fingerprint. + process.stdout.write( + `${JSON.stringify({ + operationId, + checkpointId: result.checkpointId, + manifestFingerprint: result.manifestFingerprint, + pinPath: result.pinPath, + })}\n`, + ) + }), + ), +) + +/** + * Run one calibration sample (child process). Reconciles any prior interrupted + * run INSIDE the maintenance lock (so a concurrent run cannot reconcile a live + * run's resources), then records ownership DERIVED from the operation id, + * restores a pinned checkpoint, exports a deterministic EXACT window of rows + * through the REAL shared writer with the row-count assertion, measures real + * metrics (export-section wall time, not process-launch-to-exit), and cleans up + * via the SAME authoritative reconciler. + */ +const runCalibrateSample = async ( + a: { + signal: string + operationId: Option.Option + startRow: number + sampleRows: number + maxTempDisk: number + freeSpaceReserve: number + writerThreads: number + rowGroupRows: number + maxShardRows: number + maxShardBytes: number + pauseAtPhase: Option.Option + markerDir: Option.Option + checkpointFingerprint: Option.Option + }, + dataDir: string, + archiveDir: string, + scratchRoot: string, + checkpointSelector: string, + rangeDate: string, +): Promise => { + // The parent generates the operation id; derive the exact owned paths from it. + const operationId = Option.getOrUndefined(a.operationId) + const checkpointManifestFingerprint = Option.getOrUndefined(a.checkpointFingerprint) + if ( + !operationId || + !checkpointManifestFingerprint || + checkpointSelector === "current" || + checkpointSelector === "previous" + ) { + throw new Error( + "calibrate-run requires a parent session operation id, exact checkpoint id, and checkpoint fingerprint", + ) + } + // TEST SEAM: if pauseAtPhase is set, write a marker and block after durable- + // writing the record at that phase. The crash probe waits for the marker, + // asserts the durable state, then SIGKILLs (C1). + const pausePhase = Option.getOrUndefined(a.pauseAtPhase) + const markerDir = Option.getOrUndefined(a.markerDir) + const maybePause = async (phase: string): Promise => { + if (pausePhase !== phase || !markerDir) return + const { writeFileSync } = await import("node:fs") + const { join: joinPath } = await import("node:path") + const { mkdirSync } = await import("node:fs") + mkdirSync(markerDir, { recursive: true }) + writeFileSync( + joinPath(markerDir, "paused"), + `${phase}\n${process.pid}\n${new Date().toISOString()}\n`, + ) + // Block forever until SIGKILL. A thrown error here would run the finally + // (cleanup); a SIGKILL does not, leaving the durable state for reconcile. + await new Promise(() => { + /* block forever */ + }) + } + // The parent session owns the pin and the durable checkpoint identity; the + // child only restores that pinned checkpoint into owned scratch and exports + // a sample. See assertCalibrationSession / cleanupCalibrationSample. + const scratchSubdir = derivedScratchSubdir(operationId) + const sampleDir = derivedSampleDir(archiveDir, operationId) + const settings: ExportSettings = { + writerThreads: a.writerThreads, + rowGroupRows: a.rowGroupRows, + maxShardRows: a.maxShardRows, + maxShardBytes: a.maxShardBytes, + } + // Free-space preflight with the OPERATOR-SUPPLIED reserve (not hardcoded). + await preflightCalibrationFreeSpace(archiveDir, a.freeSpaceReserve, a.maxShardBytes * 4) + const signal = archiveSignal(a.signal as Parameters[0]) + // Captured during export; emitted to stdout ONLY after successful cleanup so + // a cleanup failure causes a nonzero exit and the parent marks this candidate + // failed (C5: a run that left a pin/record/debris must not be selected). + let pendingMetrics: ChildMetrics | null = null + // The maintenance lock serializes calibration against create/GC. Reconcile + // any prior interrupted run INSIDE the lock, matching generation.ts:246-283. + await withMaintenanceLock(dataDir, operationId, async () => { + const session = await assertCalibrationSession( + archiveDir, + { dataDir, archiveDir, scratchRoot }, + { + operationId, + checkpointId: checkpointSelector, + checkpointManifestFingerprint, + }, + ) + await cleanupCalibrationSample(session) + const resolved = await resolveCheckpoint(dataDir, checkpointSelector) + const liveFingerprint = `${resolved.manifest.checkpointId}:${resolved.manifest.createdAt}:${resolved.manifest.backupBytes}` + if (liveFingerprint !== checkpointManifestFingerprint) { + throw new Error("calibration child checkpoint fingerprint changed; refusing") + } + await maybePause("pin-acquired") + try { + await withRestoredCheckpoint( + resolved, + { + scratchRoot, + scratchSubdir, + cleanup: "never", + beforeRestore: async () => { + await maybePause("scratch-allocated") + }, + }, + async ({ db }) => { + await ensurePrivateDirectory(sampleDir, archiveDir) + // The sampling seam is intentionally after both the durable phase + // record and the owned sample directory exist. Together with the + // restored scratch allocated by withRestoredCheckpoint, this lets + // the SIGKILL probe exercise cleanup of every owned resource. + await maybePause("sampling") + db.exec(`SYSTEM STOP MERGES ${signal.name}`) + const exportStart = Date.now() + try { + const sourceSchema = captureSourceSchema(db, signal) + // EXACT window: plan returns { plansByHour, totalRows } where + // totalRows is the exact matching-row count for this window. + const { plansByHour, totalRows } = planCalibrationShards( + db, + signal, + rangeDate, + settings, + a.sampleRows, + a.startRow, + ) + // The writer asserts Σ rowCount === totalRows (exact bound). + const shards = exportShardPlans( + db, + signal, + rangeDate, + sampleDir, + settings, + sourceSchema, + plansByHour, + totalRows, + ) + const exportWallMs = Date.now() - exportStart + let logicalBytes = 0 + let physicalBytes = 0 + let rowCount = 0 + for (const shard of shards) { + const measured = measureShardBytes(db, shard.path) + logicalBytes += measured.uncompressed + physicalBytes += shard.bytes + rowCount += shard.rowCount + } + const peakTempDiskBytes = + (await directoryTreeBytes(resolve(scratchRoot, scratchSubdir))) + + (await directoryTreeBytes(sampleDir)) + // Capture the metrics but DO NOT emit them yet — emit only after + // successful cleanup so a cleanup failure causes a nonzero exit and + // the parent marks the candidate failed (C5). + pendingMetrics = { + logicalBytes, + physicalBytes, + peakTempDiskBytes, + peakRssBytes: process.memoryUsage().rss, + exportWallMs, + rowCount, + // The child is the authoritative source of its exact sample scope: + // it ran planCalibrationShards(startRow, sampleRows) against this + // exact checkpoint/range and the writer asserted rowCount === totalRows. + sample: { + checkpointId: checkpointSelector, + checkpointManifestFingerprint, + rangeDate, + role: a.startRow === 0 ? "training" : "held-out", + startRow: a.startRow, + requestedRows: a.sampleRows, + rowCount, + }, + } + } finally { + db.exec(`SYSTEM START MERGES ${signal.name}`) + } + }, + ) + } finally { + // Normal cleanup calls the SAME authoritative reconciler (no duplicate + // removal logic). A cleanup-reconciliation FAILURE must propagate as a + // nonzero exit (NOT suppressed), so the parent marks the candidate + // failed and does not select a run that left a pin/record/debris. The + // record is PRESERVED for the next run by the reconciler itself. + await cleanupCalibrationSample(session) + // Emit the metrics JSON ONLY after successful cleanup. + if (pendingMetrics) { + process.stdout.write(`${JSON.stringify(pendingMetrics)}\n`) + } + } + }) +} + +export const archive = Command.make("archive").pipe( + Command.withDescription("Manage local Parquet telemetry archives exported from immutable checkpoints"), + Command.withSubcommands([ + archiveCreate, + archiveList, + archiveVerify, + archiveRebuild, + archiveReconcile, + archiveGc, + archiveCalibrate, + archiveCalibrateRun, + archiveCalibrateSession, + ]), +) diff --git a/apps/cli/src/commands/server-args.ts b/apps/cli/src/commands/server-args.ts new file mode 100644 index 000000000..438ec41c8 --- /dev/null +++ b/apps/cli/src/commands/server-args.ts @@ -0,0 +1,28 @@ +export type DirtyStorePolicy = "wipe" | "fail" | "restore-checkpoint" + +export interface DetachedChildArgs { + readonly entry: string | undefined + readonly port: number + readonly dataDir: string + readonly offline: boolean + readonly chdbConfigFile: string | undefined + readonly onDirtyStore: DirtyStorePolicy +} + +/** Build the foreground child argv without forwarding compiled-Bun virtual + * entrypoints or the background flag that caused the re-exec. */ +export const buildDetachedChildArgs = (options: DetachedChildArgs): string[] => { + const runtimeArgs = options.entry && !options.entry.startsWith("/$bunfs") ? [options.entry] : [] + return [ + ...runtimeArgs, + "start", + "--port", + String(options.port), + "--data-dir", + options.dataDir, + "--on-dirty-store", + options.onDirtyStore, + ...(options.chdbConfigFile ? ["--chdb-config-file", options.chdbConfigFile] : []), + ...(options.offline ? ["--offline"] : []), + ] +} diff --git a/apps/cli/src/commands/server.ts b/apps/cli/src/commands/server.ts index 0135faf31..6f217fca9 100644 --- a/apps/cli/src/commands/server.ts +++ b/apps/cli/src/commands/server.ts @@ -13,11 +13,17 @@ import { isStoreDirty, storeMarkerJson, storeMarkerPath, - storeOpenMarkerPath, } from "../server/store-version" +import { + createCheckpoint, + reconcileCheckpointRecovery, + resetLiveStorePreservingCheckpoints, + restoreCheckpoint, +} from "../server/checkpoints" import { resolveUiAssets } from "../server/ui-assets" import { amber, bold, cyan, dim, green, underline } from "../lib/style" import { MAPLE_VERSION } from "../version" +import { buildDetachedChildArgs, type DirtyStorePolicy } from "./server-args" /** A `maple start`/`maple stop` failure. The message is shown to the user and * the process exits non-zero — same role the old `process.exit(1)` paths had, @@ -104,6 +110,12 @@ const dataDirFlag = Flag.optional( ), ) +const chdbConfigFileFlag = Flag.optional( + Flag.string("chdb-config-file").pipe( + Flag.withDescription("Optional ClickHouse config file passed to embedded chDB"), + ), +) + const backgroundFlag = Flag.boolean("background").pipe( Flag.withAlias("d"), Flag.withDescription("Run the server detached (logs to ~/.maple/maple.log); stop with `maple stop`"), @@ -112,17 +124,28 @@ const backgroundFlag = Flag.boolean("background").pipe( const resetFlag = Flag.boolean("reset").pipe( Flag.withDescription( - "Wipe the existing store (~/.maple/data) before starting — use after an incompatible upgrade", + "Wipe live chDB data before starting while preserving checkpoints — use after an incompatible upgrade", ), Flag.withDefault(false), ) +const onDirtyStoreFlag = Flag.choice("on-dirty-store", ["wipe", "fail", "restore-checkpoint"]).pipe( + Flag.withDescription("Recovery policy when the local chDB store was not cleanly closed"), + Flag.withDefault("fail" as const), +) + const yesFlag = Flag.boolean("yes").pipe( Flag.withAlias("y"), Flag.withDescription("Skip the confirmation prompt"), Flag.withDefault(false), ) +const checkpointIdFlag = Flag.optional( + Flag.string("checkpoint-id").pipe( + Flag.withDescription("Restore one immutable checkpoint ID instead of the selected current"), + ), +) + const offlineFlag = Flag.boolean("offline").pipe( Flag.withDescription( "Use the UI bundled in this binary (served from 127.0.0.1) instead of local.maple.dev", @@ -149,24 +172,27 @@ const probeHealth = (addr: string): Effect.Effect => * file; we poll `/health` until it binds, then print a summary and return so the * parent process exits. */ -const startDetached = (port: number, dataDir: string, offline: boolean): Effect.Effect => +const startDetached = ( + port: number, + dataDir: string, + offline: boolean, + chdbConfigFile: string | undefined, + onDirtyStore: DirtyStorePolicy, +): Effect.Effect => Effect.gen(function* () { const logPath = logFilePath(dataDir) // Rebuild the command explicitly rather than slicing argv: a Bun-compiled // binary injects a virtual `/$bunfs/...` entrypoint at argv[1] that must // not be forwarded. In dev (`bun run src/bin.ts`) argv[1] is the real // script and Bun needs it; in the compiled binary execPath alone suffices. - const entry = process.argv[1] - const runtimeArgs = entry && !entry.startsWith("/$bunfs") ? [entry] : [] - const childArgs = [ - ...runtimeArgs, - "start", - "--port", - String(port), - "--data-dir", + const childArgs = buildDetachedChildArgs({ + entry: process.argv[1], + port, dataDir, - ...(offline ? ["--offline"] : []), - ] + offline, + chdbConfigFile, + onDirtyStore, + }) const child = yield* Effect.try({ try: () => { @@ -214,9 +240,11 @@ const startDetached = (port: number, dataDir: string, offline: boolean): Effect. export const start = Command.make("start", { port, dataDir: dataDirFlag, + chdbConfigFile: chdbConfigFileFlag, background: backgroundFlag, offline: offlineFlag, reset: resetFlag, + onDirtyStore: onDirtyStoreFlag, }).pipe( Command.withDescription("Start the local ingest + query server (embedded ClickHouse via chDB)"), Command.withHandler( @@ -234,12 +262,18 @@ export const start = Command.make("start", { } if (Option.isSome(existingPid)) yield* fs.remove(pidPath, { force: true }).pipe(Effect.ignore) // stale + // A restore transaction lives beside dataDir and must be reconciled + // before reset, compatibility, dirty-store, or directory creation logic. + yield* reconcileCheckpointRecovery(dataDir).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) + // `--reset`: wipe the store (and its version marker) so we bootstrap fresh. - // The escape hatch when an existing store is from an incompatible build. + // Preserve the checkpoint registry under dataDir/backups. if (a.reset) { - yield* fs.remove(dataDir, { recursive: true, force: true }).pipe(Effect.ignore) - yield* fs.remove(storeMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.remove(storeOpenMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) + yield* resetLiveStorePreservingCheckpoints(dataDir).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) } yield* fs.makeDirectory(dataDir, { recursive: true }) @@ -262,43 +296,71 @@ export const start = Command.make("start", { // which we cannot catch. Auto-wipe and bootstrap fresh instead of walking // into the crash. (`--reset` already wiped above, so the marker is gone.) if (isStoreDirty(dataDir)) { - yield* Effect.sync(() => - process.stderr.write( - amber( - "⚠ the local store was left inconsistent by an unclean shutdown — " + - "wiping it and starting fresh (local telemetry data is discarded)\n", + if (a.onDirtyStore === "fail") { + return yield* new ServerError({ + message: + `the local store at ${prettyPath(dataDir)} was not cleanly closed. ` + + `Run \`${bold("maple restore --yes")}\` to restore from the last checkpoint, ` + + `or \`${bold("maple start --reset")}\` to wipe it.`, + }) + } + if (a.onDirtyStore === "restore-checkpoint") { + yield* Effect.sync(() => + process.stderr.write( + amber( + "⚠ the local store was left inconsistent by an unclean shutdown — " + + "restoring the last checkpoint\n", + ), ), - ), - ) - yield* fs.remove(dataDir, { recursive: true, force: true }).pipe(Effect.ignore) - yield* fs.remove(storeMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.remove(storeOpenMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.makeDirectory(dataDir, { recursive: true }) + ) + const restored = yield* restoreCheckpoint(dataDir).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) + yield* Effect.sync(() => + process.stderr.write( + `${green("✓")} restored checkpoint; quarantined dirty store at ${prettyPath(restored.quarantinePath)}\n`, + ), + ) + } else { + yield* Effect.sync(() => + process.stderr.write( + amber( + "⚠ the local store was left inconsistent by an unclean shutdown — " + + "explicit wipe selected; discarding live telemetry while preserving checkpoints\n", + ), + ), + ) + yield* resetLiveStorePreservingCheckpoints(dataDir).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) + yield* fs.makeDirectory(dataDir, { recursive: true }) + } } // A store bootstrapped from an older bundled schema can't be evolved in // place: `CREATE … IF NOT EXISTS` is a no-op on existing tables, so a // column added to the schema (e.g. ServiceNamespace on trace_list_mv) // never lands and facet queries referencing it fail. Rebuild from the - // current schema. Safe to auto-wipe — local telemetry is ephemeral and - // re-ingested — and only triggers once per schema change. + // current schema. Do not silently delete telemetry or checkpoints: + // require an explicit reset, which preserves the checkpoint registry. if (isSchemaStale(dataDir, SCHEMA_FINGERPRINT)) { - yield* Effect.sync(() => - process.stderr.write( - amber( - "⚠ the local store was built from an older schema — " + - "rebuilding it from the current schema (local telemetry data is discarded)\n", - ), - ), - ) - yield* fs.remove(dataDir, { recursive: true, force: true }).pipe(Effect.ignore) - yield* fs.remove(storeMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.remove(storeOpenMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.makeDirectory(dataDir, { recursive: true }) + return yield* new ServerError({ + message: + `the local store at ${prettyPath(dataDir)} was built from an older schema. ` + + `Maple preserved it and its checkpoints; explicitly rebuild live data with ` + + `\`${bold("maple start --reset")}\` or \`${bold("maple reset --yes")}\`.`, + }) } // Detached: spawn the same command without --background and exit. - if (a.background) return yield* startDetached(a.port, dataDir, a.offline) + if (a.background) + return yield* startDetached( + a.port, + dataDir, + a.offline, + Option.getOrUndefined(a.chdbConfigFile), + a.onDirtyStore, + ) yield* Effect.sync(() => process.stderr.write( @@ -325,7 +387,12 @@ export const start = Command.make("start", { }), ) - const { port: boundPort } = yield* startServer({ port: a.port, dataDir, assets }).pipe( + const { port: boundPort } = yield* startServer({ + port: a.port, + dataDir, + configFile: Option.getOrUndefined(a.chdbConfigFile), + assets, + }).pipe( Effect.mapError((e) => new ServerError({ message: `failed to start: ${e.message}` })), ) started = true @@ -408,7 +475,7 @@ export const stop = Command.make("stop", { dataDir: dataDirFlag }).pipe( export const reset = Command.make("reset", { dataDir: dataDirFlag, yes: yesFlag }).pipe( Command.withDescription( - "Delete the local chDB store (~/.maple/data) so the next `maple start` bootstraps fresh", + "Delete live chDB data while preserving checkpoints so the next start bootstraps fresh", ), Command.withHandler( Effect.fnUntraced(function* (a) { @@ -427,18 +494,92 @@ export const reset = Command.make("reset", { dataDir: dataDirFlag, yes: yesFlag if (!a.yes) { yield* Effect.sync(() => process.stderr.write( - `This permanently deletes the local store at ${bold(prettyPath(dataDir))}.\n` + + `This permanently deletes live telemetry at ${bold(prettyPath(dataDir))}.\n` + + `The checkpoint registry under its backups directory is preserved.\n` + `Re-run with ${bold("maple reset --yes")} to confirm.\n`, ), ) return } - yield* fs.remove(dataDir, { recursive: true, force: true }).pipe(Effect.ignore) - yield* fs.remove(storeMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) - yield* fs.remove(storeOpenMarkerPath(dataDir), { force: true }).pipe(Effect.ignore) + yield* resetLiveStorePreservingCheckpoints(dataDir).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) yield* Effect.sync(() => - process.stderr.write(`${green("✓")} reset — removed ${prettyPath(dataDir)}\n`), + process.stderr.write( + `${green("✓")} reset — cleared live data and preserved checkpoints at ${prettyPath(dataDir)}\n`, + ), + ) + }), + ), +) + +export const checkpoint = Command.make("checkpoint", { dataDir: dataDirFlag, port }).pipe( + Command.withDescription("Create and validate a restorable checkpoint of the local chDB store"), + Command.withHandler( + Effect.fnUntraced(function* (a) { + const dataDir = Option.getOrUndefined(a.dataDir) ?? defaultDataDir() + const result = yield* createCheckpoint({ dataDir, port: a.port }).pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) + yield* Effect.sync(() => + process.stdout.write( + `${green("✓")} checkpoint created\n` + + ` ${dim("id")} ${result.checkpointId}\n` + + ` ${dim("path")} ${prettyPath(result.path)}\n` + + ` ${dim("traces")} ${result.manifest.validation.traces}\n` + + ` ${dim("logs")} ${result.manifest.validation.logs}\n` + + ` ${dim("metrics")} ${result.manifest.validation.metricsSum}\n` + + ` ${dim("views")} ${result.manifest.validation.materializedViews}\n`, + ), + ) + }), + ), +) + +export const restore = Command.make("restore", { + dataDir: dataDirFlag, + checkpointId: checkpointIdFlag, + yes: yesFlag, +}).pipe( + Command.withDescription("Restore the local chDB store from the last promoted checkpoint"), + Command.withHandler( + Effect.fnUntraced(function* (a) { + const fs = yield* FileSystem + const dataDir = Option.getOrUndefined(a.dataDir) ?? defaultDataDir() + + const pidOpt = yield* readPid(fs, pidFilePath(dataDir)) + if (Option.isSome(pidOpt) && isProcessAlive(pidOpt.value)) { + return yield* new ServerError({ + message: `maple is running (PID ${pidOpt.value}) — stop it first with \`maple stop\``, + }) + } + + if (!a.yes) { + yield* Effect.sync(() => + process.stderr.write( + `This replaces the local store at ${bold(prettyPath(dataDir))} with the last checkpoint.\n` + + `The existing store is moved aside for quarantine, not deleted.\n` + + `Re-run with ${bold("maple restore --yes")} to confirm.\n`, + ), + ) + return + } + + const checkpointId = Option.getOrUndefined(a.checkpointId) + const result = yield* restoreCheckpoint(dataDir, checkpointId ?? "current").pipe( + Effect.mapError((e) => new ServerError({ message: e.message })), + ) + yield* Effect.sync(() => + process.stderr.write( + `${green("✓")} restored checkpoint\n` + + ` ${dim("id")} ${result.checkpointId}\n` + + ` ${dim("quarantine")} ${prettyPath(result.quarantinePath)}\n` + + ` ${dim("traces")} ${result.validation.traces}\n` + + ` ${dim("logs")} ${result.validation.logs}\n` + + ` ${dim("metrics")} ${result.validation.metricsSum}\n` + + ` ${dim("views")} ${result.validation.materializedViews}\n`, + ), ) }), ), diff --git a/apps/cli/src/lib/style.ts b/apps/cli/src/lib/style.ts index 2d15c296f..a1052a5d6 100644 --- a/apps/cli/src/lib/style.ts +++ b/apps/cli/src/lib/style.ts @@ -13,6 +13,7 @@ export const dim = wrap(2, 22) export const underline = wrap(4, 24) export const green = wrap(32, 39) export const cyan = wrap(36, 39) +export const red = wrap(31, 39) export const gray = wrap(90, 39) /** Maple's amber/leaf accent (256-color). */ export const amber = (s: string): string => (useColor ? `\x1b[38;5;208m${s}\x1b[39m` : s) diff --git a/apps/cli/src/server/archives/calibrate.ts b/apps/cli/src/server/archives/calibrate.ts new file mode 100644 index 000000000..455d1f18e --- /dev/null +++ b/apps/cli/src/server/archives/calibrate.ts @@ -0,0 +1,745 @@ +// Archive calibration measurement engine. +// +// Calibration runs a bounded matrix of Parquet writer/shard candidates against a +// pinned checkpoint restored into sacrificial scratch, measuring real metrics +// (peak RSS via external `/usr/bin/time`, logical/physical bytes, wall time, +// write throughput, peak temporary disk) for each candidate. It selects the +// best candidate that fits the operator's declared ceilings WITH a safety +// margin applied INSIDE the ceiling, validates the selection on a held-out +// sample, and emits a versioned configuration document. +// +// The pure selection/aggregation/comparison core (selectCandidates, +// aggregateSignalResults, comparePredictedObserved) has no I/O and is unit- +// tested directly. The child-spawning, `/usr/bin/time` parsing, watchdog, and +// owned-sample execution live in the CLI layer (archive.ts); this module +// defines the types, the budget semantics, and the immutable document writer. +// +// CONTRACT (MAPLE-CHECKPOINT-ARCHIVE-PLAN.md "Calibration Acceptance Contract"): +// - The operator supplies performance goals; the calibrator never redefines +// them to make a candidate pass. +// - No configuration is emitted unless a held-out pass succeeds across the +// required signals. Insufficient held-out data yields a low-confidence +// REPORT with no config, never a config. +// - A candidate exceeding a declared budget is REJECTED (not "low confidence"); +// the next eligible candidate is tried. "low confidence" means small or +// unrepresentative data only. +// - The config document is immutable after --write-config; a later real-trial +// comparison emits a SEPARATE validation report binding the immutable config +// SHA, not a rewritten config. +// +// True external-volume, deployment-scale calibration under the deployment +// chDB/user/filesystem is a Phase 3 dependency (D-017). Phase 2 proves the +// mechanism and contract compliance in-process + native smoke. + +import { writeFileSync } from "node:fs" +import { userInfo, platform, arch, cpus, totalmem } from "node:os" +import { resolve } from "node:path" +import { + type ArchiveTuning, + CALIBRATION_HELD_OUT_TOLERANCES, + CALIBRATION_RECALIBRATION_TRIGGERS, + HELD_OUT_SAMPLE_MULTIPLIER as HELD_OUT_SAMPLE_MULTIPLIER_FROM_CONFIG, + heldOutSampleRows as heldOutSampleRowsFromConfig, + resolveArchiveTuning, + tuningRecord, + type ArchiveTuningOverrides, + TUNING_CONFIG_FORMAT_VERSION, +} from "./config" + +/** A candidate writer/shard configuration evaluated by the calibrator. */ +export interface CalibrationCandidate { + readonly writerThreads: number + readonly rowGroupRows: number + readonly maxShardRows: number + readonly maxShardBytes: number +} + +export const isSameCalibrationCandidate = ( + left: CalibrationCandidate, + right: CalibrationCandidate, +): boolean => + left.writerThreads === right.writerThreads && + left.rowGroupRows === right.rowGroupRows && + left.maxShardRows === right.maxShardRows && + left.maxShardBytes === right.maxShardBytes + +/** + * The operator-declared performance ceilings. A candidate passes only if its + * observed metrics — multiplied by the safety margin for RSS, throughput, and + * temporary disk — stay within every applicable ceiling. The margin is applied + * INSIDE the ceiling so the recommended config has headroom under the declared + * budget, not merely at its edge. + */ +export interface CalibrationBudget { + /** Maximum peak RSS in bytes allowed for any candidate (before margin). */ + readonly memoryBudget: number + /** Maximum wall-clock milliseconds for the full candidate matrix (total deadline). */ + readonly timeBudget: number + /** Rows to sample per candidate (deterministic part/offset cap). */ + readonly sampleRows: number + /** Maximum wall-clock milliseconds for a single candidate run. */ + readonly maxCandidateWallMs: number + /** Minimum logical write throughput (bytes/sec) required. */ + readonly minThroughputBytesPerSec: number + /** Maximum peak temporary disk (restored scratch + sample output) in bytes. */ + readonly maxTempDiskBytes: number + /** Minimum free-space reserve on the archive volume in bytes. */ + readonly freeSpaceReserve: number + /** + * Safety margin multiplier applied inside each ceiling: a candidate passes + * only if `observed * margin <= ceiling` (RSS, temp disk) and + * `observed / margin >= floor` (throughput). E.g. 1.1 reserves 10% headroom. + */ + readonly safetyMargin: number +} + +const POSITIVE_SAFE_INTEGER_BUDGET_FIELDS = [ + "memoryBudget", + "timeBudget", + "sampleRows", + "maxCandidateWallMs", + "maxTempDiskBytes", + "freeSpaceReserve", +] as const satisfies readonly (keyof CalibrationBudget)[] + +/** Validate every operator-controlled calibration budget value before any + * checkpoint pin, calibration session, child process, or filesystem I/O. */ +export const validateCalibrationBudget = (budget: CalibrationBudget): CalibrationBudget => { + for (const field of POSITIVE_SAFE_INTEGER_BUDGET_FIELDS) { + const value = budget[field] + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`calibration ${field} must be a positive safe integer: ${value}`) + } + } + if (!Number.isSafeInteger(budget.minThroughputBytesPerSec) || budget.minThroughputBytesPerSec < 0) { + throw new Error( + `calibration minThroughputBytesPerSec must be a non-negative safe integer: ${budget.minThroughputBytesPerSec}`, + ) + } + if (!Number.isFinite(budget.safetyMargin) || budget.safetyMargin < 1) { + throw new Error(`calibration safetyMargin must be a finite number at least 1: ${budget.safetyMargin}`) + } + return budget +} + +/** + * Precisely-defined metrics measured for one candidate on one signal. All names + * are used consistently throughout calibration, the config document, and the + * validation report. + */ +export interface CandidateMetrics { + /** Sum of shard `total_uncompressed_size` (logical, pre-compression). */ + readonly logicalBytes: number + /** Sum of shard on-disk (compressed) file sizes. */ + readonly physicalBytes: number + /** physicalBytes / logicalBytes (0 when logicalBytes is 0). */ + readonly compressionRatio: number + /** logicalBytes per wall-clock second of the write. */ + readonly writeThroughputBytesPerSec: number + /** Peak temporary disk: restored-scratch size + sample-output size, high-water. */ + readonly peakTempDiskBytes: number + /** Peak RSS in bytes, measured externally by `/usr/bin/time`. */ + readonly peakRssBytes: number + /** Wall-clock milliseconds of the candidate run. */ + readonly wallMs: number + /** Number of matching source rows in the exported sample. */ + readonly rowCount: number +} + +/** + * The role of a calibration sample within the disjoint training/held-out split. + * Training covers ordered rows `[0, trainingRows)`; held-out covers the strictly + * larger, disjoint window `[trainingRows, trainingRows + heldOutRows)`. + */ +export type CalibrationSampleRole = "training" | "held-out" + +/** + * The exact ordered-row scope one sample covered, recorded in every result so + * the config loader can prove every training and held-out sample came from one + * immutable checkpoint/range and that the two windows are disjoint and correctly + * sized. `startRow`/`requestedRows` are the inputs to `planCalibrationShards`; + * `rowCount` is the exact matching-row count the writer exported (which the + * writer also asserts equals `metrics.rowCount`). + */ +export interface CalibrationSampleScope { + readonly checkpointId: string + readonly checkpointManifestFingerprint: string + readonly rangeDate: string + readonly role: CalibrationSampleRole + /** 0-indexed start in the day's ordered (hour, part, `_part_offset`) sequence. */ + readonly startRow: number + /** The window size requested from `planCalibrationShards`. */ + readonly requestedRows: number + /** The exact matching-row count the writer exported in this window. */ + readonly rowCount: number +} + +/** A candidate's measured result for one signal, or a failure. */ +export interface CandidateResult { + readonly candidate: CalibrationCandidate + readonly signal: string + readonly metrics: CandidateMetrics | null + readonly ok: boolean + /** The exact sample scope; present iff ok (failures export nothing). */ + readonly sample?: CalibrationSampleScope + readonly error?: string +} + +/** + * Check whether a single candidate result's metrics fit every applicable + * ceiling, with the safety margin applied inside the ceiling. Pure. + * + * - RSS: `metrics.peakRssBytes * margin <= budget.memoryBudget` + * - wall: `metrics.wallMs <= budget.maxCandidateWallMs` + * - tput: `metrics.writeThroughputBytesPerSec / margin >= budget.minThroughputBytesPerSec` + * - disk: `metrics.peakTempDiskBytes * margin <= budget.maxTempDiskBytes` + * + * A failed result (ok=false or null metrics) never passes. + */ +export const meetsCeilings = (result: CandidateResult, budget: CalibrationBudget): boolean => { + if (!result.ok || result.metrics === null) return false + const m = result.metrics + const g = budget.safetyMargin + if (m.peakRssBytes * g > budget.memoryBudget) return false + if (m.wallMs > budget.maxCandidateWallMs) return false + if ( + budget.minThroughputBytesPerSec > 0 && + m.writeThroughputBytesPerSec / g < budget.minThroughputBytesPerSec + ) + return false + if (m.peakTempDiskBytes * g > budget.maxTempDiskBytes) return false + return true +} + +/** + * From per-signal results for one candidate, select the ordered list of + * candidates whose WORST-CASE result across ALL SIX signals meets every + * ceiling. The worst case is taken per-metric across signals (so arrays, maps, + * wide logs, and high-cardinality signals all weigh in). Ordered by worst-case + * peak RSS then worst-case wall time. Pure, no I/O. + * + * Requires EXACTLY one result per required signal (`requiredSignals`): + * duplicates, missing, or extra signal names make a candidate ineligible, and + * an empty result set is never eligible. This prevents a partial matrix (e.g. + * truncated by the total deadline) or a duplicate from claiming an all-six pass. + * + * Returns the eligible candidates best-first. An empty list means no candidate + * met the declared goals across all signals — calibration must fail without a + * recommendation in that case. + */ +export const selectCandidates = ( + perSignal: ReadonlyMap>, + budget: CalibrationBudget, + requiredSignals: ReadonlyArray, +): ReadonlyArray<{ candidate: CalibrationCandidate; worstCase: CandidateMetrics }> => { + const required = new Set(requiredSignals) + const eligible: { candidate: CalibrationCandidate; worstCase: CandidateMetrics }[] = [] + for (const [candidate, results] of perSignal) { + // Require exactly one result per required signal; reject duplicates/missing/extra. + const seen = new Set() + let complete = results.length === required.size + for (const r of results) { + if (!required.has(r.signal)) { + complete = false + break + } + if (seen.has(r.signal)) { + complete = false // duplicate + break + } + seen.add(r.signal) + } + if (!complete || seen.size !== required.size) continue + // Every signal's result must meet the ceilings. + const allMeet = results.every((r) => meetsCeilings(r, budget)) + if (!allMeet) continue + const worstCase = worstCaseMetrics(results) + eligible.push({ candidate, worstCase }) + } + // Best-first: lowest worst-case peak RSS, then lowest worst-case wall time. + return eligible + .slice() + .sort( + (a, b) => + a.worstCase.peakRssBytes - b.worstCase.peakRssBytes || + a.worstCase.wallMs - b.worstCase.wallMs, + ) +} + +/** + * Compute the worst-case metrics across a candidate's per-signal results: the + * MAXIMUM of each cost metric (RSS, wall, bytes, temp-disk) and the MINIMUM of + * `writeThroughputBytesPerSec` (the floor is the worst case for a throughput + * floor). Used so selection accounts for the heaviest AND slowest signal. Pure. + */ +export const worstCaseMetrics = (results: ReadonlyArray): CandidateMetrics => { + const ok = results.filter( + (r): r is CandidateResult & { metrics: CandidateMetrics } => r.ok && r.metrics !== null, + ) + if (ok.length === 0) { + return { + logicalBytes: 0, + physicalBytes: 0, + compressionRatio: 0, + writeThroughputBytesPerSec: 0, + peakTempDiskBytes: 0, + peakRssBytes: 0, + wallMs: 0, + rowCount: 0, + } + } + const max = (sel: (m: CandidateMetrics) => number): number => Math.max(...ok.map((r) => sel(r.metrics))) + // Throughput's worst case for a FLOOR is the minimum across signals (the + // slowest signal). All cost metrics (RSS, wall, bytes, temp-disk) take the max. + const min = (sel: (m: CandidateMetrics) => number): number => Math.min(...ok.map((r) => sel(r.metrics))) + return { + logicalBytes: max((m) => m.logicalBytes), + physicalBytes: max((m) => m.physicalBytes), + compressionRatio: max((m) => m.compressionRatio), + writeThroughputBytesPerSec: min((m) => m.writeThroughputBytesPerSec), + peakTempDiskBytes: max((m) => m.peakTempDiskBytes), + peakRssBytes: max((m) => m.peakRssBytes), + wallMs: max((m) => m.wallMs), + rowCount: max((m) => m.rowCount), + } +} + +/** A per-metric predicted-vs-observed comparison with a documented tolerance. */ +export interface MetricComparison { + readonly metric: + | "peakRssBytes" + | "wallMs" + | "writeThroughputBytesPerSec" + | "compressionRatio" + | "physicalBytes" + | "peakTempDiskBytes" + readonly predicted: number + readonly observed: number + /** Allowed relative deviation: |observed - predicted| / predicted <= tolerance. */ + readonly tolerance: number + readonly withinTolerance: boolean + readonly relativeDelta: number +} + +/** + * One signal's like-for-like held-out comparison. The paired raw metrics are NOT + * duplicated here — they live in the training and held-out results, and the + * loader re-derives them by exact candidate + signal identity. `scaleRatio` is + * that signal's own heldOut.logicalBytes / training.logicalBytes (used to rescale + * wallMs/physicalBytes, which are size-proportional); throughput and + * compressionRatio are compared directly; RSS and temp-disk peaks are absolute. + * `passed` requires all six metrics within tolerance for THIS signal. + */ +export interface SignalComparison { + readonly signal: string + readonly scaleRatio: number + readonly comparisons: ReadonlyArray + readonly passed: boolean +} + +/** + * Compare predicted (from calibration training) and observed (from a real or + * held-out trial) metrics within documented per-metric tolerances. Returns one + * entry per metric plus an overall pass/fail. Pure. Throughput is directional + * (higher is better), so it passes when observed >= predicted * (1 - tolerance); + * all other metrics pass when observed is within `tolerance` of predicted. + * + * When the held-out sample is larger than training, absolute size-proportional + * metrics (wallMs, physicalBytes) do not compare cleanly: a 2×-larger held-out + * naturally takes ~2× the wall time and bytes. `sizeScaling` rescales the + * training prediction for those metrics by `ratio` (= heldOut.logicalBytes / + * training.logicalBytes) before the upper-bound check. Every resource cost + * (RSS, wall time, bytes, compression ratio, and temp disk) is directional: + * lower held-out cost is safe, while only a regression beyond tolerance fails. + * Throughput is directional in the opposite sense: higher is safe. The + * recorded `predicted` is the adjusted value, and `scaleRatio` is returned so + * the document is fully auditable. + */ +export const comparePredictedObserved = ( + predicted: CandidateMetrics, + observed: CandidateMetrics, + tolerance: { + peakRssBytes: number + wallMs: number + writeThroughputBytesPerSec: number + compressionRatio: number + physicalBytes: number + peakTempDiskBytes: number + }, + sizeScaling?: { ratio: number; metrics: ReadonlySet<"wallMs" | "physicalBytes"> }, +): { comparisons: ReadonlyArray; passed: boolean; scaleRatio: number } => { + const ratio = sizeScaling?.ratio ?? 1 + const scale = (metric: "wallMs" | "physicalBytes", value: number): number => + sizeScaling && sizeScaling.metrics.has(metric) && Number.isFinite(ratio) ? value * ratio : value + const cost = (metric: MetricComparison["metric"], p: number, o: number, t: number): MetricComparison => { + // Lower resource use is not a model failure. Reject only a cost regression + // beyond tolerance; a larger held-out sample can amortize fixed overhead. + const rel = p > 0 ? Math.max(0, (o - p) / p) : o === 0 ? 0 : Number.POSITIVE_INFINITY + return { + metric, + predicted: p, + observed: o, + tolerance: t, + withinTolerance: o <= p * (1 + t), + relativeDelta: rel, + } + } + const throughput = ( + metric: MetricComparison["metric"], + p: number, + o: number, + t: number, + ): MetricComparison => { + // Higher is better: pass when o >= p * (1 - t). + const rel = p > 0 ? Math.max(0, (p - o) / p) : 0 + return { + metric, + predicted: p, + observed: o, + tolerance: t, + withinTolerance: o >= p * (1 - t), + relativeDelta: rel, + } + } + const comparisons: MetricComparison[] = [ + cost("peakRssBytes", predicted.peakRssBytes, observed.peakRssBytes, tolerance.peakRssBytes), + cost("wallMs", scale("wallMs", predicted.wallMs), observed.wallMs, tolerance.wallMs), + throughput( + "writeThroughputBytesPerSec", + predicted.writeThroughputBytesPerSec, + observed.writeThroughputBytesPerSec, + tolerance.writeThroughputBytesPerSec, + ), + cost( + "compressionRatio", + predicted.compressionRatio, + observed.compressionRatio, + tolerance.compressionRatio, + ), + cost( + "physicalBytes", + scale("physicalBytes", predicted.physicalBytes), + observed.physicalBytes, + tolerance.physicalBytes, + ), + cost( + "peakTempDiskBytes", + predicted.peakTempDiskBytes, + observed.peakTempDiskBytes, + tolerance.peakTempDiskBytes, + ), + ] + return { comparisons, passed: comparisons.every((c) => c.withinTolerance), scaleRatio: ratio } +} + +/** + * Like-for-like, PER-SIGNAL held-out comparison. For each signal (in canonical + * order), pair that signal's held-out result with the same candidate's training + * result, scale wallMs/physicalBytes by THAT signal's own + * heldOut.logicalBytes/training.logicalBytes ratio, compare throughput and + * compressionRatio directly, compare RSS/temp-disk peaks absolutely, and require + * all six metrics within tolerance for that signal. The attempt passes only when + * every signal passes — cross-signal aggregate extrema never decide acceptance. + * + * Returns `null` (incomplete) when a signal is unpaired, when either paired + * metric set is missing, or when training/held-out logicalBytes is not strictly + * positive (the ratio would be undefined; never silently substitute 1). Pure. + */ +export const compareHeldOutPerSignal = ( + training: ReadonlyArray, + heldOut: ReadonlyArray, + requiredSignals: ReadonlyArray, + candidate: CalibrationCandidate, + tolerances: typeof HELD_OUT_TOLERANCES, + scaledMetrics: ReadonlySet<"wallMs" | "physicalBytes"> = new Set(["wallMs", "physicalBytes"]), +): { signalComparisons: ReadonlyArray; passed: boolean } | null => { + const signalComparisons: SignalComparison[] = [] + for (const signal of requiredSignals) { + const trainResult = training.find( + (r) => + r.signal === signal && + r.ok && + r.metrics && + isSameCalibrationCandidate(r.candidate, candidate), + ) + const heldResult = heldOut.find( + (r) => + r.signal === signal && + r.ok && + r.metrics && + isSameCalibrationCandidate(r.candidate, candidate), + ) + if (!trainResult?.metrics || !heldResult?.metrics) return null + const trainingLogical = trainResult.metrics.logicalBytes + const heldOutLogical = heldResult.metrics.logicalBytes + if (!(trainingLogical > 0) || !(heldOutLogical > 0)) return null + const scaleRatio = heldOutLogical / trainingLogical + const comparison = comparePredictedObserved(trainResult.metrics, heldResult.metrics, tolerances, { + ratio: scaleRatio, + metrics: scaledMetrics, + }) + signalComparisons.push({ + signal, + scaleRatio, + comparisons: comparison.comparisons, + passed: comparison.passed, + }) + } + return { + signalComparisons, + passed: signalComparisons.every((entry) => entry.passed), + } +} + +/** The measured environment recorded in every calibration document. */ +export interface CalibrationEnvironment { + readonly mapleVersion: string + readonly chdbVersion: string + readonly schemaFingerprint: string + readonly executionUser: string + readonly platform: string + readonly arch: string + readonly cpuModel: string + readonly cpuCount: number + readonly totalMemoryBytes: number + /** The measurement tool used for peak RSS (e.g. "/usr/bin/time"). */ + readonly measurementTool: string + /** Archive filesystem/volume identity (from statfs f_fsid/f_type). */ + readonly archiveVolume: { + readonly fsid: string + readonly type: number + readonly archiveDir: string + } +} + +/** + * Capture the measured environment from the current process/host. Read at + * calibration time; values are not bake-time stable across runs (CPU/RAM may + * change), which is exactly why recalibration is required after hardware + * changes (see recalibrationTriggers). The archive-volume identity ties the + * calibration to the specific filesystem it measured on, so a volume change + * is detectable (a recalibration trigger). + */ +export const captureEnvironment = ( + mapleVersion: string, + chdbVersion: string, + schemaFingerprint: string, + archiveDir: string, + archiveVolume: { fsid: string; type: number }, + measurementTool = "/usr/bin/time", +): CalibrationEnvironment => { + const cpuList = cpus() + return { + mapleVersion, + chdbVersion, + schemaFingerprint, + executionUser: userInfo().username, + platform: platform(), + arch: arch(), + cpuModel: cpuList.length > 0 ? cpuList[0]!.model : "unknown", + cpuCount: cpuList.length, + totalMemoryBytes: totalmem(), + measurementTool, + archiveVolume: { fsid: archiveVolume.fsid, type: archiveVolume.type, archiveDir }, + } +} + +/** + * Conditions under which recalibration is required. Recorded in every config + * document so deployment drift is detectable and operators know when to repeat. + */ +export const RECALIBRATION_TRIGGERS: ReadonlyArray = CALIBRATION_RECALIBRATION_TRIGGERS + +export interface CalibrationRecommendation { + readonly formatVersion: typeof TUNING_CONFIG_FORMAT_VERSION + readonly checkpoint: { + readonly checkpointId: string + readonly manifestFingerprint: string + } + /** The selected candidate, or null if none met the goals on held-out validation. */ + readonly selected: { + readonly candidate: CalibrationCandidate + readonly worstCase: CandidateMetrics + } | null + /** Full per-signal, per-candidate evidence. */ + readonly results: ReadonlyArray + readonly heldOut: { + readonly results: ReadonlyArray + readonly worstCase: CandidateMetrics + /** One like-for-like entry per signal (canonical order); decides acceptance. */ + readonly signalComparisons: ReadonlyArray + readonly passed: true + readonly tolerances: typeof HELD_OUT_TOLERANCES + } | null + /** Every held-out candidate attempted, including rejected attempts. */ + readonly heldOutAttempts: ReadonlyArray<{ + readonly candidate: CalibrationCandidate + readonly results: ReadonlyArray + readonly worstCase: CandidateMetrics | null + /** Six entries when the attempt was complete (even if it failed); [] when incomplete. */ + readonly signalComparisons: ReadonlyArray + readonly passed: boolean + }> + readonly budget: CalibrationBudget + readonly environment: CalibrationEnvironment + /** + * "high" when the held-out pass succeeded across all required signals on + * representative data. "low" only when the hot store is too small or + * unrepresentative for a clean held-out split — NEVER for exceeding a budget. + * `low` is always paired with `selected: null` (no config emitted). + */ + readonly confidence: "high" | "low" + readonly measuredAt: string + readonly note: string +} + +/** + * Convert a calibration recommendation into resolved archive tuning. Used ONLY + * to compute the `effective` block written into the config document; the + * document itself is the authoritative source loaded by `loadTuningConfig`. + */ +export const recommendationToTuning = ( + rec: CalibrationRecommendation, + archiveDir: string, + scratchRoot: string, +): ArchiveTuning => { + const overrides: ArchiveTuningOverrides = + rec.selected !== null + ? { + writerThreads: rec.selected.candidate.writerThreads, + rowGroupRows: rec.selected.candidate.rowGroupRows, + maxShardRows: rec.selected.candidate.maxShardRows, + maxShardBytes: rec.selected.candidate.maxShardBytes, + minFreeSpaceReserve: rec.budget.freeSpaceReserve, + targetChunkBytes: deriveTargetChunkBytes( + rec.selected.candidate.maxShardBytes, + rec.budget.freeSpaceReserve, + ), + archiveDir, + scratchRoot, + } + : { archiveDir, scratchRoot } + return resolveArchiveTuning(overrides) +} + +export const deriveTargetChunkBytes = (maxShardBytes: number, freeSpaceReserve: number): number => { + if (!Number.isSafeInteger(freeSpaceReserve) || freeSpaceReserve <= 0) { + throw new Error(`calibration freeSpaceReserve must be a positive safe integer: ${freeSpaceReserve}`) + } + const fourShards = maxShardBytes * 4 + const reservePlusShard = freeSpaceReserve + maxShardBytes + const derived = Math.max(fourShards, reservePlusShard) + if (!Number.isSafeInteger(derived) || derived <= 0) { + throw new Error( + `calibration targetChunkBytes derivation overflow: maxShardBytes=${maxShardBytes}, reserve=${freeSpaceReserve}`, + ) + } + return derived +} + +export const HELD_OUT_TOLERANCES = CALIBRATION_HELD_OUT_TOLERANCES + +/** + * The held-out window is strictly LARGER than the training window and disjoint + * from it: training covers ordered rows `[0, sampleRows)` and held-out covers + * `[sampleRows, sampleRows + heldOutSampleRows)` where + * `heldOutSampleRows = HELD_OUT_SAMPLE_MULTIPLIER * sampleRows`. A larger + * held-out sample is required by the plan (the validation must not be weaker + * than the training measurement) and makes the recorded disjoint scope + * auditable. Pure. Defined in ./config (single source of truth, no cycle). + */ +export const HELD_OUT_SAMPLE_MULTIPLIER = HELD_OUT_SAMPLE_MULTIPLIER_FROM_CONFIG +export const heldOutSampleRows = heldOutSampleRowsFromConfig + +/** + * Write a versioned calibration config document to `path`. The document is + * IMMUTABLE after this write: it records the selected candidate, full evidence, + * the measured environment, the safety margin, and recalibration triggers. A + * later real-trial comparison does NOT rewrite this file; it emits a separate + * validation report binding this file's SHA-256. Permissions are restrictive + * (0o600) because the document records host/environment details. + */ +export const writeCalibrationConfig = ( + path: string, + rec: CalibrationRecommendation, + tuning: ArchiveTuning, +): void => { + const doc = { + formatVersion: TUNING_CONFIG_FORMAT_VERSION, + measuredAt: rec.measuredAt, + confidence: rec.confidence, + checkpoint: rec.checkpoint, + candidateMatrix: CANDIDATE_MATRIX, + requiredSignals: [ + "logs", + "traces", + "metrics_sum", + "metrics_gauge", + "metrics_histogram", + "metrics_exponential_histogram", + ], + budget: rec.budget, + selected: rec.selected, + heldOut: rec.heldOut, + heldOutAttempts: rec.heldOutAttempts, + samplePolicy: { + trainingRows: rec.budget.sampleRows, + heldOutMultiplier: HELD_OUT_SAMPLE_MULTIPLIER, + heldOutRows: heldOutSampleRows(rec.budget.sampleRows), + trainingWindow: `[0, ${rec.budget.sampleRows})`, + heldOutWindow: `[${rec.budget.sampleRows}, ${rec.budget.sampleRows + heldOutSampleRows(rec.budget.sampleRows)})`, + }, + environment: rec.environment, + effective: tuningRecord(tuning), + derivation: { + minFreeSpaceReserve: "budget.freeSpaceReserve", + targetChunkBytes: + "max(4 * selected.candidate.maxShardBytes, budget.freeSpaceReserve + selected.candidate.maxShardBytes)", + }, + safetyMargin: rec.budget.safetyMargin, + recalibrationTriggers: RECALIBRATION_TRIGGERS, + results: rec.results, + note: rec.note, + } + writeFileSync(resolve(path), `${JSON.stringify(doc, null, 2)}\n`, { mode: 0o600 }) +} + +/** + * A separate validation report binding an immutable config to a real archive + * trial. Emitted by the native probe / acceptance step, NEVER by rewriting the + * config. Binds the config SHA-256, the trial manifest identity, and the + * predicted-vs-observed evidence with a pass/fail verdict. + */ +export interface CalibrationValidationReport { + readonly formatVersion: 1 + readonly configSha256: string + readonly configName: string + readonly trial: { + readonly generationId: string + readonly signal: string + readonly rangeStart: string + readonly archivedRowCount: number + readonly shardCount: number + } + readonly comparison: { comparisons: ReadonlyArray; passed: boolean } + readonly measuredAt: string +} + +/** Write a calibration validation report (separate from the immutable config). */ +export const writeValidationReport = (path: string, report: CalibrationValidationReport): void => { + writeFileSync(resolve(path), `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 }) +} + +/** Resolve the calibration archive dir, creating it if needed. */ +export const ensureCalibrationArchiveDir = (archiveDir: string): string => { + const abs = resolve(archiveDir) + return abs +} + +/** The fixed candidate matrix evaluated by the calibrator. */ +export const CANDIDATE_MATRIX: ReadonlyArray = [ + { writerThreads: 1, rowGroupRows: 10_000, maxShardRows: 500_000, maxShardBytes: 256 * 1024 * 1024 }, + { writerThreads: 1, rowGroupRows: 5_000, maxShardRows: 250_000, maxShardBytes: 128 * 1024 * 1024 }, + { writerThreads: 2, rowGroupRows: 10_000, maxShardRows: 500_000, maxShardBytes: 256 * 1024 * 1024 }, + { writerThreads: 1, rowGroupRows: 20_000, maxShardRows: 1_000_000, maxShardBytes: 512 * 1024 * 1024 }, +] diff --git a/apps/cli/src/server/archives/calibration-recovery.ts b/apps/cli/src/server/archives/calibration-recovery.ts new file mode 100644 index 000000000..65ed18a91 --- /dev/null +++ b/apps/cli/src/server/archives/calibration-recovery.ts @@ -0,0 +1,615 @@ +// Calibration interruption recovery. +// +// A calibration run acquires a maintenance lock, a checkpoint pin, an owned +// scratch subdir, and an owned sample-output directory. A SIGKILL or crash at +// any point must leave reconcilable state: the NEXT calibration run reconciles +// the prior record and removes ONLY its exact owned paths and pin — all derived +// from the operation identifier, never accepted as arbitrary strings. This is +// the single authoritative reconciler: normal cleanup calls it too, so the +// crash path and the happy path share one proven removal routine. +// +// SAFETY INVARIANTS (every one enforced here, repairing the prior defects): +// - ownedPaths are DERIVED from operationId (scratchSubdir=`calibrate-`, +// sampleDir=`/calibration/samples/`); a record claiming any +// other paths is rejected. This was a recursive-deletion primitive before. +// - the recovery record is read/written through the archive no-symlink +// classifier + real-file checks; a planted recovery.json symlink is refused. +// - the pin is DERIVED from the recorded pinId via pinFilePath(), so a crash +// BETWEEN pin creation and the phase advance (pinPath null in the record) +// still releases the exact pin. The recorded pinPath is validated against +// the derived path; the purpose is operation-specific. +// - reconcile runs INSIDE the maintenance lock (the caller enforces this) and +// clears the record ONLY after every exact-owned resource is confirmed +// absent; a real release/removal failure PRESERVES the record for retry. +// - the checkpoint fingerprint is recorded and validated on reconcile. + +import { lstatSync, readFileSync, realpathSync, statSync } from "node:fs" +import { rm, statfs } from "node:fs/promises" +import { dirname, isAbsolute, resolve } from "node:path" +import { durableJson, durableRemove } from "../durable-files" +import { + assertCheckpointPinIdentity, + checkpointRoot, + checkpointSnapshotDir, + pinFilePath, + releaseCheckpointPin, + resolveCheckpoint, +} from "../checkpoints" +import { assertNoSymlinkSync, assertRealFileSync, classifyArchivePathSync } from "./paths" + +/** The calibration recovery record format version. */ +export const CALIBRATION_RECOVERY_FORMAT_VERSION = 1 + +/** The lifecycle phases a calibration run advances through. */ +export type CalibrationPhase = + | "intent" + | "pin-acquired" + | "scratch-allocated" + | "sampling" + | "validating" + | "cleanup" + | "complete" + +/** + * The durable ownership record for one calibration run. Written before any + * allocation; advanced per phase. The next run reconciles a prior record by + * releasing exactly the derived pin and removing exactly the derived owned + * paths. + */ +export interface CalibrationRecoveryRecord { + readonly formatVersion: typeof CALIBRATION_RECOVERY_FORMAT_VERSION + readonly phase: CalibrationPhase + readonly operationId: string + readonly pinId: string + readonly pinPurpose: string + /** The pin file path returned by acquireCheckpointPin; null until acquired. */ + readonly pinPath: string | null + readonly checkpointId: string + readonly checkpointManifestFingerprint: string + readonly boundRoots: { + readonly dataDir: string + readonly archiveDir: string + readonly scratchRoot: string + } + readonly ownedPaths: { + readonly scratchSubdir: string + readonly sampleDir: string + } + readonly updatedAt: string +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +/** The recovery record path beneath the archive root. */ +export const calibrationRecoveryPath = (archiveDir: string): string => + resolve(archiveDir, "calibration", "recovery.json") + +/** + * DERIVE the exact owned scratch subdir from the operation id. A recovery record + * may only own `calibrate-` beneath the scratch root — never an + * arbitrary path. This is the ownership binding that was missing. + */ +export const derivedScratchSubdir = (operationId: string): string => `calibrate-${operationId}` + +/** + * DERIVE the exact owned sample directory from the archive root and operation + * id. A recovery record may only own `/calibration/samples/`. + */ +export const derivedSampleDir = (archiveDir: string, operationId: string): string => + resolve(archiveDir, "calibration", "samples", operationId) + +/** + * DERIVE the exact pin file path from the recorded pin id, so a crash between + * pin creation and the phase advance still releases the exact pin. + */ +export const derivedPinPath = (dataDir: string, checkpointId: string, pinId: string): string => + pinFilePath(dataDir, checkpointId, pinId) + +/** Operation-specific pin purpose, validated on release. */ +export const calibrationPinPurpose = (operationId: string): string => `archive-calibrate:${operationId}` + +/** + * Strictly parse a recovery record, binding it to the expected roots AND + * deriving/validating the owned paths from the operation id. Rejects: + * - unknown format version / phase; + * - bound roots that don't resolve exactly to the expected roots (foreign record); + * - owned paths that are NOT the derived `calibrate-` / `samples/`; + * - a recorded pinPath that doesn't match the derived pin path; + * - a checkpoint fingerprint mismatch. + * + * Returns the parsed record plus the derived exact paths the reconciler removes. + */ +export const parseCalibrationRecoveryRecord = ( + value: unknown, + expectedRoots: { dataDir: string; archiveDir: string; scratchRoot: string }, +): CalibrationRecoveryRecord => { + if (!isRecord(value)) throw new Error("malformed calibration recovery record (not a record)") + if (value.formatVersion !== CALIBRATION_RECOVERY_FORMAT_VERSION) { + throw new Error( + `unsupported calibration recovery formatVersion ${String(value.formatVersion)} ` + + `(expected ${CALIBRATION_RECOVERY_FORMAT_VERSION})`, + ) + } + const phase = value.phase + const knownPhases: ReadonlySet = new Set([ + "intent", + "pin-acquired", + "scratch-allocated", + "sampling", + "validating", + "cleanup", + "complete", + ]) + if (typeof phase !== "string" || !knownPhases.has(phase)) { + throw new Error(`invalid calibration recovery phase: ${String(phase)}`) + } + const str = (k: string): string => { + const v = value[k] + if (typeof v !== "string" || v.length === 0) + throw new Error(`invalid calibration recovery field: ${k}`) + return v + } + const operationId = str("operationId") + const pinId = str("pinId") + const pinPurpose = str("pinPurpose") + if (pinPurpose !== calibrationPinPurpose(operationId)) { + throw new Error( + `calibration recovery pinPurpose mismatch: expected ${calibrationPinPurpose(operationId)}, got ${pinPurpose}`, + ) + } + const checkpointId = str("checkpointId") + const checkpointFingerprint = str("checkpointManifestFingerprint") + const rootsRaw = value.boundRoots + if (!isRecord(rootsRaw)) throw new Error("invalid calibration recovery field: boundRoots") + const dataDir = typeof rootsRaw.dataDir === "string" ? rootsRaw.dataDir : "" + const archiveDir = typeof rootsRaw.archiveDir === "string" ? rootsRaw.archiveDir : "" + const scratchRoot = typeof rootsRaw.scratchRoot === "string" ? rootsRaw.scratchRoot : "" + if (!dataDir || !archiveDir || !scratchRoot) { + throw new Error("invalid calibration recovery boundRoots (all roots required)") + } + if (resolve(dataDir) !== resolve(expectedRoots.dataDir)) { + throw new Error("calibration recovery dataDir mismatch; refusing to reconcile foreign record") + } + if (resolve(archiveDir) !== resolve(expectedRoots.archiveDir)) { + throw new Error("calibration recovery archiveDir mismatch; refusing to reconcile foreign record") + } + if (resolve(scratchRoot) !== resolve(expectedRoots.scratchRoot)) { + throw new Error("calibration recovery scratchRoot mismatch; refusing to reconcile foreign record") + } + // DERIVE the expected owned paths and require the record to match exactly. + const expectedScratch = derivedScratchSubdir(operationId) + const expectedSample = derivedSampleDir(archiveDir, operationId) + const ownedRaw = value.ownedPaths + if (!isRecord(ownedRaw)) throw new Error("invalid calibration recovery field: ownedPaths") + const ownedScratch = typeof ownedRaw.scratchSubdir === "string" ? ownedRaw.scratchSubdir : "" + const ownedSample = typeof ownedRaw.sampleDir === "string" ? ownedRaw.sampleDir : "" + if (ownedScratch !== expectedScratch) { + throw new Error( + `calibration recovery scratchSubdir '${ownedScratch}' != derived '${expectedScratch}'; refusing`, + ) + } + if (resolve(ownedSample) !== resolve(expectedSample)) { + throw new Error( + `calibration recovery sampleDir '${ownedSample}' != derived '${expectedSample}'; refusing`, + ) + } + // DERIVE the expected pin path from pinId and validate the recorded pinPath. + const expectedPinPath = derivedPinPath(dataDir, checkpointId, pinId) + const recordedPinPath = typeof value.pinPath === "string" ? value.pinPath : null + if (recordedPinPath !== null && resolve(recordedPinPath) !== resolve(expectedPinPath)) { + throw new Error( + `calibration recovery pinPath '${recordedPinPath}' != derived '${expectedPinPath}'; refusing`, + ) + } + return { + formatVersion: CALIBRATION_RECOVERY_FORMAT_VERSION, + phase: phase as CalibrationPhase, + operationId, + pinId, + pinPurpose, + pinPath: recordedPinPath, + checkpointId, + checkpointManifestFingerprint: checkpointFingerprint, + boundRoots: { dataDir, archiveDir, scratchRoot }, + ownedPaths: { scratchSubdir: expectedScratch, sampleDir: expectedSample }, + updatedAt: typeof value.updatedAt === "string" ? value.updatedAt : "", + } +} + +/** + * Read a prior recovery record through the archive no-symlink classifier + real + * file checks (a planted recovery.json symlink is refused). Returns null if no + * record exists. The parent chain beneath the archive root is validated. + */ +export const readPriorCalibrationRecord = ( + archiveDir: string, + expectedRoots: { dataDir: string; archiveDir: string; scratchRoot: string }, +): CalibrationRecoveryRecord | null => { + const path = calibrationRecoveryPath(archiveDir) + const topology = classifyArchivePathSync(archiveDir, path, "calibration recovery record") + if (topology === "absent") return null + if (topology !== "real-file") { + throw new Error( + `calibration recovery record is not a regular file (topology: ${topology}) at ${path}; refusing`, + ) + } + assertNoSymlinkSync(archiveDir, path, "calibration recovery record") + assertRealFileSync(path, "calibration recovery record") + const raw = JSON.parse(readFileSync(path, "utf8")) as unknown + return parseCalibrationRecoveryRecord(raw, expectedRoots) +} + +/** + * Remove a single owned directory only if it is a real (non-symlink) directory + * contained within its bound root. Idempotent: success if already absent. A + * non-directory/symlink at the owned path is a hard failure (the caller + * preserves the recovery record). `classifyArchivePathSync` proves containment + * and rejects symlinked ancestors. + */ +const removeOwnedDir = async (root: string, dir: string, label: string): Promise => { + if (!isAbsolute(dir)) throw new Error(`calibration cleanup refused non-absolute ${label}: ${dir}`) + const topology = classifyArchivePathSync(root, dir, label) + if (topology === "absent") return // already gone — success + if (topology !== "real-directory") { + throw new Error( + `calibration cleanup refused non-directory or symlinked ${label} at ${dir} (topology: ${topology})`, + ) + } + await rm(dir, { recursive: true, force: true }) +} + +/** + * An intent is safe to retire without resolving its source checkpoint only when + * it is provably inert: the record predates pin acquisition, records no pin + * path, and every exact derived resource is absent. This closes the recovery + * wedge where normal checkpoint retention removes the still-unpinned source + * snapshot after a crash at `intent`. + * + * Every classification is symlink-aware and rooted. Any present resource, + * unsafe topology, later phase, or surviving source snapshot keeps the normal + * fingerprint-validation path fail-closed. + */ +const isInertIntentWithRetiredCheckpoint = (prior: CalibrationRecoveryRecord): boolean => { + if (prior.phase !== "intent" || prior.pinPath !== null) return false + const checkpointOwner = checkpointRoot(prior.boundRoots.dataDir) + const snapshot = checkpointSnapshotDir(prior.boundRoots.dataDir, prior.checkpointId) + if (classifyArchivePathSync(checkpointOwner, snapshot, "calibration source checkpoint") !== "absent") { + return false + } + const pinPath = derivedPinPath(prior.boundRoots.dataDir, prior.checkpointId, prior.pinId) + if (classifyArchivePathSync(checkpointOwner, pinPath, "calibration checkpoint pin") !== "absent") { + return false + } + const scratchOwned = resolve(prior.boundRoots.scratchRoot, prior.ownedPaths.scratchSubdir) + if ( + classifyArchivePathSync(prior.boundRoots.scratchRoot, scratchOwned, "calibration scratch subdir") !== + "absent" + ) { + return false + } + if ( + classifyArchivePathSync( + prior.boundRoots.archiveDir, + prior.ownedPaths.sampleDir, + "calibration sample dir", + ) !== "absent" + ) { + return false + } + return true +} + +/** + * Reconcile a prior interrupted calibration run: release its exact DERIVED pin + * and remove its exact DERIVED owned scratch subdir and sample directory, then + * clear the record. The pin is derived from pinId so even an intent-phase crash + * (pinPath null in the record, but the pin was actually created) releases it. + * + * MUST be called inside the maintenance lock (the caller enforces this). + * + * The record is cleared ONLY after every exact-owned resource is confirmed + * absent. A real release/removal failure PRESERVES the record for retry — an + * already-absent resource is success, but a real error does not lose authority. + */ +export const reconcileCalibration = async ( + archiveDir: string, + expectedRoots: { dataDir: string; archiveDir: string; scratchRoot: string }, +): Promise => { + const prior = readPriorCalibrationRecord(archiveDir, expectedRoots) + if (prior === null) return // nothing to reconcile — idempotent no-op + // Validate the checkpoint fingerprint against the LIVE checkpoint manifest, so + // the recorded identity actually binds the reconciled pin to the checkpoint it + // claims (C2). A stale/foreign fingerprint refuses to reconcile (preserve). + let resolved + try { + resolved = await resolveCheckpoint(prior.boundRoots.dataDir, prior.checkpointId) + } catch (error) { + // At intent, no allocation has been authorized yet. If normal checkpoint + // retention removed the still-unpinned source and every exact derived + // resource is provably absent, the recovery record itself is the only + // remaining state and may be retired safely. Any ambiguity preserves it. + if (isInertIntentWithRetiredCheckpoint(prior)) { + await durableRemove(calibrationRecoveryPath(archiveDir)) + return + } + const message = error instanceof Error ? error.message : String(error) + throw new Error( + `calibration reconcile: source checkpoint could not be validated; preserving record: ${message}`, + ) + } + const liveFingerprint = `${resolved.manifest.checkpointId}:${resolved.manifest.createdAt}:${resolved.manifest.backupBytes}` + if (liveFingerprint !== prior.checkpointManifestFingerprint) { + throw new Error( + `calibration reconcile: checkpoint fingerprint mismatch (recorded ${prior.checkpointManifestFingerprint} != live ${liveFingerprint}); preserving record`, + ) + } + // Derive the exact pin path from pinId (works even at intent phase). + const pinPath = derivedPinPath(prior.boundRoots.dataDir, prior.checkpointId, prior.pinId) + let pinReleased = false + try { + await releaseCheckpointPin(prior.boundRoots.dataDir, prior.checkpointId, pinPath, prior.pinPurpose) + pinReleased = true + } catch (error) { + // releaseCheckpointPin fails closed on absence too (over-retention safe), + // so a missing pin is NOT an error here. A genuine identity mismatch, + // however, is — and we preserve the record so the operator can intervene. + const msg = error instanceof Error ? error.message : String(error) + if (!/already absent|already released|not exist|no such|not found/i.test(msg)) { + throw new Error( + `calibration reconcile: pin release FAILED for ${pinPath} (preserving record): ${msg}`, + ) + } + pinReleased = true // absent pin = success + } + // Remove the exact derived owned paths. + const scratchOwned = resolve(prior.boundRoots.scratchRoot, prior.ownedPaths.scratchSubdir) + await removeOwnedDir(prior.boundRoots.scratchRoot, scratchOwned, "scratch subdir") + await removeOwnedDir(prior.boundRoots.archiveDir, prior.ownedPaths.sampleDir, "sample dir") + // Clear the record ONLY after all resources are confirmed absent. + if (!pinReleased) { + throw new Error(`calibration reconcile: pin not confirmed released; preserving record`) + } + await durableRemove(calibrationRecoveryPath(archiveDir)) +} + +/** + * Validate that a child belongs to the live parent-owned calibration session. + * Children never resolve `current`, acquire a replacement pin, or release the + * session pin; they consume only this exact durable checkpoint identity. + */ +export const assertCalibrationSession = async ( + archiveDir: string, + expectedRoots: { dataDir: string; archiveDir: string; scratchRoot: string }, + expected: { + operationId: string + checkpointId: string + checkpointManifestFingerprint: string + }, +): Promise => { + const record = readPriorCalibrationRecord(archiveDir, expectedRoots) + if ( + record === null || + record.operationId !== expected.operationId || + record.phase !== "pin-acquired" || + record.pinPath === null || + record.checkpointId !== expected.checkpointId || + record.checkpointManifestFingerprint !== expected.checkpointManifestFingerprint + ) { + throw new Error("calibration child is not bound to the live parent session; refusing") + } + const pinTopology = classifyArchivePathSync( + checkpointRoot(expectedRoots.dataDir), + derivedPinPath(expectedRoots.dataDir, record.checkpointId, record.pinId), + "calibration session pin", + ) + if (pinTopology !== "real-file") { + throw new Error(`calibration parent session pin is not live (${pinTopology}); refusing child`) + } + // Topology alone is not ownership: a same-path regular file could have been + // substituted. Bind the exact pin id, checkpoint, and operation purpose. + await assertCheckpointPinIdentity( + expectedRoots.dataDir, + record.checkpointId, + derivedPinPath(expectedRoots.dataDir, record.checkpointId, record.pinId), + record.pinPurpose, + ) + return record +} + +/** Remove only the session's derived scratch/sample dirs while retaining its pin and record. */ +export const cleanupCalibrationSample = async (record: CalibrationRecoveryRecord): Promise => { + await removeOwnedDir( + record.boundRoots.scratchRoot, + resolve(record.boundRoots.scratchRoot, record.ownedPaths.scratchSubdir), + "scratch subdir", + ) + await removeOwnedDir(record.boundRoots.archiveDir, record.ownedPaths.sampleDir, "sample dir") +} + +/** + * Write (or advance) the recovery record at the calibration recovery path, + * validating the owned paths are derived from the operation id. Called before + * allocation and at each phase transition so a crash at any point leaves a + * record naming exactly what to release. Writes through the path safety checks. + */ +export const writeCalibrationRecord = async ( + archiveDir: string, + record: Omit, +): Promise => { + // Validate owned paths are derived from operationId before writing. + const expectedScratch = derivedScratchSubdir(record.operationId) + const expectedSample = derivedSampleDir(archiveDir, record.operationId) + if (record.ownedPaths.scratchSubdir !== expectedScratch) { + throw new Error( + `calibration record scratchSubdir '${record.ownedPaths.scratchSubdir}' != derived '${expectedScratch}'`, + ) + } + if (resolve(record.ownedPaths.sampleDir) !== resolve(expectedSample)) { + throw new Error( + `calibration record sampleDir '${record.ownedPaths.sampleDir}' != derived '${expectedSample}'`, + ) + } + if (record.pinPurpose !== calibrationPinPurpose(record.operationId)) { + throw new Error(`calibration record pinPurpose must be operation-specific`) + } + const path = calibrationRecoveryPath(archiveDir) + // Validate the parent chain is symlink-safe before durableJson writes. + assertNoSymlinkSync(archiveDir, path, "calibration recovery record") + await durableJson(path, { + formatVersion: CALIBRATION_RECOVERY_FORMAT_VERSION, + updatedAt: new Date().toISOString(), + ...record, + }) +} + +/** + * Measure the on-disk size of a directory tree (bytes), for peak temporary-disk + * accounting. Returns 0 if the path is absent (ENOENT). Any non-ENOENT read/stat + * error is THROWN (fail-loud) — the caller (the watchdog) treats a measurement + * error as candidate failure, not an undercount. Internal symlinks are followed + * only when their canonical target remains beneath the canonical owned root. + * A visited `(dev, ino)` set prevents symlink cycles and physical double-counts. + * Unknown special entries fail closed. + */ +export const directoryTreeBytes = async (dir: string): Promise => { + const { lstat, readdir, realpath, stat } = await import("node:fs/promises") + const { isAbsolute: pathIsAbsolute, join, relative, sep } = await import("node:path") + const root = resolve(dir) + let rootInfo + try { + rootInfo = await lstat(root) + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code === "ENOENT") return 0 + throw new Error(`directoryTreeBytes: failed to inspect root ${root}: ${code ?? error}`) + } + if (rootInfo.isSymbolicLink() || !rootInfo.isDirectory()) { + throw new Error(`directoryTreeBytes: owned root must be a real directory: ${root}`) + } + const canonicalRoot = await realpath(root) + const visited = new Set() + let total = 0 + + const contained = (target: string): boolean => { + const rel = relative(canonicalRoot, target) + return rel === "" || (!pathIsAbsolute(rel) && rel !== ".." && !rel.startsWith(`..${sep}`)) + } + + const walkResolved = async (p: string, info: Awaited>): Promise => { + const identity = `${info.dev}:${info.ino}` + if (visited.has(identity)) return + visited.add(identity) + if (info.isFile()) { + total += Number(info.size) + return + } + if (!info.isDirectory()) { + throw new Error(`directoryTreeBytes: refusing special entry ${p}`) + } + let entries + try { + entries = await readdir(p, { withFileTypes: true }) + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code === "ENOENT") return + throw new Error(`directoryTreeBytes: failed to read ${p}: ${code ?? error}`) + } + for (const entry of entries) { + await walk(join(p, entry.name)) + } + } + + const walk = async (p: string): Promise => { + let linkInfo + try { + linkInfo = await lstat(p) + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code === "ENOENT") return + throw new Error(`directoryTreeBytes: failed to inspect ${p}: ${code ?? error}`) + } + if (linkInfo.isSymbolicLink()) { + let target + try { + target = await realpath(p) + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code === "ENOENT") return + throw new Error(`directoryTreeBytes: failed to resolve symlink ${p}: ${code ?? error}`) + } + if (!contained(target)) { + throw new Error(`directoryTreeBytes: symlink escapes owned root: ${p} -> ${target}`) + } + await walkResolved(target, await stat(target)) + return + } + await walkResolved(p, linkInfo) + } + + await walk(root) + return total +} + +/** + * Capture the archive-volume identity (device id + filesystem type) for the + * calibration environment record so a volume change is detectable. Uses the + * device id from stat (cross-platform) plus the statfs filesystem type. + */ +export const archiveVolumeIdentity = async (archiveDir: string): Promise<{ fsid: string; type: number }> => { + const statPath = resolve(archiveDir) + const link = lstatSync(statPath) + if (!link.isDirectory() || link.isSymbolicLink()) { + throw new Error( + `archive volume inspection requires an existing real non-symlink directory: ${statPath}`, + ) + } + if (realpathSync(statPath) !== statPath) { + throw new Error(`archive volume inspection requires a canonical archive root: ${statPath}`) + } + const info = await statfs(statPath) + // Use the device id (cross-platform, from statSync) as the volume id, plus + // the statfs filesystem type. Together they identify the volume+filesystem. + const dev = statSync(statPath).dev.toString(16) + return { fsid: `dev:${dev}`, type: info.type } +} + +const existsSyncSafe = (p: string): boolean => { + try { + return statSync(p) !== undefined + } catch { + return false + } +} + +/** + * Free-space preflight for the archive volume, reusing the same statfs idiom as + * the production free-space check. Throws if available free bytes are below the + * required reserve plus estimated working bytes. + */ +export const preflightCalibrationFreeSpace = async ( + archiveDir: string, + freeSpaceReserve: number, + estimatedWorkingBytes: number, +): Promise => { + let statPath = resolve(archiveDir) + let climbs = 0 + while (!existsSyncSafe(statPath) && climbs < 64) { + statPath = dirname(statPath) + climbs++ + } + if (!existsSyncSafe(statPath)) { + throw new Error( + `calibration free-space preflight could not find an existing ancestor of ${archiveDir}`, + ) + } + const info = await statfs(statPath) + const free = info.bavail * info.bsize + const required = freeSpaceReserve + estimatedWorkingBytes + if (free < required) { + throw new Error( + `calibration free-space preflight failed: ${free} bytes free on ${statPath} ` + + `< ${required} required (reserve ${freeSpaceReserve} + working ${estimatedWorkingBytes})`, + ) + } +} diff --git a/apps/cli/src/server/archives/config.ts b/apps/cli/src/server/archives/config.ts new file mode 100644 index 000000000..65dc489a0 --- /dev/null +++ b/apps/cli/src/server/archives/config.ts @@ -0,0 +1,1421 @@ +// Archive tuning configuration. +// +// Every machine-sensitive archive value is centralized, documented, visible in +// command output, overridable through the CLI or a configuration file, and +// recorded in each generation manifest as the effective runtime values. The +// defaults are the measured research baselines, not universal constants: a +// deployment should calibrate its own values against its checkpoint, archive +// volume, chDB version, and memory budget (see the calibrate command). +// +// `loadTuningConfig` reads a versioned calibration config document (emitted by +// the calibrator's `writeCalibrationConfig`) from a single opened file +// descriptor — the bytes read, the SHA-256 identity, and the regular-file +// check all derive from one `open()` so there is no TOCTOU between read and +// hash and no path the archive-root classifier cannot safely validate. +// +// References: MAPLE-CHECKPOINT-ARCHIVE-PLAN.md "Configuration and Calibration" +// and the research-transfer measured starting values. + +import { createHash } from "node:crypto" +import { constants, closeSync, fstatSync, lstatSync, openSync, readSync } from "node:fs" +import { basename } from "node:path" + +/** + * The effective tuning configuration used by one archive generation. All values + * are validated at parse time; an unsafe or contradictory combination is + * rejected before any export runs. `archiveDir` and `scratchRoot` are resolved + * to absolute paths. + */ +export interface ArchiveTuning { + /** ClickHouse Parquet writer thread count (`max_threads`). */ + readonly writerThreads: number + /** Parquet row-group row count (`output_format_parquet_row_group_size`). */ + readonly rowGroupRows: number + /** Maximum rows in one physical Parquet shard before splitting. */ + readonly maxShardRows: number + /** Maximum estimated uncompressed bytes in one physical shard before splitting. */ + readonly maxShardBytes: number + /** Target logical chunk size in bytes (a provisioning hint, not a hard limit). */ + readonly targetChunkBytes: number + /** Minimum free-space reserve required on the archive volume before writing. */ + readonly minFreeSpaceReserve: number + /** Resolved absolute archive root directory. */ + readonly archiveDir: string + /** Resolved absolute scratch root for restored-checkpoint instances. */ + readonly scratchRoot: string +} + +export const DEFAULT_ARCHIVE_TUNING = { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + targetChunkBytes: 1024 * 1024 * 1024, + minFreeSpaceReserve: 512 * 1024 * 1024, +} as const + +/** + * A partial, operator-supplied override. Every field is optional; missing + * fields fall back to {@link DEFAULT_ARCHIVE_TUNING}. This is the shape accepted + * from CLI flags and configuration files. + */ +export interface ArchiveTuningOverrides { + readonly writerThreads?: number + readonly rowGroupRows?: number + readonly maxShardRows?: number + readonly maxShardBytes?: number + readonly targetChunkBytes?: number + readonly minFreeSpaceReserve?: number + readonly archiveDir?: string + readonly scratchRoot?: string +} + +const isPositiveInt = (value: unknown): value is number => + typeof value === "number" && Number.isInteger(value) && value > 0 + +const requirePositiveInt = (value: unknown, key: string): number => { + if (!isPositiveInt(value)) throw new Error(`archive tuning ${key} must be a positive integer`) + return value +} + +/** + * Build an {@link ArchiveTuning} from defaults plus optional overrides, then + * validate the combination. Rejects: + * + * - non-positive or non-integer numeric fields; + * - a row group larger than the max shard (a shard could never hold one row + * group, which would split indefinitely); + * - a max shard byte estimate smaller than a single row group's worst case; + * - a free-space reserve larger than the target chunk (nothing could ever be + * archived under that reserve on a fresh volume of that size); + * - a missing archive or scratch root. + * + * `archiveDir` and `scratchRoot` must be supplied (defaults are resolved by the + * CLI layer from the deployment's configured paths); this parser does not + * invent them. + */ +export const resolveArchiveTuning = (overrides: ArchiveTuningOverrides): ArchiveTuning => { + const writerThreads = requirePositiveInt( + overrides.writerThreads ?? DEFAULT_ARCHIVE_TUNING.writerThreads, + "writerThreads", + ) + const rowGroupRows = requirePositiveInt( + overrides.rowGroupRows ?? DEFAULT_ARCHIVE_TUNING.rowGroupRows, + "rowGroupRows", + ) + const maxShardRows = requirePositiveInt( + overrides.maxShardRows ?? DEFAULT_ARCHIVE_TUNING.maxShardRows, + "maxShardRows", + ) + const maxShardBytes = requirePositiveInt( + overrides.maxShardBytes ?? DEFAULT_ARCHIVE_TUNING.maxShardBytes, + "maxShardBytes", + ) + const targetChunkBytes = requirePositiveInt( + overrides.targetChunkBytes ?? DEFAULT_ARCHIVE_TUNING.targetChunkBytes, + "targetChunkBytes", + ) + const minFreeSpaceReserve = requirePositiveInt( + overrides.minFreeSpaceReserve ?? DEFAULT_ARCHIVE_TUNING.minFreeSpaceReserve, + "minFreeSpaceReserve", + ) + if (!overrides.archiveDir) throw new Error("archive tuning requires an archive directory") + if (!overrides.scratchRoot) throw new Error("archive tuning requires a scratch root") + if (rowGroupRows > maxShardRows) { + throw new Error("archive tuning rowGroupRows must not exceed maxShardRows") + } + // A single row group at the broadest type should fit within a shard's byte + // budget; otherwise a shard of one row group could already exceed it. + const minShardBytesForRowGroup = rowGroupRows * 1024 + if (maxShardBytes < minShardBytesForRowGroup) { + throw new Error( + `archive tuning maxShardBytes (${maxShardBytes}) is too small for rowGroupRows ` + + `(${rowGroupRows}); raise maxShardBytes or lower rowGroupRows`, + ) + } + if (minFreeSpaceReserve >= targetChunkBytes) { + throw new Error("archive tuning minFreeSpaceReserve must be smaller than targetChunkBytes") + } + if (writerThreads > 32) { + throw new Error("archive tuning writerThreads must not exceed 32") + } + return { + writerThreads, + rowGroupRows, + maxShardRows, + maxShardBytes, + targetChunkBytes, + minFreeSpaceReserve, + archiveDir: overrides.archiveDir, + scratchRoot: overrides.scratchRoot, + } +} + +/** + * The tuning-config identity recorded in a manifest so a generation is + * reproducible and deployment drift is visible. Includes both the configured + * defaults and the effective runtime values used to write the generation. + */ +export interface ArchiveTuningRecord { + readonly writerThreads: number + readonly rowGroupRows: number + readonly maxShardRows: number + readonly maxShardBytes: number + readonly targetChunkBytes: number + readonly minFreeSpaceReserve: number +} + +export const tuningRecord = (tuning: ArchiveTuning): ArchiveTuningRecord => ({ + writerThreads: tuning.writerThreads, + rowGroupRows: tuning.rowGroupRows, + maxShardRows: tuning.maxShardRows, + maxShardBytes: tuning.maxShardBytes, + targetChunkBytes: tuning.targetChunkBytes, + minFreeSpaceReserve: tuning.minFreeSpaceReserve, +}) + +/** + * The structured identity of a loaded calibration config document, recorded in + * a generation manifest so the exact config that produced a generation is + * reproducible. Replaces the prior bare `tuningConfigName: string | null` with + * a versioned, SHA-256-bound identity. An unknown `formatVersion` fails closed. + */ +export interface TuningConfigIdentity { + readonly formatVersion: number + /** A safe logical name derived from the config file's basename (no path). */ + readonly configName: string + /** SHA-256 of the exact config bytes loaded (64 lowercase hex chars). */ + readonly sha256: string +} + +/** + * The prior v2 document encoded two-sided held-out resource deltas. Keep it + * readable because calibration configs are immutable operator artifacts. + */ +export const LEGACY_TUNING_CONFIG_FORMAT_VERSION = 2 + +/** + * New calibration configs encode directional held-out resource costs: lower + * observed cost is safe, while only a regression beyond tolerance fails. + */ +export const TUNING_CONFIG_FORMAT_VERSION = 3 + +export type SupportedTuningConfigFormatVersion = + | typeof LEGACY_TUNING_CONFIG_FORMAT_VERSION + | typeof TUNING_CONFIG_FORMAT_VERSION + +/** Canonical held-out tolerances shared by calibration config formats 2 and 3. */ +export const CALIBRATION_HELD_OUT_TOLERANCES = { + peakRssBytes: 0.5, + wallMs: 0.5, + writeThroughputBytesPerSec: 0.75, + compressionRatio: 0.5, + physicalBytes: 0.5, + peakTempDiskBytes: 0.5, +} as const + +/** Exact operator-visible events that require recalibration. */ +export const CALIBRATION_RECALIBRATION_TRIGGERS = [ + "Maple version change", + "chDB version change", + "Schema fingerprint change", + "Hardware change (CPU count, memory, storage speed)", + "Archive-volume replacement or filesystem change", + "Material telemetry-shape change (row width, cardinality, signal mix)", +] as const + +/** + * The held-out window is strictly LARGER than training and disjoint from it: + * training covers ordered rows `[0, sampleRows)`; held-out covers + * `[sampleRows, sampleRows + heldOutSampleRows(sampleRows))`. Defined here + * (the low-level config module) so the loader and the calibrator share one + * source of truth without a circular import. + */ +export const HELD_OUT_SAMPLE_MULTIPLIER = 2 + +export const heldOutSampleRows = (sampleRows: number): number => { + if (!Number.isSafeInteger(sampleRows) || sampleRows <= 0) { + throw new Error(`calibration sampleRows must be a positive safe integer: ${sampleRows}`) + } + const held = HELD_OUT_SAMPLE_MULTIPLIER * sampleRows + if (!Number.isSafeInteger(held) || held <= sampleRows) { + throw new Error(`calibration held-out sample derivation overflow: ${sampleRows}`) + } + return held +} + +const SAFE_CONFIG_NAME = /^[A-Za-z0-9._-]+$/ + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const requireConfigCount = (record: Record, key: string): number => { + const value = record[key] + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`invalid calibration config field: ${key} (must be a safe non-negative integer)`) + } + return value +} + +const assertExactKeys = ( + record: Record, + keys: ReadonlySet, + label: string, + path: string, +): void => { + for (const key of Object.keys(record)) { + if (!keys.has(key)) throw new Error(`unknown calibration config ${label}.${key}: ${path}`) + } + for (const key of keys) { + if (!(key in record)) throw new Error(`missing calibration config ${label}.${key}: ${path}`) + } +} + +const CANDIDATE_KEYS = new Set(["writerThreads", "rowGroupRows", "maxShardRows", "maxShardBytes"]) +const METRIC_KEYS = new Set([ + "logicalBytes", + "physicalBytes", + "compressionRatio", + "writeThroughputBytesPerSec", + "peakTempDiskBytes", + "peakRssBytes", + "wallMs", + "rowCount", +]) + +export interface VerifiedCalibrationConfigDocument { + readonly formatVersion: SupportedTuningConfigFormatVersion + readonly measuredAt: string + readonly confidence: "high" + readonly checkpoint: { + readonly checkpointId: string + readonly manifestFingerprint: string + } + readonly candidateMatrix: ReadonlyArray> + readonly requiredSignals: ReadonlyArray + readonly budget: Record + readonly selected: { + readonly candidate: Record + readonly worstCase: Record + } + readonly heldOut: { + readonly results: ReadonlyArray> + readonly worstCase: Record + readonly signalComparisons: ReadonlyArray> + readonly passed: true + readonly tolerances: Record + } + readonly heldOutAttempts: ReadonlyArray<{ + readonly candidate: Record + readonly results: ReadonlyArray> + readonly worstCase: Record | null + readonly signalComparisons: ReadonlyArray> + readonly passed: boolean + }> + readonly environment: { + readonly mapleVersion: string + readonly chdbVersion: string + readonly schemaFingerprint: string + readonly executionUser: string + readonly platform: string + readonly arch: string + readonly cpuModel: string + readonly cpuCount: number + readonly totalMemoryBytes: number + readonly measurementTool: string + readonly archiveVolume: { + readonly fsid: string + readonly type: number + readonly archiveDir: string + } + } + readonly effective: ArchiveTuningRecord + readonly samplePolicy: { + readonly trainingRows: number + readonly heldOutMultiplier: number + readonly heldOutRows: number + readonly trainingWindow: string + readonly heldOutWindow: string + } + readonly derivation: { + readonly minFreeSpaceReserve: "budget.freeSpaceReserve" + readonly targetChunkBytes: "max(4 * selected.candidate.maxShardBytes, budget.freeSpaceReserve + selected.candidate.maxShardBytes)" + } + readonly safetyMargin: number + readonly recalibrationTriggers: ReadonlyArray + readonly results: ReadonlyArray> + readonly note: string +} + +export interface LoadedTuningConfig { + readonly overrides: ArchiveTuningOverrides + readonly identity: TuningConfigIdentity + readonly document: VerifiedCalibrationConfigDocument +} + +const EXPECTED_SIGNALS = [ + "logs", + "traces", + "metrics_sum", + "metrics_gauge", + "metrics_histogram", + "metrics_exponential_histogram", +] as const + +const EXPECTED_CANDIDATES = [ + { writerThreads: 1, rowGroupRows: 10_000, maxShardRows: 500_000, maxShardBytes: 256 * 1024 * 1024 }, + { writerThreads: 1, rowGroupRows: 5_000, maxShardRows: 250_000, maxShardBytes: 128 * 1024 * 1024 }, + { writerThreads: 2, rowGroupRows: 10_000, maxShardRows: 500_000, maxShardBytes: 256 * 1024 * 1024 }, + { writerThreads: 1, rowGroupRows: 20_000, maxShardRows: 1_000_000, maxShardBytes: 512 * 1024 * 1024 }, +] as const + +const exactJson = (a: unknown, b: unknown): boolean => JSON.stringify(a) === JSON.stringify(b) + +const resultMetrics = (value: unknown, label: string, path: string): Record => { + validateMeasuredMetricsRecord(value, label, path) + return value as Record +} + +const worstCaseFromResults = ( + results: ReadonlyArray>, + path: string, +): Record => { + if (results.length === 0) throw new Error(`calibration config has no results to aggregate: ${path}`) + const metrics = results.map((result, i) => resultMetrics(result.metrics, `aggregate[${i}].metrics`, path)) + const max = (key: string): number => Math.max(...metrics.map((entry) => entry[key]!)) + const min = (key: string): number => Math.min(...metrics.map((entry) => entry[key]!)) + return { + logicalBytes: max("logicalBytes"), + physicalBytes: max("physicalBytes"), + compressionRatio: max("compressionRatio"), + writeThroughputBytesPerSec: min("writeThroughputBytesPerSec"), + peakTempDiskBytes: max("peakTempDiskBytes"), + peakRssBytes: max("peakRssBytes"), + wallMs: max("wallMs"), + rowCount: max("rowCount"), + } +} + +const metricsMeetBudget = (metrics: Record, budget: Record): boolean => { + const margin = budget.safetyMargin! + return ( + metrics.peakRssBytes! * margin <= budget.memoryBudget! && + metrics.wallMs! <= budget.maxCandidateWallMs! && + (budget.minThroughputBytesPerSec === 0 || + metrics.writeThroughputBytesPerSec! / margin >= budget.minThroughputBytesPerSec!) && + metrics.peakTempDiskBytes! * margin <= budget.maxTempDiskBytes! + ) +} + +type HeldOutComparisonPolicy = "symmetric" | "directional" + +const expectedComparisons = ( + predicted: Record, + observed: Record, + tolerances: Record, + policy: HeldOutComparisonPolicy, + sizeScaling?: { ratio: number; metrics: ReadonlySet<"wallMs" | "physicalBytes"> }, +): ReadonlyArray> => { + const ratio = sizeScaling?.ratio ?? 1 + const scale = (metric: "wallMs" | "physicalBytes", value: number): number => + sizeScaling && sizeScaling.metrics.has(metric) && Number.isFinite(ratio) ? value * ratio : value + const metrics = [ + "peakRssBytes", + "wallMs", + "writeThroughputBytesPerSec", + "compressionRatio", + "physicalBytes", + "peakTempDiskBytes", + ] as const + return metrics.map((metric) => { + // wallMs/physicalBytes use the size-scaled prediction (held-out is larger); + // throughput/compression are size-invariant; peaks are absolute. + const rawP = predicted[metric]! + const p = + metric === "wallMs" + ? scale("wallMs", rawP) + : metric === "physicalBytes" + ? scale("physicalBytes", rawP) + : rawP + const o = observed[metric]! + const tolerance = tolerances[metric]! + const throughput = metric === "writeThroughputBytesPerSec" + const relativeDelta = throughput + ? p > 0 + ? Math.max(0, (p - o) / p) + : 0 + : p > 0 + ? policy === "directional" + ? // Lower resource use is safe. This mirrors comparePredictedObserved + // exactly for v3 (and transitional directional v2) documents. + Math.max(0, (o - p) / p) + : Math.abs(o - p) / p + : o === 0 + ? 0 + : null + const withinTolerance = throughput + ? o >= p * (1 - tolerance) + : relativeDelta !== null && relativeDelta <= tolerance + return { metric, predicted: p, observed: o, tolerance, withinTolerance, relativeDelta } + }) +} + +/** + * Recompute the per-signal held-out comparison entries from the recorded + * training and held-out results, mirroring the production + * `compareHeldOutPerSignal`. For each signal (canonical order), pair the + * same-candidate training result with the held-out result, scale wallMs/ + * physicalBytes by THAT signal's heldOut/training logical-byte ratio, compare + * throughput/compression directly and RSS/temp-disk absolutely, and require all + * six metrics within tolerance for that signal. Returns null (incomplete) when a + * signal is unpaired or either logicalBytes is not strictly positive. Pure. + */ +const expectedSignalComparisons = ( + trainingResults: ReadonlyArray>, + heldOutResults: ReadonlyArray>, + candidate: Record, + tolerances: Record, + policy: HeldOutComparisonPolicy, +): ReadonlyArray> | null => { + const findPaired = ( + results: ReadonlyArray>, + signal: string, + ): Record | undefined => + results.find( + (r) => + r.signal === signal && + r.ok === true && + isRecord(r.metrics) && + exactJson(r.candidate, candidate), + ) + const entries: Record[] = [] + for (const signal of EXPECTED_SIGNALS) { + const trainResult = findPaired(trainingResults, signal) + const heldResult = findPaired(heldOutResults, signal) + if (!trainResult || !heldResult) return null + const trainMetrics = trainResult.metrics as Record + const heldMetrics = heldResult.metrics as Record + const trainingLogical = trainMetrics.logicalBytes! + const heldOutLogical = heldMetrics.logicalBytes! + if (!(trainingLogical > 0) || !(heldOutLogical > 0)) return null + const ratio = heldOutLogical / trainingLogical + const comparisons = expectedComparisons(trainMetrics, heldMetrics, tolerances, policy, { + ratio, + metrics: new Set(["wallMs", "physicalBytes"]), + }) + entries.push({ + signal, + scaleRatio: ratio, + comparisons, + passed: comparisons.every((comparison) => comparison.withinTolerance === true), + }) + } + return entries +} + +const validateCandidateRecord = (value: unknown, label: string, path: string): void => { + if (!isRecord(value)) throw new Error(`invalid calibration config ${label} (record required): ${path}`) + assertExactKeys(value, CANDIDATE_KEYS, label, path) + for (const field of CANDIDATE_KEYS) { + const candidateValue = value[field] + if ( + typeof candidateValue !== "number" || + !Number.isSafeInteger(candidateValue) || + candidateValue <= 0 + ) { + throw new Error( + `invalid calibration config ${label}.${field} (positive safe integer required): ${path}`, + ) + } + } +} + +const validateMetricsRecord = (value: unknown, label: string, path: string): void => { + if (!isRecord(value)) throw new Error(`invalid calibration config ${label} (record required): ${path}`) + assertExactKeys(value, METRIC_KEYS, label, path) + for (const field of METRIC_KEYS) { + const metricValue = value[field] + if (typeof metricValue !== "number" || !Number.isFinite(metricValue) || metricValue < 0) { + throw new Error( + `invalid calibration config ${label}.${field} (non-negative finite number required): ${path}`, + ) + } + } + if (!Number.isSafeInteger(value.rowCount)) { + throw new Error(`invalid calibration config ${label}.rowCount (safe integer required): ${path}`) + } +} + +/** Validate metrics emitted by one real child, including derived-value coherence. */ +const validateMeasuredMetricsRecord = (value: unknown, label: string, path: string): void => { + validateMetricsRecord(value, label, path) + const metrics = value as Record + const expectedCompression = metrics.logicalBytes! > 0 ? metrics.physicalBytes! / metrics.logicalBytes! : 0 + const expectedThroughput = metrics.wallMs! > 0 ? metrics.logicalBytes! / (metrics.wallMs! / 1000) : 0 + if (metrics.compressionRatio !== expectedCompression) { + throw new Error( + `invalid calibration config ${label}.compressionRatio (not physicalBytes/logicalBytes): ${path}`, + ) + } + if (metrics.writeThroughputBytesPerSec !== expectedThroughput) { + throw new Error( + `invalid calibration config ${label}.writeThroughputBytesPerSec (not logicalBytes/wallMs): ${path}`, + ) + } +} + +const SAMPLE_KEYS = new Set([ + "checkpointId", + "checkpointManifestFingerprint", + "rangeDate", + "role", + "startRow", + "requestedRows", + "rowCount", +]) + +/** + * Validate one result's persisted sample scope. Every ok result must bind its + * measurement to one immutable checkpoint/range and an exact ordered-row window + * so the loader can prove training and held-out came from the same source on + * disjoint, correctly-sized windows. `expectedRowCount` is the metrics.rowCount + * the same result recorded (must equal sample.rowCount). Pure. + */ +const validateSampleScope = ( + value: unknown, + label: string, + path: string, + expected: { + checkpointId: string + manifestFingerprint: string + rangeDate: string + role: "training" | "held-out" + startRow: number + requestedRows: number + }, + expectedRowCount: number, +): void => { + if (!isRecord(value)) throw new Error(`invalid calibration config ${label} (record required): ${path}`) + for (const key of Object.keys(value)) { + if (!SAMPLE_KEYS.has(key)) throw new Error(`unknown calibration config ${label}.${key}: ${path}`) + } + for (const key of SAMPLE_KEYS) { + if (!(key in value)) throw new Error(`invalid calibration config ${label}.${key} (required): ${path}`) + } + const scope = value as Record + if ( + typeof scope.checkpointId !== "string" || + scope.checkpointId !== expected.checkpointId || + typeof scope.checkpointManifestFingerprint !== "string" || + scope.checkpointManifestFingerprint !== expected.manifestFingerprint || + typeof scope.rangeDate !== "string" || + scope.rangeDate !== expected.rangeDate || + scope.role !== expected.role || + typeof scope.startRow !== "number" || + !Number.isSafeInteger(scope.startRow) || + scope.startRow !== expected.startRow || + typeof scope.requestedRows !== "number" || + !Number.isSafeInteger(scope.requestedRows) || + scope.requestedRows !== expected.requestedRows || + typeof scope.rowCount !== "number" || + !Number.isSafeInteger(scope.rowCount) || + scope.rowCount !== expectedRowCount || + scope.rowCount !== expected.requestedRows + ) { + throw new Error( + `invalid calibration config ${label} (scope must bind and fully observe the exact ${expected.role} window): ${path}`, + ) + } +} + +/** + * Validate the COMPLETE versioned config schema (S10): every required field must + * be present and correctly typed, with nested unknown-field rejection. A + * document containing only `formatVersion` + `effective` is REJECTED — all + * evidence fields (environment, results, budget, confidence, safetyMargin, + * recalibrationTriggers, measuredAt, note) are required. This is the strict + * parser that the prior implementation lacked. + */ +const validateCompleteConfigSchema = ( + parsed: Record, + path: string, + formatVersion: SupportedTuningConfigFormatVersion, +): VerifiedCalibrationConfigDocument => { + const knownTopLevel = new Set([ + "formatVersion", + "effective", + "environment", + "selected", + "results", + "heldOut", + "heldOutAttempts", + "checkpoint", + "candidateMatrix", + "requiredSignals", + "samplePolicy", + "derivation", + "budget", + "confidence", + "safetyMargin", + "recalibrationTriggers", + "measuredAt", + "note", + ]) + for (const key of Object.keys(parsed)) { + if (!knownTopLevel.has(key)) { + throw new Error(`unknown calibration config field '${key}'; refusing: ${path}`) + } + } + if (parsed.confidence !== "high") { + throw new Error( + `invalid calibration config: a loadable recommendation requires confidence 'high': ${path}`, + ) + } + // safetyMargin: required finite number > 0. + if ( + typeof parsed.safetyMargin !== "number" || + !Number.isFinite(parsed.safetyMargin) || + parsed.safetyMargin <= 0 + ) { + throw new Error(`invalid calibration config safetyMargin (must be a positive finite number): ${path}`) + } + // measuredAt: the writer emits canonical UTC ISO-8601. Reject arbitrary + // non-empty strings so evidence ordering and identity remain meaningful. + if ( + typeof parsed.measuredAt !== "string" || + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(parsed.measuredAt) || + !Number.isFinite(Date.parse(parsed.measuredAt)) + ) { + throw new Error(`invalid calibration config measuredAt (canonical ISO-8601 required): ${path}`) + } + // note: required string. + if (typeof parsed.note !== "string") { + throw new Error(`invalid calibration config note: ${path}`) + } + // recalibrationTriggers: required non-empty array of strings. + if ( + !Array.isArray(parsed.recalibrationTriggers) || + !exactJson(parsed.recalibrationTriggers, CALIBRATION_RECALIBRATION_TRIGGERS) + ) { + throw new Error(`invalid calibration config recalibrationTriggers (exact policy required): ${path}`) + } + if (!Array.isArray(parsed.requiredSignals) || !exactJson(parsed.requiredSignals, EXPECTED_SIGNALS)) { + throw new Error(`invalid calibration config requiredSignals (exact six-signal set required): ${path}`) + } + if (!Array.isArray(parsed.candidateMatrix) || !exactJson(parsed.candidateMatrix, EXPECTED_CANDIDATES)) { + throw new Error( + `invalid calibration config candidateMatrix (exact supported matrix required): ${path}`, + ) + } + if (!isRecord(parsed.checkpoint)) { + throw new Error(`invalid calibration config checkpoint (record required): ${path}`) + } + assertExactKeys(parsed.checkpoint, new Set(["checkpointId", "manifestFingerprint"]), "checkpoint", path) + if ( + typeof parsed.checkpoint.checkpointId !== "string" || + parsed.checkpoint.checkpointId.length === 0 || + typeof parsed.checkpoint.manifestFingerprint !== "string" || + parsed.checkpoint.manifestFingerprint.length === 0 + ) { + throw new Error(`invalid calibration config checkpoint identity: ${path}`) + } + const configCheckpoint = { + checkpointId: parsed.checkpoint.checkpointId, + manifestFingerprint: parsed.checkpoint.manifestFingerprint, + } + // environment: required record; deep-validate with unknown-field rejection. + if (!isRecord(parsed.environment)) { + throw new Error(`invalid calibration config environment (record required): ${path}`) + } + const env = parsed.environment + const knownEnv = new Set([ + "mapleVersion", + "chdbVersion", + "schemaFingerprint", + "executionUser", + "platform", + "arch", + "cpuModel", + "cpuCount", + "totalMemoryBytes", + "measurementTool", + "archiveVolume", + ]) + for (const key of Object.keys(env)) { + if (!knownEnv.has(key)) { + throw new Error(`unknown calibration config environment.${key}: ${path}`) + } + } + for (const f of [ + "mapleVersion", + "chdbVersion", + "schemaFingerprint", + "executionUser", + "platform", + "arch", + "cpuModel", + "measurementTool", + ]) { + if (typeof env[f] !== "string") { + throw new Error(`invalid calibration config environment.${f} (string required): ${path}`) + } + } + for (const f of ["cpuCount", "totalMemoryBytes"]) { + if (typeof env[f] !== "number" || !Number.isSafeInteger(env[f]) || env[f] < 0) { + throw new Error( + `invalid calibration config environment.${f} (non-negative safe integer required): ${path}`, + ) + } + } + // archiveVolume: required record with exactly { fsid, type, archiveDir }. + if (!isRecord(env.archiveVolume)) { + throw new Error(`invalid calibration config environment.archiveVolume (record required): ${path}`) + } + const vol = env.archiveVolume + const knownVol = new Set(["fsid", "type", "archiveDir"]) + for (const key of Object.keys(vol)) { + if (!knownVol.has(key)) { + throw new Error(`unknown calibration config environment.archiveVolume.${key}: ${path}`) + } + } + if ( + typeof vol.fsid !== "string" || + vol.fsid.length === 0 || + typeof vol.archiveDir !== "string" || + vol.archiveDir.length === 0 + ) { + throw new Error( + `invalid calibration config environment.archiveVolume (fsid/archiveDir strings required): ${path}`, + ) + } + if (typeof vol.type !== "number" || !Number.isSafeInteger(vol.type)) { + throw new Error( + `invalid calibration config environment.archiveVolume.type (number required): ${path}`, + ) + } + // budget: required record; deep-validate all ceiling fields. + if (!isRecord(parsed.budget)) { + throw new Error(`invalid calibration config budget (record required): ${path}`) + } + const budget = parsed.budget + const knownBudget = new Set([ + "memoryBudget", + "timeBudget", + "sampleRows", + "maxCandidateWallMs", + "minThroughputBytesPerSec", + "maxTempDiskBytes", + "freeSpaceReserve", + "safetyMargin", + ]) + for (const key of Object.keys(budget)) { + if (!knownBudget.has(key)) { + throw new Error(`unknown calibration config budget.${key}: ${path}`) + } + } + for (const f of knownBudget) { + if (typeof budget[f] !== "number" || !Number.isFinite(budget[f]) || budget[f] < 0) { + throw new Error( + `invalid calibration config budget.${f} (non-negative finite number required): ${path}`, + ) + } + } + for (const f of [ + "memoryBudget", + "timeBudget", + "sampleRows", + "maxCandidateWallMs", + "maxTempDiskBytes", + "freeSpaceReserve", + ]) { + if (!Number.isSafeInteger(budget[f]) || budget[f] === 0) { + throw new Error( + `invalid calibration config budget.${f} (positive safe integer required): ${path}`, + ) + } + } + const budgetSafetyMargin = budget.safetyMargin + if (typeof budgetSafetyMargin !== "number" || budgetSafetyMargin <= 0) { + throw new Error(`invalid calibration config budget.safetyMargin (must be > 0): ${path}`) + } + if (parsed.safetyMargin !== budget.safetyMargin) { + throw new Error(`invalid calibration config safetyMargin != budget.safetyMargin: ${path}`) + } + // samplePolicy: the disjoint training/held-out window contract. Every result + // scope must match it exactly so the loader can prove the persisted evidence + // came from one source on disjoint, correctly-sized windows. + const trainingRows = budget.sampleRows as number + const expectedHeldOutRows = heldOutSampleRows(trainingRows) + const expectedSamplePolicy = { + trainingRows, + heldOutMultiplier: HELD_OUT_SAMPLE_MULTIPLIER, + heldOutRows: expectedHeldOutRows, + trainingWindow: `[0, ${trainingRows})`, + heldOutWindow: `[${trainingRows}, ${trainingRows + expectedHeldOutRows})`, + } + if (!isRecord(parsed.samplePolicy)) { + throw new Error(`invalid calibration config samplePolicy (record required): ${path}`) + } + if (!exactJson(parsed.samplePolicy, expectedSamplePolicy)) { + throw new Error( + `invalid calibration config samplePolicy (must bind the exact disjoint training/held-out window contract): ${path}`, + ) + } + if (!isRecord(parsed.selected)) { + throw new Error(`invalid calibration config selected (record required): ${path}`) + } + const sel = parsed.selected + assertExactKeys(sel, new Set(["candidate", "worstCase"]), "selected", path) + validateCandidateRecord(sel.candidate, "selected.candidate", path) + validateMetricsRecord(sel.worstCase, "selected.worstCase", path) + + if (!Array.isArray(parsed.results)) { + throw new Error(`invalid calibration config results (array required): ${path}`) + } + if (parsed.results.length !== EXPECTED_CANDIDATES.length * EXPECTED_SIGNALS.length) { + throw new Error( + `invalid calibration config results (complete candidate x signal matrix required): ${path}`, + ) + } + const seenTraining = new Set() + const resultsByCandidate = new Map[]>() + // Every training + held-out scope must bind to ONE rangeDate (one sealed day). + // Extract it from the first ok result's sample; all others must match. + let sharedRangeDate: string | null = null + for (const r of parsed.results) { + if (isRecord(r) && r.ok === true && isRecord(r.sample) && typeof r.sample.rangeDate === "string") { + sharedRangeDate = r.sample.rangeDate + break + } + } + if (sharedRangeDate === null) { + throw new Error(`invalid calibration config: no training result records a sample rangeDate: ${path}`) + } + for (let i = 0; i < parsed.results.length; i++) { + const r = parsed.results[i] + if (!isRecord(r)) { + throw new Error(`invalid calibration config results[${i}] (record required): ${path}`) + } + const knownResult = new Set(["candidate", "signal", "metrics", "ok", "error", "sample"]) + for (const key of Object.keys(r)) { + if (!knownResult.has(key)) { + throw new Error(`unknown calibration config results[${i}].${key}: ${path}`) + } + } + if (typeof r.signal !== "string" || typeof r.ok !== "boolean") { + throw new Error(`invalid calibration config results[${i}] (signal/ok required): ${path}`) + } + validateCandidateRecord(r.candidate, `results[${i}].candidate`, path) + const candidateKey = JSON.stringify(r.candidate) + if (!EXPECTED_CANDIDATES.some((candidate) => exactJson(candidate, r.candidate))) { + throw new Error(`invalid calibration config results[${i}].candidate (outside matrix): ${path}`) + } + if (!EXPECTED_SIGNALS.includes(r.signal as (typeof EXPECTED_SIGNALS)[number])) { + throw new Error(`invalid calibration config results[${i}].signal (outside six signals): ${path}`) + } + const evidenceKey = `${candidateKey}\u0000${r.signal}` + if (seenTraining.has(evidenceKey)) { + throw new Error(`duplicate calibration config training evidence: ${path}`) + } + seenTraining.add(evidenceKey) + if (r.error !== undefined && typeof r.error !== "string") { + throw new Error(`invalid calibration config results[${i}].error: ${path}`) + } + if (r.ok) { + validateMeasuredMetricsRecord(r.metrics, `results[${i}].metrics`, path) + const metricsRow = (r.metrics as Record).rowCount + validateSampleScope( + r.sample, + `results[${i}].sample`, + path, + { + checkpointId: configCheckpoint.checkpointId, + manifestFingerprint: configCheckpoint.manifestFingerprint, + rangeDate: sharedRangeDate, + role: "training", + startRow: 0, + requestedRows: trainingRows, + }, + metricsRow as number, + ) + } else { + if (r.metrics !== null) { + throw new Error( + `invalid calibration config results[${i}].metrics (failed result must be null): ${path}`, + ) + } + if (r.sample !== undefined) { + throw new Error( + `invalid calibration config results[${i}].sample (failed result must not record a scope): ${path}`, + ) + } + } + const candidateResults = resultsByCandidate.get(candidateKey) ?? [] + candidateResults.push(r) + resultsByCandidate.set(candidateKey, candidateResults) + } + const eligible = EXPECTED_CANDIDATES.flatMap((candidate) => { + const evidence = resultsByCandidate.get(JSON.stringify(candidate)) ?? [] + if ( + evidence.length !== EXPECTED_SIGNALS.length || + !evidence.every( + (result) => + result.ok === true && + isRecord(result.metrics) && + metricsMeetBudget( + result.metrics as Record, + budget as Record, + ), + ) + ) { + return [] + } + return [{ candidate, worstCase: worstCaseFromResults(evidence, path) }] + }).sort( + (a, b) => + a.worstCase.peakRssBytes! - b.worstCase.peakRssBytes! || + a.worstCase.wallMs! - b.worstCase.wallMs!, + ) + const selectedIndex = eligible.findIndex((entry) => exactJson(entry.candidate, sel.candidate)) + if (selectedIndex < 0) { + throw new Error( + `invalid calibration config selected candidate did not pass training ceilings: ${path}`, + ) + } + if (!exactJson(eligible[selectedIndex]!.worstCase, sel.worstCase)) { + throw new Error( + `invalid calibration config selected.worstCase was not recomputed from results: ${path}`, + ) + } + if (!isRecord(parsed.heldOut)) { + throw new Error(`invalid calibration config heldOut (record required): ${path}`) + } + assertExactKeys( + parsed.heldOut, + new Set(["results", "worstCase", "signalComparisons", "passed", "tolerances"]), + "heldOut", + path, + ) + const held = parsed.heldOut + if (!Array.isArray(held.results) || held.results.length !== EXPECTED_SIGNALS.length) { + throw new Error(`invalid calibration config heldOut.results (six entries required): ${path}`) + } + const heldSeen = new Set() + for (let i = 0; i < held.results.length; i++) { + const result = held.results[i] + if (!isRecord(result)) throw new Error(`invalid calibration config heldOut.results[${i}]: ${path}`) + assertExactKeys( + result, + new Set(["candidate", "signal", "metrics", "ok", "sample"]), + `heldOut.results[${i}]`, + path, + ) + if ( + result.ok !== true || + typeof result.signal !== "string" || + heldSeen.has(result.signal) || + !EXPECTED_SIGNALS.includes(result.signal as (typeof EXPECTED_SIGNALS)[number]) || + !exactJson(result.candidate, sel.candidate) + ) { + throw new Error(`invalid calibration config heldOut.results[${i}] identity: ${path}`) + } + heldSeen.add(result.signal) + const metrics = resultMetrics(result.metrics, `heldOut.results[${i}].metrics`, path) + if (!metricsMeetBudget(metrics, budget as Record)) { + throw new Error(`invalid calibration config heldOut.results[${i}] exceeds budget: ${path}`) + } + validateSampleScope( + result.sample, + `heldOut.results[${i}].sample`, + path, + { + checkpointId: configCheckpoint.checkpointId, + manifestFingerprint: configCheckpoint.manifestFingerprint, + rangeDate: sharedRangeDate, + role: "held-out", + startRow: trainingRows, + requestedRows: expectedHeldOutRows, + }, + metrics.rowCount!, + ) + } + const heldWorst = worstCaseFromResults(held.results as Record[], path) + validateMetricsRecord(held.worstCase, "heldOut.worstCase", path) + if (!exactJson(heldWorst, held.worstCase)) { + throw new Error(`invalid calibration config heldOut.worstCase was not recomputed: ${path}`) + } + if (!isRecord(held.tolerances)) { + throw new Error(`invalid calibration config heldOut.tolerances: ${path}`) + } + const toleranceKeys = new Set([ + "peakRssBytes", + "wallMs", + "writeThroughputBytesPerSec", + "compressionRatio", + "physicalBytes", + "peakTempDiskBytes", + ]) + assertExactKeys(held.tolerances, toleranceKeys, "heldOut.tolerances", path) + if (!exactJson(held.tolerances, CALIBRATION_HELD_OUT_TOLERANCES)) { + throw new Error(`invalid calibration config heldOut.tolerances (exact policy required): ${path}`) + } + const validateHeldOutEvidence = (policy: HeldOutComparisonPolicy): void => { + // PER-SIGNAL, like-for-like hybrid comparison: recompute each signal's + // entry by pairing the selected candidate's training result with the + // held-out result for that same signal, scaling wallMs/physicalBytes by + // that signal's own logical-byte ratio. Aggregate extrema (heldWorst) are + // descriptive only. The policy must be global to the document: accepting + // a mixture would let forged evidence combine two incompatible policies. + const recomputedSignalComparisons = expectedSignalComparisons( + parsed.results as Record[], + held.results as Record[], + sel.candidate as Record, + held.tolerances as Record, + policy, + ) + if (recomputedSignalComparisons === null) { + throw new Error( + `invalid calibration config heldOut could not pair every signal like-for-like: ${path}`, + ) + } + if ( + !Array.isArray(held.signalComparisons) || + !exactJson(recomputedSignalComparisons, held.signalComparisons) + ) { + throw new Error( + `invalid calibration config heldOut.signalComparisons were not recomputed: ${path}`, + ) + } + const heldOutPassed = recomputedSignalComparisons.every((entry) => entry.passed === true) + if (held.passed !== true || !heldOutPassed) { + throw new Error(`invalid calibration config heldOut did not pass every signal: ${path}`) + } + if (!Array.isArray(parsed.heldOutAttempts) || parsed.heldOutAttempts.length === 0) { + throw new Error( + `invalid calibration config heldOutAttempts (non-empty evidence required): ${path}`, + ) + } + for (let attemptIndex = 0; attemptIndex < parsed.heldOutAttempts.length; attemptIndex++) { + const attempt = parsed.heldOutAttempts[attemptIndex] + if (!isRecord(attempt)) { + throw new Error(`invalid calibration config heldOutAttempts[${attemptIndex}]: ${path}`) + } + assertExactKeys( + attempt, + new Set(["candidate", "results", "worstCase", "signalComparisons", "passed"]), + `heldOutAttempts[${attemptIndex}]`, + path, + ) + if ( + attemptIndex >= eligible.length || + !exactJson(attempt.candidate, eligible[attemptIndex]!.candidate) || + !Array.isArray(attempt.results) + ) { + throw new Error(`invalid calibration config heldOutAttempts candidate order: ${path}`) + } + const attemptSignals = new Set() + let completeAndWithinBudget = attempt.results.length === EXPECTED_SIGNALS.length + for (let resultIndex = 0; resultIndex < attempt.results.length; resultIndex++) { + const result = attempt.results[resultIndex] + if (!isRecord(result)) { + throw new Error(`invalid calibration config heldOutAttempts result: ${path}`) + } + for (const key of Object.keys(result)) { + if (!new Set(["candidate", "signal", "metrics", "ok", "error", "sample"]).has(key)) { + throw new Error(`unknown calibration config heldOutAttempts result.${key}: ${path}`) + } + } + if ( + typeof result.signal !== "string" || + attemptSignals.has(result.signal) || + !EXPECTED_SIGNALS.includes(result.signal as (typeof EXPECTED_SIGNALS)[number]) || + !exactJson(result.candidate, attempt.candidate) + ) { + throw new Error(`invalid calibration config heldOutAttempts result identity: ${path}`) + } + attemptSignals.add(result.signal) + if (result.ok === true && isRecord(result.metrics)) { + validateMeasuredMetricsRecord(result.metrics, "heldOutAttempts.metrics", path) + if ( + !metricsMeetBudget( + result.metrics as Record, + budget as Record, + ) + ) { + completeAndWithinBudget = false + } + validateSampleScope( + result.sample, + `heldOutAttempts[${attemptIndex}].results[${resultIndex}].sample`, + path, + { + checkpointId: configCheckpoint.checkpointId, + manifestFingerprint: configCheckpoint.manifestFingerprint, + rangeDate: sharedRangeDate, + role: "held-out", + startRow: trainingRows, + requestedRows: expectedHeldOutRows, + }, + (result.metrics as Record).rowCount as number, + ) + } else { + completeAndWithinBudget = false + if (result.metrics !== null) { + throw new Error(`invalid calibration config heldOutAttempts failed metrics: ${path}`) + } + if (result.sample !== undefined) { + throw new Error( + `invalid calibration config heldOutAttempts failed result must not record a scope: ${path}`, + ) + } + } + } + const completeWorst = completeAndWithinBudget + ? worstCaseFromResults(attempt.results as Record[], path) + : null + const attemptCandidate = eligible[attemptIndex]!.candidate as Record + // Complete attempt: recompute per-signal comparisons against this + // candidate's training results. A non-positive logical-byte pair makes + // the attempt incomplete under the same rule as the runner. + const completeSignalComparisons = completeWorst + ? expectedSignalComparisons( + parsed.results as Record[], + attempt.results as Record[], + attemptCandidate, + held.tolerances as Record, + policy, + ) + : [] + const attemptIncomplete = completeWorst === null || completeSignalComparisons === null + const attemptWorst = attemptIncomplete ? null : completeWorst + const attemptSignalComparisons = attemptIncomplete ? [] : completeSignalComparisons + const attemptPassed = + attemptWorst !== null && attemptSignalComparisons.every((entry) => entry.passed === true) + if ( + !exactJson(attempt.worstCase, attemptWorst) || + !exactJson(attempt.signalComparisons, attemptSignalComparisons) || + attempt.passed !== attemptPassed + ) { + throw new Error(`invalid calibration config heldOutAttempts semantic evidence: ${path}`) + } + if (attemptPassed && attemptIndex !== parsed.heldOutAttempts.length - 1) { + throw new Error( + `invalid calibration config continued after a passing held-out attempt: ${path}`, + ) + } + } + const finalAttempt = parsed.heldOutAttempts[parsed.heldOutAttempts.length - 1]! + if ( + !isRecord(finalAttempt) || + finalAttempt.passed !== true || + !exactJson(finalAttempt.candidate, sel.candidate) || + !exactJson(finalAttempt.results, held.results) || + !exactJson(finalAttempt.worstCase, held.worstCase) || + !exactJson(finalAttempt.signalComparisons, held.signalComparisons) + ) { + throw new Error( + `invalid calibration config selected held-out evidence is not final passing attempt: ${path}`, + ) + } + } + const allowedPolicies: ReadonlyArray = + formatVersion === LEGACY_TUNING_CONFIG_FORMAT_VERSION ? ["directional", "symmetric"] : ["directional"] + const validationErrors: Error[] = [] + let matchedPolicy = false + for (const policy of allowedPolicies) { + try { + validateHeldOutEvidence(policy) + matchedPolicy = true + } catch (error) { + validationErrors.push(error instanceof Error ? error : new Error(String(error))) + } + } + if (!matchedPolicy) { + throw validationErrors[0]! + } + if (!isRecord(parsed.derivation)) { + throw new Error(`invalid calibration config derivation (record required): ${path}`) + } + assertExactKeys( + parsed.derivation, + new Set(["minFreeSpaceReserve", "targetChunkBytes"]), + "derivation", + path, + ) + if ( + parsed.derivation.minFreeSpaceReserve !== "budget.freeSpaceReserve" || + parsed.derivation.targetChunkBytes !== + "max(4 * selected.candidate.maxShardBytes, budget.freeSpaceReserve + selected.candidate.maxShardBytes)" + ) { + throw new Error(`invalid calibration config tuning derivation formula: ${path}`) + } + if (!isRecord(parsed.effective)) { + throw new Error(`calibration config missing 'effective' tuning block: ${path}`) + } + assertExactKeys( + parsed.effective, + new Set([ + "writerThreads", + "rowGroupRows", + "maxShardRows", + "maxShardBytes", + "targetChunkBytes", + "minFreeSpaceReserve", + ]), + "effective", + path, + ) + const selectedCandidate = sel.candidate as Record + const maxShardBytes = selectedCandidate.maxShardBytes! + const freeSpaceReserve = budget.freeSpaceReserve as number + const targetChunkBytes = Math.max(4 * maxShardBytes, freeSpaceReserve + maxShardBytes) + if (!Number.isSafeInteger(targetChunkBytes)) { + throw new Error(`calibration config derived targetChunkBytes overflows safe integer range: ${path}`) + } + const expectedEffective = { + writerThreads: selectedCandidate.writerThreads, + rowGroupRows: selectedCandidate.rowGroupRows, + maxShardRows: selectedCandidate.maxShardRows, + maxShardBytes, + targetChunkBytes, + minFreeSpaceReserve: budget.freeSpaceReserve, + } + if (!exactJson(parsed.effective, expectedEffective)) { + throw new Error(`invalid calibration config effective values do not match exact derivation: ${path}`) + } + return parsed as unknown as VerifiedCalibrationConfigDocument +} + +/** + * Load and strictly validate a calibration config document from `path`, returning + * the effective tuning overrides and a SHA-256-bound identity. + * + * The file is opened ONCE; the bytes read, the SHA-256, and the regular-file + * check all derive from that single descriptor (no TOCTOU between read and + * hash). The descriptor is `fstat`-checked to be a regular file — symlinks, + * pipes, and devices are refused. This is the safety boundary for an arbitrary + * operator-supplied path; the archive-root path classifier cannot safely + * validate config paths outside the archive root. + * + * The document schema is validated strictly: required `formatVersion`, an + * `effective` tuning block whose values are routed through + * {@link resolveArchiveTuning} (so the same bounds checks as live tuning + * apply), and unknown top-level fields are rejected. `archiveDir`/`scratchRoot` + * in the config are NOT applied here; the caller resolves roots and defines + * precedence (CLI flags override config `effective` values override defaults), + * rejecting conflicting root overrides explicitly. + */ +export const loadTuningConfig = (path: string): LoadedTuningConfig => { + // lstat BEFORE open so a symlink at `path` is refused. Then open with + // O_NOFOLLOW (kernel refuses a symlink at the final component too). Then + // compare the opened fd's dev/ino against the lstat identity so a swap + // between lstat and open is detected. The content is read AND hashed from + // the single opened fd (bounded read), so read+hash are from one descriptor. + const preStat = lstatSync(path) + if (!preStat.isFile()) { + throw new Error( + `calibration config must be a regular file (refusing symlink, pipe, or device): ${path}`, + ) + } + const MAX_CONFIG_BYTES = 16 * 1024 * 1024 + if (preStat.size > MAX_CONFIG_BYTES) { + throw new Error( + `calibration config is too large (${preStat.size} bytes > ${MAX_CONFIG_BYTES}); refusing: ${path}`, + ) + } + // O_NOFOLLOW refuses a symlink at the final path component at the kernel + // level, closing the lstat/open race. + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW) + let bytes: Buffer + try { + const fdStat = fstatSync(fd) + if (!fdStat.isFile()) { + throw new Error( + `calibration config must be a regular file (refusing non-regular descriptor): ${path}`, + ) + } + // Identity check: the opened fd must be the SAME file lstat saw (same + // device + inode). A swap between lstat and open is detected here. + if (fdStat.dev !== preStat.dev || fdStat.ino !== preStat.ino) { + throw new Error(`calibration config identity changed between lstat and open (TOCTOU): ${path}`) + } + // Re-check size on the fd (the lstat size may be stale after a swap). + if (fdStat.size > MAX_CONFIG_BYTES) { + throw new Error( + `calibration config is too large on fd (${fdStat.size} bytes > ${MAX_CONFIG_BYTES}); refusing: ${path}`, + ) + } + // Bounded read from the fd: read exactly the fd's size so a huge file + // cannot exhaust memory and the SHA is over exactly the read bytes. + const size = fdStat.size + bytes = Buffer.alloc(size) + let read = 0 + while (read < size) { + const n = readSync(fd, bytes, read, size - read, null) + if (n === 0) break + read += n + } + if (read !== size) { + throw new Error(`calibration config short read (${read} of ${size} bytes): ${path}`) + } + } finally { + closeSync(fd) + } + const sha256 = createHash("sha256").update(bytes).digest("hex") + const parsed = JSON.parse(bytes.toString("utf8")) as unknown + if (!isRecord(parsed)) { + throw new Error(`malformed calibration config (not a record): ${path}`) + } + if ( + parsed.formatVersion !== LEGACY_TUNING_CONFIG_FORMAT_VERSION && + parsed.formatVersion !== TUNING_CONFIG_FORMAT_VERSION + ) { + throw new Error( + `unsupported calibration config formatVersion ${String(parsed.formatVersion)} ` + + `(expected ${LEGACY_TUNING_CONFIG_FORMAT_VERSION} or ${TUNING_CONFIG_FORMAT_VERSION}); refusing: ${path}`, + ) + } + const formatVersion: SupportedTuningConfigFormatVersion = + parsed.formatVersion === LEGACY_TUNING_CONFIG_FORMAT_VERSION + ? LEGACY_TUNING_CONFIG_FORMAT_VERSION + : TUNING_CONFIG_FORMAT_VERSION + // Complete strict schema validation (S10): all evidence fields required. + const document = validateCompleteConfigSchema(parsed, path, formatVersion) + // effective: required, six numeric knobs, no unknown fields. + const effectiveRaw = parsed.effective + if (!isRecord(effectiveRaw)) { + throw new Error(`calibration config missing 'effective' tuning block: ${path}`) + } + const knownEffective = new Set([ + "writerThreads", + "rowGroupRows", + "maxShardRows", + "maxShardBytes", + "targetChunkBytes", + "minFreeSpaceReserve", + ]) + for (const key of Object.keys(effectiveRaw)) { + if (!knownEffective.has(key)) { + throw new Error(`unknown calibration config effective field '${key}'; refusing: ${path}`) + } + } + const overrides: ArchiveTuningOverrides = { + writerThreads: requireConfigCount(effectiveRaw, "writerThreads"), + rowGroupRows: requireConfigCount(effectiveRaw, "rowGroupRows"), + maxShardRows: requireConfigCount(effectiveRaw, "maxShardRows"), + maxShardBytes: requireConfigCount(effectiveRaw, "maxShardBytes"), + targetChunkBytes: requireConfigCount(effectiveRaw, "targetChunkBytes"), + minFreeSpaceReserve: requireConfigCount(effectiveRaw, "minFreeSpaceReserve"), + } + const configName = basename(path) + if (!SAFE_CONFIG_NAME.test(configName)) { + throw new Error( + `unsafe calibration config name (must match ${SAFE_CONFIG_NAME.source}): ${configName}`, + ) + } + return { + overrides, + identity: { formatVersion, configName, sha256 }, + document, + } +} diff --git a/apps/cli/src/server/archives/errors.ts b/apps/cli/src/server/archives/errors.ts new file mode 100644 index 000000000..b820deae2 --- /dev/null +++ b/apps/cli/src/server/archives/errors.ts @@ -0,0 +1,14 @@ +import { Schema } from "effect" + +/** + * An archive operation failure. The message is shown to the user and the + * process exits non-zero, mirroring {@link ServerError} and + * {@link CheckpointError}. Archive failures are never silent: an actionable + * summary is preferable to a generic return code. + */ +export class ArchiveError extends Schema.TaggedErrorClass()("@maple/cli/ArchiveError", { + message: Schema.String, +}) {} + +/** Render an expected archive failure without Effect's diagnostic cause stack. */ +export const archiveErrorMessage = (error: ArchiveError): string => `${error.message}\n` diff --git a/apps/cli/src/server/archives/export.ts b/apps/cli/src/server/archives/export.ts new file mode 100644 index 000000000..ccf53d8d1 --- /dev/null +++ b/apps/cli/src/server/archives/export.ts @@ -0,0 +1,1045 @@ +import { createHash } from "node:crypto" +import { + closeSync, + constants, + existsSync, + fsyncSync, + openSync, + readFileSync, + renameSync, + rmSync, + statSync, +} from "node:fs" +import { basename, join } from "node:path" +import type { Chdb } from "../chdb" +import { type ArchiveSignal } from "./signals" + +// Bounded Parquet shard export from a restored checkpoint's scratch chDB. +// +// The export runs `SELECT ... INTO OUTFILE '...' FORMAT Parquet` directly on the +// restored instance. The result is a write side effect; it is never returned +// into JavaScript. One Parquet file is written per bounded slice. +// +// Sharding strategy (round 5, per D-016: NO ORDER BY on the export): a sealed +// UTC day is partitioned by UTC-hour windows, then within each hour by +// enumerating each active MergeTree part and splitting its `_part_offset` +// domain into half-open ranges (`>= lo AND < hi`) of physical width ≤ +// maxShardRows. The export and source-validation queries use the identical +// predicate `WHERE _part = ? AND _part_offset >= ? AND _part_offset < ? AND +// ` — no ORDER BY. Offset holes (other-hour rows in the range) +// are excluded by the UTC date/hour predicate, and counts/bounds are derived +// from the actual matching rows. SYSTEM STOP MERGES freezes the part layout. +// +// Byte bounds are enforced AUTHORITATIVELY: each candidate shard's actual +// `total_uncompressed_size` is measured after writing; if it exceeds +// maxShardBytes the physical range is recursively bisected and re-exported +// until every accepted shard meets both bounds. The only impassable case is a +// single matching row whose size alone exceeds maxShardBytes (distinct failure). +// +// Validation per shard (the round-5 adversarial matrix, apps/cli/test/ +// archive-adversarial-matrix.md): +// H-1 Parquet reopen: row count, UTC day, UTC hour. +// H-A Recursive schema compare (measured chDB→Parquet type normalization). +// H-B Source-slice row count == reopened shard row count. +// H-C Actual uncompressed bytes ≤ maxShardBytes (measured, not sampled). +// H-D Multiset complex-value digest: per-row position-bound hash aggregated +// as an order-independent multiset — detects same-typed column swaps, +// cross-row value reassociation, and dup/drop (the round-4 commutative +// defects). NULL-safe and DateTime-normalized (measured). +// H-E Explicit source-vs-Parquet event-time min/max in epoch nanoseconds. +// +// Every behavior above was probed against real chDB before this code was +// written: see reports/gate2-round5-implementation.md (probed behaviors). No +// chDB behavior is assumed. + +/** + * The multiset digest algorithm version recorded in each manifest shard. Bumped + * to v3 in the round-5 repair: v1 passed a bare nullable value into cityHash64 + * (collapsed the per-row hash to NULL); v2 used a sentinel string but a real + * Nullable(String) value could equal the sentinel (verified collision); v3 binds + * an EXPLICIT isNull(c) flag as a separate hash argument, so NULL-ness is never + * conflated with a value. An unknown value fails closed. + */ +export const COMPLEX_DIGEST_ALGORITHM = "cityhash64-multiset-v3" + +/** Digest algorithms this reader accepts in a manifest shard record. */ +export const KNOWN_COMPLEX_DIGEST_ALGORITHMS: ReadonlySet = new Set(["cityhash64-multiset-v3"]) + +export interface ExportSettings { + readonly writerThreads: number + readonly rowGroupRows: number + readonly maxShardRows: number + readonly maxShardBytes: number + /** + * Optional callback invoked after each shard is written and validated, before + * the next shard. Used by the adversarial merge-safety probe to inject an + * OPTIMIZE TABLE ... FINAL between shard exports, forcing a physical layout + * change that STOP MERGES must block. + */ + readonly afterShardValidated?: (db: Chdb, signal: ArchiveSignal) => void +} + +export interface WrittenShard { + readonly name: string + readonly path: string + readonly rowCount: number + /** Min event time over the reopened shard, as a UTC epoch-nanosecond decimal string. */ + readonly minEventTimeUnixNano: string + /** Max event time over the reopened shard, as a UTC epoch-nanosecond decimal string. */ + readonly maxEventTimeUnixNano: string + readonly sha256: string + readonly bytes: number + readonly columns: ReadonlyArray + /** + * Multiset complex-value digest over the reopened shard (cityHash64 of the + * sorted per-row position-bound hashes). Detects same-typed column swaps, + * cross-row value reassociation, and dup/drop that preserve count and time + * extrema. Algorithm recorded alongside it in the manifest. + */ + readonly complexDigest: string +} + +/** The UTC hours [0..23] that partition a sealed day into primary slices. */ +const HOURS_IN_DAY = Array.from({ length: 24 }, (_, hour) => hour) + +const shardName = (hour: number, seq: number): string => + `${hour.toString().padStart(2, "0")}-${seq.toString().padStart(4, "0")}.parquet` + +/** + * Parse a JSONEachRow result into rows (newline-delimited objects, not a JSON + * array — matching the checkpoint module's readJsonRows idiom). + */ +const readRows = (text: string): ReadonlyArray> => + text + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as Record) + +const parseCount = (text: string): number => { + const row = readRows(text)[0] + if (!row) return 0 + const value = row["count()"] ?? row.count + const count = typeof value === "number" ? value : Number(value ?? 0) + if (!Number.isSafeInteger(count) || count < 0) throw new Error(`invalid count result: ${value}`) + return count +} + +/** + * Count the rows in `table` whose event time falls on a given UTC date using + * toDate() equality (robust against the chDB toDateTime64 aggregate miscount). + */ +export const countRowsForDay = (db: Chdb, signal: ArchiveSignal, rangeDate: string): number => { + const sql = `SELECT count() FROM ${signal.name} WHERE toDate(${signal.eventTimeColumn}, 'UTC') = '${rangeDate}'` + return parseCount(db.query(sql, "JSONEachRow")) +} + +const sha256File = (path: string): string => { + const hash = createHash("sha256") + hash.update(readFileSync(path)) + return hash.digest("hex") +} + +/** + * Escape a filesystem path for safe embedding in a ClickHouse single-quoted + * string literal. Escapes backslashes AND single quotes so neither can break + * out of the literal or introduce escape sequences. + */ +const sqlLiteral = (path: string): string => path.replace(/\\/g, "\\\\").replace(/'/g, "\\'") + +/** + * Refuse operator-controlled archive paths containing single quotes or + * backslashes before export (M-1). The constraint is surfaced visibly rather + * than silently escaped. + */ +const assertSafePath = (path: string): void => { + if (/'/.test(path)) throw new Error(`archive path must not contain a single quote: ${path}`) + if (/\\/.test(path)) throw new Error(`archive path must not contain a backslash: ${path}`) +} + +/** The fixed UTC date + hour predicate appended to every source/export query. */ +const hourPredicate = (signal: ArchiveSignal, rangeDate: string, hour: number): string => + `toDate(${signal.eventTimeColumn}, 'UTC') = '${rangeDate}' AND toHour(${signal.eventTimeColumn}, 'UTC') = ${hour}` + +// --------------------------------------------------------------------------- +// Schema comparison (blocker #4) — recursive, grounded in measured transforms. +// --------------------------------------------------------------------------- + +/** A source column's name and type, captured before export for round-trip comparison. */ +export interface SourceColumn { + readonly name: string + readonly type: string +} + +/** + * Capture the source table's schema (name + type) via DESCRIBE. The Parquet + * shard's reopened schema is compared against this to prove the schema + * round-tripped — not just that it has "some" columns. + */ +export const captureSourceSchema = (db: Chdb, signal: ArchiveSignal): ReadonlyArray => { + const rows = readRows(db.query(`DESCRIBE ${signal.name} FORMAT JSONEachRow`, "JSONEachRow")) + const cols = rows.map((r) => ({ name: String(r.name), type: String(r.type) })) + if (cols.length === 0) throw new Error(`source table ${signal.name} has no columns`) + return cols +} + +/** + * Tokenize a ClickHouse type string into a head token and balanced parenthesized + * inner arguments, e.g. `Array(Map(String, String))` → { head: "Array", inner: + * "Map(String, String)" }. Returns null for a leaf type with no parentheses. + */ +const splitType = (type: string): { head: string; inner: string } | null => { + const open = type.indexOf("(") + if (open < 0) return null + const head = type.slice(0, open).trim() + if (!type.endsWith(")")) return null + const inner = type.slice(open + 1, -1) + return { head, inner } +} + +/** + * Split a comma-separated argument list at top-level commas (ignoring commas + * inside nested parentheses), so `Map(K, V)` arg lists and `Tuple` args parse. + */ +const splitArgs = (inner: string): string[] => { + const args: string[] = [] + let depth = 0 + let start = 0 + for (let i = 0; i < inner.length; i++) { + const ch = inner[i]! + if (ch === "(") depth++ + else if (ch === ")") depth-- + else if (ch === "," && depth === 0) { + args.push(inner.slice(start, i).trim()) + start = i + 1 + } + } + const last = inner.slice(start).trim() + if (last.length > 0) args.push(last) + return args +} + +/** + * Normalize a ClickHouse type the way chDB's Parquet writer does, per the + * measured round-trip in gate2-round4-probes.md: + * LowCardinality(T) → normalize(T) + * DateTime → DateTime64(3, 'UTC') + * DateTime64(N) → DateTime64(N, 'UTC') + * Map(K, V) → Map(normalize(K), normalize(V)) + * Array(T) → Array(normalize(T)) + * Nullable(T) → Nullable(normalize(T)) + * leaf (String, UInt*, Int*, Float*, Bool, …) → unchanged + * + * Only these transforms are applied; everything else compares exactly. This is + * what makes parameterized types survive the comparison (Array(UInt64) stays + * Array(UInt64)) while the lossy round-3 collapse (head token only) is fixed. + */ +export const normalizeType = (type: string): string => { + const t = type.trim() + const split = splitType(t) + if (!split) { + // Leaf types with no parameters. DateTime widens; DateTime64(N) gains UTC. + // DateTime64 already parameterized is handled in the recursive branch below. + if (/^DateTime$/.test(t)) return "DateTime64(3, 'UTC')" + return t + } + const { head, inner } = split + if (/^LowCardinality$/i.test(head)) { + // LowCardinality has exactly one argument; unwrap and recurse. + return normalizeType(inner) + } + if (/^DateTime64$/i.test(head)) { + // Source DateTime64(9) → Parquet DateTime64(9, 'UTC'). The first arg is the + // precision; add the UTC timezone. (If a timezone is already present we + // normalize it to 'UTC' to match the measured output.) + const args = splitArgs(inner) + const precision = args[0] ?? "9" + return `DateTime64(${precision}, 'UTC')` + } + if (/^Map$/i.test(head)) { + const args = splitArgs(inner) + return `Map(${args.map(normalizeType).join(", ")})` + } + if (/^Array$/i.test(head)) { + return `Array(${normalizeType(inner)})` + } + if (/^Nullable$/i.test(head)) { + return `Nullable(${normalizeType(inner)})` + } + // Any other parameterized type (e.g. Decimal, FixedString, Enum): keep its + // head + raw inner so an unexpected type fails closed rather than collapsing. + return `${head}(${inner})` +} + +/** + * Build the per-row position-bound hash argument list for the multiset digest: + * for each column (in order), two arguments — its column INDEX and a NULL-SAFE + * value form. `cityHash64` of a tuple is ORDER-SENSITIVE (measured: `cityHash64 + * (a,b)` ≠ `cityHash64(b,a)`), so binding index+value to position means a + * same-typed column exchange changes the row hash. + * + * NULL SAFETY (the round-5 repair): chDB's `cityHash64` returns NULL if ANY + * argument is NULL. Passing a bare nullable column therefore collapses the + * ENTIRE per-row hash to NULL — a row with NULL `Min`/`Max` (histogram tables) + * loses value sensitivity for ALL its other columns (verified: two datasets with + * NULL Min/Max but different Count/Sum produced the identical digest). So a bare + * value is NEVER passed: each column contributes `columnIndex, isNull(c), + * if(isNull(c), '', toString(norm(c)))`. The EXPLICIT `isNull(c)` flag is the + * binding for NULL-ness — a sentinel alone is insufficient because a real + * Nullable(String) value could equal the sentinel string (verified collision: + * NULL vs '\x00NULL' produced the same digest). The `if` value is then '' for + * NULLs (a constant, since the flag already distinguishes NULL) or the + * normalized value; `cityHash64` is order-sensitive, so flag+value+index bind + * position, NULL-ness, and value distinctly. + * + * TIME NORMALIZATION (round-4 finding): `toString(DateTime)` and `toString + * (DateTime64(N))` render in the session timezone on the source but UTC on the + * Parquet side, so a string hash diverges source↔Parquet. Time columns are + * normalized to a NUMERIC epoch (tz-stable, matches on both sides): bare + * DateTime → `toUnixTimestamp(c, 'UTC')`; DateTime64(N) → + * `toUnixTimestamp64Nano(toDateTime64(c, 9))`. All other types (String, UInt*, + * Map, Array, nested) hash identically via toString (measured: source↔Parquet + * match for traces/maps/arrays/Array(Map) and logs/bare-DateTime). + * + * Returns e.g. `0, isNull(OrgId), if(isNull(OrgId), '', toString(OrgId)), 1, isNull(TimestampTime), if(isNull(TimestampTime), '', toString(toUnixTimestamp(TimestampTime, 'UTC'))), ...`. + */ +const perRowHashArgs = (sourceSchema: ReadonlyArray): string => + sourceSchema + .map((c, i) => { + const t = c.type.trim() + // Normalize TIME-bearing columns to a NUMERIC epoch so toString() of the + // value matches source↔Parquet (a raw time-type's toString renders in the + // session timezone on source but UTC on the Parquet side — measured). Bare + // DateTime/DateTime64 collapse to a scalar epoch; Array(DateTime*) map each + // element to an epoch. Non-time types hash identically via toString. + // DateTime -> toUnixTimestamp(c, 'UTC') + // DateTime64(N) -> toUnixTimestamp64Nano(toDateTime64(c, 9)) + // Array(DateTime) -> arrayMap(x -> toUnixTimestamp(x, 'UTC'), c) + // Array(DateTime64) -> arrayMap(x -> toUnixTimestamp64Nano(toDateTime64(x, 9)), c) + const norm = normalizeValueForHash(c.name, t) + // Explicit isNull(c) flag is the binding for NULL-ness (a sentinel alone + // could collide with a real Nullable(String) value — verified). The + // value is '' when NULL (constant; the flag already distinguishes it). + return `${i}, isNull(${c.name}), if(isNull(${c.name}), '', toString(${norm}))` + }) + .join(", ") + +/** Normalize a column value to a timezone-stable form for the digest hash. */ +const normalizeValueForHash = (name: string, type: string): string => { + const t = type.trim() + if (t === "DateTime") return `toUnixTimestamp(${name}, 'UTC')` + if (t.startsWith("DateTime64(")) return `toUnixTimestamp64Nano(toDateTime64(${name}, 9))` + if (t === "Array(DateTime)") return `arrayMap(x -> toUnixTimestamp(x, 'UTC'), ${name})` + if (t.startsWith("Array(DateTime64(")) + return `arrayMap(x -> toUnixTimestamp64Nano(toDateTime64(x, 9)), ${name})` + return name +} + +/** + * The multiset complex-value digest of a slice: the sorted multiset of per-row + * position-bound hashes, folded into one hash. Order-independent (sorted) so it + * tolerates row-order differences between source and reopened Parquet, yet it + * preserves row identity + multiplicity — so it detects: + * - a same-typed column swap (each affected row's hash changes), + * - cross-row value reassociation (a row's hash changes), + * - duplicate-one/drop-another (the multiset of row hashes changes), + * all of which preserve count and time extrema and defeated the round-4 + * commutative per-column sum. Measured at maxShardRows (500k): 41ms, +15MiB RSS. + * + * `sliceFrom` is the FROM clause (e.g. `traces` or `file('p', Parquet)`, with a + * WHERE predicate already applied where needed). The sort is inside chDB; no + * rows are materialized in JavaScript. + */ +const multisetDigestSql = (sourceSchema: ReadonlyArray, sliceFrom: string): string => { + const args = perRowHashArgs(sourceSchema) + return `SELECT toString(cityHash64(groupArray(h))) AS d FROM (SELECT cityHash64(${args}) AS h FROM ${sliceFrom} ORDER BY h)` +} + +/** + * Compare a reopened Parquet shard's schema against the captured source schema. + * Source types are normalized to their Parquet-round-trip form, then compared + * exactly — so parameterized inner types survive (Array(UInt64) ≠ Array(String)) + * while the measured lossless transforms (LowCardinality unwrap, DateTime widen, + * timezone add) are tolerated. Exact column name, count, and order are enforced. + */ +export const compareSchema = ( + source: ReadonlyArray, + parquetRows: ReadonlyArray>, + shardPath: string, +): ReadonlyArray => { + const parquetCols = parquetRows.map((r) => ({ name: String(r.name), type: String(r.type) })) + if (parquetCols.length === 0) { + throw new Error( + `archive shard validation failed: ${shardPath} reopened with no columns (schema lost)`, + ) + } + // Enforce exact column set and order: every source column must have a + // positionally-matching Parquet column with an exactly-equal normalized type, + // and no extra Parquet columns may exist (a drift that adds/drops/reorders + // columns fails closed). + if (parquetCols.length !== source.length) { + throw new Error( + `archive shard validation failed: ${shardPath} column count mismatch: source ${source.length}, Parquet ${parquetCols.length}`, + ) + } + for (let i = 0; i < source.length; i++) { + const src = source[i]! + const par = parquetCols[i]! + if (src.name !== par.name) { + throw new Error( + `archive shard validation failed: ${shardPath} column ${i} name mismatch: source ${src.name}, Parquet ${par.name}`, + ) + } + const srcNorm = normalizeType(src.type) + const parNorm = normalizeType(par.type) + if (srcNorm !== parNorm) { + throw new Error( + `archive shard validation failed: ${shardPath} column ${src.name} type mismatch: source ${src.type} (→${srcNorm}), Parquet ${par.type} (→${parNorm})`, + ) + } + } + return parquetCols.map((c) => c.name) +} + +// --------------------------------------------------------------------------- +// Per-shard validation (H-1 reopen, H-A schema, H-B source count, H-D digest). +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Per-shard validation: H-1 reopen, H-A schema, H-B source count, H-D multiset +// digest, H-E explicit source-vs-Parquet nanosecond event-time bounds. +// --------------------------------------------------------------------------- + +/** + * The physical slice a shard covers: one part, a half-open `_part_offset` range, + * and the UTC date/hour predicate. The export query and the source re-query use + * the IDENTICAL predicate, so the shard's rows and the source slice match. + */ +export interface PartRange { + readonly part: string + readonly offsetLo: number + readonly offsetHiExclusive: number +} + +/** Build the exact WHERE predicate for a physical slice (no ORDER BY). */ +const slicePredicate = (signal: ArchiveSignal, rangeDate: string, hour: number, range: PartRange): string => + `${hourPredicate(signal, rangeDate, hour)} ` + + `AND _part = '${sqlLiteral(range.part)}' ` + + `AND _part_offset >= ${range.offsetLo} AND _part_offset < ${range.offsetHiExclusive}` + +/** + * Reopen a written shard and validate it against the source slice it came from. + * Throws (fails closed) on any mismatch. Time evidence is read back as UTC epoch + * nanoseconds (H-E), independent of the host timezone. + */ +const validateShard = ( + db: Chdb, + shardPath: string, + signal: ArchiveSignal, + rangeDate: string, + hour: number, + sourceSchema: ReadonlyArray, + range: PartRange, +): { + rowCount: number + minEventTimeUnixNano: string + maxEventTimeUnixNano: string + columns: ReadonlyArray + complexDigest: string +} => { + const lit = sqlLiteral(shardPath) + const pred = slicePredicate(signal, rangeDate, hour, range) + // H-1: reopen; a garbage file throws here. + const rowCount = parseCount(db.query(`SELECT count() FROM file('${lit}', Parquet)`, "JSONEachRow")) + if (rowCount === 0) { + throw new Error( + `archive shard validation failed: ${shardPath} reopened with 0 rows (empty or corrupt Parquet)`, + ) + } + // UTC date + hour containment over the reopened shard. + const dateRow = readRows( + db.query( + `SELECT min(toDate(${signal.eventTimeColumn}, 'UTC')) AS dmn, max(toDate(${signal.eventTimeColumn}, 'UTC')) AS dmx, ` + + `min(toHour(${signal.eventTimeColumn}, 'UTC')) AS hmn, max(toHour(${signal.eventTimeColumn}, 'UTC')) AS hmx ` + + `FROM file('${lit}', Parquet)`, + "JSONEachRow", + ), + )[0] + if (String(dateRow?.dmn ?? "") !== rangeDate || String(dateRow?.dmx ?? "") !== rangeDate) { + throw new Error( + `archive shard validation failed: ${shardPath} contains rows outside date ${rangeDate}`, + ) + } + if (Number(dateRow?.hmn ?? -1) !== hour || Number(dateRow?.hmx ?? -1) !== hour) { + throw new Error(`archive shard validation failed: ${shardPath} contains rows outside hour ${hour}`) + } + // H-A: schema compare. + const parquetSchemaRows = readRows( + db.query(`DESCRIBE file('${lit}', Parquet) FORMAT JSONEachRow`, "JSONEachRow"), + ) + const columns = compareSchema(sourceSchema, parquetSchemaRows, shardPath) + // H-B: source-slice count == reopened shard count. + const sourceCount = parseCount( + db.query(`SELECT count() FROM ${signal.name} WHERE ${pred}`, "JSONEachRow"), + ) + if (sourceCount !== rowCount) { + throw new Error( + `archive shard validation failed: ${shardPath} source slice has ${sourceCount} rows but Parquet has ${rowCount}`, + ) + } + // H-D: multiset complex-value digest. Source slice and reopened Parquet must + // produce the identical multiset digest. Detects column swaps, row-value + // reassociation, and dup/drop that preserve count and time extrema. + const srcDigest = String( + readRows(db.query(multisetDigestSql(sourceSchema, `${signal.name} WHERE ${pred}`), "JSONEachRow"))[0] + ?.d ?? "", + ) + const parDigest = String( + readRows(db.query(multisetDigestSql(sourceSchema, `file('${lit}', Parquet)`), "JSONEachRow"))[0]?.d ?? + "", + ) + if (srcDigest.length === 0 || parDigest.length === 0) { + throw new Error( + `archive shard validation failed: ${shardPath} complex-value digest is empty (src=${srcDigest}, par=${parDigest}); NULL handling regression`, + ) + } + if (srcDigest !== parDigest) { + throw new Error( + `archive shard validation failed: ${shardPath} complex-value digest mismatch: source ${srcDigest} != Parquet ${parDigest}`, + ) + } + // H-E: explicit source-vs-Parquet event-time min/max in epoch NANOSECONDS. + // toUnixTimestamp64Nano rejects bare DateTime (code 43), so wrap to + // DateTime64(col, 9) first (measured). Comparing nanos is timezone-independent. + const nanoCol = `toUnixTimestamp64Nano(toDateTime64(${signal.eventTimeColumn}, 9))` + const srcBounds = readRows( + db.query( + `SELECT min(${nanoCol}) AS mn, max(${nanoCol}) AS mx FROM ${signal.name} WHERE ${pred}`, + "JSONEachRow", + ), + )[0] + const parBounds = readRows( + db.query( + `SELECT min(${nanoCol}) AS mn, max(${nanoCol}) AS mx FROM file('${lit}', Parquet)`, + "JSONEachRow", + ), + )[0] + const srcMin = String(srcBounds?.mn ?? "") + const srcMax = String(srcBounds?.mx ?? "") + const parMin = String(parBounds?.mn ?? "") + const parMax = String(parBounds?.mx ?? "") + if (srcMin !== parMin || srcMax !== parMax) { + throw new Error( + `archive shard validation failed: ${shardPath} event-time bounds mismatch: source [${srcMin}, ${srcMax}] != Parquet [${parMin}, ${parMax}] (nanos)`, + ) + } + return { + rowCount, + minEventTimeUnixNano: parMin, + maxEventTimeUnixNano: parMax, + columns, + complexDigest: parDigest, + } +} + +/** + * Read a shard's ACTUAL total uncompressed size from its Parquet metadata + * (H-C). The plan's bound is on estimated uncompressed bytes, not compressed + * on-disk size; compression can keep a 1 GiB-uncompressed shard under a 256 MiB + * compressed ceiling, so the on-disk stat is insufficient. This is AUTHORITATIVE + * measurement, not a sample-based estimate — the caller refines by bisecting the + * physical range if this exceeds maxShardBytes. + * + * Returns { uncompressed, onDiskBytes }. Does NOT throw on overflow; the caller + * decides whether to refine or (for a single-row range) fail distinctly. + */ +export const measureShardBytes = ( + db: Chdb, + shardPath: string, +): { uncompressed: number; onDiskBytes: number } => { + const lit = sqlLiteral(shardPath) + // ClickHouse's real interface is `file('', ParquetMetadata)` exposing + // `total_uncompressed_size` — NOT DuckDB's `parquet_metadata()` function + // (which does not exist in bundled chDB). + const row = readRows( + db.query( + `SELECT total_uncompressed_size AS uncompressed FROM file('${lit}', ParquetMetadata)`, + "JSONEachRow", + ), + )[0] + return { uncompressed: Number(row?.uncompressed ?? 0), onDiskBytes: statSync(shardPath).size } +} + +// --------------------------------------------------------------------------- +// Sharding plan — enumerate each active MergeTree part and split its +// _part_offset domain into half-open ranges (no ORDER BY; D-016). +// --------------------------------------------------------------------------- + +/** + * A planned shard: one part, a half-open `_part_offset` range, and the UTC + * date/hour. The export and source-validation use the identical predicate (no + * ORDER BY). The byte bound is enforced authoritatively in the export loop by + * measuring each candidate and bisecting if it overflows. + */ +export interface ShardPlan { + readonly hour: number + readonly range: PartRange + /** Matching source rows in this slice (from the actual predicate count). */ + readonly matchingRows: number +} + +/** Count the source rows for one UTC hour of a sealed date. */ +export const countHourRows = (db: Chdb, signal: ArchiveSignal, rangeDate: string, hour: number): number => + parseCount( + db.query( + `SELECT count() FROM ${signal.name} WHERE ${hourPredicate(signal, rangeDate, hour)}`, + "JSONEachRow", + ), + ) + +/** + * Enumerate the active MergeTree parts for one UTC hour, each with its physical + * `_part_offset` [min, max] domain and the count of rows in that domain that + * ALSO match the UTC date/hour predicate. Re-derived per restored snapshot under + * SYSTEM STOP MERGES, so it is a stable snapshot of the physical layout. + * + * A part's offset domain is its min..max `_part_offset` (inclusive); rows in + * that domain whose event time is OUTSIDE the sealed hour are offset holes, + * excluded later by the predicate. We do NOT assume the matching rows are + * contiguous within the domain. + */ +export interface PartDomain { + readonly part: string + readonly offsetMin: number + readonly offsetMax: number + readonly matchingRows: number +} +export const enumeratePartsForHour = ( + db: Chdb, + signal: ArchiveSignal, + rangeDate: string, + hour: number, +): ReadonlyArray => { + const pred = hourPredicate(signal, rangeDate, hour) + // Per-part offset domain over the WHOLE part (min/max _part_offset), plus the + // count of rows in that part that match the hour predicate. We page the + // physical offset domain and let the hour predicate filter holes. + const rows = readRows( + db.query( + `SELECT _part AS part, min(_part_offset) AS lo, max(_part_offset) AS hi, ` + + `countIf(${pred}) AS matching ` + + `FROM ${signal.name} GROUP BY _part ` + + `HAVING matching > 0 ORDER BY _part`, + "JSONEachRow", + ), + ) + return rows.map((r) => ({ + part: String(r.part), + offsetMin: Number(r.lo), + offsetMax: Number(r.hi), + matchingRows: Number(r.matching), + })) +} + +/** + * Build the shard plan for one hour: for each active part, split its physical + * `_part_offset` domain [offsetMin, offsetMax] into half-open ranges of width at + * most `maxShardRows`. `expectedRows` is the COUNT of matching rows (from the + * hour predicate), used only as an upper bound sanity check; the actual matching + * count per range is recounted during export. A conservative byte-aware sizing + * halves the row width when the part's average bytes/row (matchingRows over the + * domain) suggests the byte bound would be exceeded — this only picks the + * INITIAL range size; authoritative measurement refines afterward. + */ +export const planHourShards = ( + hour: number, + parts: ReadonlyArray, + settings: ExportSettings, +): ShardPlan[] => { + const plans: ShardPlan[] = [] + for (const p of parts) { + const domainWidth = p.offsetMax - p.offsetMin + 1 + const rowsPerShard = Math.max(1, settings.maxShardRows) + for (let lo = p.offsetMin; lo <= p.offsetMax; lo += rowsPerShard) { + const hiExclusive = Math.min(lo + rowsPerShard, p.offsetMax + 1) + plans.push({ + hour, + range: { part: p.part, offsetLo: lo, offsetHiExclusive: hiExclusive }, + matchingRows: p.matchingRows, + }) + } + // domainWidth is informational; if a part's domain is empty of matching + // rows it was filtered out by enumeratePartsForHour (HAVING matching > 0). + void domainWidth + } + return plans +} + +/** + * Return the `_part_offset` of the `n`th (1-indexed) matching row at or after + * `offsetLo` in one part/hour, ordered ascending by `_part_offset`. Used to + * build EXACT physical windows: a window `[offsetLo, nthOffset + 1)` contains + * exactly `n` matching rows (the offset is the physical position, so the + * half-open range includes it). Returns `null` if fewer than `n` matching rows + * remain. Grounded in the same frozen-merge enumeration as the export. + * + * This is the row-exact bound that a SQL `LIMIT` cannot provide (ClickHouse + * `count()` ignores LIMIT on the aggregate, so the export and the validation + * re-count would diverge). Bounding via `_part_offset` keeps both on the + * identical predicate. + */ +const nthMatchingOffset = ( + db: Chdb, + signal: ArchiveSignal, + rangeDate: string, + hour: number, + part: string, + offsetLo: number, + n: number, +): number | null => { + const pred = + `${hourPredicate(signal, rangeDate, hour)} ` + + `AND _part = '${sqlLiteral(part)}' ` + + `AND _part_offset >= ${offsetLo}` + // The nth matching row ordered by _part_offset. LIMIT 1 OFFSET (n-1) selects + // exactly that row's offset. readRows yields one row with field "off". + const rows = readRows( + db.query( + `SELECT _part_offset AS off FROM ${signal.name} WHERE ${pred} ` + + `ORDER BY _part_offset ASC LIMIT 1 OFFSET ${n - 1} FORMAT JSONEachRow`, + "JSONEachRow", + ), + ) + if (rows.length === 0) return null + return Number(rows[0]!.off) +} + +/** + * Count the matching rows at or after `offsetLo` in one part/hour. Used to know + * a part's remaining capacity when building windows. + */ +const countMatchingFrom = ( + db: Chdb, + signal: ArchiveSignal, + rangeDate: string, + hour: number, + part: string, + offsetLo: number, +): number => { + const pred = + `${hourPredicate(signal, rangeDate, hour)} ` + + `AND _part = '${sqlLiteral(part)}' ` + + `AND _part_offset >= ${offsetLo}` + return parseCount(db.query(`SELECT count() FROM ${signal.name} WHERE ${pred}`, "JSONEachRow")) +} + +/** + * Build a deterministic, EXACT calibration sample plan covering exactly + * `sampleRows` matching rows starting at `startRow` (0-indexed) in the day's + * ordered (hour, part, `_part_offset`) sequence. Training uses `startRow=0`; + * held-out validation uses `startRow=sampleRows` for a provably disjoint window. + * + * Each emitted {@link ShardPlan.range} is a half-open `_part_offset` window + * whose matching-row count is determined AUTHORITATIVELY (via + * {@link nthMatchingOffset}), not estimated. The last window in a part is + * truncated at the exact offset of the final included row, so the writer's + * actual exported total equals the planned total exactly. A part with a single + * matching row is one window; a part never crosses its offset domain. + * + * The caller passes `expectedTotalRows` to {@link exportShardPlans}, which + * asserts `Σ validated.rowCount === expectedTotalRows` after export. + * + * Returns `{ plansByHour, totalRows }` where `totalRows` is the exact planned + * count (may be less than `sampleRows` if the day has fewer matching rows + * starting at `startRow`). + */ +export const planCalibrationShards = ( + db: Chdb, + signal: ArchiveSignal, + rangeDate: string, + settings: ExportSettings, + sampleRows: number, + startRow = 0, +): { plansByHour: Map; totalRows: number } => { + const rowsPerShard = Math.max(1, settings.maxShardRows) + const plansByHour = new Map() + // Skip `startRow` matching rows across the day, then collect `sampleRows`. + let toSkip = startRow + let budget = sampleRows + let cumulative = 0 + for (const hour of HOURS_IN_DAY) { + if (budget <= 0) break + const parts = enumeratePartsForHour(db, signal, rangeDate, hour) + const hourPlans: ShardPlan[] = [] + for (const p of parts) { + if (budget <= 0) break + // Advance past any rows to skip within this part. The part's matching + // rows may be fewer than the remaining skip; consume what we can. + let cursor = p.offsetMin + if (toSkip > 0) { + const partMatching = countMatchingFrom(db, signal, rangeDate, hour, p.part, cursor) + if (partMatching <= toSkip) { + // The entire part is skipped. + toSkip -= partMatching + continue + } + // Skip `toSkip` rows within this part: the window start is the offset + // AFTER the toSkip-th matching row. + const afterSkip = nthMatchingOffset(db, signal, rangeDate, hour, p.part, cursor, toSkip) + if (afterSkip === null) { + toSkip = 0 + } else { + cursor = afterSkip + 1 + toSkip = 0 + } + } + // Now collect windows of up to rowsPerShard matching rows each, until the + // part or the budget is exhausted. + while (budget > 0) { + const remainingInPart = countMatchingFrom(db, signal, rangeDate, hour, p.part, cursor) + if (remainingInPart === 0) break + const take = Math.min(rowsPerShard, remainingInPart, budget) + // The exact offset of the `take`-th matching row at/after cursor. + const nthOff = nthMatchingOffset(db, signal, rangeDate, hour, p.part, cursor, take) + if (nthOff === null) break + const hiExclusive = nthOff + 1 + hourPlans.push({ + hour, + range: { part: p.part, offsetLo: cursor, offsetHiExclusive: hiExclusive }, + matchingRows: take, + }) + cumulative += take + budget -= take + cursor = hiExclusive + if (take < rowsPerShard) break // part boundary reached within this shard + } + } + if (hourPlans.length > 0) plansByHour.set(hour, hourPlans) + } + return { plansByHour, totalRows: cumulative } +} + +/** + * Export one signal for a sealed UTC day as bounded Parquet shards under + * `shardsDir`. Flow: + * + * 1. SYSTEM STOP MERGES freezes the part layout. The try/finally begins + * IMMEDIATELY after a successful stop so any later failure (schema capture, + * planning, write, validation, callback) always restarts merges. + * 2. For each UTC hour with rows: enumerate the active parts and split each + * part's `_part_offset` domain into half-open ranges of width ≤ maxShardRows. + * 3. Export each range with the identical predicate (NO ORDER BY; D-016) to a + * uniquely owned candidate file; measure its actual uncompressed size. + * 4. AUTHORITATIVE byte refinement: if a candidate exceeds maxShardBytes, + * recursively bisect the physical range and re-export each half, skipping + * empty halves, until every accepted shard meets both bounds. The only + * impassable case is a single matching row whose size alone exceeds + * maxShardBytes — failed distinctly. + * 5. Validate each accepted shard (reopen, schema, source count, multiset + * digest, nanosecond bounds) and assign its final sequential name. + * 6. After all shards for the hour, re-count and verify the hour total is + * unchanged (detects concurrent data loss/gain even though merges are frozen). + */ +export const exportSignalShards = ( + db: Chdb, + signal: ArchiveSignal, + rangeDate: string, + shardsDir: string, + settings: ExportSettings, +): WrittenShard[] => { + assertSafePath(shardsDir) + db.exec(`SYSTEM STOP MERGES ${signal.name}`) + try { + // captureSourceSchema runs INSIDE the try so a throw (e.g. an empty + // source table) always reaches the SYSTEM START MERGES finally. Placing + // it before the try would leave merges stopped on a schema-capture + // failure — a production regression caught in review. + const sourceSchema = captureSourceSchema(db, signal) + // Build the full-day shard plan: every UTC hour with rows, every active + // part, split into half-open ranges of width <= maxShardRows. + const plansByHour = new Map() + const hourRowCounts = new Map() + for (const hour of HOURS_IN_DAY) { + const hourRows = countHourRows(db, signal, rangeDate, hour) + if (hourRows === 0) continue + const parts = enumeratePartsForHour(db, signal, rangeDate, hour) + plansByHour.set(hour, planHourShards(hour, parts, settings)) + hourRowCounts.set(hour, hourRows) + } + const shards = exportShardPlans(db, signal, rangeDate, shardsDir, settings, sourceSchema, plansByHour) + // Per-hour re-count over the WHOLE hour: detects concurrent data loss/gain + // even though merges are frozen. This full-day guard lives only in the + // production path; calibration intentionally subsets hours. + for (const [hour, preExportRows] of hourRowCounts) { + const liveTotal = countHourRows(db, signal, rangeDate, hour) + if (liveTotal !== preExportRows) { + throw new Error( + `archive export hour ${hour} row count changed from ${preExportRows} to ${liveTotal} during export; aborting`, + ) + } + } + return shards + } finally { + // Always restart merges, even on failure, so the scratch store is clean. + db.exec(`SYSTEM START MERGES ${signal.name}`) + } +} + +/** + * The shared write→measure→refine→validate→name pipeline, parameterized by a + * pre-built per-hour shard plan. Production ({@link exportSignalShards}) builds + * the full-day plan; calibration ({@link planCalibrationShards}) builds a + * deterministic sample capped at `sampleRows`. Both execute the IDENTICAL + * pipeline, so `maxShardRows`/`maxShardBytes` bisection and reopen validation + * (count/schema/digest/UTC-time) apply identically. This is the single + * writer/validator: calibration does not duplicate it. + * + * `plansByHour` maps each UTC hour to its ordered shard plans. A per-hour + * sequential counter names that hour's shards `HH-NNNN.parquet`. The caller + * must have already issued `SYSTEM STOP MERGES` and captured `sourceSchema`. + */ +export const exportShardPlans = ( + db: Chdb, + signal: ArchiveSignal, + rangeDate: string, + shardsDir: string, + settings: ExportSettings, + sourceSchema: ReadonlyArray, + plansByHour: ReadonlyMap>, + expectedTotalRows?: number, +): WrittenShard[] => { + assertSafePath(shardsDir) + const shards: WrittenShard[] = [] + /** Owned candidate files created during byte refinement, cleaned in finally. */ + const candidates: string[] = [] + // A monotonic counter for owned candidate file names, so bisect recursion + // never collides. Final sequential shard names are assigned only after a + // candidate passes all validation. + let candidateSeq = 0 + // Recursively export a physical range, refining by bisection when the byte + // bound is exceeded. Accepts validated shards into `shards` with their final + // HH-NNNN name; throws on the single-row-oversize impossibility. + const exportRange = (hour: number, range: PartRange, finalSeq: () => number): void => { + const pred = slicePredicate(signal, rangeDate, hour, range) + const matching = parseCount( + db.query(`SELECT count() FROM ${signal.name} WHERE ${pred}`, "JSONEachRow"), + ) + if (matching === 0) return // empty half (no matching rows in this range) — skip + const width = range.offsetHiExclusive - range.offsetLo + const candidate = join(shardsDir, `.candidate-${candidateSeq++}.parquet`) + candidates.push(candidate) + rmSync(candidate, { force: true }) // INTO OUTFILE refuses to overwrite + assertSafePath(candidate) + db.query( + `SELECT * FROM ${signal.name} WHERE ${pred} ` + + `INTO OUTFILE '${sqlLiteral(candidate)}' FORMAT Parquet ` + + `SETTINGS max_threads = ${settings.writerThreads}, ` + + `output_format_parquet_row_group_size = ${settings.rowGroupRows}`, + "Null", + ) + const { uncompressed, onDiskBytes } = measureShardBytes(db, candidate) + if (uncompressed > settings.maxShardBytes) { + // Refine: remove the proven-owned candidate, bisect the physical range. + rmSync(candidate) + if (width <= 1 || matching === 1) { + // A single matching row whose size alone exceeds the bound: the one + // genuinely impossible case. Fail distinctly, not "recalibrate". + throw new Error( + `archive single row exceeds maxShardBytes uncompressed (${uncompressed} > ${settings.maxShardBytes}) ` + + `in ${signal.name} hour ${hour} part ${range.part} offset ${range.offsetLo}; ` + + `raise maxShardBytes or recalibrate to a wider row budget`, + ) + } + const mid = range.offsetLo + Math.floor(width / 2) + exportRange( + hour, + { part: range.part, offsetLo: range.offsetLo, offsetHiExclusive: mid }, + finalSeq, + ) + exportRange( + hour, + { part: range.part, offsetLo: mid, offsetHiExclusive: range.offsetHiExclusive }, + finalSeq, + ) + return + } + // Candidate passes the byte bound: validate it, then assign its final + // name. validateShard reopens and checks schema/count/digest/nanos. + const validated = validateShard(db, candidate, signal, rangeDate, hour, sourceSchema, range) + const name = shardName(hour, finalSeq()) + const finalPath = join(shardsDir, name) + assertSafePath(finalPath) + if (existsSync(finalPath)) + throw new Error(`archive shard already exists; refusing to overwrite: ${finalPath}`) + // Promote the candidate to its final name (rename is atomic on same fs). + renameSync(candidate, finalPath) + candidates[candidates.length - 1] = finalPath + shards.push({ + name, + path: finalPath, + rowCount: validated.rowCount, + minEventTimeUnixNano: validated.minEventTimeUnixNano, + maxEventTimeUnixNano: validated.maxEventTimeUnixNano, + sha256: sha256File(finalPath), + bytes: onDiskBytes, + columns: validated.columns, + complexDigest: validated.complexDigest, + }) + // The crash seam below is authoritative only after this individual + // shard and its directory entry are durable. syncTree after the whole + // export is still retained as the aggregate durability barrier. + const shardFd = openSync(finalPath, constants.O_RDONLY) + try { + fsyncSync(shardFd) + } finally { + closeSync(shardFd) + } + const shardsDirFd = openSync(shardsDir, constants.O_RDONLY) + try { + fsyncSync(shardsDirFd) + } finally { + closeSync(shardsDirFd) + } + settings.afterShardValidated?.(db, signal) + } + try { + for (const [, plans] of plansByHour) { + let seq = 0 + const nextSeq = () => seq++ + for (const plan of plans) { + exportRange(plan.hour, plan.range, nextSeq) + } + } + // The calibration planner builds EXACT physical windows whose cumulative + // matching rows equal `expectedTotalRows`. Byte bisection splits ranges + // but preserves the total row count (each bisected half's rows sum to the + // parent's), so the writer's actual exported total must equal the planned + // total exactly. Assert it so a planning/writer divergence fails closed + // rather than silently exporting the wrong number of rows. + if (expectedTotalRows !== undefined) { + const actualTotal = shards.reduce((sum, s) => sum + s.rowCount, 0) + if (actualTotal !== expectedTotalRows) { + throw new Error( + `calibration export row-count mismatch: writer exported ${actualTotal} rows ` + + `but the planned exact window total was ${expectedTotalRows} ` + + `(signal ${signal.name} ${rangeDate}); the sampleRows bound was not honored`, + ) + } + } + return shards + } finally { + // Remove any candidate files that were not promoted (e.g. after a failure + // during refinement). Promoted shards were renamed and their entries updated + // to the final HH-NNNN.parquet path; unpromoted candidates keep the + // `.candidate-N.parquet` name and must not survive a failure. + for (const c of candidates) { + if (existsSync(c) && basename(c).startsWith(".candidate-")) { + try { + rmSync(c) + } catch { + // best-effort cleanup + } + } + } + } +} diff --git a/apps/cli/src/server/archives/gc.ts b/apps/cli/src/server/archives/gc.ts new file mode 100644 index 000000000..914ce339b --- /dev/null +++ b/apps/cli/src/server/archives/gc.ts @@ -0,0 +1,829 @@ +import { createHash, randomUUID } from "node:crypto" +import { existsSync, lstatSync, readdirSync, readFileSync, statSync } from "node:fs" +import { lstat, rm } from "node:fs/promises" +import { dirname, join } from "node:path" +import { durableRename, syncDirectory } from "../durable-files" +import { + activePointerPath, + archiveRoot, + assertNoSymlink, + assertNoSymlinkSync, + assertRealFile, + assertRealFileSync, + classifyArchivePathSync, + ensurePrivateDirectory, + generationManifestPath, + generationRoot, + generationsRoot, + signalRoot, +} from "./paths" +import { + readArchiveGenerationManifest, + type ArchiveGenerationManifest, + type ArchiveShardRecord, +} from "./manifest" +import { archiveSignal, type ArchiveSignalName, ARCHIVE_SIGNALS } from "./signals" +import { + archiveCompletedOperation, + operationDir, + persistGcProgress, + readActiveOperation, + writeGcIntent, + type GcOperationIntent, + type GcTarget, +} from "./journal" +import { assertCatalogExact, rebuildCatalog } from "./listing" +import { withMaintenanceLock } from "../checkpoints" + +// Garbage collection of superseded archive generations (Gate 3b). +// +// GC reclaims disk space by deleting generations that a later generation +// superseded. It is the ONLY archive operation that deletes published +// generations, so it is conservative to a fault: it deletes only superseded +// generations it can PROVE are not the active pointer target, with manifest + +// per-shard evidence recorded in a frozen journal before any deletion, and it +// collects via a tombstone rename (never an in-place recursive delete) so a +// SIGKILL mid-collection leaves state the next reconcile can prove it owns. +// +// A GC operation shares the single permitted `operations/active/` slot with +// create operations and is reconciled by the same +// `reconcileArchiveGeneration` entry point (dispatched on `kind: "gc"`). So a +// crashed GC must be reconcilable or it blocks all future archive work. + +export interface GcShardEvidence { + readonly name: string + readonly bytes: number + readonly sha256: string +} + +export interface GcDeleteCandidate { + readonly signal: string + readonly rangeStart: string + readonly generationId: string + readonly createdAt: string + readonly manifestSha256: string + readonly bytes: number + readonly shards: ReadonlyArray + readonly recordedActiveGenerationId: string + readonly sourcePath: string +} + +export interface GcRetained { + readonly signal: string + readonly rangeStart: string + readonly generationId: string + readonly createdAt: string + readonly reason: "active" | "kept" | "uncertain" +} + +export interface GcExcludedRange { + readonly signal: string + readonly rangeStart: string + readonly reason: string +} + +export interface GcExcludedSignal { + readonly signal: string + readonly reason: string +} + +export interface GcPlan { + readonly archiveDir: string + readonly keep: number + readonly deleteSet: ReadonlyArray + readonly retained: ReadonlyArray + readonly excludedRanges: ReadonlyArray + readonly excludedSignals: ReadonlyArray + readonly reclaimableBytes: number +} + +/** + * Fault seams for crash-safety validation (Gate 3b). Committed test seam, not a + * production switch. The crash harness SIGKILLs the worker at these exact + * intra-boundary points where unwinding would mask a real crash. Each is AWAITED + * AFTER the named boundary is durable, before the next destructive step. + */ +export interface GcFaults { + /** After the frozen GC intent is durably written, before any collection. */ + readonly afterIntentDurable?: () => void | Promise + /** After the source→tombstone rename of the FIRST target, before its removal. */ + readonly afterFirstTargetRenamed?: () => void | Promise + /** + * After a target's gc-collecting progress record is durably written (the target + * fully removed + progress persisted). `index` is the 0-based target index; + * `total` is targets.length. Fired for EVERY target including the final one. + * This is the authoritative seam for the nonfinal-progress boundary (index < + * total-1), which exposes the premature-complete defect: a SIGKILL here must + * leave the op at gc-collecting (not complete), so reconcile resumes and + * collects the remaining targets. + */ + readonly afterTargetProgress?: (index: number, total: number) => void | Promise + /** After every target is removed (cursor full), before catalog rebuild. */ + readonly afterAllRemovals?: () => void | Promise + /** After the affected catalogs are rebuilt, before the journal is marked complete. */ + readonly afterCatalogRebuilt?: () => void | Promise +} + +/** SHA-256 of a file's contents (whole-file read; GC targets are bounded shards). */ +const sha256File = (path: string): string => createHash("sha256").update(readFileSync(path)).digest("hex") + +/** Validate a generation dir + manifest + shards, returning the evidence. Throws on any defect. */ +const verifyGenerationEvidence = ( + archiveDir: string, + signal: string, + rangeStart: string, + generationId: string, +): { + readonly manifest: ArchiveGenerationManifest + readonly manifestSha256: string + readonly bytes: number + readonly shards: ReadonlyArray +} => { + const sourcePath = generationRoot(archiveDir, signal, rangeStart, generationId) + assertNoSymlinkSync(archiveDir, sourcePath, "archive generation") + const manifestPath = generationManifestPath(archiveDir, signal, rangeStart, generationId) + assertNoSymlinkSync(archiveDir, manifestPath, "archive generation manifest") + assertRealFile(manifestPath, "archive generation manifest") + const manifestSha256 = sha256File(manifestPath) + const manifest = readArchiveGenerationManifest(archiveDir, signal, rangeStart, generationId) + if ( + manifest.generationId !== generationId || + manifest.signal !== signal || + manifest.rangeStart !== rangeStart + ) { + throw new Error(`archive generation manifest identity mismatch: ${sourcePath}`) + } + let bytes = 0 + const shards: GcShardEvidence[] = [] + for (const shard of manifest.shards as ReadonlyArray) { + const shardPath = join(sourcePath, "shards", shard.name) + assertNoSymlinkSync(archiveDir, shardPath, `archive shard ${shard.name}`) + assertRealFile(shardPath, `archive shard ${shard.name}`) + const actualBytes = statSync(shardPath).size + if (actualBytes !== shard.bytes) { + throw new Error( + `archive shard ${shard.name} byte mismatch: manifest ${shard.bytes}, actual ${actualBytes}`, + ) + } + const actualSha = sha256File(shardPath) + if (actualSha !== shard.sha256) { + throw new Error(`archive shard ${shard.name} SHA-256 mismatch (tampered)`) + } + bytes += actualBytes + shards.push({ name: shard.name, bytes: actualBytes, sha256: actualSha }) + } + return { manifest, manifestSha256, bytes, shards } +} + +/** + * Plan a GC run: enumerate superseded generations and decide the delete/retain + * sets. Strict and fail-closed; NEVER called before acquiring the maintenance + * lock. If a signal cannot be authoritatively catalog-reconstructed (any + * malformed manifest/shard/pointer), the ENTIRE signal is excluded — GC never + * deletes a range and then discovers reconstruction is impossible. + * + * `keep` is the number of newest superseded generations to retain per range. + */ +export const planArchiveGc = (archiveDir: string, keep: number): GcPlan => { + if (!Number.isSafeInteger(keep) || keep < 0) { + throw new Error(`invalid gc keep value: ${keep}`) + } + const deleteSet: GcDeleteCandidate[] = [] + const retained: GcRetained[] = [] + const excludedRanges: GcExcludedRange[] = [] + const excludedSignals: GcExcludedSignal[] = [] + let reclaimableBytes = 0 + + for (const signalEntry of ARCHIVE_SIGNALS) { + const signal = signalEntry.name + const sRoot = signalRoot(archiveDir, signal) + if (!existsSync(sRoot)) continue + // Signal-level proof: can this signal's catalog be authoritatively + // reconstructed? If any range/generation is malformed, exclude the WHOLE + // signal before touching any of its ranges. + try { + // Authoritative reconstruction requires the signal root to exist; if it + // does, assert the catalog is provably reconstructable by attempting the + // exact check against current manifests (rebuild is non-mutating on + // failure, but assertCatalogExact only reads — it does not write). + const sigName = archiveSignal(signal).name as ArchiveSignalName + if (existsSync(signalRoot(archiveDir, signal))) assertCatalogExact(archiveDir, sigName) + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + excludedSignals.push({ signal, reason: `signal catalog not provably reconstructable: ${reason}` }) + continue + } + let ranges: string[] + try { + ranges = readdirSync(sRoot) + .filter((e) => /^\d{4}-\d{2}-\d{2}$/.test(e)) + .sort() + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + excludedSignals.push({ signal, reason: `signal root unreadable: ${reason}` }) + continue + } + for (const rangeStart of ranges) { + const gensRoot = generationsRoot(archiveDir, signal, rangeStart) + if (!existsSync(gensRoot)) continue + // Resolve the active generation for this range strictly. + let activeGenerationId: string | null + try { + activeGenerationId = readActiveGenerationIdStrict(archiveDir, signal, rangeStart) + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + excludedRanges.push({ signal, rangeStart, reason: `ambiguous active pointer: ${reason}` }) + continue + } + // A MISSING active pointer is uncertain state: without it, every + // generation is ambiguous (none is provably active), and recording an + // empty-string sentinel would produce an invalid journal the parser + // rejects on recovery (blocker 4). Over-retain the ENTIRE range. + if (activeGenerationId === null) { + excludedRanges.push({ + signal, + rangeStart, + reason: "no active pointer; range is uncertain (over-retained)", + }) + continue + } + let generationIds: string[] + try { + generationIds = readdirSync(gensRoot).sort() + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + excludedRanges.push({ signal, rangeStart, reason: `generations root unreadable: ${reason}` }) + continue + } + // Verify every generation in the range up front; any defect excludes the + // range (conservative — never partially collect a range). + const verified: Array<{ + generationId: string + createdAt: string + manifestSha256: string + bytes: number + shards: ReadonlyArray + }> = [] + try { + for (const generationId of generationIds) { + const ev = verifyGenerationEvidence(archiveDir, signal, rangeStart, generationId) + verified.push({ + generationId, + createdAt: ev.manifest.createdAt, + manifestSha256: ev.manifestSha256, + bytes: ev.bytes, + shards: ev.shards, + }) + } + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + excludedRanges.push({ + signal, + rangeStart, + reason: `malformed generation prevents range collection: ${reason}`, + }) + continue + } + // Partition: active (never deleted) vs superseded. + const superseded = verified + .filter((g) => g.generationId !== activeGenerationId) + .sort((a, b) => + a.createdAt < b.createdAt + ? 1 + : a.createdAt > b.createdAt + ? -1 + : a.generationId < b.generationId + ? 1 + : -1, + ) + for (const g of verified.filter((g) => g.generationId === activeGenerationId)) { + retained.push({ + signal, + rangeStart, + generationId: g.generationId, + createdAt: g.createdAt, + reason: "active", + }) + } + // Keep newest N superseded; delete the older ones. + const keepers = superseded.slice(0, keep) + const targets = superseded.slice(keep) + for (const g of keepers) { + retained.push({ + signal, + rangeStart, + generationId: g.generationId, + createdAt: g.createdAt, + reason: "kept", + }) + } + for (const g of targets) { + deleteSet.push({ + signal, + rangeStart, + generationId: g.generationId, + createdAt: g.createdAt, + manifestSha256: g.manifestSha256, + bytes: g.bytes, + shards: g.shards, + // activeGenerationId is guaranteed non-null here: a missing + // pointer excludes the whole range above (blocker 4). + recordedActiveGenerationId: activeGenerationId as string, + sourcePath: generationRoot(archiveDir, signal, rangeStart, g.generationId), + }) + reclaimableBytes += g.bytes + } + } + } + return { archiveDir, keep, deleteSet, retained, excludedRanges, excludedSignals, reclaimableBytes } +} + +/** Strictly read the active pointer; throws on a malformed/ambiguous pointer. */ +const readActiveGenerationIdStrict = ( + archiveDir: string, + signal: string, + rangeDate: string, +): string | null => { + const pointerPath = activePointerPath(archiveDir, signal, rangeDate) + if (!existsSync(pointerPath)) return null + assertNoSymlinkSync(archiveDir, pointerPath, "archive active pointer") + assertRealFile(pointerPath, "archive active pointer") + const raw = JSON.parse(readFileSync(pointerPath, "utf8")) as Record + if ( + raw.formatVersion !== 1 || + typeof raw.generationId !== "string" || + raw.signal !== signal || + raw.rangeStart !== rangeDate + ) { + throw new Error(`malformed active pointer at ${pointerPath}`) + } + return raw.generationId +} + +/** Deterministic tombstone path for a GC target beneath the operation dir. */ +const tombstonePath = ( + archiveDir: string, + operationId: string, + target: GcTarget | GcDeleteCandidate, +): string => join(operationDir(archiveDir, operationId), "tombstones", target.generationId) + +const revalidateSource = (archiveDir: string, target: GcTarget | GcDeleteCandidate): void => { + const sourcePath = generationRoot(archiveDir, target.signal, target.rangeStart, target.generationId) + if (!existsSync(sourcePath)) return // absent handled by topology switch + assertNoSymlinkSync(archiveDir, sourcePath, "archive generation") + const manifestPath = generationManifestPath( + archiveDir, + target.signal, + target.rangeStart, + target.generationId, + ) + assertNoSymlinkSync(archiveDir, manifestPath, "archive generation manifest") + assertRealFile(manifestPath, "archive generation manifest") + const actualManifestSha = sha256File(manifestPath) + if (actualManifestSha !== target.manifestSha256) { + throw new Error(`archive gc target manifest changed after planning: ${sourcePath}`) + } + const manifest = readArchiveGenerationManifest( + archiveDir, + target.signal, + target.rangeStart, + target.generationId, + ) + if (manifest.generationId !== target.generationId) { + throw new Error(`archive gc target identity changed after planning: ${sourcePath}`) + } + for (const shardEv of target.shards) { + const shardPath = join(sourcePath, "shards", shardEv.name) + assertNoSymlinkSync(archiveDir, shardPath, `archive shard ${shardEv.name}`) + assertRealFile(shardPath, `archive shard ${shardEv.name}`) + const actualSha = sha256File(shardPath) + if (actualSha !== shardEv.sha256) { + throw new Error(`archive gc target shard ${shardEv.name} changed after planning: ${sourcePath}`) + } + } +} + +const revalidatePointer = (archiveDir: string, target: GcTarget | GcDeleteCandidate): void => { + // The pointer must STILL select the exact recorded active generation — not + // merely "some generation other than the target." A pointer that came back + // onto the target (re-selection) stops collection and preserves it. + const current = readActiveGenerationIdStrict(archiveDir, target.signal, target.rangeStart) + if (current !== target.recordedActiveGenerationId) { + throw new Error( + `archive gc pointer changed for ${target.signal}/${target.rangeStart}: ` + + `recorded active ${target.recordedActiveGenerationId}, now ${current} (refusing to collect)`, + ) + } + if (current === target.generationId) { + // The pointer now selects our target — it was re-selected. Preserve it. + throw new Error( + `archive gc target is now the active generation (re-selected): ${target.generationId}`, + ) + } +} + +/** + * Run a GC operation under the maintenance lock. Apply mode: reconcile any prior + * op, plan, journal the frozen set, collect via tombstone rename, rebuild + * catalogs, prove terminal invariants, complete. Dry-run mode: take the lock for + * a consistent view, plan, and return the plan WITHOUT any mutation — including + * NO implicit reconciliation (a dry-run that reconciles would rename tombstones, + * repair catalogs, release pins, and archive an op). If an active op prevents a + * trustworthy GC plan, dry-run reports that blocker rather than predicting a + * deletion set from unstable state. + */ +export const runArchiveGc = async (args: { + readonly dataDir: string + readonly archiveDir: string + readonly scratchRoot: string + readonly keep: number + readonly dryRun: boolean + readonly faults?: GcFaults +}): Promise<{ readonly plan: GcPlan; readonly deleted: ReadonlyArray }> => { + const { dataDir, archiveDir, scratchRoot, keep, dryRun } = args + const operationId = cryptoRandomUuid() + return withMaintenanceLock(dataDir, operationId, async () => { + if (!dryRun) { + // Reconcile any prior op first (create or gc) before planning new work. + // DRY-RUN must NOT do this — it would mutate (blocker 3). + const { reconcileArchiveGeneration } = await import("./generation") + await reconcileArchiveGeneration(dataDir, archiveDir, scratchRoot) + } + const plan = planArchiveGc(archiveDir, keep) + // If an active op is present, a dry-run cannot predict a trustworthy + // deletion set from unstable state; report the blocker instead. + const activePresent = + existsSync(join(archiveDir, "operations", "active")) && + readdirSync(join(archiveDir, "operations", "active")).length > 0 + if (dryRun) { + if (activePresent) { + return { + plan: { + ...plan, + deleteSet: [], + excludedSignals: [ + ...plan.excludedSignals, + { + signal: "(active operation)", + reason: "an active operation is present; reconcile before planning GC", + }, + ], + }, + deleted: [], + } + } + return { plan, deleted: [] } + } + if (plan.deleteSet.length === 0) return { plan, deleted: [] } + const targets: GcTarget[] = plan.deleteSet.map((c) => ({ + signal: c.signal, + rangeStart: c.rangeStart, + generationId: c.generationId, + createdAt: c.createdAt, + manifestSha256: c.manifestSha256, + bytes: c.bytes, + shards: c.shards, + recordedActiveGenerationId: c.recordedActiveGenerationId, + })) + await writeGcIntent({ archiveDir, operationId, dataDir, scratchRoot, keep, targets }) + await args.faults?.afterIntentDurable?.() + const deleted = await collectTargets(archiveDir, operationId, targets, args.faults) + // Recheck every affected pointer, rebuild affected catalogs, assert exact. + await args.faults?.afterAllRemovals?.() + const affectedSignals = new Set(targets.map((t) => t.signal)) + for (const signal of affectedSignals) { + const sigName = archiveSignal(signal).name as ArchiveSignalName + await rebuildCatalog(archiveDir, sigName) + assertCatalogExact(archiveDir, sigName) + } + await args.faults?.afterCatalogRebuilt?.() + // complete is written ONLY after every affected catalog is asserted exact. + // Then the terminal invariant is proved before archival by re-reading the + // durable record (a phase label is never proof — observe the reality). + await persistGcProgress(archiveDir, operationId, targets.length, "complete") + const completed = readActiveOperation(archiveDir, dataDir, scratchRoot) + if (completed === null || completed.intent.kind !== "gc") { + throw new Error("archive gc operation vanished after completion") + } + await verifyCompletedGcInvariants(archiveDir, completed.intent) + await archiveCompletedOperation(archiveDir, operationId) + return { plan, deleted } + }) +} + +/** + * Crash-safe collection via tombstone rename. For each target, revalidate source + * + pointer, atomically rename source → tombstone, then remove only the + * tombstone. Progress is persisted per target as the NONTERMINAL gc-collecting + * phase (NEVER complete) so a crash here leaves the op resumable. `complete` is + * written only by the caller after catalog rebuild + assert. Returns the + * targets that completed (including those already done). + */ +const collectTargets = async ( + archiveDir: string, + operationId: string, + targets: ReadonlyArray, + faults?: GcFaults, +): Promise => { + const completed: GcTarget[] = [] + for (let i = 0; i < targets.length; i++) { + const target = targets[i]! + await collectOneTarget(archiveDir, operationId, target, faults) + completed.push(target) + // Persist the NONTERMINAL gc-collecting phase with the advanced cursor. + // This is true for EVERY target including the final one — the legitimate + // crash state after the final deletion but before catalog repair is + // gc-collecting with a full cursor, NOT complete. + await persistGcProgress(archiveDir, operationId, i + 1, "gc-collecting") + // Awaited crash seam: fires after the durable progress write. For a + // nonfinal target (index < total-1) this is the window where the old code + // wrote a premature complete; a SIGKILL here must leave gc-collecting. + await faults?.afterTargetProgress?.(i, targets.length) + } + return completed +} + +const collectOneTarget = async ( + archiveDir: string, + operationId: string, + target: GcTarget, + faults?: GcFaults, +): Promise => { + const sourcePath = generationRoot(archiveDir, target.signal, target.rangeStart, target.generationId) + const tomb = tombstonePath(archiveDir, operationId, target) + const sourceTopology = classifyArchivePathSync(archiveDir, sourcePath, "archive gc source") + const tombTopology = classifyArchivePathSync(archiveDir, tomb, "archive gc tombstone") + if (sourceTopology !== "absent" && tombTopology !== "absent") { + throw new Error(`archive gc target has both source and tombstone: ${target.generationId}`) + } + if (sourceTopology === "absent" && tombTopology === "absent") { + // Already completed (idempotent resume). Still CAS-check the pointer. + revalidatePointer(archiveDir, target) + return + } + if (sourceTopology === "absent" && tombTopology !== "absent") { + // Resume: source already renamed to tombstone; finish removing the tombstone. + revalidatePointer(archiveDir, target) + await removeTombstone(tomb) + return + } + // sourceTopology !== "absent" && tombTopology === "absent": revalidate + rename. + revalidateSource(archiveDir, target) + revalidatePointer(archiveDir, target) + await ensurePrivateDirectory(dirname(tomb), archiveRoot(archiveDir)) + await assertNoSymlink(archiveDir, tomb, "archive gc tombstone") + await durableRename(sourcePath, tomb) + await syncDirectory(dirname(sourcePath)) + await syncDirectory(dirname(tomb)) + // Crash seam: AFTER the source→tombstone rename of the first target, BEFORE + // tombstone removal. A SIGKILL here leaves source absent + tombstone present, + // which reconcile resumes (the "during-removal" boundary). + await faults?.afterFirstTargetRenamed?.() + // Now the tombstone is the sole copy; remove only it. + await removeTombstone(tomb) +} + +const removeTombstone = async (tomb: string): Promise => { + // Containment: the tombstone must be a real directory (not a symlink). + const info = await lstat(tomb) + if (info.isSymbolicLink() || !info.isDirectory()) { + throw new Error(`refusing to remove non-directory tombstone: ${tomb}`) + } + await rm(tomb, { recursive: true, force: true }) + await syncDirectory(dirname(tomb)) +} + +/** + * Exported alias of {@link collectOneTarget} for the action-driven reconcile + * executor (Gate 3b r4): the plan decides WHICH targets to collect; this helper + * executes ONE target's topology switch (rename/remove/verify) with its source + + * pointer precondition revalidation. + */ +export const collectOneTargetForReconcile = collectOneTarget + +/** + * Reconcile an interrupted GC operation. Drives the FROZEN target set to + * completion idempotently — NEVER re-expands the set. A pointer change or source + * divergence at any stage stops collection and preserves remaining state. + * + * State machine (the core fix for the premature-complete defect): + * - `gc-collecting` (any cursor, including a full cursor after the final + * deletion but before catalog repair): resume collection from the cursor, + * collect the remainder, then rebuild + assert affected catalogs. + * - `complete`: prove all terminal invariants (verifyCompletedGcInvariants), + * then retire the journal. A phase label is NEVER proof — observe the reality. + */ +/** + * Verify a complete GC operation's terminal invariants, then archive it. + * NO repair — a phase label is never proof. Called directly by the + * GcVerifyComplete decision branch (no phase re-branch). + */ +export const verifyCompleteAndArchiveGc = async ( + archiveDir: string, + intent: GcOperationIntent, +): Promise => { + if (intent.kind !== "gc") throw new Error(`verifyCompleteAndArchiveGc called on non-gc intent`) + await verifyCompletedGcInvariants(archiveDir, intent) + await archiveCompletedOperation(archiveDir, intent.operationId) +} + +/** + * Resume collection of the frozen target set, rebuild affected catalogs, + * complete, verify, and archive. Called directly by the GcResume decision + * branch (no phase re-branch). collectOneTarget is idempotent per target. + */ +export const resumeFrozenTargetsAndCompleteGc = async ( + archiveDir: string, + intent: GcOperationIntent, +): Promise => { + if (intent.kind !== "gc") throw new Error(`resumeFrozenTargetsAndCompleteGc called on non-gc intent`) + for (let i = intent.completedTargets; i < intent.targets.length; i++) { + const target = intent.targets[i]! + await collectOneTarget(archiveDir, intent.operationId, target) + await persistGcProgress(archiveDir, intent.operationId, i + 1, "gc-collecting") + } + // Rebuild ALL affected catalogs + assert exact. + const affectedSignals = new Set(intent.targets.map((t) => t.signal)) + for (const signal of affectedSignals) { + const sigName = archiveSignal(signal).name as ArchiveSignalName + await rebuildCatalog(archiveDir, sigName) + assertCatalogExact(archiveDir, sigName) + } + await persistGcProgress(archiveDir, intent.operationId, intent.targets.length, "complete") + // Re-read the durable record to verify (a phase label is never proof). + const retired = readActiveOperation(archiveDir, intent.dataDir, intent.scratchRoot) + if (retired === null || retired.intent.kind !== "gc") { + throw new Error("archive gc operation vanished after completion") + } + await verifyCompletedGcInvariants(archiveDir, retired.intent) + await archiveCompletedOperation(archiveDir, intent.operationId) +} + +/** + * Validate a tombstone present at the owned path: real directory (not symlink), + * containing a generation matching the frozen manifest/shard evidence. + */ +const revalidateTombstone = ( + archiveDir: string, + tomb: string, + target: GcTarget | GcDeleteCandidate, +): void => { + assertNoSymlinkSync(archiveDir, tomb, "archive gc tombstone") + const info = lstatSync(tomb) + if (!info.isDirectory()) { + throw new Error(`archive gc tombstone is not a real directory: ${tomb}`) + } + // Verify the tombstone's manifest matches the frozen evidence. + const tombManifestPath = join(tomb, "manifest.json") + assertNoSymlinkSync(archiveDir, tombManifestPath, "archive gc tombstone manifest") + assertRealFileSync(tombManifestPath, "archive gc tombstone manifest") + const actualManifestSha = sha256File(tombManifestPath) + if (actualManifestSha !== target.manifestSha256) { + throw new Error(`archive gc tombstone manifest evidence mismatch: ${tomb}`) + } + // Verify each shard matches the frozen evidence. + for (const shardEv of target.shards) { + const shardPath = join(tomb, "shards", shardEv.name) + assertNoSymlinkSync(archiveDir, shardPath, `archive gc tombstone shard ${shardEv.name}`) + assertRealFileSync(shardPath, `archive gc tombstone shard ${shardEv.name}`) + const actualSha = sha256File(shardPath) + if (actualSha !== shardEv.sha256) { + throw new Error(`archive gc tombstone shard ${shardEv.name} evidence mismatch: ${tomb}`) + } + } +} + +/** + * Preflight the ENTIRE frozen target set read-only before the decision authorizes + * any mutation. For every target — prefix (already completed), current (resume + * cursor), and suffix (not yet attempted): + * + * - prefix (index < completedTargets): source absent, tombstone absent, pointer + * still equals the recorded active generation. + * - current (index === completedTargets): the documented crash topologies + * (source+no-tombstone, source-absent+tombstone-present, both-absent), each + * with evidence validation and pointer CAS; + * - suffix (index > completedTargets): source present with exact frozen + * evidence, tombstone absent, and pointer CAS. + * + * Throws on any defect so the inspection returns FailClosed — preventing partial + * deletion where target 1 succeeds but a later target (or a corrupted prefix) + * fails. + */ +export const preflightGcTargets = async (archiveDir: string, intent: GcOperationIntent): Promise => { + for (let i = 0; i < intent.targets.length; i++) { + const target = intent.targets[i]! + const sourcePath = generationRoot(archiveDir, target.signal, target.rangeStart, target.generationId) + const tomb = tombstonePath(archiveDir, intent.operationId, target) + const sourceTopology = classifyArchivePathSync(archiveDir, sourcePath, "archive gc source") + const tombTopology = classifyArchivePathSync(archiveDir, tomb, "archive gc tombstone") + + if (i < intent.completedTargets) { + // Prefix: already completed. Both source and tombstone must be absent. + if (sourceTopology !== "absent") + throw new Error(`archive gc completed target still has source: ${target.generationId}`) + if (tombTopology !== "absent") + throw new Error(`archive gc completed target still has tombstone: ${target.generationId}`) + revalidatePointer(archiveDir, target) + continue + } + + if (i === intent.completedTargets) { + // Current target: the documented crash topologies are permitted. + if (sourceTopology !== "absent" && tombTopology !== "absent") { + throw new Error(`archive gc target has both source and tombstone: ${target.generationId}`) + } + if (sourceTopology === "absent" && tombTopology === "absent") { + revalidatePointer(archiveDir, target) + continue + } + if (sourceTopology !== "absent" && tombTopology === "absent") { + revalidateSource(archiveDir, target) + revalidatePointer(archiveDir, target) + continue + } + // Source absent, tombstone present: mid-removal crash topology. + revalidateTombstone(archiveDir, tomb, target) + revalidatePointer(archiveDir, target) + continue + } + + // Suffix (i > cursor): must be source-present + tombstone-absent. + // Only the current target may be ahead of the cursor (crash between + // rename and progress persistence). A suffix target that is already + // tombstoned or absent indicates impossible out-of-order mutation. + if (sourceTopology === "absent") { + throw new Error(`archive gc suffix target source absent (impossible): ${target.generationId}`) + } + if (tombTopology !== "absent") { + throw new Error(`archive gc suffix target tombstone present (impossible): ${target.generationId}`) + } + revalidateSource(archiveDir, target) + revalidatePointer(archiveDir, target) + } +} + +/** Legacy compatibility wrapper — delegates to the split helpers by phase. */ +export const reconcileGcOperation = async ( + _dataDir: string, + archiveDir: string, + intent: GcOperationIntent, +): Promise => { + if (intent.phase === "complete") { + await verifyCompleteAndArchiveGc(archiveDir, intent) + } else { + await resumeFrozenTargetsAndCompleteGc(archiveDir, intent) + } +} + +/** + * Prove a GC operation's terminal invariants before retiring its journal — a + * phase label is never proof of durable reality (mirrors 3a's + * verifyCompletedOperationInvariants). Verifies: + * - completedTargets === targets.length; + * - every frozen target's source is absent AND no operation tombstone holds data; + * - every affected active pointer still equals its recorded CAS identity; + * - every affected catalog is assertCatalogExact. + * Any failure throws (the caller preserves the active journal; fail closed). + */ +export const verifyCompletedGcInvariants = async ( + archiveDir: string, + intent: GcOperationIntent, +): Promise => { + if (intent.completedTargets !== intent.targets.length) { + throw new Error( + `archive gc operation complete requires completedTargets === targets.length (${intent.targets.length}), got ${intent.completedTargets}`, + ) + } + for (const target of intent.targets) { + const sourcePath = generationRoot(archiveDir, target.signal, target.rangeStart, target.generationId) + const sourceTopology = classifyArchivePathSync(archiveDir, sourcePath, "archive gc complete source") + if (sourceTopology !== "absent") { + throw new Error(`archive gc complete but target source still exists: ${sourcePath}`) + } + // No operation tombstone may hold a generation dir. + const tomb = tombstonePath(archiveDir, intent.operationId, target) + const tombTopology = classifyArchivePathSync(archiveDir, tomb, "archive gc complete tombstone") + if (tombTopology !== "absent") { + throw new Error(`archive gc complete but tombstone still exists: ${tomb}`) + } + } + // Every affected active pointer must still equal its recorded identity. + for (const target of intent.targets) { + const current = readActiveGenerationIdStrict(archiveDir, target.signal, target.rangeStart) + if (current !== target.recordedActiveGenerationId) { + throw new Error( + `archive gc complete but active pointer changed for ${target.signal}/${target.rangeStart}: ` + + `recorded ${target.recordedActiveGenerationId}, now ${current}`, + ) + } + } + // Every affected catalog must be exact. + const affectedSignals = new Set(intent.targets.map((t) => t.signal)) + for (const signal of affectedSignals) { + const sigName = archiveSignal(signal).name as ArchiveSignalName + assertCatalogExact(archiveDir, sigName) + } +} + +const cryptoRandomUuid = (): string => randomUUID() diff --git a/apps/cli/src/server/archives/generation.ts b/apps/cli/src/server/archives/generation.ts new file mode 100644 index 000000000..ad1200ab5 --- /dev/null +++ b/apps/cli/src/server/archives/generation.ts @@ -0,0 +1,1412 @@ +import { createHash, randomUUID } from "node:crypto" +import { existsSync, readFileSync, statSync } from "node:fs" +import { lstat, rm, statfs } from "node:fs/promises" +import { dirname, join, parse, relative, resolve, sep } from "node:path" +import { arch, cpus, platform, totalmem, userInfo } from "node:os" +import { CHDB_VERSION, MAPLE_VERSION } from "../../version" +import { SCHEMA_FINGERPRINT } from "../serve" +import { + acquireCheckpointPin, + checkpointPinsRoot, + releaseCheckpointPin, + resolveCheckpoint, + withMaintenanceLock, + withRestoredCheckpoint, + type CheckpointManifest, +} from "../checkpoints" +import { durableJson, durableRename, durableWrite, syncDirectory, syncTree } from "../durable-files" +import { type ArchiveTuning, tuningRecord, type LoadedTuningConfig } from "./config" +import { archiveVolumeIdentity } from "./calibration-recovery" +import { + type ArchiveShardRecord, + type ArchiveGenerationManifest, + parseArchiveActivePointer, + readArchiveGenerationManifest, +} from "./manifest" +import { + activePointerPath, + archiveRoot, + assertArchiveRootSeparate, + assertNoSymlink, + assertRealDirectory, + assertRealFile, + buildingGenerationRoot, + buildingRoot, + catalogPath, + classifyArchivePathSync, + ensurePrivateDirectory, + generationManifestPath, + generationRoot, + newArchiveGenerationId, + nextMidnightUtc, + rangeRoot, + validateRangeDate, +} from "./paths" +import { type ArchiveSignal, archiveSignal } from "./signals" +import { COMPLEX_DIGEST_ALGORITHM, exportSignalShards, type WrittenShard } from "./export" +import { + advancePhase, + archiveCompletedOperation, + assertPointerConsistent, + inspectActiveOperation, + migrateActiveIntentIfLegacy, + migrateV2CreateIntent, + ownedPathsFor, + parseArchiveOperationIntent, + phaseAtLeast, + resolveBaseActiveGenerationId, + writeInitialIntent, + type ArchiveOperationIntent, + type ArchiveOperationPhase, + type CreateOperationIntent, + type GcOperationIntent, +} from "./journal" +import { assertCatalogExact, rebuildCatalog } from "./listing" +import { + decideReconciliation, + digestOfIntent, + type ReconciliationDecision, + type ReconciliationInspection, + type ReconciliationSnapshot, +} from "./reconcile" + +// Archive generation write, validation, promotion, and reconciliation. +// +// One archive operation seals a fixed UTC day for one signal by exporting it +// from a restored checkpoint into bounded Parquet shards, validating every +// shard, publishing an authoritative manifest, and atomically selecting the new +// generation through the active pointer. Late arrivals create a new generation +// that supersedes the old; the old generation is retained, never deleted and +// never scanned for TraceId deduplication. +// +// The whole operation holds the checkpoint maintenance lock so it cannot overlap +// checkpoint creation, restore, reset, or another archive operation. It pins +// the source checkpoint inside the lock so retention cannot delete it between +// resolution and export. Uncertain or incomplete state is preserved and +// reported; only provably owned `building//` temporary output is removed. + +export interface ArchiveGenerationFaults { + readonly afterPinAcquired?: () => void | Promise + readonly afterScratchRestored?: () => void | Promise + readonly afterBuildingCreated?: () => void | Promise + readonly afterShardsWritten?: () => void | Promise + readonly afterFirstDurableShard?: () => void + readonly afterValidationComplete?: () => void | Promise + readonly beforePublicationVolumeRecheck?: () => void | Promise + readonly afterManifestWritten?: () => void | Promise + readonly afterGenerationRenamed?: () => void | Promise + readonly afterGenerationPromoted?: () => void | Promise + readonly afterCatalogAppended?: () => void | Promise + readonly afterPinRemovedBeforeJournal?: () => void | Promise + readonly afterPinReleased?: () => void | Promise + // Pre-boundary seams for crash-safety validation (Gate 3). The after-* hooks + // above fire AFTER a durable boundary completes; they cannot inject a crash + // DURING the boundary (e.g. between a durable write and the journal advance + // that records it). These pre-boundary seams let the crash harness SIGKILL at + // the exact intra-boundary points where unwinding or the finally would mask a + // real crash. They are a committed test seam, not a production switch. + readonly beforeIntentDurable?: () => void | Promise + readonly beforePinAcquired?: () => void | Promise + readonly beforeScratchAllocated?: () => void | Promise + readonly beforeBuildingCreated?: () => void | Promise + readonly beforeManifestDurable?: () => void | Promise + readonly beforeGenerationPromoted?: () => void | Promise + readonly beforeActivePointerUpdated?: () => void | Promise + readonly beforeCatalogAppended?: () => void | Promise + readonly beforePinReleased?: () => void | Promise + readonly beforeScratchRemoved?: () => void | Promise + readonly beforeOperationArchived?: () => void | Promise +} + +export interface ArchiveGenerationResult { + readonly generationId: string + readonly signal: string + readonly rangeStart: string + readonly shardCount: number + readonly archivedRowCount: number + readonly superseded: string | null +} + +const checkpointFingerprint = (manifest: CheckpointManifest): string => + `${manifest.checkpointId}:${manifest.createdAt}:${manifest.backupBytes}` + +const sha256File = (path: string): string => createHash("sha256").update(readFileSync(path)).digest("hex") + +const toShardRecord = (shard: WrittenShard): ArchiveShardRecord => ({ + name: shard.name, + rowCount: shard.rowCount, + minEventTimeUnixNano: shard.minEventTimeUnixNano, + maxEventTimeUnixNano: shard.maxEventTimeUnixNano, + sha256: shard.sha256, + bytes: shard.bytes, + columns: shard.columns, + complexDigest: shard.complexDigest, + complexDigestAlgorithm: COMPLEX_DIGEST_ALGORITHM, +}) + +export interface FreeSpaceSnapshot { + readonly identity: string + readonly path: string + readonly freeBytes: number +} + +const addRequiredBytes = (label: string, ...parts: readonly number[]): number => { + if (parts.some((part) => !Number.isSafeInteger(part) || part < 0)) { + throw new Error(`${label} free-space requirement contains an invalid byte count`) + } + const total = parts.reduce((sum, part) => sum + part, 0) + if (!Number.isSafeInteger(total)) + throw new Error(`${label} free-space requirement exceeds the safe integer range`) + return total +} + +/** Apply the archive-output and checkpoint-restore requirements independently, + * or as one combined requirement when both paths live on the same device. */ +export const assertArchiveScratchFreeSpace = ( + archive: FreeSpaceSnapshot, + scratch: FreeSpaceSnapshot, + minFreeSpaceReserve: number, + estimatedArchiveBytes: number, + checkpointBackupBytes: number, +): void => { + const archiveRequired = addRequiredBytes("archive", minFreeSpaceReserve, estimatedArchiveBytes) + if (archive.identity === scratch.identity) { + const combinedRequired = addRequiredBytes( + "archive/scratch", + minFreeSpaceReserve, + estimatedArchiveBytes, + checkpointBackupBytes, + ) + const free = Math.min(archive.freeBytes, scratch.freeBytes) + if (free < combinedRequired) { + throw new Error( + `archive/scratch volume has ${free} bytes free, below the required ${combinedRequired} bytes ` + + `(archive reserve ${minFreeSpaceReserve} + archive working ${estimatedArchiveBytes} + ` + + `checkpoint restore ${checkpointBackupBytes}); free space or recalibrate`, + ) + } + return + } + if (archive.freeBytes < archiveRequired) { + throw new Error( + `archive volume has ${archive.freeBytes} bytes free, below the required ${archiveRequired} bytes ` + + `(reserve ${minFreeSpaceReserve} + working ${estimatedArchiveBytes}); free space or recalibrate`, + ) + } + if (scratch.freeBytes < checkpointBackupBytes) { + throw new Error( + `scratch volume has ${scratch.freeBytes} bytes free, below the required ${checkpointBackupBytes} bytes ` + + `for checkpoint restore; free space or choose a larger scratch volume`, + ) + } +} + +/** Inspect the containing volume even when the configured leaf does not yet + * exist. Device id plus filesystem type distinguishes separate filesystems. */ +const freeSpaceSnapshot = async (path: string, label: string): Promise => { + let statPath = resolve(path) + let climbs = 0 + while (!existsSync(statPath) && climbs < 64) { + statPath = resolve(statPath, "..") + climbs++ + } + if (!existsSync(statPath)) { + throw new Error(`cannot determine volume for ${label} ${path} (no existing ancestor)`) + } + const info = await statfs(statPath) + return { + identity: `dev:${statSync(statPath).dev.toString(16)}/type:${info.type}`, + path: statPath, + freeBytes: info.bavail * info.bsize, + } +} + +/** Preflight archive output and restored-checkpoint scratch before publishing + * an operation intent or acquiring a checkpoint pin. */ +const preflightArchiveScratchFreeSpace = async ( + archiveDir: string, + tuningArchiveDir: string, + scratchRoot: string, + minFreeSpaceReserve: number, + estimatedArchiveBytes: number, + checkpointBackupBytes: number, +): Promise => { + if (resolve(archiveDir) !== resolve(tuningArchiveDir)) { + throw new Error( + `archive directory mismatch: output target ${archiveDir} != tuning.archiveDir ${tuningArchiveDir}`, + ) + } + const [archive, scratch] = await Promise.all([ + freeSpaceSnapshot(archiveDir, "archive dir"), + freeSpaceSnapshot(scratchRoot, "scratch root"), + ]) + assertArchiveScratchFreeSpace( + archive, + scratch, + minFreeSpaceReserve, + estimatedArchiveBytes, + checkpointBackupBytes, + ) +} + +const assertCalibrationArchiveVolume = async ( + config: LoadedTuningConfig, + archiveDir: string, +): Promise => { + const expected = config.document.environment.archiveVolume + const canonicalArchiveDir = resolve(archiveDir) + if (expected.archiveDir !== canonicalArchiveDir) { + throw new Error( + `calibration environment mismatch: archive path ${canonicalArchiveDir} != ${expected.archiveDir}`, + ) + } + const actual = await archiveVolumeIdentity(canonicalArchiveDir) + if (actual.fsid !== expected.fsid || actual.type !== expected.type) { + throw new Error( + `calibration environment mismatch: archive volume ${actual.fsid}/${actual.type} != ${expected.fsid}/${expected.type}`, + ) + } +} + +const assertCalibrationEnvironment = async ( + config: LoadedTuningConfig, + archiveDir: string, +): Promise => { + const expected = config.document.environment + const cpuList = cpus() + const actual = { + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + executionUser: userInfo().username, + platform: platform(), + arch: arch(), + cpuModel: cpuList.length > 0 ? cpuList[0]!.model : "unknown", + cpuCount: cpuList.length, + totalMemoryBytes: totalmem(), + } + for (const key of Object.keys(actual) as Array) { + if (actual[key] !== expected[key]) { + throw new Error( + `calibration environment mismatch: ${key} ${String(actual[key])} != ${String(expected[key])}; recalibrate`, + ) + } + } + await assertCalibrationArchiveVolume(config, archiveDir) +} + +/** + * Seal one UTC day of one signal into a new archive generation. + * + * Crash-safe via a durable operation journal (Gate 3). Each boundary below is + * recorded as a phase BEFORE the next destructive step, so a SIGKILL at any + * point leaves a reconcilable record. The journal is written BEFORE pin + * acquisition (closing the orphan-pin window) and uses deterministic identities + * (pinId, scratchSubdir, generationId) so reconciliation knows exactly what an + * interrupted operation owned. + * + * The lifecycle, inside the maintenance lock: + * 1. reconcile any existing active operation (see {@link reconcileArchiveGeneration}); + * 2. resolve checkpoint; read the current active pointer as the CAS base; + * 3. write the initial intent (phase "intent"); + * 4. acquire the deterministic pin (phase "pin-acquired"); + * 5. allocate deterministic scratch + restore (phase "scratch-allocated"→"restored"); + * 6. create owned building (phase "building-created"); + * 7. export + validate shards (phase "shards-written"); + * 8. write manifest inside building/ (phase "manifest-written"); + * 9. rename building → final generation (phase "promoted"); + * 10. CAS pointer update (phase "pointer-complete"); + * 11. rebuild catalog idempotently (phase "catalog-complete"); + * 12. release owned pin (phase "pin-released"); + * 13. remove owned scratch (phase "scratch-removed"); + * 14. archive the operation journal to operations/completed/ (phase "complete"). + * + * Thrown errors and SIGKILL deliberately leave the same journal-described + * topology. Reconciliation — not exception unwinding — owns pin release, + * building quarantine, and scratch removal. + */ +export const createArchiveGeneration = async ( + dataDir: string, + archiveDir: string, + signalName: string, + rangeDate: string, + tuning: ArchiveTuning, + checkpointSelector: "current" | "previous" | string = "current", + faults: ArchiveGenerationFaults = {}, + loadedTuningConfig: LoadedTuningConfig | null = null, +): Promise => { + validateRangeDate(rangeDate) + assertArchiveRootSeparate(archiveDir, dataDir) + if (resolve(archiveDir) !== resolve(tuning.archiveDir)) { + throw new Error( + `archive directory mismatch: invocation ${resolve(archiveDir)} != configured ${resolve(tuning.archiveDir)}`, + ) + } + await assertReconciliationRoots(dataDir, archiveDir, tuning.scratchRoot) + if (loadedTuningConfig !== null) { + await assertCalibrationEnvironment(loadedTuningConfig, archiveDir) + } + const signal = archiveSignal(signalName) + const estimatedArchiveBytes = tuning.targetChunkBytes + const generationId = newArchiveGenerationId() + const operationId = randomUUID() + // Deterministic identities recorded in the journal BEFORE allocation. + const pinId = randomUUID() + const pinPurpose = `archive:${generationId}` + const scratchSubdir = `archive-${operationId}` + + return withMaintenanceLock(dataDir, operationId, async () => { + // Step 1: reconcile any prior interrupted operation before allocating a + // new one. This is the crash-recovery entry point. + await reconcileArchiveGeneration(dataDir, archiveDir, tuning.scratchRoot, faults) + // Step 2: resolve and validate the checkpoint so its immutable backup size + // can be included in scratch-volume capacity planning. This is read-only. + const resolved = await resolveCheckpoint(dataDir, checkpointSelector) + // Reject impossible archive and scratch capacity before pointer/base reads + // or creation of a durable intent/pin. A failed preflight leaves no new + // operation for reconciliation to clean up. + await preflightArchiveScratchFreeSpace( + archiveDir, + tuning.archiveDir, + tuning.scratchRoot, + tuning.minFreeSpaceReserve, + estimatedArchiveBytes, + resolved.manifest.backupBytes, + ) + // Read the CAS base (current active pointer). + const baseActiveGenerationId = resolveBaseActiveGenerationId(archiveDir, signal.name, rangeDate) + // Step 3: write the initial intent BEFORE the pin or any allocation. A + // crash here leaves only the journal; reconciliation quarantines it. + await faults.beforeIntentDurable?.() + await writeInitialIntent({ + archiveDir, + operationId, + generationId, + signal: signal.name, + rangeStart: rangeDate, + checkpointId: resolved.checkpointId, + dataDir, + scratchRoot: tuning.scratchRoot, + pinId, + pinPurpose, + scratchSubdir, + baseActiveGenerationId, + }) + // Step 4: acquire the deterministic pin. The journal already names pinId, + // so a crash between pin-write and the phase advance is reconcilable. + await faults.beforePinAcquired?.() + const pinPath = await acquireCheckpointPin(dataDir, resolved.checkpointId, pinPurpose, pinId) + await advancePhase(archiveDir, operationId, "pin-acquired") + await faults.afterPinAcquired?.() + + try { + // Steps 5–7: scratch restore + export. The beforeRestore seam records + // "scratch-allocated" after the owned scratch dir is created but before + // restore; "restored" after the db is usable. + return await withRestoredCheckpoint( + resolved, + { + scratchRoot: tuning.scratchRoot, + scratchSubdir, + cleanup: "never", + beforeRestore: async () => { + await faults.beforeScratchAllocated?.() + await advancePhase(archiveDir, operationId, "scratch-allocated") + }, + }, + async ({ db, manifest: checkpointManifest }) => { + await advancePhase(archiveDir, operationId, "restored") + await faults.afterScratchRestored?.() + const dayEndExclusiveIso = nextMidnightUtc(rangeDate) + const sourceRowCount = countSignalRowsForDay(db, signal, rangeDate) + + // Step 6: create owned building. + const building = buildingGenerationRoot(archiveDir, generationId) + await faults.beforeBuildingCreated?.() + await ensureOwnedBuilding(archiveDir, building) + await advancePhase(archiveDir, operationId, "building-created") + await faults.afterBuildingCreated?.() + + // Step 7: export + validate shards. + const shardsDir = join(building, "shards") + await ensurePrivateDirectory(shardsDir, archiveRoot(archiveDir)) + const writtenShards = exportSignalShards(db, signal, rangeDate, shardsDir, { + writerThreads: tuning.writerThreads, + rowGroupRows: tuning.rowGroupRows, + maxShardRows: tuning.maxShardRows, + maxShardBytes: tuning.maxShardBytes, + afterShardValidated: (() => { + let seen = false + return () => { + if (!seen) { + seen = true + faults.afterFirstDurableShard?.() + } + } + })(), + }) + await syncTree(shardsDir) + const archivedRowCount = writtenShards.reduce((sum, s) => sum + s.rowCount, 0) + if (archivedRowCount !== sourceRowCount) { + throw new Error( + `archive row-count mismatch for ${signal.name} ${rangeDate}: source ${sourceRowCount}, ` + + `archived ${archivedRowCount}`, + ) + } + await advancePhase(archiveDir, operationId, "shards-written") + await faults.afterShardsWritten?.() + await faults.afterValidationComplete?.() + // The volume is checked once before any durable intent and again + // immediately before publication. A replacement/mount swap during + // export must never publish a config-bound generation. + if (loadedTuningConfig !== null) { + await faults.beforePublicationVolumeRecheck?.() + await assertCalibrationArchiveVolume(loadedTuningConfig, archiveDir) + } + + // Step 8: manifest (written inside building/ by promote). + const manifest: ArchiveGenerationManifest = { + formatVersion: 3, + generationId, + signal: signal.name, + rangeStart: rangeDate, + rangeEndExclusive: dayEndExclusiveIso, + checkpointId: resolved.checkpointId, + checkpointManifestFingerprint: checkpointFingerprint(checkpointManifest), + createdAt: new Date().toISOString(), + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + sourceRowCount, + archivedRowCount, + tuning: tuningRecord(tuning), + tuningConfig: loadedTuningConfig?.identity ?? null, + shards: writtenShards.map(toShardRecord), + } + // Step 9: promote building → final generation + manifest. + await promoteGeneration( + archiveDir, + signal.name, + rangeDate, + generationId, + manifest, + building, + { + ...faults, + afterManifestWritten: async () => { + const manifestPath = join(building, "manifest.json") + const manifestSha256 = sha256File(manifestPath) + await advancePhase( + archiveDir, + operationId, + "manifest-written", + manifestSha256, + ) + await faults.afterManifestWritten?.() + }, + }, + ) + await advancePhase(archiveDir, operationId, "promoted") + // Step 10: CAS pointer update. + const superseded = await selectActiveGeneration( + archiveDir, + signal.name, + rangeDate, + generationId, + baseActiveGenerationId, + faults, + ) + await advancePhase(archiveDir, operationId, "pointer-complete") + // Step 11: rebuild catalog idempotently from manifests (never a + // blind append — a duplicate after recovery would corrupt the index). + await faults.beforeCatalogAppended?.() + await rebuildCatalog(archiveDir, signal.name) + await advancePhase(archiveDir, operationId, "catalog-complete") + await faults.afterCatalogAppended?.() + // Steps 12–14: release the owned pin, remove owned scratch, then + // archive the completed journal. Each is a recorded durable boundary + // so a SIGKILL at any of them is reconcilable. These run on the happy + // path INSIDE the journal. + await releaseCheckpointPin(dataDir, resolved.checkpointId, pinPath, pinPurpose) + await faults.afterPinRemovedBeforeJournal?.() + await advancePhase(archiveDir, operationId, "pin-released") + await faults.afterPinReleased?.() + await faults.beforeScratchRemoved?.() + await removeOwnedScratch(tuning.scratchRoot, scratchSubdir) + await advancePhase(archiveDir, operationId, "scratch-removed") + // Advance to "complete" BEFORE archiving the journal: archiving MOVES + // the op dir out of active/, so a phase advance after it would read a + // path that no longer exists. The "complete" phase is the last record + // written while the op is still in active/; archiving then retires it. + await advancePhase(archiveDir, operationId, "complete") + await faults.beforeOperationArchived?.() + await archiveCompletedOperation(archiveDir, operationId) + return { + generationId, + signal: signal.name, + rangeStart: rangeDate, + shardCount: writtenShards.length, + archivedRowCount, + superseded, + } + }, + ) + } finally { + // Deliberately no durable-state mutation here. Throw and SIGKILL must + // leave the same journal-described topology; reconciliation is the sole + // authority for pin release, quarantine, and scratch cleanup. + } + }) +} + +const countSignalRowsForDay = ( + db: { query: (sql: string, format?: string) => string }, + signal: ArchiveSignal, + rangeDate: string, +): number => { + // Use toDate() equality, not a toDateTime64: chDB's bundled ClickHouse + // miscounts aggregate count() over a toDateTime64-vs-DateTime predicate. + const sql = `SELECT count() FROM ${signal.name} WHERE toDate(${signal.eventTimeColumn}, 'UTC') = '${rangeDate}'` + return parseCount(db.query(sql, "JSONEachRow")) +} + +/** + * Strictly read the previous active pointer and return its generation id, binding + * the pointer's recorded signal/range to its on-disk location so a pointer + * copied or moved to the wrong range cannot be silently superseded (H-7). + * Throws on a malformed, mismatched, or unreadable pointer. + */ +const readPreviousPointerGenerationId = ( + pointerPath: string, + expectedSignal: string, + expectedRange: string, +): string | null => { + const parsed = JSON.parse(readFileSync(pointerPath, "utf8")) as unknown + const pointer = parseArchiveActivePointer(parsed) + if (pointer.signal !== expectedSignal) { + throw new Error( + `archive active pointer signal mismatch at ${pointerPath}: ` + + `expected ${expectedSignal}, recorded ${pointer.signal}`, + ) + } + if (pointer.rangeStart !== expectedRange) { + throw new Error( + `archive active pointer range mismatch at ${pointerPath}: ` + + `expected ${expectedRange}, recorded ${pointer.rangeStart}`, + ) + } + return pointer.generationId +} + +/** Parse a JSONEachRow count result (newline-delimited objects, not a JSON array). */ +const parseCount = (text: string): number => { + const rows = text + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as Record) + const row = rows[0] + if (!row) return 0 + const value = row["count()"] ?? row.count + const count = typeof value === "number" ? value : Number(value ?? 0) + if (!Number.isSafeInteger(count) || count < 0) throw new Error(`invalid count result: ${value}`) + return count +} + +const ensureOwnedBuilding = async (archiveDir: string, building: string): Promise => { + const root = buildingRoot(archiveDir) + // Refuse a symlinked building root or any symlinked ancestor beneath the + // archive root before creating anything (C-1): mkdir -p would otherwise + // silently create the tree under a symlink target outside the archive root. + if (existsSync(root)) { + await assertNoSymlink(archiveDir, root, "archive building root") + await assertRealDirectory(root, "archive building root") + } + await ensurePrivateDirectory(root, archiveRoot(archiveDir)) + if (existsSync(building)) { + throw new Error(`archive building generation already exists; refusing to overwrite: ${building}`) + } + await ensurePrivateDirectory(building, archiveRoot(archiveDir)) + await assertNoSymlink(archiveDir, building, "archive building generation") +} + +/** + * Move the validated building generation into its final location and write its + * manifest there. This is the "promote" boundary: after it returns, the + * generation exists at its final path with its manifest, but the active pointer + * does NOT yet select it. A separate {@link selectActiveGeneration} call flips + * the pointer — the two are split so the journal can record each as a distinct + * durable boundary (promoted → pointer-complete), making promotion crash-safe. + * + * Returns the previously-active generation id (the CAS base), or null. The old + * generation directory is retained (never deleted). + * + * Exported for filesystem-level testing of promotion without a restored chDB. + */ +export const promoteGeneration = async ( + archiveDir: string, + signal: string, + rangeDate: string, + generationId: string, + manifestValue: ArchiveGenerationManifest, + building: string, + faults: ArchiveGenerationFaults = {}, +): Promise => { + const finalGeneration = generationRoot(archiveDir, signal, rangeDate, generationId) + if (existsSync(finalGeneration)) { + await assertNoSymlink(archiveDir, finalGeneration, "archive generation") + throw new Error(`archive generation already exists; refusing to overwrite: ${finalGeneration}`) + } + const range = rangeRoot(archiveDir, signal, rangeDate) + const generationsRootAbs = generationsRootPath(archiveDir, signal, rangeDate) + // Refuse symlinked ancestors on every path we are about to create or write + // (C-1): the signal/range/generations chain is operator-controlled on disk. + await ensurePrivateDirectory(range, archiveRoot(archiveDir)) + await assertNoSymlink(archiveDir, range, "archive range") + await ensurePrivateDirectory(generationsRootAbs, archiveRoot(archiveDir)) + await assertNoSymlink(archiveDir, generationsRootAbs, "archive generations root") + // The complete manifest becomes durable INSIDE building before publication. + // The subsequent directory rename therefore publishes shards + manifest as + // one atomic unit; no final generation can exist without its manifest. + const manifestPath = join(building, "manifest.json") + await assertNoSymlink(archiveDir, manifestPath, "archive building manifest") + await faults.beforeManifestDurable?.() + await durableJson(manifestPath, manifestValue) + await syncDirectory(dirname(manifestPath)) + await faults.afterManifestWritten?.() + await faults.beforeGenerationPromoted?.() + await durableRename(building, finalGeneration) + await syncDirectory(dirname(finalGeneration)) + await faults.afterGenerationRenamed?.() +} + +/** + * Atomically select `generationId` through the active pointer for (signal, + * rangeDate). CAS-guarded: the pointer must currently equal `baseGenerationId` + * (the recorded base) OR already select `generationId` (idempotent replay). + * Anything else means concurrent activity moved the pointer and a blind + * overwrite would clobber it — fail closed. Returns the superseded generation + * id (the prior pointer value), or null. + * + * Exported for filesystem-level testing of pointer atomicity. + */ +export const selectActiveGeneration = async ( + archiveDir: string, + signal: string, + rangeDate: string, + generationId: string, + baseGenerationId: string | null, + faults: ArchiveGenerationFaults = {}, +): Promise => { + const pointerPath = activePointerPath(archiveDir, signal, rangeDate) + await assertNoSymlink(archiveDir, pointerPath, "archive active pointer") + let current: string | null = null + let superseded: string | null = null + if (existsSync(pointerPath)) { + await assertRealFile(pointerPath, "archive active pointer") + superseded = readPreviousPointerGenerationId(pointerPath, signal, rangeDate) + current = superseded + } + // CAS: the pointer must still match the recorded base, or already select the + // intended generation (idempotent). Concurrent supersession fails closed. + if (current !== baseGenerationId && current !== generationId) { + throw new Error( + `archive active pointer no longer matches base for ${signal}/${rangeDate}: ` + + `expected base ${baseGenerationId}, now ${current} (refusing to clobber)`, + ) + } + // Idempotent: if the pointer already selects this generation, nothing to do. + if (current === generationId) return superseded + await faults.beforeActivePointerUpdated?.() + await durableWrite( + pointerPath, + `${JSON.stringify({ + formatVersion: 1, + generationId, + signal, + rangeStart: rangeDate, + selectedAt: new Date().toISOString(), + })}\n`, + ) + await syncDirectory(dirname(pointerPath)) + await faults.afterGenerationPromoted?.() + return superseded +} + +const generationsRootPath = (archiveDir: string, signal: string, rangeDate: string): string => + join(rangeRoot(archiveDir, signal, rangeDate), "generations") + +/** + * Append a generation to the per-signal catalog. Exported for testing catalog + * append durability and rebuild. + */ +export const appendCatalog = async ( + archiveDir: string, + signal: string, + manifest: ArchiveGenerationManifest, + faults: ArchiveGenerationFaults = {}, +): Promise => { + const path = catalogPath(archiveDir, signal) + // Refuse a symlinked catalog (C-1): a symlinked catalog.jsonl could point + // outside the archive root and be overwritten by this append. + if (existsSync(path)) await assertRealFile(path, "archive catalog") + else await assertNoSymlink(archiveDir, path, "archive catalog") + const existing = existsSync(path) ? `${readFileSync(path, "utf8")}` : "" + const line = `${JSON.stringify({ + formatVersion: 1, + generationId: manifest.generationId, + signal: manifest.signal, + rangeStart: manifest.rangeStart, + checkpointId: manifest.checkpointId, + archivedRowCount: manifest.archivedRowCount, + shardCount: manifest.shards.length, + createdAt: manifest.createdAt, + })}\n` + // Catalog append is a durable full rewrite so the appended line is fsynced. + // A truncated final line is ignored on rebuild (see catalog rebuild). + await durableWrite(path, existing + line) + await syncDirectory(dirname(path)) + await faults.afterCatalogAppended?.() +} + +/** + * Remove the owned deterministic scratch subdirectory the operation allocated. + * Only the exact journal-named subdir beneath scratchRoot is removed; anything + * else (other operations' scratch, the scratch root itself) is over-retained. + */ +const removeOwnedScratch = async (scratchRoot: string, scratchSubdir: string): Promise => { + if (!existsSync(scratchRoot)) return + await assertExistingPathComponentsNoSymlinks(scratchRoot, "scratch root") + const scratchInfo = await lstat(resolve(scratchRoot)) + if (scratchInfo.isSymbolicLink() || !scratchInfo.isDirectory()) { + throw new Error(`refusing unsafe scratch root: ${scratchRoot}`) + } + const owned = join(resolve(scratchRoot), scratchSubdir) + if (!existsSync(owned)) return + // Containment: the subdir must be a direct child of the scratch root. + const rel = relative(resolve(scratchRoot), resolve(owned)) + if (rel === "" || rel === ".." || rel.startsWith(`..${sep}`) || rel.includes(sep)) { + throw new Error(`refusing to remove scratch path outside its root: ${owned}`) + } + await assertFilesystemPathNoSymlinks(owned, "owned scratch directory") + const ownedInfo = await lstat(owned) + if (ownedInfo.isSymbolicLink() || !ownedInfo.isDirectory()) { + throw new Error(`refusing unsafe owned scratch directory: ${owned}`) + } + // Restored ClickHouse stores legitimately contain internal table symlinks. + // fs.rm removes those links as directory entries; it does not traverse their + // targets. The security boundary is therefore the real configured root and + // exact real operation subdirectory checked above. + await rm(owned, { recursive: true, force: true }) + await syncDirectory(resolve(scratchRoot)) +} + +const pathExistsIncludingSymlinks = (root: string, path: string, label: string): boolean => + classifyArchivePathSync(root, path, label) !== "absent" + +const assertFilesystemPathNoSymlinks = async (path: string, label: string): Promise => { + const absolute = resolve(path) + if (!existsSync(absolute)) return + const info = await lstat(absolute) + if (info.isSymbolicLink()) throw new Error(`refusing symlink in ${label}: ${absolute}`) +} + +const assertExistingPathComponentsNoSymlinks = async (path: string, label: string): Promise => { + const absolute = resolve(path) + const root = parse(absolute).root + let current = root + for (const component of relative(root, absolute).split(sep).filter(Boolean)) { + current = join(current, component) + if (!existsSync(current)) break + const info = await lstat(current) + if (info.isSymbolicLink()) throw new Error(`refusing symlink in ${label}: ${current}`) + } +} + +/** + * Reconcile any active (interrupted) archive operation, driving it to its exact + * intended state or failing closed (preserving everything; D-004). Called at the + * top of {@link createArchiveGeneration} inside the maintenance lock, BEFORE + * allocating a new operation. + * + * Policy: + * - At most one active operation is permitted; more is ambiguous (fail closed). + * - A pre-publication op (phase before "promoted") owns no published generation. + * Its incomplete building output is QUARANTINED (retained, not deleted), its + * owned scratch removed, its owned pin released, and the op marked "aborted". + * - A post-promotion op (phase "promoted" onward) has a published generation. + * Reconciliation verifies it, finishes the pointer/catalog if needed, releases + * the owned pin, removes owned scratch, and archives the op to "complete". + * - Pin absence is success only at a phase where release was already authorized + * ("pin-released" onward). Otherwise it is an identity/topology error. + * + * Reconciliation is idempotent: running it twice converges to the same state. + */ +export const reconcileArchiveGeneration = async ( + dataDir: string, + archiveDir: string, + scratchRoot: string, + faults: ArchiveGenerationFaults = {}, +): Promise => { + void faults + // The SINGLE decision-driven executor. inspect → decide → switch on the + // decision and execute ONLY that branch's helpers. The decision IS the + // operative state machine — no independent re-branching. + const inspection = await inspectReconciliationState(dataDir, archiveDir, scratchRoot) + const decision = decideReconciliation(inspection) + if (decision.kind === "NoOp") return + if (decision.kind === "FailClosed") { + throw new Error(`refusing to reconcile unsafe archive state: ${decision.reason}`) + } + // v2 migration is the first action (if required). The inspector already + // validated the lifted record; write it now. + if ("migrationRequired" in decision && decision.migrationRequired) { + await migrateActiveIntentIfLegacy(archiveDir, dataDir, scratchRoot) + } + // Switch on the computed decision — each branch executes ONLY its specific + // helpers, no re-branching. + switch (decision.kind) { + case "CreateVerifyComplete": { + const { finalGeneration, building } = ownedPathsFor(decision.intent) + await verifyCompletedOperationInvariants( + dataDir, + archiveDir, + decision.intent, + finalGeneration, + building, + ) + await archiveCompletedOperation(archiveDir, decision.operationId) + return + } + case "CreateAbortPrepublication": { + const { building } = ownedPathsFor(decision.intent) + await reconcilePrePublication(dataDir, archiveDir, decision.intent, building, "aborted") + return + } + case "CreateFinishPublication": { + await verifyPublishedGeneration(archiveDir, decision.intent) + await reconcilePostPromotion(dataDir, archiveDir, decision.intent, decision.operationId) + return + } + case "GcVerifyComplete": { + const { verifyCompleteAndArchiveGc } = await import("./gc") + await verifyCompleteAndArchiveGc(archiveDir, decision.intent) + return + } + case "GcResume": { + const { resumeFrozenTargetsAndCompleteGc } = await import("./gc") + await resumeFrozenTargetsAndCompleteGc(archiveDir, decision.intent) + return + } + } +} + +const validateReconciliationTopology = async ( + dataDir: string, + archiveDir: string, + intent: CreateOperationIntent, + finalGeneration: string, + building: string, +): Promise => { + for (const [path, label] of [ + [building, "archive building generation"], + [finalGeneration, "archive final generation"], + ] as const) { + if (!existsSync(path)) continue + await assertNoSymlink(archiveDir, path, label) + await assertRealDirectory(path, label) + } + if (existsSync(building) && existsSync(finalGeneration)) { + throw new Error("archive operation has both building and final generation state") + } + const ownedScratch = join(resolve(intent.scratchRoot), intent.scratchSubdir) + if (existsSync(intent.scratchRoot)) { + await assertExistingPathComponentsNoSymlinks(intent.scratchRoot, "scratch root") + } + if (existsSync(ownedScratch)) { + await assertFilesystemPathNoSymlinks(ownedScratch, "owned scratch directory") + const info = await lstat(ownedScratch) + if (!info.isDirectory()) throw new Error(`owned scratch path is not a directory: ${ownedScratch}`) + } + await validateOwnedPinState(dataDir, intent) +} + +const validateOwnedPinState = async (dataDir: string, intent: CreateOperationIntent): Promise => { + const expectedPinPath = join(checkpointPinsRoot(dataDir), intent.checkpointId, `${intent.pinId}.json`) + if (!existsSync(expectedPinPath)) { + const absenceAuthorized = intent.phase === "intent" || phaseAtLeast(intent.phase, "catalog-complete") + if (!absenceAuthorized) { + throw new Error( + `archive operation pin is missing before release was authorized: ${expectedPinPath} (phase ${intent.phase})`, + ) + } + return + } + await assertFilesystemPathNoSymlinks(expectedPinPath, "archive operation pin") + await assertRealFile(expectedPinPath, "archive operation pin") + const raw = JSON.parse(readFileSync(expectedPinPath, "utf8")) as Record + if ( + raw.formatVersion !== 1 || + raw.pinId !== intent.pinId || + raw.checkpointId !== intent.checkpointId || + raw.purpose !== intent.pinPurpose + ) { + throw new Error(`archive operation pin identity or purpose mismatch: ${expectedPinPath}`) + } +} + +const assertReconciliationRoots = async ( + dataDir: string, + archiveDir: string, + scratchRoot: string, +): Promise => { + for (const [label, path] of [ + ["archive", resolve(archiveDir)], + ["data", resolve(dataDir)], + ["scratch", resolve(scratchRoot)], + ] as const) { + // The configured scratch leaf may not exist yet. Its existing ancestors + // are still security-critical because a later mkdir/restore would follow + // an ancestor symlink out of the configured topology. + if (label === "scratch") await assertExistingPathComponentsNoSymlinks(path, `${label} root`) + if (!existsSync(path)) continue + if (label !== "scratch") await assertFilesystemPathNoSymlinks(path, `${label} root`) + const info = await lstat(path) + if (info.isSymbolicLink() || !info.isDirectory()) { + throw new Error(`refusing unsafe ${label} root: ${path}`) + } + } +} + +const verifyPublishedGeneration = async ( + archiveDir: string, + intent: CreateOperationIntent, +): Promise => { + const finalGeneration = generationRoot(archiveDir, intent.signal, intent.rangeStart, intent.generationId) + await assertNoSymlink(archiveDir, finalGeneration, "archive final generation") + await assertRealDirectory(finalGeneration, "archive final generation") + const manifestPath = generationManifestPath( + archiveDir, + intent.signal, + intent.rangeStart, + intent.generationId, + ) + await assertNoSymlink(archiveDir, manifestPath, "archive final manifest") + await assertRealFile(manifestPath, "archive final manifest") + const actualManifestSha256 = sha256File(manifestPath) + if (actualManifestSha256 !== intent.manifestSha256) { + throw new Error( + `archive final manifest SHA-256 mismatch: journal ${intent.manifestSha256}, actual ${actualManifestSha256}`, + ) + } + const manifest = readArchiveGenerationManifest( + archiveDir, + intent.signal, + intent.rangeStart, + intent.generationId, + ) + if (manifest.checkpointId !== intent.checkpointId) { + throw new Error( + `archive final manifest checkpoint mismatch: journal ${intent.checkpointId}, manifest ${manifest.checkpointId}`, + ) + } + for (const shard of manifest.shards) { + const shardPath = join(finalGeneration, "shards", shard.name) + await assertNoSymlink(archiveDir, shardPath, `archive shard ${shard.name}`) + await assertRealFile(shardPath, `archive shard ${shard.name}`) + const actualBytes = statSync(shardPath).size + if (actualBytes !== shard.bytes) { + throw new Error( + `archive shard ${shard.name} byte size mismatch: manifest ${shard.bytes}, actual ${actualBytes}`, + ) + } + const actualSha256 = sha256File(shardPath) + if (actualSha256 !== shard.sha256) { + throw new Error( + `archive shard ${shard.name} SHA-256 mismatch: manifest ${shard.sha256}, actual ${actualSha256}`, + ) + } + } + return manifest +} + +/** + * Reconcile a pre-publication operation: the generation was never durably + * published. Quarantine any incomplete building output (retain it for + * inspection — D-004), remove only the owned scratch, release only the owned + * pin (if not already released), and mark the operation with the given end + * phase. Idempotent. + */ +const reconcilePrePublication = async ( + dataDir: string, + archiveDir: string, + intent: CreateOperationIntent, + building: string, + endPhase: ArchiveOperationPhase, +): Promise => { + // Quarantine incomplete building output if present (retain, don't delete). + if (existsSync(building)) { + // Move the building debris into a quarantine subdir named for the + // operation, retaining it for inspection. + const quarantineBuilding = join( + archiveRoot(archiveDir), + "quarantine", + `building-${intent.operationId}`, + ) + await ensurePrivateDirectory(join(archiveRoot(archiveDir), "quarantine"), archiveRoot(archiveDir)) + if (existsSync(quarantineBuilding)) { + throw new Error( + `archive operation has both building and quarantine state; refusing to retire authority: ${quarantineBuilding}`, + ) + } + await durableRename(building, quarantineBuilding) + await syncDirectory(buildingRoot(archiveDir)) + } else { + const quarantineBuilding = join( + archiveRoot(archiveDir), + "quarantine", + `building-${intent.operationId}`, + ) + if (existsSync(quarantineBuilding)) { + await assertNoSymlink(archiveDir, quarantineBuilding, "archive building quarantine") + await assertRealDirectory(quarantineBuilding, "archive building quarantine") + } + } + // Remove owned scratch. + await removeOwnedScratch(intent.scratchRoot, intent.scratchSubdir) + // Release the owned pin if it still exists. A pin absence before + // "pin-released" would be an error, but a pre-publication abort releasing its + // own pin is the intended recovery — so tolerate already-absent here. + await releaseOwnedPin(dataDir, intent) + await advancePhase(archiveDir, intent.operationId, endPhase) + // Archive the aborted operation journal to completed/ (retained for audit). + await archiveCompletedOperation(archiveDir, intent.operationId) +} + +/** + * Reconcile a post-promotion operation: the generation + manifest are + * durably published. Finish pointer/catalog if not done, release the owned pin, + * remove owned scratch, and archive the operation to "complete". Idempotent. + */ +const reconcilePostPromotion = async ( + dataDir: string, + archiveDir: string, + intent: CreateOperationIntent, + operationId: string, +): Promise => { + // Never trust pointer/catalog phase labels. Observe the pointer, require its + // CAS topology to be either the recorded base or intended generation, then + // idempotently select the intended generation. + assertPointerConsistent(archiveDir, intent) + await selectActiveGeneration( + archiveDir, + intent.signal, + intent.rangeStart, + intent.generationId, + intent.baseActiveGenerationId, + ) + if (!phaseAtLeast(intent.phase, "pointer-complete")) { + await advancePhase(archiveDir, operationId, "pointer-complete") + } + // Rebuild even when the label says complete: this safely repairs missing, + // truncated, duplicated, or stale catalog state from authoritative manifests. + await rebuildCatalog(archiveDir, archiveSignal(intent.signal).name) + assertCatalogExact(archiveDir, archiveSignal(intent.signal).name) + if (!phaseAtLeast(intent.phase, "catalog-complete")) { + await advancePhase(archiveDir, operationId, "catalog-complete") + } + if (!phaseAtLeast(intent.phase, "pin-released")) { + await releaseOwnedPin(dataDir, intent) + await advancePhase(archiveDir, operationId, "pin-released") + } else { + assertOwnedPinAbsent(dataDir, intent) + } + if (!phaseAtLeast(intent.phase, "scratch-removed")) { + await removeOwnedScratch(intent.scratchRoot, intent.scratchSubdir) + await advancePhase(archiveDir, operationId, "scratch-removed") + } else { + assertOwnedScratchAbsent(intent) + } + await advancePhase(archiveDir, operationId, "complete") + await archiveCompletedOperation(archiveDir, operationId) +} + +const assertOwnedPinAbsent = (dataDir: string, intent: CreateOperationIntent): void => { + const expectedPinPath = join(checkpointPinsRoot(dataDir), intent.checkpointId, `${intent.pinId}.json`) + if (existsSync(expectedPinPath)) { + throw new Error( + `archive operation phase requires its exact owned pin to be absent: ${expectedPinPath}`, + ) + } +} + +const assertOwnedScratchAbsent = (intent: CreateOperationIntent): void => { + const ownedScratch = join(resolve(intent.scratchRoot), intent.scratchSubdir) + if (existsSync(ownedScratch)) { + throw new Error( + `archive operation phase requires its exact owned scratch to be absent: ${ownedScratch}`, + ) + } +} + +const verifyCompletedOperationInvariants = async ( + dataDir: string, + archiveDir: string, + intent: CreateOperationIntent, + finalGeneration: string, + building: string, +): Promise => { + if (!existsSync(finalGeneration)) { + throw new Error(`complete archive operation is missing its final generation: ${finalGeneration}`) + } + if (existsSync(building)) { + throw new Error(`complete archive operation retains building state: ${building}`) + } + await verifyPublishedGeneration(archiveDir, intent) + const current = resolveBaseActiveGenerationId(archiveDir, intent.signal, intent.rangeStart) + if (current !== intent.generationId) { + throw new Error( + `complete archive operation pointer mismatch: expected ${intent.generationId}, actual ${current}`, + ) + } + assertCatalogExact(archiveDir, archiveSignal(intent.signal).name) + assertOwnedPinAbsent(dataDir, intent) + assertOwnedScratchAbsent(intent) +} + +/** + * Release the journal-owned pin, tolerating its absence ONLY if the recorded + * phase is already at-or-past "pin-released", or when recovering a + * pre-publication abort (the operation never published and owns nothing). + * Otherwise a missing pin is an identity/topology error (fail closed). This + * implements the plan rule: pin absence is success only where release was + * already authorized. + */ +const releaseOwnedPin = async (dataDir: string, intent: CreateOperationIntent): Promise => { + const expectedPinPath = join(checkpointPinsRoot(dataDir), intent.checkpointId, `${intent.pinId}.json`) + if (!existsSync(expectedPinPath)) { + const absenceAuthorized = + intent.phase === "intent" || + phaseAtLeast(intent.phase, "catalog-complete") || + phaseAtLeast(intent.phase, "pin-released") + if (absenceAuthorized) return + throw new Error( + `archive operation pin is missing before release was authorized: ${expectedPinPath} (phase ${intent.phase})`, + ) + } + await releaseCheckpointPin(dataDir, intent.checkpointId, expectedPinPath, intent.pinPurpose) +} + +// --------------------------------------------------------------------------- +// Reconciliation as ONE protocol: one inspector → one pure decision → one +// mutating executor (Gate 3b r5). The pure decideReconciliation is the sole +// branch logic. All entry points route through reconcileArchiveGenerationUnderLock. +// --------------------------------------------------------------------------- + +/** + * Inspect the active operation and produce a complete validated snapshot (or + * null / FailClosed). Runs ALL read-only validation; any failure → FailClosed. + * For v2, lifts in-memory (preserving exact phase) and sets migrationRequired. + */ +export const inspectReconciliationState = async ( + dataDir: string, + archiveDir: string, + scratchRoot: string, +): Promise => { + const inspection = inspectActiveOperation(archiveDir, dataDir, scratchRoot) + if (inspection === null) return null + if (inspection.kind === "fail-closed") return { kind: "FailClosed", reason: inspection.reason } + let migrationRequired = false + let intent: ArchiveOperationIntent + if (inspection.kind === "v2") { + migrationRequired = true + const lifted = migrateV2CreateIntent(archiveDir, inspection.raw, dataDir, scratchRoot) + intent = parseArchiveOperationIntent(archiveDir, lifted, dataDir, scratchRoot) + } else { + intent = inspection.intent + } + if (intent.kind === "create") { + try { + await assertReconciliationRoots(dataDir, archiveDir, scratchRoot) + const { finalGeneration, building } = ownedPathsFor(intent) + await validateReconciliationTopology(dataDir, archiveDir, intent, finalGeneration, building) + } catch (error) { + return { kind: "FailClosed", reason: error instanceof Error ? error.message : String(error) } + } + const { finalGeneration, building } = ownedPathsFor(intent) + const promoted = existsSync(finalGeneration) + const manifestAtFinal = existsSync( + generationManifestPath(archiveDir, intent.signal, intent.rangeStart, intent.generationId), + ) + // Branch-significant read-only preconditions: run the same checks the + // executor's branch will run, so dry-run returns FailClosed for the same + // state apply would reject. BUT only for VALID topology (not impossible + // states — the decision function gates those from the snapshot fields). + try { + if (promoted && manifestAtFinal && intent.phase !== "aborted") { + // Post-promotion with a manifest: verify it (manifest SHA + shards). + await verifyPublishedGeneration(archiveDir, intent) + // Pointer CAS: the pointer must still match the recorded base or + // already select the intended generation. A conflicting pointer is + // branch-significant — apply's reconcilePostPromotion would fail here. + assertPointerConsistent(archiveDir, intent) + } + if (phaseAtLeast(intent.phase, "complete") && promoted) { + // Terminal: verify all complete invariants (no repair). + await verifyCompletedOperationInvariants( + dataDir, + archiveDir, + intent, + finalGeneration, + building, + ) + } + // Preflight destination collisions: if the completed-op destination or + // quarantine destination already exists (including as a broken symlink), + // apply would mutate before failing. Use lstatSync (catches dangling + // symlinks that existsSync misses) before the decision authorizes actions. + const completedDest = join( + join(archiveRoot(archiveDir), "operations", "completed"), + `archive-${inspection.operationId}`, + ) + if ( + pathExistsIncludingSymlinks( + archiveDir, + completedDest, + "completed archive operation destination", + ) + ) { + throw new Error( + `completed archive operation already exists; refusing to overwrite: ${completedDest}`, + ) + } + if (!promoted && existsSync(building)) { + const quarantineDest = join( + archiveRoot(archiveDir), + "quarantine", + `building-${inspection.operationId}`, + ) + if ( + pathExistsIncludingSymlinks( + archiveDir, + quarantineDest, + "archive building quarantine destination", + ) + ) { + throw new Error( + `archive operation has both building and quarantine state; refusing to retire authority: ${quarantineDest}`, + ) + } + } + } catch (error) { + return { kind: "FailClosed", reason: error instanceof Error ? error.message : String(error) } + } + const snapshot: ReconciliationSnapshot = { + operationId: inspection.operationId, + journalDigest: digestOfIntent(intent), + migrationRequired, + intent, + promoted, + manifestAtFinal, + buildingPresent: existsSync(building), + buildingAndFinalBothPresent: existsSync(building) && existsSync(finalGeneration), + remainingTargets: 0, + affectedSignals: [], + } + return { kind: "ValidSnapshot", snapshot } + } + // GC: run root safety + the same terminal/resume preconditions the executor checks. + try { + await assertReconciliationRoots(dataDir, archiveDir, scratchRoot) + } catch (error) { + return { kind: "FailClosed", reason: error instanceof Error ? error.message : String(error) } + } + const gc = intent as GcOperationIntent + try { + // Preflight destination collision (same as create, symlink-aware). + const completedDest = join( + join(archiveRoot(archiveDir), "operations", "completed"), + `archive-${inspection.operationId}`, + ) + if ( + pathExistsIncludingSymlinks(archiveDir, completedDest, "completed archive operation destination") + ) { + throw new Error( + `completed archive operation already exists; refusing to overwrite: ${completedDest}`, + ) + } + if (gc.phase === "complete") { + // Terminal GC: verify all invariants (no repair). + const { verifyCompletedGcInvariants } = await import("./gc") + await verifyCompletedGcInvariants(archiveDir, gc) + } else { + // GC resume: preflight EVERY remaining target before the decision + // authorizes any mutation. Source/tombstone topology, evidence, pointer + // CAS, and both-present detection — all checked read-only here, so + // dry-run returns FailClosed for the same state apply would reject + // AFTER partially deleting earlier targets. + const { preflightGcTargets } = await import("./gc") + await preflightGcTargets(archiveDir, gc) + } + } catch (error) { + return { kind: "FailClosed", reason: error instanceof Error ? error.message : String(error) } + } + const snapshot: ReconciliationSnapshot = { + operationId: inspection.operationId, + journalDigest: digestOfIntent(intent), + migrationRequired, + intent, + promoted: false, + manifestAtFinal: false, + buildingPresent: false, + buildingAndFinalBothPresent: false, + remainingTargets: gc.targets.length - gc.completedTargets, + affectedSignals: [...new Set(gc.targets.map((t) => t.signal))], + } + return { kind: "ValidSnapshot", snapshot } +} + +/** + * The ONE under-lock reconciliation function, shared by CLI, automatic create, + * and automatic GC. Dry-run: inspect → decide → return the decision (no + * mutation). Apply: call reconcileArchiveGeneration (which itself inspects → + * decides → executes the branch — the decision IS operative). Then capture the + * postcondition under the same lock. + */ +export const reconcileArchiveGenerationUnderLock = async ( + dataDir: string, + archiveDir: string, + scratchRoot: string, + options: { readonly dryRun: boolean } = { dryRun: false }, +): Promise => { + if (options.dryRun) { + const inspection = await inspectReconciliationState(dataDir, archiveDir, scratchRoot) + return decideReconciliation(inspection) + } + // Apply: reconcileArchiveGeneration is the decision-driven executor (it + // inspects, decides, and switches on the decision). All entry points call + // this same function. + await reconcileArchiveGeneration(dataDir, archiveDir, scratchRoot) + // Capture the terminal postcondition under the lock. + return decideReconciliation(await inspectReconciliationState(dataDir, archiveDir, scratchRoot)) +} + +/** + * The CLI entry point. Acquires the lock, then calls the under-lock function. + * A FailClosed decision in apply mode surfaces as a throw (nonzero, preserve). + */ +export const runArchiveReconciliation = async ( + dataDir: string, + archiveDir: string, + scratchRoot: string, + options: { readonly dryRun: boolean } = { dryRun: false }, +): Promise => { + const lockOperationId = randomUUID() + return withMaintenanceLock(dataDir, lockOperationId, async () => { + const decision = await reconcileArchiveGenerationUnderLock(dataDir, archiveDir, scratchRoot, options) + if (!options.dryRun && decision.kind === "FailClosed") { + throw new Error(`refusing to reconcile unsafe archive state: ${decision.reason}`) + } + return decision + }) +} diff --git a/apps/cli/src/server/archives/journal.ts b/apps/cli/src/server/archives/journal.ts new file mode 100644 index 000000000..3e68d531d --- /dev/null +++ b/apps/cli/src/server/archives/journal.ts @@ -0,0 +1,1260 @@ +import { existsSync, lstatSync, readdirSync, readFileSync } from "node:fs" +import { join, resolve } from "node:path" +import { durableJson, durableRename, durableRemove, syncDirectory } from "../durable-files" +import { + activePointerPath, + archiveQuarantineRoot, + archiveRoot, + assertNoSymlink, + assertNoSymlinkSync, + assertRealDirectory, + assertRealFile, + assertRealFileSync, + ensurePrivateDirectory, + rangeRoot, + signalRoot, + validateArchiveId, + validateRangeDate, +} from "./paths" +import { parseArchiveActivePointer } from "./manifest" +import { archiveSignal } from "./signals" + +// Archive generation operation journal and reconciliation (Gate 3). +// +// `createArchiveGeneration` performs a multi-step durable state transition +// (resolve → pin → scratch restore → export → validate → promote → pointer → +// catalog → unpin → cleanup). A process kill at any step can leave an orphan +// pin, dangling scratch, or a half-published generation that the next run must +// reconcile correctly. The `finally` block of the operation runs on a thrown +// error but NOT on a real SIGKILL, so the journal — not the finally — is the +// authority for crash recovery. +// +// This module ports the checkpoint subsystem's proven crash-safety pattern +// (reconcileCheckpointOperations in checkpoints.ts): a versioned intent journal +// written BEFORE any destructive boundary, recording exact identities, that the +// next operation reconciles to its exact intended state or fails closed +// (preserving everything; D-004). The journal may be behind filesystem reality +// but never ahead: it records the LAST completed durable boundary, so +// reconciliation validates recorded identity against observed topology before +// acting. +// +// One active operation is permitted at a time. The maintenance lock serializes +// operations, so at most one `operations/active/` entry should exist; if more +// than one is found, the state is ambiguous and reconciliation fails closed. + +/** + * Versioned journal format. The parser accepts only this version (fail-closed on + * any other). Gate 3b raises v2 → v3 to introduce the `kind` discriminator + * (create vs gc) so reconcile can dispatch on operation type. A v2 create intent + * is migrated to v3 under the maintenance lock by {@link migrateV2CreateIntent}; + * the parser never silently reinterprets a v2 record. + */ +export const ARCHIVE_OPERATION_FORMAT_VERSION = 3 as const + +/** Operation kind discriminator recorded in every v3 intent. */ +export const ARCHIVE_OPERATION_KINDS = ["create", "gc"] as const +export type ArchiveOperationKind = (typeof ARCHIVE_OPERATION_KINDS)[number] + +/** + * Phases record the last COMPLETED durable boundary. Advancement happens only + * AFTER the named boundary is fsync-durable, so the journal is never ahead of + * the filesystem. Reconciliation reads the phase to know what is owned and what + * remains. + * + * Ordering: each phase implies every earlier boundary is also durable. + */ +export const ARCHIVE_OPERATION_PHASES = [ + "intent", // journal durably written; pin not yet acquired + "pin-acquired", // the journal-named pin exists + "scratch-allocated", // owned scratch subdir created + "restored", // checkpoint restored into scratch; db open was possible + "building-created", // owned building// created + "shards-written", // all shards durably written under building//shards/ + "manifest-written", // generation manifest written inside building// + "promoted", // building/ renamed to final generations// location + "pointer-complete", // active pointer durably selects this generation + "catalog-complete", // catalog rebuilt/upserted + "pin-released", // the journal-named pin removed + "scratch-removed", // owned scratch subdir removed + "gc-collecting", // (GC only) ≥1 target collected; catalog not yet rebuilt. Cursor in completedTargets. + "complete", // operation journal moved to operations/completed/ + "aborted", // pre-publication op reconciled away cleanly (nothing published) +] as const +export type ArchiveOperationPhase = (typeof ARCHIVE_OPERATION_PHASES)[number] + +export const PHASE_ORDER: Readonly> = Object.fromEntries( + ARCHIVE_OPERATION_PHASES.map((phase, index) => [phase, index]), +) as Readonly> + +export const phaseAtLeast = (a: ArchiveOperationPhase, b: ArchiveOperationPhase): boolean => + PHASE_ORDER[a] >= PHASE_ORDER[b] + +/** + * Phases valid for each operation kind. The parser rejects kind-incompatible + * phases so a GC intent can never carry a create-only phase (e.g. pin-acquired) + * and a create intent can never carry gc-collecting. This closes the + * "phase label substitutes for reality" defect: a GC op's progress is recorded + * ONLY as gc-collecting (with a cursor) or complete, never as a create phase. + */ +export const CREATE_PHASES: ReadonlySet = new Set([ + "intent", + "pin-acquired", + "scratch-allocated", + "restored", + "building-created", + "shards-written", + "manifest-written", + "promoted", + "pointer-complete", + "catalog-complete", + "pin-released", + "scratch-removed", + "complete", + "aborted", +]) +export const GC_PHASES: ReadonlySet = new Set(["intent", "gc-collecting", "complete"]) + +const phaseRequiresManifest = (phase: ArchiveOperationPhase): boolean => + phase !== "aborted" && phaseAtLeast(phase, "manifest-written") + +/** Fields common to every operation kind. */ +export interface ArchiveOperationBase { + readonly formatVersion: typeof ARCHIVE_OPERATION_FORMAT_VERSION + readonly kind: ArchiveOperationKind + readonly operationId: string + /** Configured roots recorded so reconciliation can locate owned state. */ + readonly archiveDir: string + readonly dataDir: string + readonly scratchRoot: string + readonly phase: ArchiveOperationPhase + readonly createdAt: string + readonly updatedAt: string +} + +/** + * A create-generation operation intent (Gate 3a). Carries the deterministic + * pin/scratch/generation identities and the manifest SHA-256 once published. + */ +export interface CreateOperationIntent extends ArchiveOperationBase { + readonly kind: "create" + readonly generationId: string + readonly signal: string + readonly rangeStart: string + readonly checkpointId: string + /** Deterministic identities recorded BEFORE allocation. */ + readonly pinId: string + readonly pinPurpose: string + readonly scratchSubdir: string + /** SHA-256 of the exact durable manifest bytes once phase >= manifest-written. */ + readonly manifestSha256: string | null + /** The generation this operation supersedes, or null if none (CAS base). */ + readonly baseActiveGenerationId: string | null +} + +/** + * A single target of garbage collection: one superseded generation to delete, + * with the evidence (manifest SHA + per-shard bytes/SHA) reconciliation uses to + * revalidate the source before each tombstone rename, and the recorded active + * generation for its range so collection can refuse a pointer that came back. + */ +export interface GcTarget { + readonly signal: string + readonly rangeStart: string + readonly generationId: string + readonly createdAt: string + readonly manifestSha256: string + readonly bytes: number + readonly shards: ReadonlyArray<{ readonly name: string; readonly bytes: number; readonly sha256: string }> + /** The exact active generation recorded for this target's range at plan time. */ + readonly recordedActiveGenerationId: string +} + +/** + * A garbage-collection operation intent (Gate 3b). Records the FROZEN deletion + * set computed under the maintenance lock. Reconciliation drives the frozen set + * to completion idempotently and NEVER expands it — a resumed GC deletes exactly + * what the original decided. + */ +export interface GcOperationIntent extends ArchiveOperationBase { + readonly kind: "gc" + readonly keep: number + readonly targets: ReadonlyArray + /** Number of targets whose deletion has completed (progress cursor). */ + readonly completedTargets: number +} + +export type ArchiveOperationIntent = CreateOperationIntent | GcOperationIntent + +/** Directory holding a single active operation's journal. */ +export const operationDir = (archiveDir: string, operationId: string): string => + join(activeOperationsRoot(archiveDir), `archive-${validateArchiveId(operationId, "operation")}`) + +/** `/operations/active/` — holds the single permitted active op. */ +export const activeOperationsRoot = (archiveDir: string): string => join(operationsRoot(archiveDir), "active") + +/** `/operations/completed/` — retained records of completed ops. */ +export const completedOperationsRoot = (archiveDir: string): string => + join(operationsRoot(archiveDir), "completed") + +const operationsRoot = (archiveDir: string): string => join(archiveRoot(archiveDir), "operations") + +const intentPath = (archiveDir: string, operationId: string): string => + join(operationDir(archiveDir, operationId), "intent.json") + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null + +const requiredString = (value: unknown, field: string): string => { + if (typeof value !== "string" || value.length === 0) + throw new Error(`journal field ${field} missing or not a string`) + return value +} + +/** + * Strict parse of an operation intent. Validates format version, the `kind` + * discriminator, every identity, phase, and path containment, then dispatches + * create-vs-gc field parsing. Throws on any defect (fail-closed); the caller + * preserves the offending files. Parsed identities are validated to be real + * archive IDs / range dates so a corrupted or hand-edited journal cannot direct + * reconciliation at arbitrary paths. A v2 (pre-kind) record is rejected here; + * use {@link migrateV2CreateIntent} to lift a v2 create intent to v3 under the + * maintenance lock before parsing. + */ +export const parseArchiveOperationIntent = ( + archiveDir: string, + raw: unknown, + expectedDataDir?: string, + expectedScratchRoot?: string, +): ArchiveOperationIntent => { + if (!isRecord(raw)) throw new Error("archive operation intent is not a record") + if (raw.formatVersion !== ARCHIVE_OPERATION_FORMAT_VERSION) { + throw new Error(`unsupported archive operation format version: ${String(raw.formatVersion)}`) + } + const kind = requiredString(raw.kind, "kind") as ArchiveOperationKind + if (!ARCHIVE_OPERATION_KINDS.includes(kind)) { + throw new Error(`invalid archive operation kind: ${kind}`) + } + const operationId = validateArchiveId(requiredString(raw.operationId, "operationId"), "operation") + const phase = requiredString(raw.phase, "phase") as ArchiveOperationPhase + if (!ARCHIVE_OPERATION_PHASES.includes(phase)) { + throw new Error(`invalid archive operation phase: ${phase}`) + } + // Kind-phase strictness: a phase label must be valid for the operation kind. + // A GC intent may only be intent / gc-collecting / complete; a create intent + // may only use create-eligible phases. This prevents a GC op from carrying a + // create-only phase (which would let a phase label substitute for reality). + const validPhases = kind === "create" ? CREATE_PHASES : GC_PHASES + if (!validPhases.has(phase)) { + throw new Error(`archive operation phase ${phase} is not valid for kind ${kind}`) + } + const recordedArchiveDir = resolve(requiredString(raw.archiveDir, "archiveDir")) + const recordedDataDir = resolve(requiredString(raw.dataDir, "dataDir")) + const recordedScratchRoot = resolve(requiredString(raw.scratchRoot, "scratchRoot")) + if (recordedArchiveDir !== resolve(archiveDir)) { + throw new Error( + `archive operation root mismatch: journal ${recordedArchiveDir}, invocation ${resolve(archiveDir)}`, + ) + } + if (expectedDataDir !== undefined && recordedDataDir !== resolve(expectedDataDir)) { + throw new Error( + `archive operation data root mismatch: journal ${recordedDataDir}, invocation ${resolve(expectedDataDir)}`, + ) + } + if (expectedScratchRoot !== undefined && recordedScratchRoot !== resolve(expectedScratchRoot)) { + throw new Error( + `archive operation scratch root mismatch: journal ${recordedScratchRoot}, invocation ${resolve(expectedScratchRoot)}`, + ) + } + const createdAt = requiredString(raw.createdAt, "createdAt") + const updatedAt = requiredString(raw.updatedAt, "updatedAt") + // Roots are recorded for inspection/recovery; they are not authority to act + // outside the archive root. The archive root itself is re-derived. + if (kind === "create") + return parseCreateIntent( + raw, + operationId, + phase, + recordedArchiveDir, + recordedDataDir, + recordedScratchRoot, + createdAt, + updatedAt, + ) + return parseGcIntent( + raw, + operationId, + phase, + recordedArchiveDir, + recordedDataDir, + recordedScratchRoot, + createdAt, + updatedAt, + ) +} + +const parseCreateIntent = ( + raw: Record, + operationId: string, + phase: ArchiveOperationPhase, + recordedArchiveDir: string, + recordedDataDir: string, + recordedScratchRoot: string, + createdAt: string, + updatedAt: string, +): CreateOperationIntent => { + const generationId = validateArchiveId(requiredString(raw.generationId, "generationId"), "generation") + const signal = archiveSignal(requiredString(raw.signal, "signal")).name + const rangeStart = validateRangeDate(requiredString(raw.rangeStart, "rangeStart")) + const checkpointId = validateArchiveId(requiredString(raw.checkpointId, "checkpointId"), "checkpoint") + const pinId = validateArchiveId(requiredString(raw.pinId, "pinId"), "pin") + const scratchSubdir = requiredString(raw.scratchSubdir, "scratchSubdir") + if (scratchSubdir !== `archive-${operationId}`) { + throw new Error(`invalid scratch subdir in journal: ${scratchSubdir}`) + } + const pinPurpose = requiredString(raw.pinPurpose, "pinPurpose") + if (pinPurpose !== `archive:${generationId}`) { + throw new Error(`archive operation pin purpose does not match generation: ${pinPurpose}`) + } + const manifestSha256Raw = raw.manifestSha256 + const manifestSha256 = + manifestSha256Raw === null ? null : requiredString(manifestSha256Raw, "manifestSha256").toLowerCase() + if (manifestSha256 !== null && !/^[0-9a-f]{64}$/.test(manifestSha256)) { + throw new Error("invalid archive operation manifestSha256") + } + if (phaseRequiresManifest(phase) !== (manifestSha256 !== null)) { + throw new Error(`archive operation manifest hash is inconsistent with phase ${phase}`) + } + const baseActiveGenerationIdRaw = raw.baseActiveGenerationId + const baseActiveGenerationId = + baseActiveGenerationIdRaw === null + ? null + : validateArchiveId( + requiredString(baseActiveGenerationIdRaw, "baseActiveGenerationId"), + "base generation", + ) + return { + formatVersion: ARCHIVE_OPERATION_FORMAT_VERSION, + kind: "create", + operationId, + generationId, + signal, + rangeStart, + checkpointId, + archiveDir: recordedArchiveDir, + dataDir: recordedDataDir, + scratchRoot: recordedScratchRoot, + pinId, + pinPurpose, + scratchSubdir, + manifestSha256, + baseActiveGenerationId, + phase, + createdAt, + updatedAt, + } +} + +const parseGcIntent = ( + raw: Record, + operationId: string, + phase: ArchiveOperationPhase, + recordedArchiveDir: string, + recordedDataDir: string, + recordedScratchRoot: string, + createdAt: string, + updatedAt: string, +): GcOperationIntent => { + const keepRaw = raw.keep + if (typeof keepRaw !== "number" || !Number.isSafeInteger(keepRaw) || keepRaw < 0) { + throw new Error(`archive operation gc intent has invalid keep: ${String(keepRaw)}`) + } + const completedTargetsRaw = raw.completedTargets + if ( + typeof completedTargetsRaw !== "number" || + !Number.isSafeInteger(completedTargetsRaw) || + completedTargetsRaw < 0 + ) { + throw new Error( + `archive operation gc intent has invalid completedTargets: ${String(completedTargetsRaw)}`, + ) + } + const targetsRaw = raw.targets + if (!Array.isArray(targetsRaw)) { + throw new Error("archive operation gc intent targets is not an array") + } + const targets: GcTarget[] = targetsRaw.map((t, i) => parseGcTarget(t, i)) + if (completedTargetsRaw > targets.length) { + throw new Error( + `archive operation gc intent completedTargets ${completedTargetsRaw} exceeds targets ${targets.length}`, + ) + } + // Duplicate-target detection: each (signal, range, generationId) must be unique. + // A duplicate would let collection double-count or confuse the resume cursor. + const seenKeys = new Set() + for (const t of targets) { + const key = `${t.signal}/${t.rangeStart}/${t.generationId}` + if (seenKeys.has(key)) { + throw new Error(`archive operation gc intent has a duplicate target: ${key}`) + } + seenKeys.add(key) + } + // Phase/cursor consistency (the core fix for the premature-complete defect): + // - intent requires completedTargets === 0 + // - gc-collecting allows 0 <= completedTargets <= targets.length + // - complete REQUIRES completedTargets === targets.length (the terminal state + // is only reachable after every target is collected + catalogs repaired) + if (phase === "intent" && completedTargetsRaw !== 0) { + throw new Error( + `archive operation gc intent phase intent requires completedTargets 0, got ${completedTargetsRaw}`, + ) + } + if (phase === "complete" && completedTargetsRaw !== targets.length) { + throw new Error( + `archive operation gc intent phase complete requires completedTargets === targets.length (${targets.length}), got ${completedTargetsRaw}`, + ) + } + return { + formatVersion: ARCHIVE_OPERATION_FORMAT_VERSION, + kind: "gc", + operationId, + keep: keepRaw, + targets, + completedTargets: completedTargetsRaw, + archiveDir: recordedArchiveDir, + dataDir: recordedDataDir, + scratchRoot: recordedScratchRoot, + phase, + createdAt, + updatedAt, + } +} + +const parseGcTarget = (raw: unknown, index: number): GcTarget => { + if (!isRecord(raw)) throw new Error(`archive gc target ${index} is not a record`) + const signal = archiveSignal(requiredString(raw.signal, "signal")).name + const rangeStart = validateRangeDate(requiredString(raw.rangeStart, "rangeStart")) + const generationId = validateArchiveId(requiredString(raw.generationId, "generationId"), "generation") + const createdAt = requiredString(raw.createdAt, "createdAt") + const manifestSha256 = requiredString(raw.manifestSha256, "manifestSha256").toLowerCase() + if (!/^[0-9a-f]{64}$/.test(manifestSha256)) { + throw new Error(`archive gc target ${index} has invalid manifestSha256`) + } + const bytesRaw = raw.bytes + if (typeof bytesRaw !== "number" || !Number.isSafeInteger(bytesRaw) || bytesRaw < 0) { + throw new Error(`archive gc target ${index} has invalid bytes: ${String(bytesRaw)}`) + } + const recordedActiveGenerationId = validateArchiveId( + requiredString(raw.recordedActiveGenerationId, "recordedActiveGenerationId"), + "active generation", + ) + const shardsRaw = raw.shards + if (!Array.isArray(shardsRaw)) { + throw new Error(`archive gc target ${index} shards is not an array`) + } + const shards = shardsRaw.map((s, j) => { + if (!isRecord(s)) throw new Error(`archive gc target ${index} shard ${j} is not a record`) + const name = requiredString(s.name, "name") + const bytes = s.bytes + if (typeof bytes !== "number" || !Number.isSafeInteger(bytes) || bytes < 0) { + throw new Error(`archive gc target ${index} shard ${j} invalid bytes`) + } + const sha256 = requiredString(s.sha256, "sha256").toLowerCase() + if (!/^[0-9a-f]{64}$/.test(sha256)) { + throw new Error(`archive gc target ${index} shard ${j} invalid sha256`) + } + return { name, bytes, sha256 } + }) + return { + signal, + rangeStart, + generationId, + createdAt, + manifestSha256, + bytes: bytesRaw, + shards, + recordedActiveGenerationId, + } +} + +/** + * Migrate a v2 (pre-kind) create intent record to v3 under the maintenance lock. + * This is a mechanical, validated field addition: `kind: "create"` is the only + * new field, and no existing semantics change. The input is re-validated strictly + * so a corrupt v2 record fails migration (and reconciliation) rather than being + * silently lifted. Returns the v3 record (unparsed, for durably rewriting the + * intent.json) — callers should re-parse via {@link parseArchiveOperationIntent}. + * + * Never used for a v1 record or any record that is not a clean v2 create intent. + */ +export const migrateV2CreateIntent = ( + archiveDir: string, + raw: unknown, + expectedDataDir?: string, + expectedScratchRoot?: string, +): Record => { + if (!isRecord(raw)) throw new Error("archive operation intent is not a record (v2 migration)") + if (raw.formatVersion !== 2) { + throw new Error(`v2 migration requires formatVersion 2, got ${String(raw.formatVersion)}`) + } + // Re-validate every v2 field strictly before lifting. Reuse the create parser + // by synthesizing a v3 record: parse it, then emit it. This guarantees the + // migrated record passes the v3 parser byte-for-byte. + const lifted: Record = { + ...raw, + formatVersion: ARCHIVE_OPERATION_FORMAT_VERSION, + kind: "create", + } + // parseArchiveOperationIntent validates all fields + the kind discriminator. + parseArchiveOperationIntent(archiveDir, lifted, expectedDataDir, expectedScratchRoot) + return lifted +} + +/** + * If the single active operation's intent is a legacy v2 record, durably migrate + * it to v3 under the maintenance lock BEFORE reading/parsing it. Returns true if + * a migration occurred. A v2 record left by a pre-v3 binary (Gate 3a) would + * otherwise strand — the v3 parser rejects it and reconciliation fails closed, + * blocking all future archive work. This lifts it so reconcile can proceed. + * + * No-op when there is no active op, more than one (ambiguous), or the record is + * already v3. A malformed v2 record fails migration and reconcile fails closed. + */ +export const migrateActiveIntentIfLegacy = async ( + archiveDir: string, + expectedDataDir?: string, + expectedScratchRoot?: string, +): Promise => { + // Consume the ONE authoritative inspector. All pre-write checks — no-symlink, + // real-file, directory/record identity binding, clean v2 lift — run INSIDE the + // inspector BEFORE this function rewrites anything. A symlinked intent, a + // mismatched ID, or a corrupt v2 record is detected (and surfaced fail-closed) + // before any durableJson write. Never reread through a weaker path. + const inspection = inspectActiveOperation(archiveDir, expectedDataDir, expectedScratchRoot) + if (inspection === null) return false + if (inspection.kind === "fail-closed") { + // Unsafe state: surface as an error (reconciliation must fail closed, not + // silently skip migration). The caller is inside the maintenance lock and + // will propagate this as a nonzero failure. + throw new Error(`refusing to migrate unsafe active operation: ${inspection.reason}`) + } + if (inspection.kind !== "v2") return false // v3 (or gc): nothing to migrate. + const path = intentPath(archiveDir, inspection.operationId) + // Re-derive the lifted record (the inspector validated it already; recompute + // rather than stash, to keep the inspector read-only). + const lifted = migrateV2CreateIntent(archiveDir, inspection.raw, expectedDataDir, expectedScratchRoot) + const { durableJson } = await import("../durable-files") + await durableJson(path, lifted) + await syncDirectory(operationDir(archiveDir, inspection.operationId)) + return true +} + +/** Read and strictly parse the intent for an operation dir. */ +const readIntent = ( + archiveDir: string, + operationId: string, + expectedDataDir?: string, + expectedScratchRoot?: string, +): ArchiveOperationIntent => { + const path = intentPath(archiveDir, operationId) + assertNoSymlinkSync(archiveDir, path, "archive operation intent") + assertRealFileSync(path, "archive operation intent") + const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown + return parseArchiveOperationIntent(archiveDir, parsed, expectedDataDir, expectedScratchRoot) +} + +/** + * Persist the initial intent BEFORE pin acquisition or any allocation. The + * recorded identities (pinId, scratchSubdir, generationId) are the exact ones + * the operation will allocate, so a crash between journal-write and allocation + * leaves a reconcilable record of intended ownership. + */ +export const writeInitialIntent = async (intent: { + readonly archiveDir: string + readonly operationId: string + readonly generationId: string + readonly signal: string + readonly rangeStart: string + readonly checkpointId: string + readonly dataDir: string + readonly scratchRoot: string + readonly pinId: string + readonly pinPurpose: string + readonly scratchSubdir: string + readonly baseActiveGenerationId: string | null +}): Promise => { + const dir = operationDir(intent.archiveDir, intent.operationId) + await ensurePrivateDirectory(dir, archiveRoot(intent.archiveDir)) + await assertNoSymlink(intent.archiveDir, dir, "archive operation") + const now = new Date().toISOString() + const record: CreateOperationIntent = { + formatVersion: ARCHIVE_OPERATION_FORMAT_VERSION, + kind: "create", + operationId: intent.operationId, + generationId: intent.generationId, + signal: intent.signal, + rangeStart: intent.rangeStart, + checkpointId: intent.checkpointId, + archiveDir: resolve(intent.archiveDir), + dataDir: resolve(intent.dataDir), + scratchRoot: resolve(intent.scratchRoot), + pinId: intent.pinId, + pinPurpose: intent.pinPurpose, + scratchSubdir: intent.scratchSubdir, + manifestSha256: null, + baseActiveGenerationId: intent.baseActiveGenerationId, + phase: "intent", + createdAt: now, + updatedAt: now, + } + await durableJson(intentPath(intent.archiveDir, intent.operationId), record) + await syncDirectory(dir) +} + +/** + * Advance the recorded phase to the next completed durable boundary. Called + * only AFTER the named boundary is fsync-durable. Reads the current intent, + * validates the transition is a forward step, and rewrites it durably. + * `manifestSha256` applies only to create intents (ignored for gc). + */ +export const advancePhase = async ( + archiveDir: string, + operationId: string, + next: ArchiveOperationPhase, + manifestSha256?: string, +): Promise => { + const current = readIntent(archiveDir, operationId) + // Allow re-advancing to the same phase (idempotent reconciliation replay) + // but refuse a backward or invalid transition. + if (PHASE_ORDER[next] < PHASE_ORDER[current.phase]) { + throw new Error(`archive operation phase regression: ${current.phase} -> ${next}`) + } + let updated: ArchiveOperationIntent + if (current.kind === "create") { + const nextManifestSha256 = next === "aborted" ? null : (manifestSha256 ?? current.manifestSha256) + if ( + phaseRequiresManifest(next) && + (nextManifestSha256 === null || !/^[0-9a-f]{64}$/.test(nextManifestSha256)) + ) { + throw new Error(`archive operation phase ${next} requires a manifest SHA-256`) + } + updated = { + ...current, + phase: next, + manifestSha256: nextManifestSha256, + updatedAt: new Date().toISOString(), + } + } else { + updated = { ...current, phase: next, updatedAt: new Date().toISOString() } + } + await durableJson(intentPath(archiveDir, operationId), updated) + await syncDirectory(operationDir(archiveDir, operationId)) + return updated +} + +/** + * Persist the initial GC intent BEFORE any collection. Records the FROZEN + * deletion set computed under the maintenance lock, so a crashed/resumed GC + * deletes exactly what the original decided — never re-expanded. + */ +export const writeGcIntent = async (intent: { + readonly archiveDir: string + readonly operationId: string + readonly dataDir: string + readonly scratchRoot: string + readonly keep: number + readonly targets: ReadonlyArray +}): Promise => { + const dir = operationDir(intent.archiveDir, intent.operationId) + await ensurePrivateDirectory(dir, archiveRoot(intent.archiveDir)) + await assertNoSymlink(intent.archiveDir, dir, "archive operation") + const now = new Date().toISOString() + const record: GcOperationIntent = { + formatVersion: ARCHIVE_OPERATION_FORMAT_VERSION, + kind: "gc", + operationId: intent.operationId, + keep: intent.keep, + targets: intent.targets, + completedTargets: 0, + archiveDir: resolve(intent.archiveDir), + dataDir: resolve(intent.dataDir), + scratchRoot: resolve(intent.scratchRoot), + phase: "intent", + createdAt: now, + updatedAt: now, + } + await durableJson(intentPath(intent.archiveDir, intent.operationId), record) + await syncDirectory(dir) +} + +/** + * Rewrite a GC intent's progress cursor (completedTargets) + phase. Used by the + * crash-safe collector after each target completes so a resume resumes at the + * right point. The frozen target list is never mutated. + */ +export const persistGcProgress = async ( + archiveDir: string, + operationId: string, + completedTargets: number, + phase: ArchiveOperationPhase, +): Promise => { + const current = readIntent(archiveDir, operationId) + if (current.kind !== "gc") { + throw new Error(`archive operation is not a gc operation: ${operationId}`) + } + if (PHASE_ORDER[phase] < PHASE_ORDER[current.phase]) { + throw new Error(`archive operation phase regression: ${current.phase} -> ${phase}`) + } + const updated: GcOperationIntent = { + ...current, + completedTargets, + phase, + updatedAt: new Date().toISOString(), + } + await durableJson(intentPath(archiveDir, operationId), updated) + await syncDirectory(operationDir(archiveDir, operationId)) + return updated +} + +/** + * Enumerate active operation dirs under `operations/active/`. Returns the + * validated operation IDs. Fails closed on any non-conforming entry, symlink, + * or unexpected content — these signal ambiguous or corrupt state that + * reconciliation must surface, not silently act on. + * + * Returns at most the IDs present; the caller enforces "at most one". + */ +export const listActiveOperationIds = (archiveDir: string): string[] => { + const root = activeOperationsRoot(archiveDir) + if (!existsSync(root)) return [] + assertNoSymlinkSync(archiveDir, root, "archive active operations root") + const rootInfo = lstatSync(root) + if (!rootInfo.isDirectory()) { + throw new Error(`archive active operations root is not a real directory: ${root}`) + } + const entries = readdirSync(root, { withFileTypes: true }) + const ids: string[] = [] + for (const entry of entries) { + // Any non-directory entry (file, symlink, socket) is unrecognized debris. + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new Error(`unrecognized active operation debris: ${join(root, entry.name)}`) + } + const prefix = "archive-" + if (!entry.name.startsWith(prefix)) { + throw new Error(`unrecognized active operation entry: ${join(root, entry.name)}`) + } + ids.push(validateArchiveId(entry.name.slice(prefix.length), "operation")) + } + return ids +} + +export interface ActiveOperation { + readonly operationId: string + readonly dir: string + readonly intent: ArchiveOperationIntent +} + +/** + * Read the single permitted active operation, or null if none. Fails closed if + * there is more than one active operation dir (ambiguous state; the maintenance + * lock should prevent this, so its presence signals corruption or a bug). + */ +export const readActiveOperation = ( + archiveDir: string, + expectedDataDir?: string, + expectedScratchRoot?: string, +): ActiveOperation | null => { + const inspected = inspectActiveOperation(archiveDir, expectedDataDir, expectedScratchRoot) + if (inspected === null) return null + if (inspected.kind === "fail-closed") { + throw new Error(inspected.reason) + } + // inspectActiveOperation returns a v3 snapshot only (it fail-closes on v2, + // since v2 must be migrated before it can be a v3 ActiveOperation). + if (inspected.formatVersion !== 3) { + throw new Error(`unexpected inspection format version: ${inspected.formatVersion}`) + } + return { operationId: inspected.operationId, dir: inspected.dir, intent: inspected.intent } +} + +/** + * A V2 active-operation snapshot: the record has been read through the guarded + * (no-symlink, real-file) path and its `operationId` bound to its directory, and + * it has been validated to lift cleanly to v3 — but it has NOT been rewritten. + * Migration consumes this and performs the single durable rewrite. + */ +export interface V2ActiveOperationSnapshot { + readonly kind: "v2" + readonly operationId: string + readonly dir: string + readonly formatVersion: 2 + /** The raw v2 record (already validated to lift cleanly). */ + readonly raw: Record +} + +/** + * A V3 active-operation snapshot: the strict v3 reader has accepted it and bound + * its `operationId` to its directory. + */ +export interface V3ActiveOperationSnapshot { + readonly kind: "create-v3" | "gc" + readonly operationId: string + readonly dir: string + readonly formatVersion: 3 + readonly intent: ArchiveOperationIntent +} + +/** + * A fail-closed inspection: the active state is unsafe (multiple ops, a + * missing/unreadable/strict-invalid intent, unknown debris, an unreadable root, + * or a directory/record identity mismatch). Reconciliation must surface this and + * preserve state — never act. + */ +export interface FailClosedInspection { + readonly kind: "fail-closed" + readonly reason: string +} + +export type ActiveOperationInspection = + | null + | V2ActiveOperationSnapshot + | V3ActiveOperationSnapshot + | FailClosedInspection + +/** + * The ONE authoritative, symlink-safe V2/V3 inspector. Both dry-run and apply + * consume this; migration consumes it; nothing rereads through a weaker path. + * + * Reuses `listActiveOperationIds` (active-root containment + no-symlink + per- + * entry debris/prefix/`validateArchiveId`) and reads `intent.json` through the + * SAME guarded path as `readIntent` (`assertNoSymlinkSync` + `assertRealFileSync` + * before `readFileSync`). Binds the directory ID to the record's `operationId`. + * + * Returns: + * - `null` — no active operation; + * - `{ kind: "fail-closed", reason }` — unsafe active state (multiple ops, a + * missing/unreadable/strict-invalid intent, unknown debris, an unreadable root, + * or a directory/record identity mismatch); + * - `{ kind: "v2", ... }` — a valid v2 record (read safely + bound), validated + * to lift cleanly, NOT rewritten (migration does the rewrite); + * - `{ kind: "create-v3" | "gc", ... }` — a valid v3 record, strictly parsed + bound. + */ +export const inspectActiveOperation = ( + archiveDir: string, + expectedDataDir?: string, + expectedScratchRoot?: string, +): ActiveOperationInspection => { + let ids: string[] + try { + ids = listActiveOperationIds(archiveDir) + } catch (error) { + // The authoritative enumerator throws on debris / unreadable root (rather + // than filtering to absence). Surface as fail-closed. + const reason = error instanceof Error ? error.message : String(error) + return { kind: "fail-closed", reason: `active operations directory is unsafe: ${reason}` } + } + if (ids.length === 0) return null + if (ids.length > 1) { + return { + kind: "fail-closed", + reason: `${ids.length} active operations are ambiguous; manual inspection required`, + } + } + const operationId = ids[0]! + const dir = operationDir(archiveDir, operationId) + const path = intentPath(archiveDir, operationId) + if (!existsSync(path)) { + return { kind: "fail-closed", reason: `active operation is missing its intent.json: ${dir}` } + } + // Read through the SAME guarded path as readIntent — no bare readFileSync. + // These assertions throw on a symlinked intent or a non-regular file. + let raw: unknown + try { + assertNoSymlinkSync(archiveDir, path, "archive operation intent") + assertRealFileSync(path, "archive operation intent") + raw = JSON.parse(readFileSync(path, "utf8")) as unknown + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + return { kind: "fail-closed", reason: `active operation intent is unreadable/unsafe: ${reason}` } + } + if (!isRecord(raw)) { + return { kind: "fail-closed", reason: `active operation intent is not a record: ${dir}` } + } + if (raw.formatVersion === 2) { + // V2: validate it lifts cleanly AND bind its operationId to the directory. + // migrateV2CreateIntent parses the lifted record (internal-field validation); + // it does NOT know the directory name, so bind it here. + let lifted: Record + try { + lifted = migrateV2CreateIntent(archiveDir, raw, expectedDataDir, expectedScratchRoot) + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + return { + kind: "fail-closed", + reason: `active v2 intent is corrupt and will not migrate: ${reason}`, + } + } + // Bind the directory ID to the record's operationId (the check the old + // migration skipped). A v2 record for operation B inside directory A is + // fail-closed — never rewritten. + const recordedOperationId = lifted.operationId + if (typeof recordedOperationId !== "string" || recordedOperationId !== operationId) { + return { + kind: "fail-closed", + reason: `archive operation identity mismatch (directory: ${operationId}; intent: ${String(recordedOperationId)})`, + } + } + return { kind: "v2", operationId, dir, formatVersion: 2, raw } + } + if (raw.formatVersion === ARCHIVE_OPERATION_FORMAT_VERSION) { + // V3: strict parse + directory/record identity binding. + let intent: ArchiveOperationIntent + try { + intent = parseArchiveOperationIntent(archiveDir, raw, expectedDataDir, expectedScratchRoot) + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + return { kind: "fail-closed", reason: `active operation intent is strict-invalid: ${reason}` } + } + if (intent.operationId !== operationId) { + return { + kind: "fail-closed", + reason: `archive operation identity mismatch (directory: ${operationId}; intent: ${intent.operationId})`, + } + } + return { + kind: intent.kind === "create" ? "create-v3" : "gc", + operationId, + dir, + formatVersion: 3, + intent, + } + } + return { + kind: "fail-closed", + reason: `active operation intent has unsupported format version: ${String(raw.formatVersion)}`, + } +} + +// --------------------------------------------------------------------------- +// Reconciliation plan: a discriminated union of CONCRETE ordered actions. +// The plan is the decision model apply executes verbatim; it never rediscovers +// the branch. Each action names an exact operation with its precondition; apply +// revalidates the precondition immediately before the mutation (Gate 3b r4). +// --------------------------------------------------------------------------- + +/** A SHA-256 of the intent record bytes, binding the plan to the exact journal observed at plan time. */ +export const journalDigest = (raw: Record): string => { + const c = require("node:crypto") as typeof import("node:crypto") + return c.createHash("sha256").update(JSON.stringify(raw)).digest("hex") +} + +/** Concrete create-reconciliation actions, in execution order. */ +export type CreateAction = + | { readonly type: "migrate-v2" } + | { readonly type: "quarantine-building" } + | { readonly type: "verify-building-absent" } + | { readonly type: "remove-owned-scratch" } + | { readonly type: "verify-scratch-absent" } + | { readonly type: "release-owned-pin" } + | { readonly type: "verify-pin-absent" } + | { readonly type: "verify-published-generation" } + | { readonly type: "select-pointer" } + | { readonly type: "verify-pointer-selects-intended" } + | { readonly type: "rebuild-catalog" } + | { readonly type: "verify-catalog-exact" } + | { readonly type: "advance-phase"; readonly to: ArchiveOperationPhase } + | { readonly type: "verify-terminal-invariants" } + | { readonly type: "archive-operation" } + +/** Concrete GC-reconciliation actions, in execution order. */ +export type GcAction = + | { readonly type: "collect-target"; readonly index: number } + | { + readonly type: "persist-cursor" + readonly completedTargets: number + readonly to: ArchiveOperationPhase + } + | { readonly type: "rebuild-catalog" } + | { readonly type: "verify-catalog-exact" } + | { readonly type: "verify-terminal-invariants" } + | { readonly type: "archive-operation" } + +export type ReconciliationPlan = + | { readonly kind: "no-op" } + | { readonly kind: "fail-closed"; readonly reason: string } + | { + readonly kind: "create" + readonly operationId: string + readonly journalDigest: string + readonly actions: ReadonlyArray + } + | { + readonly kind: "gc" + readonly operationId: string + readonly journalDigest: string + readonly actions: ReadonlyArray + } + +/** A topology observer the plan builder uses to decide the branch ONCE. */ +export interface ReconcileTopology { + readonly promoted: boolean + readonly buildingPresent: boolean + readonly phase: ArchiveOperationPhase +} + +/** + * Build the immutable reconciliation plan from an inspection + the observed + * create-kind topology. Makes the branch decision ONCE (pre-publication-abort vs + * post-promotion-complete vs already-complete-verify) and emits the exact ordered + * concrete actions; apply executes them without re-branching. + */ +export const buildCreatePlan = ( + inspection: V2ActiveOperationSnapshot | V3ActiveOperationSnapshot, + topology: ReconcileTopology, +): ReconciliationPlan => { + // For a v2 snapshot, the only known-safe action is migrate then reconcile; + // the v2 phase is always "intent" (the only phase a v2 record can hold), so + // after migration the v3 reconciler will observe the post-migration topology. + // The plan therefore emits migrate-v2 as the first action, then defers the + // concrete post-migration steps to a re-plan under the lock at apply time + // (the topology can only be observed AFTER migration rewrites the record). + if (inspection.kind === "v2") { + return { + kind: "create", + operationId: inspection.operationId, + journalDigest: journalDigest(inspection.raw), + actions: [{ type: "migrate-v2" }, { type: "advance-phase", to: "intent" }], + } + } + const intent = inspection.intent as CreateOperationIntent + const actions: CreateAction[] = [] + const phase = topology.phase + // Already complete: verify terminal invariants, then archive. + if (phaseAtLeast(phase, "complete")) { + actions.push({ type: "verify-terminal-invariants" }, { type: "archive-operation" }) + return { + kind: "create", + operationId: inspection.operationId, + journalDigest: journalDigestOfIntent(intent), + actions, + } + } + // Already aborted in active/ is fail-closed (should have been quarantined). + if (phaseAtLeast(phase, "aborted")) { + return { + kind: "fail-closed", + reason: `aborted archive operation still in active dir: ${inspection.dir}`, + } + } + if (!topology.promoted) { + // Pre-publication: quarantine building (if present), remove scratch, release pin, abort, archive. + if (topology.buildingPresent) actions.push({ type: "quarantine-building" }) + else actions.push({ type: "verify-building-absent" }) + actions.push( + { type: "remove-owned-scratch" }, + { type: "release-owned-pin" }, + { type: "advance-phase", to: "aborted" }, + { type: "archive-operation" }, + ) + return { + kind: "create", + operationId: inspection.operationId, + journalDigest: journalDigestOfIntent(intent), + actions, + } + } + // Post-promotion: verify published, finish pointer/catalog/pin/scratch, complete, archive. + actions.push({ type: "verify-published-generation" }) + if (!phaseAtLeast(phase, "pointer-complete")) + actions.push({ type: "select-pointer" }, { type: "advance-phase", to: "pointer-complete" }) + else actions.push({ type: "verify-pointer-selects-intended" }) + actions.push({ type: "rebuild-catalog" }, { type: "verify-catalog-exact" }) + if (!phaseAtLeast(phase, "catalog-complete")) + actions.push({ type: "advance-phase", to: "catalog-complete" }) + if (!phaseAtLeast(phase, "pin-released")) + actions.push({ type: "release-owned-pin" }, { type: "advance-phase", to: "pin-released" }) + else actions.push({ type: "verify-pin-absent" }) + if (!phaseAtLeast(phase, "scratch-removed")) + actions.push({ type: "remove-owned-scratch" }, { type: "advance-phase", to: "scratch-removed" }) + else actions.push({ type: "verify-scratch-absent" }) + actions.push( + { type: "advance-phase", to: "complete" }, + { type: "verify-terminal-invariants" }, + { type: "archive-operation" }, + ) + return { + kind: "create", + operationId: inspection.operationId, + journalDigest: journalDigestOfIntent(intent), + actions, + } +} + +/** + * Build the GC plan: collect each remaining target (cursor..end), persist cursor + * per target (nonterminal gc-collecting), rebuild + verify affected catalogs, + * persist complete, verify terminal, archive. + */ +export const buildGcPlan = (inspection: V3ActiveOperationSnapshot): ReconciliationPlan => { + const intent = inspection.intent as GcOperationIntent + const actions: GcAction[] = [] + for (let i = intent.completedTargets; i < intent.targets.length; i++) { + actions.push( + { type: "collect-target", index: i }, + { type: "persist-cursor", completedTargets: i + 1, to: "gc-collecting" }, + ) + } + // Affected-signal catalog rebuild + verify. + const signals = [...new Set(intent.targets.map((t) => t.signal))] + for (const _signal of signals) { + actions.push({ type: "rebuild-catalog" }, { type: "verify-catalog-exact" }) + } + actions.push( + { type: "persist-cursor", completedTargets: intent.targets.length, to: "complete" }, + { type: "verify-terminal-invariants" }, + { type: "archive-operation" }, + ) + return { + kind: "gc", + operationId: inspection.operationId, + journalDigest: journalDigestOfIntent(intent), + actions, + } +} + +const journalDigestOfIntent = (intent: ArchiveOperationIntent): string => { + const c = require("node:crypto") as typeof import("node:crypto") + return c.createHash("sha256").update(JSON.stringify(intent)).digest("hex") +} + +/** + * Move a completed operation's journal from `operations/active/` to the retained + * `operations/completed/` location so it no longer blocks later work. The + * completed record is retained for inspection (D-004: never silently deleted). + */ +export const archiveCompletedOperation = async (archiveDir: string, operationId: string): Promise => { + const activeDir = operationDir(archiveDir, operationId) + const completedDir = completedOperationsRoot(archiveDir) + await ensurePrivateDirectory(completedDir, archiveRoot(archiveDir)) + const dest = join(completedDir, `archive-${validateArchiveId(operationId, "operation")}`) + if (existsSync(dest)) { + // A completed record already exists for this id — ambiguous; fail closed + // rather than overwriting retained history. + throw new Error(`completed archive operation already exists; refusing to overwrite: ${dest}`) + } + await durableRename(activeDir, dest) + await syncDirectory(activeOperationsRoot(archiveDir)) +} + +/** + * Quarantine an operation dir (pre-publication incomplete output) by renaming it + * under `quarantine/` with a stable, owned name, so archive evidence is retained + * for inspection rather than silently deleted (D-004). Returns the quarantine + * destination path. + */ +export const quarantineOperation = async (archiveDir: string, operationId: string): Promise => { + const activeDir = operationDir(archiveDir, operationId) + const quarantineRoot = archiveQuarantineRoot(archiveDir) + await ensurePrivateDirectory(quarantineRoot, archiveRoot(archiveDir)) + const dest = join(quarantineRoot, `operation-${validateArchiveId(operationId, "operation")}`) + if (existsSync(dest)) { + throw new Error(`quarantined operation already exists; refusing to overwrite: ${dest}`) + } + await durableRename(activeDir, dest) + await syncDirectory(activeOperationsRoot(archiveDir)) + return dest +} + +/** Remove the active operation dir entirely (used after a clean abort). */ +export const removeActiveOperation = async (archiveDir: string, operationId: string): Promise => { + const activeDir = operationDir(archiveDir, operationId) + if (existsSync(activeDir)) { + await durableRemove(activeDir) + await syncDirectory(activeOperationsRoot(archiveDir)) + } +} + +/** + * Read the active generation id currently selected by the pointer for a + * (signal, range), or null if no pointer exists. Throws on a malformed or + * location-mismatched pointer (binding the pointer to its on-disk location). + */ +export const readActiveGenerationId = ( + archiveDir: string, + signal: string, + rangeDate: string, +): string | null => { + const pointerPath = activePointerPath(archiveDir, signal, rangeDate) + if (!existsSync(pointerPath)) return null + assertNoSymlinkSync(archiveDir, pointerPath, "archive active pointer") + assertRealFileSync(pointerPath, "archive active pointer") + const parsed = JSON.parse(readFileSync(pointerPath, "utf8")) as unknown + const pointer = parseArchiveActivePointer(parsed, signal, rangeDate) + return pointer.generationId +} + +/** + * Resolve the base active generation id strictly, returning null only when there + * is genuinely no pointer. Used to record the CAS base before promotion. + */ +export const resolveBaseActiveGenerationId = ( + archiveDir: string, + signal: string, + rangeDate: string, +): string | null => readActiveGenerationId(archiveDir, signal, rangeDate) + +/** + * Pre-allocate the owned building and final-generation paths from the archive + * root and identities, for inspection and for the operation to record. These are + * pure path computations; they do not create anything. + */ +export const ownedPathsFor = (intent: { + readonly archiveDir: string + readonly generationId: string + readonly signal: string + readonly rangeStart: string +}): { readonly finalGeneration: string; readonly building: string } => { + const finalGeneration = join( + rangeRoot(intent.archiveDir, intent.signal, intent.rangeStart), + "generations", + intent.generationId, + ) + const building = join(archiveRoot(intent.archiveDir), "building", intent.generationId) + void signalRoot + return { finalGeneration, building } +} + +/** + * Assert the journal's recorded (signal, range) topology exists consistently + * with the on-disk pointer for that location. Used by reconcile to validate that + * the recorded CAS base still matches reality before flipping the pointer. + */ +export const assertPointerConsistent = (archiveDir: string, intent: CreateOperationIntent): void => { + const current = readActiveGenerationId(archiveDir, intent.signal, intent.rangeStart) + // The pointer must either still select the recorded base, or already select + // the intended generation (an earlier promotion completed). Anything else + // means concurrent activity moved the pointer and a blind flip would clobber + // it — fail closed. + if (current !== intent.baseActiveGenerationId && current !== intent.generationId) { + throw new Error( + `archive active pointer no longer matches the recorded base for ${intent.signal}/${intent.rangeStart}: ` + + `recorded base ${intent.baseActiveGenerationId}, now ${current} (concurrent activity; refusing to clobber)`, + ) + } +} + +/** + * Assert that a path is a real directory beneath the archive root (no symlink), + * if it exists. Used by reconcile to validate owned topology before acting. + */ +export const assertOwnedDirectoryIfPresent = async ( + archiveDir: string, + path: string, + label: string, +): Promise => { + if (!existsSync(path)) return + await assertNoSymlink(archiveDir, path, label) + await assertRealDirectory(path, label) + await assertRealFile(join(path, "intent.json"), `${label} intent`).catch(() => { + throw new Error(`${label} is missing its intent.json: ${path}`) + }) +} diff --git a/apps/cli/src/server/archives/listing.ts b/apps/cli/src/server/archives/listing.ts new file mode 100644 index 000000000..4883fddaf --- /dev/null +++ b/apps/cli/src/server/archives/listing.ts @@ -0,0 +1,388 @@ +import { createHash } from "node:crypto" +import { createReadStream, existsSync, readdirSync, readFileSync, statSync } from "node:fs" +import { + readArchiveGenerationManifest, + parseArchiveActivePointer, + shardFilePath, + type ArchiveGenerationManifest, +} from "./manifest" +import { + activePointerPath, + assertNoSymlinkSync, + assertRealFileSync, + catalogPath, + generationManifestPath, + generationsRoot, + signalRoot, +} from "./paths" +import { ARCHIVE_SIGNALS, type ArchiveSignalName } from "./signals" + +// Archive read-side: listing, active-path resolution, and catalog rebuild. +// +// `archive list` reports the active generation per (signal, range) with sizes +// and paths, and only ever exposes the active generation's Parquet paths — a +// superseded generation is retained on disk but never returned to queries or to +// the listing, so late-arrival history cannot be double-counted. The catalog is +// a rebuildable index: if `catalog.jsonl` is missing or truncated, it can be +// regenerated from the authoritative generation manifests without rescanning +// Parquet bytes. + +export interface ActiveGenerationSummary { + readonly signal: string + readonly rangeStart: string + readonly generationId: string + readonly archivedRowCount: number + readonly shardCount: number + readonly createdAt: string + readonly checkpointId: string + /** Absolute paths of the active generation's Parquet shards, in order. */ + readonly shardPaths: ReadonlyArray + /** Total bytes of the active generation's shards. */ + readonly shardBytes: number +} + +export interface ArchiveListingError { + readonly signal: string + readonly rangeStart: string + readonly error: string +} + +export interface ArchiveListing { + readonly archiveDir: string + /** Listing validates topology and manifest-bound sizes, but does not hash shard contents. */ + readonly integrity: "metadata-only" + readonly active: ReadonlyArray + readonly signals: ReadonlyArray + /** Preserved errors surfaced (not silently skipped) so a corrupt range is visible (H-7). */ + readonly errors: ReadonlyArray +} + +/** Sum the byte sizes of a generation's shard records. */ +const shardBytes = (manifest: Pick): number => + manifest.shards.reduce((sum, shard) => sum + shard.bytes, 0) + +/** + * List the active generation for every (signal, range) that has an `active.json` + * pointer. Superseded generations are present on disk but never appear here. A + * malformed or unreadable active pointer or manifest for one range is SURFACED in + * `errors` (not silently skipped) so the operator sees corrupt state; unaffected + * ranges are still listed. The pointer/manifest files themselves are preserved + * untouched. + */ +export const listActiveGenerations = (archiveDir: string): ArchiveListing => { + const active: ActiveGenerationSummary[] = [] + const signalsPresent: string[] = [] + const errors: ArchiveListingError[] = [] + for (const signal of ARCHIVE_SIGNALS) { + const sRoot = signalRoot(archiveDir, signal.name) + if (!existsSync(sRoot)) continue + let ranges: string[] + try { + ranges = readdirSync(sRoot).filter((entry) => /^\d{4}-\d{2}-\d{2}$/.test(entry)) + } catch (error) { + errors.push({ + signal: signal.name, + rangeStart: "", + error: `signal root unreadable: ${messageOf(error)}`, + }) + continue + } + let signalHasActive = false + for (const rangeDate of ranges) { + const pointerPath = activePointerPath(archiveDir, signal.name, rangeDate) + if (!existsSync(pointerPath)) continue + let generationId: string + try { + // Refuse a symlinked or non-regular pointer path (HIGH-1 read-side): + // a symlinked range dir or a non-file (socket, device) would make + // this read attacker-controlled or undefined content. + assertNoSymlinkSync(archiveDir, pointerPath, "archive active pointer") + assertRealFileSync(pointerPath, "archive active pointer") + const pointer = parseArchiveActivePointer( + JSON.parse(readFileSync(pointerPath, "utf8")) as unknown, + signal.name, + rangeDate, + ) + generationId = pointer.generationId + } catch (error) { + errors.push({ + signal: signal.name, + rangeStart: rangeDate, + error: `active pointer: ${messageOf(error)}`, + }) + continue + } + let manifest: ArchiveGenerationManifest + try { + manifest = readArchiveGenerationManifest(archiveDir, signal.name, rangeDate, generationId) + } catch (error) { + errors.push({ + signal: signal.name, + rangeStart: rangeDate, + error: `manifest: ${messageOf(error)}`, + }) + continue + } + signalHasActive = true + // Cheap metadata listing verifies path topology, regular-file type, and + // manifest-bound byte size. Content hashing is deliberately reserved for + // explicit `archive verify`, which streams every shard with bounded memory. + let shardPaths: string[] + try { + shardPaths = manifest.shards.map((shard) => { + const p = shardFilePath(archiveDir, signal.name, rangeDate, generationId, shard.name) + assertNoSymlinkSync(archiveDir, p, "archive shard") + assertRealFileSync(p, "archive shard") + const actualBytes = statSync(p).size + if (actualBytes !== shard.bytes) { + throw new Error( + `shard ${shard.name} byte size mismatch: manifest ${shard.bytes}, actual ${actualBytes}`, + ) + } + return p + }) + } catch (error) { + errors.push({ + signal: signal.name, + rangeStart: rangeDate, + error: `shard path: ${messageOf(error)}`, + }) + continue + } + active.push({ + signal: signal.name, + rangeStart: rangeDate, + generationId, + archivedRowCount: manifest.archivedRowCount, + shardCount: manifest.shards.length, + createdAt: manifest.createdAt, + checkpointId: manifest.checkpointId, + shardPaths, + shardBytes: shardBytes(manifest), + }) + } + if (signalHasActive) signalsPresent.push(signal.name) + } + return { archiveDir, integrity: "metadata-only", active, signals: signalsPresent, errors } +} + +const messageOf = (error: unknown): string => (error instanceof Error ? error.message : String(error)) + +export const ARCHIVE_VERIFY_BUFFER_BYTES = 1024 * 1024 + +/** Compute SHA-256 with a fixed 1 MiB stream buffer rather than allocating a + * full shard-sized buffer. Verification processes one shard at a time. */ +const sha256FileStreaming = async (path: string): Promise => { + const hash = createHash("sha256") + const stream = createReadStream(path, { highWaterMark: ARCHIVE_VERIFY_BUFFER_BYTES }) + for await (const chunk of stream) hash.update(chunk) + return hash.digest("hex") +} + +export interface ArchiveVerification { + readonly archiveDir: string + readonly signals: ReadonlyArray + readonly generationCount: number + readonly shardCount: number + readonly verifiedBytes: number +} + +/** Explicitly verify every selected active shard against its manifest SHA-256. + * Listing errors fail the operation before any partial success is reported. */ +export const verifyActiveGenerations = async ( + archiveDir: string, + signal?: ArchiveSignalName, +): Promise => { + const listing = listActiveGenerations(archiveDir) + const relevantErrors = signal ? listing.errors.filter((error) => error.signal === signal) : listing.errors + if (relevantErrors.length > 0) { + const detail = relevantErrors + .map((error) => `${error.signal}/${error.rangeStart || "(root)"}: ${error.error}`) + .join("; ") + throw new Error(`refusing archive integrity verification: ${detail}`) + } + const active = signal ? listing.active.filter((summary) => summary.signal === signal) : listing.active + let shardCount = 0 + let verifiedBytes = 0 + for (const summary of active) { + const manifest = readArchiveGenerationManifest( + archiveDir, + summary.signal, + summary.rangeStart, + summary.generationId, + ) + for (const shard of manifest.shards) { + const path = shardFilePath( + archiveDir, + summary.signal, + summary.rangeStart, + summary.generationId, + shard.name, + ) + assertNoSymlinkSync(archiveDir, path, "archive shard") + assertRealFileSync(path, "archive shard") + const actualBytes = statSync(path).size + if (actualBytes !== shard.bytes) { + throw new Error( + `shard ${summary.signal}/${summary.rangeStart}/${shard.name} byte size mismatch: ` + + `manifest ${shard.bytes}, actual ${actualBytes}`, + ) + } + const actualSha = await sha256FileStreaming(path) + if (actualSha !== shard.sha256) { + throw new Error( + `shard ${summary.signal}/${summary.rangeStart}/${shard.name} SHA-256 mismatch: ` + + `manifest ${shard.sha256.slice(0, 16)}…, actual ${actualSha.slice(0, 16)}…`, + ) + } + shardCount++ + verifiedBytes += actualBytes + } + } + return { + archiveDir, + signals: signal ? [signal] : listing.signals, + generationCount: active.length, + shardCount, + verifiedBytes, + } +} + +/** + * Resolve the active Parquet shard paths for one signal across all sealed + * ranges, excluding superseded generations. This is the machine-readable output + * an operator feeds to DuckDB's `read_parquet`. Returns the paths grouped by + * range in ascending order. + * + * Fail-closed on malformed topology or manifest-bound byte-size mismatches, but + * deliberately does not hash shard contents. Run `archive verify` for explicit + * bounded-memory integrity verification before querying when required. + */ +export const activeParquetPaths = (archiveDir: string, signal: ArchiveSignalName): ReadonlyArray => { + const listing = listActiveGenerations(archiveDir) + const relevantErrors = listing.errors.filter((e) => e.signal === signal) + if (relevantErrors.length > 0) { + const detail = relevantErrors + .map((e) => `${e.signal}/${e.rangeStart || "(root)"}: ${e.error}`) + .join("; ") + throw new Error( + `refusing to return active Parquet paths for ${signal}: ${relevantErrors.length} malformed range(s) — ` + + `${detail}. Run 'maple archive rebuild ${signal}' or inspect the archive.`, + ) + } + const forSignal = listing.active + .filter((summary) => summary.signal === signal) + .sort((a, b) => a.rangeStart.localeCompare(b.rangeStart)) + return forSignal.flatMap((summary) => summary.shardPaths) +} + +export interface CatalogEntry { + readonly generationId: string + readonly signal: string + readonly rangeStart: string + readonly checkpointId: string + readonly archivedRowCount: number + readonly shardCount: number + readonly createdAt: string +} + +const serializeCatalogEntries = (entries: ReadonlyArray): string => + `${entries.map((entry) => JSON.stringify({ ...entry, formatVersion: 1 as const })).join("\n")}\n` + +/** + * Read every authoritative manifest for a signal and derive the exact canonical + * catalog entries without mutating the catalog. This is shared by rebuild and + * crash reconciliation so a journal phase can never substitute for observed + * catalog state. + */ +export const authoritativeCatalogEntries = ( + archiveDir: string, + signal: ArchiveSignalName, +): ReadonlyArray => { + const sRoot = signalRoot(archiveDir, signal) + if (!existsSync(sRoot)) return [] + let ranges: string[] + try { + ranges = readdirSync(sRoot).filter((entry) => /^\d{4}-\d{2}-\d{2}$/.test(entry)) + } catch (error) { + throw new Error(`archive catalog rebuild: signal root unreadable: ${messageOf(error)}`) + } + const entries: CatalogEntry[] = [] + for (const rangeDate of ranges.sort()) { + const gensRoot = generationsRoot(archiveDir, signal, rangeDate) + if (!existsSync(gensRoot)) continue + let generationIds: string[] + try { + generationIds = readdirSync(gensRoot) + } catch (error) { + throw new Error( + `archive catalog rebuild: generations root unreadable for ${signal}/${rangeDate}: ${messageOf(error)}`, + ) + } + for (const generationId of generationIds.sort()) { + const manifestPath = generationManifestPath(archiveDir, signal, rangeDate, generationId) + if (!existsSync(manifestPath)) { + throw new Error( + `archive catalog rebuild: generation ${signal}/${rangeDate}/${generationId} is missing its manifest; ` + + `remove the orphan generation directory or restore the manifest before rebuilding`, + ) + } + const manifest = readArchiveGenerationManifest(archiveDir, signal, rangeDate, generationId) + entries.push({ + generationId: manifest.generationId, + signal: manifest.signal, + rangeStart: manifest.rangeStart, + checkpointId: manifest.checkpointId, + archivedRowCount: manifest.archivedRowCount, + shardCount: manifest.shards.length, + createdAt: manifest.createdAt, + }) + } + } + return entries +} + +/** + * Assert that the on-disk catalog is byte-for-byte the canonical index derived + * from authoritative manifests. Missing, duplicated, reordered, truncated, or + * tampered entries all fail closed. + */ +export const assertCatalogExact = (archiveDir: string, signal: ArchiveSignalName): void => { + const path = catalogPath(archiveDir, signal) + assertNoSymlinkSync(archiveDir, path, "archive catalog") + assertRealFileSync(path, "archive catalog") + const expected = serializeCatalogEntries(authoritativeCatalogEntries(archiveDir, signal)) + const actual = readFileSync(path, "utf8") + if (actual !== expected) { + throw new Error(`archive catalog does not exactly match authoritative manifests for ${signal}`) + } +} + +/** + * Rebuild `catalog.jsonl` for a signal from the authoritative generation + * manifests. Every promoted generation (active or superseded) appears once, + * because the catalog indexes all retained generations, not just the active one. + * + * Fail-closed (H-7): the rebuild PREFLIGHTS every manifest before writing. If + * any generation manifest is missing, malformed, or on a symlinked path, the + * existing catalog is PRESERVED untouched and the call throws. A partial rebuild + * that silently drops corrupt generations would make the catalog lie about what + * is archived, which is worse than a visible error. The operator inspects the + * named generation and recovers. + */ +export const rebuildCatalog = async ( + archiveDir: string, + signal: ArchiveSignalName, +): Promise> => { + if (!existsSync(signalRoot(archiveDir, signal))) return [] + // Phase 1 — preflight every authoritative manifest before touching catalog. + const entries = authoritativeCatalogEntries(archiveDir, signal) + // Phase 2 — write: only reached if every manifest preflighted clean. Use the + // durable atomic-write primitive (temp + fsync + rename + dir sync) so an + // ENOSPC, short write, or interruption cannot destroy the prior catalog. + const path = catalogPath(archiveDir, signal) + assertNoSymlinkSync(archiveDir, path, "archive catalog") + const { durableWrite } = await import("../durable-files") + await durableWrite(path, serializeCatalogEntries(entries)) + return entries +} diff --git a/apps/cli/src/server/archives/manifest.ts b/apps/cli/src/server/archives/manifest.ts new file mode 100644 index 000000000..50f263c91 --- /dev/null +++ b/apps/cli/src/server/archives/manifest.ts @@ -0,0 +1,497 @@ +import { readFileSync } from "node:fs" +import { join } from "node:path" +import { + type ArchiveTuningRecord, + type TuningConfigIdentity, + LEGACY_TUNING_CONFIG_FORMAT_VERSION, + TUNING_CONFIG_FORMAT_VERSION, +} from "./config" +import { KNOWN_COMPLEX_DIGEST_ALGORITHMS } from "./export" +import { + assertNoSymlinkSync, + assertRealFileSync, + generationManifestPath, + nextMidnightUtc, + validateArchiveId, + validateRangeDate, +} from "./paths" +import { isArchiveSignalName } from "./signals" + +// Versioned, strict archive manifest and pointer formats. +// +// A generation manifest is the authoritative completion record for one sealed +// UTC-day export of one signal. It is written only after every shard is +// validated and is never edited after commit. The active pointer selects +// exactly one generation per (signal, range); selection changes only by atomic +// replacement of `active.json`. Unknown format versions, missing/wrong fields, +// path escape, count mismatch, or checksum mismatch fail closed. Mirrors the +// checkpoint module's `formatVersion` discipline. + +// Manifest format version history: +// 1 — round 4. Shard time evidence as timezone-less ISO strings parsed with +// Date.parse (host-timezone-dependent); commutative per-column-sum digest. +// 2 — round 5. Shard time evidence as UTC epoch-nanosecond DECIMAL STRINGS +// (parsed with BigInt, host-timezone-independent); multiset digest with an +// explicit algorithm field; bare `tuningConfigName` string. +// 3 — config/calibration gate. `tuningConfigName` replaced by structured, +// SHA-256-bound `tuningConfig` identity ({ formatVersion, configName, +// sha256 } | null). A v2 manifest lacks this structured field; a v3 +// reader rejects v2 (and v1) fail-closed, preserving files, because the +// config-identity semantics changed and a silent null would lose identity. +// (Older archives are not migrated in place; re-export to re-validate.) +const MANIFEST_FORMAT_VERSION = 3 +const ACTIVE_POINTER_FORMAT_VERSION = 1 + +export interface ArchiveShardRecord { + /** Shard file name, e.g. `00-0000.parquet` (hour + sequence). */ + readonly name: string + /** Row count READ BACK from the reopened Parquet file (not the source count). */ + readonly rowCount: number + /** Min event time, UTC epoch nanoseconds as a decimal string (host-tz-independent). */ + readonly minEventTimeUnixNano: string + /** Max event time, UTC epoch nanoseconds as a decimal string (host-tz-independent). */ + readonly maxEventTimeUnixNano: string + /** SHA-256 of the shard file bytes. */ + readonly sha256: string + /** Shard file size in bytes (on-disk, compressed). */ + readonly bytes: number + /** Column names read back from the reopened Parquet (schema round-trip proof). */ + readonly columns: ReadonlyArray + /** + * Complex-value digest over the reopened shard (algorithm named by + * complexDigestAlgorithm). Detects same-typed column swaps, cross-row value + * reassociation, and dup/drop that preserve count and time extrema. + */ + readonly complexDigest: string + /** The digest algorithm that produced {@link complexDigest} (e.g. cityhash64-multiset-v3). */ + readonly complexDigestAlgorithm: string +} + +export interface ArchiveGenerationManifest { + readonly formatVersion: 3 + readonly generationId: string + readonly signal: string + readonly rangeStart: string + readonly rangeEndExclusive: string + readonly checkpointId: string + readonly checkpointManifestFingerprint: string + readonly createdAt: string + readonly mapleVersion: string + readonly chdbVersion: string + readonly schemaFingerprint: string + readonly sourceRowCount: number + readonly archivedRowCount: number + readonly tuning: ArchiveTuningRecord + /** + * Structured identity of the calibration config that produced the effective + * tuning, or `null` when defaults/CLI overrides were used. Versioned and + * SHA-256-bound so a generation's exact config is reproducible. Replaces the + * prior bare `tuningConfigName` string. + */ + readonly tuningConfig: TuningConfigIdentity | null + readonly shards: ReadonlyArray +} + +export interface ArchiveActivePointer { + readonly formatVersion: 1 + readonly generationId: string + readonly signal: string + readonly rangeStart: string + readonly selectedAt: string +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const MANIFEST_KEYS = new Set([ + "formatVersion", + "generationId", + "signal", + "rangeStart", + "rangeEndExclusive", + "checkpointId", + "checkpointManifestFingerprint", + "createdAt", + "mapleVersion", + "chdbVersion", + "schemaFingerprint", + "sourceRowCount", + "archivedRowCount", + "tuning", + "tuningConfig", + "shards", +]) + +const TUNING_KEYS = new Set([ + "writerThreads", + "rowGroupRows", + "maxShardRows", + "maxShardBytes", + "targetChunkBytes", + "minFreeSpaceReserve", +]) + +const assertExactOwnKeys = (record: Record, keys: ReadonlySet, label = ""): void => { + const location = label.length > 0 ? `${label} ` : "" + for (const key of keys) { + if (!Object.prototype.hasOwnProperty.call(record, key)) { + throw new Error(`invalid archive manifest ${location}field: ${key} (required in formatVersion 3)`) + } + } + for (const key of Object.keys(record)) { + if (!keys.has(key)) throw new Error(`unknown archive manifest ${location}field: ${key}`) + } +} + +const requiredString = (record: Record, key: string): string => { + const value = record[key] + if (typeof value !== "string" || value.length === 0) + throw new Error(`invalid archive manifest field: ${key}`) + return value +} + +const requiredCount = (record: Record, key: string): number => { + const value = record[key] + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`invalid archive manifest field: ${key} (must be a safe non-negative integer)`) + } + return value +} + +const requiredPositiveInteger = (record: Record, key: string): number => { + const value = record[key] + if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) { + throw new Error(`invalid archive manifest tuning field: ${key} (must be a positive integer)`) + } + return value +} + +const SHA256_HEX = /^[0-9a-f]{64}$/ + +/** A safe logical config name (no path separators, no traversal). */ +const SAFE_CONFIG_NAME = /^[A-Za-z0-9._-]+$/ + +/** + * Strictly parse the structured `tuningConfig` identity field of a manifest. + * Accepts `null` (no config was loaded) or a record with exactly + * `{ formatVersion, configName, sha256 }`. Rejects unknown subfields, a bad + * SHA-256, an unsafe config name, or a config formatVersion outside the two + * explicitly known identities. Manifest v3 stores an opaque, hash-bound config + * identity and can therefore safely describe legacy v1, symmetric v2, and + * directional v3 documents; only the config loader refuses v1 for new writes. + */ +const parseTuningConfig = (value: unknown): TuningConfigIdentity | null => { + if (value === null) return null + if (!isRecord(value)) { + throw new Error("invalid archive manifest field: tuningConfig (must be null or a record)") + } + const knownKeys = new Set(["formatVersion", "configName", "sha256"]) + for (const key of Object.keys(value)) { + if (!knownKeys.has(key)) { + throw new Error(`unknown archive manifest tuningConfig field: ${key}`) + } + } + const formatVersion = value.formatVersion + if ( + typeof formatVersion !== "number" || + !Number.isSafeInteger(formatVersion) || + (formatVersion !== 1 && + formatVersion !== LEGACY_TUNING_CONFIG_FORMAT_VERSION && + formatVersion !== TUNING_CONFIG_FORMAT_VERSION) + ) { + throw new Error( + `invalid archive manifest tuningConfig.formatVersion (known versions: 1, ${LEGACY_TUNING_CONFIG_FORMAT_VERSION}, ${TUNING_CONFIG_FORMAT_VERSION}): ${String(formatVersion)}`, + ) + } + const configName = requiredString(value, "configName") + if (!SAFE_CONFIG_NAME.test(configName)) { + throw new Error(`invalid archive manifest tuningConfig.configName (unsafe name): ${configName}`) + } + const sha256 = requiredString(value, "sha256") + if (!SHA256_HEX.test(sha256)) { + throw new Error(`invalid archive manifest tuningConfig.sha256 (must be 64 hex chars): ${sha256}`) + } + return { formatVersion, configName, sha256 } +} + +/** + * Parse the path-independent portion of resolveArchiveTuning's effective + * values. A manifest records no roots, but must preserve the same numeric and + * cross-field invariants that governed its writer. + */ +const parseTuningRecord = (value: unknown): ArchiveTuningRecord => { + if (!isRecord(value)) throw new Error("invalid archive manifest field: tuning") + assertExactOwnKeys(value, TUNING_KEYS, "tuning") + const tuning = { + writerThreads: requiredPositiveInteger(value, "writerThreads"), + rowGroupRows: requiredPositiveInteger(value, "rowGroupRows"), + maxShardRows: requiredPositiveInteger(value, "maxShardRows"), + maxShardBytes: requiredPositiveInteger(value, "maxShardBytes"), + targetChunkBytes: requiredPositiveInteger(value, "targetChunkBytes"), + minFreeSpaceReserve: requiredPositiveInteger(value, "minFreeSpaceReserve"), + } + if (tuning.rowGroupRows > tuning.maxShardRows) { + throw new Error("archive tuning rowGroupRows must not exceed maxShardRows") + } + const minShardBytesForRowGroup = tuning.rowGroupRows * 1024 + if (tuning.maxShardBytes < minShardBytesForRowGroup) { + throw new Error( + `archive tuning maxShardBytes (${tuning.maxShardBytes}) is too small for rowGroupRows ` + + `(${tuning.rowGroupRows}); raise maxShardBytes or lower rowGroupRows`, + ) + } + if (tuning.minFreeSpaceReserve >= tuning.targetChunkBytes) { + throw new Error("archive tuning minFreeSpaceReserve must be smaller than targetChunkBytes") + } + if (tuning.writerThreads > 32) { + throw new Error("archive tuning writerThreads must not exceed 32") + } + return tuning +} + +/** + * A required ISO-8601 string for MANIFEST-LEVEL timestamps (createdAt, + * selectedAt) and the canonical `rangeEndExclusive` (always a `...Z` ISO from + * nextMidnightUtc). These are NOT shard event-time evidence — that uses epoch + * nanoseconds (see requiredNanoDecimal) to be host-timezone-independent. + */ +const requiredIso = (record: Record, key: string): string => { + const value = requiredString(record, key) + if (!Number.isFinite(Date.parse(value))) throw new Error(`invalid archive manifest field: ${key}`) + return value +} + +/** A non-negative decimal integer string (epoch nanoseconds), parsed as BigInt. */ +const NANO_DECIMAL = /^[0-9]+$/ +const requiredNanoDecimal = (record: Record, key: string): bigint => { + const value = requiredString(record, key) + if (!NANO_DECIMAL.test(value)) { + throw new Error( + `invalid archive manifest field: ${key} (must be a non-negative decimal integer string)`, + ) + } + return BigInt(value) +} + +const parseShardRecord = ( + value: unknown, + rangeStart: string, + rangeEndExclusive: string, +): ArchiveShardRecord => { + if (!isRecord(value)) throw new Error("invalid archive shard record") + const name = requiredString(value, "name") + if (!/^[0-9a-z._-]+\.parquet$/i.test(name)) throw new Error(`invalid archive shard name: ${name}`) + const columnsRaw = value.columns + if (!Array.isArray(columnsRaw) || columnsRaw.length === 0) { + throw new Error("invalid archive shard record field: columns (must be a nonempty array)") + } + const columns = columnsRaw.map((c) => { + if (typeof c !== "string" || c.length === 0) throw new Error("invalid archive shard column name") + return c + }) + const sha256 = requiredString(value, "sha256") + if (!SHA256_HEX.test(sha256)) + throw new Error(`invalid archive shard sha256 (must be 64 hex chars): ${sha256}`) + const rowCount = requiredCount(value, "rowCount") + const minNano = requiredNanoDecimal(value, "minEventTimeUnixNano") + const maxNano = requiredNanoDecimal(value, "maxEventTimeUnixNano") + if (minNano > maxNano) { + throw new Error(`archive shard ${name}: minEventTimeUnixNano > maxEventTimeUnixNano`) + } + // Bind shard time evidence to the sealed range in EPOCH NANOSECONDS + // (host-timezone-independent). The range bounds are computed from the UTC + // rangeDate and its next-midnight ISO as nanos; a shard whose min/max falls + // outside [rangeStart 00:00:00 UTC, next midnight UTC) is rejected. A valid + // 23:30 UTC late-day shard is accepted under ANY host timezone. + const rangeStartNano = BigInt(Date.parse(`${rangeStart}T00:00:00.000Z`)) * 1_000_000n + const rangeEndNano = BigInt(Date.parse(rangeEndExclusive)) * 1_000_000n + if (minNano < rangeStartNano || maxNano >= rangeEndNano) { + throw new Error( + `archive shard ${name}: event time [${minNano}, ${maxNano}] ns outside sealed range ` + + `[${rangeStartNano}, ${rangeEndNano}) ns`, + ) + } + const bytes = requiredCount(value, "bytes") + const complexDigest = requiredString(value, "complexDigest") + if (!/^[0-9]+$/.test(complexDigest)) { + throw new Error(`invalid archive shard complexDigest (must be a numeric digest): ${complexDigest}`) + } + const complexDigestAlgorithm = requiredString(value, "complexDigestAlgorithm") + if (!KNOWN_COMPLEX_DIGEST_ALGORITHMS.has(complexDigestAlgorithm)) { + throw new Error( + `invalid archive shard complexDigestAlgorithm: ${complexDigestAlgorithm} ` + + `(known: ${[...KNOWN_COMPLEX_DIGEST_ALGORITHMS].join(", ")}); the manifest is preserved as-is`, + ) + } + return { + name, + rowCount, + minEventTimeUnixNano: minNano.toString(), + maxEventTimeUnixNano: maxNano.toString(), + sha256, + bytes, + columns, + complexDigest, + complexDigestAlgorithm, + } +} + +/** + * Strictly parse an archive generation manifest. Binds the manifest to its + * expected (signal, range, generation) location and rejects unknown format + * versions, absent/wrongly typed fields, negative or non-finite counts, signal + * or range mismatch, and malformed shard records. + */ +export const parseArchiveGenerationManifest = ( + value: unknown, + expectedSignal?: string, + expectedRange?: string, + expectedGenerationId?: string, +): ArchiveGenerationManifest => { + if (!isRecord(value)) { + throw new Error("malformed archive generation manifest (not a record)") + } + // Fail closed on an unknown OR older format version, preserving the files for + // inspection. v3 introduced the structured, SHA-256-bound `tuningConfig` + // identity; a v2 manifest (bare `tuningConfigName`) is incompatible with this + // reader because the config-identity semantics changed — silently treating + // the missing field as null would lose the config identity. Surface it + // distinctly so the operator re-exports. + if (value.formatVersion !== MANIFEST_FORMAT_VERSION) { + throw new Error( + `unsupported archive manifest formatVersion ${String(value.formatVersion)} (expected ${MANIFEST_FORMAT_VERSION}); ` + + `the manifest is preserved as-is. v3 introduced the structured tuningConfig identity (v2 used a bare name and v1 used timezone-dependent time evidence); re-export the range to re-validate.`, + ) + } + assertExactOwnKeys(value, MANIFEST_KEYS) + const signal = requiredString(value, "signal") + if (!isArchiveSignalName(signal)) throw new Error(`unknown archive signal: ${signal}`) + if (expectedSignal && signal !== expectedSignal) { + throw new Error(`archive manifest signal mismatch: expected ${expectedSignal}, got ${signal}`) + } + const rangeStart = validateRangeDate(requiredString(value, "rangeStart")) + if (expectedRange && rangeStart !== expectedRange) { + throw new Error(`archive manifest range mismatch: expected ${expectedRange}, got ${rangeStart}`) + } + const generationId = validateArchiveId(requiredString(value, "generationId"), "generation") + if (expectedGenerationId && generationId !== expectedGenerationId) { + throw new Error( + `archive manifest generation mismatch: expected ${expectedGenerationId}, got ${generationId}`, + ) + } + // Parse and validate rangeEndExclusive BEFORE the shards so each shard record + // can be bound to the sealed range (blocker #6). + const rangeEndExclusive = requiredIso(value, "rangeEndExclusive") + // rangeEndExclusive must be the next midnight after rangeStart (exclusive end). + const expectedEnd = nextMidnightUtc(rangeStart) + if (rangeEndExclusive !== expectedEnd) { + throw new Error( + `archive manifest rangeEndExclusive must be next-midnight ${expectedEnd}, got ${rangeEndExclusive}`, + ) + } + const shardsRaw = value.shards + if (!Array.isArray(shardsRaw)) throw new Error("invalid archive manifest field: shards") + const shards = shardsRaw.map((s) => parseShardRecord(s, rangeStart, rangeEndExclusive)) + // Cross-field validation (H-7): unique shard names, shard-row sum equals + // archivedRowCount, source count equals archived count. + const shardNames = new Set() + let shardRowSum = 0 + for (const shard of shards) { + if (shardNames.has(shard.name)) { + throw new Error(`archive manifest has duplicate shard name: ${shard.name}`) + } + shardNames.add(shard.name) + shardRowSum += shard.rowCount + } + const sourceRowCount = requiredCount(value, "sourceRowCount") + const archivedRowCount = requiredCount(value, "archivedRowCount") + if (shardRowSum !== archivedRowCount) { + throw new Error( + `archive manifest shard row sum (${shardRowSum}) != archivedRowCount (${archivedRowCount})`, + ) + } + if (sourceRowCount !== archivedRowCount) { + throw new Error( + `archive manifest sourceRowCount (${sourceRowCount}) != archivedRowCount (${archivedRowCount})`, + ) + } + return { + formatVersion: MANIFEST_FORMAT_VERSION, + generationId, + signal, + rangeStart, + rangeEndExclusive, + checkpointId: validateArchiveId(requiredString(value, "checkpointId"), "checkpoint"), + checkpointManifestFingerprint: requiredString(value, "checkpointManifestFingerprint"), + createdAt: requiredIso(value, "createdAt"), + mapleVersion: requiredString(value, "mapleVersion"), + chdbVersion: requiredString(value, "chdbVersion"), + schemaFingerprint: requiredString(value, "schemaFingerprint"), + sourceRowCount, + archivedRowCount, + tuning: parseTuningRecord(value.tuning), + // v3 requires its own explicit tuningConfig key. The value is either null + // or the strict, SHA-256-bound identity parsed above. + tuningConfig: parseTuningConfig(value.tuningConfig), + shards, + } +} + +/** + * Read and strictly parse a generation manifest from its on-disk path. Binds + * the manifest to its (signal, range, generation) directory so a manifest + * copied or moved to the wrong location is rejected. + */ +export const readArchiveGenerationManifest = ( + archiveDir: string, + signal: string, + rangeDate: string, + generationId: string, +): ArchiveGenerationManifest => { + const path = generationManifestPath(archiveDir, signal, rangeDate, generationId) + // Refuse a symlinked descendant on the READ path (the C-1 write fix's mirror): + // a planted symlink on the signal/range/generation/manifest chain would be + // followed by readFileSync, reading attacker-controlled content from outside + // the archive root. This is the single chokepoint for manifest reads. + assertNoSymlinkSync(archiveDir, path, "archive manifest") + assertRealFileSync(path, "archive manifest") + const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown + return parseArchiveGenerationManifest(parsed, signal, rangeDate, generationId) +} + +export const parseArchiveActivePointer = ( + value: unknown, + expectedSignal?: string, + expectedRange?: string, +): ArchiveActivePointer => { + if (!isRecord(value) || value.formatVersion !== ACTIVE_POINTER_FORMAT_VERSION) { + throw new Error("unsupported or malformed archive active pointer") + } + const signal = requiredString(value, "signal") + const rangeStart = validateRangeDate(requiredString(value, "rangeStart")) + // Bind the pointer to its on-disk (signal, range) directory so a pointer + // copied or moved to the wrong range cannot be silently accepted (H-7). + if (expectedSignal && signal !== expectedSignal) { + throw new Error(`active pointer signal mismatch: expected ${expectedSignal}, recorded ${signal}`) + } + if (expectedRange && rangeStart !== expectedRange) { + throw new Error(`active pointer range mismatch: expected ${expectedRange}, recorded ${rangeStart}`) + } + return { + formatVersion: ACTIVE_POINTER_FORMAT_VERSION, + generationId: validateArchiveId(requiredString(value, "generationId"), "generation"), + signal, + rangeStart, + selectedAt: requiredIso(value, "selectedAt"), + } +} + +/** Resolve the shard file path for a record within a generation. */ +export const shardFilePath = ( + archiveDir: string, + signal: string, + rangeDate: string, + generationId: string, + shardName: string, +): string => + join(generationManifestPath(archiveDir, signal, rangeDate, generationId), "..", "shards", shardName) diff --git a/apps/cli/src/server/archives/paths.ts b/apps/cli/src/server/archives/paths.ts new file mode 100644 index 000000000..f3198b858 --- /dev/null +++ b/apps/cli/src/server/archives/paths.ts @@ -0,0 +1,344 @@ +import { lstat, mkdir, readdir } from "node:fs/promises" +import { existsSync, lstatSync } from "node:fs" +import { isAbsolute, join, relative, resolve, sep } from "node:path" +import { randomUUID } from "node:crypto" + +// Archive path model and path-safety primitives. +// +// The archive root is operator-configured (an external volume in deployment). +// It never lives inside the live Maple data directory. Every component below it +// is constructed from validated IDs and a validated signal/range, then resolved +// and proven to stay inside the configured root before any mutation — mirroring +// the checkpoint module's path discipline. Symlinks are rejected at every level +// a path is used as state, operation, manifest, shard, quarantine, or building +// input, because a symlinked descendant can escape the configured root and +// mutate unrelated filesystem content (a defect the Phase 1 review caught and +// closed for checkpoints; the same hazard exists for archives). + +const ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i + +/** A UTC date in `YYYY-MM-DD` form, naming a sealed archive range's start. */ +const RANGE_DATE = /^\d{4}-\d{2}-\d{2}$/ + +export const validateArchiveId = (value: string, kind: string): string => { + if (!ID.test(value)) throw new Error(`invalid ${kind} ID: ${value}`) + return value.toLowerCase() +} + +export const validateRangeDate = (value: string): string => { + if (!RANGE_DATE.test(value)) throw new Error(`invalid archive range date: ${value}`) + // Reject impossible calendar dates (e.g. 2026-02-31). JavaScript's Date + // constructor normalizes impossible dates (rolls Feb 31 to Mar 3) rather + // than returning NaN, so we must verify the date round-trips: construct the + // date, then check its UTC year/month/day match the input exactly. + const [, y, m, d] = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value)! + const year = Number(y) + const month = Number(m) - 1 // JS months are 0-based + const day = Number(d) + const date = new Date(Date.UTC(year, month, day)) + if (date.getUTCFullYear() !== year || date.getUTCMonth() !== month || date.getUTCDate() !== day) { + throw new Error(`invalid archive range date (impossible calendar date): ${value}`) + } + return value +} + +/** + * Compute the exclusive end of a UTC day as the next day's midnight in ISO form + * (e.g. `2026-06-01` → `2026-06-02T00:00:00.000Z`). Used for the + * `rangeEndExclusive` manifest field; the prior `23:59:59.999999999Z` was + * inclusive, not exclusive. + */ +export const nextMidnightUtc = (rangeDate: string): string => { + const validated = validateRangeDate(rangeDate) + const [, y, m, d] = /^(\d{4})-(\d{2})-(\d{2})$/.exec(validated)! + const date = new Date(Date.UTC(Number(y), Number(m) - 1, Number(d))) + date.setUTCDate(date.getUTCDate() + 1) + return date.toISOString() +} + +export const newArchiveGenerationId = (): string => validateArchiveId(randomUUID(), "archive generation") + +export const archiveRoot = (archiveDir: string): string => resolve(archiveDir) + +export const signalRoot = (archiveDir: string, signal: string): string => + join(archiveRoot(archiveDir), signal) + +export const rangeRoot = (archiveDir: string, signal: string, rangeDate: string): string => + join(signalRoot(archiveDir, signal), validateRangeDate(rangeDate)) + +export const generationsRoot = (archiveDir: string, signal: string, rangeDate: string): string => + join(rangeRoot(archiveDir, signal, rangeDate), "generations") + +export const generationRoot = ( + archiveDir: string, + signal: string, + rangeDate: string, + generationId: string, +): string => + join(generationsRoot(archiveDir, signal, rangeDate), validateArchiveId(generationId, "generation")) + +export const generationManifestPath = ( + archiveDir: string, + signal: string, + rangeDate: string, + generationId: string, +): string => join(generationRoot(archiveDir, signal, rangeDate, generationId), "manifest.json") + +export const shardsRoot = ( + archiveDir: string, + signal: string, + rangeDate: string, + generationId: string, +): string => join(generationRoot(archiveDir, signal, rangeDate, generationId), "shards") + +export const activePointerPath = (archiveDir: string, signal: string, rangeDate: string): string => + join(rangeRoot(archiveDir, signal, rangeDate), "active.json") + +export const catalogPath = (archiveDir: string, signal: string): string => + join(signalRoot(archiveDir, signal), "catalog.jsonl") + +export const buildingRoot = (archiveDir: string): string => join(archiveRoot(archiveDir), "building") + +export const buildingGenerationRoot = (archiveDir: string, generationId: string): string => + join(buildingRoot(archiveDir), validateArchiveId(generationId, "generation")) + +export const archiveQuarantineRoot = (archiveDir: string): string => + join(archiveRoot(archiveDir), "quarantine") + +/** + * Resolve `candidate` and prove it stays inside `root`. Returns the absolute + * candidate. Anything that resolves outside the root, or to the root itself via + * `..`, is rejected. This is the same containment check the checkpoint module + * uses; path string-prefix checks alone are insufficient. + */ +export const assertContained = (root: string, candidate: string, label: string): string => { + const absoluteRoot = resolve(root) + const absoluteCandidate = resolve(candidate) + const rel = relative(absoluteRoot, absoluteCandidate) + if (rel === "" || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { + throw new Error(`${label} escapes configured archive root`) + } + return absoluteCandidate +} + +/** + * Refuse a symlink at any depth of `candidate` beneath `root`. Walks each path + * component with `lstat` immediately before use; a symlink anywhere on the path + * fails closed. Missing components are allowed (the path may not exist yet). + */ +export const assertNoSymlink = async (root: string, candidate: string, label: string): Promise => { + const absoluteRoot = resolve(root) + const absoluteCandidate = assertContained(absoluteRoot, candidate, label) + try { + if ((await lstat(absoluteRoot)).isSymbolicLink()) { + throw new Error(`refusing symlink archive root: ${absoluteRoot}`) + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + } + const rel = relative(absoluteRoot, absoluteCandidate) + let current = absoluteRoot + for (const part of rel.split(sep)) { + current = join(current, part) + try { + if ((await lstat(current)).isSymbolicLink()) { + throw new Error(`refusing symlink in ${label}: ${current}`) + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + return + } + } +} + +/** + * Synchronous variant of {@link assertNoSymlink} for use in synchronous + * read-side code (listing, catalog rebuild). Walks each existing component with + * `lstatSync`; a symlink anywhere on the path from `root` to `candidate` fails + * closed. + */ +export const assertNoSymlinkSync = (root: string, candidate: string, label: string): void => { + const absoluteRoot = resolve(root) + const absoluteCandidate = assertContained(absoluteRoot, candidate, label) + try { + if (lstatSync(absoluteRoot).isSymbolicLink()) { + throw new Error(`refusing symlink archive root: ${absoluteRoot}`) + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + } + const rel = relative(absoluteRoot, absoluteCandidate) + let current = absoluteRoot + for (const part of rel.split(sep)) { + current = join(current, part) + try { + if (lstatSync(current).isSymbolicLink()) { + throw new Error(`refusing symlink in ${label}: ${current}`) + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + return + } + } +} + +export type ArchivePathTopology = "absent" | "real-directory" | "real-file" + +/** + * Classify a path beneath a trusted root without following symlinks. + * + * A final-path `lstatSync` alone is insufficient: an absent leaf beneath an + * existing symlinked ancestor also reports ENOENT. Walk and validate every + * existing component first, then classify the final entry. Only a genuinely + * missing component on a symlink-free path is reported as absent. + */ +export const classifyArchivePathSync = ( + root: string, + candidate: string, + label: string, +): ArchivePathTopology => { + const absoluteCandidate = assertContained(root, candidate, label) + assertNoSymlinkSync(root, absoluteCandidate, label) + try { + const info = lstatSync(absoluteCandidate) + if (info.isDirectory()) return "real-directory" + if (info.isFile()) return "real-file" + if (info.isSymbolicLink()) { + // assertNoSymlinkSync already rejects this. Keep the explicit branch + // as a fail-closed guard against a same-user race between checks. + throw new Error(`refusing symlink in ${label}: ${absoluteCandidate}`) + } + throw new Error(`unexpected entry type for ${label}: ${absoluteCandidate}`) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return "absent" + throw error + } +} + +export const assertRealDirectory = async (path: string, label: string): Promise => { + const info = await lstat(path) + if (info.isSymbolicLink() || !info.isDirectory()) { + throw new Error(`${label} must be a real directory: ${path}`) + } +} + +export const assertRealFile = async (path: string, label: string): Promise => { + const info = await lstat(path) + if (info.isSymbolicLink() || !info.isFile()) { + throw new Error(`${label} must be a real file: ${path}`) + } +} + +/** + * Recursively walk a directory tree, refusing symlinks and unsupported special + * files at every depth. Returns the total byte size of real files. Used to + * validate a Parquet shard tree and to measure generated output before any + * manifest or pointer commit — a symlinked shard could otherwise point outside + * the archive root. + */ +export const treeBytes = async (path: string): Promise => { + let total = 0 + const stack: string[] = [path] + while (stack.length > 0) { + const current = stack.pop()! + const info = await lstat(current) + if (info.isSymbolicLink()) throw new Error(`refusing symlink in archive tree: ${current}`) + if (info.isFile()) { + total += info.size + continue + } + if (!info.isDirectory()) throw new Error(`unsupported archive entry type: ${current}`) + for (const entry of await readdir(current)) stack.push(join(current, entry)) + } + return total +} + +/** + * Ensure `path` exists with restrictive permissions, refusing a pre-existing + * symlink or non-directory at ANY ancestor. `mkdir -p` followed by a single + * `lstat` of the final entry is unsafe: a symlinked ancestor (e.g. + * `/traces -> /outside`) is followed by recursive mkdir, silently + * creating the tree under the symlink target outside the configured root. + * + * This walks each existing ancestor with `lstat` first, creates missing + * components one at a time (refusing to cross a symlink), then verifies the + * final entry. `root` must be an ancestor of `path`; every component from `root` + * to `path` is checked. + */ +export const ensurePrivateDirectory = async (path: string, root: string): Promise => { + const absolute = resolve(path) + const absoluteRoot = resolve(root) + // The root must exist or be creatable as the first component. If the root + // itself does not exist yet (fresh archive root), create it with restrictive + // permissions before walking. A fresh root that fails ENOENT on lstat is + // expected here; the prior code treated a missing root as the walk start and + // then failed when lstat on the first child hit a non-existent parent. + const rel = relative(absoluteRoot, absolute) + if (rel === "") { + // path IS the root: ensure it exists as a real private directory. + try { + const info = await lstat(absoluteRoot) + if (info.isSymbolicLink()) throw new Error(`refusing symlink archive root: ${absoluteRoot}`) + if (!info.isDirectory()) throw new Error(`archive root must be a directory: ${absoluteRoot}`) + return + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + await mkdir(absoluteRoot, { recursive: true, mode: 0o700 }) + return + } + } + if (rel.startsWith("..")) throw new Error(`archive path escapes root: ${path}`) + // Ensure the root exists first (handles fresh-root creation). + try { + const rootInfo = await lstat(absoluteRoot) + if (rootInfo.isSymbolicLink()) throw new Error(`refusing symlink archive root: ${absoluteRoot}`) + if (!rootInfo.isDirectory()) throw new Error(`archive root must be a directory: ${absoluteRoot}`) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + await mkdir(absoluteRoot, { recursive: true, mode: 0o700 }) + } + // Walk from the root down, checking each existing component is a real dir and + // creating missing ones. This refuses to cross a symlink at any depth. + let current = absoluteRoot + for (const part of rel.split(sep)) { + if (part === "") continue + current = join(current, part) + let info + try { + info = await lstat(current) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + await mkdir(current, { mode: 0o700 }) + continue + } + if (info.isSymbolicLink()) throw new Error(`refusing symlink in archive path: ${current}`) + if (!info.isDirectory()) throw new Error(`archive path component is not a directory: ${current}`) + } +} + +/** + * Synchronously verify a path is a real (non-symlink) regular file before + * reading it. Use at every read site so a symlinked or non-file entry cannot + * feed attacker-controlled or undefined content to a parser. + */ +export const assertRealFileSync = (path: string, label: string): void => { + const info = lstatSync(path) + if (info.isSymbolicLink() || !info.isFile()) { + throw new Error(`${label} must be a real file: ${path}`) + } +} + +/** Reject an archive root that is, or sits inside, the live Maple data dir. */ +export const assertArchiveRootSeparate = (archiveDir: string, dataDir: string): void => { + const archive = resolve(archiveDir) + const data = resolve(dataDir) + const rel = relative(data, archive) + if (archive === data || rel === "" || (rel !== ".." && !rel.startsWith(`..${sep}`))) { + throw new Error( + `archive root must not be the live data directory or one of its descendants: ${archiveDir}`, + ) + } + if (existsSync(archive) && lstatSync(archive).isSymbolicLink()) { + throw new Error(`archive root must not be a symlink: ${archive}`) + } +} diff --git a/apps/cli/src/server/archives/reconcile.ts b/apps/cli/src/server/archives/reconcile.ts new file mode 100644 index 000000000..ababb195b --- /dev/null +++ b/apps/cli/src/server/archives/reconcile.ts @@ -0,0 +1,239 @@ +// The single reconciliation decision engine (Gate 3b r5). +// +// ONE pure transition-table function (`decideReconciliation`) consumes a +// validated snapshot (or a fail-closed/no-op inspection result) and returns a +// `ReconciliationDecision`. Dry-run renders it; apply executes its branch via +// the proven mutation helpers. This pure function is the sole branch logic — +// there is no second `if phase...` implementation anywhere. +// +// Validation failures (root/pin/scratch/pointer-CAS/owned-path/identity) surface +// as FailClosed inspections that enter the SAME decision function, so every +// unsafe state is decided uniformly: fail-closed, zero mutation. + +import { + PHASE_ORDER, + phaseAtLeast, + type ArchiveOperationIntent, + type CreateOperationIntent, + type GcOperationIntent, +} from "./journal" + +// --------------------------------------------------------------------------- +// Inspection result: the complete output of read-only validation. +// --------------------------------------------------------------------------- + +/** + * The read-only validation outcome. `inspectReconciliationState` returns one of: + * - `null` — no active operation; + * - `{ kind: "FailClosed", reason }` — unsafe/ambiguous state (the validation + * threw; nothing was mutated); + * - `{ kind: "ValidSnapshot", snapshot }` — a fully validated snapshot. + * + * This is the sole input shape to `decideReconciliation`, so validation failures + * enter the SAME decision function as valid state. + */ +export type ReconciliationInspection = null | FailClosedInspection | ValidSnapshotInspection + +export interface FailClosedInspection { + readonly kind: "FailClosed" + readonly reason: string +} + +export interface ValidSnapshotInspection { + readonly kind: "ValidSnapshot" + readonly snapshot: ReconciliationSnapshot +} + +/** + * A complete, read-only, validated snapshot of the active operation + topology. + * Every field the pure decision function needs; nothing observed twice. + * + * The inspector (`inspectReconciliationState`, to be implemented in the wiring + * commit) runs ALL read-only validation before producing this: + * - guarded V2/V3 journal (no-symlink, real-file, directory/record binding); + * - the lifted v3 record for v2 (preserving its EXACT phase/topology — a v2 + * record can hold any create-eligible phase, not just "intent"); + * - `assertReconciliationRoots` (archive/data/scratch root safety); + * - `validateReconciliationTopology` (building/final real non-symlink dirs, the + * both-present gate, owned-scratch safety); + * - `validateOwnedPinState` (pin presence matches phase + identity); + * - pointer/manifest/catalog observations; + * - GC target/tombstone topology. + * Any failure → FailClosed (zero mutation). + */ +export interface ReconciliationSnapshot { + readonly operationId: string + readonly journalDigest: string + readonly migrationRequired: boolean + /** The v3 intent selecting the branch (on-disk v3, or the lifted v2 record). */ + readonly intent: ArchiveOperationIntent + // Create-kind topology (meaningful when intent.kind === "create"): + readonly promoted: boolean + readonly manifestAtFinal: boolean + readonly buildingPresent: boolean + /** building && finalGeneration both present — an impossible topology. */ + readonly buildingAndFinalBothPresent: boolean + // GC-kind observations (meaningful when intent.kind === "gc"): + readonly remainingTargets: number + readonly affectedSignals: ReadonlyArray +} + +// --------------------------------------------------------------------------- +// Decision: the sole output of the pure transition table. +// --------------------------------------------------------------------------- + +export type ReconciliationDecision = + | { readonly kind: "NoOp" } + | { readonly kind: "FailClosed"; readonly reason: string; readonly operationId?: string } + | { + readonly kind: "CreateVerifyComplete" + readonly operationId: string + readonly journalDigest: string + readonly migrationRequired: boolean + readonly intent: CreateOperationIntent + } + | { + readonly kind: "CreateAbortPrepublication" + readonly operationId: string + readonly journalDigest: string + readonly migrationRequired: boolean + readonly intent: CreateOperationIntent + readonly buildingPresent: boolean + } + | { + readonly kind: "CreateFinishPublication" + readonly operationId: string + readonly journalDigest: string + readonly migrationRequired: boolean + readonly intent: CreateOperationIntent + readonly affectedSignals: ReadonlyArray + } + | { + readonly kind: "GcVerifyComplete" + readonly operationId: string + readonly journalDigest: string + readonly migrationRequired: boolean + readonly intent: GcOperationIntent + } + | { + readonly kind: "GcResume" + readonly operationId: string + readonly journalDigest: string + readonly migrationRequired: boolean + readonly intent: GcOperationIntent + readonly remainingTargets: number + readonly affectedSignals: ReadonlyArray + } + +/** + * The ONE pure transition table. No I/O, no mutation. Consumes a + * `ReconciliationInspection` (which already encodes no-op / fail-closed / valid) + * and returns the exact decision branch. + * + * The impossible-topology gates (building+final both present, promoted without + * manifest, final before manifest-written, phase≥promoted without final, aborted + * in active) are decided HERE from the validated snapshot — they are the sole + * branch logic, testable without I/O. + */ +export const decideReconciliation = (inspection: ReconciliationInspection): ReconciliationDecision => { + if (inspection === null) return { kind: "NoOp" } + if (inspection.kind === "FailClosed") { + return { kind: "FailClosed", reason: inspection.reason } + } + const snapshot = inspection.snapshot + const { intent } = snapshot + + if (intent.kind === "gc") { + return decideGc(snapshot, intent) + } + return decideCreate(snapshot, intent) +} + +const decideGc = (snapshot: ReconciliationSnapshot, intent: GcOperationIntent): ReconciliationDecision => { + const { operationId, journalDigest, migrationRequired } = snapshot + if (intent.phase === "complete") { + return { kind: "GcVerifyComplete", operationId, journalDigest, migrationRequired, intent } + } + return { + kind: "GcResume", + operationId, + journalDigest, + migrationRequired, + intent, + remainingTargets: snapshot.remainingTargets, + affectedSignals: snapshot.affectedSignals, + } +} + +const decideCreate = ( + snapshot: ReconciliationSnapshot, + intent: CreateOperationIntent, +): ReconciliationDecision => { + const phase = intent.phase + const { operationId, journalDigest, migrationRequired } = snapshot + + // Impossible-topology gates — every one is FailClosed (zero mutation): + // 1. building && finalGeneration both present. + if (snapshot.buildingAndFinalBothPresent) { + return { + kind: "FailClosed", + reason: "archive operation has both building and final generation state", + operationId, + } + } + // 2. promoted without a manifest at the final location. + if (phase !== "aborted" && snapshot.promoted && !snapshot.manifestAtFinal) { + return { kind: "FailClosed", reason: "published a generation without a manifest", operationId } + } + // 3. final generation exists before the manifest-written phase. + if (snapshot.promoted && !phaseAtLeast(phase, "manifest-written") && phase !== "aborted") { + return { + kind: "FailClosed", + reason: "final generation exists before manifest-written phase", + operationId, + } + } + // 4. phase >= promoted but no final generation (phase ahead of reality). + if (!snapshot.promoted && phaseAtLeast(phase, "promoted") && phase !== "aborted") { + return { kind: "FailClosed", reason: `phase ${phase} requires its final generation`, operationId } + } + // 5. aborted operation still in the active directory. + if (phase === "aborted") { + return { kind: "FailClosed", reason: "aborted operation still in active dir", operationId } + } + + // Terminal verify-only: phase >= complete. + if (phaseAtLeast(phase, "complete")) { + return { kind: "CreateVerifyComplete", operationId, journalDigest, migrationRequired, intent } + } + + // Pre-publication abort: not promoted. + if (!snapshot.promoted) { + return { + kind: "CreateAbortPrepublication", + operationId, + journalDigest, + migrationRequired, + intent, + buildingPresent: snapshot.buildingPresent, + } + } + + // Post-promotion repair: promoted (always re-select pointer + rebuild catalog). + return { + kind: "CreateFinishPublication", + operationId, + journalDigest, + migrationRequired, + intent, + affectedSignals: [intent.signal], + } +} + +/** Helper for tests/snapshot builders: derive the journal digest of an intent. */ +export const digestOfIntent = (intent: ArchiveOperationIntent): string => { + const c = require("node:crypto") as typeof import("node:crypto") + return c.createHash("sha256").update(JSON.stringify(intent)).digest("hex") +} + +void PHASE_ORDER diff --git a/apps/cli/src/server/archives/signals.ts b/apps/cli/src/server/archives/signals.ts new file mode 100644 index 000000000..4b46d676d --- /dev/null +++ b/apps/cli/src/server/archives/signals.ts @@ -0,0 +1,44 @@ +// The six raw telemetry tables that Maple archives to Parquet. Aggregation and +// materialized-view tables are intentionally excluded: they are rebuildable from +// raw telemetry and would balloon archive volume without preserving any fact the +// raw tables do not already carry. +// +// Each signal names its event-time column, which drives the fixed half-open +// UTC-day range predicate (`>= start AND < end`). `logs` uses `TimestampTime` +// (the partition/TTL driver) for partition alignment while still bounding on +// `Timestamp` for nanosecond precision when needed; here we partition and range +// on the same column the store TTLs on, so an archived day is exactly the set of +// rows ClickHouse would have retained for that day. + +export type ArchiveSignalName = + | "logs" + | "traces" + | "metrics_sum" + | "metrics_gauge" + | "metrics_histogram" + | "metrics_exponential_histogram" + +export interface ArchiveSignal { + /** The raw table name; also the on-disk signal directory name. */ + readonly name: ArchiveSignalName + /** Event-time column used for the UTC-day range predicate. */ + readonly eventTimeColumn: string +} + +export const ARCHIVE_SIGNALS: ReadonlyArray = [ + { name: "logs", eventTimeColumn: "TimestampTime" }, + { name: "traces", eventTimeColumn: "Timestamp" }, + { name: "metrics_sum", eventTimeColumn: "TimeUnix" }, + { name: "metrics_gauge", eventTimeColumn: "TimeUnix" }, + { name: "metrics_histogram", eventTimeColumn: "TimeUnix" }, + { name: "metrics_exponential_histogram", eventTimeColumn: "TimeUnix" }, +] + +export const isArchiveSignalName = (value: string): value is ArchiveSignalName => + ARCHIVE_SIGNALS.some((s) => s.name === value) + +export const archiveSignal = (name: string): ArchiveSignal => { + const signal = ARCHIVE_SIGNALS.find((s) => s.name === name) + if (!signal) throw new Error(`unknown archive signal: ${name}`) + return signal +} diff --git a/apps/cli/src/server/archives/timed-process.ts b/apps/cli/src/server/archives/timed-process.ts new file mode 100644 index 000000000..de1a34d53 --- /dev/null +++ b/apps/cli/src/server/archives/timed-process.ts @@ -0,0 +1,94 @@ +import type { ChildProcess } from "node:child_process" +import { mkdtempSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" + +/** `/usr/bin/time` argv: `-lp` on macOS/BSD, `-v` on GNU/Linux. */ +export const timeArgv = (platform: NodeJS.Platform = process.platform): string[] => + platform === "darwin" ? ["-lp"] : ["-v"] + +/** Parse peak RSS (bytes) from `/usr/bin/time` output; fail-closed if unparseable. */ +export const parsePeakRss = (report: string, platform: NodeJS.Platform): number | null => { + // macOS/BSD `-lp`: "123456 maximum resident set size" (bytes). + // GNU/Linux `-v`: "Maximum resident set size (kbytes): 12345" (kbytes). + if (platform === "darwin") { + const match = report.match(/(\d+)\s+maximum resident set size/i) + return match ? Number.parseInt(match[1]!, 10) : null + } + const match = report.match(/Maximum resident set size \(kbytes\):\s*(\d+)/i) + return match ? Number.parseInt(match[1]!, 10) * 1024 : null +} + +export interface TimeReportRead { + readonly report: string + readonly error?: string +} + +export interface TimeReport { + /** Private path passed to `/usr/bin/time -o`; never inherited through stderr. */ + readonly path: string + /** Read the completed report and remove its private directory in every outcome. */ + readonly readAndRemove: () => TimeReportRead + /** Best-effort idempotent cleanup for spawn errors. */ + readonly remove: () => void +} + +/** + * Allocate a private, one-child timing-report directory. This keeps GNU/BSD + * `time` from writing its verbose report into Bun's nonblocking stderr pipe. + */ +export const createTimeReport = (temporaryRoot: string = tmpdir()): TimeReport => { + const directory = mkdtempSync(join(temporaryRoot, "maple-calibration-time-")) + const path = join(directory, "report.txt") + let removed = false + const remove = () => { + if (removed) return + removed = true + try { + rmSync(directory, { recursive: true, force: true }) + } catch { + // Best effort only: cleanup must never mask the worker outcome. + } + } + return { + path, + remove, + readAndRemove: () => { + try { + return { report: readFileSync(path, "utf8") } + } catch (error) { + return { + report: "", + error: `failed to read /usr/bin/time report: ${error instanceof Error ? error.message : String(error)}`, + } + } finally { + remove() + } + }, + } +} + +export interface ChildOutput { + readonly code: number | null + readonly signal: NodeJS.Signals | null + readonly stdout: string + readonly stderr: string +} + +/** + * Collect child diagnostics through `close`, not `exit`: Node emits `exit` + * before its stdio pipes are guaranteed to drain. The parent can therefore + * report complete worker output before launching the next calibration trial. + */ +export const collectChildOutputAfterClose = (child: ChildProcess): Promise => + new Promise((resolve) => { + let stdout = "" + let stderr = "" + child.stdout?.on("data", (chunk: Buffer) => { + stdout += chunk.toString() + }) + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString() + }) + child.once("close", (code, signal) => resolve({ code, signal, stdout, stderr })) + }) diff --git a/apps/cli/src/server/chdb.ts b/apps/cli/src/server/chdb.ts index c13613f73..f7209a187 100644 --- a/apps/cli/src/server/chdb.ts +++ b/apps/cli/src/server/chdb.ts @@ -81,8 +81,31 @@ export interface ChdbOptions { readonly dataDir: string /** Full DDL applied once at open (idempotent `IF NOT EXISTS`). */ readonly schemaSql: string + /** Optional ClickHouse config file passed through to chDB. */ + readonly configFile?: string + /** Apply the Maple schema after connect. Defaults to true. */ + readonly bootstrapSchema?: boolean } +/** Build the embedded ClickHouse argv. Keep table metadata loading and restore + * work serialized: + * chDB v26.1.0 can otherwise fail nondeterministically while its loader resolves + * Maple's materialized-view dependency graph (`recursive_mutex lock failed` / + * `ASYNC_LOAD_WAIT_FAILED`). `async_load_databases=0` waits for loading, but it + * does not make the loader pools single-threaded. RESTORE uses a separate + * 16-thread pool by default and can trip the same invalid recursive-mutex state + * while restoring that dependency graph. */ +export const chdbArgv = (options: Pick): string[] => [ + "clickhouse", + "--async_load_databases=0", + "--async_load_system_database=0", + "--tables_loader_foreground_pool_size=1", + "--tables_loader_background_pool_size=1", + "--restore_threads=1", + `--path=${options.dataDir}`, + ...(options.configFile ? [`--config-file=${options.configFile}`] : []), +] + /** * A live chDB connection. `query` runs read SQL and returns the raw result * bytes (in whatever `format` was requested — default JSONEachRow). `exec` runs @@ -101,38 +124,32 @@ export class Chdb { static open(options: ChdbOptions): Chdb { const sym = symbols() - // `--async_load_databases=0`: make chdb_connect BLOCK until every persisted - // table has finished loading before it returns. chDB v26.1.0 defaults this to - // true, so connect returns while the existing tables (our ~30 MVs feeding - // MergeTree targets) are still loading on background loader threads — and the - // `#bootstrap` DDL we run immediately after (`CREATE TABLE IF NOT EXISTS …`) - // then races the concurrent load of the same table, tripping a - // `recursive_mutex lock failed: Invalid argument` → `ASYNC_LOAD_WAIT_FAILED` - // → chdb_connect returns NULL. Most visible after an unclean kill, when more - // load/merge work is still in flight on reopen. Waiting serializes load before - // bootstrap and removes the race. (`--async_load_system_database=0` is already - // the default in this build; set for symmetry / future-proofing.) - const args = [ - "clickhouse", - "--async_load_databases=0", - "--async_load_system_database=0", - `--path=${options.dataDir}`, - ] + const args = chdbArgv(options) const argBufs = args.map(cstr) const argv = new BigUint64Array(args.length) argBufs.forEach((b, i) => { argv[i] = BigInt(ptr(b)) }) const connPtrPtr = sym.chdb_connect(args.length, ptr(argv)) - if (!connPtrPtr) throw new Error(Chdb.#connectFailure(options.dataDir, "chdb_connect returned NULL")) + if (!connPtrPtr) { + throw new Error( + Chdb.#connectFailure(options.dataDir, "chdb_connect returned NULL", options.configFile), + ) + } // chdb_connect returns chdb_connection* (a double pointer); chdb_query // wants chdb_connection — dereference once. const conn = read.ptr(connPtrPtr, 0) as Pointer if (!conn) - throw new Error(Chdb.#connectFailure(options.dataDir, "chdb_connect produced a NULL connection")) + throw new Error( + Chdb.#connectFailure( + options.dataDir, + "chdb_connect produced a NULL connection", + options.configFile, + ), + ) const db = new Chdb(sym, connPtrPtr, conn) - db.#bootstrap(options.schemaSql) + if (options.bootstrapSchema !== false) db.#bootstrap(options.schemaSql) return db } @@ -141,7 +158,13 @@ export class Chdb { // kill); point the user at the recovery path rather than the raw libchdb // message. A failure over an empty dir is a different problem (missing/broken // libchdb), so keep the generic message there. - static #connectFailure(dataDir: string, raw: string): string { + static #connectFailure(dataDir: string, raw: string, configFile?: string): string { + if (configFile) { + return ( + `${raw} while loading the explicit chDB config at ${configFile}. ` + + "The existing store was not modified; correct or remove the config before retrying." + ) + } if (!storeHasData(dataDir)) return raw return ( `${raw} — the local store at ${dataDir} could not be opened ` + diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts new file mode 100644 index 000000000..a1ed02d37 --- /dev/null +++ b/apps/cli/src/server/checkpoints.ts @@ -0,0 +1,1979 @@ +import { randomUUID } from "node:crypto" +import { spawnSync } from "node:child_process" +import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { cp, lstat, mkdir, readFile, readdir, rm, stat } from "node:fs/promises" +import { tmpdir } from "node:os" +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path" +import { Effect, Schema } from "effect" +import { CHDB_VERSION, MAPLE_VERSION } from "../version" +import { Chdb } from "./chdb" +import { + type DurabilityFaults, + durableJson, + durableRemove, + durableRename, + ensurePrivateDirectory, + syncDirectory, + syncTree, +} from "./durable-files" +import { SCHEMA_FINGERPRINT } from "./serve" +import schemaSql from "./schema/local-schema.sql" with { type: "text" } +import { + markStoreClosedDurable, + storeMarkerPath, + storeOpenMarkerPath, + writeStoreMarkerDurable, +} from "./store-version" + +const STATE_FORMAT_VERSION = 1 +const MANIFEST_FORMAT_VERSION = 1 +const OPERATION_FORMAT_VERSION = 1 +const RESTORE_TRANSACTION_FORMAT_VERSION = 1 +const RESET_TRANSACTION_FORMAT_VERSION = 1 +const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i +const RESETTABLE_CHDB_ENTRIES = new Set(["data", "metadata", "store", "tmp"]) +export const CHECKPOINT_REOPEN_PROBE_ENV = "MAPLE_INTERNAL_CHECKPOINT_REOPEN_DATA_DIR" + +export class CheckpointError extends Schema.TaggedErrorClass()( + "@maple/cli/CheckpointError", + { message: Schema.String }, +) {} + +export interface CheckpointOptions { + readonly dataDir: string + readonly port: number + readonly faults?: DurabilityFaults +} + +export interface CheckpointValidation { + readonly validatedAt: string + readonly traces: number + readonly logs: number + readonly metricsSum: number + readonly metricsGauge: number + readonly metricsHistogram: number + readonly metricsExponentialHistogram: number + readonly materializedViews: number +} + +export interface CheckpointManifest { + readonly formatVersion: 1 + readonly checkpointId: string + readonly operationId: string + readonly mapleVersion: string + readonly chdbVersion: string + readonly schemaFingerprint: string + readonly createdAt: string + readonly sourceDataDir: string + readonly backupRelativePath: string + readonly backupBytes: number + readonly validation: CheckpointValidation +} + +export interface CheckpointState { + readonly formatVersion: 1 + readonly revision: string + readonly current: string + readonly previous: string | null + readonly committedAt: string +} + +export interface ResolvedCheckpoint { + readonly checkpointId: string + readonly snapshotDir: string + readonly backupDir: string + readonly backupSqlPath: string + readonly manifest: CheckpointManifest +} + +interface CheckpointOperation { + readonly formatVersion: 1 + readonly operationId: string + readonly checkpointId: string + readonly baseRevision: string | null + readonly baseCurrent: string | null + readonly basePrevious: string | null + readonly phase: + | "intent" + | "backup-complete" + | "manifest-complete" + | "pointer-complete" + | "retention-complete" + readonly startedAt: string +} + +interface MaintenanceOwner { + readonly formatVersion: 1 + readonly operationId: string + readonly pid: number + readonly startedAt: string +} + +interface RestoreTransaction { + readonly formatVersion: 1 + readonly operationId: string + readonly checkpointId: string + readonly quarantineId: string + readonly phase: "intent" | "restore-ready" | "old-quarantined" | "new-live" | "markers-committed" + readonly createdAt: string + readonly validation: CheckpointValidation | null +} + +interface ResetTransaction { + readonly formatVersion: 1 + readonly operationId: string + readonly dataDir: string + readonly targets: ReadonlyArray + readonly phase: "intent" | "live-cleared" | "markers-cleared" + readonly createdAt: string +} + +export interface RestoreRecoveryFaults { + readonly afterLiveQuarantineRename?: () => void | Promise + readonly afterOldQuarantinedRecord?: () => void | Promise + readonly afterRestoredLiveRename?: () => void | Promise + readonly afterNewLiveRecord?: () => void | Promise + readonly afterStoreMarkerWrite?: () => void | Promise + readonly afterOpenMarkerRemoval?: () => void | Promise + readonly afterMarkersCommittedRecord?: () => void | Promise + readonly afterReadyMarkerRemoval?: () => void | Promise + readonly afterRestoreRootRemoval?: () => void | Promise + readonly afterResetIntent?: () => void | Promise + readonly afterResetEntryRemoval?: (entry: string) => void | Promise + readonly afterResetLiveClearedRecord?: () => void | Promise + readonly afterResetStoreMarkerRemoval?: () => void | Promise + readonly afterResetOpenMarkerRemoval?: () => void | Promise + readonly afterResetMarkersClearedRecord?: () => void | Promise + readonly afterResetTransactionRemoval?: () => void | Promise +} + +export const checkpointRoot = (dataDir: string): string => join(resolve(dataDir), "backups") +export const checkpointStatePath = (dataDir: string): string => join(checkpointRoot(dataDir), "state.json") +export const checkpointSnapshotsRoot = (dataDir: string): string => join(checkpointRoot(dataDir), "snapshots") +export const checkpointOperationsRoot = (dataDir: string): string => + join(checkpointRoot(dataDir), "operations") +export const checkpointPinsRoot = (dataDir: string): string => join(checkpointRoot(dataDir), "pins") +export const checkpointQuarantineRoot = (dataDir: string): string => + join(checkpointRoot(dataDir), "quarantine") +export const checkpointRetiringRoot = (dataDir: string): string => join(checkpointRoot(dataDir), "retiring") +export const checkpointSnapshotDir = (dataDir: string, checkpointId: string): string => + join(checkpointSnapshotsRoot(dataDir), validateId(checkpointId, "checkpoint")) + +const maintenanceLockPath = (dataDir: string): string => `${resolve(dataDir)}.maple-maintenance-lock` +export const restoreTransactionPath = (dataDir: string): string => + `${resolve(dataDir)}.restore-transaction.json` +export const resetTransactionPath = (dataDir: string): string => `${resolve(dataDir)}.reset-transaction.json` +export const restoreRootPath = (dataDir: string, operationId: string): string => + `${resolve(dataDir)}.restore-${validateId(operationId, "restore operation")}` +export const restoreDataPath = (dataDir: string, operationId: string): string => + join(restoreRootPath(dataDir, operationId), "data") +export const restoreQuarantinePath = (dataDir: string, operationId: string, quarantineId: string): string => + `${resolve(dataDir)}.quarantine-${validateId(operationId, "restore operation")}-${validateId( + quarantineId, + "quarantine", + )}` +const operationDir = (dataDir: string, operationId: string): string => + join(checkpointOperationsRoot(dataDir), `checkpoint-${validateId(operationId, "operation")}`) +const operationPath = (dataDir: string, operationId: string): string => + join(operationDir(dataDir, operationId), "intent.json") +const snapshotManifestPath = (dataDir: string, checkpointId: string): string => + join(checkpointSnapshotDir(dataDir, checkpointId), "manifest.json") +const snapshotBackupDir = (dataDir: string, checkpointId: string): string => + join(checkpointSnapshotDir(dataDir, checkpointId), "backup") +const snapshotBackupRelativePath = (checkpointId: string): string => + `snapshots/${validateId(checkpointId, "checkpoint")}/backup` +const snapshotBackupSqlPath = (checkpointId: string): string => + `backups/${snapshotBackupRelativePath(checkpointId)}` + +const validateId = (value: string, kind: string): string => { + if (!CHECKPOINT_ID.test(value)) throw new Error(`invalid ${kind} ID: ${value}`) + return value.toLowerCase() +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const requiredString = (record: Record, key: string): string => { + const value = record[key] + if (typeof value !== "string" || value.length === 0) throw new Error(`invalid ${key}`) + return value +} + +const requiredCount = (record: Record, key: string): number => { + const value = record[key] + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`invalid ${key}`) + } + return value +} + +const requiredIso = (record: Record, key: string): string => { + const value = requiredString(record, key) + if (!Number.isFinite(Date.parse(value))) throw new Error(`invalid ${key}`) + return value +} + +const assertContained = (root: string, candidate: string, label: string): string => { + const absoluteRoot = resolve(root) + const absoluteCandidate = resolve(candidate) + const rel = relative(absoluteRoot, absoluteCandidate) + if (rel === "" || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { + throw new Error(`${label} escapes configured root`) + } + return absoluteCandidate +} + +const assertNoSymlink = async (root: string, candidate: string): Promise => { + const absoluteRoot = resolve(root) + const absoluteCandidate = assertContained(absoluteRoot, candidate, "checkpoint path") + try { + if ((await lstat(absoluteRoot)).isSymbolicLink()) { + throw new Error(`refusing symlink checkpoint root: ${absoluteRoot}`) + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + } + const rel = relative(absoluteRoot, absoluteCandidate) + let current = absoluteRoot + for (const part of rel.split(sep)) { + current = join(current, part) + try { + if ((await lstat(current)).isSymbolicLink()) { + throw new Error(`refusing symlink in checkpoint path: ${current}`) + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + return + } + } +} + +const assertRealDirectory = async (path: string, label: string): Promise => { + const info = await lstat(path) + if (info.isSymbolicLink() || !info.isDirectory()) { + throw new Error(`${label} must be a real directory: ${path}`) + } +} + +const assertRealFile = async (path: string, label: string): Promise => { + const info = await lstat(path) + if (info.isSymbolicLink() || !info.isFile()) { + throw new Error(`${label} must be a real file: ${path}`) + } +} + +const assertCheckpointInfrastructureSafe = async (dataDir: string): Promise => { + const live = resolve(dataDir) + if (existsSync(live)) await assertRealDirectory(live, "live data directory") + const root = checkpointRoot(dataDir) + if (!existsSync(root)) return + await assertRealDirectory(root, "checkpoint root") + for (const child of ["snapshots", "operations", "pins", "quarantine", "retiring"]) { + const path = join(root, child) + if (existsSync(path)) await assertRealDirectory(path, `checkpoint ${child}`) + } + const statePath = checkpointStatePath(dataDir) + if (existsSync(statePath)) await assertRealFile(statePath, "checkpoint state") +} + +const xmlEscape = (value: string): string => + value + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'") + +const dataDirWithSlash = (dataDir: string): string => { + const abs = resolve(dataDir) + return abs.endsWith(sep) ? abs : `${abs}${sep}` +} + +export const writeBackupConfig = (path: string, sourceDataDir?: string): void => { + const sourceDisk = sourceDataDir + ? ` + + + + ${xmlEscape(dataDirWithSlash(sourceDataDir))} + + + ` + : "" + writeFileSync( + path, + ` + + ${sourceDataDir ? "src" : "default"} + backups + ${sourceDisk} + +`, + { mode: 0o600 }, + ) +} + +export class LocalQueryError extends Error { + readonly status: number + readonly detail: string + + constructor(status: number, detail: string) { + super(`local query failed (${status})${detail ? `: ${detail}` : ""}`) + this.name = "LocalQueryError" + this.status = status + this.detail = detail + } +} + +const postLocalQuery = async (port: number, sql: string): Promise => { + const response = await fetch(`http://127.0.0.1:${port}/local/query`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ sql }), + }) + if (!response.ok) { + const detail = await response.text().catch(() => "") + throw new LocalQueryError(response.status, detail) + } + return response.json() +} + +export const isMissingBackupConfigurationError = (error: unknown): boolean => { + const detail = + error instanceof LocalQueryError + ? error.detail + : error instanceof Error + ? error.message + : String(error) + const lower = detail.toLowerCase() + const backupSpecific = + lower.includes("backups.allowed_disk") || + lower.includes("backups.allowed_path") || + (lower.includes("backup") && + (lower.includes("not allowed") || + lower.includes("unknown disk") || + lower.includes("allowed disk"))) + return backupSpecific +} + +const readJsonRows = (text: string): ReadonlyArray> => + text + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as Record) + +const countFrom = (rows: ReadonlyArray>): number => { + const row = rows[0] + if (!row) return 0 + const value = row["count()"] ?? row.count + const count = typeof value === "number" ? value : Number(value ?? 0) + if (!Number.isSafeInteger(count) || count < 0) throw new Error(`invalid count result: ${value}`) + return count +} + +const queryCount = (db: Chdb, sql: string): number => countFrom(readJsonRows(db.query(sql))) + +const validateRestoredDatabase = (db: Chdb): CheckpointValidation => ({ + validatedAt: new Date().toISOString(), + traces: queryCount(db, "SELECT count() FROM traces"), + logs: queryCount(db, "SELECT count() FROM logs"), + metricsSum: queryCount(db, "SELECT count() FROM metrics_sum"), + metricsGauge: queryCount(db, "SELECT count() FROM metrics_gauge"), + metricsHistogram: queryCount(db, "SELECT count() FROM metrics_histogram"), + metricsExponentialHistogram: queryCount(db, "SELECT count() FROM metrics_exponential_histogram"), + materializedViews: queryCount( + db, + "SELECT count() FROM system.tables WHERE database = 'default' AND engine = 'MaterializedView'", + ), +}) + +/** Open and validate a restored store in the current process. Production + * restore calls this only from the dedicated child-process entrypoint in + * `bin.ts`; keeping it synchronous ensures the child exits only after chDB has + * closed cleanly. */ +export const validateCheckpointDataDir = (dataDir: string): CheckpointValidation => { + const resolvedDataDir = resolve(dataDir) + if (!isAbsolute(dataDir) || resolvedDataDir !== dataDir) { + throw new Error("checkpoint reopen probe requires a normalized absolute data directory") + } + if (!existsSync(resolvedDataDir)) { + throw new Error("checkpoint reopen probe data directory is missing") + } + const info = lstatSync(resolvedDataDir) + if (info.isSymbolicLink() || !info.isDirectory()) { + throw new Error("checkpoint reopen probe data directory is not a real directory") + } + const db = Chdb.open({ + dataDir: resolvedDataDir, + schemaSql, + bootstrapSchema: false, + }) + try { + return validateRestoredDatabase(db) + } finally { + db.close() + } +} + +const dirSize = async (path: string): Promise => { + let total = 0 + const entries = await readdir(path, { withFileTypes: true }) + for (const entry of entries) { + const child = join(path, entry.name) + if (entry.isSymbolicLink()) throw new Error(`refusing symlink in checkpoint backup: ${child}`) + if (entry.isDirectory()) total += await dirSize(child) + else if (entry.isFile()) total += (await stat(child)).size + else throw new Error(`refusing unsupported checkpoint entry: ${child}`) + } + return total +} + +const parseValidation = (value: unknown): CheckpointValidation => { + if (!isRecord(value)) throw new Error("invalid validation") + return { + validatedAt: requiredIso(value, "validatedAt"), + traces: requiredCount(value, "traces"), + logs: requiredCount(value, "logs"), + metricsSum: requiredCount(value, "metricsSum"), + metricsGauge: requiredCount(value, "metricsGauge"), + metricsHistogram: requiredCount(value, "metricsHistogram"), + metricsExponentialHistogram: requiredCount(value, "metricsExponentialHistogram"), + materializedViews: requiredCount(value, "materializedViews"), + } +} + +const validationCountsMatch = (left: CheckpointValidation, right: CheckpointValidation): boolean => + left.traces === right.traces && + left.logs === right.logs && + left.metricsSum === right.metricsSum && + left.metricsGauge === right.metricsGauge && + left.metricsHistogram === right.metricsHistogram && + left.metricsExponentialHistogram === right.metricsExponentialHistogram && + left.materializedViews === right.materializedViews + +/** Re-exec Maple and prove a wholly fresh process can load the restored + * representation. A successful query in the restoring connection is not + * sufficient: chDB's persisted metadata is loaded again only on process start. */ +const validateRestoredDatabaseInFreshProcess = ( + dataDir: string, + expected: CheckpointValidation, +): CheckpointValidation => { + const entry = process.argv[1] + const childArgs = entry && !entry.startsWith("/$bunfs") ? [entry] : [] + const child = spawnSync(process.execPath, childArgs, { + env: { ...process.env, [CHECKPOINT_REOPEN_PROBE_ENV]: resolve(dataDir) }, + encoding: "utf8", + timeout: 30_000, + maxBuffer: 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }) + const stderr = child.stderr.trim() + if (child.error) { + throw new Error(`fresh-process checkpoint reopen probe failed to run: ${child.error.message}`) + } + if (child.status !== 0) { + throw new Error( + `fresh-process checkpoint reopen probe failed${child.signal ? ` (${child.signal})` : ""}` + + `${stderr ? `: ${stderr.slice(-4096)}` : ""}`, + ) + } + let parsed: CheckpointValidation + try { + parsed = parseValidation(JSON.parse(child.stdout.trim()) as unknown) + } catch (error) { + throw new Error( + `fresh-process checkpoint reopen probe returned invalid output: ${error instanceof Error ? error.message : String(error)}`, + ) + } + if (!validationCountsMatch(expected, parsed)) { + throw new Error("fresh-process checkpoint reopen validation does not match the restoring process") + } + return parsed +} + +export const parseCheckpointManifest = ( + value: unknown, + expectedCheckpointId?: string, + expectedSourceDataDir?: string, +): CheckpointManifest => { + if (!isRecord(value) || value.formatVersion !== MANIFEST_FORMAT_VERSION) { + throw new Error("unsupported or malformed checkpoint manifest") + } + const checkpointId = validateId(requiredString(value, "checkpointId"), "checkpoint") + if (expectedCheckpointId && checkpointId !== validateId(expectedCheckpointId, "checkpoint")) { + throw new Error("checkpoint manifest ID does not match its snapshot directory") + } + const operationId = validateId(requiredString(value, "operationId"), "operation") + const sourceDataDir = requiredString(value, "sourceDataDir") + if (!isAbsolute(sourceDataDir)) throw new Error("checkpoint sourceDataDir must be absolute") + if (expectedSourceDataDir && resolve(sourceDataDir) !== resolve(expectedSourceDataDir)) { + throw new Error("checkpoint sourceDataDir does not match its configured owner") + } + const backupRelativePath = requiredString(value, "backupRelativePath") + if (backupRelativePath !== snapshotBackupRelativePath(checkpointId)) { + throw new Error("checkpoint backup path does not match its immutable ID") + } + const manifest: CheckpointManifest = { + formatVersion: 1, + checkpointId, + operationId, + mapleVersion: requiredString(value, "mapleVersion"), + chdbVersion: requiredString(value, "chdbVersion"), + schemaFingerprint: requiredString(value, "schemaFingerprint"), + createdAt: requiredIso(value, "createdAt"), + sourceDataDir, + backupRelativePath, + backupBytes: requiredCount(value, "backupBytes"), + validation: parseValidation(value.validation), + } + if (manifest.chdbVersion !== CHDB_VERSION) { + throw new Error( + `checkpoint chDB version mismatch (checkpoint: ${manifest.chdbVersion}; build: ${CHDB_VERSION})`, + ) + } + if (manifest.schemaFingerprint !== SCHEMA_FINGERPRINT) { + throw new Error( + `checkpoint schema mismatch (checkpoint: ${manifest.schemaFingerprint}; build: ${SCHEMA_FINGERPRINT})`, + ) + } + return manifest +} + +export const parseCheckpointState = (value: unknown): CheckpointState => { + if (!isRecord(value) || value.formatVersion !== STATE_FORMAT_VERSION) { + throw new Error("unsupported or malformed checkpoint state") + } + const current = validateId(requiredString(value, "current"), "current checkpoint") + const previousValue = value.previous + const previous = + previousValue === null + ? null + : validateId( + typeof previousValue === "string" + ? previousValue + : (() => { + throw new Error("invalid previous checkpoint") + })(), + "previous checkpoint", + ) + if (previous === current) throw new Error("checkpoint current and previous IDs must differ") + return { + formatVersion: 1, + revision: validateId(requiredString(value, "revision"), "state revision"), + current, + previous, + committedAt: requiredIso(value, "committedAt"), + } +} + +const readStateFileOptional = async (dataDir: string): Promise => { + try { + await assertCheckpointInfrastructureSafe(dataDir) + return parseCheckpointState(JSON.parse(await readFile(checkpointStatePath(dataDir), "utf8"))) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null + throw error + } +} + +const checkpointLikePaths = async (dataDir: string): Promise => { + const root = checkpointRoot(dataDir) + try { + const entries = await readdir(root, { withFileTypes: true }) + const unsafe: string[] = [] + for (const entry of entries) { + if (entry.name === "state.json" || entry.name === "quarantine") continue + const path = join(root, entry.name) + if (["snapshots", "operations", "pins", "retiring"].includes(entry.name)) { + if (!entry.isDirectory() || (await readdir(path)).length > 0) unsafe.push(path) + continue + } + unsafe.push(path) + } + return unsafe + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return [] + throw error + } +} + +const assertNoLegacyLayout = (dataDir: string): void => { + const legacy = ["building", "current", "previous"] + .map((name) => join(checkpointRoot(dataDir), name)) + .filter(existsSync) + if (legacy.length > 0) { + throw new Error( + `legacy preview checkpoint layout detected; refusing to infer or migrate state. Preserve and move aside after inspection: ${legacy.join(", ")}`, + ) + } +} + +export const readCheckpointState = async (dataDir: string): Promise => { + assertNoLegacyLayout(dataDir) + const state = await readStateFileOptional(dataDir) + if (!state) { + const paths = await checkpointLikePaths(dataDir) + throw new Error( + paths.length === 0 + ? `checkpoint state not found at ${checkpointStatePath(dataDir)}` + : `checkpoint state missing while checkpoint data exists; refusing to infer selection: ${paths.join(", ")}`, + ) + } + await resolveCheckpoint(dataDir, state.current, state) + if (state.previous) await resolveCheckpoint(dataDir, state.previous, state) + return state +} + +const resolveCheckpointById = async (dataDir: string, checkpointId: string): Promise => { + await assertCheckpointInfrastructureSafe(dataDir) + const validatedCheckpointId = validateId(checkpointId, "checkpoint") + const snapshotDir = checkpointSnapshotDir(dataDir, validatedCheckpointId) + const snapshotsRoot = checkpointSnapshotsRoot(dataDir) + await assertNoSymlink(checkpointRoot(dataDir), snapshotsRoot) + await assertNoSymlink(snapshotsRoot, snapshotDir) + await assertNoSymlink(snapshotsRoot, snapshotManifestPath(dataDir, validatedCheckpointId)) + await assertNoSymlink(snapshotsRoot, snapshotBackupDir(dataDir, validatedCheckpointId)) + await assertRealDirectory(snapshotDir, "checkpoint snapshot") + await assertRealFile(snapshotManifestPath(dataDir, validatedCheckpointId), "checkpoint manifest") + const manifest = parseCheckpointManifest( + JSON.parse(await readFile(snapshotManifestPath(dataDir, validatedCheckpointId), "utf8")), + validatedCheckpointId, + dataDir, + ) + const backupDir = snapshotBackupDir(dataDir, validatedCheckpointId) + await assertRealDirectory(backupDir, "checkpoint backup") + const actualBackupBytes = await dirSize(backupDir) + if (actualBackupBytes !== manifest.backupBytes) { + throw new Error( + `checkpoint backup size mismatch (manifest: ${manifest.backupBytes}; actual: ${actualBackupBytes})`, + ) + } + return { + checkpointId: validatedCheckpointId, + snapshotDir, + backupDir, + backupSqlPath: snapshotBackupSqlPath(validatedCheckpointId), + manifest, + } +} + +export const resolveCheckpoint = async ( + dataDir: string, + selector: "current" | "previous" | string = "current", + knownState?: CheckpointState, +): Promise => { + const state = knownState ?? (await readCheckpointState(dataDir)) + const checkpointId = + selector === "current" + ? state.current + : selector === "previous" + ? state.previous + : validateId(selector, "checkpoint") + if (!checkpointId) throw new Error("no previous checkpoint is selected") + return resolveCheckpointById(dataDir, checkpointId) +} + +export const readCheckpointManifest = async ( + dataDir: string, + selector: "current" | "previous" | string = "current", +): Promise => (await resolveCheckpoint(dataDir, selector)).manifest + +const restoreResolvedInto = async ( + resolvedCheckpoint: ResolvedCheckpoint, + targetDataDir: string, +): Promise<{ readonly db: Chdb; readonly validation: CheckpointValidation }> => { + const scratchParent = dirname(targetDataDir) + const scratchConfig = join(scratchParent, `checkpoint-${randomUUID()}.xml`) + writeBackupConfig(scratchConfig, resolvedCheckpoint.manifest.sourceDataDir) + let db: Chdb | undefined + try { + db = Chdb.open({ + dataDir: targetDataDir, + schemaSql, + configFile: scratchConfig, + bootstrapSchema: false, + }) + db.exec("CREATE DATABASE IF NOT EXISTS default") + db.exec( + `RESTORE DATABASE default FROM Disk('src', '${resolvedCheckpoint.backupSqlPath}') ` + + "SETTINGS allow_different_database_def=1", + ) + return { db, validation: validateRestoredDatabase(db) } + } catch (error) { + db?.close() + throw error + } finally { + rmSync(scratchConfig, { force: true }) + } +} + +export const withRestoredCheckpoint = async ( + resolvedCheckpoint: ResolvedCheckpoint, + options: { + readonly scratchRoot?: string + readonly cleanup?: "always" | "never" + /** + * A caller-supplied deterministic subdirectory name beneath scratchRoot, + * instead of the default random `maple-checkpoint-`. Used by the + * archive generation journal so an interrupted operation records the exact + * scratch path it owns and reconciliation can remove only that path. + * Must be a single path segment (no separators) and not already in use. + */ + readonly scratchSubdir?: string + /** + * Invoked after the owned scratch directory is created (and synced) but + * BEFORE the checkpoint is restored into it. Used by the archive journal + * to advance its phase to "scratch-allocated" so a kill during restore is + * reconcilable. No-op for non-archive callers. + */ + readonly beforeRestore?: (scratchDataDir: string) => void | Promise + }, + use: (restored: { + readonly checkpointId: string + readonly manifest: CheckpointManifest + readonly scratchDataDir: string + readonly db: Chdb + readonly validation: CheckpointValidation + }) => A | Promise, +): Promise => { + const scratchRoot = resolve(options.scratchRoot ?? tmpdir()) + const sourceDataDir = resolve(resolvedCheckpoint.manifest.sourceDataDir) + const sourceRelation = relative(sourceDataDir, scratchRoot) + if ( + scratchRoot === sourceDataDir || + (sourceRelation !== "" && sourceRelation !== ".." && !sourceRelation.startsWith(`..${sep}`)) + ) { + throw new Error("scratch root must not be the live data directory or one of its descendants") + } + if (existsSync(scratchRoot) && (await lstat(scratchRoot)).isSymbolicLink()) { + throw new Error(`scratch root must not be a symlink: ${scratchRoot}`) + } + await mkdir(scratchRoot, { recursive: true }) + const scratchParentName = options.scratchSubdir ?? `maple-checkpoint-${randomUUID()}` + // A deterministic scratchSubdir (archive journal) must be a single path + // segment so it cannot escape the scratch root, and must not already exist so + // reuse after a crash is unambiguous (the caller reconciles the prior op + // before allocating a new one). + if (options.scratchSubdir !== undefined) { + if ( + scratchParentName.length === 0 || + scratchParentName.includes(sep) || + scratchParentName.includes("/") || + scratchParentName === "." || + scratchParentName === ".." + ) { + throw new Error(`invalid scratch subdirectory: ${scratchParentName}`) + } + } + const scratchParent = join(scratchRoot, scratchParentName) + await mkdir(scratchParent, { mode: 0o700 }) + const scratchDataDir = join(scratchParent, "data") + let db: Chdb | undefined + try { + // The owned scratch dir now exists. Give the caller (archive journal) a + // seam to durably record that fact BEFORE the restore begins, so a kill + // mid-restore leaves a reconcilable "scratch-allocated" phase. + if (options.beforeRestore) await options.beforeRestore(scratchDataDir) + const restored = await restoreResolvedInto(resolvedCheckpoint, scratchDataDir) + db = restored.db + return await use({ + checkpointId: resolvedCheckpoint.checkpointId, + manifest: resolvedCheckpoint.manifest, + scratchDataDir, + db, + validation: restored.validation, + }) + } finally { + db?.close() + if (options.cleanup !== "never") await rm(scratchParent, { recursive: true, force: true }) + } +} + +const parseOperation = (value: unknown): CheckpointOperation => { + if (!isRecord(value) || value.formatVersion !== OPERATION_FORMAT_VERSION) { + throw new Error("unsupported or malformed checkpoint operation") + } + const phase = requiredString(value, "phase") + if ( + ![ + "intent", + "backup-complete", + "manifest-complete", + "pointer-complete", + "retention-complete", + ].includes(phase) + ) { + throw new Error("invalid checkpoint operation phase") + } + const baseRevision = + value.baseRevision === null + ? null + : validateId(requiredString(value, "baseRevision"), "base state revision") + const baseCurrent = + value.baseCurrent === null + ? null + : validateId(requiredString(value, "baseCurrent"), "base current checkpoint") + const basePrevious = + value.basePrevious === null + ? null + : validateId(requiredString(value, "basePrevious"), "base previous checkpoint") + if ((baseRevision === null) !== (baseCurrent === null)) { + throw new Error("checkpoint operation has an inconsistent base state") + } + if (baseCurrent === null && basePrevious !== null) { + throw new Error("checkpoint operation cannot have a previous checkpoint without a current checkpoint") + } + if (baseCurrent !== null && baseCurrent === basePrevious) { + throw new Error("checkpoint operation base current and previous must differ") + } + return { + formatVersion: 1, + operationId: validateId(requiredString(value, "operationId"), "operation"), + checkpointId: validateId(requiredString(value, "checkpointId"), "checkpoint"), + baseRevision, + baseCurrent, + basePrevious, + phase: phase as CheckpointOperation["phase"], + startedAt: requiredIso(value, "startedAt"), + } +} + +const writeOperation = async ( + dataDir: string, + operation: CheckpointOperation, + faults: DurabilityFaults = {}, +): Promise => durableJson(operationPath(dataDir, operation.operationId), operation, faults) + +const preserveCompletedOperation = async ( + dataDir: string, + operationDirPath: string, + operationId: string, + faults: DurabilityFaults = {}, +): Promise => { + const quarantineRoot = checkpointQuarantineRoot(dataDir) + if (existsSync(quarantineRoot)) { + await assertNoSymlink(checkpointRoot(dataDir), quarantineRoot) + await assertRealDirectory(quarantineRoot, "checkpoint quarantine") + } + await ensurePrivateDirectory(quarantineRoot) + const preserved = join( + quarantineRoot, + `completed-operation-${validateId(operationId, "operation")}-${randomUUID()}`, + ) + await durableRename(operationDirPath, preserved, faults) + await faults.afterCompletedOperationPreserved?.(preserved) +} + +const processIsAlive = (pid: number): boolean => { + try { + process.kill(pid, 0) + return true + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM" + } +} + +const acquireMaintenance = async (dataDir: string, operationId: string): Promise<() => Promise> => { + const lockPath = maintenanceLockPath(dataDir) + if (existsSync(lockPath)) await assertRealDirectory(lockPath, "maintenance lock") + try { + await mkdir(lockPath, { mode: 0o700 }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error + let owner: MaintenanceOwner + try { + const ownerPath = join(lockPath, "owner.json") + await assertRealFile(ownerPath, "maintenance lock owner") + const parsed = JSON.parse(readFileSync(ownerPath, "utf8")) as unknown + if (!isRecord(parsed) || parsed.formatVersion !== 1) throw new Error("malformed owner") + owner = { + formatVersion: 1, + operationId: validateId(requiredString(parsed, "operationId"), "operation"), + pid: requiredCount(parsed, "pid"), + startedAt: requiredIso(parsed, "startedAt"), + } + } catch { + throw new Error(`maintenance lock is present but ownership is uncertain: ${lockPath}`) + } + if (processIsAlive(owner.pid)) { + throw new Error(`another Maple maintenance operation is active (PID ${owner.pid})`) + } + const quarantinedLock = `${lockPath}.quarantine-${randomUUID()}` + await durableRename(lockPath, quarantinedLock) + await mkdir(lockPath, { mode: 0o700 }) + } + const owner: MaintenanceOwner = { + formatVersion: 1, + operationId, + pid: process.pid, + startedAt: new Date().toISOString(), + } + await durableJson(join(lockPath, "owner.json"), owner) + return async () => { + const ownerPath = join(lockPath, "owner.json") + await assertRealFile(ownerPath, "maintenance lock owner") + const current = JSON.parse(await readFile(ownerPath, "utf8")) as unknown + if ( + !isRecord(current) || + requiredString(current, "operationId") !== operationId || + current.pid !== process.pid + ) { + throw new Error(`maintenance lock ownership changed unexpectedly: ${lockPath}`) + } + await rm(lockPath, { recursive: true }) + await syncDirectory(dirname(lockPath)) + } +} + +export const reconcileCheckpointOperations = async ( + dataDir: string, + faults: DurabilityFaults = {}, +): Promise => { + await assertCheckpointInfrastructureSafe(dataDir) + const state = await readStateFileOptional(dataDir) + const operationsRoot = checkpointOperationsRoot(dataDir) + if (existsSync(operationsRoot)) { + await assertNoSymlink(checkpointRoot(dataDir), operationsRoot) + await assertRealDirectory(operationsRoot, "checkpoint operations") + } + let entries + try { + entries = await readdir(operationsRoot, { withFileTypes: true }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + if (entries.length === 0) return + if (entries.length !== 1) { + throw new Error( + `multiple checkpoint operations require operator inspection: ${entries + .map((entry) => join(operationsRoot, entry.name)) + .join(", ")}`, + ) + } + const entry = entries[0]! + if (!entry.isDirectory() || !entry.name.startsWith("checkpoint-")) { + throw new Error(`unrecognized checkpoint operation debris: ${join(operationsRoot, entry.name)}`) + } + const entryOperationId = validateId(entry.name.slice("checkpoint-".length), "operation directory") + const entryDir = join(operationsRoot, entry.name) + const intentPath = join(entryDir, "intent.json") + await assertNoSymlink(operationsRoot, entryDir) + await assertNoSymlink(operationsRoot, intentPath) + await assertRealDirectory(entryDir, "checkpoint operation") + await assertRealFile(intentPath, "checkpoint operation intent") + const operation = parseOperation(JSON.parse(await readFile(intentPath, "utf8"))) + if (operation.operationId !== entryOperationId) { + throw new Error( + `checkpoint operation identity mismatch (directory: ${entryOperationId}; intent: ${operation.operationId})`, + ) + } + + const baseMatches = + operation.baseRevision === null + ? state === null + : state !== null && + state.revision === operation.baseRevision && + state.current === operation.baseCurrent && + state.previous === operation.basePrevious + const expectedMatches = + state !== null && + state.revision === operation.operationId && + state.current === operation.checkpointId && + state.previous === operation.baseCurrent + const snapshot = checkpointSnapshotDir(dataDir, operation.checkpointId) + const manifestPath = snapshotManifestPath(dataDir, operation.checkpointId) + + if (existsSync(manifestPath)) { + const resolved = await resolveCheckpointById(dataDir, operation.checkpointId) + if (resolved.manifest.operationId !== operation.operationId) { + throw new Error("checkpoint manifest operation identity does not match its operation") + } + if (!baseMatches && !expectedMatches) { + throw new Error("checkpoint operation base no longer matches authoritative state") + } + if (baseMatches) { + if (!["backup-complete", "manifest-complete"].includes(operation.phase)) { + throw new Error(`checkpoint operation phase ${operation.phase} cannot publish its snapshot`) + } + if (operation.baseCurrent) await resolveCheckpointById(dataDir, operation.baseCurrent) + if (operation.basePrevious) await resolveCheckpointById(dataDir, operation.basePrevious) + const manifestComplete: CheckpointOperation = { + ...operation, + phase: "manifest-complete", + } + await writeOperation(dataDir, manifestComplete) + const nextState: CheckpointState = { + formatVersion: 1, + revision: operation.operationId, + current: operation.checkpointId, + previous: operation.baseCurrent, + committedAt: new Date().toISOString(), + } + await durableJson(checkpointStatePath(dataDir), nextState) + } + const publishedState = await readCheckpointState(dataDir) + if ( + publishedState.revision !== operation.operationId || + publishedState.current !== operation.checkpointId || + publishedState.previous !== operation.baseCurrent + ) { + throw new Error("checkpoint operation publication did not produce its exact intended state") + } + let retirement: string | null = null + if (operation.phase === "retention-complete") { + const candidate = join(checkpointRetiringRoot(dataDir), `retirement-${operation.operationId}`) + retirement = existsSync(candidate) ? candidate : null + } else { + await writeOperation(dataDir, { ...operation, phase: "pointer-complete" }) + retirement = await retireCheckpointIfEligible( + dataDir, + operation.basePrevious, + publishedState, + operation.operationId, + ) + await writeOperation(dataDir, { ...operation, phase: "retention-complete" }) + } + await removeCompletedRetirement(retirement, faults) + await preserveCompletedOperation(dataDir, entryDir, operation.operationId, faults) + return + } + + if (!baseMatches || !["intent", "backup-complete"].includes(operation.phase)) { + throw new Error("checkpoint operation is missing its manifest in an unsafe state") + } + const quarantineRoot = checkpointQuarantineRoot(dataDir) + if (existsSync(quarantineRoot)) { + await assertNoSymlink(checkpointRoot(dataDir), quarantineRoot) + await assertRealDirectory(quarantineRoot, "checkpoint quarantine") + } + const quarantine = join(quarantineRoot, `operation-${operation.operationId}-${randomUUID()}`) + await ensurePrivateDirectory(quarantineRoot) + await ensurePrivateDirectory(quarantine) + if (existsSync(snapshot)) { + await assertNoSymlink(checkpointSnapshotsRoot(dataDir), snapshot) + await assertRealDirectory(snapshot, "incomplete checkpoint snapshot") + await durableRename(snapshot, join(quarantine, "incomplete-snapshot")) + } + await durableRename(entryDir, join(quarantine, "operation")) +} + +const hasPins = async (dataDir: string, checkpointId: string): Promise => { + const path = join(checkpointPinsRoot(dataDir), checkpointId) + try { + await assertNoSymlink(checkpointRoot(dataDir), checkpointPinsRoot(dataDir)) + await assertNoSymlink(checkpointPinsRoot(dataDir), path) + await assertRealDirectory(path, "checkpoint pin reservation") + return (await readdir(path)).length > 0 + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false + throw error + } +} + +export interface CheckpointPin { + readonly formatVersion: 1 + readonly pinId: string + readonly checkpointId: string + readonly purpose: string + readonly createdAt: string +} + +/** Derive the exact pin-file path from a data dir, checkpoint id, and pin id. */ +export const pinFilePath = (dataDir: string, checkpointId: string, pinId: string): string => + join(checkpointPinsRoot(dataDir), checkpointId, `${validateId(pinId, "pin")}.json`) + +const PIN_PURPOSE = /^[A-Za-z0-9 _./:-]{0,128}$/ + +/** + * Acquire a persistent pin on a checkpoint so retention cannot delete its + * snapshot while the pin is held. The pin record is durably written under + * `backups/pins//.json`; `retireCheckpointIfEligible` + * already honors a non-empty pin directory. Callers that need pin acquisition + * to race neither GC nor a concurrent checkpoint operation should hold the + * maintenance lock (see {@link withMaintenanceLock}) while resolving and + * pinning. A stale pin over-retains data rather than risking deletion. + */ +export const acquireCheckpointPin = async ( + dataDir: string, + checkpointId: string, + purpose = "archive", + pinId: string = randomUUID(), +): Promise => { + const validatedCheckpointId = validateId(checkpointId, "checkpoint") + if (!PIN_PURPOSE.test(purpose)) throw new Error(`invalid checkpoint pin purpose: ${purpose}`) + const validatedPinId = validateId(pinId, "pin") + // A pin on a checkpoint that does not resolve cannot protect anything; force + // the caller to pin real, validated state. + await resolveCheckpoint(dataDir, validatedCheckpointId) + const pinsRoot = checkpointPinsRoot(dataDir) + const pinDir = join(pinsRoot, validatedCheckpointId) + await ensurePrivateDirectory(pinsRoot) + await assertNoSymlink(checkpointRoot(dataDir), pinsRoot) + await ensurePrivateDirectory(pinDir) + await assertNoSymlink(pinsRoot, pinDir) + const path = pinFilePath(dataDir, validatedCheckpointId, validatedPinId) + // A deterministic caller-supplied pinId (archive journal) must target an + // unused path so post-crash ownership is unambiguous: the journal records + // this exact pinId before acquisition, and reconciliation releases exactly it. + if (existsSync(path)) { + throw new Error(`checkpoint pin already exists; refusing to overwrite: ${path}`) + } + const pin: CheckpointPin = { + formatVersion: 1, + pinId: validatedPinId, + checkpointId: validatedCheckpointId, + purpose, + createdAt: new Date().toISOString(), + } + await durableJson(path, pin) + return path +} + +/** + * Validate one exact pin record without removing it. This is shared by session + * consumers and pin release so a same-path regular-file substitution cannot be + * mistaken for a live pin. + */ +export const assertCheckpointPinIdentity = async ( + dataDir: string, + checkpointId: string, + pinPath: string, + expectedPurpose?: string, +): Promise => { + const validatedCheckpointId = validateId(checkpointId, "checkpoint") + const pinsRoot = checkpointPinsRoot(dataDir) + const pinDir = join(pinsRoot, validatedCheckpointId) + // The pin file must live directly under the named checkpoint's pin dir. + const resolvedPinPath = resolve(pinPath) + if (relative(pinDir, resolvedPinPath).startsWith(`..${sep}`)) { + throw new Error(`pin path escapes checkpoint pin directory: ${pinPath}`) + } + if (!resolvedPinPath.endsWith(".json") || dirname(resolvedPinPath) !== resolve(pinDir)) { + throw new Error(`pin path is not a direct child of the checkpoint pin directory: ${pinPath}`) + } + const baseName = basename(resolvedPinPath, ".json") + await assertNoSymlink(checkpointRoot(dataDir), pinsRoot) + await assertNoSymlink(pinsRoot, pinDir) + if (!existsSync(resolvedPinPath)) { + throw new Error(`checkpoint pin not found (already released?): ${pinPath}`) + } + await assertNoSymlink(pinDir, resolvedPinPath) + await assertRealFile(resolvedPinPath, "checkpoint pin") + const parsed = JSON.parse(await readFile(resolvedPinPath, "utf8")) as unknown + const expectedKeys = new Set(["formatVersion", "pinId", "checkpointId", "purpose", "createdAt"]) + if ( + !isRecord(parsed) || + Object.keys(parsed).some((key) => !expectedKeys.has(key)) || + [...expectedKeys].some((key) => !(key in parsed)) || + parsed.formatVersion !== 1 || + validateId(requiredString(parsed, "pinId"), "pin") !== baseName || + validateId(requiredString(parsed, "checkpointId"), "checkpoint") !== validatedCheckpointId || + (expectedPurpose !== undefined && requiredString(parsed, "purpose") !== expectedPurpose) || + !PIN_PURPOSE.test(requiredString(parsed, "purpose")) || + !Number.isFinite(Date.parse(requiredString(parsed, "createdAt"))) + ) { + throw new Error(`checkpoint pin identity mismatch: ${pinPath}`) + } + return parsed as unknown as CheckpointPin +} + +/** + * Release a pin acquired by {@link acquireCheckpointPin}. Only the exact owned + * pin record at `pinPath` is removed. If the path is absent, belongs to a + * different checkpoint, or does not match the recorded pin identity, nothing is + * deleted and the call fails closed — over-retention is always preferred. + */ +export const releaseCheckpointPin = async ( + dataDir: string, + checkpointId: string, + pinPath: string, + expectedPurpose?: string, +): Promise => { + await assertCheckpointPinIdentity(dataDir, checkpointId, pinPath, expectedPurpose) + const resolvedPinPath = resolve(pinPath) + await durableRemove(resolvedPinPath) +} + +/** + * Run `fn` while holding the sibling maintenance lock so checkpoint creation, + * restore, reset, and archive operations cannot overlap. A live owner is busy; + * an unprovable owner fails closed; a provably dead owner is reconciled by + * exact operation identity (see {@link acquireMaintenance}). PID age alone never + * authorizes deletion. The lock is released when `fn` settles. + */ +export const withMaintenanceLock = async ( + dataDir: string, + operationId: string, + fn: () => A | Promise, +): Promise => { + const release = await acquireMaintenance(dataDir, validateId(operationId, "operation")) + try { + return await fn() + } finally { + await release() + } +} + +export const retireCheckpointIfEligible = async ( + dataDir: string, + checkpointId: string | null, + state: CheckpointState, + retirementId: string = randomUUID(), + faults: DurabilityFaults = {}, +): Promise => { + if (!checkpointId || state.current === checkpointId || state.previous === checkpointId) return null + const validatedCheckpointId = validateId(checkpointId, "checkpoint") + if (await hasPins(dataDir, validatedCheckpointId)) return null + const validatedRetirementId = validateId(retirementId, "retirement") + const retirementRoot = checkpointRetiringRoot(dataDir) + const retirement = join(retirementRoot, `retirement-${validatedRetirementId}`) + const retirementIntent = join(retirement, "intent.json") + const retirementComplete = join(retirement, "complete.json") + const retiredSnapshot = join(retirement, validatedCheckpointId) + if (existsSync(checkpointRetiringRoot(dataDir))) { + await assertNoSymlink(checkpointRoot(dataDir), retirementRoot) + await assertRealDirectory(retirementRoot, "checkpoint retirement root") + } + if (!existsSync(retirement)) { + await resolveCheckpoint(dataDir, validatedCheckpointId, state) + await ensurePrivateDirectory(retirement) + await durableJson(retirementIntent, { + formatVersion: 1, + retirementId: validatedRetirementId, + checkpointId: validatedCheckpointId, + stateRevision: state.revision, + }) + await faults.afterRetirementIntent?.(retirement) + } else { + await assertNoSymlink(retirementRoot, retirement) + await assertRealDirectory(retirement, "checkpoint retirement") + await assertNoSymlink(retirement, retirementIntent) + await assertRealFile(retirementIntent, "checkpoint retirement intent") + const parsed = JSON.parse(await readFile(retirementIntent, "utf8")) as unknown + if ( + !isRecord(parsed) || + parsed.formatVersion !== 1 || + validateId(requiredString(parsed, "retirementId"), "retirement") !== validatedRetirementId || + validateId(requiredString(parsed, "checkpointId"), "checkpoint") !== validatedCheckpointId || + validateId(requiredString(parsed, "stateRevision"), "state revision") !== state.revision + ) { + throw new Error(`checkpoint retirement identity mismatch: ${retirement}`) + } + } + if (existsSync(retirementComplete)) { + await assertNoSymlink(retirement, retirementComplete) + await assertRealFile(retirementComplete, "checkpoint retirement completion") + const complete = JSON.parse(await readFile(retirementComplete, "utf8")) as unknown + if ( + !isRecord(complete) || + complete.formatVersion !== 1 || + validateId(requiredString(complete, "retirementId"), "retirement") !== validatedRetirementId || + validateId(requiredString(complete, "checkpointId"), "checkpoint") !== validatedCheckpointId || + validateId(requiredString(complete, "stateRevision"), "state revision") !== state.revision + ) { + throw new Error(`checkpoint retirement completion identity mismatch: ${retirement}`) + } + if ( + existsSync(checkpointSnapshotDir(dataDir, validatedCheckpointId)) || + existsSync(retiredSnapshot) + ) { + throw new Error("completed checkpoint retirement still has snapshot data") + } + return retirement + } + const source = checkpointSnapshotDir(dataDir, validatedCheckpointId) + const sourceExists = existsSync(source) + const retiredExists = existsSync(retiredSnapshot) + if (sourceExists && retiredExists) { + throw new Error("checkpoint retirement has both source and retired snapshots") + } + if (sourceExists) { + await assertNoSymlink(checkpointSnapshotsRoot(dataDir), source) + await durableRename(source, retiredSnapshot, faults) + await faults.afterRetirementRename?.(retirement) + } + if (existsSync(retiredSnapshot)) { + await rm(retiredSnapshot, { recursive: true }) + await syncDirectory(retirement) + await faults.afterRetiredSnapshotRemoval?.(retirement) + } + await durableJson(retirementComplete, { + formatVersion: 1, + retirementId: validatedRetirementId, + checkpointId: validatedCheckpointId, + stateRevision: state.revision, + }) + await faults.afterRetirementComplete?.(retirement) + return retirement +} + +const removeCompletedRetirement = async ( + retirement: string | null, + faults: DurabilityFaults = {}, +): Promise => { + if (!retirement || !existsSync(retirement)) return + const intent = join(retirement, "intent.json") + const complete = join(retirement, "complete.json") + await assertNoSymlink(dirname(retirement), retirement) + await assertRealDirectory(retirement, "checkpoint retirement") + const entries = (await readdir(retirement)).sort() + if (entries.length !== 2 || entries[0] !== "complete.json" || entries[1] !== "intent.json") { + throw new Error(`completed checkpoint retirement contains unexpected state: ${retirement}`) + } + await assertNoSymlink(retirement, intent) + await assertNoSymlink(retirement, complete) + await assertRealFile(intent, "checkpoint retirement intent") + await assertRealFile(complete, "checkpoint retirement completion") + const intentValue = JSON.parse(await readFile(intent, "utf8")) as unknown + const completeValue = JSON.parse(await readFile(complete, "utf8")) as unknown + if ( + !isRecord(intentValue) || + !isRecord(completeValue) || + JSON.stringify(intentValue) !== JSON.stringify(completeValue) + ) { + throw new Error(`checkpoint retirement records do not match: ${retirement}`) + } + const cleanup = `${retirement}.cleanup-${randomUUID()}` + await durableRename(retirement, cleanup, faults) + await faults.afterRetirementCleanupRename?.(cleanup) + await rm(cleanup, { recursive: true }) + await syncDirectory(dirname(cleanup)) + await faults.afterRetirementCleanupRemoval?.(cleanup) +} + +export const createCheckpoint = ( + options: CheckpointOptions, +): Effect.Effect< + { + readonly checkpointId: string + readonly path: string + readonly state: CheckpointState + readonly manifest: CheckpointManifest + }, + CheckpointError +> => + Effect.tryPromise({ + try: async () => { + const operationId = randomUUID() + const checkpointId = randomUUID() + const release = await acquireMaintenance(options.dataDir, operationId) + try { + assertCheckpointRootSafe(options.dataDir) + await assertCheckpointInfrastructureSafe(options.dataDir) + assertNoLegacyLayout(options.dataDir) + await reconcileCheckpointOperations(options.dataDir, options.faults) + const oldState = await readStateFileOptional(options.dataDir) + if (!oldState && (await checkpointLikePaths(options.dataDir)).length > 0) { + throw new Error( + "checkpoint state is missing while checkpoint data exists; refusing to infer selection", + ) + } + if (oldState) { + await resolveCheckpoint(options.dataDir, oldState.current, oldState) + if (oldState.previous) { + await resolveCheckpoint(options.dataDir, oldState.previous, oldState) + } + } + for (const path of [ + checkpointRoot(options.dataDir), + checkpointSnapshotsRoot(options.dataDir), + checkpointOperationsRoot(options.dataDir), + checkpointPinsRoot(options.dataDir), + checkpointQuarantineRoot(options.dataDir), + checkpointRetiringRoot(options.dataDir), + ]) { + await ensurePrivateDirectory(path) + } + const startedAt = new Date().toISOString() + let operation: CheckpointOperation = { + formatVersion: 1, + operationId, + checkpointId, + baseRevision: oldState?.revision ?? null, + baseCurrent: oldState?.current ?? null, + basePrevious: oldState?.previous ?? null, + phase: "intent", + startedAt, + } + await writeOperation(options.dataDir, operation, options.faults) + const snapshot = checkpointSnapshotDir(options.dataDir, checkpointId) + await assertNoSymlink(checkpointSnapshotsRoot(options.dataDir), snapshot) + await mkdir(snapshot, { mode: 0o700 }) + try { + await postLocalQuery( + options.port, + `BACKUP DATABASE default TO Disk('default', '${snapshotBackupSqlPath(checkpointId)}')`, + ) + } catch (error) { + if (isMissingBackupConfigurationError(error)) { + throw new Error( + "checkpoints require the local server to be started with `--chdb-config-file` " + + "pointing at a ClickHouse backups config", + { cause: error }, + ) + } + throw error + } + await syncTree(snapshotBackupDir(options.dataDir, checkpointId)) + operation = { ...operation, phase: "backup-complete" } + await writeOperation(options.dataDir, operation, options.faults) + const provisionalManifest: CheckpointManifest = { + formatVersion: 1, + checkpointId, + operationId, + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + createdAt: startedAt, + sourceDataDir: resolve(options.dataDir), + backupRelativePath: snapshotBackupRelativePath(checkpointId), + backupBytes: await dirSize(snapshotBackupDir(options.dataDir, checkpointId)), + validation: { + validatedAt: startedAt, + traces: 0, + logs: 0, + metricsSum: 0, + metricsGauge: 0, + metricsHistogram: 0, + metricsExponentialHistogram: 0, + materializedViews: 0, + }, + } + const provisional: ResolvedCheckpoint = { + checkpointId, + snapshotDir: snapshot, + backupDir: snapshotBackupDir(options.dataDir, checkpointId), + backupSqlPath: snapshotBackupSqlPath(checkpointId), + manifest: provisionalManifest, + } + const validation = await withRestoredCheckpoint( + provisional, + { cleanup: "always" }, + (restored) => restored.validation, + ) + const manifest: CheckpointManifest = { ...provisionalManifest, validation } + await durableJson( + snapshotManifestPath(options.dataDir, checkpointId), + manifest, + options.faults, + ) + await syncDirectory(snapshot) + operation = { ...operation, phase: "manifest-complete" } + await writeOperation(options.dataDir, operation, options.faults) + const state: CheckpointState = { + formatVersion: 1, + revision: operationId, + current: checkpointId, + previous: oldState?.current ?? null, + committedAt: new Date().toISOString(), + } + await durableJson(checkpointStatePath(options.dataDir), state, options.faults) + operation = { ...operation, phase: "pointer-complete" } + await writeOperation(options.dataDir, operation, options.faults) + const retirement = await retireCheckpointIfEligible( + options.dataDir, + oldState?.previous ?? null, + state, + operationId, + options.faults, + ) + operation = { ...operation, phase: "retention-complete" } + await writeOperation(options.dataDir, operation, options.faults) + await removeCompletedRetirement(retirement, options.faults) + await preserveCompletedOperation( + options.dataDir, + operationDir(options.dataDir, operationId), + operationId, + options.faults, + ) + return { checkpointId, path: snapshot, state, manifest } + } finally { + await release() + } + }, + catch: (error) => + new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), + }) + +const parseResetTransaction = (value: unknown, expectedDataDir: string): ResetTransaction => { + if (!isRecord(value) || value.formatVersion !== RESET_TRANSACTION_FORMAT_VERSION) { + throw new Error("unsupported or malformed reset transaction") + } + const phase = requiredString(value, "phase") + if (!["intent", "live-cleared", "markers-cleared"].includes(phase)) { + throw new Error("invalid reset transaction phase") + } + const dataDir = requiredString(value, "dataDir") + if (!isAbsolute(dataDir) || resolve(dataDir) !== resolve(expectedDataDir)) { + throw new Error("reset transaction data directory does not match its configured owner") + } + if (!Array.isArray(value.targets)) throw new Error("invalid reset transaction targets") + const targets = value.targets.map((target) => { + if (typeof target !== "string" || !RESETTABLE_CHDB_ENTRIES.has(target)) { + throw new Error(`unsafe reset transaction target: ${String(target)}`) + } + return target + }) + if (new Set(targets).size !== targets.length || [...targets].sort().join("\0") !== targets.join("\0")) { + throw new Error("reset transaction targets must be unique and sorted") + } + return { + formatVersion: 1, + operationId: validateId(requiredString(value, "operationId"), "reset operation"), + dataDir: resolve(dataDir), + targets, + phase: phase as ResetTransaction["phase"], + createdAt: requiredIso(value, "createdAt"), + } +} + +const readResetTransaction = async (dataDir: string): Promise => { + const path = resetTransactionPath(dataDir) + try { + await assertRealFile(path, "reset transaction") + return parseResetTransaction(JSON.parse(await readFile(path, "utf8")), dataDir) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null + throw error + } +} + +const writeResetTransaction = async (dataDir: string, transaction: ResetTransaction): Promise => + durableJson(resetTransactionPath(dataDir), transaction) + +const reconcileResetTransactionUnlocked = async ( + dataDir: string, + faults: RestoreRecoveryFaults = {}, +): Promise => { + let transaction = await readResetTransaction(dataDir) + if (!transaction) return false + if (existsSync(restoreTransactionPath(dataDir))) { + throw new Error("reset and restore transactions both exist; refusing to choose one") + } + const live = resolve(dataDir) + if (existsSync(live)) await assertRealDirectory(live, "live data directory") + + if (transaction.phase === "intent") { + for (const target of transaction.targets) { + const path = join(live, target) + if (!existsSync(path)) continue + await assertRealDirectory(path, `reset target ${target}`) + await rm(path, { recursive: true }) + await syncDirectory(live) + await faults.afterResetEntryRemoval?.(target) + } + transaction = { ...transaction, phase: "live-cleared" } + await writeResetTransaction(dataDir, transaction) + await faults.afterResetLiveClearedRecord?.() + } + + if (transaction.phase === "live-cleared") { + for (const target of transaction.targets) { + if (existsSync(join(live, target))) { + throw new Error(`reset transaction target reappeared before marker removal: ${target}`) + } + } + const marker = storeMarkerPath(dataDir) + if (existsSync(marker)) await durableRemove(marker) + await faults.afterResetStoreMarkerRemoval?.() + const openMarker = storeOpenMarkerPath(dataDir) + if (existsSync(openMarker)) await durableRemove(openMarker) + await faults.afterResetOpenMarkerRemoval?.() + transaction = { ...transaction, phase: "markers-cleared" } + await writeResetTransaction(dataDir, transaction) + await faults.afterResetMarkersClearedRecord?.() + } + + for (const target of transaction.targets) { + if (existsSync(join(live, target))) { + throw new Error(`reset transaction target reappeared after deletion: ${target}`) + } + } + await durableRemove(resetTransactionPath(dataDir)) + await faults.afterResetTransactionRemoval?.() + return true +} + +const beginResetTransactionUnlocked = async ( + dataDir: string, + operationId: string, + faults: RestoreRecoveryFaults = {}, +): Promise => { + await assertCheckpointInfrastructureSafe(dataDir) + const live = resolve(dataDir) + const targets: string[] = [] + const unknown: string[] = [] + if (existsSync(live)) { + await assertRealDirectory(live, "live data directory") + const entries = await readdir(live, { withFileTypes: true }) + for (const entry of entries) { + if (entry.name === "backups") continue + if (!RESETTABLE_CHDB_ENTRIES.has(entry.name)) { + unknown.push(join(live, entry.name)) + continue + } + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new Error(`reset target is not a real chDB directory: ${join(live, entry.name)}`) + } + targets.push(entry.name) + } + } + if (unknown.length > 0) { + throw new Error( + `unrecognized data-directory entries were preserved; refusing reset: ${unknown.sort().join(", ")}`, + ) + } + const transaction: ResetTransaction = { + formatVersion: 1, + operationId: validateId(operationId, "reset operation"), + dataDir: live, + targets: targets.sort(), + phase: "intent", + createdAt: new Date().toISOString(), + } + await writeResetTransaction(dataDir, transaction) + await faults.afterResetIntent?.() + await reconcileResetTransactionUnlocked(dataDir, faults) +} + +const parseRestoreTransaction = (value: unknown): RestoreTransaction => { + if (!isRecord(value) || value.formatVersion !== RESTORE_TRANSACTION_FORMAT_VERSION) { + throw new Error("unsupported or malformed restore transaction") + } + const phase = requiredString(value, "phase") + if (!["intent", "restore-ready", "old-quarantined", "new-live", "markers-committed"].includes(phase)) { + throw new Error("invalid restore transaction phase") + } + return { + formatVersion: 1, + operationId: validateId(requiredString(value, "operationId"), "restore operation"), + checkpointId: validateId(requiredString(value, "checkpointId"), "checkpoint"), + quarantineId: validateId(requiredString(value, "quarantineId"), "quarantine"), + phase: phase as RestoreTransaction["phase"], + createdAt: requiredIso(value, "createdAt"), + validation: value.validation === null ? null : parseValidation(value.validation), + } +} + +const readRestoreTransaction = async (dataDir: string): Promise => { + try { + await assertRealFile(restoreTransactionPath(dataDir), "restore transaction") + return parseRestoreTransaction(JSON.parse(await readFile(restoreTransactionPath(dataDir), "utf8"))) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null + throw error + } +} + +const writeRestoreTransaction = async (dataDir: string, transaction: RestoreTransaction) => + durableJson(restoreTransactionPath(dataDir), transaction) + +const restoreReadyPath = (dataDir: string, operationId: string): string => + join(restoreDataPath(dataDir, operationId), ".maple-restore-ready.json") + +const readyIdentityMatches = (dataDir: string, transaction: RestoreTransaction): boolean => { + const candidates = [ + restoreReadyPath(dataDir, transaction.operationId), + join(resolve(dataDir), ".maple-restore-ready.json"), + ] + for (const path of candidates) { + if (!existsSync(path)) continue + try { + const info = lstatSync(path) + if (info.isSymbolicLink() || !info.isFile()) return false + const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown + if ( + isRecord(parsed) && + parsed.formatVersion === 1 && + parsed.operationId === transaction.operationId && + parsed.checkpointId === transaction.checkpointId + ) { + return true + } + } catch { + return false + } + } + return false +} + +const finalizeRestoreMarkers = async ( + dataDir: string, + transaction: RestoreTransaction, + faults: RestoreRecoveryFaults = {}, +): Promise => { + if (!readyIdentityMatches(dataDir, transaction)) { + throw new Error("restored live store identity is missing or does not match the transaction") + } + await writeStoreMarkerDurable(dataDir, MAPLE_VERSION, new Date().toISOString(), SCHEMA_FINGERPRINT) + await faults.afterStoreMarkerWrite?.() + await markStoreClosedDurable(dataDir) + await faults.afterOpenMarkerRemoval?.() + const committed: RestoreTransaction = { ...transaction, phase: "markers-committed" } + await writeRestoreTransaction(dataDir, committed) + await faults.afterMarkersCommittedRecord?.() + return committed +} + +const completeRestoreTransaction = async ( + dataDir: string, + transaction: RestoreTransaction, + faults: RestoreRecoveryFaults = {}, +): Promise => { + const readyPath = join(resolve(dataDir), ".maple-restore-ready.json") + if (existsSync(readyPath)) await durableRemove(readyPath) + await faults.afterReadyMarkerRemoval?.() + const root = restoreRootPath(dataDir, transaction.operationId) + if (existsSync(root)) { + await rm(root, { recursive: true }) + await syncDirectory(dirname(root)) + } + await faults.afterRestoreRootRemoval?.() + await durableRemove(restoreTransactionPath(dataDir)) +} + +const reconcileRestoreTransactionUnlocked = async ( + dataDir: string, + faults: RestoreRecoveryFaults = {}, +): Promise => { + if (existsSync(resolve(dataDir))) { + await assertRealDirectory(resolve(dataDir), "live data directory") + } + let transaction = await readRestoreTransaction(dataDir) + if (!transaction) { + const parent = dirname(resolve(dataDir)) + const prefix = `${basename(resolve(dataDir))}.restore-` + let names: string[] + try { + names = await readdir(parent) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + const debris = names + .filter( + (name) => + (name.startsWith(prefix) && !name.includes(".quarantine-")) || + name === `${basename(resolve(dataDir))}.restore-transaction.json`, + ) + .map((name) => join(parent, name)) + if (debris.length > 0) { + throw new Error( + `restore-like paths exist without a valid transaction; refusing to infer ownership: ${debris.join(", ")}`, + ) + } + return + } + const live = resolve(dataDir) + const restoreRoot = restoreRootPath(dataDir, transaction.operationId) + const restoreData = restoreDataPath(dataDir, transaction.operationId) + const quarantine = restoreQuarantinePath(dataDir, transaction.operationId, transaction.quarantineId) + const liveExists = existsSync(live) + const restoreExists = existsSync(restoreData) + const quarantineExists = existsSync(quarantine) + if (liveExists) await assertRealDirectory(live, "live data directory") + if (existsSync(restoreRoot)) await assertRealDirectory(restoreRoot, "restore root") + if (restoreExists) await assertRealDirectory(restoreData, "restored data directory") + if (quarantineExists) await assertRealDirectory(quarantine, "restore quarantine") + + if (transaction.phase === "intent") { + if (!liveExists || quarantineExists) { + throw new Error( + `ambiguous restore intent topology; live=${liveExists} restore=${restoreExists} quarantine=${quarantineExists}`, + ) + } + if (existsSync(restoreRoot)) { + await durableRename(restoreRoot, `${restoreRoot}.quarantine-${randomUUID()}`) + } + await durableRename( + restoreTransactionPath(dataDir), + `${restoreTransactionPath(dataDir)}.quarantine-${randomUUID()}`, + ) + return + } + + if (transaction.phase === "restore-ready" && liveExists && restoreExists && !quarantineExists) { + if (!readyIdentityMatches(dataDir, transaction)) { + throw new Error("restore-ready identity is missing or mismatched") + } + await durableRename(live, quarantine) + await faults.afterLiveQuarantineRename?.() + transaction = { ...transaction, phase: "old-quarantined" } + await writeRestoreTransaction(dataDir, transaction) + await faults.afterOldQuarantinedRecord?.() + } + + if ( + (transaction.phase === "restore-ready" || transaction.phase === "old-quarantined") && + !existsSync(live) && + existsSync(restoreData) && + existsSync(quarantine) + ) { + await durableRename(restoreData, live) + await faults.afterRestoredLiveRename?.() + transaction = { ...transaction, phase: "new-live" } + await writeRestoreTransaction(dataDir, transaction) + await faults.afterNewLiveRecord?.() + } + + if ( + (transaction.phase === "old-quarantined" || transaction.phase === "new-live") && + existsSync(live) && + !existsSync(restoreData) && + existsSync(quarantine) + ) { + transaction = await finalizeRestoreMarkers( + dataDir, + { + ...transaction, + phase: "new-live", + }, + faults, + ) + } + + if (transaction.phase === "markers-committed" && existsSync(live) && existsSync(quarantine)) { + await completeRestoreTransaction(dataDir, transaction, faults) + return + } + + throw new Error( + `restore transaction could not be reconciled safely; phase=${transaction.phase} live=${existsSync(live)} restore=${existsSync(restoreData)} quarantine=${existsSync(quarantine)}`, + ) +} + +export const reconcileCheckpointRecovery = ( + dataDir: string, + faults: RestoreRecoveryFaults = {}, +): Effect.Effect => + Effect.tryPromise({ + try: async () => { + const operationId = randomUUID() + const release = await acquireMaintenance(dataDir, operationId) + try { + const resetReconciled = await reconcileResetTransactionUnlocked(dataDir, faults) + if (!resetReconciled) await reconcileRestoreTransactionUnlocked(dataDir, faults) + } finally { + await release() + } + }, + catch: (error) => + new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), + }) + +/** + * Explicitly remove the live chDB store while preserving the checkpoint + * registry below `/backups`. The maintenance lock serializes this + * destructive operation with checkpoint, restore, and archive work. + */ +export const resetLiveStorePreservingCheckpoints = ( + dataDir: string, + faults: RestoreRecoveryFaults = {}, +): Effect.Effect => + Effect.tryPromise({ + try: async () => { + const operationId = randomUUID() + const release = await acquireMaintenance(dataDir, operationId) + try { + const resetReconciled = await reconcileResetTransactionUnlocked(dataDir, faults) + if (!resetReconciled) { + await reconcileRestoreTransactionUnlocked(dataDir) + await beginResetTransactionUnlocked(dataDir, operationId, faults) + } + } finally { + await release() + } + }, + catch: (error) => + new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), + }) + +export const restoreCheckpoint = ( + dataDir: string, + selector: "current" | "previous" | string = "current", +): Effect.Effect< + { + readonly checkpointId: string + readonly quarantinePath: string + readonly validation: CheckpointValidation + }, + CheckpointError +> => + Effect.tryPromise({ + try: async () => { + const operationId = randomUUID() + const quarantineId = randomUUID() + const release = await acquireMaintenance(dataDir, operationId) + try { + await reconcileResetTransactionUnlocked(dataDir) + await reconcileRestoreTransactionUnlocked(dataDir) + const resolvedCheckpoint = await resolveCheckpoint(dataDir, selector) + const restoreRoot = restoreRootPath(dataDir, operationId) + const restoreData = restoreDataPath(dataDir, operationId) + const quarantinePath = restoreQuarantinePath(dataDir, operationId, quarantineId) + if (existsSync(restoreRoot) || existsSync(quarantinePath)) { + throw new Error("restore or quarantine path already exists") + } + let transaction: RestoreTransaction = { + formatVersion: 1, + operationId, + checkpointId: resolvedCheckpoint.checkpointId, + quarantineId, + phase: "intent", + createdAt: new Date().toISOString(), + validation: null, + } + await writeRestoreTransaction(dataDir, transaction) + await mkdir(restoreRoot, { mode: 0o700 }) + const restored = await restoreResolvedInto(resolvedCheckpoint, restoreData) + const restoringProcessValidation = restored.validation + restored.db.close() + // Do not publish a representation that only the restoring connection can + // read. Re-exec Maple after close and require the persisted metadata to + // load and produce the same counts in a wholly fresh chDB process. + const validation = validateRestoredDatabaseInFreshProcess( + restoreData, + restoringProcessValidation, + ) + await cp(checkpointRoot(dataDir), join(restoreData, "backups"), { + recursive: true, + force: false, + errorOnExist: true, + }) + await syncTree(join(restoreData, "backups")) + await durableJson(restoreReadyPath(dataDir, operationId), { + formatVersion: 1, + operationId, + checkpointId: resolvedCheckpoint.checkpointId, + }) + await syncTree(restoreData, { allowSymlinks: true }) + transaction = { ...transaction, phase: "restore-ready", validation } + await writeRestoreTransaction(dataDir, transaction) + await reconcileRestoreTransactionUnlocked(dataDir) + return { + checkpointId: resolvedCheckpoint.checkpointId, + quarantinePath, + validation, + } + } finally { + await release() + } + }, + catch: (error) => + new CheckpointError({ message: error instanceof Error ? error.message : String(error) }), + }) + +// Test helper: assert generated operation IDs remain unique and valid without +// exposing an override in production command paths. +export const newCheckpointId = (): string => validateId(randomUUID(), "checkpoint") + +// Refuse pre-existing symlink roots even before an operation allocates paths. +export const assertCheckpointRootSafe = (dataDir: string): void => { + const root = checkpointRoot(dataDir) + if (existsSync(root) && lstatSync(root).isSymbolicLink()) { + throw new Error(`refusing symlink checkpoint root: ${root}`) + } + if (basename(root) !== "backups") throw new Error("invalid checkpoint root") +} diff --git a/apps/cli/src/server/durable-files.ts b/apps/cli/src/server/durable-files.ts new file mode 100644 index 000000000..685fd67de --- /dev/null +++ b/apps/cli/src/server/durable-files.ts @@ -0,0 +1,136 @@ +import { randomUUID } from "node:crypto" +import { constants } from "node:fs" +import { chmod, lstat, mkdir, open, readdir, rename, rm } from "node:fs/promises" +import { dirname, join } from "node:path" + +export interface DurabilityFaults { + readonly beforeFileSync?: (path: string) => void | Promise + readonly beforeRename?: (from: string, to: string) => void | Promise + readonly beforeDirectorySync?: (path: string) => void | Promise + readonly beforeRemove?: (path: string) => void | Promise + readonly afterRetirementIntent?: (path: string) => void | Promise + readonly afterRetirementRename?: (path: string) => void | Promise + readonly afterRetiredSnapshotRemoval?: (path: string) => void | Promise + readonly afterRetirementComplete?: (path: string) => void | Promise + readonly afterRetirementCleanupRename?: (path: string) => void | Promise + readonly afterRetirementCleanupRemoval?: (path: string) => void | Promise + readonly afterCompletedOperationPreserved?: (path: string) => void | Promise +} + +// APFS and the target Linux filesystems support directory fsync. Keep the +// fallback deliberately narrow for filesystems that explicitly report that +// the operation is unsupported; descriptor/type errors are programming bugs. +const unsupportedDirectorySyncCodes = new Set(["EINVAL", "ENOTSUP", "EOPNOTSUPP"]) + +export const isUnsupportedDirectorySyncError = (error: unknown): boolean => + typeof error === "object" && + error !== null && + "code" in error && + unsupportedDirectorySyncCodes.has(String((error as NodeJS.ErrnoException).code)) + +export const ensurePrivateDirectory = async (path: string): Promise => { + const before = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null + throw error + }) + if (before?.isSymbolicLink()) throw new Error(`refusing symlink directory: ${path}`) + if (before && !before.isDirectory()) throw new Error(`refusing non-directory path: ${path}`) + await mkdir(path, { recursive: true, mode: 0o700 }) + const after = await lstat(path) + if (after.isSymbolicLink() || !after.isDirectory()) { + throw new Error(`private directory is not a real directory: ${path}`) + } + await chmod(path, 0o700) +} + +export const syncDirectory = async (path: string, faults: DurabilityFaults = {}): Promise => { + await faults.beforeDirectorySync?.(path) + let handle + try { + handle = await open(path, constants.O_RDONLY) + await handle.sync() + } catch (error) { + if (!isUnsupportedDirectorySyncError(error)) throw error + } finally { + await handle?.close() + } +} + +export const durableWrite = async ( + path: string, + bytes: string | Uint8Array, + faults: DurabilityFaults = {}, +): Promise => { + const parent = dirname(path) + await ensurePrivateDirectory(parent) + const temporary = join(parent, `.${randomUUID()}.tmp`) + const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600) + try { + await handle.writeFile(bytes) + await faults.beforeFileSync?.(path) + await handle.sync() + } catch (error) { + await handle.close().catch(() => undefined) + await rm(temporary, { force: true }).catch(() => undefined) + throw error + } + await handle.close() + try { + await faults.beforeRename?.(temporary, path) + await rename(temporary, path) + await syncDirectory(parent, faults) + } catch (error) { + await rm(temporary, { force: true }).catch(() => undefined) + throw error + } +} + +export const durableJson = async ( + path: string, + value: unknown, + faults: DurabilityFaults = {}, +): Promise => durableWrite(path, `${JSON.stringify(value, null, 2)}\n`, faults) + +export const durableRemove = async (path: string, faults: DurabilityFaults = {}): Promise => { + await faults.beforeRemove?.(path) + await rm(path, { force: true }) + await syncDirectory(dirname(path), faults) +} + +export const durableRename = async ( + from: string, + to: string, + faults: DurabilityFaults = {}, +): Promise => { + await faults.beforeRename?.(from, to) + await rename(from, to) + await syncDirectory(dirname(from), faults) + if (dirname(to) !== dirname(from)) await syncDirectory(dirname(to), faults) +} + +export const syncTree = async ( + path: string, + options: { readonly allowSymlinks?: boolean } = {}, +): Promise => { + const entries = await readdir(path, { withFileTypes: true }) + for (const entry of entries) { + const child = join(path, entry.name) + if (entry.isDirectory()) { + await syncTree(child, options) + } else if (entry.isFile()) { + const handle = await open(child, constants.O_RDONLY) + try { + await handle.sync() + } finally { + await handle.close() + } + } else if (entry.isSymbolicLink() && options.allowSymlinks) { + // A closed chDB store contains engine-managed symlinks. Do not + // follow them; syncing this directory below durably records the link. + continue + } else { + throw new Error(`refusing to sync non-file checkpoint entry at ${child}`) + } + } + await syncDirectory(path) +} diff --git a/apps/cli/src/server/serve.ts b/apps/cli/src/server/serve.ts index 05ee77908..5bef6986a 100644 --- a/apps/cli/src/server/serve.ts +++ b/apps/cli/src/server/serve.ts @@ -26,6 +26,7 @@ export interface AssetResolver { export interface ServerOptions { readonly port: number readonly dataDir: string + readonly configFile?: string /** Serves the bundled SPA; omit to disable the UI (API-only). */ readonly assets?: AssetResolver } @@ -335,7 +336,11 @@ export const startServer = ( options: ServerOptions, ): Effect.Effect<{ readonly port: number }, ChdbError, Scope.Scope> => Effect.gen(function* () { - const db = yield* acquireChdb({ dataDir: options.dataDir, schemaSql }) + const db = yield* acquireChdb({ + dataDir: options.dataDir, + schemaSql, + configFile: options.configFile, + }) // A dedicated runtime carrying the OTel tracer for per-request spans: the // Bun.serve handler runs outside Effect, so each request's span effect is // run through this runtime. Disposed on scope close, which flushes any diff --git a/apps/cli/src/server/store-version.ts b/apps/cli/src/server/store-version.ts index 6d6198d44..1b97ecb95 100644 --- a/apps/cli/src/server/store-version.ts +++ b/apps/cli/src/server/store-version.ts @@ -14,6 +14,7 @@ import { createHash } from "node:crypto" import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs" import { dirname, join } from "node:path" import { CHDB_VERSION } from "../version" +import { durableRemove, durableWrite } from "./durable-files" export interface StoreMarker { /** libchdb version that created the store (e.g. "v26.1.0"); "dev" in dev builds. */ @@ -61,6 +62,12 @@ export const markStoreClosed = (dataDir: string): void => { } } +/** Durably clear the clean-shutdown sentinel for a restore transaction. */ +export const markStoreClosedDurable = async (dataDir: string): Promise => { + const path = storeOpenMarkerPath(dataDir) + if (existsSync(path)) await durableRemove(path) +} + /** True when the store holds data AND was not cleanly closed (the sentinel * survives). Gated on `storeHasData`: a marker over an empty store means a * fresh open that never persisted anything — safe to reopen. */ @@ -89,6 +96,14 @@ export const readMarker = (dataDir: string): StoreMarker | null => { export const storeMarkerJson = (maple: string, now: string, schema: string): string => `${JSON.stringify({ chdb: CHDB_VERSION, maple, createdAt: now, schema } satisfies StoreMarker, null, 2)}\n` +/** Atomically and durably stamp the store version after a live restore swap. */ +export const writeStoreMarkerDurable = async ( + dataDir: string, + maple: string, + now: string, + schema: string, +): Promise => durableWrite(storeMarkerPath(dataDir), storeMarkerJson(maple, now, schema)) + /** * Stable fingerprint of the bundled schema DDL. Comments and whitespace are * stripped first so cosmetic edits (a reworded comment, reindentation) don't diff --git a/apps/cli/test/archive-adversarial-matrix.md b/apps/cli/test/archive-adversarial-matrix.md new file mode 100644 index 000000000..c21b12346 --- /dev/null +++ b/apps/cli/test/archive-adversarial-matrix.md @@ -0,0 +1,313 @@ +# Archive export — adversarial validation matrix + +This is a **permanent gate**, not a one-time review. It exists because four +consecutive Gate 2 rounds failed for the same root cause: the implementation +and its tests shared one mental model, so the tests confirmed the author's +intent while the independent review attacked the author's assumptions. Each +repair taught a lesson that then disappeared into conversation. This matrix +captures the lessons so the next change to archive export must answer them +explicitly, in code, before it can be considered done. + +**Working rule:** before any change to archive export can be called complete, +answer this question in writing for the diff: + +> How could an _incorrect_ archive preserve every metric I currently check? + +Every cell below is a concrete instance of that question, the transformation +that realizes it, the named probe that must catch it, the independent oracle +that confirms the verdict, and the required result. A probe must be hermetic +(owned `mkdtemp`, cleans only its own state, no fixed `/tmp` paths, runs from a +fresh clone with otherwise-empty `/tmp`) and use consistent exit semantics: +**nonzero when corruption is accepted, zero when corruption is correctly +rejected.** + +The red/green columns record the state at the round this matrix was introduced +(Gate 2 round 5). Red = the current code fails to detect the corruption; the +repair must turn it green. + +## Invariants and counterexamples + +### 1. Exact row identity + +**Invariant:** a shard contains exactly the source rows for its sealed slice, +each row's values bound to its columns and its row identity — not merely the +same aggregate of values. + +| Counterexample transformation | Named probe | Independent oracle | Required | +| --------------------------------------------------------------------------------------------------------------------------- | ------------------------------------- | ------------------------------------------------------ | -------------------------------------------- | +| Swap two same-typed Map columns within rows (e.g. `SpanAttributes`↔`ResourceAttributes`), preserving count and time extrema | `archive-probe-digest-column-swap.ts` | canonical full source rows vs DuckDB-read Parquet rows | rejected (red at round 4 → green at round 5) | +| Reassociate values between two rows (move row A's map to row B and vice versa), preserving count and time extrema | `archive-probe-digest-row-swap.ts` | per-row canonical comparison | rejected (red → green) | +| Duplicate one row and drop another of equal count, preserving count and time extrema | `archive-probe-digest-dup-drop.ts` | per-row multiset equality | rejected (red → green) | + +The digest construction must bind (a) column index/name + position, (b) an +EXPLICIT NULL flag (a sentinel alone is insufficient — a real Nullable(String) +value can equal the sentinel), (c) normalized value, and aggregate rows as an +order-independent multiset that preserves duplicates. A commutative sum of +independent per-column hashes fails all three transformations. + +### 2. Stable physical sharding + +**Invariant:** every archived row is archived exactly once across all shards, +and no row outside the sealed slice is archived, for any physical layout. + +| Counterexample transformation | Named probe | Independent oracle | Required | +| ------------------------------------------------------------ | ---------------------------------- | --------------------------------------- | ------------- | +| Offset holes within a part (matching offsets non-contiguous) | `archive-probe-mixed-hour.ts` | source ID set vs union of shard ID sets | exact (green) | +| Multiple parts for one hour, out-of-order insertion | `archive-probe-multipart.ts` | same | exact (green) | +| A background merge injected between shard pages | `archive-probe-merge-injection.ts` | same; merges must be blocked | exact (green) | + +Paging must derive counts and cut points from the **actual** matching rows, not +from an assumed contiguous offset range. `_part_offset` repeats across parts, so +any predicate must bind `_part` together with the offset range. + +### 3. Complex-value fidelity + +**Invariant:** Map, Array, nested-Map/Array, NULL, and high-precision +timestamp values round-trip exactly through Parquet. + +| Counterexample transformation | Named probe | Independent oracle | Required | +| --------------------------------------------------------------- | ------------------------------------------------------------------------- | --------------------------------------------------- | ---------------------------- | +| NULL in any column collapses the digest to empty | `archive-probe-null-digest.ts` | digest string non-empty | digest non-empty (green) | +| NULL-bearing rows lose value sensitivity in NON-NULL columns | `archive-probe-null-value-sensitivity.ts` | two NULL-Min datasets, different non-null values | digests differ (red → green) | +| NULL collides with a real Nullable(String) sentinel value | `archive-probe-null-flag-binding.ts` | NULL vs '\x00NULL' string | digests differ (red → green) | +| Bare `DateTime` / `DateTime64(N)` render diverge source↔Parquet | covered by `archive-probe-duckdb-oracle.ts` (epoch_us on raw TIMESTAMPTZ) | numeric epoch normalization in the digest | match (green) | +| Schema substitution `Array(UInt64)`↔`Array(String)` | `archive-probe-schema-substitution.ts` | recursive type compare after measured normalization | rejected (green) | +| A non-null map/value changed with identical count/time | `archive-probe-complex-alter.ts` | export twice, compare digests | digests differ (green) | +| Read-back fidelity via an INDEPENDENT reader (not the digest) | `archive-probe-duckdb-oracle.ts` | DuckDB epoch_us / NULL count / array contents | match source (green) | + +No chDB Parquet type/value behavior may be assumed; it is measured (see +`reports/gate2-round4-probes.md` and the round-5 probe report) before any +comparison logic is written. + +### 4. Byte bounds + +**Invariant:** every shard satisfies both `maxShardRows` and `maxShardBytes` +(uncompressed). The planner refines by measurement, not by sampling. + +| Counterexample transformation | Named probe | Independent oracle | Required | +| --------------------------------------------------------------------------- | ------------------------------------- | --------------------------------------------------- | --------------------------------- | +| Narrow prefix + wide incompressible tail (sample-based plan underestimates) | `archive-probe-byte-heterogeneous.ts` | actual `total_uncompressed_size` per shard ≤ bound | every shard ≤ bound (red → green) | +| Uniform wide rows | `archive-probe-byte-uniform.ts` | same | ≤ bound (green) | +| One genuinely oversized row that cannot fit alone | `archive-probe-byte-single-row.ts` | distinct `single row exceeds maxShardBytes` failure | distinct failure (red → green) | + +Sampling may choose an initial range size but cannot determine correctness. +The only impassable case is a single matching row whose uncompressed size +exceeds `maxShardBytes`. + +### 5. UTC time bounds + +**Invariant:** shard time evidence and range binding are independent of the host +timezone. + +| Counterexample transformation | Named probe | Independent oracle | Required | +| -------------------------------------------------------------- | -------------------------------------------- | ---------------------------------------- | ------------------------------ | +| Valid 23:30 UTC shard bound parsed under `TZ=America/New_York` | `archive-probe-timezone-bound.ts` | `BigInt` epoch-nanosecond comparison | accepted (red → green) | +| Out-of-range bound (2027 shard for a 2026 range) | unit test in `archive-export-round5.test.ts` | integer range comparison | rejected (green) | +| Timezone-less string `"2026-06-29 23:30:00..."` serialized | covered by the timezone-bound probe | canonical UTC epoch-nano decimal strings | never serialized (red → green) | + +Shard bounds are persisted as decimal-string epoch nanoseconds and parsed with +`BigInt`, never as timezone-dependent ISO via `Date.parse`. + +### 6. Cleanup at every boundary + +**Invariant:** a failure at any point in the export lifecycle leaves merges +restarted and only proven-owned temporary output removed. + +| Counterexample transformation | Named probe | Independent oracle | Required | +| --------------------------------------------------------------------- | --------------------------------------- | ----------------------------------------------- | -------------------------- | +| Setup failure immediately after `STOP MERGES` (before the main `try`) | `archive-probe-merge-freeze-leak.ts` | `OPTIMIZE` succeeds after failure (no code 236) | merges restarted (green) | +| Mid-export shard failure | covered by the heterogeneous-byte probe | merges restarted; only owned candidate removed | restarted, no leak (green) | + +`try/finally` begins immediately after a successful `STOP MERGES`. + +### 7. Malformed-state fail-closed + +**Invariant:** an unknown manifest format version or a malformed field fails +closed while preserving the offending files. + +| Counterexample transformation | Named probe | Independent oracle | Required | +| ------------------------------------------------------ | ----------- | ----------------------------- | --------------------------- | +| Manifest `formatVersion` from a future/unknown version | unit test | parse throws, files untouched | rejected, preserved (green) | +| Missing/empty `complexDigest`, non-numeric digest | unit test | parse throws | rejected (green) | +| Shard time outside sealed range | unit test | `BigInt` range comparison | rejected (green) | + +A manifest format-version bump must reject older formats explicitly (the +on-disk files are preserved for inspection). + +### 8. Recovery reproducibility + +**Invariant:** the recovery bundle clones to the exact reviewed commit with +complete ancestry, verifiable with the original repository and `/tmp` +unavailable. + +| Counterexample transformation | Named probe | Independent oracle | Required | +| ----------------------------------------------------------- | ------------------------------------------------------------------------ | ----------------------------------------------- | ------------------------------- | +| Clone bundle from isolated dir, original repo + `/tmp` gone | `git clone -b codex/local-telemetry-archives-impl ` + `git fsck` | cloned HEAD == final round-5 commit; clean fsck | exact HEAD, clean (red → green) | +| Run the committed probe runner from that clone | the native runner | all probes green | green (red → green) | + +The bundle must contain the exact branch and complete ancestry, never rewritten +alternate history. + +### 9. Crash recovery (Gate 3a) — authoritative SIGKILL oracle + +**Invariant:** a process kill at ANY point in the archive generation lifecycle +leaves state that the next operation reconciles to its exact intended outcome — +no orphaned pin, no orphaned scratch, no half-published generation, no duplicate +catalog entry, no clobbered pointer, and the live store unchanged. The operation +has no cleanup `finally`: hook-throw and SIGKILL therefore leave the same +journal-described durable state, and reconciliation is the only cleanup +authority. + +**Authoritative oracle:** the native harness +`native-archive-crash-recovery-probe.sh` injects a real SIGKILL at each boundary +via a committed child worker paused at a fault seam, then reconciles WITHOUT a +fresh export and verifies exact convergence + idempotence. Hook-throw results are +secondary deterministic coverage; they cannot substitute for the real process +kills below. + +| Kill-point (SIGKILL at…) | Published? | Required oracle | +| ---------------------------------------------------------- | ---------- | ------------------------------------------------------------ | +| before initial intent durability | no | no final/pointer/catalog/quarantine; no debris | +| intent durable, before pin acquisition | no | exact journal-owned pin absent or released; clean abort | +| scratch allocated, immediately before synchronous restore | no | scratch removed; no final generation | +| restore complete | no | scratch removed; no final generation | +| building created | no | exact building quarantine retained | +| after first individually fsynced shard of a 3-shard export | no | exactly owned incomplete building quarantined; no final | +| all shard validation complete | no | exact complete shard set quarantined; no final | +| before in-building manifest write | no | quarantine retained; no final | +| in-building manifest + journal hash durable, before rename | no | manifest-bearing building quarantined; no final | +| complete manifest-bearing generation renamed | yes | strict manifest/hash/shard verification, then CAS pointer | +| before pointer update | yes | same as above | +| pointer durable, before catalog update | yes | catalog exactly one authoritative entry | +| catalog durable, before pin release | yes | exact-purpose pin released | +| pin removed, before journal phase advance | yes | absence accepted because catalog-complete authorizes release | +| pin-released phase durable | yes | scratch cleanup completes | +| before scratch removal | yes | exact scratch removed | +| complete phase, before operation-journal archival | yes | active journal archived exactly once | +| fresh `archive create` after crash | yes | dead-owner lock reconciled first, then new generation sealed | + +The journal is written BEFORE pin acquisition and records the deterministic +resolved archive/data/scratch roots, pinId/purpose, scratchSubdir, signal/range, +and generationId up front — closing the orphan-pin window +where a SIGKILL between pin creation and journal write would be unreconcilable. +The complete manifest is written and synced inside `building/`, its exact +SHA-256 is recorded in the journal, and only then is the whole directory renamed +to its final location. Reconciliation strictly binds that manifest and every +shard before pointer or catalog mutation. +The pointer flip is CAS-guarded (must equal the recorded base or already select +the intended generation) so post-crash concurrent activity is never clobbered. +Pin absence is success ONLY at a phase where release was already authorized. + +### 9a. Reconciliation labels are claims, not evidence + +| Hostile topology | Required oracle | +| ----------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | +| free-space preflight fails | no new active intent exists | +| configured scratch root has a symlinked ancestor | fail closed; outside sentinel unchanged; active journal retained | +| partial restored store contains an internal table symlink | unlink owned tree without following link; outside target survives; evidence kept | +| pointer-complete/catalog-complete label but pointer or catalog is missing | repair from recorded CAS topology and authoritative manifests | +| pointer selects neither recorded base nor intended generation | fail closed without clobbering pointer; active journal retained | +| catalog-complete label but catalog is tampered, duplicated, or truncated | rebuild exact canonical catalog from all authoritative manifests | +| complete label with manifest/shard/pointer/catalog/pin/scratch/building drift | fail closed without repair or journal retirement | + +The `complete` phase is uniquely non-repairing: before its journal can move out +of `operations/active/`, reconciliation must prove the final manifest hash and +identity, every shard hash/size, intended pointer, exact canonical catalog, +owned-pin absence, owned-scratch absence, and building absence. Any mismatch +retains the active journal as the only authority over uncertain state. + +**Restore limitation, stated precisely:** chDB exposes RESTORE as one synchronous +FFI call and provides no callback from inside that call. The authoritative +matrix therefore covers the durable boundary immediately before RESTORE and the +boundary after RESTORE returns; it does not mislabel the pre-call pause as +"during restore." A real OS-level arbitrary-time kill inside the FFI call +remains outside this deterministic seam, while its possible durable topology +(journal at scratch-allocated with partial scratch) is the same topology the +pre-restore recovery case exercises. + +**Working rule for this section:** _how could a crash at this kill-point preserve +every metric I currently check, or appear recovered while leaving corrupt state?_ +The answer the harness enforces: reconcile-without-export → verify exact +convergence → reconcile AGAIN (idempotence). The recovery code returning success +is NOT the oracle; the on-disk state after a kill is. + +### 10. Garbage collection (Gate 3b) — conservative journaled deletion + +**Invariant:** GC deletes only superseded generations it can PROVE are not the +active pointer target — never the active generation, never quarantined/malformed/ +symlinked/ambiguous state, never a range with NO active pointer (uncertain → +over-retained). It is the only archive operation that deletes published +generations, so it journals a frozen deletion set (computed under the maintenance +lock) and collects via a **tombstone rename**, never an in-place recursive delete, +so a SIGKILL mid-collection leaves only whole, owned state that reconcile can +prove ownership of. A crashed GC shares the single `operations/active/` slot with +create and is reconciled by the same entry point (dispatched on `kind: "gc"`); a +stranded GC must reconcile or it blocks all future archive work. + +**State machine (the core repair):** GC uses a nonterminal `gc-collecting` phase +(`0 ≤ completedTargets ≤ targets.length` — the full cursor is the legitimate +post-final-deletion state before catalog repair). Progress is persisted as +`gc-collecting` per target, NEVER `complete`. `complete` is written only after +every target is absent + every affected catalog passes `assertCatalogExact`, and +a `complete` journal is verified before archival. The parser rejects +kind-incompatible phases, `aborted` for GC, and inconsistent phase/cursor +combinations (a phase label is never proof of durable reality — the exact defect +repaired in 3a, applied to GC). + +**Policy:** `--keep N` default **1** (retain the newest superseded generation per +signal/range; `--keep 0` reclaims all). A signal whose catalog cannot be +authoritatively reconstructed, OR a range with no active pointer, is excluded +ENTIRELY before any mutation. Both `reconcile` and `gc` acquire the maintenance +lock; dry-run consumes a shared nonmutating planner (never reconciles). + +| Kill-point (SIGKILL at…) | Probe | Oracle + required recovery | +| ------------------------------------------------------------------------------ | -------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| after GC intent durability, before any collection | `native-archive-gc-probe.sh` | reconcile completes the frozen set; only the active generation remains; idempotent | +| after first source→tombstone rename, before removal | " | reconcile resumes: source absent + tombstone present → remove the tombstone; both superseded deleted | +| **nonfinal target removed + gc-collecting progress durable** (index < total-1) | " | reconcile resumes from the cursor + collects the remaining target; NEVER archives prematurely | +| after all removals, before catalog rebuild | " | reconcile rebuilds affected catalogs from manifests; `assertCatalogExact` passes | +| after catalog rebuild, before journal completion | " | reconcile verifies terminal invariants + archives the journal; create-after succeeds | +| `gc --dry-run` (with an active op present) | " + `archive-gc.test.ts` | reports the blocker; NO mutation (snapshot before == after; no reconcile, no journal, no deletion) | +| `reconcile --dry-run` | `archive.ts` + journal | shared `planArchiveReconciliation`; reports kind/phase/actions without mutating (no migration) | +| keep-N retention ordering | `archive-gc.test.ts` | newest N superseded retained per range; older ones targeted; active never selected | +| pointer re-selection (CAS) | `archive-gc.test.ts` + reconcile | if the pointer returns to a target, collection stops and preserves it; never deletes a re-selected generation | +| source replaced/both-present topology | `archive-gc.test.ts` | source+tombstone both present, or identity differs → fail closed (preserve everything) | +| malformed manifest / tampered shard / symlinked gen | `archive-gc.test.ts` | range/signal excluded before any mutation; over-retained; reported | +| absent leaf below symlinked generation/tombstone ancestor | `archive-reconcile.test.ts` | root-to-leaf classifier rejects before mutation; outside sentinel and complete structural snapshot unchanged | +| absent completed/quarantine destination below symlinked ancestor | `archive-reconcile.test.ts` | dry-run and apply both fail closed before collection/quarantine; journal and building state unchanged | +| missing active pointer (uncertain range) | `archive-gc.test.ts` | range excluded entirely; nothing targeted; no journal written; no invalid sentinel | +| terminal invariant (complete journal) | `archive-gc.test.ts` | completedTargets===length, every source/tombstone absent, pointer unchanged, catalog exact — else fail closed | +| legacy v2 create intent (pre-kind) | `archive-journal.test.ts` | `migrateV2CreateIntent` lifts to v3 under the lock; a stranded 3a intent reconciles; corrupt v2 fails closed | + +For every boundary the harness verifies the EXACT invariants (not just counts): +only the FROZEN target IDs are removed; the EXACT active generation remains with +its pointer identity unchanged; the completed GC journal has `phase: complete` + +`completedTargets === targets.length` + the unchanged frozen set; the catalog +exactly matches authoritative manifests; no tombstone retains a generation; a +second reconcile is a no-op; and a subsequent `archive create` succeeds (a crashed +GC never blocks future work). Reconciliation NEVER expands the frozen set — a +resumed GC deletes exactly what the original decided. + +Zero-mutation parity uses a fail-loud structural snapshot, not shell `find | +shasum`: it records every relative path, entry type, symlink target, file size +and hash, and empty directory. A dangling symlink is evidence and snapshot +failure is a test failure, never a comparable sentinel value. + +**Working rule for this section:** _GC is the only path that deletes published +evidence, so it must prove it owns and may delete each generation twice — once at +plan time (under the lock) and once at collection time (the CAS re-check)._ +"Recovery succeeded" is never the oracle; the on-disk state after a kill is. + +## How to use this matrix + +1. For any archive-export change, identify which invariants the diff touches. +2. Write/extend the corresponding probe so it fails against the current code. +3. Implement the change; require the probe to pass and the six-signal smoke to + pass. +4. Re-answer, in the change description: _how could an incorrect archive + preserve every metric this change checks?_ +5. Update this matrix's red/green columns if a new transformation is + discovered. + +The matrix is the gate; the ledgers (`STATUS.md`/`TESTS.md`/`DECISIONS.md`) +record only the verdict. diff --git a/apps/cli/test/archive-calibrate.test.ts b/apps/cli/test/archive-calibrate.test.ts new file mode 100644 index 000000000..4648ceeef --- /dev/null +++ b/apps/cli/test/archive-calibrate.test.ts @@ -0,0 +1,1375 @@ +import { describe, it } from "@effect/vitest" +import { Effect } from "effect" +import { ok, rejects, strictEqual, throws } from "node:assert" +import { + writeFileSync as writeFileSyncSync, + mkdtempSync, + mkdirSync, + mkdtempSync as mktmp, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, + existsSync, +} from "node:fs" +import { arch, cpus, platform, tmpdir, totalmem, userInfo } from "node:os" +import { join } from "node:path" +import { + acquireCheckpointPin, + checkpointRoot, + checkpointSnapshotDir, + checkpointStatePath, +} from "../src/server/checkpoints" +import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +import { SCHEMA_FINGERPRINT } from "../src/server/serve" + +/** Seed a minimal checkpoint snapshot + state so resolveCheckpoint succeeds in unit tests. */ +const seedCheckpoint = (dataDir: string, checkpointId: string): string => { + const createdAt = "2026-01-01T00:00:00.000Z" + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + mkdirSync(join(snapshot, "backup"), { recursive: true }) + writeFileSyncSync(join(snapshot, "backup", "data.bin"), "backup") + writeFileSyncSync( + join(snapshot, "manifest.json"), + `${JSON.stringify({ + formatVersion: 1, + checkpointId, + operationId: "00000000-0000-4000-8000-000000000000", + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + createdAt, + sourceDataDir: dataDir, + backupRelativePath: `snapshots/${checkpointId}/backup`, + backupBytes: 6, + validation: { + validatedAt: createdAt, + traces: 0, + logs: 0, + metricsSum: 0, + metricsGauge: 0, + metricsHistogram: 0, + metricsExponentialHistogram: 0, + materializedViews: 0, + }, + })}\n`, + ) + mkdirSync(checkpointRoot(dataDir), { recursive: true }) + writeFileSyncSync( + checkpointStatePath(dataDir), + `${JSON.stringify({ formatVersion: 1, revision: "00000000-0000-4000-8000-000000000001", current: checkpointId, previous: null, committedAt: createdAt })}\n`, + ) + // Return the canonical fingerprint the recovery record must match. + return `${checkpointId}:${createdAt}:6` +} +import { + type CalibrationBudget, + type CalibrationCandidate, + type CandidateMetrics, + type CandidateResult, + meetsCeilings, + selectCandidates, + worstCaseMetrics, + comparePredictedObserved, + compareHeldOutPerSignal, + HELD_OUT_TOLERANCES, + isSameCalibrationCandidate, + heldOutSampleRows, + RECALIBRATION_TRIGGERS, + recommendationToTuning, + writeCalibrationConfig, + type CalibrationRecommendation, + CANDIDATE_MATRIX, + deriveTargetChunkBytes, + validateCalibrationBudget, +} from "../src/server/archives/calibrate" +import { ARCHIVE_SIGNALS } from "../src/server/archives/signals" +import { + reconcileCalibration, + writeCalibrationRecord, + calibrationRecoveryPath, + calibrationPinPurpose, + derivedScratchSubdir, + derivedSampleDir, + directoryTreeBytes, + preflightCalibrationFreeSpace, + assertCalibrationSession, + cleanupCalibrationSample, + archiveVolumeIdentity, +} from "../src/server/archives/calibration-recovery" +import { createArchiveGeneration } from "../src/server/archives/generation" +import { listActiveOperationIds } from "../src/server/archives/journal" +import { + loadTuningConfig, + resolveArchiveTuning, + TUNING_CONFIG_FORMAT_VERSION, + type LoadedTuningConfig, +} from "../src/server/archives/config" +import { decodeChildMetrics, requireCalibrationSelection } from "../src/commands/archive" +import { ArchiveError, archiveErrorMessage } from "../src/server/archives/errors" + +const baseMetrics = (over: Partial = {}): CandidateMetrics => ({ + logicalBytes: 1_000_000, + physicalBytes: 300_000, + compressionRatio: 0.3, + writeThroughputBytesPerSec: 200_000, + peakTempDiskBytes: 500_000, + peakRssBytes: 200_000_000, + wallMs: 5_000, + rowCount: 10_000, + ...over, +}) + +const okResult = ( + candidate: CalibrationCandidate, + signal: string, + metrics: CandidateMetrics, +): CandidateResult => ({ + candidate, + signal, + metrics, + ok: true, +}) + +const baseBudget = (over: Partial = {}): CalibrationBudget => ({ + memoryBudget: 1_000_000_000, + timeBudget: 60_000, + sampleRows: 10_000, + maxCandidateWallMs: 30_000, + minThroughputBytesPerSec: 0, + maxTempDiskBytes: 2_000_000_000, + freeSpaceReserve: 512 * 1024 * 1024, + safetyMargin: 1.1, + ...over, +}) + +const cand = (wt: number, rg: number): CalibrationCandidate => ({ + writerThreads: wt, + rowGroupRows: rg, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, +}) + +describe("calibration budget validation", () => { + it("accepts a complete valid budget", () => { + const budget = baseBudget() + strictEqual(validateCalibrationBudget(budget), budget) + }) + + it("requires positive safe-integer ceilings, sample sizes, and reserves", () => { + for (const field of [ + "memoryBudget", + "timeBudget", + "sampleRows", + "maxCandidateWallMs", + "maxTempDiskBytes", + "freeSpaceReserve", + ] as const) { + for (const value of [0, -1, Number.MAX_SAFE_INTEGER + 1]) { + throws( + () => validateCalibrationBudget(baseBudget({ [field]: value })), + new RegExp(`${field} must be a positive safe integer`), + ) + } + } + }) + + it("requires non-negative safe-integer throughput", () => { + for (const value of [-1, Number.MAX_SAFE_INTEGER + 1]) { + throws( + () => validateCalibrationBudget(baseBudget({ minThroughputBytesPerSec: value })), + /minThroughputBytesPerSec must be a non-negative safe integer/, + ) + } + }) + + it("requires a finite safety margin of at least one", () => { + for (const value of [0, 0.999, Number.NaN, Number.POSITIVE_INFINITY]) { + throws( + () => validateCalibrationBudget(baseBudget({ safetyMargin: value })), + /safetyMargin must be a finite number at least 1/, + ) + } + }) +}) + +const childScope = { + checkpointId: "00000000-0000-4000-8000-000000000000", + checkpointManifestFingerprint: "checkpoint:fingerprint:6", + rangeDate: "2026-01-01", + role: "held-out" as const, + startRow: 10_000, + requestedRows: 20_000, +} + +const childMetrics = () => ({ + logicalBytes: 1_000_000, + physicalBytes: 300_000, + peakTempDiskBytes: 500_000, + peakRssBytes: 200_000_000, + exportWallMs: 5_000, + rowCount: 19_500, + sample: { ...childScope, rowCount: 19_500 }, +}) + +describe("calibration child protocol", () => { + it("decodes an exact finite metrics document bound to the requested sample scope", () => { + strictEqual(decodeChildMetrics(childMetrics(), childScope).sample.startRow, childScope.startRow) + }) + + it("rejects missing, null, string, non-finite, negative, unsafe, and excess fields", () => { + const invalid: Array = [ + { ...childMetrics(), logicalBytes: undefined }, + { ...childMetrics(), logicalBytes: null }, + { ...childMetrics(), logicalBytes: "1000000" }, + { ...childMetrics(), logicalBytes: Number.NaN }, + { ...childMetrics(), physicalBytes: Number.POSITIVE_INFINITY }, + { ...childMetrics(), peakTempDiskBytes: -1 }, + { ...childMetrics(), rowCount: Number.MAX_SAFE_INTEGER + 1 }, + { ...childMetrics(), unexpected: true }, + ] + for (const value of invalid) throws(() => decodeChildMetrics(value, childScope)) + }) + + it("binds role, requested window, and returned row count exactly", () => { + for (const sample of [ + { ...childMetrics().sample, role: "training" }, + { ...childMetrics().sample, startRow: 0 }, + { ...childMetrics().sample, requestedRows: 10_000 }, + { ...childMetrics().sample, rowCount: 19_499 }, + ]) { + throws( + () => decodeChildMetrics({ ...childMetrics(), sample }, childScope), + /inconsistent sample scope/, + ) + } + }) +}) + +describe("calibration recommendation error handling", () => { + it("returns no-recommendation as a typed ArchiveError instead of a fiber defect", async () => { + const error = await Effect.runPromise( + Effect.flip( + requireCalibrationSelection({ + selected: null, + note: "no candidate met the declared goals", + }), + ), + ) + + ok(error instanceof ArchiveError) + ok(error.message.includes("calibration did not produce a recommendation")) + ok(error.message.includes("no candidate met the declared goals")) + }) + + it("returns the selected recommendation on success", async () => { + const selected = { candidate: CANDIDATE_MATRIX[0]!, worstCase: baseMetrics() } + strictEqual( + await Effect.runPromise(requireCalibrationSelection({ selected, note: "selected" })), + selected, + ) + }) + + it("renders an expected archive failure without diagnostic stack frames", () => { + const error = new ArchiveError({ message: "calibration did not produce a recommendation" }) + strictEqual(archiveErrorMessage(error), "calibration did not produce a recommendation\n") + }) +}) + +/** Create isolated data/archive/scratch roots under the real temp volume. */ +const withRoots = async ( + run: (roots: { dataDir: string; archiveDir: string; scratchRoot: string }) => Promise, +): Promise => { + const parent = realpathSync(mktmp(join(tmpdir(), "maple-calrec-"))) + const dataDir = join(parent, "data") + const archiveDir = join(parent, "archive") + const scratchRoot = join(parent, "scratch") + mkdirSync(dataDir, { recursive: true }) + mkdirSync(archiveDir, { recursive: true }) + mkdirSync(scratchRoot, { recursive: true }) + try { + await run({ dataDir, archiveDir, scratchRoot }) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +describe("calibration candidate identity", () => { + it("does not let same-thread candidates lend representative rows", () => { + const selected = CANDIDATE_MATRIX[0]! + strictEqual(isSameCalibrationCandidate(selected, { ...selected }), true) + strictEqual(isSameCalibrationCandidate(selected, CANDIDATE_MATRIX[1]!), false) + strictEqual(isSameCalibrationCandidate(selected, CANDIDATE_MATRIX[3]!), false) + }) +}) + +describe("calibration held-out window is larger and disjoint", () => { + it("heldOutSampleRows is a strict multiple > training size and yields a disjoint window", () => { + const training = 1000 + const held = heldOutSampleRows(training) + ok(held > training, "held-out must be larger than training") + // Training [0, training); held-out [training, training+held). Disjoint. + const trainingEnd = training + const heldOutStart = training + strictEqual(heldOutStart, trainingEnd, "held-out must start where training ends") + ok(heldOutStart >= trainingEnd) + // A larger training keeps the multiplier invariant. + strictEqual(heldOutSampleRows(50_000), 100_000) + }) +}) + +describe("calibration measurement engine — meetsCeilings", () => { + it("passes when all metrics are within every ceiling with margin applied inside", () => { + const budget = baseBudget({ memoryBudget: 250_000_000, safetyMargin: 1.1 }) + const r = okResult(cand(1, 10_000), "logs", baseMetrics({ peakRssBytes: 200_000_000 })) + strictEqual(meetsCeilings(r, budget), true) + }) + + it("fails when peak RSS * margin exceeds the memory budget", () => { + const budget = baseBudget({ memoryBudget: 250_000_000, safetyMargin: 1.1 }) + // 230M * 1.1 = 253M > 250M + const r = okResult(cand(1, 10_000), "logs", baseMetrics({ peakRssBytes: 230_000_000 })) + strictEqual(meetsCeilings(r, budget), false) + }) + + it("fails when wall time exceeds the per-candidate deadline", () => { + const budget = baseBudget({ maxCandidateWallMs: 10_000 }) + const r = okResult(cand(1, 10_000), "logs", baseMetrics({ wallMs: 15_000 })) + strictEqual(meetsCeilings(r, budget), false) + }) + + it("fails when throughput / margin is below the floor", () => { + const budget = baseBudget({ minThroughputBytesPerSec: 100_000, safetyMargin: 1.1 }) + // 100000 / 1.1 = 90909 < 100000 + const r = okResult(cand(1, 10_000), "logs", baseMetrics({ writeThroughputBytesPerSec: 100_000 })) + strictEqual(meetsCeilings(r, budget), false) + }) + + it("fails when peak temp disk * margin exceeds the ceiling", () => { + const budget = baseBudget({ maxTempDiskBytes: 1_000_000_000, safetyMargin: 1.1 }) + const r = okResult(cand(1, 10_000), "logs", baseMetrics({ peakTempDiskBytes: 950_000_000 })) + strictEqual(meetsCeilings(r, budget), false) + }) + + it("never passes a failed result (ok=false or null metrics)", () => { + const budget = baseBudget() + const failed: CandidateResult = { + candidate: cand(1, 10_000), + signal: "logs", + metrics: null, + ok: false, + error: "boom", + } + strictEqual(meetsCeilings(failed, budget), false) + }) +}) + +describe("calibration measurement engine — worstCaseMetrics", () => { + it("takes the MAXIMUM of cost metrics and the MINIMUM of throughput across signals", () => { + const results: CandidateResult[] = [ + okResult( + cand(1, 10_000), + "logs", + baseMetrics({ + peakRssBytes: 100_000_000, + rowCount: 5_000, + writeThroughputBytesPerSec: 200_000, + }), + ), + okResult( + cand(1, 10_000), + "traces", + baseMetrics({ + peakRssBytes: 200_000_000, + rowCount: 8_000, + writeThroughputBytesPerSec: 80_000, + }), + ), + okResult( + cand(1, 10_000), + "metrics_sum", + baseMetrics({ + peakRssBytes: 150_000_000, + rowCount: 12_000, + writeThroughputBytesPerSec: 150_000, + }), + ), + ] + const wc = worstCaseMetrics(results) + strictEqual(wc.peakRssBytes, 200_000_000) // max + strictEqual(wc.rowCount, 12_000) // max + strictEqual(wc.writeThroughputBytesPerSec, 80_000) // MIN (the slowest signal is the floor worst case) + }) + + it("returns zeroed metrics when no result is ok", () => { + const wc = worstCaseMetrics([ + { candidate: cand(1, 10_000), signal: "logs", metrics: null, ok: false, error: "x" }, + ]) + strictEqual(wc.peakRssBytes, 0) + strictEqual(wc.rowCount, 0) + }) +}) + +describe("calibration measurement engine — selectCandidates", () => { + it("returns eligible candidates best-first (lowest worst-case RSS, then wall) and only those meeting every required signal", () => { + const budget = baseBudget({ memoryBudget: 300_000_000 }) + const c1 = cand(1, 10_000) + const c2 = cand(2, 10_000) + // c1 passes both required signals; c2 fails one signal (RSS too high). + const perSignal = new Map([ + [ + c1, + [ + okResult(c1, "logs", baseMetrics({ peakRssBytes: 100_000_000 })), + okResult(c1, "traces", baseMetrics({ peakRssBytes: 150_000_000 })), + ], + ], + [ + c2, + [ + okResult(c2, "logs", baseMetrics({ peakRssBytes: 400_000_000 })), + okResult(c2, "traces", baseMetrics({ peakRssBytes: 200_000_000 })), + ], + ], + ]) + const eligible = selectCandidates(perSignal, budget, ["logs", "traces"]) + strictEqual(eligible.length, 1) + strictEqual(eligible[0]!.candidate.writerThreads, 1) + strictEqual(eligible[0]!.worstCase.peakRssBytes, 150_000_000) + }) + + it("rejects an incomplete signal set (missing a required signal)", () => { + const budget = baseBudget({ memoryBudget: 300_000_000 }) + const perSignal = new Map([ + // Only logs present, traces MISSING — incomplete. + [ + cand(1, 10_000), + [okResult(cand(1, 10_000), "logs", baseMetrics({ peakRssBytes: 100_000_000 }))], + ], + ]) + const eligible = selectCandidates(perSignal, budget, ["logs", "traces"]) + strictEqual(eligible.length, 0) + }) + + it("rejects a duplicate signal", () => { + const budget = baseBudget({ memoryBudget: 300_000_000 }) + const perSignal = new Map([ + [ + cand(1, 10_000), + [ + okResult(cand(1, 10_000), "logs", baseMetrics()), + okResult(cand(1, 10_000), "logs", baseMetrics()), // duplicate + ], + ], + ]) + const eligible = selectCandidates(perSignal, budget, ["logs", "traces"]) + strictEqual(eligible.length, 0) + }) + + it("returns an empty list when no candidate meets every signal (impossible budget)", () => { + const budget = baseBudget({ memoryBudget: 50_000_000 }) + const perSignal = new Map([ + [ + cand(1, 10_000), + [okResult(cand(1, 10_000), "logs", baseMetrics({ peakRssBytes: 200_000_000 }))], + ], + ]) + const eligible = selectCandidates(perSignal, budget, ["logs"]) + strictEqual(eligible.length, 0) + }) +}) + +describe("calibration measurement engine — comparePredictedObserved", () => { + it("passes when every metric is within its tolerance", () => { + const pred = baseMetrics() + const obs = baseMetrics({ peakRssBytes: 210_000_000 }) // 5% over + const result = comparePredictedObserved(pred, obs, { + peakRssBytes: 0.1, + wallMs: 0.1, + writeThroughputBytesPerSec: 0.1, + compressionRatio: 0.1, + physicalBytes: 0.1, + peakTempDiskBytes: 0.1, + }) + strictEqual(result.passed, true) + }) + + it("fails when a metric exceeds its tolerance", () => { + const pred = baseMetrics() + const obs = baseMetrics({ peakRssBytes: 300_000_000 }) // 50% over + const result = comparePredictedObserved(pred, obs, { + peakRssBytes: 0.1, + wallMs: 0.1, + writeThroughputBytesPerSec: 0.1, + compressionRatio: 0.1, + physicalBytes: 0.1, + peakTempDiskBytes: 0.1, + }) + strictEqual(result.passed, false) + const rssCmp = result.comparisons.find((c) => c.metric === "peakRssBytes")! + ok(!rssCmp.withinTolerance) + }) + + it("accepts lower cost across every resource metric, including scaled held-out costs", () => { + const pred = baseMetrics({ + peakRssBytes: 200_000_000, + wallMs: 1_000, + compressionRatio: 0.5, + physicalBytes: 100_000, + peakTempDiskBytes: 1_000_000, + }) + const obs = baseMetrics({ + peakRssBytes: 100_000_000, + wallMs: 250, + compressionRatio: 0.25, + physicalBytes: 25_000, + peakTempDiskBytes: 500_000, + }) + const result = comparePredictedObserved( + pred, + obs, + { + peakRssBytes: 0.1, + wallMs: 0.1, + writeThroughputBytesPerSec: 0.1, + compressionRatio: 0.1, + physicalBytes: 0.1, + peakTempDiskBytes: 0.1, + }, + { ratio: 2, metrics: new Set(["wallMs", "physicalBytes"]) }, + ) + strictEqual(result.passed, true) + for (const metric of [ + "peakRssBytes", + "wallMs", + "compressionRatio", + "physicalBytes", + "peakTempDiskBytes", + ] as const) { + strictEqual(result.comparisons.find((c) => c.metric === metric)!.relativeDelta, 0) + } + }) + + it("rejects a regression beyond tolerance for each directional resource cost", () => { + const tolerance = { + peakRssBytes: 0.1, + wallMs: 0.1, + writeThroughputBytesPerSec: 0.1, + compressionRatio: 0.1, + physicalBytes: 0.1, + peakTempDiskBytes: 0.1, + } + const costs = [ + "peakRssBytes", + "wallMs", + "compressionRatio", + "physicalBytes", + "peakTempDiskBytes", + ] as const + for (const metric of costs) { + const predicted = baseMetrics() + const observed = { ...predicted, [metric]: predicted[metric] * 1.11 } + const result = comparePredictedObserved(predicted, observed, tolerance) + strictEqual(result.passed, false, `${metric} regression must fail`) + ok(!result.comparisons.find((comparison) => comparison.metric === metric)!.withinTolerance) + } + }) + + it("handles a zero predicted resource cost fail-closed", () => { + const tolerance = { + peakRssBytes: 0.1, + wallMs: 0.1, + writeThroughputBytesPerSec: 0.1, + compressionRatio: 0.1, + physicalBytes: 0.1, + peakTempDiskBytes: 0.1, + } + const predicted = baseMetrics({ peakTempDiskBytes: 0 }) + strictEqual( + comparePredictedObserved(predicted, baseMetrics({ peakTempDiskBytes: 0 }), tolerance).passed, + true, + ) + const regression = comparePredictedObserved( + predicted, + baseMetrics({ peakTempDiskBytes: 1 }), + tolerance, + ) + strictEqual(regression.passed, false) + strictEqual( + regression.comparisons.find((comparison) => comparison.metric === "peakTempDiskBytes")! + .withinTolerance, + false, + ) + }) + + it("throughput is directional (higher observed is better, always passes)", () => { + const pred = baseMetrics({ writeThroughputBytesPerSec: 100_000 }) + const obs = baseMetrics({ writeThroughputBytesPerSec: 200_000 }) + const result = comparePredictedObserved(pred, obs, { + peakRssBytes: 0.1, + wallMs: 0.1, + writeThroughputBytesPerSec: 0.1, + compressionRatio: 0.1, + physicalBytes: 0.1, + peakTempDiskBytes: 0.1, + }) + const tputCmp = result.comparisons.find((c) => c.metric === "writeThroughputBytesPerSec")! + ok(tputCmp.withinTolerance) + }) + + it("rejects throughput below its floor and allows a zero predicted throughput", () => { + const tolerance = { + peakRssBytes: 0.1, + wallMs: 0.1, + writeThroughputBytesPerSec: 0.1, + compressionRatio: 0.1, + physicalBytes: 0.1, + peakTempDiskBytes: 0.1, + } + const predicted = baseMetrics({ writeThroughputBytesPerSec: 100_000 }) + const tooSlow = comparePredictedObserved( + predicted, + baseMetrics({ writeThroughputBytesPerSec: 89_999 }), + tolerance, + ) + strictEqual(tooSlow.passed, false) + strictEqual( + tooSlow.comparisons.find((comparison) => comparison.metric === "writeThroughputBytesPerSec")! + .withinTolerance, + false, + ) + const zeroBaseline = comparePredictedObserved( + baseMetrics({ writeThroughputBytesPerSec: 0 }), + baseMetrics({ writeThroughputBytesPerSec: 1 }), + tolerance, + ) + strictEqual(zeroBaseline.passed, true) + }) +}) + +describe("per-signal held-out comparison rejects cross-signal aggregate masking", () => { + // The reviewer's executable counterexample: aggregate logical-byte ratio is 4, + // so aggregate wall prediction becomes 100*4=400 matching observed 400 (pass), + // but like-for-like signal A has ratio 2, adjusted prediction 200 vs observed + // 400 = delta 1.0, which exceeds the canonical 0.5 wall tolerance (fail). + // Per-signal comparison must reject what aggregate scaling would accept. + const candidate = CANDIDATE_MATRIX[0]! + const metrics = (over: Partial): CandidateMetrics => ({ + logicalBytes: 1_000_000, + physicalBytes: 300_000, + compressionRatio: 0.3, + writeThroughputBytesPerSec: 100_000, + peakTempDiskBytes: 500_000, + peakRssBytes: 200_000_000, + wallMs: 5_000, + rowCount: 10_000, + ...over, + }) + const result = (signal: string, m: CandidateMetrics): CandidateResult => ({ + candidate, + signal, + metrics: m, + ok: true, + }) + + it("fails when one signal regresses even though the aggregate ratio would pass", () => { + const training = [ + result("logs", metrics({ logicalBytes: 1_000, wallMs: 100, physicalBytes: 300 })), + result("traces", metrics({ logicalBytes: 10_000, wallMs: 10, physicalBytes: 3_000 })), + ] + const heldOut = [ + result("logs", metrics({ logicalBytes: 2_000, wallMs: 400, physicalBytes: 600 })), + result("traces", metrics({ logicalBytes: 40_000, wallMs: 40, physicalBytes: 12_000 })), + ] + const perSignal = compareHeldOutPerSignal( + training, + heldOut, + ["logs", "traces"], + candidate, + HELD_OUT_TOLERANCES, + ) + if (perSignal === null) throw new Error("per-signal comparison returned null unexpectedly") + const logsWall = perSignal.signalComparisons + .find((s) => s.signal === "logs")! + .comparisons.find((c) => c.metric === "wallMs")! + // Signal A: ratio 2, adjusted prediction 200, observed 400 → delta 1.0 > 0.5. + ok(!logsWall.withinTolerance, "signal A wall regression must fail the per-signal check") + strictEqual(perSignal.passed, false, "the attempt must not pass when any signal fails") + }) + + it("returns null when a signal cannot be paired (incomplete)", () => { + const training = [result("logs", metrics())] + const heldOut = [result("logs", metrics()), result("traces", metrics())] + strictEqual( + compareHeldOutPerSignal(training, heldOut, ["logs", "traces"], candidate, HELD_OUT_TOLERANCES), + null, + ) + }) + + it("returns null when a paired signal has non-positive training logicalBytes", () => { + const training = [result("logs", metrics({ logicalBytes: 0 }))] + const heldOut = [result("logs", metrics())] + strictEqual( + compareHeldOutPerSignal(training, heldOut, ["logs"], candidate, HELD_OUT_TOLERANCES), + null, + ) + }) +}) + +describe("calibration tuning derivation and strict volume binding", () => { + it("derives both non-candidate knobs exactly and rejects overflow", () => { + strictEqual(deriveTargetChunkBytes(256 * 1024 * 1024, 512 * 1024 * 1024), 1024 * 1024 * 1024) + strictEqual(deriveTargetChunkBytes(100, 10_000), 10_100) + throws(() => deriveTargetChunkBytes(Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER), /overflow/) + }) + + it("inspects only an existing canonical non-symlink archive root", async () => { + const parent = realpathSync(mkdtempSync(join(tmpdir(), "maple-bound-volume-"))) + try { + const root = join(parent, "archive") + const link = join(parent, "archive-link") + mkdirSync(root) + symlinkSync(root, link) + const identity = await archiveVolumeIdentity(root) + ok(identity.fsid.startsWith("dev:")) + await rejects(archiveVolumeIdentity(link), /real non-symlink|canonical/) + await rejects(archiveVolumeIdentity(join(parent, "missing")), /ENOENT|existing/) + } finally { + rmSync(parent, { recursive: true, force: true }) + } + }) +}) + +describe("calibration config document — writeCalibrationConfig emits required fields", () => { + it("writes environment, evidence, safetyMargin, recalibrationTriggers, and schemaFingerprint", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-cfg-")) + try { + const path = join(dir, "cfg.json") + const rec: CalibrationRecommendation = { + formatVersion: TUNING_CONFIG_FORMAT_VERSION, + checkpoint: { + checkpointId: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee", + manifestFingerprint: "checkpoint:fingerprint", + }, + selected: { candidate: CANDIDATE_MATRIX[0]!, worstCase: baseMetrics() }, + results: [okResult(CANDIDATE_MATRIX[0]!, "logs", baseMetrics())], + heldOut: { + results: [okResult(CANDIDATE_MATRIX[0]!, "logs", baseMetrics())], + worstCase: baseMetrics(), + comparisons: comparePredictedObserved(baseMetrics(), baseMetrics(), { + peakRssBytes: 0.5, + wallMs: 1, + writeThroughputBytesPerSec: 0.75, + compressionRatio: 0.5, + physicalBytes: 1, + peakTempDiskBytes: 0.5, + }).comparisons, + passed: true, + tolerances: { + peakRssBytes: 0.5, + wallMs: 1, + writeThroughputBytesPerSec: 0.75, + compressionRatio: 0.5, + physicalBytes: 1, + peakTempDiskBytes: 0.5, + }, + }, + heldOutAttempts: [], + budget: baseBudget(), + environment: { + mapleVersion: "test", + chdbVersion: "v26", + schemaFingerprint: "abc123", + executionUser: "tester", + platform: "darwin", + arch: "arm64", + cpuModel: "test-cpu", + cpuCount: 8, + totalMemoryBytes: 16_000_000_000, + measurementTool: "/usr/bin/time", + archiveVolume: { fsid: "dev:abc", type: 17, archiveDir: "/tmp/archive" }, + }, + confidence: "high", + measuredAt: "2026-07-01T00:00:00.000Z", + note: "test", + } + const tuning = recommendationToTuning(rec, "/tmp/archive", "/tmp/scratch") + writeCalibrationConfig(path, rec, tuning) + const doc = JSON.parse(require("node:fs").readFileSync(path, "utf8")) as Record + strictEqual(doc.formatVersion, TUNING_CONFIG_FORMAT_VERSION) + ok(doc.environment !== undefined) + ok(Array.isArray(doc.results)) + ok(doc.safetyMargin !== undefined) + ok(Array.isArray(doc.recalibrationTriggers)) + strictEqual((doc.environment as { schemaFingerprint: string }).schemaFingerprint, "abc123") + ok(doc.effective !== undefined) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +}) + +describe("calibration recovery — idempotent reconcile", () => { + it("reconciling when no prior record exists is a no-op", async () => { + await withRoots(async (roots) => { + await reconcileCalibration(roots.archiveDir, roots) + strictEqual(existsSync(calibrationRecoveryPath(roots.archiveDir)), false) + }) + }) + + it("reconciling a record whose phase precedes pin creation removes owned paths (pin derived from pinId, absent = success)", async () => { + await withRoots(async (roots) => { + const operationId = "deadbeef-1111-4aaa-9bbb-deadbeefdead" + const checkpointId = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + const pinId = "11111111-2222-4333-8444-555555555555" + // Seed a real checkpoint so resolveCheckpoint + fingerprint validation pass. + const fingerprint = seedCheckpoint(roots.dataDir, checkpointId) + // DERIVED owned dirs from the operation id. + const scratchOwned = join(roots.scratchRoot, derivedScratchSubdir(operationId)) + const sampleDir = derivedSampleDir(roots.archiveDir, operationId) + mkdirSync(scratchOwned, { recursive: true }) + mkdirSync(sampleDir, { recursive: true }) + writeFileSync(join(scratchOwned, "junk"), "x") + // Record at intent phase (pinPath null). The pin is DERIVED from pinId; + // an absent pin is success (over-retention safe), so reconcile proceeds. + await writeCalibrationRecord(roots.archiveDir, { + phase: "intent", + operationId, + pinId, + pinPurpose: calibrationPinPurpose(operationId), + pinPath: null, + checkpointId, + checkpointManifestFingerprint: fingerprint, + boundRoots: roots, + ownedPaths: { scratchSubdir: derivedScratchSubdir(operationId), sampleDir }, + }) + await reconcileCalibration(roots.archiveDir, roots) + strictEqual(existsSync(scratchOwned), false) + strictEqual(existsSync(sampleDir), false) + strictEqual(existsSync(calibrationRecoveryPath(roots.archiveDir)), false) + }) + }) + + it("retires an inert intent after normal retention removes its still-unpinned source checkpoint", async () => { + await withRoots(async (roots) => { + const operationId = "deadbeef-1111-4aaa-9bbb-deadbeefdead" + const checkpointId = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + const replacementId = "bbbbbbbb-cccc-4ddd-8eee-ffffffffffff" + const pinId = "11111111-2222-4333-8444-555555555555" + const fingerprint = seedCheckpoint(roots.dataDir, checkpointId) + // A later checkpoint becomes current, then normal retention removes the + // unpinned source selected by the interrupted intent. + seedCheckpoint(roots.dataDir, replacementId) + rmSync(checkpointSnapshotDir(roots.dataDir, checkpointId), { recursive: true, force: true }) + const sampleDir = derivedSampleDir(roots.archiveDir, operationId) + await writeCalibrationRecord(roots.archiveDir, { + phase: "intent", + operationId, + pinId, + pinPurpose: calibrationPinPurpose(operationId), + pinPath: null, + checkpointId, + checkpointManifestFingerprint: fingerprint, + boundRoots: roots, + ownedPaths: { scratchSubdir: derivedScratchSubdir(operationId), sampleDir }, + }) + + await reconcileCalibration(roots.archiveDir, roots) + + strictEqual(existsSync(calibrationRecoveryPath(roots.archiveDir)), false) + strictEqual(existsSync(checkpointSnapshotDir(roots.dataDir, replacementId)), true) + }) + }) + + it("preserves a missing-checkpoint intent when an exact derived resource is still present", async () => { + await withRoots(async (roots) => { + const operationId = "deadbeef-1111-4aaa-9bbb-deadbeefdead" + const checkpointId = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + const replacementId = "bbbbbbbb-cccc-4ddd-8eee-ffffffffffff" + const pinId = "11111111-2222-4333-8444-555555555555" + const fingerprint = seedCheckpoint(roots.dataDir, checkpointId) + seedCheckpoint(roots.dataDir, replacementId) + rmSync(checkpointSnapshotDir(roots.dataDir, checkpointId), { recursive: true, force: true }) + const sampleDir = derivedSampleDir(roots.archiveDir, operationId) + mkdirSync(sampleDir, { recursive: true }) + await writeCalibrationRecord(roots.archiveDir, { + phase: "intent", + operationId, + pinId, + pinPurpose: calibrationPinPurpose(operationId), + pinPath: null, + checkpointId, + checkpointManifestFingerprint: fingerprint, + boundRoots: roots, + ownedPaths: { scratchSubdir: derivedScratchSubdir(operationId), sampleDir }, + }) + + await rejects( + reconcileCalibration(roots.archiveDir, roots), + /source checkpoint.*preserving record/i, + ) + strictEqual(existsSync(calibrationRecoveryPath(roots.archiveDir)), true) + strictEqual(existsSync(sampleDir), true) + }) + }) + + it("re-running reconcile after cleanup is a no-op (idempotent)", async () => { + await withRoots(async (roots) => { + await reconcileCalibration(roots.archiveDir, roots) + await reconcileCalibration(roots.archiveDir, roots) + strictEqual(existsSync(calibrationRecoveryPath(roots.archiveDir)), false) + }) + }) + + it("cleans one child sample while retaining the parent session pin and durable identity", async () => { + await withRoots(async (roots) => { + const operationId = "deadbeef-1111-4aaa-9bbb-deadbeefdead" + const checkpointId = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + const pinId = "11111111-2222-4333-8444-555555555555" + const fingerprint = seedCheckpoint(roots.dataDir, checkpointId) + const purpose = calibrationPinPurpose(operationId) + const pinPath = await acquireCheckpointPin(roots.dataDir, checkpointId, purpose, pinId) + const sampleDir = derivedSampleDir(roots.archiveDir, operationId) + const scratchSubdir = derivedScratchSubdir(operationId) + mkdirSync(join(roots.scratchRoot, scratchSubdir), { recursive: true }) + mkdirSync(sampleDir, { recursive: true }) + await writeCalibrationRecord(roots.archiveDir, { + phase: "pin-acquired", + operationId, + pinId, + pinPurpose: purpose, + pinPath, + checkpointId, + checkpointManifestFingerprint: fingerprint, + boundRoots: roots, + ownedPaths: { scratchSubdir, sampleDir }, + }) + + const session = await assertCalibrationSession(roots.archiveDir, roots, { + operationId, + checkpointId, + checkpointManifestFingerprint: fingerprint, + }) + await cleanupCalibrationSample(session) + + strictEqual(existsSync(pinPath), true) + strictEqual(existsSync(calibrationRecoveryPath(roots.archiveDir)), true) + strictEqual(existsSync(join(roots.scratchRoot, scratchSubdir)), false) + strictEqual(existsSync(sampleDir), false) + await reconcileCalibration(roots.archiveDir, roots) + }) + }) + + it("rejects malformed or substituted parent-session pin identities", async () => { + const cases = [ + { name: "malformed", value: {} }, + { + name: "foreign-pin-id", + value: { + formatVersion: 1, + pinId: "99999999-9999-4999-8999-999999999999", + checkpointId: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee", + purpose: "archive-calibrate:deadbeef-1111-4aaa-9bbb-deadbeefdead", + createdAt: "2026-01-01T00:00:00.000Z", + }, + }, + { + name: "foreign-checkpoint", + value: { + formatVersion: 1, + pinId: "11111111-2222-4333-8444-555555555555", + checkpointId: "bbbbbbbb-cccc-4ddd-8eee-ffffffffffff", + purpose: "archive-calibrate:deadbeef-1111-4aaa-9bbb-deadbeefdead", + createdAt: "2026-01-01T00:00:00.000Z", + }, + }, + { + name: "foreign-purpose", + value: { + formatVersion: 1, + pinId: "11111111-2222-4333-8444-555555555555", + checkpointId: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee", + purpose: "archive-calibrate:ffffffff-ffff-4fff-8fff-ffffffffffff", + createdAt: "2026-01-01T00:00:00.000Z", + }, + }, + ] + for (const testCase of cases) { + await withRoots(async (roots) => { + const operationId = "deadbeef-1111-4aaa-9bbb-deadbeefdead" + const checkpointId = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + const pinId = "11111111-2222-4333-8444-555555555555" + const fingerprint = seedCheckpoint(roots.dataDir, checkpointId) + const purpose = calibrationPinPurpose(operationId) + const pinPath = await acquireCheckpointPin(roots.dataDir, checkpointId, purpose, pinId) + writeFileSync(pinPath, JSON.stringify(testCase.value)) + await writeCalibrationRecord(roots.archiveDir, { + phase: "pin-acquired", + operationId, + pinId, + pinPurpose: purpose, + pinPath, + checkpointId, + checkpointManifestFingerprint: fingerprint, + boundRoots: roots, + ownedPaths: { + scratchSubdir: derivedScratchSubdir(operationId), + sampleDir: derivedSampleDir(roots.archiveDir, operationId), + }, + }) + + await rejects( + assertCalibrationSession(roots.archiveDir, roots, { + operationId, + checkpointId, + checkpointManifestFingerprint: fingerprint, + }), + /checkpoint pin identity mismatch/, + testCase.name, + ) + }) + } + }) + + it("refuses a record whose bound roots do not match (foreign record)", async () => { + await withRoots(async (roots) => { + const operationId = "x-y-z-w" + // writeCalibrationRecord validates derived paths from the archiveDir, so + // write a FOREIGN dataDir directly into the record file via a manual + // write (the bound-root check happens at parse, not write). + const { writeFileSync } = await import("node:fs") + const record = { + formatVersion: 1, + phase: "intent", + operationId, + pinId: "p", + pinPurpose: calibrationPinPurpose(operationId), + pinPath: null, + checkpointId: "c", + checkpointManifestFingerprint: "c:2026:1", + boundRoots: { + dataDir: "/different/data", + archiveDir: roots.archiveDir, + scratchRoot: roots.scratchRoot, + }, + ownedPaths: { + scratchSubdir: derivedScratchSubdir(operationId), + sampleDir: derivedSampleDir(roots.archiveDir, operationId), + }, + updatedAt: new Date().toISOString(), + } + mkdirSync(join(roots.archiveDir, "calibration"), { recursive: true }) + writeFileSync(calibrationRecoveryPath(roots.archiveDir), JSON.stringify(record)) + await rejects(reconcileCalibration(roots.archiveDir, roots), /dataDir mismatch/) + }) + }) + + it("refuses a record with non-derived owned paths (rejects arbitrary deletion targets)", async () => { + await withRoots(async (roots) => { + await rejects( + writeCalibrationRecord(roots.archiveDir, { + phase: "intent", + operationId: "op-x", + pinId: "pin-x", + pinPurpose: calibrationPinPurpose("op-x"), + pinPath: null, + checkpointId: "cp-x", + checkpointManifestFingerprint: "cp:2026:1", + boundRoots: roots, + // Non-derived paths must be rejected: scratchSubdir !== calibrate-op-x. + ownedPaths: { scratchSubdir: ".", sampleDir: roots.archiveDir }, + }), + /!= derived|refusing/i, + ) + }) + }) +}) + +describe("calibration recovery — directoryTreeBytes and preflightFreeSpace", () => { + it("directoryTreeBytes sums file sizes in a tree and returns 0 for absent paths", async () => { + const dir = mkdtempSync(join(tmpdir(), "maple-tree-")) + try { + mkdirSync(join(dir, "sub"), { recursive: true }) + writeFileSync(join(dir, "a.bin"), "aaaa") + writeFileSync(join(dir, "sub", "b.bin"), "bbbbbb") + const total = await directoryTreeBytes(dir) + strictEqual(total, 10) + strictEqual(await directoryTreeBytes(join(dir, "nonexistent")), 0) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("directoryTreeBytes follows contained symlinks once and rejects escapes", async () => { + const dir = mkdtempSync(join(tmpdir(), "maple-tree-")) + const outside = mkdtempSync(join(tmpdir(), "maple-tree-outside-")) + try { + mkdirSync(join(dir, "sub"), { recursive: true }) + writeFileSync(join(dir, "sub", "data.bin"), "123456") + symlinkSync("sub", join(dir, "sub-link")) + // The directory and its contained alias identify the same physical + // inode, so the bytes are counted once. + strictEqual(await directoryTreeBytes(dir), 6) + writeFileSync(join(outside, "foreign.bin"), "outside") + symlinkSync(outside, join(dir, "escape")) + await rejects(directoryTreeBytes(dir), /symlink escapes owned root/) + } finally { + rmSync(dir, { recursive: true, force: true }) + rmSync(outside, { recursive: true, force: true }) + } + }) + + it("preflightCalibrationFreeSpace passes on a writable temp volume with a small reserve", async () => { + const dir = mkdtempSync(join(tmpdir(), "maple-fs-")) + try { + // A tiny reserve + tiny working set should pass on the temp volume. + await preflightCalibrationFreeSpace(dir, 1024, 1024) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("preflightCalibrationFreeSpace fails when the reserve+working exceeds free space", async () => { + const dir = mkdtempSync(join(tmpdir(), "maple-fs2-")) + try { + // An impossibly large requirement. + await rejects( + preflightCalibrationFreeSpace(dir, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER), + /free-space preflight failed/, + ) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +}) + +describe("config-bound create enforces environment and volume identity", () => { + /** Capture the live host's environment + the real archive-volume identity. */ + const liveEnvironment = async (archiveDir: string) => { + const cpuList = cpus() + const vol = await archiveVolumeIdentity(archiveDir) + return { + environment: { + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + executionUser: userInfo().username, + platform: platform(), + arch: arch(), + cpuModel: cpuList.length > 0 ? cpuList[0]!.model : "unknown", + cpuCount: cpuList.length, + totalMemoryBytes: totalmem(), + measurementTool: "/usr/bin/time", + archiveVolume: { ...vol, archiveDir }, + }, + } + } + + /** A minimal internally-consistent v2 config document bound to a checkpoint + archive. */ + const configDocumentFor = async ( + archiveDir: string, + checkpointId: string, + fingerprint: string, + env: Awaited>, + ) => { + const candidate = CANDIDATE_MATRIX[0]! + const metrics = baseMetrics() + const heldOutMetrics = baseMetrics({ + logicalBytes: metrics.logicalBytes * 2, + physicalBytes: metrics.physicalBytes * 2, + wallMs: metrics.wallMs * 2, + rowCount: metrics.rowCount * 2, + }) + const freeSpaceReserve = 1_000_000 + const sampleRows = metrics.rowCount + const heldOutRows = 2 * sampleRows + const rangeDate = "2026-06-01" + const trainingSample = { + checkpointId, + checkpointManifestFingerprint: fingerprint, + rangeDate, + role: "training" as const, + startRow: 0, + requestedRows: sampleRows, + rowCount: metrics.rowCount, + } + const heldOutSample = { + checkpointId, + checkpointManifestFingerprint: fingerprint, + rangeDate, + role: "held-out" as const, + startRow: sampleRows, + requestedRows: heldOutRows, + rowCount: heldOutMetrics.rowCount, + } + const effective = { + ...candidate, + targetChunkBytes: deriveTargetChunkBytes(candidate.maxShardBytes, freeSpaceReserve), + minFreeSpaceReserve: freeSpaceReserve, + } + // Every candidate/signal uses identical metrics so every recomputed worst + // case (training and held-out) equals `metrics`, keeping the document + // internally consistent for the loader's semantic re-derivation. + const results = CANDIDATE_MATRIX.flatMap((matrixCandidate) => + ARCHIVE_SIGNALS.map((signal) => ({ + candidate: matrixCandidate, + signal: signal.name, + metrics, + ok: true, + sample: trainingSample, + })), + ) + const selectedWorstCase = metrics + const heldOutResults = ARCHIVE_SIGNALS.map((signal) => ({ + candidate, + signal: signal.name, + metrics: heldOutMetrics, + ok: true, + sample: heldOutSample, + })) + const heldOutRatio = heldOutMetrics.logicalBytes / metrics.logicalBytes + const signalComparisons = ARCHIVE_SIGNALS.map((signal) => { + const comparison = comparePredictedObserved(metrics, heldOutMetrics, HELD_OUT_TOLERANCES, { + ratio: heldOutRatio, + metrics: new Set(["wallMs", "physicalBytes"]), + }) + return { + signal: signal.name, + scaleRatio: heldOutRatio, + comparisons: comparison.comparisons, + passed: comparison.passed, + } + }) + return { + formatVersion: TUNING_CONFIG_FORMAT_VERSION, + measuredAt: "2026-07-01T00:00:00.000Z", + confidence: "high" as const, + checkpoint: { checkpointId, manifestFingerprint: fingerprint }, + candidateMatrix: CANDIDATE_MATRIX, + requiredSignals: ARCHIVE_SIGNALS.map((signal) => signal.name), + budget: { + memoryBudget: 1e9, + timeBudget: 60000, + sampleRows, + maxCandidateWallMs: 30000, + minThroughputBytesPerSec: 0, + maxTempDiskBytes: 2e9, + freeSpaceReserve, + safetyMargin: 1.1, + }, + selected: { candidate, worstCase: selectedWorstCase }, + heldOut: { + results: heldOutResults, + worstCase: heldOutMetrics, + signalComparisons, + passed: true, + tolerances: HELD_OUT_TOLERANCES, + }, + heldOutAttempts: [ + { + candidate, + results: heldOutResults, + worstCase: heldOutMetrics, + signalComparisons, + passed: true, + }, + ], + environment: env.environment, + effective, + samplePolicy: { + trainingRows: sampleRows, + heldOutMultiplier: 2, + heldOutRows, + trainingWindow: `[0, ${sampleRows})`, + heldOutWindow: `[${sampleRows}, ${sampleRows + heldOutRows})`, + }, + derivation: { + minFreeSpaceReserve: "budget.freeSpaceReserve", + targetChunkBytes: + "max(4 * selected.candidate.maxShardBytes, budget.freeSpaceReserve + selected.candidate.maxShardBytes)", + }, + safetyMargin: 1.1, + recalibrationTriggers: RECALIBRATION_TRIGGERS, + results, + note: "test", + } + } + + /** Write a config doc + load it, returning a LoadedTuningConfig bound to the roots. */ + const loadedConfigFor = async ( + roots: { dataDir: string; archiveDir: string; scratchRoot: string }, + checkpointId: string, + fingerprint: string, + ): Promise<{ config: LoadedTuningConfig; dir: string }> => { + const env = await liveEnvironment(roots.archiveDir) + const doc = await configDocumentFor(roots.archiveDir, checkpointId, fingerprint, env) + const dir = mkdtempSync(join(tmpdir(), "maple-cfgenv-")) + const path = join(dir, "cfg.json") + writeFileSync(path, JSON.stringify(doc)) + const config = loadTuningConfig(path) + return { config, dir } + } + + it("rejects create before any mutation when the recorded environment mismatches the live host", async () => { + await withRoots(async (roots) => { + const checkpointId = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + const fingerprint = seedCheckpoint(roots.dataDir, checkpointId) + const { config, dir } = await loadedConfigFor(roots, checkpointId, fingerprint) + try { + // Forge a single environment field; the live host's schema differs. + ;(config.document.environment as { schemaFingerprint: string }).schemaFingerprint += "-forged" + const tuning = resolveArchiveTuning({ ...config.overrides, ...roots }) + await rejects( + createArchiveGeneration( + roots.dataDir, + roots.archiveDir, + "logs", + "2026-06-01", + tuning, + "current", + {}, + config, + ), + /calibration environment mismatch: schemaFingerprint/, + ) + // No mutation: the env check precedes intent publication. + strictEqual(listActiveOperationIds(roots.archiveDir).length, 0) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + }) + + it("rejects create before any mutation when the recorded archive volume differs", async () => { + await withRoots(async (roots) => { + const checkpointId = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + const fingerprint = seedCheckpoint(roots.dataDir, checkpointId) + const { config, dir } = await loadedConfigFor(roots, checkpointId, fingerprint) + try { + // Forge the volume device id while keeping the canonical path. + ;(config.document.environment.archiveVolume as { fsid: string }).fsid = "dev:deadbeef" + const tuning = resolveArchiveTuning({ ...config.overrides, ...roots }) + await rejects( + createArchiveGeneration( + roots.dataDir, + roots.archiveDir, + "logs", + "2026-06-01", + tuning, + "current", + {}, + config, + ), + /calibration environment mismatch: archive volume/, + ) + strictEqual(listActiveOperationIds(roots.archiveDir).length, 0) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + }) + + // NOTE: the publication-time volume re-check (beforePublicationVolumeRecheck) + // fires AFTER the full chDB export, so it is proven by the NATIVE calibrate + // probe's config-bound create step, not by this chDB-free unit harness. +}) diff --git a/apps/cli/test/archive-config.test.ts b/apps/cli/test/archive-config.test.ts new file mode 100644 index 000000000..a4bbdf301 --- /dev/null +++ b/apps/cli/test/archive-config.test.ts @@ -0,0 +1,674 @@ +import { describe, it } from "@effect/vitest" +import { deepStrictEqual, strictEqual, throws } from "node:assert" +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { + ArchiveTuningRecord, + DEFAULT_ARCHIVE_TUNING, + resolveArchiveTuning, + tuningRecord, + loadTuningConfig, + LEGACY_TUNING_CONFIG_FORMAT_VERSION, + TUNING_CONFIG_FORMAT_VERSION, +} from "../src/server/archives/config" +import { + CANDIDATE_MATRIX, + comparePredictedObserved, + HELD_OUT_TOLERANCES, + RECALIBRATION_TRIGGERS, +} from "../src/server/archives/calibrate" +import { ARCHIVE_SIGNALS } from "../src/server/archives/signals" + +const base = { archiveDir: "/tmp/archive", scratchRoot: "/tmp/scratch" } + +describe("archive tuning config", () => { + it("applies research-baseline defaults when only directories are supplied", () => { + const tuning = resolveArchiveTuning(base) + strictEqual(tuning.writerThreads, DEFAULT_ARCHIVE_TUNING.writerThreads) + strictEqual(tuning.rowGroupRows, DEFAULT_ARCHIVE_TUNING.rowGroupRows) + strictEqual(tuning.maxShardRows, DEFAULT_ARCHIVE_TUNING.maxShardRows) + strictEqual(tuning.maxShardBytes, DEFAULT_ARCHIVE_TUNING.maxShardBytes) + strictEqual(tuning.targetChunkBytes, DEFAULT_ARCHIVE_TUNING.targetChunkBytes) + strictEqual(tuning.minFreeSpaceReserve, DEFAULT_ARCHIVE_TUNING.minFreeSpaceReserve) + }) + + it("overrides individual knobs while keeping the rest at defaults", () => { + const tuning = resolveArchiveTuning({ ...base, writerThreads: 4, rowGroupRows: 50_000 }) + strictEqual(tuning.writerThreads, 4) + strictEqual(tuning.rowGroupRows, 50_000) + strictEqual(tuning.maxShardRows, DEFAULT_ARCHIVE_TUNING.maxShardRows) + }) + + it("records the effective values in a manifest-shaped tuning record", () => { + const tuning = resolveArchiveTuning({ ...base, maxShardRows: 250_000 }) + const record: ArchiveTuningRecord = tuningRecord(tuning) + deepStrictEqual(record, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 250_000, + maxShardBytes: 256 * 1024 * 1024, + targetChunkBytes: 1024 * 1024 * 1024, + minFreeSpaceReserve: 512 * 1024 * 1024, + }) + }) + + it("rejects a non-positive writer thread count", () => { + throws(() => resolveArchiveTuning({ ...base, writerThreads: 0 }), /writerThreads/) + }) + + it("rejects a fractional row-group size", () => { + throws(() => resolveArchiveTuning({ ...base, rowGroupRows: 10.5 }), /rowGroupRows/) + }) + + it("rejects a row group larger than the max shard", () => { + throws( + () => resolveArchiveTuning({ ...base, rowGroupRows: 1_000_000, maxShardRows: 500_000 }), + /rowGroupRows must not exceed maxShardRows/, + ) + }) + + it("rejects a max shard byte budget too small for one row group", () => { + throws( + () => resolveArchiveTuning({ ...base, maxShardBytes: 1024, rowGroupRows: 10_000 }), + /too small for rowGroupRows/, + ) + }) + + it("rejects a free-space reserve larger than the target chunk", () => { + throws( + () => + resolveArchiveTuning({ + ...base, + minFreeSpaceReserve: 2 * 1024 * 1024 * 1024, + targetChunkBytes: 1024 * 1024 * 1024, + }), + /minFreeSpaceReserve must be smaller than targetChunkBytes/, + ) + }) + + it("rejects an implausibly large writer thread count", () => { + throws(() => resolveArchiveTuning({ ...base, writerThreads: 100 }), /writerThreads/) + }) + + it("rejects a missing archive directory", () => { + throws(() => resolveArchiveTuning({ scratchRoot: "/tmp/scratch" }), /archive directory/) + }) + + it("rejects a missing scratch root", () => { + throws(() => resolveArchiveTuning({ archiveDir: "/tmp/archive" }), /scratch root/) + }) +}) + +describe("loadTuningConfig", () => { + /** A minimal valid calibration config document for round-trip testing. */ + const validConfigDoc = (selectedCandidateIndex = 0, prependZeroLogicalAttempt = false) => { + const candidate = CANDIDATE_MATRIX[selectedCandidateIndex]! + const metrics = { + logicalBytes: 1000, + physicalBytes: 300, + compressionRatio: 0.3, + writeThroughputBytesPerSec: 200_000, + peakTempDiskBytes: 500, + peakRssBytes: 200, + wallMs: 5, + rowCount: 1000, + } + const heldOutMetrics = { + ...metrics, + logicalBytes: 2000, + physicalBytes: 600, + wallMs: 10, + rowCount: 2000, + } + const freeSpaceReserve = 500_000_000 + const effective = { + ...candidate, + targetChunkBytes: Math.max( + 4 * candidate.maxShardBytes, + freeSpaceReserve + candidate.maxShardBytes, + ), + minFreeSpaceReserve: freeSpaceReserve, + } + const sampleRows = 1000 + const heldOutRows = 2 * sampleRows + const trainingSample = (rowCount: number) => ({ + checkpointId: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee", + checkpointManifestFingerprint: "checkpoint:fingerprint", + rangeDate: "2026-07-01", + role: "training" as const, + startRow: 0, + requestedRows: sampleRows, + rowCount, + }) + const heldOutSample = (rowCount: number) => ({ + checkpointId: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee", + checkpointManifestFingerprint: "checkpoint:fingerprint", + rangeDate: "2026-07-01", + role: "held-out" as const, + startRow: sampleRows, + requestedRows: heldOutRows, + rowCount, + }) + const results = CANDIDATE_MATRIX.flatMap((matrixCandidate, candidateIndex) => + ARCHIVE_SIGNALS.map((signal) => ({ + candidate: matrixCandidate, + signal: signal.name, + metrics: { ...metrics, peakRssBytes: 200 + candidateIndex }, + ok: true, + sample: trainingSample(1000), + })), + ) + const selectedWorstCase = { ...metrics, peakRssBytes: 200 + selectedCandidateIndex } + const heldOutResults = ARCHIVE_SIGNALS.map((signal) => ({ + candidate, + signal: signal.name, + metrics: heldOutMetrics, + ok: true, + sample: heldOutSample(2000), + })) + // Per-signal like-for-like comparison: each signal pairs training metrics + // with held-out metrics, scaled by that signal's own logical-byte ratio. + const heldOutRatio = heldOutMetrics.logicalBytes / metrics.logicalBytes + const signalComparisons = ARCHIVE_SIGNALS.map((signal) => { + const comparison = comparePredictedObserved( + selectedWorstCase, + heldOutMetrics, + HELD_OUT_TOLERANCES, + { + ratio: heldOutRatio, + metrics: new Set(["wallMs", "physicalBytes"]), + }, + ) + return { + signal: signal.name, + scaleRatio: heldOutRatio, + comparisons: comparison.comparisons, + passed: comparison.passed, + } + }) + return { + formatVersion: TUNING_CONFIG_FORMAT_VERSION, + measuredAt: "2026-07-01T00:00:00.000Z", + confidence: "high", + checkpoint: { + checkpointId: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee", + manifestFingerprint: "checkpoint:fingerprint", + }, + candidateMatrix: CANDIDATE_MATRIX, + requiredSignals: ARCHIVE_SIGNALS.map((signal) => signal.name), + budget: { + memoryBudget: 1e9, + timeBudget: 60000, + sampleRows: 1000, + maxCandidateWallMs: 30000, + minThroughputBytesPerSec: 0, + maxTempDiskBytes: 2e9, + freeSpaceReserve, + safetyMargin: 1.1, + }, + selected: { + candidate, + worstCase: selectedWorstCase, + }, + heldOut: { + results: heldOutResults, + worstCase: heldOutMetrics, + signalComparisons, + passed: true, + tolerances: HELD_OUT_TOLERANCES, + }, + heldOutAttempts: [ + ...(prependZeroLogicalAttempt + ? [ + { + candidate: CANDIDATE_MATRIX[0]!, + results: ARCHIVE_SIGNALS.map((signal) => ({ + candidate: CANDIDATE_MATRIX[0]!, + signal: signal.name, + metrics: { + ...heldOutMetrics, + logicalBytes: 0, + physicalBytes: 0, + compressionRatio: 0, + writeThroughputBytesPerSec: 0, + }, + ok: true, + sample: heldOutSample(2000), + })), + worstCase: null, + signalComparisons: [], + passed: false, + }, + ] + : []), + { + candidate, + results: heldOutResults, + worstCase: heldOutMetrics, + signalComparisons, + passed: true, + }, + ], + environment: { + mapleVersion: "x", + chdbVersion: "y", + schemaFingerprint: "z", + executionUser: "tester", + platform: "darwin", + arch: "arm64", + cpuModel: "test-cpu", + cpuCount: 8, + totalMemoryBytes: 16_000_000_000, + measurementTool: "/usr/bin/time", + archiveVolume: { fsid: "dev:1", type: 17, archiveDir: "/tmp/archive" }, + }, + effective, + samplePolicy: { + trainingRows: sampleRows, + heldOutMultiplier: 2, + heldOutRows, + trainingWindow: `[0, ${sampleRows})`, + heldOutWindow: `[${sampleRows}, ${sampleRows + heldOutRows})`, + }, + derivation: { + minFreeSpaceReserve: "budget.freeSpaceReserve", + targetChunkBytes: + "max(4 * selected.candidate.maxShardBytes, budget.freeSpaceReserve + selected.candidate.maxShardBytes)", + }, + safetyMargin: 1.1, + recalibrationTriggers: RECALIBRATION_TRIGGERS, + results, + note: "test", + } + } + + /** + * Model the actual Phase 3B shape: held-out RSS is lower than the selected + * training worst case. The evidence representation is what distinguishes a + * legacy v2 document (symmetric delta) from the repaired directional form. + */ + const makeHeldOutRssCheaper = ( + doc: ReturnType, + policy: "directional" | "symmetric", + ): void => { + for (const result of doc.heldOut.results) result.metrics.peakRssBytes = 100 + doc.heldOut.worstCase.peakRssBytes = 100 + const heldOutMetrics = doc.heldOut.results[0]!.metrics + const ratio = heldOutMetrics.logicalBytes / doc.selected.worstCase.logicalBytes + const signalComparisons = ARCHIVE_SIGNALS.map((signal) => { + const comparison = comparePredictedObserved( + doc.selected.worstCase, + heldOutMetrics, + HELD_OUT_TOLERANCES, + { ratio, metrics: new Set(["wallMs", "physicalBytes"]) }, + ) + return { + signal: signal.name, + scaleRatio: ratio, + comparisons: comparison.comparisons.map((entry) => ({ ...entry })), + passed: comparison.passed, + } + }) + if (policy === "symmetric") { + for (const signal of signalComparisons) { + const rss = signal.comparisons.find((entry) => entry.metric === "peakRssBytes")! + // 100 observed versus 200 predicted: a valid v2 two-sided delta. + rss.relativeDelta = 0.5 + } + } + doc.heldOut.signalComparisons = signalComparisons + doc.heldOutAttempts[doc.heldOutAttempts.length - 1]!.signalComparisons = signalComparisons + } + + const setConfigFormat = (doc: ReturnType, formatVersion: number): void => { + ;(doc as { formatVersion: number }).formatVersion = formatVersion + } + + const cloneSignalComparisons = (doc: ReturnType) => + doc.heldOut.signalComparisons.map((signal) => ({ + ...signal, + comparisons: signal.comparisons.map((comparison) => ({ ...comparison })), + })) + + it("round-trips an earlier non-positive-logical attempt before a later passing candidate", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-zero-logical-")) + try { + const path = join(dir, "config.json") + writeFileSync(path, JSON.stringify(validConfigDoc(1, true))) + const loaded = loadTuningConfig(path) + strictEqual(loaded.document.selected.candidate.writerThreads, CANDIDATE_MATRIX[1]!.writerThreads) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("round-trips a valid config: loads effective overrides + SHA-256 identity", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-")) + try { + const path = join(dir, "cfg.json") + const doc = validConfigDoc() + writeFileSync(path, JSON.stringify(doc)) + const { overrides, identity } = loadTuningConfig(path) + strictEqual(overrides.writerThreads, 1) + strictEqual(overrides.rowGroupRows, 10_000) + strictEqual(identity.formatVersion, TUNING_CONFIG_FORMAT_VERSION) + strictEqual(identity.configName, "cfg.json") + strictEqual(identity.sha256.length, 64) + // The SHA is stable for identical content. + const again = loadTuningConfig(path) + strictEqual(again.identity.sha256, identity.sha256) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("accepts the directional v3 evidence emitted after the calibration repair", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-directional-v3-")) + try { + const path = join(dir, "cfg.json") + const doc = validConfigDoc() + makeHeldOutRssCheaper(doc, "directional") + writeFileSync(path, JSON.stringify(doc)) + const loaded = loadTuningConfig(path) + strictEqual(loaded.identity.formatVersion, TUNING_CONFIG_FORMAT_VERSION) + strictEqual(loaded.identity.configName, "cfg.json") + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("loads valid legacy v2 symmetric evidence and records its actual identity version", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-symmetric-v2-")) + try { + const path = join(dir, "cfg.json") + const doc = validConfigDoc() + setConfigFormat(doc, LEGACY_TUNING_CONFIG_FORMAT_VERSION) + makeHeldOutRssCheaper(doc, "symmetric") + writeFileSync(path, JSON.stringify(doc)) + strictEqual(loadTuningConfig(path).identity.formatVersion, LEGACY_TUNING_CONFIG_FORMAT_VERSION) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("loads the deployed transitional v2 directional evidence and records version 2", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-directional-v2-")) + try { + const path = join(dir, "cfg.json") + const doc = validConfigDoc() + setConfigFormat(doc, LEGACY_TUNING_CONFIG_FORMAT_VERSION) + makeHeldOutRssCheaper(doc, "directional") + writeFileSync(path, JSON.stringify(doc)) + strictEqual(loadTuningConfig(path).identity.formatVersion, LEGACY_TUNING_CONFIG_FORMAT_VERSION) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("rejects a v3 document with legacy evidence and a v2 document that mixes policies", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-mixed-policy-")) + try { + const legacyInV3 = validConfigDoc() + makeHeldOutRssCheaper(legacyInV3, "symmetric") + const legacyInV3Path = join(dir, "legacy-in-v3.json") + writeFileSync(legacyInV3Path, JSON.stringify(legacyInV3)) + throws(() => loadTuningConfig(legacyInV3Path), /signalComparisons.*recomputed/i) + + const mixedV2 = validConfigDoc() + setConfigFormat(mixedV2, LEGACY_TUNING_CONFIG_FORMAT_VERSION) + makeHeldOutRssCheaper(mixedV2, "directional") + const legacyAttempt = cloneSignalComparisons(mixedV2) + for (const signal of legacyAttempt) { + signal.comparisons.find((entry) => entry.metric === "peakRssBytes")!.relativeDelta = 0.5 + } + mixedV2.heldOutAttempts[0]!.signalComparisons = legacyAttempt + const mixedV2Path = join(dir, "mixed-v2.json") + writeFileSync(mixedV2Path, JSON.stringify(mixedV2)) + throws(() => loadTuningConfig(mixedV2Path), /signalComparisons.*recomputed|semantic evidence/i) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("rejects an unknown top-level field (strict schema)", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-")) + try { + const path = join(dir, "cfg.json") + const doc = { ...validConfigDoc(), rogue: "evil" } + writeFileSync(path, JSON.stringify(doc)) + throws(() => loadTuningConfig(path), /unknown calibration config field 'rogue'/) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("rejects an unknown effective field", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-")) + try { + const path = join(dir, "cfg.json") + const doc = validConfigDoc() + ;(doc.effective as typeof doc.effective & { bogus: number }).bogus = 1 + writeFileSync(path, JSON.stringify(doc)) + throws(() => loadTuningConfig(path), /unknown calibration config effective\.bogus/) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("rejects hostile semantic rewrites even when every field remains well typed", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-semantic-")) + try { + const cases: Array<{ name: string; mutate: (doc: ReturnType) => void }> = [ + { + name: "forged-selected-worst-case", + mutate: (doc) => { + doc.selected.worstCase.peakRssBytes++ + }, + }, + { + name: "missing-training-cell", + mutate: (doc) => { + doc.results.pop() + }, + }, + { + name: "forged-held-out-comparison", + mutate: (doc) => { + doc.heldOut.signalComparisons[0]!.comparisons[0]!.withinTolerance = false + }, + }, + { + name: "forged-effective-reserve", + mutate: (doc) => { + doc.effective.minFreeSpaceReserve++ + }, + }, + { + name: "forged-derivation", + mutate: (doc) => { + doc.derivation.targetChunkBytes = "selected.maxShardBytes" as never + }, + }, + { + name: "wrong-checkpoint-shape", + mutate: (doc) => { + doc.checkpoint.manifestFingerprint = "" + }, + }, + { + name: "forged-training-scope-checkpoint", + mutate: (doc) => { + doc.results[0]!.sample!.checkpointId = "11111111-1111-4111-8111-111111111111" + }, + }, + { + name: "forged-training-scope-role", + mutate: (doc) => { + doc.results[0]!.sample!.role = "held-out" + }, + }, + { + name: "forged-held-out-scope-non-disjoint", + mutate: (doc) => { + doc.heldOut.results[0]!.sample!.startRow = 0 + }, + }, + { + name: "forged-scope-rowcount-mismatch", + mutate: (doc) => { + doc.results[0]!.sample!.rowCount = 1 + }, + }, + { + name: "forged-sample-policy-multiplier", + mutate: (doc) => { + doc.samplePolicy.heldOutMultiplier = 1 + doc.samplePolicy.heldOutRows = 1000 + doc.samplePolicy.heldOutWindow = "[1000, 2000)" + }, + }, + { + name: "forged-scale-ratio", + mutate: (doc) => { + doc.heldOut.signalComparisons[0]!.scaleRatio = 0.5 + }, + }, + { + name: "short-observed-held-out-window", + mutate: (doc) => { + for (const result of doc.heldOut.results) { + result.metrics.rowCount = doc.budget.sampleRows + result.sample.rowCount = doc.budget.sampleRows + } + }, + }, + { + name: "forged-canonical-tolerances-with-recomputed-comparisons", + mutate: (doc) => { + // The original 1000x hostile reproduction: redefine tolerances + // and recompute every per-signal comparison with them, so the + // document is internally consistent yet the loader must reject + // because the tolerance policy is not canonical. + const forged = { + peakRssBytes: 10_000, + wallMs: 10_000, + writeThroughputBytesPerSec: 10_000, + compressionRatio: 10_000, + physicalBytes: 10_000, + peakTempDiskBytes: 10_000, + } + doc.heldOut.tolerances = forged as typeof doc.heldOut.tolerances + const trainingMetrics = doc.results[0]!.metrics! + const heldOutMetrics = doc.heldOut.results[0]!.metrics! + const ratio = heldOutMetrics.logicalBytes / trainingMetrics.logicalBytes + const recomputed = ARCHIVE_SIGNALS.map((signal) => { + const comparison = comparePredictedObserved( + trainingMetrics, + heldOutMetrics, + forged, + { ratio, metrics: new Set(["wallMs", "physicalBytes"]) }, + ) + return { + signal: signal.name, + scaleRatio: ratio, + comparisons: comparison.comparisons, + passed: comparison.passed, + } + }) + doc.heldOut.signalComparisons = recomputed + doc.heldOutAttempts[0]!.signalComparisons = recomputed + }, + }, + ] + for (const testCase of cases) { + const doc = validConfigDoc() + testCase.mutate(doc) + const path = join(dir, `${testCase.name}.json`) + writeFileSync(path, JSON.stringify(doc)) + throws(() => loadTuningConfig(path), /invalid|missing|recomputed|derivation/i) + } + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("rejects malformed nested selected/results/metrics evidence and non-ISO timestamps", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-")) + try { + const cases: Array<{ name: string; mutate: (doc: ReturnType) => void }> = [ + { + name: "missing-worst-case", + mutate: (doc) => { + doc.selected = { candidate: doc.selected.candidate } as typeof doc.selected + }, + }, + { + name: "invalid-result-metrics", + mutate: (doc) => { + doc.results[0]!.metrics = "garbage" as never + doc.results[0]!.ok = true + }, + }, + { + name: "invalid-result-candidate", + mutate: (doc) => { + doc.results[0]!.candidate = null as never + }, + }, + { + name: "non-iso-time", + mutate: (doc) => { + doc.measuredAt = "not-an-ISO-timestamp" + }, + }, + ] + for (const testCase of cases) { + const doc = validConfigDoc() + testCase.mutate(doc) + const path = join(dir, `${testCase.name}.json`) + writeFileSync(path, JSON.stringify(doc)) + throws(() => loadTuningConfig(path), /invalid|missing/i) + } + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("rejects an unsupported formatVersion", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-")) + try { + const path = join(dir, "cfg.json") + writeFileSync(path, JSON.stringify({ ...validConfigDoc(), formatVersion: 99 })) + throws(() => loadTuningConfig(path), /unsupported calibration config formatVersion/) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("refuses a non-regular file (symlink) — one-fd regular-file check", () => { + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-")) + try { + const real = join(dir, "real.json") + const link = join(dir, "link.json") + writeFileSync(real, JSON.stringify(validConfigDoc())) + symlinkSync(real, link) + throws(() => loadTuningConfig(link), /regular file/) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("rejects an unsafe config name (path-like basename)", () => { + // A basename with a slash is not possible as a single path segment; test a + // name that fails the safe-name regex (e.g. contains a space). + const dir = mkdtempSync(join(tmpdir(), "maple-loadcfg-")) + try { + const path = join(dir, "bad name.json") + writeFileSync(path, JSON.stringify(validConfigDoc())) + throws(() => loadTuningConfig(path), /unsafe calibration config name/) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/apps/cli/test/archive-export-round5.test.ts b/apps/cli/test/archive-export-round5.test.ts new file mode 100644 index 000000000..924edae6b --- /dev/null +++ b/apps/cli/test/archive-export-round5.test.ts @@ -0,0 +1,248 @@ +import { describe, it } from "@effect/vitest" +import { deepStrictEqual, throws } from "node:assert" +import { randomUUID } from "node:crypto" +import { + normalizeType, + planHourShards, + COMPLEX_DIGEST_ALGORITHM, + type ExportSettings, +} from "../src/server/archives/export" +import { parseArchiveGenerationManifest } from "../src/server/archives/manifest" +import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +import { SCHEMA_FINGERPRINT } from "../src/server/serve" + +// Pure-logic tests for the round-5 fixes. The end-to-end adversarial coverage +// (column swap, row reassociation, byte refinement, timezone, recovery) lives +// in apps/cli/test/probes/ and the matrix in archive-adversarial-matrix.md. + +const settings = (overrides: Partial = {}): ExportSettings => ({ + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + ...overrides, +}) + +describe("schema normalizeType (measured chDB Parquet mapping)", () => { + it("does NOT collapse parameterized array element types", () => { + if (normalizeType("Array(UInt64)") === normalizeType("Array(String)")) { + throw new Error("Array(UInt64) collapsed to Array(String)") + } + }) + + it("applies the measured LowCardinality/DateTime/DateTime64/Map transforms", () => { + deepStrictEqual(normalizeType("LowCardinality(String)"), "String") + deepStrictEqual(normalizeType("LowCardinality(LowCardinality(String))"), "String") + deepStrictEqual(normalizeType("Map(LowCardinality(String), String)"), "Map(String, String)") + deepStrictEqual(normalizeType("DateTime"), "DateTime64(3, 'UTC')") + deepStrictEqual(normalizeType("DateTime64(9)"), "DateTime64(9, 'UTC')") + deepStrictEqual(normalizeType("Array(DateTime64(9))"), "Array(DateTime64(9, 'UTC'))") + deepStrictEqual( + normalizeType("Array(Map(LowCardinality(String), String))"), + "Array(Map(String, String))", + ) + deepStrictEqual(normalizeType("Nullable(Float64)"), "Nullable(Float64)") + }) + + it("leaves simple leaf types untouched", () => { + for (const t of ["String", "UInt8", "UInt64", "Int32", "Float64", "Bool"]) { + deepStrictEqual(normalizeType(t), t) + } + }) + + it("makes source vs Parquet normalized forms equal for every measured round-trip", () => { + const pairs: ReadonlyArray<[string, string]> = [ + ["LowCardinality(String)", "String"], + ["DateTime", "DateTime64(3, 'UTC')"], + ["DateTime64(9)", "DateTime64(9, 'UTC')"], + ["Map(LowCardinality(String), String)", "Map(String, String)"], + ["Array(UInt64)", "Array(UInt64)"], + ["Array(String)", "Array(String)"], + ] + for (const [src, par] of pairs) { + if (normalizeType(src) !== normalizeType(par)) { + throw new Error(`normalized source ${src} must equal normalized parquet ${par}`) + } + } + }) +}) + +describe("planHourShards — per-part physical offset ranges (no ORDER BY)", () => { + it("splits a part's offset domain into maxShardRows-width half-open ranges", () => { + // One part, offset domain [0, 1199], maxShardRows 500 -> 3 ranges. + const parts = [{ part: "p1", offsetMin: 0, offsetMax: 1199, matchingRows: 1200 }] + const plans = planHourShards(12, parts, settings({ maxShardRows: 500 })) + deepStrictEqual(plans.length, 3) + // Half-open ranges covering [0, 1200): [0,500),[500,1000),[1000,1200) + deepStrictEqual( + plans.map((p) => [p.range.offsetLo, p.range.offsetHiExclusive]), + [ + [0, 500], + [500, 1000], + [1000, 1200], + ], + ) + deepStrictEqual( + plans.map((p) => p.range.part), + ["p1", "p1", "p1"], + ) + // Each range's physical width <= maxShardRows. + for (const p of plans) { + if (p.range.offsetHiExclusive - p.range.offsetLo > 500) { + throw new Error(`range exceeds maxShardRows: ${JSON.stringify(p.range)}`) + } + } + }) + + it("produces one range when the part fits in one shard", () => { + const parts = [{ part: "p1", offsetMin: 0, offsetMax: 99, matchingRows: 100 }] + const plans = planHourShards(12, parts, settings()) + deepStrictEqual(plans.length, 1) + deepStrictEqual([plans[0]!.range.offsetLo, plans[0]!.range.offsetHiExclusive], [0, 100]) + }) + + it("handles multiple parts, paging each independently", () => { + const parts = [ + { part: "a", offsetMin: 0, offsetMax: 999, matchingRows: 1000 }, + { part: "b", offsetMin: 0, offsetMax: 499, matchingRows: 500 }, + ] + const plans = planHourShards(12, parts, settings({ maxShardRows: 500 })) + // part a -> [0,500),[500,1000); part b -> [0,500) + deepStrictEqual( + plans.map((p) => [p.range.part, p.range.offsetLo, p.range.offsetHiExclusive]), + [ + ["a", 0, 500], + ["a", 500, 1000], + ["b", 0, 500], + ], + ) + }) +}) + +// Build a minimal valid v2 manifest with one shard, for the time-evidence tests. +const nano = (iso: string): string => `${BigInt(Date.parse(iso)) * 1_000_000n}` +const manifestWith = ( + overrides: Record, + shardOverrides: Record = {}, +): Record => ({ + formatVersion: 3, + generationId: randomUUID(), + signal: "traces", + rangeStart: "2026-06-29", + rangeEndExclusive: "2026-06-30T00:00:00.000Z", + checkpointId: randomUUID(), + checkpointManifestFingerprint: "fp", + createdAt: "2026-06-29T12:00:00.000Z", + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + sourceRowCount: 1, + archivedRowCount: 1, + tuning: { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + targetChunkBytes: 1024 * 1024 * 1024, + minFreeSpaceReserve: 512 * 1024 * 1024, + }, + tuningConfig: null, + shards: [ + { + name: "12-0000.parquet", + rowCount: 1, + minEventTimeUnixNano: nano("2026-06-29T12:00:00.000Z"), + maxEventTimeUnixNano: nano("2026-06-29T12:30:00.000Z"), + sha256: "a".repeat(64), + bytes: 4096, + columns: ["Timestamp"], + complexDigest: "123456789", + complexDigestAlgorithm: COMPLEX_DIGEST_ALGORITHM, + ...shardOverrides, + }, + ], + ...overrides, +}) + +describe("shard time evidence — UTC nanoseconds, host-timezone-independent (round 5)", () => { + it("accepts a valid in-range shard bound", () => { + parseArchiveGenerationManifest(manifestWith({}), "traces", "2026-06-29") // must not throw + }) + + it("accepts a valid 23:30 UTC late-day bound (the round-4 timezone defect)", () => { + // Under any host timezone this must parse, because bounds are epoch nanos. + parseArchiveGenerationManifest( + manifestWith( + {}, + { + minEventTimeUnixNano: nano("2026-06-29T23:30:00.000Z"), + maxEventTimeUnixNano: nano("2026-06-29T23:30:00.000Z"), + }, + ), + "traces", + "2026-06-29", + ) + }) + + it("rejects an out-of-range (2027) shard bound", () => { + throws( + () => + parseArchiveGenerationManifest( + manifestWith( + {}, + { + minEventTimeUnixNano: nano("2027-01-01T00:00:00.000Z"), + maxEventTimeUnixNano: nano("2027-01-01T00:00:00.000Z"), + }, + ), + "traces", + "2026-06-29", + ), + /outside sealed range/, + ) + }) + + it("rejects a shard reaching the exclusive range end (next midnight)", () => { + throws( + () => + parseArchiveGenerationManifest( + manifestWith({}, { maxEventTimeUnixNano: nano("2026-06-30T00:00:00.000Z") }), + "traces", + "2026-06-29", + ), + /outside sealed range/, + ) + }) + + it("rejects a missing complexDigestAlgorithm (round 5 manifest field)", () => { + const m = manifestWith({}) + delete (m.shards as Array>)[0]!.complexDigestAlgorithm + throws(() => parseArchiveGenerationManifest(m, "traces", "2026-06-29"), /complexDigestAlgorithm/) + }) + + it("rejects an unknown complexDigestAlgorithm fail-closed (round 5)", () => { + // v2 made digest semantics versioned; an unknown algorithm must not be + // silently re-interpreted (a v1 digest, a future v3, or a bogus string). + throws( + () => + parseArchiveGenerationManifest( + manifestWith({}, { complexDigestAlgorithm: "bogus-digest" }), + "traces", + "2026-06-29", + ), + /invalid archive shard complexDigestAlgorithm: bogus-digest/, + ) + }) + + it("rejects a v1 manifest fail-closed (round 5 version bump)", () => { + throws( + () => + parseArchiveGenerationManifest( + { ...manifestWith({}), formatVersion: 1 }, + "traces", + "2026-06-29", + ), + /unsupported archive manifest formatVersion 1/, + ) + }) +}) diff --git a/apps/cli/test/archive-export-safety.test.ts b/apps/cli/test/archive-export-safety.test.ts new file mode 100644 index 000000000..c8bdea3e1 --- /dev/null +++ b/apps/cli/test/archive-export-safety.test.ts @@ -0,0 +1,39 @@ +import { describe, it } from "@effect/vitest" +import { throws } from "node:assert" +import { exportSignalShards } from "../src/server/archives/export" + +// Pure-logic tests for export safety primitives that don't require chDB. +// The full Parquet write+reopen+validation path is exercised by the native +// archive smoke (Gate 5). These tests pin the safety invariants: path +// rejection and SQL escaping. + +describe("export path safety (M-1)", () => { + it("rejects a shardsDir containing a single quote", () => { + throws( + // A null db is never reached: assertSafePath runs before any query. + () => + exportSignalShards( + null as never, + { name: "traces", eventTimeColumn: "Timestamp" } as never, + "2026-06-01", + "/tmp/o'clock", + { writerThreads: 1, rowGroupRows: 100, maxShardRows: 10, maxShardBytes: 1024 } as never, + ), + /single quote/, + ) + }) + + it("rejects a shardsDir containing a backslash", () => { + throws( + () => + exportSignalShards( + null as never, + { name: "traces", eventTimeColumn: "Timestamp" } as never, + "2026-06-01", + "/tmp/back\\slash", + { writerThreads: 1, rowGroupRows: 100, maxShardRows: 10, maxShardBytes: 1024 } as never, + ), + /backslash/, + ) + }) +}) diff --git a/apps/cli/test/archive-gc.test.ts b/apps/cli/test/archive-gc.test.ts new file mode 100644 index 000000000..525b6f734 --- /dev/null +++ b/apps/cli/test/archive-gc.test.ts @@ -0,0 +1,464 @@ +import { describe, it } from "@effect/vitest" +import { ok, rejects, strictEqual } from "node:assert" +import { createHash, randomUUID } from "node:crypto" +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs" +import { tmpdir } from "node:os" +import { dirname, join } from "node:path" +import { + activePointerPath, + buildingGenerationRoot, + generationManifestPath, + generationRoot, + nextMidnightUtc, +} from "../src/server/archives/paths" +import { promoteGeneration, selectActiveGeneration } from "../src/server/archives/generation" +import { type ArchiveGenerationManifest, parseArchiveActivePointer } from "../src/server/archives/manifest" +import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +import { SCHEMA_FINGERPRINT } from "../src/server/serve" +import { rebuildCatalog } from "../src/server/archives/listing" +import { planArchiveGc, runArchiveGc, verifyCompletedGcInvariants } from "../src/server/archives/gc" +import { writeInitialIntent, type GcOperationIntent } from "../src/server/archives/journal" + +// Hostile unit tests for archive garbage collection (Gate 3b). These cover the +// deterministic deletion-set logic, keep-N retention, fail-closed on +// malformed/symlinked/ambiguous state, signal-level exclusion, and dry-run +// mutates-nothing. The AUTHORITATIVE interrupted-GC crash oracle is the native +// SIGKILL probe; these unit tests cover the invariants the harness does not +// isolate. + +const withArchive = async (run: (archiveDir: string) => Promise | void): Promise => { + const parent = realpathSync(mkdtempSync(join(tmpdir(), "maple-gc-test-"))) + const archiveDir = join(parent, "archive") + const dataDir = join(parent, "data") + const scratchRoot = join(parent, "scratch") + mkdirSync(archiveDir, { recursive: true }) + mkdirSync(dataDir, { recursive: true }) + mkdirSync(scratchRoot, { recursive: true }) + try { + await run(archiveDir, dataDir, scratchRoot, parent) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const sha256 = (s: string): string => createHash("sha256").update(s).digest("hex") + +/** + * Seed a published, GC-verifiable generation: a real shard file whose bytes + + * SHA-256 match the manifest, promoted to its final location with the active + * pointer optionally selecting it. GC verifies manifest + per-shard SHA, so the + * seeded evidence must be internally consistent. + */ +const seedPublishedGeneration = async ( + archiveDir: string, + opts: { + signal?: string + rangeDate?: string + createdAt: string + shardContents?: string + selectActive?: boolean + }, +): Promise<{ generationId: string; manifestSha256: string }> => { + const generationId = randomUUID() + const signal = opts.signal ?? "traces" + const rangeDate = opts.rangeDate ?? "2026-06-01" + const shardContents = opts.shardContents ?? `PAR1-${generationId}` + const building = buildingGenerationRoot(archiveDir, generationId) + const shardsDir = join(building, "shards") + mkdirSync(shardsDir, { recursive: true }) + writeFileSync(join(shardsDir, "00.parquet"), shardContents) + // Event-time bounds must fall within the sealed UTC day [rangeStart, nextMidnight). + const noonNano = `${BigInt(Date.parse(`${rangeDate}T12:00:00.000Z`)) * 1_000_000n}` + const manifest: ArchiveGenerationManifest = { + formatVersion: 3, + generationId, + signal, + rangeStart: rangeDate, + rangeEndExclusive: nextMidnightUtc(rangeDate), + checkpointId: randomUUID(), + checkpointManifestFingerprint: `cid:${rangeDate}:100`, + createdAt: opts.createdAt, + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + sourceRowCount: 1, + archivedRowCount: 1, + tuning: { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + targetChunkBytes: 1024 * 1024 * 1024, + minFreeSpaceReserve: 512 * 1024 * 1024, + }, + tuningConfig: null, + shards: [ + { + name: "00.parquet", + rowCount: 1, + minEventTimeUnixNano: noonNano, + maxEventTimeUnixNano: noonNano, + sha256: sha256(shardContents), + bytes: shardContents.length, + columns: ["TimestampTime", "ServiceName"], + complexDigest: "0", + complexDigestAlgorithm: "cityhash64-multiset-v3", + }, + ], + } + await promoteGeneration(archiveDir, signal, rangeDate, generationId, manifest, building) + const manifestSha256 = createHash("sha256") + .update(JSON.stringify({ ...manifest, createdAt: opts.createdAt })) + .digest("hex") + if (opts.selectActive) { + await selectActiveGeneration(archiveDir, signal, rangeDate, generationId, null) + } + return { generationId, manifestSha256 } +} + +const rebuildSignalCatalog = async (archiveDir: string, signal: string): Promise => { + await rebuildCatalog(archiveDir, signal as never) +} + +describe("archive gc planning", () => { + it("keep=0 targets all superseded generations and retains only the active", async () => { + await withArchive(async (archiveDir, dataDir, scratchRoot) => { + const old = await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-02T00:00:00.000Z" }) + const mid = await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-03T00:00:00.000Z" }) + const active = await seedPublishedGeneration(archiveDir, { + createdAt: "2026-06-04T00:00:00.000Z", + selectActive: true, + }) + await rebuildSignalCatalog(archiveDir, "traces") + const plan = planArchiveGc(archiveDir, 0) + strictEqual(plan.deleteSet.length, 2) + strictEqual( + plan.deleteSet.some((c) => c.generationId === old.generationId), + true, + ) + strictEqual( + plan.deleteSet.some((c) => c.generationId === mid.generationId), + true, + ) + strictEqual( + plan.deleteSet.some((c) => c.generationId === active.generationId), + false, + ) + strictEqual( + plan.retained.some((r) => r.generationId === active.generationId && r.reason === "active"), + true, + ) + void dataDir + void scratchRoot + }) + }) + + it("keep=1 retains the newest superseded and targets only older ones", async () => { + await withArchive(async (archiveDir) => { + const oldest = await seedPublishedGeneration(archiveDir, { + createdAt: "2026-06-02T00:00:00.000Z", + }) + const newer = await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-03T00:00:00.000Z" }) + await seedPublishedGeneration(archiveDir, { + createdAt: "2026-06-04T00:00:00.000Z", + selectActive: true, + }) + await rebuildSignalCatalog(archiveDir, "traces") + const plan = planArchiveGc(archiveDir, 1) + strictEqual(plan.deleteSet.length, 1) + strictEqual(plan.deleteSet[0]!.generationId, oldest.generationId) + strictEqual( + plan.retained.some((r) => r.generationId === newer.generationId && r.reason === "kept"), + true, + ) + }) + }) + + it("over-retains a range with no active pointer (missing pointer is uncertain)", async () => { + await withArchive(async (archiveDir) => { + // No pointer at all: every generation is ambiguous (none is provably + // active). A missing pointer is uncertain state — the range is excluded + // entirely and nothing is targeted (blocker 4: never encode absence as + // an invalid empty-string sentinel that strands the journal). + await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-02T00:00:00.000Z" }) + await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-03T00:00:00.000Z" }) + await rebuildSignalCatalog(archiveDir, "traces") + const plan = planArchiveGc(archiveDir, 0) + strictEqual(plan.deleteSet.length, 0, "nothing targeted without an active pointer") + ok( + plan.excludedRanges.some((r) => r.rangeStart === "2026-06-01"), + "range excluded as uncertain", + ) + }) + }) + + it("excludes a range whose active pointer is ambiguous/malformed", async () => { + await withArchive(async (archiveDir) => { + const old = await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-02T00:00:00.000Z" }) + await seedPublishedGeneration(archiveDir, { + createdAt: "2026-06-03T00:00:00.000Z", + selectActive: true, + }) + await rebuildSignalCatalog(archiveDir, "traces") + // Corrupt the pointer so it no longer matches its location. + const pointerPath = activePointerPath(archiveDir, "traces", "2026-06-01") + writeFileSync( + pointerPath, + JSON.stringify({ + formatVersion: 1, + generationId: randomUUID(), + signal: "logs", + rangeStart: "2026-06-01", + }), + ) + const plan = planArchiveGc(archiveDir, 0) + // The range is excluded (ambiguous pointer); neither gen is targeted. + strictEqual(plan.deleteSet.length, 0) + ok(plan.excludedRanges.length >= 1, "range excluded for ambiguous pointer") + void old + }) + }) + + it("excludes an entire signal when a generation manifest is malformed", async () => { + await withArchive(async (archiveDir) => { + await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-02T00:00:00.000Z" }) + const active = await seedPublishedGeneration(archiveDir, { + createdAt: "2026-06-03T00:00:00.000Z", + selectActive: true, + }) + // Tamper the active generation's manifest so catalog reconstruction fails. + const manifestPath = generationManifestPath( + archiveDir, + "traces", + "2026-06-01", + active.generationId, + ) + writeFileSync(manifestPath, "{not valid json") + const plan = planArchiveGc(archiveDir, 0) + // Signal excluded entirely; nothing deleted. + strictEqual(plan.deleteSet.length, 0) + ok( + plan.excludedSignals.length >= 1 || plan.excludedRanges.length >= 1, + "signal/range excluded for malformed state", + ) + }) + }) + + it("rejects an invalid keep value", async () => { + await withArchive(async (archiveDir) => { + await rejects(async () => planArchiveGc(archiveDir, -1), /invalid gc keep/) + await rejects(async () => planArchiveGc(archiveDir, 1.5), /invalid gc keep/) + }) + }) +}) + +describe("archive gc execution", () => { + it("dry-run mutates nothing but reports the delete set", async () => { + await withArchive(async (archiveDir, dataDir, scratchRoot) => { + const old = await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-02T00:00:00.000Z" }) + await seedPublishedGeneration(archiveDir, { + createdAt: "2026-06-03T00:00:00.000Z", + selectActive: true, + }) + await rebuildSignalCatalog(archiveDir, "traces") + const oldGenPath = generationRoot(archiveDir, "traces", "2026-06-01", old.generationId) + ok(existsSync(oldGenPath), "old gen exists before dry-run") + const result = await runArchiveGc({ dataDir, archiveDir, scratchRoot, keep: 0, dryRun: true }) + strictEqual(result.plan.deleteSet.length, 1) + strictEqual(result.deleted.length, 0) + ok(existsSync(oldGenPath), "old gen STILL exists after dry-run (nothing mutated)") + }) + }) + + it("apply deletes the targeted superseded generation and keeps the active", async () => { + await withArchive(async (archiveDir, dataDir, scratchRoot) => { + const old = await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-02T00:00:00.000Z" }) + const active = await seedPublishedGeneration(archiveDir, { + createdAt: "2026-06-03T00:00:00.000Z", + selectActive: true, + }) + await rebuildSignalCatalog(archiveDir, "traces") + const result = await runArchiveGc({ dataDir, archiveDir, scratchRoot, keep: 0, dryRun: false }) + strictEqual(result.deleted.length, 1) + ok( + !existsSync(generationRoot(archiveDir, "traces", "2026-06-01", old.generationId)), + "old gen deleted", + ) + ok( + existsSync(generationRoot(archiveDir, "traces", "2026-06-01", active.generationId)), + "active gen retained", + ) + // The pointer still selects the active generation. + const pointer = parseArchiveActivePointer( + JSON.parse( + require("node:fs").readFileSync( + activePointerPath(archiveDir, "traces", "2026-06-01"), + "utf8", + ), + ), + "traces", + "2026-06-01", + ) + strictEqual(pointer.generationId, active.generationId) + }) + }) + + it("GC leaves no active operation journal after completing", async () => { + await withArchive(async (archiveDir, dataDir, scratchRoot) => { + await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-02T00:00:00.000Z" }) + await seedPublishedGeneration(archiveDir, { + createdAt: "2026-06-03T00:00:00.000Z", + selectActive: true, + }) + await rebuildSignalCatalog(archiveDir, "traces") + await runArchiveGc({ dataDir, archiveDir, scratchRoot, keep: 0, dryRun: false }) + // No active operation entries (the op dir is moved to completed/; the + // empty active/ parent may remain but must hold no operation dirs). + const activeDir = join(archiveDir, "operations", "active") + const activeCount = existsSync(activeDir) ? readdirSync(activeDir).length : 0 + strictEqual(activeCount, 0, "no active op journal after gc") + }) + }) +}) + +describe("archive gc dry-run nonmutation with an active operation (Gate 3b repair)", () => { + // A snapshot of the durable state (journal/pins/pointers/catalogs/generations) + // to compare before/after a dry-run that must not mutate. + const snapshot = (archiveDir: string, dataDir: string): string => { + const { execSync } = require("node:child_process") as typeof import("node:child_process") + try { + return execSync( + `find "${archiveDir}" "${dataDir}" -type f 2>/dev/null | sort | xargs shasum -a 256 2>/dev/null | shasum -a 256`, + { encoding: "utf8" }, + ).trim() + } catch { + return "snapshot-failed" + } + } + + it("gc --dry-run mutates nothing even when an active operation is present", async () => { + await withArchive(async (archiveDir, dataDir, scratchRoot) => { + await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-02T00:00:00.000Z" }) + await seedPublishedGeneration(archiveDir, { + createdAt: "2026-06-03T00:00:00.000Z", + selectActive: true, + }) + await rebuildSignalCatalog(archiveDir, "traces") + // Seed an active CREATE operation journal (an interrupted op present). + const op = randomUUID() + await writeInitialIntent({ + archiveDir, + operationId: op, + generationId: randomUUID(), + signal: "traces", + rangeStart: "2026-06-01", + checkpointId: randomUUID(), + dataDir, + scratchRoot, + pinId: randomUUID(), + pinPurpose: `archive:${op}`, + scratchSubdir: `archive-${op}`, + baseActiveGenerationId: null, + }) + const before = snapshot(archiveDir, dataDir) + // dry-run must NOT reconcile the active op or delete anything. + const result = await runArchiveGc({ dataDir, archiveDir, scratchRoot, keep: 0, dryRun: true }) + const after = snapshot(archiveDir, dataDir) + strictEqual(before, after, "dry-run mutated durable state with an active op present") + // With an active op present, dry-run reports the blocker (no deletion set predicted). + strictEqual(result.deleted.length, 0) + }) + }) + + it("gc --dry-run mutates nothing on a clean archive", async () => { + await withArchive(async (archiveDir, dataDir, scratchRoot) => { + await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-02T00:00:00.000Z" }) + await seedPublishedGeneration(archiveDir, { + createdAt: "2026-06-03T00:00:00.000Z", + selectActive: true, + }) + await rebuildSignalCatalog(archiveDir, "traces") + const before = snapshot(archiveDir, dataDir) + await runArchiveGc({ dataDir, archiveDir, scratchRoot, keep: 0, dryRun: true }) + const after = snapshot(archiveDir, dataDir) + strictEqual(before, after, "dry-run mutated durable state on a clean archive") + }) + }) +}) + +describe("archive gc terminal invariant verification (Gate 3b repair)", () => { + it("verifyCompletedGcInvariants fails when a frozen target source still exists", async () => { + await withArchive(async (archiveDir) => { + const active = await seedPublishedGeneration(archiveDir, { + createdAt: "2026-06-03T00:00:00.000Z", + selectActive: true, + }) + const target = { + signal: "traces", + rangeStart: "2026-06-01", + generationId: active.generationId, + createdAt: "2026-06-03T00:00:00.000Z", + manifestSha256: "a".repeat(64), + bytes: 100, + shards: [{ name: "00.parquet", bytes: 100, sha256: "b".repeat(64) }], + recordedActiveGenerationId: active.generationId, + } + const intent = { + kind: "gc" as const, + operationId: randomUUID(), + keep: 0, + targets: [target], + completedTargets: 1, + formatVersion: 3 as const, + archiveDir, + dataDir: "/d", + scratchRoot: "/s", + phase: "complete" as const, + createdAt: "x", + updatedAt: "x", + } as GcOperationIntent + // The target source (the active gen) still exists → must fail closed. + await rejects( + async () => verifyCompletedGcInvariants(archiveDir, intent), + /target source still exists/, + ) + }) + }) + + it("verifyCompletedGcInvariants fails when completedTargets !== targets.length", async () => { + await withArchive(async (archiveDir) => { + const intent = { + kind: "gc" as const, + operationId: randomUUID(), + keep: 0, + targets: [], + completedTargets: 1, + formatVersion: 3 as const, + archiveDir, + dataDir: "/d", + scratchRoot: "/s", + phase: "complete" as const, + createdAt: "x", + updatedAt: "x", + } as GcOperationIntent + await rejects( + async () => verifyCompletedGcInvariants(archiveDir, intent), + /completedTargets === targets.length/, + ) + }) + }) +}) + +void dirname +void symlinkSync +void readdirSync +void rmSync diff --git a/apps/cli/test/archive-generation.test.ts b/apps/cli/test/archive-generation.test.ts new file mode 100644 index 000000000..5512ef928 --- /dev/null +++ b/apps/cli/test/archive-generation.test.ts @@ -0,0 +1,742 @@ +import { describe, it } from "@effect/vitest" +import { ok, rejects, strictEqual, throws } from "node:assert" +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { createHash, randomUUID } from "node:crypto" +import { statSync } from "node:fs" +import { + activePointerPath, + buildingGenerationRoot, + catalogPath, + generationRoot, + generationManifestPath, + shardsRoot, +} from "../src/server/archives/paths" +import { parseArchiveActivePointer, type ArchiveGenerationManifest } from "../src/server/archives/manifest" +import { + assertArchiveScratchFreeSpace, + appendCatalog, + createArchiveGeneration, + promoteGeneration, + reconcileArchiveGeneration, + selectActiveGeneration, +} from "../src/server/archives/generation" +import { + advancePhase, + listActiveOperationIds, + operationDir, + writeInitialIntent, + type ArchiveOperationPhase, +} from "../src/server/archives/journal" +import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +import { SCHEMA_FINGERPRINT } from "../src/server/serve" +import { + checkpointPinsRoot, + checkpointRoot, + checkpointSnapshotDir, + checkpointStatePath, +} from "../src/server/checkpoints" +import { assertCatalogExact, rebuildCatalog } from "../src/server/archives/listing" + +// Filesystem-level tests for generation promotion, supersession, and catalog +// append. These exercise the durable state machine without a restored chDB; the +// full export path is covered by the native smoke script. + +const withArchive = async (run: (archiveDir: string) => Promise | void): Promise => { + const parent = realpathSync(mkdtempSync(join(tmpdir(), "maple-archive-gen-test-"))) + const archiveDir = join(parent, "archive") + mkdirSync(archiveDir, { recursive: true }) + try { + await run(archiveDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const seedCheckpoint = (dataDir: string, checkpointId: string, backupBytes = 6): void => { + const createdAt = "2026-01-01T00:00:00.000Z" + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + mkdirSync(join(snapshot, "backup"), { recursive: true }) + writeFileSync(join(snapshot, "backup", "data.bin"), "x".repeat(backupBytes)) + writeFileSync( + join(snapshot, "manifest.json"), + `${JSON.stringify({ + formatVersion: 1, + checkpointId, + operationId: randomUUID(), + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + createdAt, + sourceDataDir: dataDir, + backupRelativePath: `snapshots/${checkpointId}/backup`, + backupBytes, + validation: { + validatedAt: createdAt, + traces: 0, + logs: 0, + metricsSum: 0, + metricsGauge: 0, + metricsHistogram: 0, + metricsExponentialHistogram: 0, + materializedViews: 0, + }, + })}\n`, + ) + mkdirSync(checkpointRoot(dataDir), { recursive: true }) + writeFileSync( + checkpointStatePath(dataDir), + `${JSON.stringify({ + formatVersion: 1, + revision: randomUUID(), + current: checkpointId, + previous: null, + committedAt: createdAt, + })}\n`, + ) +} + +const manifest = ( + generationId: string, + signal = "traces", + archivedRowCount = 10, +): ArchiveGenerationManifest => ({ + formatVersion: 3, + generationId, + signal, + rangeStart: "2026-06-01", + rangeEndExclusive: "2026-06-02T00:00:00.000Z", + checkpointId: randomUUID(), + checkpointManifestFingerprint: "cid:2026-01-01:100", + createdAt: "2026-06-02T00:00:00.000Z", + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + sourceRowCount: archivedRowCount, + archivedRowCount, + tuning: { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + targetChunkBytes: 1024 * 1024 * 1024, + minFreeSpaceReserve: 512 * 1024 * 1024, + }, + tuningConfig: null, + shards: [ + { + name: "00.parquet", + rowCount: archivedRowCount, + minEventTimeUnixNano: `${BigInt(Date.parse("2026-06-01T00:00:00.000Z")) * 1_000_000n}`, + maxEventTimeUnixNano: `${BigInt(Date.parse("2026-06-01T00:30:00.000Z")) * 1_000_000n}`, + sha256: "a".repeat(64), + bytes: 4096, + columns: ["TimestampTime", "ServiceName"], + complexDigest: "123456789", + complexDigestAlgorithm: "cityhash64-multiset-v3", + }, + ], +}) + +/** Build a fake building generation with a shards dir and a placeholder shard. */ +const seedBuilding = (archiveDir: string, generationId: string): string => { + const building = buildingGenerationRoot(archiveDir, generationId) + const shards = join(building, "shards") + mkdirSync(shards, { recursive: true }) + writeFileSync(join(shards, "00.parquet"), "PAR1-placeholder") + return building +} + +const seedPublishedOperation = async ( + archiveDir: string, + phase: ArchiveOperationPhase, + options: { pointer?: boolean; catalog?: boolean } = {}, +) => { + const parent = join(archiveDir, "..") + const dataDir = join(parent, "data") + const scratchRoot = join(parent, "scratch") + mkdirSync(dataDir, { recursive: true }) + mkdirSync(scratchRoot, { recursive: true }) + const operationId = randomUUID() + const generationId = randomUUID() + const checkpointId = randomUUID() + const pinId = randomUUID() + const building = seedBuilding(archiveDir, generationId) + const shardPath = join(building, "shards", "00.parquet") + const baseManifest = manifest(generationId) + const exactManifest: ArchiveGenerationManifest = { + ...baseManifest, + checkpointId, + shards: [ + { + ...baseManifest.shards[0]!, + bytes: statSync(shardPath).size, + sha256: createHash("sha256").update(readFileSync(shardPath)).digest("hex"), + }, + ], + } + await writeInitialIntent({ + archiveDir, + operationId, + generationId, + signal: "traces", + rangeStart: "2026-06-01", + checkpointId, + dataDir, + scratchRoot, + pinId, + pinPurpose: `archive:${generationId}`, + scratchSubdir: `archive-${operationId}`, + baseActiveGenerationId: null, + }) + await promoteGeneration(archiveDir, "traces", "2026-06-01", generationId, exactManifest, building) + const finalManifestPath = generationManifestPath(archiveDir, "traces", "2026-06-01", generationId) + const manifestSha256 = createHash("sha256").update(readFileSync(finalManifestPath)).digest("hex") + await advancePhase(archiveDir, operationId, "manifest-written", manifestSha256) + await advancePhase(archiveDir, operationId, "promoted") + if (options.pointer) { + await selectActiveGeneration(archiveDir, "traces", "2026-06-01", generationId, null) + } + if (options.catalog) await rebuildCatalog(archiveDir, "traces") + if (phase !== "promoted") await advancePhase(archiveDir, operationId, phase) + return { + archiveDir, + dataDir, + scratchRoot, + operationId, + generationId, + checkpointId, + pinId, + finalManifestPath, + finalShardPath: join(shardsRoot(archiveDir, "traces", "2026-06-01", generationId), "00.parquet"), + } +} + +describe("archive generation promotion", () => { + it("preflights archive and scratch independently on separate devices", () => { + assertArchiveScratchFreeSpace( + { identity: "archive", path: "/archive", freeBytes: 150 }, + { identity: "scratch", path: "/scratch", freeBytes: 80 }, + 50, + 100, + 80, + ) + throws( + () => + assertArchiveScratchFreeSpace( + { identity: "archive", path: "/archive", freeBytes: 149 }, + { identity: "scratch", path: "/scratch", freeBytes: 80 }, + 50, + 100, + 80, + ), + /archive volume .* below the required 150 bytes/, + ) + throws( + () => + assertArchiveScratchFreeSpace( + { identity: "archive", path: "/archive", freeBytes: 150 }, + { identity: "scratch", path: "/scratch", freeBytes: 79 }, + 50, + 100, + 80, + ), + /scratch volume .* below the required 80 bytes/, + ) + }) + + it("combines archive and scratch requirements exactly once on one device", () => { + assertArchiveScratchFreeSpace( + { identity: "shared", path: "/archive", freeBytes: 230 }, + { identity: "shared", path: "/scratch", freeBytes: 230 }, + 50, + 100, + 80, + ) + throws( + () => + assertArchiveScratchFreeSpace( + { identity: "shared", path: "/archive", freeBytes: 229 }, + { identity: "shared", path: "/scratch", freeBytes: 229 }, + 50, + 100, + 80, + ), + /archive\/scratch volume .* below the required 230 bytes/, + ) + }) + + it("fails free-space preflight before creating an active operation journal", async () => { + await withArchive(async (archiveDir) => { + const parent = join(archiveDir, "..") + const dataDir = join(parent, "data") + const scratchRoot = join(parent, "scratch") + mkdirSync(dataDir, { recursive: true }) + mkdirSync(scratchRoot, { recursive: true }) + seedCheckpoint(dataDir, randomUUID()) + await rejects( + createArchiveGeneration(dataDir, archiveDir, "traces", "2026-06-01", { + writerThreads: 1, + rowGroupRows: 1, + maxShardRows: 1, + maxShardBytes: 1, + targetChunkBytes: 1, + minFreeSpaceReserve: Number.MAX_SAFE_INTEGER - 100, + archiveDir, + scratchRoot, + }), + /below the required/, + ) + strictEqual(listActiveOperationIds(archiveDir).length, 0) + }) + }) + + it("makes the complete manifest durable in building before publishing the generation", async () => { + await withArchive(async (archiveDir) => { + const generationId = randomUUID() + const building = seedBuilding(archiveDir, generationId) + await rejects( + promoteGeneration( + archiveDir, + "traces", + "2026-06-01", + generationId, + manifest(generationId), + building, + { + beforeGenerationPromoted: () => { + throw new Error("stop before rename") + }, + }, + ), + /stop before rename/, + ) + ok(existsSync(join(building, "manifest.json")), "manifest is durable inside building") + ok( + !existsSync(generationManifestPath(archiveDir, "traces", "2026-06-01", generationId)), + "no final generation exists before the atomic directory rename", + ) + }) + }) + + it("does not publish a final generation when interrupted before manifest durability", async () => { + await withArchive(async (archiveDir) => { + const generationId = randomUUID() + const building = seedBuilding(archiveDir, generationId) + await rejects( + promoteGeneration( + archiveDir, + "traces", + "2026-06-01", + generationId, + manifest(generationId), + building, + { + beforeManifestDurable: () => { + throw new Error("stop before manifest") + }, + }, + ), + /stop before manifest/, + ) + ok(!existsSync(join(building, "manifest.json"))) + ok(!existsSync(generationManifestPath(archiveDir, "traces", "2026-06-01", generationId))) + }) + }) + + it("moves the building generation into place and selects it through the active pointer", async () => { + await withArchive(async (archiveDir) => { + const generationId = randomUUID() + const building = seedBuilding(archiveDir, generationId) + // Promotion moves building → final + writes manifest (does not touch the + // pointer). A separate CAS pointer update selects the generation. + await promoteGeneration( + archiveDir, + "traces", + "2026-06-01", + generationId, + manifest(generationId), + building, + {}, + ) + const superseded = await selectActiveGeneration( + archiveDir, + "traces", + "2026-06-01", + generationId, + null, + {}, + ) + strictEqual(superseded, null) + // The generation dir now exists with a manifest and shards. + ok(existsSync(generationManifestPath(archiveDir, "traces", "2026-06-01", generationId))) + ok(existsSync(join(shardsRoot(archiveDir, "traces", "2026-06-01", generationId), "00.parquet"))) + // The active pointer selects this generation. + const pointer = parseArchiveActivePointer( + JSON.parse(readFileSync(activePointerPath(archiveDir, "traces", "2026-06-01"), "utf8")), + ) + strictEqual(pointer.generationId, generationId) + // The building dir is gone after promotion. + ok(!existsSync(building)) + }) + }) + + it("supersedes a previous generation and retains the old one", async () => { + await withArchive(async (archiveDir) => { + const old = randomUUID() + const oldBuilding = seedBuilding(archiveDir, old) + await promoteGeneration(archiveDir, "traces", "2026-06-01", old, manifest(old), oldBuilding, {}) + await selectActiveGeneration(archiveDir, "traces", "2026-06-01", old, null, {}) + + const next = randomUUID() + const nextBuilding = seedBuilding(archiveDir, next) + await promoteGeneration( + archiveDir, + "traces", + "2026-06-01", + next, + manifest(next), + nextBuilding, + {}, + ) + // CAS base is the previously-active generation (old). selectActiveGeneration + // returns the superseded id. + const superseded = await selectActiveGeneration(archiveDir, "traces", "2026-06-01", next, old, {}) + strictEqual(superseded, old) + // The active pointer now selects the new generation... + const pointer = parseArchiveActivePointer( + JSON.parse(readFileSync(activePointerPath(archiveDir, "traces", "2026-06-01"), "utf8")), + ) + strictEqual(pointer.generationId, next) + // ...but the old generation directory is retained, never deleted. + ok(existsSync(generationManifestPath(archiveDir, "traces", "2026-06-01", old))) + ok(existsSync(generationManifestPath(archiveDir, "traces", "2026-06-01", next))) + }) + }) + + it("selectActiveGeneration refuses to clobber a pointer that moved off the recorded base", async () => { + // CAS: if the pointer no longer matches the recorded base AND does not + // already select the intended generation, a blind flip would clobber + // concurrent activity — fail closed. + await withArchive(async (archiveDir) => { + const gen = randomUUID() + const building = seedBuilding(archiveDir, gen) + await promoteGeneration(archiveDir, "traces", "2026-06-01", gen, manifest(gen), building, {}) + // Record a base that does NOT match reality (no pointer exists; base + // claims a different generation). + await rejects( + selectActiveGeneration(archiveDir, "traces", "2026-06-01", gen, randomUUID(), {}), + /no longer matches base/, + ) + }) + }) + + it("selectActiveGeneration is idempotent when the pointer already selects the generation", async () => { + await withArchive(async (archiveDir) => { + const gen = randomUUID() + const building = seedBuilding(archiveDir, gen) + await promoteGeneration(archiveDir, "traces", "2026-06-01", gen, manifest(gen), building, {}) + await selectActiveGeneration(archiveDir, "traces", "2026-06-01", gen, null, {}) + // Re-selecting with a base equal to the generation is a no-op (no throw). + const superseded = await selectActiveGeneration(archiveDir, "traces", "2026-06-01", gen, gen, {}) + strictEqual(superseded, gen) + }) + }) + + it("refuses to promote into an existing generation directory", async () => { + await withArchive(async (archiveDir) => { + const generationId = randomUUID() + const building = seedBuilding(archiveDir, generationId) + await promoteGeneration( + archiveDir, + "traces", + "2026-06-01", + generationId, + manifest(generationId), + building, + {}, + ) + // A second promotion of the same id must fail closed; the existing + // generation directory is not overwritten. + const dupBuilding = seedBuilding(archiveDir, generationId) + await rejects( + promoteGeneration( + archiveDir, + "traces", + "2026-06-01", + generationId, + manifest(generationId), + dupBuilding, + {}, + ), + /already exists/, + ) + }) + }) + + it("reconciliation rejects a tampered published shard before pointer or catalog mutation", async () => { + await withArchive(async (archiveDir) => { + const parent = join(archiveDir, "..") + const dataDir = join(parent, "data") + const scratchRoot = join(parent, "scratch") + mkdirSync(dataDir, { recursive: true }) + mkdirSync(scratchRoot, { recursive: true }) + const operationId = randomUUID() + const generationId = randomUUID() + const checkpointId = randomUUID() + const pinId = randomUUID() + const building = seedBuilding(archiveDir, generationId) + const shardPath = join(building, "shards", "00.parquet") + const shardBytes = statSync(shardPath).size + const shardSha256 = createHash("sha256").update(readFileSync(shardPath)).digest("hex") + const baseManifest = manifest(generationId) + const exactManifest: ArchiveGenerationManifest = { + ...baseManifest, + checkpointId, + shards: [{ ...baseManifest.shards[0]!, bytes: shardBytes, sha256: shardSha256 }], + } + await writeInitialIntent({ + archiveDir, + operationId, + generationId, + signal: "traces", + rangeStart: "2026-06-01", + checkpointId, + dataDir, + scratchRoot, + pinId, + pinPurpose: `archive:${generationId}`, + scratchSubdir: `archive-${operationId}`, + baseActiveGenerationId: null, + }) + await promoteGeneration(archiveDir, "traces", "2026-06-01", generationId, exactManifest, building) + const finalManifestPath = generationManifestPath(archiveDir, "traces", "2026-06-01", generationId) + const manifestSha256 = createHash("sha256").update(readFileSync(finalManifestPath)).digest("hex") + await advancePhase(archiveDir, operationId, "manifest-written", manifestSha256) + await advancePhase(archiveDir, operationId, "promoted") + const pinPath = join(checkpointPinsRoot(dataDir), checkpointId, `${pinId}.json`) + mkdirSync(join(pinPath, ".."), { recursive: true }) + writeFileSync( + pinPath, + `${JSON.stringify({ + formatVersion: 1, + pinId, + checkpointId, + purpose: `archive:${generationId}`, + createdAt: new Date().toISOString(), + })}\n`, + ) + const finalShardPath = join( + shardsRoot(archiveDir, "traces", "2026-06-01", generationId), + "00.parquet", + ) + writeFileSync(finalShardPath, "attacker bytes") + await rejects( + reconcileArchiveGeneration(dataDir, archiveDir, scratchRoot), + /shard 00\.parquet (byte size|SHA-256) mismatch/, + ) + ok(!existsSync(activePointerPath(archiveDir, "traces", "2026-06-01"))) + ok(!existsSync(catalogPath(archiveDir, "traces"))) + }) + }) + + it("preserves authority over an impossible final generation without a manifest", async () => { + await withArchive(async (archiveDir) => { + const parent = join(archiveDir, "..") + const dataDir = join(parent, "data") + const scratchRoot = join(parent, "scratch") + mkdirSync(dataDir, { recursive: true }) + mkdirSync(scratchRoot, { recursive: true }) + const operationId = randomUUID() + const generationId = randomUUID() + await writeInitialIntent({ + archiveDir, + operationId, + generationId, + signal: "traces", + rangeStart: "2026-06-01", + checkpointId: randomUUID(), + dataDir, + scratchRoot, + pinId: randomUUID(), + pinPurpose: `archive:${generationId}`, + scratchSubdir: `archive-${operationId}`, + baseActiveGenerationId: null, + }) + const impossibleFinal = generationRoot(archiveDir, "traces", "2026-06-01", generationId) + mkdirSync(join(impossibleFinal, "shards"), { recursive: true }) + await rejects( + reconcileArchiveGeneration(dataDir, archiveDir, scratchRoot), + /published a generation without a manifest/, + ) + ok(existsSync(impossibleFinal), "uncertain final state retained") + ok(existsSync(operationDir(archiveDir, operationId)), "journal authority retained") + }) + }) + + it("repairs missing pointer and catalog despite journal completion labels", async () => { + await withArchive(async (archiveDir) => { + const seeded = await seedPublishedOperation(archiveDir, "catalog-complete") + await reconcileArchiveGeneration(seeded.dataDir, archiveDir, seeded.scratchRoot) + const pointer = parseArchiveActivePointer( + JSON.parse(readFileSync(activePointerPath(archiveDir, "traces", "2026-06-01"), "utf8")), + "traces", + "2026-06-01", + ) + strictEqual(pointer.generationId, seeded.generationId) + assertCatalogExact(archiveDir, "traces") + strictEqual(listActiveOperationIds(archiveDir).length, 0) + }) + }) + + it("fails closed when the observed pointer conflicts with the journal CAS topology", async () => { + await withArchive(async (archiveDir) => { + const seeded = await seedPublishedOperation(archiveDir, "catalog-complete") + writeFileSync( + activePointerPath(archiveDir, "traces", "2026-06-01"), + `${JSON.stringify({ + formatVersion: 1, + generationId: randomUUID(), + signal: "traces", + rangeStart: "2026-06-01", + selectedAt: new Date().toISOString(), + })}\n`, + ) + await rejects( + reconcileArchiveGeneration(seeded.dataDir, archiveDir, seeded.scratchRoot), + /no longer matches the recorded base/, + ) + ok(existsSync(operationDir(archiveDir, seeded.operationId)), "journal authority retained") + }) + }) + + it("repairs a tampered catalog from authoritative manifests despite catalog-complete", async () => { + await withArchive(async (archiveDir) => { + const seeded = await seedPublishedOperation(archiveDir, "catalog-complete", { + pointer: true, + catalog: true, + }) + writeFileSync(catalogPath(archiveDir, "traces"), '{"attacker":true}\n') + await reconcileArchiveGeneration(seeded.dataDir, archiveDir, seeded.scratchRoot) + assertCatalogExact(archiveDir, "traces") + const lines = readFileSync(catalogPath(archiveDir, "traces"), "utf8").trim().split("\n") + strictEqual(lines.length, 1) + strictEqual((JSON.parse(lines[0]!) as { generationId: string }).generationId, seeded.generationId) + }) + }) + + it("retains a complete journal unless every implied durable invariant is exact", async () => { + await withArchive(async (outerArchiveDir) => { + const root = join(outerArchiveDir, "..") + const cases: ReadonlyArray<{ + name: string + mutate: (seeded: Awaited>) => void + error: RegExp + }> = [ + { + name: "manifest", + mutate: (s) => writeFileSync(s.finalManifestPath, "{}\n"), + error: /manifest SHA-256 mismatch/, + }, + { + name: "shard", + mutate: (s) => writeFileSync(s.finalShardPath, "tampered"), + error: /shard 00\.parquet (byte size|SHA-256) mismatch/, + }, + { + name: "pointer", + mutate: (s) => rmSync(activePointerPath(s.archiveDir, "traces", "2026-06-01")), + error: /pointer mismatch/, + }, + { + name: "catalog", + mutate: (s) => writeFileSync(catalogPath(s.archiveDir, "traces"), "{}\n"), + error: /catalog does not exactly match/, + }, + { + name: "pin", + mutate: (s) => { + const pinPath = join(checkpointPinsRoot(s.dataDir), s.checkpointId, `${s.pinId}.json`) + mkdirSync(join(pinPath, ".."), { recursive: true }) + writeFileSync( + pinPath, + `${JSON.stringify({ + formatVersion: 1, + pinId: s.pinId, + checkpointId: s.checkpointId, + purpose: `archive:${s.generationId}`, + createdAt: new Date().toISOString(), + })}\n`, + ) + }, + error: /requires its exact owned pin to be absent/, + }, + { + name: "scratch", + mutate: (s) => + mkdirSync(join(s.scratchRoot, `archive-${s.operationId}`), { recursive: true }), + error: /requires its exact owned scratch to be absent/, + }, + { + name: "building", + mutate: (s) => + mkdirSync(buildingGenerationRoot(s.archiveDir, s.generationId), { recursive: true }), + error: /both building and final generation state/, + }, + ] + for (const scenario of cases) { + const archiveDir = join(root, `complete-${scenario.name}`, "archive") + mkdirSync(archiveDir, { recursive: true }) + const seeded = await seedPublishedOperation(archiveDir, "complete", { + pointer: true, + catalog: true, + }) + scenario.mutate(seeded) + await rejects( + reconcileArchiveGeneration(seeded.dataDir, archiveDir, seeded.scratchRoot), + scenario.error, + ) + ok( + existsSync(operationDir(archiveDir, seeded.operationId)), + `${scenario.name}: active journal retained`, + ) + } + }) + }) +}) + +describe("archive catalog append", () => { + it("appends one line per generation and survives a rebuild from manifests", async () => { + await withArchive(async (archiveDir) => { + const g1 = randomUUID() + await appendCatalog(archiveDir, "traces", manifest(g1, "traces", 10)) + const g2 = randomUUID() + await appendCatalog(archiveDir, "traces", manifest(g2, "traces", 20)) + const catalog = readFileSync(catalogPath(archiveDir, "traces"), "utf8").trim().split("\n") + strictEqual(catalog.length, 2) + const first = JSON.parse(catalog[0]!) as { generationId: string; archivedRowCount: number } + const second = JSON.parse(catalog[1]!) as { generationId: string; archivedRowCount: number } + strictEqual(first.generationId, g1) + strictEqual(first.archivedRowCount, 10) + strictEqual(second.generationId, g2) + strictEqual(second.archivedRowCount, 20) + }) + }) + + it("creates the catalog on first append when none exists", async () => { + await withArchive(async (archiveDir) => { + const g = randomUUID() + await appendCatalog(archiveDir, "logs", manifest(g, "logs", 5)) + ok(existsSync(catalogPath(archiveDir, "logs"))) + }) + }) +}) diff --git a/apps/cli/test/archive-journal.test.ts b/apps/cli/test/archive-journal.test.ts new file mode 100644 index 000000000..6e4ef7244 --- /dev/null +++ b/apps/cli/test/archive-journal.test.ts @@ -0,0 +1,640 @@ +import { describe, it } from "@effect/vitest" +import { ok, rejects, strictEqual } from "node:assert" +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { randomUUID } from "node:crypto" +import { + ARCHIVE_OPERATION_FORMAT_VERSION, + advancePhase, + archiveCompletedOperation, + activeOperationsRoot, + listActiveOperationIds, + migrateActiveIntentIfLegacy, + migrateV2CreateIntent, + operationDir, + parseArchiveOperationIntent, + readActiveOperation, + writeInitialIntent, + type ArchiveOperationIntent, +} from "../src/server/archives/journal" +import { reconcileArchiveGeneration } from "../src/server/archives/generation" +import { checkpointPinsRoot } from "../src/server/checkpoints" + +// Filesystem-level tests for the archive operation journal (Gate 3). These are +// the fast in-process checks of the journal's fail-closed parsing, phase +// transitions, and at-most-one-active-operation invariant. The AUTHORITATIVE +// crash-safety oracle is the native SIGKILL harness +// (native-archive-crash-recovery-probe.sh); these unit tests cover the +// deterministic invariants the harness does not isolate. + +const withArchive = async (run: (archiveDir: string) => Promise | void): Promise => { + const parent = realpathSync(mkdtempSync(join(tmpdir(), "maple-archive-journal-test-"))) + const archiveDir = join(parent, "archive") + mkdirSync(archiveDir, { recursive: true }) + try { + await run(archiveDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const baseIntent = (overrides: Partial<{ operationId: string; generationId: string }> = {}) => { + const operationId = overrides.operationId ?? randomUUID() + const generationId = overrides.generationId ?? randomUUID() + return { + kind: "create" as const, + archiveDir: "", // set by withArchive caller + operationId, + generationId, + signal: "traces", + rangeStart: "2026-06-01", + checkpointId: randomUUID(), + dataDir: "/data", + scratchRoot: "/scratch", + pinId: randomUUID(), + pinPurpose: `archive:${generationId}`, + scratchSubdir: `archive-${operationId}`, + baseActiveGenerationId: null, + } +} + +describe("archive operation journal", () => { + it("writeInitialIntent persists a parseable intent at phase intent", async () => { + await withArchive(async (archiveDir) => { + const intent = baseIntent() + await writeInitialIntent({ ...intent, archiveDir }) + const active = readActiveOperation(archiveDir) + ok(active !== null) + strictEqual(active.intent.phase, "intent") + strictEqual(active.intent.operationId, intent.operationId) + strictEqual(active.intent.pinId, intent.pinId) + strictEqual(active.intent.scratchSubdir, intent.scratchSubdir) + strictEqual(active.intent.formatVersion, ARCHIVE_OPERATION_FORMAT_VERSION) + }) + }) + + it("listActiveOperationIds returns empty when no operations exist", async () => { + await withArchive(async (archiveDir) => { + strictEqual(listActiveOperationIds(archiveDir).length, 0) + strictEqual(readActiveOperation(archiveDir), null) + }) + }) + + it("advancePhase records a forward transition and refuses regression", async () => { + await withArchive(async (archiveDir) => { + const op = randomUUID() + await writeInitialIntent({ ...baseIntent({ operationId: op }), archiveDir }) + await advancePhase(archiveDir, op, "pin-acquired") + let active = readActiveOperation(archiveDir) + strictEqual(active!.intent.phase, "pin-acquired") + // Idempotent re-advance to the same phase is allowed. + await advancePhase(archiveDir, op, "pin-acquired") + // Backward transition is refused. + await rejects(advancePhase(archiveDir, op, "intent"), /regression/) + active = readActiveOperation(archiveDir) + strictEqual(active!.intent.phase, "pin-acquired") + }) + }) + + it("clears published-manifest authority when a prepublication operation aborts", async () => { + await withArchive(async (archiveDir) => { + const op = randomUUID() + await writeInitialIntent({ ...baseIntent({ operationId: op }), archiveDir }) + await advancePhase(archiveDir, op, "manifest-written", "a".repeat(64)) + await advancePhase(archiveDir, op, "aborted") + const active = readActiveOperation(archiveDir) + strictEqual(active!.intent.phase, "aborted") + strictEqual(active!.intent.manifestSha256, null) + }) + }) + + it("readActiveOperation fails closed on more than one active operation", async () => { + await withArchive(async (archiveDir) => { + await writeInitialIntent({ ...baseIntent({ operationId: randomUUID() }), archiveDir }) + await writeInitialIntent({ ...baseIntent({ operationId: randomUUID() }), archiveDir }) + // Two active operation dirs -> ambiguous -> fail closed. + await rejects(async () => readActiveOperation(archiveDir), /ambiguous|multiple active/) + }) + }) + + it("readActiveOperation fails closed on an unrecognized active entry", async () => { + await withArchive(async (archiveDir) => { + // A non-conforming entry (not archive-) is unrecognized debris. + mkdirSync(join(activeOperationsRoot(archiveDir), "junk"), { recursive: true }) + await rejects(async () => readActiveOperation(archiveDir), /unrecognized/) + }) + }) + + it("archiveCompletedOperation moves the journal out of active/ and retains it", async () => { + await withArchive(async (archiveDir) => { + const op = randomUUID() + await writeInitialIntent({ ...baseIntent({ operationId: op }), archiveDir }) + await archiveCompletedOperation(archiveDir, op) + // No longer in active/. + strictEqual(listActiveOperationIds(archiveDir).length, 0) + // Retained under completed/. + const completed = join(archiveDir, "operations", "completed", `archive-${op}`, "intent.json") + ok(existsSync(completed), "completed journal retained") + }) + }) +}) + +describe("archive operation journal strict parsing (fail-closed)", () => { + it("rejects an unknown format version", async () => { + const raw = { ...baseIntent(), formatVersion: 99, phase: "intent" } + await rejects(async () => parseArchiveOperationIntent("/archive", raw), /format version/) + }) + + it("rejects an invalid phase", async () => { + const raw: Record = { + ...baseIntent(), + formatVersion: ARCHIVE_OPERATION_FORMAT_VERSION, + phase: "nope", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + } + await rejects(async () => parseArchiveOperationIntent("/archive", raw), /invalid.*phase/) + }) + + it("rejects a malformed/missing identity", async () => { + const raw: Record = { + formatVersion: ARCHIVE_OPERATION_FORMAT_VERSION, + kind: "create", + phase: "intent", + // missing operationId, generationId, etc. + } + await rejects(async () => parseArchiveOperationIntent("/archive", raw), /operationId/) + }) + + it("rejects a scratchSubdir containing a path separator (escape attempt)", async () => { + const intent = baseIntent() + const raw: ArchiveOperationIntent = { + formatVersion: ARCHIVE_OPERATION_FORMAT_VERSION, + kind: "create", + operationId: intent.operationId, + generationId: intent.generationId, + signal: intent.signal, + rangeStart: intent.rangeStart, + checkpointId: intent.checkpointId, + archiveDir: "/archive", + dataDir: intent.dataDir, + scratchRoot: intent.scratchRoot, + pinId: intent.pinId, + pinPurpose: intent.pinPurpose, + scratchSubdir: "../escape", + manifestSha256: null, + baseActiveGenerationId: null, + phase: "intent", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + } + await rejects(async () => parseArchiveOperationIntent("/archive", raw), /scratch subdir/) + }) + + it("readActiveOperation fails closed when the intent operationId mismatches its directory", async () => { + await withArchive(async (archiveDir) => { + const op = randomUUID() + await writeInitialIntent({ ...baseIntent({ operationId: op }), archiveDir }) + // Corrupt: rename the dir to a different operation id while leaving the + // intent recording the original. Identity binding must catch this. + const other = randomUUID() + const fs = await import("node:fs/promises") + await fs.rename(operationDir(archiveDir, op), operationDir(archiveDir, other)) + await rejects(async () => readActiveOperation(archiveDir), /identity mismatch/) + }) + }) + + it("readActiveOperation fails closed on a hand-edited malformed intent file", async () => { + await withArchive(async (archiveDir) => { + const op = randomUUID() + await writeInitialIntent({ ...baseIntent({ operationId: op }), archiveDir }) + // Overwrite the intent with garbage. + writeFileSync(join(operationDir(archiveDir, op), "intent.json"), "{not json") + await rejects(async () => readActiveOperation(archiveDir)) + }) + }) + + it("reconciliation rejects altered configured roots without touching outside state", async () => { + await withArchive(async (archiveDir) => { + const root = join(archiveDir, "..") + const dataDir = join(root, "data") + const scratchRoot = join(root, "scratch") + const outside = join(root, "outside") + const marker = join(outside, "KEEP") + mkdirSync(dataDir, { recursive: true }) + mkdirSync(scratchRoot, { recursive: true }) + mkdirSync(outside, { recursive: true }) + writeFileSync(marker, "preserve") + const op = randomUUID() + await writeInitialIntent({ + ...baseIntent({ operationId: op }), + archiveDir, + dataDir, + scratchRoot, + }) + const path = join(operationDir(archiveDir, op), "intent.json") + const raw = JSON.parse(readFileSync(path, "utf8")) as Record + writeFileSync(path, `${JSON.stringify({ ...raw, scratchRoot: outside })}\n`) + await rejects( + reconcileArchiveGeneration(dataDir, archiveDir, scratchRoot), + /scratch root mismatch/, + ) + strictEqual(readFileSync(marker, "utf8"), "preserve") + ok(existsSync(path), "active journal retained on root mismatch") + }) + }) + + it("reconciliation refuses a symlinked owned scratch directory and preserves its target", async () => { + await withArchive(async (archiveDir) => { + const root = join(archiveDir, "..") + const dataDir = join(root, "data") + const scratchRoot = join(root, "scratch") + const outside = join(root, "outside") + const marker = join(outside, "KEEP") + mkdirSync(dataDir, { recursive: true }) + mkdirSync(scratchRoot, { recursive: true }) + mkdirSync(outside, { recursive: true }) + writeFileSync(marker, "preserve") + const op = randomUUID() + await writeInitialIntent({ + ...baseIntent({ operationId: op }), + archiveDir, + dataDir, + scratchRoot, + }) + symlinkSync(outside, join(scratchRoot, `archive-${op}`)) + await rejects( + reconcileArchiveGeneration(dataDir, archiveDir, scratchRoot), + /refusing symlink in owned scratch/, + ) + strictEqual(readFileSync(marker, "utf8"), "preserve") + ok(readActiveOperation(archiveDir) !== null, "active journal retained") + }) + }) + + it("reconciliation refuses a symlinked configured root and preserves its target", async () => { + await withArchive(async (archiveDir) => { + const root = join(archiveDir, "..") + const dataDir = join(root, "data") + const realScratch = join(root, "real-scratch") + const scratchRoot = join(root, "scratch-link") + const marker = join(realScratch, "KEEP") + mkdirSync(dataDir, { recursive: true }) + mkdirSync(realScratch, { recursive: true }) + writeFileSync(marker, "preserve") + symlinkSync(realScratch, scratchRoot) + const op = randomUUID() + await writeInitialIntent({ + ...baseIntent({ operationId: op }), + archiveDir, + dataDir, + scratchRoot, + }) + await rejects( + reconcileArchiveGeneration(dataDir, archiveDir, scratchRoot), + /refusing symlink in scratch root/, + ) + strictEqual(readFileSync(marker, "utf8"), "preserve") + ok(readActiveOperation(archiveDir) !== null) + }) + }) + + it("reconciliation refuses a symlinked scratch-root ancestor and preserves outside state", async () => { + await withArchive(async (archiveDir) => { + const root = join(archiveDir, "..") + const dataDir = join(root, "data") + const realParent = join(root, "real-parent") + const linkedParent = join(root, "linked-parent") + const scratchRoot = join(linkedParent, "scratch") + const marker = join(realParent, "KEEP") + mkdirSync(dataDir, { recursive: true }) + mkdirSync(realParent, { recursive: true }) + writeFileSync(marker, "preserve") + symlinkSync(realParent, linkedParent) + const op = randomUUID() + await writeInitialIntent({ + ...baseIntent({ operationId: op }), + archiveDir, + dataDir, + scratchRoot, + }) + await rejects( + reconcileArchiveGeneration(dataDir, archiveDir, scratchRoot), + /refusing symlink in scratch root/, + ) + strictEqual(readFileSync(marker, "utf8"), "preserve") + ok( + !existsSync(join(realParent, "scratch")), + "missing scratch leaf was not created through symlink", + ) + ok(readActiveOperation(archiveDir) !== null, "active journal retained") + }) + }) + + it("partial-restore recovery unlinks internal symlinks without following them", async () => { + await withArchive(async (archiveDir) => { + const root = join(archiveDir, "..") + const dataDir = join(root, "data") + const scratchRoot = join(root, "scratch") + const outside = join(root, "outside") + const marker = join(outside, "KEEP") + mkdirSync(dataDir, { recursive: true }) + mkdirSync(scratchRoot, { recursive: true }) + mkdirSync(outside, { recursive: true }) + writeFileSync(marker, "preserve") + const op = randomUUID() + const initial = { ...baseIntent({ operationId: op }), archiveDir, dataDir, scratchRoot } + await writeInitialIntent(initial) + const pinPath = join(checkpointPinsRoot(dataDir), initial.checkpointId, `${initial.pinId}.json`) + mkdirSync(join(pinPath, ".."), { recursive: true }) + writeFileSync( + pinPath, + `${JSON.stringify({ + formatVersion: 1, + pinId: initial.pinId, + checkpointId: initial.checkpointId, + purpose: initial.pinPurpose, + createdAt: new Date().toISOString(), + })}\n`, + ) + await advancePhase(archiveDir, op, "pin-acquired") + const ownedScratch = join(scratchRoot, initial.scratchSubdir) + mkdirSync(join(ownedScratch, "store"), { recursive: true }) + writeFileSync(join(ownedScratch, "partial"), "restore debris") + symlinkSync(outside, join(ownedScratch, "store", "table-link")) + await advancePhase(archiveDir, op, "scratch-allocated") + + await reconcileArchiveGeneration(dataDir, archiveDir, scratchRoot) + + strictEqual(readFileSync(marker, "utf8"), "preserve") + ok(!existsSync(pinPath), "exact owned pin released") + ok(!existsSync(ownedScratch), "exact owned scratch removed") + ok(readActiveOperation(archiveDir) === null, "active authority retired") + ok( + existsSync(join(archiveDir, "operations", "completed", `archive-${op}`, "intent.json")), + "aborted operation evidence retained", + ) + }) + }) + + it("strictly binds known signal and deterministic identities", async () => { + const base = baseIntent() + const raw = { + ...base, + formatVersion: ARCHIVE_OPERATION_FORMAT_VERSION, + archiveDir: "/archive", + phase: "intent", + manifestSha256: null, + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + } + await rejects( + async () => parseArchiveOperationIntent("/archive", { ...raw, signal: "unknown" }), + /unknown archive signal/, + ) + await rejects( + async () => parseArchiveOperationIntent("/archive", { ...raw, pinPurpose: "archive:other" }), + /pin purpose/, + ) + await rejects( + async () => parseArchiveOperationIntent("/archive", { ...raw, scratchSubdir: "archive-other" }), + /scratch subdir/, + ) + }) + + it("reconciliation rejects a mismatched pin purpose and premature pin absence", async () => { + await withArchive(async (archiveDir) => { + const root = join(archiveDir, "..") + const dataDir = join(root, "data") + const scratchRoot = join(root, "scratch") + mkdirSync(dataDir, { recursive: true }) + mkdirSync(scratchRoot, { recursive: true }) + const op = randomUUID() + const initial = { ...baseIntent({ operationId: op }), archiveDir, dataDir, scratchRoot } + await writeInitialIntent(initial) + const pinPath = join(checkpointPinsRoot(dataDir), initial.checkpointId, `${initial.pinId}.json`) + mkdirSync(join(pinPath, ".."), { recursive: true }) + writeFileSync( + pinPath, + `${JSON.stringify({ + formatVersion: 1, + pinId: initial.pinId, + checkpointId: initial.checkpointId, + purpose: "archive:attacker", + createdAt: new Date().toISOString(), + })}\n`, + ) + await rejects( + reconcileArchiveGeneration(dataDir, archiveDir, scratchRoot), + /pin identity or purpose mismatch/, + ) + rmSync(pinPath) + await advancePhase(archiveDir, op, "pin-acquired") + await rejects( + reconcileArchiveGeneration(dataDir, archiveDir, scratchRoot), + /pin is missing before release was authorized/, + ) + ok(readActiveOperation(archiveDir) !== null) + }) + }) +}) + +describe("archive operation journal v2 → v3 migration (Gate 3b)", () => { + it("migrateV2CreateIntent lifts a clean v2 record to v3 and round-trips through the parser", () => { + const operationId = randomUUID() + const generationId = randomUUID() + // A v2 record (no `kind`, formatVersion 2) as a Gate 3a binary would write. + const v2: Record = { + formatVersion: 2, + operationId, + generationId, + signal: "traces", + rangeStart: "2026-06-01", + checkpointId: randomUUID(), + archiveDir: "/archive", + dataDir: "/data", + scratchRoot: "/scratch", + pinId: randomUUID(), + pinPurpose: `archive:${generationId}`, + scratchSubdir: `archive-${operationId}`, + manifestSha256: null, + baseActiveGenerationId: null, + phase: "intent", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + } + const lifted = migrateV2CreateIntent("/archive", v2) + strictEqual(lifted.formatVersion, 3) + strictEqual(lifted.kind, "create") + // The lifted record parses cleanly as v3. + const parsed = parseArchiveOperationIntent("/archive", lifted) + strictEqual(parsed.kind, "create") + strictEqual(parsed.operationId, operationId) + }) + + it("migrateV2CreateIntent rejects a corrupt v2 record rather than silently lifting it", () => { + const corrupt: Record = { + formatVersion: 2, + // missing every required field + } + rejects( + async () => migrateV2CreateIntent("/archive", corrupt), + /missing or not a string|operationId|phase|kind|invalid/, + ) + }) + + it("migrateV2CreateIntent refuses a non-v2 record", () => { + rejects( + async () => migrateV2CreateIntent("/archive", { formatVersion: 1 }), + /v2 migration requires formatVersion 2/, + ) + rejects( + async () => migrateV2CreateIntent("/archive", { formatVersion: 3, kind: "create" }), + /v2 migration requires formatVersion 2/, + ) + }) + + it("migrateActiveIntentIfLegacy lifts a stranded v2 intent on disk under the lock", async () => { + await withArchive(async (archiveDir) => { + const op = randomUUID() + const generationId = randomUUID() + // Write a v2 intent directly to disk (as a Gate 3a binary would leave). + const dir = operationDir(archiveDir, op) + mkdirSync(dir, { recursive: true }) + const v2 = { + formatVersion: 2, + operationId: op, + generationId, + signal: "traces", + rangeStart: "2026-06-01", + checkpointId: randomUUID(), + archiveDir, + dataDir: join(archiveDir, "..", "data"), + scratchRoot: join(archiveDir, "..", "scratch"), + pinId: randomUUID(), + pinPurpose: `archive:${generationId}`, + scratchSubdir: `archive-${op}`, + manifestSha256: null, + baseActiveGenerationId: null, + phase: "intent", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + } + writeFileSync(join(dir, "intent.json"), JSON.stringify(v2)) + // A direct v3 parse REJECTS the v2 record (fail-closed, no silent reinterpret). + rejects(async () => parseArchiveOperationIntent(archiveDir, v2), /format version/) + // Migration lifts it to v3 on disk. + const migrated = await migrateActiveIntentIfLegacy(archiveDir) + strictEqual(migrated, true, "v2 intent was migrated") + // Now the on-disk record parses cleanly as v3. + const active = readActiveOperation(archiveDir) + ok(active !== null) + strictEqual(active!.intent.kind, "create") + strictEqual(active!.intent.formatVersion, 3) + // A second migration is a no-op (already v3). + const migrated2 = await migrateActiveIntentIfLegacy(archiveDir) + strictEqual(migrated2, false, "already-v3 intent is not re-migrated") + }) + }) + + it("migrateActiveIntentIfLegacy is a no-op when there is no active operation", async () => { + await withArchive(async (archiveDir) => { + const migrated = await migrateActiveIntentIfLegacy(archiveDir) + strictEqual(migrated, false) + }) + }) +}) + +describe("archive gc journal phase/cursor strictness (Gate 3b repair)", () => { + const gcTarget = (generationId: string) => ({ + signal: "traces", + rangeStart: "2026-06-01", + generationId, + createdAt: "2026-06-02T00:00:00.000Z", + manifestSha256: "a".repeat(64), + bytes: 100, + shards: [{ name: "00.parquet", bytes: 100, sha256: "b".repeat(64) }], + recordedActiveGenerationId: randomUUID(), + }) + const gcIntentBase = (overrides: Record = {}) => ({ + formatVersion: ARCHIVE_OPERATION_FORMAT_VERSION, + kind: "gc", + operationId: randomUUID(), + keep: 0, + targets: [gcTarget(randomUUID()), gcTarget(randomUUID())], + completedTargets: 0, + archiveDir: "/archive", + dataDir: "/data", + scratchRoot: "/scratch", + phase: "intent", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + ...overrides, + }) + + it("rejects a GC intent with a create-only phase (pin-acquired)", async () => { + await rejects( + async () => parseArchiveOperationIntent("/archive", gcIntentBase({ phase: "pin-acquired" })), + /not valid for kind gc/, + ) + }) + + it("rejects a GC intent with the aborted phase", async () => { + await rejects( + async () => parseArchiveOperationIntent("/archive", gcIntentBase({ phase: "aborted" })), + /not valid for kind gc/, + ) + }) + + it("rejects a GC intent at complete with an incomplete cursor", async () => { + await rejects( + async () => + parseArchiveOperationIntent( + "/archive", + gcIntentBase({ phase: "complete", completedTargets: 1 }), + ), + /phase complete requires completedTargets === targets.length/, + ) + }) + + it("rejects a GC intent at intent with a nonzero cursor", async () => { + await rejects( + async () => + parseArchiveOperationIntent( + "/archive", + gcIntentBase({ phase: "intent", completedTargets: 1 }), + ), + /phase intent requires completedTargets 0/, + ) + }) + + it("accepts a GC intent at gc-collecting with a full cursor (legitimate post-final-deletion state)", async () => { + const parsed = parseArchiveOperationIntent( + "/archive", + gcIntentBase({ phase: "gc-collecting", completedTargets: 2 }), + ) + strictEqual(parsed.kind, "gc") + }) + + it("rejects a GC intent with duplicate targets", async () => { + const dup = randomUUID() + await rejects( + async () => + parseArchiveOperationIntent( + "/archive", + gcIntentBase({ targets: [gcTarget(dup), gcTarget(dup)] }), + ), + /duplicate target/, + ) + }) +}) diff --git a/apps/cli/test/archive-listing.test.ts b/apps/cli/test/archive-listing.test.ts new file mode 100644 index 000000000..3ae206fe1 --- /dev/null +++ b/apps/cli/test/archive-listing.test.ts @@ -0,0 +1,324 @@ +import { describe, it } from "@effect/vitest" +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert" +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { createHash, randomUUID } from "node:crypto" +import { + activePointerPath, + catalogPath, + generationManifestPath, + generationsRoot, + nextMidnightUtc, + shardsRoot, +} from "../src/server/archives/paths" +import { + ARCHIVE_VERIFY_BUFFER_BYTES, + activeParquetPaths, + listActiveGenerations, + rebuildCatalog, + verifyActiveGenerations, +} from "../src/server/archives/listing" +import { type ArchiveGenerationManifest } from "../src/server/archives/manifest" +import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +import { SCHEMA_FINGERPRINT } from "../src/server/serve" + +const withArchive = async (run: (archiveDir: string) => Promise | void): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-archive-listing-test-")) + const archiveDir = join(parent, "archive") + mkdirSync(archiveDir, { recursive: true }) + try { + await run(archiveDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const manifest = ( + generationId: string, + signal: string, + rangeDate: string, + rowCount: number, + shardSha = "a".repeat(64), + shardBytes = 4096, +): ArchiveGenerationManifest => ({ + formatVersion: 3, + generationId, + signal, + rangeStart: rangeDate, + rangeEndExclusive: nextMidnightUtc(rangeDate), + checkpointId: randomUUID(), + checkpointManifestFingerprint: "cid:2026-01-01:100", + createdAt: "2026-06-02T00:00:00.000Z", + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + sourceRowCount: rowCount, + archivedRowCount: rowCount, + tuning: { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + targetChunkBytes: 1024 * 1024 * 1024, + minFreeSpaceReserve: 512 * 1024 * 1024, + }, + tuningConfig: null, + shards: [ + { + name: "00-0000.parquet", + rowCount, + minEventTimeUnixNano: `${BigInt(Date.parse(`${rangeDate}T00:00:00.000Z`)) * 1_000_000n}`, + maxEventTimeUnixNano: `${BigInt(Date.parse(`${rangeDate}T00:30:00.000Z`)) * 1_000_000n}`, + sha256: shardSha, + bytes: shardBytes, + columns: ["TimestampTime", "ServiceName"], + complexDigest: "123456789", + complexDigestAlgorithm: "cityhash64-multiset-v3", + }, + ], +}) + +/** Seed a complete, promoted generation on disk (manifest + shard + active pointer). */ +const seedActiveGeneration = ( + archiveDir: string, + signal: string, + rangeDate: string, + generationId: string, + rowCount: number, +): void => { + const shardsDir = shardsRoot(archiveDir, signal, rangeDate, generationId) + mkdirSync(shardsDir, { recursive: true }) + const shardPath = join(shardsDir, "00-0000.parquet") + const shardContent = "PAR1" + writeFileSync(shardPath, shardContent) + const shardStat = statSync(shardPath) + const shardSha = sha256Hex(shardContent) + writeFileSync( + generationManifestPath(archiveDir, signal, rangeDate, generationId), + `${JSON.stringify(manifest(generationId, signal, rangeDate, rowCount, shardSha, shardStat.size))}\n`, + ) + writeFileSync( + activePointerPath(archiveDir, signal, rangeDate), + `${JSON.stringify({ + formatVersion: 1, + generationId, + signal, + rangeStart: rangeDate, + selectedAt: "2026-06-02T00:00:00.000Z", + })}\n`, + ) +} + +/** Seed a superseded generation: manifest + shard present, but NOT active. */ +const seedSupersededGeneration = ( + archiveDir: string, + signal: string, + rangeDate: string, + generationId: string, + rowCount: number, +): void => { + const shardsDir = shardsRoot(archiveDir, signal, rangeDate, generationId) + mkdirSync(shardsDir, { recursive: true }) + const shardPath = join(shardsDir, "00-0000.parquet") + const shardContent = "PAR1-old" + writeFileSync(shardPath, shardContent) + const shardStat = statSync(shardPath) + const shardSha = sha256Hex(shardContent) + writeFileSync( + generationManifestPath(archiveDir, signal, rangeDate, generationId), + `${JSON.stringify(manifest(generationId, signal, rangeDate, rowCount, shardSha, shardStat.size))}\n`, + ) +} + +describe("archive listing", () => { + it("lists the active generation with its shard paths and excludes superseded generations", async () => { + await withArchive(async (archiveDir) => { + const old = randomUUID() + const active = randomUUID() + seedSupersededGeneration(archiveDir, "traces", "2026-06-01", old, 5) + seedActiveGeneration(archiveDir, "traces", "2026-06-01", active, 10) + const listing = listActiveGenerations(archiveDir) + strictEqual(listing.active.length, 1) + const summary = listing.active[0]! + strictEqual(summary.generationId, active) + strictEqual(summary.archivedRowCount, 10) + strictEqual(summary.shardCount, 1) + strictEqual(summary.shardPaths.length, 1) + ok(summary.shardPaths[0]!.endsWith("00-0000.parquet")) + strictEqual(summary.shardBytes, 4) + deepStrictEqual(listing.signals, ["traces"]) + }) + }) + + it("returns empty when no archive exists", async () => { + await withArchive(async (archiveDir) => { + const listing = listActiveGenerations(archiveDir) + strictEqual(listing.active.length, 0) + }) + }) + + it("lists metadata without hashing shard contents", async () => { + await withArchive(async (archiveDir) => { + const generationId = randomUUID() + seedActiveGeneration(archiveDir, "logs", "2026-06-01", generationId, 3) + const shardPath = join( + shardsRoot(archiveDir, "logs", "2026-06-01", generationId), + "00-0000.parquet", + ) + writeFileSync(shardPath, "XXXX") + const listing = listActiveGenerations(archiveDir) + strictEqual(listing.integrity, "metadata-only") + strictEqual(listing.active.length, 1) + }) + }) + + it("explicit verification streams multi-buffer shards and rejects same-size tampering", async () => { + await withArchive(async (archiveDir) => { + const generationId = randomUUID() + const content = Buffer.alloc(ARCHIVE_VERIFY_BUFFER_BYTES * 2 + 17, 0x61) + const shardsDir = shardsRoot(archiveDir, "logs", "2026-06-01", generationId) + mkdirSync(shardsDir, { recursive: true }) + const shardPath = join(shardsDir, "00-0000.parquet") + writeFileSync(shardPath, content) + writeFileSync( + generationManifestPath(archiveDir, "logs", "2026-06-01", generationId), + `${JSON.stringify( + manifest( + generationId, + "logs", + "2026-06-01", + 3, + createHash("sha256").update(content).digest("hex"), + content.length, + ), + )}\n`, + ) + writeFileSync( + activePointerPath(archiveDir, "logs", "2026-06-01"), + `${JSON.stringify({ + formatVersion: 1, + generationId, + signal: "logs", + rangeStart: "2026-06-01", + selectedAt: "2026-06-02T00:00:00.000Z", + })}\n`, + ) + const verified = await verifyActiveGenerations(archiveDir, "logs") + strictEqual(verified.shardCount, 1) + strictEqual(verified.verifiedBytes, content.length) + content[0] = 0x62 + writeFileSync(shardPath, content) + await rejects(verifyActiveGenerations(archiveDir, "logs"), /SHA-256 mismatch/) + }) + }) + + it("resolves active parquet paths across ranges in ascending order", async () => { + await withArchive(async (archiveDir) => { + seedActiveGeneration(archiveDir, "logs", "2026-06-02", randomUUID(), 3) + seedActiveGeneration(archiveDir, "logs", "2026-06-01", randomUUID(), 7) + const paths = activeParquetPaths(archiveDir, "logs") + strictEqual(paths.length, 2) + // Ascending range order: June 1 before June 2. + ok(paths[0]!.includes("2026-06-01")) + ok(paths[1]!.includes("2026-06-02")) + }) + }) + + it("skips a malformed active pointer without hiding other ranges", async () => { + await withArchive(async (archiveDir) => { + seedActiveGeneration(archiveDir, "traces", "2026-06-01", randomUUID(), 4) + // Corrupt the pointer for a second range. + mkdirSync(join(archiveDir, "traces", "2026-06-02"), { recursive: true }) + writeFileSync(activePointerPath(archiveDir, "traces", "2026-06-02"), "{bad json") + const listing = listActiveGenerations(archiveDir) + strictEqual(listing.active.length, 1) + strictEqual(listing.active[0]!.rangeStart, "2026-06-01") + }) + }) + + it("activeParquetPaths throws when any relevant range is malformed (no partial DuckDB output)", async () => { + await withArchive(async (archiveDir) => { + seedActiveGeneration(archiveDir, "traces", "2026-06-01", randomUUID(), 4) + // Corrupt the pointer for a second range of the SAME signal. + mkdirSync(join(archiveDir, "traces", "2026-06-02"), { recursive: true }) + writeFileSync(activePointerPath(archiveDir, "traces", "2026-06-02"), "{bad json") + throwsSync(() => activeParquetPaths(archiveDir, "traces"), /malformed range/) + }) + }) + + it("activeParquetPaths succeeds when a DIFFERENT signal has errors", async () => { + await withArchive(async (archiveDir) => { + seedActiveGeneration(archiveDir, "traces", "2026-06-01", randomUUID(), 4) + // Corrupt a LOGS range; traces must still be queryable. + mkdirSync(join(archiveDir, "logs", "2026-06-02"), { recursive: true }) + writeFileSync(activePointerPath(archiveDir, "logs", "2026-06-02"), "{bad json") + const paths = activeParquetPaths(archiveDir, "traces") + strictEqual(paths.length, 1) + }) + }) +}) + +describe("archive catalog rebuild", () => { + it("rebuilds the catalog from manifests after truncation, including superseded generations", async () => { + await withArchive(async (archiveDir) => { + const old = randomUUID() + const active = randomUUID() + seedSupersededGeneration(archiveDir, "traces", "2026-06-01", old, 5) + seedActiveGeneration(archiveDir, "traces", "2026-06-01", active, 10) + // Truncate the catalog if it exists, then rebuild. + const entries = await rebuildCatalog(archiveDir, "traces") + // Both the superseded and the active generation appear, because the + // catalog indexes all retained generations. + strictEqual(entries.length, 2) + const ids = entries.map((e) => e.generationId).sort() + deepStrictEqual(ids, [active, old].sort()) + ok(existsSync(catalogPath(archiveDir, "traces"))) + }) + }) + + it("fails closed and preserves the existing catalog when a generation is missing its manifest", async () => { + await withArchive(async (archiveDir) => { + // Seed a valid active generation first and build its catalog. + seedActiveGeneration(archiveDir, "traces", "2026-06-01", randomUUID(), 8) + await rebuildCatalog(archiveDir, "traces") + const catalogFile = catalogPath(archiveDir, "traces") + const originalCatalog = readFileSync(catalogFile, "utf8") + // Add a stray generation dir with no manifest. + const stray = randomUUID() + mkdirSync(generationsRoot(archiveDir, "traces", "2026-06-01"), { recursive: true }) + mkdirSync(join(generationsRoot(archiveDir, "traces", "2026-06-01"), stray), { recursive: true }) + // Rebuild must THROW (preflight fails) and preserve the existing catalog. + await rejects(rebuildCatalog(archiveDir, "traces"), /missing its manifest/) + strictEqual( + readFileSync(catalogFile, "utf8"), + originalCatalog, + "existing catalog preserved on error", + ) + }) + }) + + it("produces a catalog with one valid JSON line per generation", async () => { + await withArchive(async (archiveDir) => { + seedActiveGeneration(archiveDir, "logs", "2026-06-01", randomUUID(), 12) + await rebuildCatalog(archiveDir, "logs") + const lines = readFileSync(catalogPath(archiveDir, "logs"), "utf8").trim().split("\n") + strictEqual(lines.length, 1) + const entry = JSON.parse(lines[0]!) as { signal: string; archivedRowCount: number } + strictEqual(entry.signal, "logs") + strictEqual(entry.archivedRowCount, 12) + }) + }) +}) + +const sha256Hex = (content: string): string => createHash("sha256").update(content).digest("hex") + +const throwsSync = (fn: () => unknown, pattern: RegExp): void => { + try { + fn() + throw new Error("expected function to throw") + } catch (error) { + const msg = error instanceof Error ? error.message : String(error) + if (!pattern.test(msg)) throw new Error(`expected ${pattern}, got: ${msg}`) + } +} diff --git a/apps/cli/test/archive-manifest.test.ts b/apps/cli/test/archive-manifest.test.ts new file mode 100644 index 000000000..098baf402 --- /dev/null +++ b/apps/cli/test/archive-manifest.test.ts @@ -0,0 +1,414 @@ +import { describe, it } from "@effect/vitest" +import { ok, strictEqual, throws } from "node:assert" +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { dirname, join } from "node:path" +import { + activePointerPath, + catalogPath, + generationManifestPath, + generationsRoot, + rangeRoot, +} from "../src/server/archives/paths" +import { parseArchiveActivePointer, parseArchiveGenerationManifest } from "../src/server/archives/manifest" +import { TUNING_CONFIG_FORMAT_VERSION } from "../src/server/archives/config" +import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +import { SCHEMA_FINGERPRINT } from "../src/server/serve" +import { randomUUID } from "node:crypto" + +const withArchive = async (run: (archiveDir: string) => Promise | void): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-archive-manifest-test-")) + const archiveDir = join(parent, "archive") + mkdirSync(archiveDir, { recursive: true }) + try { + await run(archiveDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const validGenerationManifest = (overrides: Record = {}) => ({ + formatVersion: 3, + generationId: randomUUID(), + signal: "traces", + rangeStart: "2026-06-01", + rangeEndExclusive: "2026-06-02T00:00:00.000Z", + checkpointId: randomUUID(), + checkpointManifestFingerprint: "cid:2026-01-01:100", + createdAt: "2026-06-02T00:00:00.000Z", + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + sourceRowCount: 100, + archivedRowCount: 100, + tuning: { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + targetChunkBytes: 1024 * 1024 * 1024, + minFreeSpaceReserve: 512 * 1024 * 1024, + }, + tuningConfig: null, + shards: [ + { + name: "00-0000.parquet", + rowCount: 100, + minEventTimeUnixNano: `${BigInt(Date.parse("2026-06-01T00:00:00.000Z")) * 1_000_000n}`, + maxEventTimeUnixNano: `${BigInt(Date.parse("2026-06-01T00:30:00.000Z")) * 1_000_000n}`, + sha256: "a".repeat(64), + bytes: 4096, + columns: ["TimestampTime", "ServiceName"], + complexDigest: "123456789", + complexDigestAlgorithm: "cityhash64-multiset-v3", + }, + ], + ...overrides, +}) + +describe("archive generation manifest parser", () => { + it("parses a valid manifest and binds it to its location", () => { + const generationId = randomUUID() + const manifest = validGenerationManifest({ generationId }) + const parsed = parseArchiveGenerationManifest(manifest, "traces", "2026-06-01", generationId) + strictEqual(parsed.generationId, generationId) + strictEqual(parsed.signal, "traces") + strictEqual(parsed.shards.length, 1) + strictEqual(parsed.shards[0]!.bytes, 4096) + }) + + it("rejects an unknown (future) format version", () => { + throws( + () => parseArchiveGenerationManifest({ ...validGenerationManifest(), formatVersion: 99 }), + /unsupported archive manifest formatVersion 99/, + ) + }) + + it("rejects an older (v1) format version fail-closed (round 5)", () => { + // A round-4 v1 manifest carried timezone-dependent time evidence and a + // commutative digest; the reader must not silently re-interpret it. + throws( + () => parseArchiveGenerationManifest({ ...validGenerationManifest(), formatVersion: 1 }), + /unsupported archive manifest formatVersion 1/, + ) + }) + + it("rejects a v2 manifest fail-closed (config-identity semantics changed)", () => { + // v3 introduced the structured, SHA-256-bound tuningConfig identity. A v2 + // manifest (bare tuningConfigName) is incompatible; silently treating the + // missing field as null would lose the config identity. Re-export required. + throws( + () => parseArchiveGenerationManifest({ ...validGenerationManifest(), formatVersion: 2 }), + /unsupported archive manifest formatVersion 2.*v3 introduced the structured tuningConfig identity/, + ) + }) + + it("rejects a signal mismatch with its directory", () => { + throws( + () => parseArchiveGenerationManifest(validGenerationManifest(), "logs", "2026-06-01"), + /signal mismatch/, + ) + }) + + it("rejects a range mismatch with its directory", () => { + throws( + () => parseArchiveGenerationManifest(validGenerationManifest(), "traces", "2026-06-02"), + /range mismatch/, + ) + }) + + it("rejects a generation id mismatch with its directory", () => { + throws( + () => + parseArchiveGenerationManifest( + validGenerationManifest(), + "traces", + "2026-06-01", + randomUUID(), + ), + /generation mismatch/, + ) + }) + + it("rejects an unknown signal name", () => { + throws( + () => parseArchiveGenerationManifest(validGenerationManifest({ signal: "bogus" })), + /unknown archive signal/, + ) + }) + + it("rejects a negative source row count", () => { + throws( + () => parseArchiveGenerationManifest(validGenerationManifest({ sourceRowCount: -1 })), + /sourceRowCount/, + ) + }) + + it("rejects a malformed shard name", () => { + const bad = validGenerationManifest({ + shards: [{ ...validGenerationManifest().shards[0]!, name: "../escape.parquet" }], + }) + throws(() => parseArchiveGenerationManifest(bad), /shard name/) + }) + + it("rejects a missing tuning block", () => { + const bad = validGenerationManifest() + delete (bad as Record).tuning + throws(() => parseArchiveGenerationManifest(bad), /tuning/) + }) + + it("rejects an unknown top-level key", () => { + throws( + () => parseArchiveGenerationManifest(validGenerationManifest({ rogue: true })), + /unknown archive manifest field: rogue/, + ) + }) + + it("reads a manifest from disk bound to its location", async () => { + await withArchive(async (archiveDir) => { + const { readArchiveGenerationManifest } = await import("../src/server/archives/manifest") + const generationId = randomUUID() + const manifestPath = generationManifestPath(archiveDir, "traces", "2026-06-01", generationId) + mkdirSync(dirname(manifestPath), { recursive: true }) + writeFileSync(manifestPath, `${JSON.stringify(validGenerationManifest({ generationId }))}\n`) + const read = readArchiveGenerationManifest(archiveDir, "traces", "2026-06-01", generationId) + strictEqual(read.generationId, generationId) + }) + }) +}) + +describe("archive active pointer parser", () => { + it("parses a valid pointer", () => { + const parsed = parseArchiveActivePointer({ + formatVersion: 1, + generationId: randomUUID(), + signal: "logs", + rangeStart: "2026-06-01", + selectedAt: "2026-06-02T00:00:00.000Z", + }) + strictEqual(parsed.signal, "logs") + }) + + it("rejects an unknown format version", () => { + throws( + () => + parseArchiveActivePointer({ + formatVersion: 3, + generationId: randomUUID(), + signal: "logs", + rangeStart: "2026-06-01", + selectedAt: "2026-06-02T00:00:00.000Z", + }), + /unsupported/, + ) + }) +}) + +describe("archive path model", () => { + it("places generations under signal/range/generations", async () => { + await withArchive(async (archiveDir) => { + const gen = generationsRoot(archiveDir, "traces", "2026-06-01") + ok(gen.endsWith(join("traces", "2026-06-01", "generations"))) + }) + }) + + it("rejects an invalid range date", () => { + throws(() => rangeRoot("/tmp/a", "traces", "2026-6-1"), /range date/) + }) + + it("rejects a malformed generation id in path construction", () => { + throws(() => generationManifestPath("/tmp/a", "traces", "2026-06-01", "not-a-uuid"), /generation/) + }) + + it("catalog and active pointer live under the signal/range roots", async () => { + await withArchive(async (archiveDir) => { + const cat = catalogPath(archiveDir, "traces") + ok(cat.endsWith(join("traces", "catalog.jsonl"))) + const active = activePointerPath(archiveDir, "traces", "2026-06-01") + ok(active.endsWith(join("traces", "2026-06-01", "active.json"))) + }) + }) +}) + +describe("archive manifest tuningConfig identity", () => { + it("parses a structured tuningConfig identity (configName + sha256 + formatVersion)", () => { + const generationId = randomUUID() + const manifest = validGenerationManifest({ + generationId, + tuningConfig: { formatVersion: 1, configName: "calib-2026.json", sha256: "b".repeat(64) }, + }) + const parsed = parseArchiveGenerationManifest(manifest, "traces", "2026-06-01", generationId) + ok(parsed.tuningConfig !== null) + strictEqual(parsed.tuningConfig!.configName, "calib-2026.json") + strictEqual(parsed.tuningConfig!.sha256, "b".repeat(64)) + strictEqual(parsed.tuningConfig!.formatVersion, 1) + }) + + it("preserves a loaded format-2 calibration config identity", () => { + const generationId = randomUUID() + const manifest = validGenerationManifest({ + generationId, + tuningConfig: { formatVersion: 2, configName: "phase3b-tuning.json", sha256: "c".repeat(64) }, + }) + const parsed = parseArchiveGenerationManifest(manifest, "traces", "2026-06-01", generationId) + strictEqual(parsed.tuningConfig!.formatVersion, 2) + strictEqual(parsed.tuningConfig!.configName, "phase3b-tuning.json") + }) + + it("preserves a current format-3 calibration config identity", () => { + const generationId = randomUUID() + const manifest = validGenerationManifest({ + generationId, + tuningConfig: { + formatVersion: TUNING_CONFIG_FORMAT_VERSION, + configName: "new-tuning.json", + sha256: "d".repeat(64), + }, + }) + const parsed = parseArchiveGenerationManifest(manifest, "traces", "2026-06-01", generationId) + strictEqual(parsed.tuningConfig!.formatVersion, TUNING_CONFIG_FORMAT_VERSION) + }) + + it("accepts null tuningConfig (no config loaded)", () => { + const generationId = randomUUID() + const manifest = validGenerationManifest({ generationId, tuningConfig: null }) + const parsed = parseArchiveGenerationManifest(manifest, "traces", "2026-06-01", generationId) + strictEqual(parsed.tuningConfig, null) + }) + + it("rejects a v3 manifest missing the tuningConfig key", () => { + const manifest = validGenerationManifest() + delete (manifest as Record).tuningConfig + throws(() => parseArchiveGenerationManifest(manifest), /tuningConfig \(required in formatVersion 3\)/) + }) + + it("rejects a legacy-shaped v3 manifest with tuningConfigName but no tuningConfig key", () => { + const manifest = validGenerationManifest({ tuningConfigName: "calib-2026.json" }) + delete (manifest as Record).tuningConfig + throws(() => parseArchiveGenerationManifest(manifest), /tuningConfig \(required in formatVersion 3\)/) + }) + + it("rejects a tuningConfig with a malformed sha256", () => { + const generationId = randomUUID() + const manifest = validGenerationManifest({ + generationId, + tuningConfig: { formatVersion: 1, configName: "c.json", sha256: "tooshort" }, + }) + throws( + () => parseArchiveGenerationManifest(manifest, "traces", "2026-06-01", generationId), + /must be 64 hex chars/, + ) + }) + + it("rejects an unknown tuningConfig subfield", () => { + const generationId = randomUUID() + const manifest = validGenerationManifest({ + generationId, + tuningConfig: { formatVersion: 1, configName: "c.json", sha256: "b".repeat(64), rogue: "x" }, + }) + throws( + () => parseArchiveGenerationManifest(manifest, "traces", "2026-06-01", generationId), + /unknown archive manifest tuningConfig field: rogue/, + ) + }) + + it("rejects an unsafe tuningConfig configName", () => { + const generationId = randomUUID() + const manifest = validGenerationManifest({ + generationId, + tuningConfig: { formatVersion: 1, configName: "../evil", sha256: "b".repeat(64) }, + }) + throws( + () => parseArchiveGenerationManifest(manifest, "traces", "2026-06-01", generationId), + /unsafe name/, + ) + }) +}) + +describe("archive manifest tuning", () => { + it("rejects all-zero tuning", () => { + throws( + () => + parseArchiveGenerationManifest( + validGenerationManifest({ + tuning: { + writerThreads: 0, + rowGroupRows: 0, + maxShardRows: 0, + maxShardBytes: 0, + targetChunkBytes: 0, + minFreeSpaceReserve: 0, + }, + }), + ), + /tuning field: writerThreads \(must be a positive integer\)/, + ) + }) + + it("rejects writerThreads greater than 32", () => { + const tuning = validGenerationManifest().tuning + throws( + () => + parseArchiveGenerationManifest( + validGenerationManifest({ tuning: { ...tuning, writerThreads: 33 } }), + ), + /writerThreads must not exceed 32/, + ) + }) + + it("rejects rowGroupRows greater than maxShardRows", () => { + const tuning = validGenerationManifest().tuning + throws( + () => + parseArchiveGenerationManifest( + validGenerationManifest({ + tuning: { ...tuning, rowGroupRows: tuning.maxShardRows + 1 }, + }), + ), + /rowGroupRows must not exceed maxShardRows/, + ) + }) + + it("rejects maxShardBytes smaller than rowGroupRows * 1024", () => { + const tuning = validGenerationManifest().tuning + throws( + () => + parseArchiveGenerationManifest( + validGenerationManifest({ + tuning: { + ...tuning, + maxShardBytes: tuning.rowGroupRows * 1024 - 1, + }, + }), + ), + /maxShardBytes .* is too small for rowGroupRows/, + ) + }) + + it("rejects minFreeSpaceReserve greater than or equal to targetChunkBytes", () => { + const tuning = validGenerationManifest().tuning + throws( + () => + parseArchiveGenerationManifest( + validGenerationManifest({ + tuning: { + ...tuning, + minFreeSpaceReserve: tuning.targetChunkBytes, + }, + }), + ), + /minFreeSpaceReserve must be smaller than targetChunkBytes/, + ) + }) + + it("rejects an unknown tuning key", () => { + const tuning = validGenerationManifest().tuning + throws( + () => + parseArchiveGenerationManifest( + validGenerationManifest({ tuning: { ...tuning, rogue: true } }), + ), + /unknown archive manifest tuning field: rogue/, + ) + }) +}) diff --git a/apps/cli/test/archive-path-safety.test.ts b/apps/cli/test/archive-path-safety.test.ts new file mode 100644 index 000000000..30b20ada0 --- /dev/null +++ b/apps/cli/test/archive-path-safety.test.ts @@ -0,0 +1,328 @@ +import { describe, it } from "@effect/vitest" +import { ok, rejects, strictEqual } from "node:assert" +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { createHash, randomUUID } from "node:crypto" +import { + activePointerPath, + assertNoSymlink, + assertNoSymlinkSync, + buildingGenerationRoot, + catalogPath, + ensurePrivateDirectory, + generationManifestPath, + rangeRoot, + shardsRoot, +} from "../src/server/archives/paths" +import { appendCatalog, promoteGeneration } from "../src/server/archives/generation" +import { rebuildCatalog, listActiveGenerations } from "../src/server/archives/listing" +import { type ArchiveGenerationManifest } from "../src/server/archives/manifest" +import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +import { SCHEMA_FINGERPRINT } from "../src/server/serve" + +// External-sentinel tests proving archive-root path escapes are closed (C-1). +// These mirror the reviewer's deterministic probes: a symlinked descendant must +// fail closed and the outside target must be untouched. + +const withArchive = async ( + run: (archiveDir: string, outside: string) => Promise | void, +): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-archive-pathsafe-")) + const archiveDir = join(parent, "archive") + const outside = join(parent, "outside") + mkdirSync(archiveDir, { recursive: true }) + mkdirSync(outside, { recursive: true }) + try { + await run(archiveDir, outside) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const manifest = ( + generationId: string, + signal = "traces", + rowCount = 10, + shardSha = "a".repeat(64), + shardBytes = 4096, +): ArchiveGenerationManifest => ({ + formatVersion: 3, + generationId, + signal, + rangeStart: "2026-06-01", + rangeEndExclusive: "2026-06-02T00:00:00.000Z", + checkpointId: randomUUID(), + checkpointManifestFingerprint: "cid:2026-01-01:100", + createdAt: "2026-06-02T00:00:00.000Z", + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + sourceRowCount: rowCount, + archivedRowCount: rowCount, + tuning: { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + targetChunkBytes: 1024 * 1024 * 1024, + minFreeSpaceReserve: 512 * 1024 * 1024, + }, + tuningConfig: null, + shards: [ + { + name: "00-0000.parquet", + rowCount, + minEventTimeUnixNano: `${BigInt(Date.parse("2026-06-01T00:00:00.000Z")) * 1_000_000n}`, + maxEventTimeUnixNano: `${BigInt(Date.parse("2026-06-01T00:30:00.000Z")) * 1_000_000n}`, + sha256: shardSha, + bytes: shardBytes, + columns: ["TimestampTime", "ServiceName"], + complexDigest: "123456789", + complexDigestAlgorithm: "cityhash64-multiset-v3", + }, + ], +}) + +const seedBuilding = (archiveDir: string, generationId: string): string => { + const building = buildingGenerationRoot(archiveDir, generationId) + const shards = join(building, "shards") + mkdirSync(shards, { recursive: true }) + writeFileSync(join(shards, "00.parquet"), "PAR1-placeholder") + return building +} + +describe("archive path safety — symlink escapes (C-1)", () => { + it("ensurePrivateDirectory creates a fresh archive root and nested dir safely", async () => { + await withArchive(async (archiveDir) => { + // A completely fresh nested path: archive root exists but the signal/ + // range/generations chain does not. Must create all without ENOENT. + const nested = join(archiveDir, "traces", "2026-06-01", "generations", "sub") + await ensurePrivateDirectory(nested, archiveDir) + ok(existsSync(nested), "nested dir created") + }) + }) + + it("ensurePrivateDirectory refuses a symlinked ancestor beneath the archive root", async () => { + await withArchive(async (archiveDir, outside) => { + // /traces -> outside/traces-evil + mkdirSync(join(outside, "traces-evil"), { recursive: true }) + symlinkSync(join(outside, "traces-evil"), join(archiveDir, "traces")) + // Creating a dir under the symlinked signal root must fail. + await rejects( + ensurePrivateDirectory(join(archiveDir, "traces", "2026-06-01"), archiveDir), + /symlink/, + ) + }) + }) + + it("assertNoSymlink refuses a symlinked signal root", async () => { + await withArchive(async (archiveDir, outside) => { + mkdirSync(join(outside, "traces-evil"), { recursive: true }) + symlinkSync(join(outside, "traces-evil"), join(archiveDir, "traces")) + await rejects( + assertNoSymlink(archiveDir, rangeRoot(archiveDir, "traces", "2026-06-01"), "test"), + /symlink/, + ) + }) + }) + + it("assertNoSymlinkSync refuses a symlinked catalog path", async () => { + await withArchive(async (archiveDir, outside) => { + const sentinel = join(outside, "catalog-sentinel.jsonl") + writeFileSync(sentinel, "sensitive") + mkdirSync(signalRootPath(archiveDir, "traces"), { recursive: true }) + symlinkSync(sentinel, catalogPath(archiveDir, "traces")) + throwsSync( + () => assertNoSymlinkSync(archiveDir, catalogPath(archiveDir, "traces"), "catalog"), + /symlink/, + ) + // The outside sentinel is untouched. + strictEqual(readFileSync(sentinel, "utf8"), "sensitive") + }) + }) + + it("promoteGeneration refuses a symlinked signal root and does not write outside", async () => { + await withArchive(async (archiveDir, outside) => { + // Point the signal root at an outside dir. + mkdirSync(join(outside, "traces-out"), { recursive: true }) + symlinkSync(join(outside, "traces-out"), join(archiveDir, "traces")) + const generationId = randomUUID() + const building = seedBuilding(archiveDir, generationId) + await rejects( + promoteGeneration( + archiveDir, + "traces", + "2026-06-01", + generationId, + manifest(generationId), + building, + {}, + ), + /symlink/, + ) + // No generation manifest or pointer was created outside the root. + ok( + !existsSync( + join(outside, "traces-out", "2026-06-01", "generations", generationId, "manifest.json"), + ), + ) + ok(!existsSync(join(outside, "traces-out", "2026-06-01", "active.json"))) + }) + }) + + it("appendCatalog refuses a symlinked catalog and leaves the outside target untouched", async () => { + await withArchive(async (archiveDir, outside) => { + const sentinel = join(outside, "catalog-sentinel.jsonl") + writeFileSync(sentinel, "preserve") + mkdirSync(signalRootPath(archiveDir, "traces"), { recursive: true }) + symlinkSync(sentinel, catalogPath(archiveDir, "traces")) + await rejects(appendCatalog(archiveDir, "traces", manifest(randomUUID())), /symlink|real file/) + strictEqual(readFileSync(sentinel, "utf8"), "preserve") + }) + }) + + it("rebuildCatalog refuses a symlinked catalog and leaves the outside target untouched", async () => { + await withArchive(async (archiveDir, outside) => { + const sentinel = join(outside, "catalog-sentinel.jsonl") + writeFileSync(sentinel, "preserve") + mkdirSync(signalRootPath(archiveDir, "traces"), { recursive: true }) + symlinkSync(sentinel, catalogPath(archiveDir, "traces")) + await rejects(rebuildCatalog(archiveDir, "traces"), /symlink/) + strictEqual(readFileSync(sentinel, "utf8"), "preserve") + }) + }) + + it("listActiveGenerations surfaces a malformed pointer as an error instead of silently skipping", async () => { + await withArchive(async (archiveDir) => { + // A valid active generation for traces/2026-06-01. + const generationId = randomUUID() + const shardsDir = shardsRoot(archiveDir, "traces", "2026-06-01", generationId) + mkdirSync(shardsDir, { recursive: true }) + const shardContent = "PAR1" + writeFileSync(join(shardsDir, "00-0000.parquet"), shardContent) + const shardSha = createHash("sha256").update(shardContent).digest("hex") + const shardBytes = shardContent.length + writeFileSync( + generationManifestPath(archiveDir, "traces", "2026-06-01", generationId), + `${JSON.stringify(manifest(generationId, undefined, undefined, shardSha, shardBytes))}\n`, + ) + writeFileSync( + activePointerPath(archiveDir, "traces", "2026-06-01"), + `${JSON.stringify({ formatVersion: 1, generationId, signal: "traces", rangeStart: "2026-06-01", selectedAt: "2026-06-02T00:00:00.000Z" })}\n`, + ) + // A malformed pointer for a second range. + mkdirSync(rangeRoot(archiveDir, "traces", "2026-06-02"), { recursive: true }) + writeFileSync(activePointerPath(archiveDir, "traces", "2026-06-02"), "{bad json") + const listing = listActiveGenerations(archiveDir) + strictEqual(listing.active.length, 1) + strictEqual(listing.active[0]!.rangeStart, "2026-06-01") + ok(listing.errors.length >= 1, "malformed pointer surfaced as error") + ok( + listing.errors.some((e) => e.rangeStart === "2026-06-02"), + "the corrupt range is named", + ) + }) + }) + + it("listActiveGenerations rejects a pointer whose recorded signal/range mismatches its directory", async () => { + await withArchive(async (archiveDir) => { + // Pointer physically under traces/2026-06-01 but claiming logs/2025-01-01. + mkdirSync(rangeRoot(archiveDir, "traces", "2026-06-01"), { recursive: true }) + writeFileSync( + activePointerPath(archiveDir, "traces", "2026-06-01"), + `${JSON.stringify({ formatVersion: 1, generationId: randomUUID(), signal: "logs", rangeStart: "2025-01-01", selectedAt: "2026-06-02T00:00:00.000Z" })}\n`, + ) + const listing = listActiveGenerations(archiveDir) + strictEqual(listing.active.length, 0) + ok(listing.errors.some((e) => e.rangeStart === "2026-06-01" && /mismatch/.test(e.error))) + }) + }) +}) + +describe("archive path safety — read-side symlink escapes (HIGH-1/HIGH-2)", () => { + it("readArchiveGenerationManifest refuses a symlinked generation dir and does not read outside content", async () => { + await withArchive(async (archiveDir, outside) => { + const { readArchiveGenerationManifest } = await import("../src/server/archives/manifest") + const generationId = randomUUID() + // Plant a real generation outside the root with an attacker manifest. + const outsideGen = join(outside, "evil-gen") + mkdirSync(join(outsideGen, "shards"), { recursive: true }) + writeFileSync(join(outsideGen, "manifest.json"), `${JSON.stringify(manifest(generationId))}\n`) + writeFileSync(join(outsideGen, "shards", "00.parquet"), "ATTACKER-SHARDS") + // Symlink the in-root generation dir at the outside one. + mkdirSync(generationsRootPath(archiveDir, "traces", "2026-06-01"), { recursive: true }) + symlinkSync( + outsideGen, + join(generationsRootPath(archiveDir, "traces", "2026-06-01"), generationId), + ) + throwsSync( + () => readArchiveGenerationManifest(archiveDir, "traces", "2026-06-01", generationId), + /symlink/, + ) + }) + }) + + it("listActiveGenerations refuses a symlinked shard path and surfaces it as an error", async () => { + await withArchive(async (archiveDir, outside) => { + const generationId = randomUUID() + // Build a valid generation with a real manifest + pointer. + const shardsDir = shardsRoot(archiveDir, "traces", "2026-06-01", generationId) + mkdirSync(shardsDir, { recursive: true }) + writeFileSync(join(shardsDir, "00.parquet"), "PAR1") + writeFileSync( + generationManifestPath(archiveDir, "traces", "2026-06-01", generationId), + `${JSON.stringify(manifest(generationId))}\n`, + ) + writeFileSync( + activePointerPath(archiveDir, "traces", "2026-06-01"), + `${JSON.stringify({ formatVersion: 1, generationId, signal: "traces", rangeStart: "2026-06-01", selectedAt: "2026-06-02T00:00:00.000Z" })}\n`, + ) + // Now symlink the shard file at an outside target. + writeFileSync(join(outside, "evil.parquet"), "ATTACKER-PARQUET") + rmSync(join(shardsDir, "00.parquet")) + symlinkSync(join(outside, "evil.parquet"), join(shardsDir, "00.parquet")) + const listing = listActiveGenerations(archiveDir) + // The symlinked shard must not appear in active paths. + strictEqual(listing.active.length, 0) + ok( + listing.errors.some((e) => /shard path/.test(e.error)), + "symlinked shard surfaced as error", + ) + }) + }) + + it("rebuildCatalog does not trust a symlinked generation dir's manifest", async () => { + await withArchive(async (archiveDir, outside) => { + const generationId = randomUUID() + // Plant attacker manifest outside the root. + const outsideGen = join(outside, "evil-gen") + mkdirSync(join(outsideGen, "shards"), { recursive: true }) + writeFileSync(join(outsideGen, "manifest.json"), `${JSON.stringify(manifest(generationId))}\n`) + // Symlink the in-root generation at the outside one. + mkdirSync(generationsRootPath(archiveDir, "traces", "2026-06-01"), { recursive: true }) + symlinkSync( + outsideGen, + join(generationsRootPath(archiveDir, "traces", "2026-06-01"), generationId), + ) + // rebuildCatalog must THROW (symlink detected at preflight) and not + // trust the attacker's manifest. + await rejects(rebuildCatalog(archiveDir, "traces"), /symlink/) + }) + }) +}) + +const signalRootPath = (archiveDir: string, signal: string): string => join(archiveDir, signal) + +const generationsRootPath = (archiveDir: string, signal: string, rangeDate: string): string => + join(archiveDir, signal, rangeDate, "generations") + +const throwsSync = (fn: () => unknown, pattern: RegExp): void => { + try { + fn() + throw new Error("expected function to throw") + } catch (error) { + const msg = error instanceof Error ? error.message : String(error) + if (!pattern.test(msg)) throw new Error(`expected ${pattern}, got: ${msg}`) + } +} diff --git a/apps/cli/test/archive-pins.test.ts b/apps/cli/test/archive-pins.test.ts new file mode 100644 index 000000000..601ddf99f --- /dev/null +++ b/apps/cli/test/archive-pins.test.ts @@ -0,0 +1,273 @@ +import { describe, it } from "@effect/vitest" +import { ok, rejects, strictEqual } from "node:assert" +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs" +import { tmpdir } from "node:os" +import { dirname, join } from "node:path" +import { + acquireCheckpointPin, + checkpointPinsRoot, + checkpointRoot, + checkpointSnapshotDir, + checkpointStatePath, + newCheckpointId, + releaseCheckpointPin, + retireCheckpointIfEligible, + readCheckpointState, + withMaintenanceLock, +} from "../src/server/checkpoints" +import { SCHEMA_FINGERPRINT } from "../src/server/serve" +import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" + +// Pin and maintenance-lock API for the dependent archive branch. These tests +// mirror the checkpoint suite's conventions: a throwaway parent/data dir, +// hand-built fixtures (no real chDB), node:assert, and fail-closed assertions +// proving uncertain state is preserved rather than deleted. +const withDataDir = async (run: (dataDir: string) => Promise | void): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-archive-pin-test-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir, { recursive: true }) + try { + await run(dataDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const manifest = (checkpointId: string, sourceDataDir: string) => ({ + formatVersion: 1 as const, + checkpointId, + operationId: newCheckpointId(), + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + createdAt: new Date().toISOString(), + sourceDataDir, + backupRelativePath: `snapshots/${checkpointId}/backup`, + backupBytes: 6, + validation: { + validatedAt: new Date().toISOString(), + traces: 0, + logs: 0, + metricsSum: 0, + metricsGauge: 0, + metricsHistogram: 0, + metricsExponentialHistogram: 0, + materializedViews: 0, + }, +}) + +const writeSnapshot = (dataDir: string, checkpointId: string): void => { + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + mkdirSync(join(snapshot, "backup"), { recursive: true }) + writeFileSync(join(snapshot, "backup", "data.bin"), "backup") + writeFileSync(join(snapshot, "manifest.json"), `${JSON.stringify(manifest(checkpointId, dataDir))}\n`) +} + +const writeState = (dataDir: string, current: string, previous: string | null = null): void => { + mkdirSync(checkpointRoot(dataDir), { recursive: true }) + writeFileSync( + checkpointStatePath(dataDir), + `${JSON.stringify({ + formatVersion: 1, + revision: newCheckpointId(), + current, + previous, + committedAt: "2026-01-01T00:00:02.000Z", + })}\n`, + ) +} + +describe("checkpoint pin API", () => { + it("acquires a persistent pin that prevents retirement and resolves by exact identity", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId) + writeState(dataDir, checkpointId) + const pinPath = await acquireCheckpointPin(dataDir, checkpointId, "archive") + ok(existsSync(pinPath), "pin file written") + const parsed = JSON.parse(readFileSync(pinPath, "utf8")) as { + checkpointId: string + purpose: string + pinId: string + } + strictEqual(parsed.checkpointId, checkpointId) + strictEqual(parsed.purpose, "archive") + ok(parsed.pinId.length > 0) + strictEqual(readdirSync(join(checkpointPinsRoot(dataDir), checkpointId)).length, 1) + }) + }) + + it("a pinned unreferenced snapshot is retained by retirement", async () => { + await withDataDir(async (dataDir) => { + const current = newCheckpointId() + const old = newCheckpointId() + writeSnapshot(dataDir, current) + writeSnapshot(dataDir, old) + writeState(dataDir, current, old) + const state = await readCheckpointState(dataDir) + await acquireCheckpointPin(dataDir, old) + // old is neither current nor previous-resolvable-after-promotion; with a + // pin it must survive retirement. + const retired = await retireCheckpointIfEligible(dataDir, old, state) + strictEqual(retired, null, "pinned snapshot not retired") + ok(existsSync(checkpointSnapshotDir(dataDir, old)), "pinned snapshot retained") + }) + }) + + it("releases an owned pin and removes only that pin file", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId) + writeState(dataDir, checkpointId) + const pinPath = await acquireCheckpointPin(dataDir, checkpointId) + await releaseCheckpointPin(dataDir, checkpointId, pinPath) + ok(!existsSync(pinPath), "pin file removed") + }) + }) + + it("preserves an owned pin when its purpose does not match the journal expectation", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId) + writeState(dataDir, checkpointId) + const pinPath = await acquireCheckpointPin(dataDir, checkpointId, "archive:generation-a") + await rejects( + releaseCheckpointPin(dataDir, checkpointId, pinPath, "archive:generation-b"), + /identity mismatch/, + ) + ok(existsSync(pinPath), "purpose-mismatched pin preserved") + }) + }) + + it("fails closed and preserves a pin whose recorded identity does not match", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + const other = newCheckpointId() + writeSnapshot(dataDir, checkpointId) + writeSnapshot(dataDir, other) + writeState(dataDir, checkpointId, other) + // A pin file physically under `checkpointId`'s pin dir, but whose recorded + // checkpointId is `other`. Releasing it as `checkpointId` must refuse. + const pinDir = join(checkpointPinsRoot(dataDir), checkpointId) + mkdirSync(pinDir, { recursive: true }) + const bogusPinPath = join(pinDir, `${newCheckpointId()}.json`) + writeFileSync( + bogusPinPath, + `${JSON.stringify({ + formatVersion: 1, + pinId: newCheckpointId(), + checkpointId: other, + purpose: "archive", + createdAt: new Date().toISOString(), + })}\n`, + ) + await rejects(releaseCheckpointPin(dataDir, checkpointId, bogusPinPath), /identity mismatch/) + ok(existsSync(bogusPinPath), "mismatched pin preserved") + }) + }) + + it("fails closed when releasing an already-absent pin", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId) + writeState(dataDir, checkpointId) + const pinPath = await acquireCheckpointPin(dataDir, checkpointId) + await releaseCheckpointPin(dataDir, checkpointId, pinPath) + await rejects(releaseCheckpointPin(dataDir, checkpointId, pinPath), /not found/) + }) + }) + + it("refuses to pin a checkpoint that is not selected", async () => { + await withDataDir(async (dataDir) => { + const missing = newCheckpointId() + writeSnapshot(dataDir, missing) + // No state.json selecting `missing` -> resolveCheckpoint fails closed. + await rejects(acquireCheckpointPin(dataDir, missing), /state not found|refusing to infer/) + }) + }) + + it("rejects an invalid pin purpose", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId) + writeState(dataDir, checkpointId) + await rejects(acquireCheckpointPin(dataDir, checkpointId, "bad;purpose"), /purpose/) + }) + }) + + it("refuses a pin path that escapes the checkpoint pin directory", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId) + writeState(dataDir, checkpointId) + await acquireCheckpointPin(dataDir, checkpointId) + await rejects(releaseCheckpointPin(dataDir, checkpointId, "/tmp/escape.json"), /pin directory/) + }) + }) + + it("rejects a symlinked pin reservation without touching the target", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId) + writeState(dataDir, checkpointId) + const outside = join(dirname(dataDir), "outside-pin-target") + mkdirSync(outside, { recursive: true }) + writeFileSync(join(outside, "sensitive"), "preserve") + // Point the pin reservation dir at an outside target. + mkdirSync(checkpointPinsRoot(dataDir), { recursive: true }) + symlinkSync(outside, join(checkpointPinsRoot(dataDir), checkpointId)) + await rejects(acquireCheckpointPin(dataDir, checkpointId), /symlink/) + // The outside target is untouched. + strictEqual(readFileSync(join(outside, "sensitive"), "utf8"), "preserve") + }) + }) +}) + +describe("maintenance lock", () => { + it("runs the task and releases the lock on success", async () => { + await withDataDir(async (dataDir) => { + const result = await withMaintenanceLock(dataDir, newCheckpointId(), async () => "done") + strictEqual(result, "done") + ok(!existsSync(`${dataDir}.maple-maintenance-lock`), "maintenance lock released") + }) + }) + + it("releases the lock even when the task throws", async () => { + await withDataDir(async (dataDir) => { + await rejects( + withMaintenanceLock(dataDir, newCheckpointId(), async () => { + throw new Error("boom") + }), + /boom/, + ) + ok(!existsSync(`${dataDir}.maple-maintenance-lock`), "maintenance lock released after failure") + }) + }) + + it("rejects a concurrent owner while the lock is held", async () => { + await withDataDir(async (dataDir) => { + // Acquire the lock directly and hold it across the second attempt. + const firstOperation = newCheckpointId() + const held = await withMaintenanceLock(dataDir, firstOperation, async () => { + // While this call holds the lock, a second acquirer must be refused. + // The first owner's PID is this process (alive), so the refusal is + // "another Maple maintenance operation is active". + await rejects( + withMaintenanceLock(dataDir, newCheckpointId(), async () => "nope"), + /active/, + ) + }) + await held + ok(!existsSync(`${dataDir}.maple-maintenance-lock`), "lock released after nested refusal") + }) + }) +}) diff --git a/apps/cli/test/archive-probe-helpers.ts b/apps/cli/test/archive-probe-helpers.ts new file mode 100644 index 000000000..ca6784973 --- /dev/null +++ b/apps/cli/test/archive-probe-helpers.ts @@ -0,0 +1,106 @@ +// Hermetic helpers for archive adversarial probes. +// +// Every probe must run from a fresh clone with an otherwise-empty /tmp, use an +// OWNED mkdtemp directory (never a fixed /tmp path), and clean only its own +// state. This module gives probes a uniform lifecycle and a consistent contract: +// exit nonzero when corruption is ACCEPTED (the bug is present), exit zero when +// corruption is correctly REJECTED (the fix is present). +// +// Usage: +// const h = await ArchiveProbe.create("digest-column-swap") +// const db = h.openDb(SCHEMA) +// try { ... h.ok("message") } finally { await h.cleanup() } + +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Chdb } from "../src/server/chdb" + +export interface ArchiveProbeHandle { + /** The probe's owned root directory (created via mkdtemp, unique per run). */ + readonly root: string + /** A subdirectory for chDB data, unique per probe. */ + readonly dataDir: string + /** A subdirectory for shard output, unique per probe. */ + readonly outDir: string + /** + * Open a chDB connection. `dbSubdir` selects an independent data directory + * (default "db"); use distinct subdirs when a probe needs multiple separate + * databases (chDB allows only one LIVE connection, but successive connections + * over the SAME data dir see persisted rows — so two logically-distinct + * sources must use different subdirs). + */ + openDb: (schemaSql: string, dbSubdir?: string) => Chdb + /** Report success (corruption correctly rejected) and exit 0. */ + ok: (message: string) => never + /** Report failure (corruption accepted — the bug is present) and exit 1. */ + fail: (message: string) => never + /** Remove the owned root. Safe to call in finally; idempotent. */ + cleanup: () => void +} + +const slug = (name: string): string => name.replace(/[^a-z0-9-]/gi, "-").slice(0, 48) + +export const ArchiveProbe = { + /** Create an owned, unique probe workspace under the system temp dir. */ + create(name: string): ArchiveProbeHandle { + const root = mkdtempSync(join(tmpdir(), `maple-archive-${slug(name)}-`)) + const dataDir = join(root, "db") + const outDir = join(root, "out") + const connections: Chdb[] = [] + const handle: ArchiveProbeHandle = { + root, + dataDir, + outDir, + openDb: (schemaSql: string, dbSubdir = "db") => { + // chDB allows exactly one live connection per process; open serially. + // Distinct dbSubdir => independent persisted database, so a probe that + // needs two logically-separate sources (e.g. original vs swapped) + // must pass different subdirs (successive connections over the SAME + // data dir see the prior rows). + const dd = join(root, dbSubdir) + const db = Chdb.open({ dataDir: dd, schemaSql, bootstrapSchema: true }) + connections.push(db) + return db + }, + ok: (message: string): never => { + console.log(`PASS: ${message}`) + handle.cleanup() + process.exit(0) + }, + fail: (message: string): never => { + console.error(`FAIL: ${message}`) + handle.cleanup() + process.exit(1) + }, + cleanup: () => { + // Close any connection that is still open (best effort — chDB close + // is safe to call once). Then remove only this probe's owned root. + for (const db of connections) { + try { + db.close() + } catch { + // best effort + } + } + connections.length = 0 + rmSync(root, { recursive: true, force: true }) + }, + } + return handle + }, +} + +/** Parse JSONEachRow text into an array of objects. */ +export const readRows = (text: string): ReadonlyArray> => + text + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as Record) + +/** First JSONEachRow row, or undefined. */ +export const firstRow = (text: string): Record | undefined => readRows(text)[0] + +/** Escape a path for a ClickHouse single-quoted literal. */ +export const sqlLiteral = (path: string): string => path.replace(/\\/g, "\\\\").replace(/'/g, "\\'") diff --git a/apps/cli/test/archive-reconcile-decision.test.ts b/apps/cli/test/archive-reconcile-decision.test.ts new file mode 100644 index 000000000..5b791551a --- /dev/null +++ b/apps/cli/test/archive-reconcile-decision.test.ts @@ -0,0 +1,299 @@ +import { describe, it } from "@effect/vitest" +import { strictEqual } from "node:assert" +import { randomUUID } from "node:crypto" +import { + decideReconciliation, + digestOfIntent, + type ReconciliationInspection, + type ReconciliationSnapshot, +} from "../src/server/archives/reconcile" +import type { CreateOperationIntent, GcOperationIntent } from "../src/server/archives/journal" + +// Exhaustive transition-table tests for the pure decision function (Gate 3b r5). +// These test the SOLE branch logic with NO I/O: every (kind, phase, topology) +// row maps to exactly one decision. The wiring commit will add integration tests +// proving CLI/create/GC route through this same function. + +// Use the indexed-access type so we don't import ArchiveOperationPhase (which +// oxlint's no-unused-vars flags in type-only positions). +const createIntent = (phase: CreateOperationIntent["phase"]): CreateOperationIntent => { + const gid = randomUUID() + return { + formatVersion: 3, + kind: "create", + operationId: randomUUID(), + generationId: gid, + signal: "traces", + rangeStart: "2026-06-01", + checkpointId: randomUUID(), + archiveDir: "/archive", + dataDir: "/data", + scratchRoot: "/scratch", + pinId: randomUUID(), + pinPurpose: `archive:${gid}`, + scratchSubdir: `archive-${randomUUID()}`, + manifestSha256: null, + baseActiveGenerationId: null, + phase, + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + } as CreateOperationIntent +} + +const gcIntent = ( + phase: "intent" | "gc-collecting" | "complete", + completedTargets: number, + targetCount: number, +): GcOperationIntent => { + const targets = Array.from({ length: targetCount }, () => ({ + signal: "traces", + rangeStart: "2026-06-01", + generationId: randomUUID(), + createdAt: "2026-06-02T00:00:00.000Z", + manifestSha256: "b".repeat(64), + bytes: 100, + shards: [{ name: "00.parquet", bytes: 100, sha256: "c".repeat(64) }], + recordedActiveGenerationId: randomUUID(), + })) + return { + formatVersion: 3, + kind: "gc", + operationId: randomUUID(), + keep: 0, + targets, + completedTargets, + archiveDir: "/archive", + dataDir: "/data", + scratchRoot: "/scratch", + phase, + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + } as GcOperationIntent +} + +const createSnapshot = ( + intent: CreateOperationIntent, + topology: { + promoted?: boolean + manifestAtFinal?: boolean + buildingPresent?: boolean + buildingAndFinalBothPresent?: boolean + migrationRequired?: boolean + }, +): ReconciliationSnapshot => ({ + operationId: intent.operationId, + journalDigest: digestOfIntent(intent), + migrationRequired: topology.migrationRequired ?? false, + intent, + promoted: topology.promoted ?? false, + manifestAtFinal: topology.manifestAtFinal ?? false, + buildingPresent: topology.buildingPresent ?? false, + buildingAndFinalBothPresent: topology.buildingAndFinalBothPresent ?? false, + remainingTargets: 0, + affectedSignals: [], +}) + +const gcSnapshot = ( + intent: GcOperationIntent, + affectedSignals: string[] = ["traces"], +): ReconciliationSnapshot => ({ + operationId: intent.operationId, + journalDigest: digestOfIntent(intent), + migrationRequired: false, + intent, + promoted: false, + manifestAtFinal: false, + buildingPresent: false, + buildingAndFinalBothPresent: false, + remainingTargets: intent.targets.length - intent.completedTargets, + affectedSignals, +}) + +const valid = (snapshot: ReconciliationSnapshot): ReconciliationInspection => ({ + kind: "ValidSnapshot", + snapshot, +}) +const failClosed = (reason: string): ReconciliationInspection => ({ kind: "FailClosed", reason }) + +describe("decideReconciliation — null and fail-closed inputs", () => { + it("returns NoOp for null (no active operation)", () => { + strictEqual(decideReconciliation(null).kind, "NoOp") + }) + it("returns FailClosed for a FailClosed inspection (validation failure enters the same function)", () => { + const d = decideReconciliation(failClosed("unsafe state: debris")) + strictEqual(d.kind, "FailClosed") + if (d.kind === "FailClosed") strictEqual(d.reason, "unsafe state: debris") + }) +}) + +describe("decideReconciliation — CREATE transition table", () => { + it("phase >= complete → CreateVerifyComplete (verify-only, no repair)", () => { + const intent = createIntent("complete") + const d = decideReconciliation( + valid(createSnapshot(intent, { promoted: true, manifestAtFinal: true })), + ) + strictEqual(d.kind, "CreateVerifyComplete") + }) + + it("phase = scratch-removed (< complete) → CreateFinishPublication (NOT verify-only)", () => { + const intent = createIntent("scratch-removed") + // scratch-removed is BEFORE complete in PHASE_ORDER, so it is repair not verify-only. + const d = decideReconciliation( + valid(createSnapshot(intent, { promoted: true, manifestAtFinal: true })), + ) + strictEqual(d.kind, "CreateFinishPublication") + }) + + it("!promoted → CreateAbortPrepublication (repair: quarantine + abort)", () => { + const intent = createIntent("intent") + const d = decideReconciliation( + valid(createSnapshot(intent, { promoted: false, buildingPresent: true })), + ) + strictEqual(d.kind, "CreateAbortPrepublication") + if (d.kind === "CreateAbortPrepublication") strictEqual(d.buildingPresent, true) + }) + + it("!promoted with no building → CreateAbortPrepublication (buildingPresent=false)", () => { + const intent = createIntent("restored") + const d = decideReconciliation( + valid(createSnapshot(intent, { promoted: false, buildingPresent: false })), + ) + strictEqual(d.kind, "CreateAbortPrepublication") + if (d.kind === "CreateAbortPrepublication") strictEqual(d.buildingPresent, false) + }) + + it("promoted (post-promotion) → CreateFinishPublication (repair: unconditional select + rebuild)", () => { + const intent = createIntent("promoted") + const d = decideReconciliation( + valid(createSnapshot(intent, { promoted: true, manifestAtFinal: true })), + ) + strictEqual(d.kind, "CreateFinishPublication") + }) + + it("promoted at pointer-complete → CreateFinishPublication (repair, NOT verify-only)", () => { + const intent = createIntent("pointer-complete") + const d = decideReconciliation( + valid(createSnapshot(intent, { promoted: true, manifestAtFinal: true })), + ) + strictEqual(d.kind, "CreateFinishPublication") + }) + + it("promoted at catalog-complete → CreateFinishPublication (repair, NOT verify-only)", () => { + const intent = createIntent("catalog-complete") + const d = decideReconciliation( + valid(createSnapshot(intent, { promoted: true, manifestAtFinal: true })), + ) + strictEqual(d.kind, "CreateFinishPublication") + }) +}) + +describe("decideReconciliation — CREATE impossible-topology FailClosed gates", () => { + it("building && final both present → FailClosed", () => { + const intent = createIntent("intent") + const d = decideReconciliation( + valid( + createSnapshot(intent, { + promoted: true, + buildingPresent: true, + manifestAtFinal: true, + buildingAndFinalBothPresent: true, + }), + ), + ) + strictEqual(d.kind, "FailClosed") + }) + + it("promoted && !manifestAtFinal → FailClosed", () => { + const intent = createIntent("promoted") + const d = decideReconciliation( + valid(createSnapshot(intent, { promoted: true, manifestAtFinal: false })), + ) + strictEqual(d.kind, "FailClosed") + }) + + it("promoted && phase < manifest-written → FailClosed", () => { + const intent = createIntent("shards-written") + const d = decideReconciliation( + valid(createSnapshot(intent, { promoted: true, manifestAtFinal: true })), + ) + strictEqual(d.kind, "FailClosed") + }) + + it("phase >= promoted && !promoted → FailClosed (phase ahead of reality)", () => { + const intent = createIntent("promoted") + const d = decideReconciliation(valid(createSnapshot(intent, { promoted: false }))) + strictEqual(d.kind, "FailClosed") + }) + + it("aborted in active/ → FailClosed", () => { + const intent = createIntent("aborted") + const d = decideReconciliation(valid(createSnapshot(intent, { promoted: false }))) + strictEqual(d.kind, "FailClosed") + }) +}) + +describe("decideReconciliation — GC transition table", () => { + it("phase = complete → GcVerifyComplete (verify-only, no repair)", () => { + const intent = gcIntent("complete", 2, 2) + const d = decideReconciliation(valid(gcSnapshot(intent))) + strictEqual(d.kind, "GcVerifyComplete") + }) + + it("phase = gc-collecting → GcResume (repair)", () => { + const intent = gcIntent("gc-collecting", 1, 2) + const d = decideReconciliation(valid(gcSnapshot(intent))) + strictEqual(d.kind, "GcResume") + if (d.kind === "GcResume") { + strictEqual(d.remainingTargets, 1) + strictEqual(d.affectedSignals.length, 1) + } + }) + + it("phase = intent (cursor 0) → GcResume", () => { + const intent = gcIntent("intent", 0, 3) + const d = decideReconciliation(valid(gcSnapshot(intent))) + strictEqual(d.kind, "GcResume") + if (d.kind === "GcResume") strictEqual(d.remainingTargets, 3) + }) + + it("multi-signal GC carries all affected signals", () => { + const intent = gcIntent("gc-collecting", 0, 2) + const d = decideReconciliation(valid(gcSnapshot(intent, ["traces", "logs"]))) + strictEqual(d.kind, "GcResume") + if (d.kind === "GcResume") strictEqual(d.affectedSignals.length, 2) + }) +}) + +describe("decideReconciliation — V2 migration flag preserved", () => { + it("a v2 snapshot (migrationRequired=true) preserves the flag in the decision", () => { + const intent = createIntent("intent") + const d = decideReconciliation( + valid(createSnapshot(intent, { promoted: false, migrationRequired: true })), + ) + strictEqual(d.kind, "CreateAbortPrepublication") + if (d.kind === "CreateAbortPrepublication") strictEqual(d.migrationRequired, true) + }) + + it("a v2 snapshot at a non-intent phase preserves the phase exactly", () => { + // A v2 record can hold any create-eligible phase; the lifted record's phase + // selects the branch, not a hardcoded "intent". + const intent = createIntent("promoted") + const d = decideReconciliation( + valid(createSnapshot(intent, { promoted: true, manifestAtFinal: true, migrationRequired: true })), + ) + strictEqual(d.kind, "CreateFinishPublication") + if (d.kind === "CreateFinishPublication") strictEqual(d.migrationRequired, true) + }) +}) + +describe("decideReconciliation — decision identities", () => { + it("every valid decision carries operationId + journalDigest", () => { + const intent = createIntent("promoted") + const snap = createSnapshot(intent, { promoted: true, manifestAtFinal: true }) + const d = decideReconciliation(valid(snap)) + if (d.kind !== "NoOp" && d.kind !== "FailClosed") { + strictEqual(d.operationId, intent.operationId) + strictEqual(d.journalDigest, snap.journalDigest) + } + }) +}) diff --git a/apps/cli/test/archive-reconcile.test.ts b/apps/cli/test/archive-reconcile.test.ts new file mode 100644 index 000000000..4aff4e82a --- /dev/null +++ b/apps/cli/test/archive-reconcile.test.ts @@ -0,0 +1,1171 @@ +import { describe, it } from "@effect/vitest" +import { ok, rejects, strictEqual } from "node:assert" +import { createHash, randomUUID } from "node:crypto" +import { MAPLE_VERSION, CHDB_VERSION } from "../src/version" +import { SCHEMA_FINGERPRINT } from "../src/server/serve" +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + readlinkSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs" +import { tmpdir } from "node:os" +import { dirname, join } from "node:path" +import { runArchiveReconciliation } from "../src/server/archives/generation" +import type { ReconciliationDecision } from "../src/server/archives/reconcile" + +const withRoots = async ( + run: (archiveDir: string, dataDir: string, scratchRoot: string) => Promise | void, +): Promise => { + const parent = realpathSync(mkdtempSync(join(tmpdir(), "maple-reconcile-test-"))) + const archiveDir = join(parent, "archive") + const dataDir = join(parent, "data") + const scratchRoot = join(parent, "scratch") + for (const d of [archiveDir, dataDir, scratchRoot]) mkdirSync(d, { recursive: true }) + try { + await run(archiveDir, dataDir, scratchRoot) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} +const writeActiveIntent = ( + archiveDir: string, + operationId: string, + record: Record, +): void => { + const dir = join(archiveDir, "operations", "active", `archive-${operationId}`) + mkdirSync(dir, { recursive: true }) + writeFileSync(join(dir, "intent.json"), JSON.stringify(record)) +} +const validV2 = (archiveDir: string, dataDir: string, scratchRoot: string) => { + const operationId = randomUUID() + const generationId = randomUUID() + return { + operationId, + record: { + formatVersion: 2, + operationId, + generationId, + signal: "traces", + rangeStart: "2026-06-01", + checkpointId: randomUUID(), + archiveDir, + dataDir, + scratchRoot, + pinId: randomUUID(), + pinPurpose: `archive:${generationId}`, + scratchSubdir: `archive-${operationId}`, + manifestSha256: null, + baseActiveGenerationId: null, + phase: "intent", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + }, + } +} +const sha = (path: string): string => createHash("sha256").update(readFileSync(path)).digest("hex") +const durableStateSnapshot = (...roots: string[]): string => { + const records: string[] = [] + const walk = (path: string, relativePath: string): void => { + const info = lstatSync(path) + if (info.isSymbolicLink()) { + records.push(`link\0${relativePath}\0${readlinkSync(path)}`) + return + } + if (info.isFile()) { + records.push(`file\0${relativePath}\0${info.size}\0${sha(path)}`) + return + } + if (!info.isDirectory()) { + throw new Error(`unsupported snapshot entry type: ${path}`) + } + // Record every directory so empty-directory creation/removal is visible. + records.push(`directory\0${relativePath}`) + for (const name of readdirSync(path).sort()) { + walk(join(path, name), `${relativePath}/${name}`) + } + } + for (const [index, root] of roots.entries()) walk(root, `root-${index}`) + return createHash("sha256").update(records.join("\n")).digest("hex") +} +const isFailClosed = ( + d: ReconciliationDecision, +): d is Extract => d.kind === "FailClosed" + +describe("archive reconciliation protocol (Gate 3b r5)", () => { + it("dry-run treats a valid v2 intent as a decision with migrationRequired", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const { operationId, record } = validV2(archiveDir, dataDir, scratchRoot) + writeActiveIntent(archiveDir, operationId, record) + const d = await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: true }) + if (d.kind !== "CreateAbortPrepublication") + ok(false, `expected CreateAbortPrepublication, got ${d.kind}`) + if (d.kind === "CreateAbortPrepublication") strictEqual(d.migrationRequired, true) + }) + }) + it("dry-run marks a malformed v3 intent FailClosed", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + writeActiveIntent(archiveDir, randomUUID(), { + formatVersion: 3, + kind: "create", + operationId: randomUUID(), + phase: "bogus-phase", + }) + ok( + isFailClosed( + await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: true }), + ), + ) + }) + }) + it("dry-run marks unknown active-dir debris FailClosed", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + mkdirSync(join(archiveDir, "operations", "active", "junk-debris"), { recursive: true }) + ok( + isFailClosed( + await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: true }), + ), + ) + }) + }) + it("dry-run marks multiple active operations FailClosed", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const a = validV2(archiveDir, dataDir, scratchRoot), + b = validV2(archiveDir, dataDir, scratchRoot) + writeActiveIntent(archiveDir, a.operationId, a.record) + writeActiveIntent(archiveDir, b.operationId, b.record) + ok( + isFailClosed( + await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: true }), + ), + ) + }) + }) + it("dry-run marks a corrupt v2 intent FailClosed", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + writeActiveIntent(archiveDir, randomUUID(), { formatVersion: 2 }) + ok( + isFailClosed( + await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: true }), + ), + ) + }) + }) + it("dry-run marks a v2 dir/record mismatch FailClosed and does NOT rewrite it", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const dirId = randomUUID() + const rec = validV2(archiveDir, dataDir, scratchRoot) + rec.record.operationId = randomUUID() + rec.record.scratchSubdir = `archive-${rec.record.operationId}` + rec.record.pinPurpose = `archive:${rec.record.generationId}` + writeActiveIntent(archiveDir, dirId, rec.record) + const intentPath = join(archiveDir, "operations", "active", `archive-${dirId}`, "intent.json") + const before = sha(intentPath) + ok( + isFailClosed( + await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: true }), + ), + ) + strictEqual(sha(intentPath), before, "dry-run must not rewrite mismatched v2") + }) + }) + it("dry-run never mutates a valid v2 intent", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const { operationId, record } = validV2(archiveDir, dataDir, scratchRoot) + writeActiveIntent(archiveDir, operationId, record) + const intentPath = join( + archiveDir, + "operations", + "active", + `archive-${operationId}`, + "intent.json", + ) + const before = sha(intentPath) + await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: true }) + strictEqual(sha(intentPath), before, "dry-run mutated v2") + strictEqual(JSON.parse(readFileSync(intentPath, "utf8")).formatVersion, 2) + }) + }) + it("apply throws on FailClosed and preserves state", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + mkdirSync(join(archiveDir, "operations", "active", "junk-debris"), { recursive: true }) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /unsafe|FailClosed|debris|ambiguous/i, + ) + ok(existsSync(join(archiveDir, "operations", "active", "junk-debris"))) + }) + }) + it("apply with no active operation returns NoOp", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + strictEqual( + (await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false })).kind, + "NoOp", + ) + }) + }) + it("apply does not rewrite a v2 mismatch before rejecting it", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const dirId = randomUUID() + const rec = validV2(archiveDir, dataDir, scratchRoot) + rec.record.operationId = randomUUID() + rec.record.scratchSubdir = `archive-${rec.record.operationId}` + rec.record.pinPurpose = `archive:${rec.record.generationId}` + writeActiveIntent(archiveDir, dirId, rec.record) + const intentPath = join(archiveDir, "operations", "active", `archive-${dirId}`, "intent.json") + const before = sha(intentPath) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /unsafe|identity mismatch/i, + ) + strictEqual(sha(intentPath), before) + }) + }) + it("dry-run on a symlinked intent is FailClosed; outside target survives", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const opId = randomUUID() + const outside = join(archiveDir, "..", "outside-intent") + mkdirSync(outside, { recursive: true }) + const { record } = validV2(archiveDir, dataDir, scratchRoot) + record.operationId = opId + record.scratchSubdir = `archive-${opId}` + record.pinPurpose = `archive:${record.generationId}` + writeFileSync(join(outside, "intent.json"), JSON.stringify(record)) + writeFileSync(join(outside, "SENTINEL"), "preserve") + const dirOp = join(archiveDir, "operations", "active", `archive-${opId}`) + mkdirSync(dirOp, { recursive: true }) + symlinkSync(join(outside, "intent.json"), join(dirOp, "intent.json")) + ok( + isFailClosed( + await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: true }), + ), + ) + strictEqual(readFileSync(join(outside, "SENTINEL"), "utf8"), "preserve") + }) + }) + it("apply on a symlinked intent is FailClosed; outside target survives", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const opId = randomUUID() + const outside = join(archiveDir, "..", "outside-intent") + mkdirSync(outside, { recursive: true }) + const { record } = validV2(archiveDir, dataDir, scratchRoot) + record.operationId = opId + record.scratchSubdir = `archive-${opId}` + record.pinPurpose = `archive:${record.generationId}` + writeFileSync(join(outside, "intent.json"), JSON.stringify(record)) + writeFileSync(join(outside, "SENTINEL"), "preserve") + const dirOp = join(archiveDir, "operations", "active", `archive-${opId}`) + mkdirSync(dirOp, { recursive: true }) + symlinkSync(join(outside, "intent.json"), join(dirOp, "intent.json")) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /unsafe|unreadable|symlink/i, + ) + strictEqual(readFileSync(join(outside, "SENTINEL"), "utf8"), "preserve") + }) + }) + it("dry-run fails nonzero while a live owner holds the lock", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const { withMaintenanceLock } = await import("../src/server/checkpoints") + const { record } = validV2(archiveDir, dataDir, scratchRoot) + writeActiveIntent(archiveDir, record.operationId as string, record) + await withMaintenanceLock(dataDir, randomUUID(), async () => { + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: true }), + /active|lock/i, + ) + }) + }) + }) +}) + +describe("dry-run/apply parity — hostile preflight fixtures", () => { + it("create post-promotion with a conflicting pointer: dry-run FailClosed, apply nonzero, zero mutation", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + // Seed a v3 create intent at "promoted" with a published generation + manifest, + // but plant a pointer selecting a THIRD generation. + const gid = randomUUID(), + opId = randomUUID() + const opDir = join(archiveDir, "operations", "active", `archive-${opId}`) + mkdirSync(opDir, { recursive: true }) + const finalGen = join(archiveDir, "traces", "2026-06-01", "generations", gid) + mkdirSync(join(finalGen, "shards"), { recursive: true }) + // Minimal valid manifest+shard for verifyPublishedGeneration to pass, + // then assertPointerConsistent will fail (pointer selects a third gen). + const shardContents = "PAR1" + writeFileSync(join(finalGen, "shards", "00.parquet"), shardContents) + const shardSha = createHash("sha256").update(shardContents).digest("hex") + const manifest = { + formatVersion: 3, + generationId: gid, + signal: "traces", + rangeStart: "2026-06-01", + rangeEndExclusive: "2026-06-02T00:00:00.000Z", + checkpointId: randomUUID(), + checkpointManifestFingerprint: "cid", + createdAt: "2026-06-02T00:00:00.000Z", + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + sourceRowCount: 1, + archivedRowCount: 1, + tuning: { + writerThreads: 1, + rowGroupRows: 10000, + maxShardRows: 500000, + maxShardBytes: 268435456, + targetChunkBytes: 1073741824, + minFreeSpaceReserve: 536870912, + }, + tuningConfig: null, + shards: [ + { + name: "00.parquet", + rowCount: 1, + minEventTimeUnixNano: `${BigInt(Date.parse("2026-06-01T12:00:00.000Z")) * 1000000n}`, + maxEventTimeUnixNano: `${BigInt(Date.parse("2026-06-01T12:00:00.000Z")) * 1000000n}`, + sha256: shardSha, + bytes: shardContents.length, + columns: ["TimestampTime", "ServiceName"], + complexDigest: "0", + complexDigestAlgorithm: "cityhash64-multiset-v3", + }, + ], + } + const manifestJson = JSON.stringify(manifest) + const manifestSha = createHash("sha256").update(manifestJson).digest("hex") + writeFileSync(join(finalGen, "manifest.json"), manifestJson) + // Plant a pointer selecting a THIRD generation (not the base, not the intended). + const rangeDir = join(archiveDir, "traces", "2026-06-01") + mkdirSync(rangeDir, { recursive: true }) + writeFileSync( + join(rangeDir, "active.json"), + JSON.stringify({ + formatVersion: 1, + generationId: randomUUID(), + signal: "traces", + rangeStart: "2026-06-01", + }), + ) + // Create a valid pin so pin validation passes; the pointer CAS is the failure. + const pinId = randomUUID(), + checkpointId = randomUUID() + const pinDir = join(dataDir, "backups", "pins", checkpointId) + mkdirSync(pinDir, { recursive: true }) + writeFileSync( + join(pinDir, `${pinId}.json`), + JSON.stringify({ + formatVersion: 1, + pinId, + checkpointId, + purpose: `archive:${gid}`, + createdAt: "2026-06-01T00:00:00.000Z", + }), + ) + // Write a v3 intent at "promoted" with the manifest SHA. + writeFileSync( + join(opDir, "intent.json"), + JSON.stringify({ + formatVersion: 3, + kind: "create", + operationId: opId, + generationId: gid, + signal: "traces", + rangeStart: "2026-06-01", + checkpointId, + archiveDir, + dataDir, + scratchRoot, + pinId, + pinPurpose: `archive:${gid}`, + scratchSubdir: `archive-${opId}`, + manifestSha256: manifestSha, + baseActiveGenerationId: null, + phase: "promoted", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + }), + ) + // Dry-run: should return FailClosed (pointer CAS fails). + const dryDecision = await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { + dryRun: true, + }) + ok( + isFailClosed(dryDecision), + `dry-run should be FailClosed for conflicting pointer, got ${dryDecision.kind}`, + ) + // Apply: should throw (nonzero). + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /pointer|clobber|CAS|unsafe/i, + ) + // State preserved (generation + journal still present). + ok(existsSync(finalGen), "generation preserved after failed apply") + ok(existsSync(opDir), "journal preserved after failed apply") + }) + }) +}) + +describe("dry-run/apply parity — GC multi-target hostile preflight", () => { + const seedGcOperation = ( + archiveDir: string, + dataDir: string, + scratchRoot: string, + targets: Array<{ + generationId: string + signal: string + rangeStart: string + manifestSha256: string + shardName: string + shardSha: string + shardBytes: number + sourcePath: string + recordedActive: string + }>, + completedTargets: number, + ) => { + const opId = randomUUID() + const opDir = join(archiveDir, "operations", "active", `archive-${opId}`) + mkdirSync(opDir, { recursive: true }) + const gcTargets = targets.map((t) => ({ + signal: t.signal, + rangeStart: t.rangeStart, + generationId: t.generationId, + createdAt: "2026-06-02T00:00:00.000Z", + manifestSha256: t.manifestSha256, + bytes: t.shardBytes, + shards: [{ name: t.shardName, bytes: t.shardBytes, sha256: t.shardSha }], + recordedActiveGenerationId: t.recordedActive, + })) + writeFileSync( + join(opDir, "intent.json"), + JSON.stringify({ + formatVersion: 3, + kind: "gc", + operationId: opId, + keep: 0, + targets: gcTargets, + completedTargets, + archiveDir, + dataDir, + scratchRoot, + phase: completedTargets > 0 ? "gc-collecting" : "intent", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + }), + ) + return { opId, opDir } + } + + const seedGeneration = ( + archiveDir: string, + signal: string, + rangeDate: string, + generationId: string, + shardContents: string, + ) => { + const genDir = join(archiveDir, signal, rangeDate, "generations", generationId) + mkdirSync(join(genDir, "shards"), { recursive: true }) + writeFileSync(join(genDir, "shards", "00.parquet"), shardContents) + const shardSha = createHash("sha256").update(shardContents).digest("hex") + const manifest = { + formatVersion: 3, + generationId, + signal, + rangeStart: rangeDate, + rangeEndExclusive: `${rangeDate}T23:59:59.000000000Z`, + checkpointId: randomUUID(), + checkpointManifestFingerprint: "cid", + createdAt: "2026-06-02T00:00:00.000Z", + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + sourceRowCount: 1, + archivedRowCount: 1, + tuning: { + writerThreads: 1, + rowGroupRows: 10000, + maxShardRows: 500000, + maxShardBytes: 268435456, + targetChunkBytes: 1073741824, + minFreeSpaceReserve: 536870912, + }, + tuningConfig: null, + shards: [ + { + name: "00.parquet", + rowCount: 1, + minEventTimeUnixNano: `${BigInt(Date.parse(`${rangeDate}T12:00:00.000Z`)) * 1000000n}`, + maxEventTimeUnixNano: `${BigInt(Date.parse(`${rangeDate}T12:00:00.000Z`)) * 1000000n}`, + sha256: shardSha, + bytes: shardContents.length, + columns: ["TimestampTime", "ServiceName"], + complexDigest: "0", + complexDigestAlgorithm: "cityhash64-multiset-v3", + }, + ], + } + const manifestJson = JSON.stringify(manifest) + const manifestSha = createHash("sha256").update(manifestJson).digest("hex") + writeFileSync(join(genDir, "manifest.json"), manifestJson) + return { genDir, manifestSha, shardSha } + } + + it("cursor-ahead (completedTargets=1 but target 0 source still exists): dry-run FailClosed, apply nonzero, zero mutation", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const gid1 = randomUUID(), + gid2 = randomUUID(), + activeGid = randomUUID() + const ev1 = seedGeneration(archiveDir, "traces", "2026-06-01", gid1, "PAR1-old") + const ev2 = seedGeneration(archiveDir, "traces", "2026-06-01", gid2, "PAR1-mid") + seedGeneration(archiveDir, "traces", "2026-06-01", activeGid, "PAR1-active") + // Plant active pointer. + const rangeDir = join(archiveDir, "traces", "2026-06-01") + writeFileSync( + join(rangeDir, "active.json"), + JSON.stringify({ + formatVersion: 1, + generationId: activeGid, + signal: "traces", + rangeStart: "2026-06-01", + }), + ) + // Cursor=1 but target 0 (gid1) source still exists — impossible prefix. + seedGcOperation( + archiveDir, + dataDir, + scratchRoot, + [ + { + generationId: gid1, + signal: "traces", + rangeStart: "2026-06-01", + manifestSha256: ev1.manifestSha, + shardName: "00.parquet", + shardSha: ev1.shardSha, + shardBytes: 4, + sourcePath: "", + recordedActive: activeGid, + }, + { + generationId: gid2, + signal: "traces", + rangeStart: "2026-06-01", + manifestSha256: ev2.manifestSha, + shardName: "00.parquet", + shardSha: ev2.shardSha, + shardBytes: 7, + sourcePath: "", + recordedActive: activeGid, + }, + ], + 1, + ) + const before = durableStateSnapshot(archiveDir, dataDir) + const dryDecision = await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { + dryRun: true, + }) + ok( + isFailClosed(dryDecision), + `dry-run should be FailClosed for cursor-ahead, got ${dryDecision.kind}`, + ) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /completed target still has source|unsafe/i, + ) + const after = durableStateSnapshot(archiveDir, dataDir) + strictEqual(before, after, "zero mutation on cursor-ahead preflight failure") + }) + }) + + it("source-absent/tombstone-present with symlinked tombstone: dry-run FailClosed, apply nonzero", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const gid1 = randomUUID(), + activeGid = randomUUID() + const ev1 = seedGeneration(archiveDir, "traces", "2026-06-01", gid1, "PAR1-old") + seedGeneration(archiveDir, "traces", "2026-06-01", activeGid, "PAR1-active") + const rangeDir = join(archiveDir, "traces", "2026-06-01") + writeFileSync( + join(rangeDir, "active.json"), + JSON.stringify({ + formatVersion: 1, + generationId: activeGid, + signal: "traces", + rangeStart: "2026-06-01", + }), + ) + // Seed GC op at cursor=0, but manually create a symlinked tombstone for target 0 + // while removing its source — simulating a mid-removal crash where the tombstone + // is a symlink (uncertain state). + const { opId } = seedGcOperation( + archiveDir, + dataDir, + scratchRoot, + [ + { + generationId: gid1, + signal: "traces", + rangeStart: "2026-06-01", + manifestSha256: ev1.manifestSha, + shardName: "00.parquet", + shardSha: ev1.shardSha, + shardBytes: 7, + sourcePath: "", + recordedActive: activeGid, + }, + ], + 0, + ) + // Remove source, create a symlinked tombstone. + rmSync(join(archiveDir, "traces", "2026-06-01", "generations", gid1), { + recursive: true, + force: true, + }) + const tombDir = join(archiveDir, "operations", "active", `archive-${opId}`, "tombstones", gid1) + mkdirSync(join(tombDir, "shards"), { recursive: true }) + // Replace tombstone dir with a symlink to /tmp (uncertain). + rmSync(tombDir, { recursive: true, force: true }) + symlinkSync("/tmp", tombDir) + const dryDecision = await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { + dryRun: true, + }) + ok( + isFailClosed(dryDecision), + `dry-run should be FailClosed for symlinked tombstone, got ${dryDecision.kind}`, + ) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /symlink|tombstone|unsafe/i, + ) + }) + }) +}) + +describe("dry-run/apply parity — dangling symlinks and impossible suffix (r9)", () => { + it("dangling source symlink is FailClosed (not treated as absent), zero mutation", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const gid1 = randomUUID(), + activeGid = randomUUID() + const ev1 = seedGeneration2(archiveDir, "traces", "2026-06-01", gid1, "PAR1-old") + seedGeneration2(archiveDir, "traces", "2026-06-01", activeGid, "PAR1-active") + const rangeDir = join(archiveDir, "traces", "2026-06-01") + mkdirSync(rangeDir, { recursive: true }) + writeFileSync( + join(rangeDir, "active.json"), + JSON.stringify({ + formatVersion: 1, + generationId: activeGid, + signal: "traces", + rangeStart: "2026-06-01", + }), + ) + // Seed a GC operation targeting gid1. + seedGcOpWithEvidence( + archiveDir, + dataDir, + scratchRoot, + [ + { + gid: gid1, + signal: "traces", + rangeDate: "2026-06-01", + contents: "PAR1-old", + recordedActive: activeGid, + }, + ], + 0, + ) + // Replace gid1's generation dir with a dangling symlink. + const genDir = join(archiveDir, "traces", "2026-06-01", "generations", gid1) + rmSync(genDir, { recursive: true, force: true }) + symlinkSync(join(archiveDir, "nonexistent-target"), genDir) + const before = durableStateSnapshot(archiveDir, dataDir) + const dryDecision = await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { + dryRun: true, + }) + ok( + isFailClosed(dryDecision), + `dry-run should be FailClosed for dangling source symlink, got ${dryDecision.kind}`, + ) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /symlink|unsafe|source/i, + ) + strictEqual( + durableStateSnapshot(archiveDir, dataDir), + before, + "zero mutation on dangling source symlink", + ) + void ev1 + }) + }) + + it("dangling tombstone symlink is FailClosed (not treated as absent), zero mutation", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const gid1 = randomUUID(), + activeGid = randomUUID() + seedGeneration2(archiveDir, "traces", "2026-06-01", gid1, "PAR1-old") + seedGeneration2(archiveDir, "traces", "2026-06-01", activeGid, "PAR1-active") + const rangeDir = join(archiveDir, "traces", "2026-06-01") + mkdirSync(rangeDir, { recursive: true }) + writeFileSync( + join(rangeDir, "active.json"), + JSON.stringify({ + formatVersion: 1, + generationId: activeGid, + signal: "traces", + rangeStart: "2026-06-01", + }), + ) + const { opId, manifestSha1, shardSha1 } = seedGcOp2( + archiveDir, + dataDir, + scratchRoot, + gid1, + activeGid, + ) + // Remove source, create a DANGLING tombstone symlink. + rmSync(join(archiveDir, "traces", "2026-06-01", "generations", gid1), { + recursive: true, + force: true, + }) + const tombDir = join(archiveDir, "operations", "active", `archive-${opId}`, "tombstones", gid1) + mkdirSync(dirname(tombDir), { recursive: true }) + symlinkSync(join(archiveDir, "nonexistent-tomb"), tombDir) + const before = durableStateSnapshot(archiveDir, dataDir) + const dryDecision = await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { + dryRun: true, + }) + ok( + isFailClosed(dryDecision), + `dry-run should be FailClosed for dangling tombstone symlink, got ${dryDecision.kind}`, + ) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /symlink|tombstone|unsafe/i, + ) + strictEqual( + durableStateSnapshot(archiveDir, dataDir), + before, + "zero mutation on dangling tombstone symlink", + ) + void manifestSha1 + void shardSha1 + }) + }) + + it("impossible suffix (target 2 absent while cursor=0): FailClosed, zero mutation", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const gid1 = randomUUID(), + gid2 = randomUUID(), + activeGid = randomUUID() + seedGeneration2(archiveDir, "traces", "2026-06-01", gid1, "PAR1-t1") + seedGeneration2(archiveDir, "traces", "2026-06-01", activeGid, "PAR1-active") + // gid2 is a suffix target but we DON'T create its generation — it's absent. + const rangeDir = join(archiveDir, "traces", "2026-06-01") + mkdirSync(rangeDir, { recursive: true }) + writeFileSync( + join(rangeDir, "active.json"), + JSON.stringify({ + formatVersion: 1, + generationId: activeGid, + signal: "traces", + rangeStart: "2026-06-01", + }), + ) + // Seed GC op with 2 targets, cursor=0. Target 2 (gid2) is absent — impossible suffix. + seedGcOpWithEvidence( + archiveDir, + dataDir, + scratchRoot, + [ + { + gid: gid1, + signal: "traces", + rangeDate: "2026-06-01", + contents: "PAR1-t1", + recordedActive: activeGid, + }, + { + gid: gid2, + signal: "traces", + rangeDate: "2026-06-01", + contents: "PAR1-t2", + recordedActive: activeGid, + }, + ], + 0, + ) + // Remove gid2's generation dir — it's absent as a suffix target (impossible). + rmSync(join(archiveDir, "traces", "2026-06-01", "generations", gid2), { + recursive: true, + force: true, + }) + const before = durableStateSnapshot(archiveDir, dataDir) + const dryDecision = await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { + dryRun: true, + }) + ok( + isFailClosed(dryDecision), + `dry-run should be FailClosed for impossible suffix, got ${dryDecision.kind}`, + ) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /suffix.*absent|impossible|unsafe/i, + ) + strictEqual( + durableStateSnapshot(archiveDir, dataDir), + before, + "zero mutation on impossible suffix", + ) + }) + }) +}) + +describe("dry-run/apply parity — root-aware topology (r10)", () => { + const writePointer = (archiveDir: string, activeGid: string): void => { + const rangeDir = join(archiveDir, "traces", "2026-06-01") + mkdirSync(rangeDir, { recursive: true }) + writeFileSync( + join(rangeDir, "active.json"), + JSON.stringify({ + formatVersion: 1, + generationId: activeGid, + signal: "traces", + rangeStart: "2026-06-01", + }), + ) + } + + it("rejects an absent source leaf beneath a symlinked generations ancestor", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const targetGid = randomUUID() + const activeGid = randomUUID() + seedGeneration2(archiveDir, "traces", "2026-06-01", activeGid, "PAR1-active") + writePointer(archiveDir, activeGid) + seedGcOpWithEvidence( + archiveDir, + dataDir, + scratchRoot, + [ + { + gid: targetGid, + signal: "traces", + rangeDate: "2026-06-01", + contents: "PAR1-old", + recordedActive: activeGid, + }, + ], + 0, + ) + const generations = join(archiveDir, "traces", "2026-06-01", "generations") + const outside = join(dirname(archiveDir), "outside-generations") + rmSync(generations, { recursive: true, force: true }) + mkdirSync(outside) + writeFileSync(join(outside, "sentinel"), "outside") + symlinkSync(outside, generations) + + const before = durableStateSnapshot(archiveDir, dataDir, outside) + const dryDecision = await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { + dryRun: true, + }) + ok(isFailClosed(dryDecision), `expected FailClosed, got ${dryDecision.kind}`) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /symlink|source|unsafe/i, + ) + strictEqual(durableStateSnapshot(archiveDir, dataDir, outside), before) + }) + }) + + it("rejects an absent tombstone leaf beneath a symlinked ancestor at a full cursor", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const targetGid = randomUUID() + const activeGid = randomUUID() + seedGeneration2(archiveDir, "traces", "2026-06-01", activeGid, "PAR1-active") + writePointer(archiveDir, activeGid) + const { opDir } = seedGcOpWithEvidence( + archiveDir, + dataDir, + scratchRoot, + [ + { + gid: targetGid, + signal: "traces", + rangeDate: "2026-06-01", + contents: "PAR1-old", + recordedActive: activeGid, + }, + ], + 1, + ) + rmSync(join(archiveDir, "traces", "2026-06-01", "generations", targetGid), { + recursive: true, + force: true, + }) + const outside = join(dirname(archiveDir), "outside-tombstones") + mkdirSync(outside) + writeFileSync(join(outside, "sentinel"), "outside") + symlinkSync(outside, join(opDir, "tombstones")) + + const before = durableStateSnapshot(archiveDir, dataDir, outside) + const dryDecision = await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { + dryRun: true, + }) + ok(isFailClosed(dryDecision), `expected FailClosed, got ${dryDecision.kind}`) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /symlink|tombstone|unsafe/i, + ) + strictEqual(durableStateSnapshot(archiveDir, dataDir, outside), before) + }) + }) + + it("rejects a dangling completed-operations ancestor before GC mutation", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const targetGid = randomUUID() + const activeGid = randomUUID() + seedGeneration2(archiveDir, "traces", "2026-06-01", activeGid, "PAR1-active") + writePointer(archiveDir, activeGid) + seedGcOpWithEvidence( + archiveDir, + dataDir, + scratchRoot, + [ + { + gid: targetGid, + signal: "traces", + rangeDate: "2026-06-01", + contents: "PAR1-old", + recordedActive: activeGid, + }, + ], + 0, + ) + const completed = join(archiveDir, "operations", "completed") + symlinkSync(join(dirname(archiveDir), "missing-completed-target"), completed) + + const before = durableStateSnapshot(archiveDir, dataDir) + const dryDecision = await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { + dryRun: true, + }) + ok(isFailClosed(dryDecision), `expected FailClosed, got ${dryDecision.kind}`) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /symlink|completed|unsafe/i, + ) + strictEqual(durableStateSnapshot(archiveDir, dataDir), before) + }) + }) + + it("rejects a symlinked quarantine ancestor before moving building state", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const operationId = randomUUID() + const generationId = randomUUID() + const checkpointId = randomUUID() + const pinId = randomUUID() + const opDir = join(archiveDir, "operations", "active", `archive-${operationId}`) + const building = join(archiveDir, "building", generationId) + mkdirSync(opDir, { recursive: true }) + mkdirSync(building, { recursive: true }) + writeFileSync(join(building, "partial"), "retain me") + writeFileSync( + join(opDir, "intent.json"), + JSON.stringify({ + formatVersion: 3, + kind: "create", + operationId, + generationId, + signal: "traces", + rangeStart: "2026-06-01", + checkpointId, + archiveDir, + dataDir, + scratchRoot, + pinId, + pinPurpose: `archive:${generationId}`, + scratchSubdir: `archive-${operationId}`, + manifestSha256: null, + baseActiveGenerationId: null, + phase: "intent", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + }), + ) + const outside = join(dirname(archiveDir), "outside-quarantine") + mkdirSync(outside) + writeFileSync(join(outside, "sentinel"), "outside") + symlinkSync(outside, join(archiveDir, "quarantine")) + + const before = durableStateSnapshot(archiveDir, dataDir, outside) + const dryDecision = await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { + dryRun: true, + }) + ok(isFailClosed(dryDecision), `expected FailClosed, got ${dryDecision.kind}`) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /symlink|quarantine|unsafe/i, + ) + strictEqual(durableStateSnapshot(archiveDir, dataDir, outside), before) + }) + }) +}) + +// Shared test helpers for r9 parity tests. +function seedGeneration2( + archiveDir: string, + signal: string, + rangeDate: string, + generationId: string, + shardContents: string, +): { manifestSha: string; shardSha: string } { + const genDir = join(archiveDir, signal, rangeDate, "generations", generationId) + mkdirSync(join(genDir, "shards"), { recursive: true }) + writeFileSync(join(genDir, "shards", "00.parquet"), shardContents) + const shardSha = createHash("sha256").update(shardContents).digest("hex") + const manifest = { + formatVersion: 3, + generationId, + signal, + rangeStart: rangeDate, + rangeEndExclusive: "2026-06-02T00:00:00.000Z", + checkpointId: randomUUID(), + checkpointManifestFingerprint: "cid", + createdAt: "2026-06-02T00:00:00.000Z", + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + sourceRowCount: 1, + archivedRowCount: 1, + tuning: { + writerThreads: 1, + rowGroupRows: 10000, + maxShardRows: 500000, + maxShardBytes: 268435456, + targetChunkBytes: 1073741824, + minFreeSpaceReserve: 536870912, + }, + tuningConfig: null, + shards: [ + { + name: "00.parquet", + rowCount: 1, + minEventTimeUnixNano: `${BigInt(Date.parse(`${rangeDate}T12:00:00.000Z`)) * 1000000n}`, + maxEventTimeUnixNano: `${BigInt(Date.parse(`${rangeDate}T12:00:00.000Z`)) * 1000000n}`, + sha256: shardSha, + bytes: shardContents.length, + columns: ["TimestampTime", "ServiceName"], + complexDigest: "0", + complexDigestAlgorithm: "cityhash64-multiset-v3", + }, + ], + } + const manifestJson = JSON.stringify(manifest) + const manifestSha = createHash("sha256").update(manifestJson).digest("hex") + writeFileSync(join(genDir, "manifest.json"), manifestJson) + return { manifestSha, shardSha } +} + +function seedGcOp2( + archiveDir: string, + dataDir: string, + scratchRoot: string, + gid: string, + activeGid: string, +): { opId: string; manifestSha1: string; shardSha1: string } { + const { manifestSha, shardSha } = seedGeneration2(archiveDir, "traces", "2026-06-01", gid, "PAR1-old") + const opId = randomUUID() + const opDir = join(archiveDir, "operations", "active", `archive-${opId}`) + mkdirSync(opDir, { recursive: true }) + writeFileSync( + join(opDir, "intent.json"), + JSON.stringify({ + formatVersion: 3, + kind: "gc", + operationId: opId, + keep: 0, + targets: [ + { + signal: "traces", + rangeStart: "2026-06-01", + generationId: gid, + createdAt: "2026-06-02T00:00:00.000Z", + manifestSha256: manifestSha, + bytes: 7, + shards: [{ name: "00.parquet", bytes: 7, sha256: shardSha }], + recordedActiveGenerationId: activeGid, + }, + ], + completedTargets: 0, + archiveDir, + dataDir, + scratchRoot, + phase: "intent", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + }), + ) + return { opId, manifestSha1: manifestSha, shardSha1: shardSha } +} + +function seedGcOpWithEvidence( + archiveDir: string, + dataDir: string, + scratchRoot: string, + targets: Array<{ + gid: string + signal: string + rangeDate: string + contents: string + recordedActive: string + }>, + completedTargets: number, +): { opId: string; opDir: string } { + const opId = randomUUID() + const opDir = join(archiveDir, "operations", "active", `archive-${opId}`) + mkdirSync(opDir, { recursive: true }) + const gcTargets = targets.map((t) => { + const { manifestSha, shardSha } = seedGeneration2( + archiveDir, + t.signal, + t.rangeDate, + t.gid, + t.contents, + ) + return { + signal: t.signal, + rangeStart: t.rangeDate, + generationId: t.gid, + createdAt: "2026-06-02T00:00:00.000Z", + manifestSha256: manifestSha, + bytes: t.contents.length, + shards: [{ name: "00.parquet", bytes: t.contents.length, sha256: shardSha }], + recordedActiveGenerationId: t.recordedActive, + } + }) + writeFileSync( + join(opDir, "intent.json"), + JSON.stringify({ + formatVersion: 3, + kind: "gc", + operationId: opId, + keep: 0, + targets: gcTargets, + completedTargets, + archiveDir, + dataDir, + scratchRoot, + phase: completedTargets > 0 ? "gc-collecting" : "intent", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + }), + ) + return { opId, opDir } +} diff --git a/apps/cli/test/archive-timed-process.test.ts b/apps/cli/test/archive-timed-process.test.ts new file mode 100644 index 000000000..112f0d711 --- /dev/null +++ b/apps/cli/test/archive-timed-process.test.ts @@ -0,0 +1,109 @@ +import { describe, it } from "@effect/vitest" +import { ok, strictEqual } from "node:assert" +import { spawn, type ChildProcess } from "node:child_process" +import { EventEmitter } from "node:events" +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { dirname, join } from "node:path" +import { PassThrough } from "node:stream" +import { + collectChildOutputAfterClose, + createTimeReport, + parsePeakRss, + timeArgv, +} from "../src/server/archives/timed-process" + +const waitForClose = ( + child: ChildProcess, +): Promise<{ code: number | null; stdout: string; stderr: string }> => + new Promise((resolve) => { + let stdout = "" + let stderr = "" + child.stdout?.on("data", (chunk: Buffer) => { + stdout += chunk.toString() + }) + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString() + }) + child.once("close", (code) => resolve({ code, stdout, stderr })) + }) + +describe("calibration timed-process transport", () => { + it("uses the platform-specific /usr/bin/time RSS formats and fails closed on malformed output", () => { + strictEqual(parsePeakRss(" 123456 maximum resident set size", "darwin"), 123_456) + strictEqual(parsePeakRss("Maximum resident set size (kbytes): 12345", "linux"), 12_345 * 1024) + strictEqual(parsePeakRss("no RSS field", "darwin"), null) + strictEqual(parsePeakRss("no RSS field", "linux"), null) + strictEqual(timeArgv("darwin").join(" "), "-lp") + strictEqual(timeArgv("linux").join(" "), "-v") + }) + + it("keeps each report private and removes it after both success and read failure", () => { + const root = mkdtempSync(join(tmpdir(), "maple-timed-process-test-")) + try { + const success = createTimeReport(root) + writeFileSync(success.path, "timing report") + strictEqual(success.readAndRemove().report, "timing report") + strictEqual(existsSync(dirname(success.path)), false) + + const missing = createTimeReport(root) + const result = missing.readAndRemove() + ok(result.error?.includes("failed to read /usr/bin/time report")) + strictEqual(existsSync(dirname(missing.path)), false) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + it("directs /usr/bin/time output to its file without contaminating worker stderr", async () => { + const report = createTimeReport() + try { + const child = spawn( + "/usr/bin/time", + [ + ...timeArgv(), + "-o", + report.path, + process.execPath, + "-e", + 'process.stdout.write("worker stdout"); process.stderr.write("worker stderr")', + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ) + const result = await waitForClose(child) + // macOS's sandbox can deny BSD time's optional `kern.clockrate` lookup, + // which makes time exit 1 after the worker still ran and wrote its report. + ok(result.code === 0 || result.stderr.includes("sysctl kern.clockrate: Operation not permitted")) + strictEqual(result.stdout, "worker stdout") + ok(result.stderr.includes("worker stderr")) + ok(!result.stderr.includes("maximum resident set size")) + const timing = report.readAndRemove() + strictEqual(timing.error, undefined) + ok(timing.report.length > 0) + } finally { + report.remove() + } + }) + + it("does not resolve completion at exit before the worker pipes drain", async () => { + const child = Object.assign(new EventEmitter(), { + stdout: new PassThrough(), + stderr: new PassThrough(), + }) as unknown as ChildProcess + const completion = collectChildOutputAfterClose(child) + let settled = false + void completion.then(() => { + settled = true + }) + child.emit("exit", 0, null) + await Promise.resolve() + strictEqual(settled, false) + child.stdout!.write("complete stdout") + child.stderr!.write("complete stderr") + child.emit("close", 0, null) + const result = await completion + strictEqual(result.code, 0) + strictEqual(result.stdout, "complete stdout") + strictEqual(result.stderr, "complete stderr") + }) +}) diff --git a/apps/cli/test/chdb-args.test.ts b/apps/cli/test/chdb-args.test.ts new file mode 100644 index 000000000..ed9fca70b --- /dev/null +++ b/apps/cli/test/chdb-args.test.ts @@ -0,0 +1,30 @@ +import { describe, it } from "@effect/vitest" +import { deepStrictEqual } from "node:assert" +import { chdbArgv } from "../src/server/chdb" + +describe("embedded chDB arguments", () => { + it("waits for metadata and serializes table loading and restore work", () => { + deepStrictEqual(chdbArgv({ dataDir: "/tmp/maple-data" }), [ + "clickhouse", + "--async_load_databases=0", + "--async_load_system_database=0", + "--tables_loader_foreground_pool_size=1", + "--tables_loader_background_pool_size=1", + "--restore_threads=1", + "--path=/tmp/maple-data", + ]) + }) + + it("keeps an explicit config after the loader and restore safety settings", () => { + deepStrictEqual(chdbArgv({ dataDir: "/tmp/maple-data", configFile: "/tmp/backups.xml" }), [ + "clickhouse", + "--async_load_databases=0", + "--async_load_system_database=0", + "--tables_loader_foreground_pool_size=1", + "--tables_loader_background_pool_size=1", + "--restore_threads=1", + "--path=/tmp/maple-data", + "--config-file=/tmp/backups.xml", + ]) + }) +}) diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts new file mode 100644 index 000000000..f06e1d054 --- /dev/null +++ b/apps/cli/test/checkpoints.test.ts @@ -0,0 +1,1026 @@ +import { describe, it } from "@effect/vitest" +import { Effect } from "effect" +import { deepStrictEqual, match, ok, rejects, strictEqual, throws } from "node:assert" +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs" +import { tmpdir } from "node:os" +import { basename, dirname, join } from "node:path" +import { + assertCheckpointRootSafe, + checkpointRoot, + checkpointSnapshotDir, + checkpointStatePath, + isMissingBackupConfigurationError, + LocalQueryError, + newCheckpointId, + parseCheckpointManifest, + parseCheckpointState, + readCheckpointState, + reconcileCheckpointRecovery, + reconcileCheckpointOperations, + resetTransactionPath, + resetLiveStorePreservingCheckpoints, + type RestoreRecoveryFaults, + resolveCheckpoint, + restoreDataPath, + restoreQuarantinePath, + restoreRootPath, + restoreTransactionPath, + retireCheckpointIfEligible, + validateCheckpointDataDir, + writeBackupConfig, +} from "../src/server/checkpoints" +import { + durableWrite, + isUnsupportedDirectorySyncError, + syncDirectory, + syncTree, +} from "../src/server/durable-files" +import { SCHEMA_FINGERPRINT } from "../src/server/serve" +import { storeMarkerPath, storeOpenMarkerPath } from "../src/server/store-version" +import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" + +const withDataDir = async (run: (dataDir: string) => Promise | void): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-checkpoint-test-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir, { recursive: true }) + try { + await run(dataDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const manifest = ( + checkpointId: string, + operationId = newCheckpointId(), + sourceDataDir = "/tmp/maple-data", +): Record => ({ + formatVersion: 1, + checkpointId, + operationId, + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + createdAt: "2026-01-01T00:00:00.000Z", + sourceDataDir, + backupRelativePath: `snapshots/${checkpointId}/backup`, + backupBytes: 123, + validation: { + validatedAt: "2026-01-01T00:00:01.000Z", + traces: 1, + logs: 2, + metricsSum: 3, + metricsGauge: 4, + metricsHistogram: 5, + metricsExponentialHistogram: 6, + materializedViews: 33, + }, +}) + +describe("fresh-process checkpoint reopen probe", () => { + it("rejects a non-normalized or relative data directory before opening chDB", () => { + throws(() => validateCheckpointDataDir("relative/data"), /normalized absolute data directory/) + }) +}) + +const writeSnapshot = (dataDir: string, checkpointId: string, operationId = newCheckpointId()): void => { + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + mkdirSync(join(snapshot, "backup"), { recursive: true }) + writeFileSync(join(snapshot, "backup", "data.bin"), "backup") + const value = manifest(checkpointId, operationId, dataDir) + value.backupBytes = 6 + writeFileSync(join(snapshot, "manifest.json"), `${JSON.stringify(value)}\n`) +} + +const writeState = ( + dataDir: string, + current: string, + previous: string | null = null, + revision = newCheckpointId(), +): void => { + mkdirSync(checkpointRoot(dataDir), { recursive: true }) + writeFileSync( + checkpointStatePath(dataDir), + `${JSON.stringify({ + formatVersion: 1, + revision, + current, + previous, + committedAt: "2026-01-01T00:00:02.000Z", + })}\n`, + ) +} + +const writeCheckpointOperation = ( + dataDir: string, + operationId: string, + checkpointId: string, + phase: "intent" | "backup-complete" | "manifest-complete" | "pointer-complete" | "retention-complete", + base: { + readonly revision: string | null + readonly current: string | null + readonly previous: string | null + } = { revision: null, current: null, previous: null }, +): string => { + const dir = join(checkpointRoot(dataDir), "operations", `checkpoint-${operationId}`) + mkdirSync(dir, { recursive: true }) + writeFileSync( + join(dir, "intent.json"), + `${JSON.stringify({ + formatVersion: 1, + operationId, + checkpointId, + baseRevision: base.revision, + baseCurrent: base.current, + basePrevious: base.previous, + phase, + startedAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ) + return dir +} + +const restoreValidation = { + validatedAt: "2026-01-01T00:00:01.000Z", + traces: 1, + logs: 2, + metricsSum: 3, + metricsGauge: 4, + metricsHistogram: 5, + metricsExponentialHistogram: 6, + materializedViews: 33, +} + +const writeRestoreTransaction = ( + dataDir: string, + operationId: string, + checkpointId: string, + quarantineId: string, + phase: "intent" | "restore-ready" | "old-quarantined" | "new-live" | "markers-committed", +): void => { + writeFileSync( + restoreTransactionPath(dataDir), + `${JSON.stringify({ + formatVersion: 1, + operationId, + checkpointId, + quarantineId, + phase, + createdAt: "2026-01-01T00:00:00.000Z", + validation: phase === "intent" ? null : restoreValidation, + })}\n`, + ) +} + +const writeRestoreReady = (dataDir: string, operationId: string, checkpointId: string): void => { + const restoreData = restoreDataPath(dataDir, operationId) + mkdirSync(restoreData, { recursive: true }) + writeFileSync( + join(restoreData, ".maple-restore-ready.json"), + `${JSON.stringify({ formatVersion: 1, operationId, checkpointId })}\n`, + ) +} + +describe("writeBackupConfig", () => { + it("writes restrictive runtime and escaped restore configurations", async () => { + await withDataDir((dataDir) => { + const runtimePath = join(dataDir, "runtime.xml") + writeBackupConfig(runtimePath) + const runtime = readFileSync(runtimePath, "utf8") + ok(runtime.includes("default")) + ok(runtime.includes("backups")) + strictEqual(lstatSync(runtimePath).mode & 0o777, 0o600) + + const restorePath = join(dataDir, "restore.xml") + writeBackupConfig(restorePath, join(dataDir, "source & ")) + const restore = readFileSync(restorePath, "utf8") + ok(restore.includes("src")) + ok(restore.includes("source & <store>")) + }) + }) +}) + +describe("checkpoint IDs and strict parsers", () => { + it("generates collision-resistant UUIDs", () => { + const ids = new Set(Array.from({ length: 2_000 }, () => newCheckpointId())) + strictEqual(ids.size, 2_000) + for (const id of ids) match(id, /^[0-9a-f-]{36}$/) + }) + + it("accepts a complete manifest and rejects ID, path, compatibility, and count corruption", () => { + const id = newCheckpointId() + strictEqual(parseCheckpointManifest(manifest(id), id).checkpointId, id) + strictEqual(parseCheckpointManifest(manifest(id), id, "/tmp/maple-data").checkpointId, id) + throwsMessage( + () => parseCheckpointManifest(manifest(id), id, "/tmp/different-owner"), + /configured owner/, + ) + const wrong = newCheckpointId() + ok(wrong !== id) + throwsMessage(() => parseCheckpointManifest(manifest(id), wrong), /does not match/) + throwsMessage( + () => parseCheckpointManifest({ ...manifest(id), backupRelativePath: "../escape" }, id), + /backup path/, + ) + throwsMessage( + () => parseCheckpointManifest({ ...manifest(id), chdbVersion: "v0.0.0" }, id), + /version mismatch/, + ) + throwsMessage( + () => + parseCheckpointManifest({ + ...manifest(id), + validation: { ...(manifest(id).validation as object), logs: -1 }, + }), + /invalid logs/, + ) + }) + + it("accepts versioned current/previous state and rejects malformed selection", () => { + const current = newCheckpointId() + const previous = newCheckpointId() + const revision = newCheckpointId() + deepStrictEqual( + parseCheckpointState({ + formatVersion: 1, + revision, + current, + previous, + committedAt: "2026-01-01T00:00:00.000Z", + }), + { + formatVersion: 1, + revision, + current, + previous, + committedAt: "2026-01-01T00:00:00.000Z", + }, + ) + throwsMessage( + () => + parseCheckpointState({ + formatVersion: 1, + revision, + current, + previous: current, + committedAt: "2026-01-01T00:00:00.000Z", + }), + /must differ/, + ) + throwsMessage( + () => + parseCheckpointState({ + formatVersion: 99, + revision, + current, + previous: null, + committedAt: "2026-01-01T00:00:00.000Z", + }), + /unsupported/, + ) + }) +}) + +describe("checkpoint state resolution", () => { + it("resolves immutable current, previous, and explicit IDs", async () => { + await withDataDir(async (dataDir) => { + const current = newCheckpointId() + const previous = newCheckpointId() + writeSnapshot(dataDir, current) + writeSnapshot(dataDir, previous) + writeState(dataDir, current, previous) + + const state = await readCheckpointState(dataDir) + strictEqual(state.current, current) + strictEqual((await resolveCheckpoint(dataDir, "current")).checkpointId, current) + strictEqual((await resolveCheckpoint(dataDir, "previous")).checkpointId, previous) + strictEqual((await resolveCheckpoint(dataDir, previous)).checkpointId, previous) + }) + }) + + it("fails closed for missing/malformed state, incomplete snapshots, and legacy aliases", async () => { + await withDataDir(async (dataDir) => { + await rejects(readCheckpointState(dataDir), /state not found/) + + mkdirSync(join(checkpointRoot(dataDir), "snapshots", newCheckpointId()), { + recursive: true, + }) + await rejects(readCheckpointState(dataDir), /state missing while checkpoint data exists/) + + writeFileSync(checkpointStatePath(dataDir), "{bad json") + await rejects(readCheckpointState(dataDir), /JSON/) + + rmSync(checkpointRoot(dataDir), { recursive: true }) + mkdirSync(join(checkpointRoot(dataDir), "current"), { recursive: true }) + await rejects(readCheckpointState(dataDir), /legacy preview/) + }) + }) + + it("rejects symlink roots and symlinked snapshot paths", async () => { + await withDataDir(async (dataDir) => { + const outside = join(dirname(dataDir), "outside") + mkdirSync(outside) + symlinkSync(outside, checkpointRoot(dataDir)) + throwsMessage(() => assertCheckpointRootSafe(dataDir), /symlink/) + }) + }) + + it("rejects symlinked manifest and backup leaves outside the registry", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + const outside = join(dirname(dataDir), "outside-checkpoint") + mkdirSync(outside) + writeFileSync( + join(outside, "manifest.json"), + `${JSON.stringify(manifest(checkpointId, newCheckpointId(), dataDir))}\n`, + ) + mkdirSync(join(outside, "backup")) + mkdirSync(snapshot, { recursive: true }) + symlinkSync(join(outside, "manifest.json"), join(snapshot, "manifest.json")) + symlinkSync(join(outside, "backup"), join(snapshot, "backup")) + writeState(dataDir, checkpointId) + + await rejects(readCheckpointState(dataDir), /symlink/) + ok(existsSync(join(outside, "manifest.json"))) + ok(existsSync(join(outside, "backup"))) + }) + }) + + it("rejects nested symlinks inside an otherwise real checkpoint backup", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + const outside = join(dirname(dataDir), "outside-backup.bin") + writeFileSync(outside, "sensitive") + writeSnapshot(dataDir, checkpointId) + symlinkSync(outside, join(checkpointSnapshotDir(dataDir, checkpointId), "backup", "nested-link")) + writeState(dataDir, checkpointId) + + await rejects(readCheckpointState(dataDir), /symlink/) + strictEqual(readFileSync(outside, "utf8"), "sensitive") + }) + }) +}) + +describe("checkpoint reconciliation and retention", () => { + it("quarantines only an exactly owned incomplete operation and preserves its bytes", async () => { + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + mkdirSync(join(snapshot, "backup"), { recursive: true }) + writeFileSync(join(snapshot, "backup", "partial.bin"), "partial") + writeCheckpointOperation(dataDir, operationId, checkpointId, "backup-complete") + + await reconcileCheckpointOperations(dataDir) + + ok(!existsSync(snapshot)) + const quarantineRoot = join(checkpointRoot(dataDir), "quarantine") + const quarantines = readdirSync(quarantineRoot) + strictEqual(quarantines.length, 1) + ok( + existsSync( + join(quarantineRoot, quarantines[0]!, "incomplete-snapshot", "backup", "partial.bin"), + ), + ) + ok(existsSync(join(quarantineRoot, quarantines[0]!, "operation", "intent.json"))) + }) + }) + + it("fails closed and preserves a malformed operation", async () => { + await withDataDir(async (dataDir) => { + const operationDir = join(checkpointRoot(dataDir), "operations", "checkpoint-not-a-uuid") + mkdirSync(operationDir, { recursive: true }) + writeFileSync(join(operationDir, "intent.json"), "{bad json") + await rejects(reconcileCheckpointOperations(dataDir)) + ok(existsSync(join(operationDir, "intent.json"))) + }) + }) + + it("rejects a symlinked operations root without touching its target", async () => { + await withDataDir(async (dataDir) => { + const outside = join(dirname(dataDir), "outside-operations") + const sentinel = join(outside, "sentinel") + mkdirSync(outside) + writeFileSync(sentinel, "preserve") + mkdirSync(checkpointRoot(dataDir), { recursive: true }) + symlinkSync(outside, join(checkpointRoot(dataDir), "operations")) + + await rejects(reconcileCheckpointOperations(dataDir), /real directory|symlink/) + strictEqual(readFileSync(sentinel, "utf8"), "preserve") + }) + }) + + it("rejects mismatched operation directory and intent identities without mutation", async () => { + await withDataDir(async (dataDir) => { + const directoryId = newCheckpointId() + const intentId = newCheckpointId() + const checkpointId = newCheckpointId() + const directory = writeCheckpointOperation(dataDir, directoryId, checkpointId, "backup-complete") + const intent = JSON.parse(readFileSync(join(directory, "intent.json"), "utf8")) + intent.operationId = intentId + writeFileSync(join(directory, "intent.json"), `${JSON.stringify(intent)}\n`) + + await rejects(reconcileCheckpointOperations(dataDir), /identity mismatch/) + ok(existsSync(join(directory, "intent.json"))) + }) + }) + + it("publishes a completed first checkpoint after an interrupted pointer update", async () => { + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId, operationId) + writeCheckpointOperation(dataDir, operationId, checkpointId, "manifest-complete") + + await reconcileCheckpointOperations(dataDir) + + const state = await readCheckpointState(dataDir) + strictEqual(state.revision, operationId) + strictEqual(state.current, checkpointId) + strictEqual(state.previous, null) + ok(!existsSync(join(checkpointRoot(dataDir), "operations", `checkpoint-${operationId}`))) + }) + }) + + it("resumes a based promotion and retires only the superseded previous checkpoint", async () => { + await withDataDir(async (dataDir) => { + const oldCurrent = newCheckpointId() + const oldPrevious = newCheckpointId() + const baseRevision = newCheckpointId() + writeSnapshot(dataDir, oldCurrent) + writeSnapshot(dataDir, oldPrevious) + writeState(dataDir, oldCurrent, oldPrevious, baseRevision) + + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId, operationId) + writeCheckpointOperation(dataDir, operationId, checkpointId, "manifest-complete", { + revision: baseRevision, + current: oldCurrent, + previous: oldPrevious, + }) + + await reconcileCheckpointOperations(dataDir) + + const state = await readCheckpointState(dataDir) + strictEqual(state.current, checkpointId) + strictEqual(state.previous, oldCurrent) + ok(existsSync(checkpointSnapshotDir(dataDir, oldCurrent))) + ok(!existsSync(checkpointSnapshotDir(dataDir, oldPrevious))) + }) + }) + + it("converges after every retirement intent, data-removal, and completion boundary", async () => { + for (const boundary of [ + "afterRetirementIntent", + "afterRetirementRename", + "afterRetiredSnapshotRemoval", + "afterRetirementComplete", + ] as const) { + await withDataDir(async (dataDir) => { + const current = newCheckpointId() + const previous = newCheckpointId() + const old = newCheckpointId() + const retirementId = newCheckpointId() + writeSnapshot(dataDir, current) + writeSnapshot(dataDir, previous) + writeSnapshot(dataDir, old) + writeState(dataDir, current, previous) + const state = await readCheckpointState(dataDir) + + await rejects( + retireCheckpointIfEligible(dataDir, old, state, retirementId, { + [boundary]: () => { + throw new Error(`injected ${boundary}`) + }, + }), + /injected/, + ) + const retirement = await retireCheckpointIfEligible(dataDir, old, state, retirementId) + + ok(!existsSync(checkpointSnapshotDir(dataDir, old)), boundary) + ok(retirement !== null && existsSync(join(retirement, "complete.json")), boundary) + ok(existsSync(checkpointSnapshotDir(dataDir, current)), boundary) + ok(existsSync(checkpointSnapshotDir(dataDir, previous)), boundary) + }) + } + }) + + it("converges after every retirement cleanup and completed-operation boundary", async () => { + for (const boundary of [ + "afterRetirementCleanupRename", + "afterRetirementCleanupRemoval", + "afterCompletedOperationPreserved", + ] as const) { + await withDataDir(async (dataDir) => { + const oldCurrent = newCheckpointId() + const oldPrevious = newCheckpointId() + const baseRevision = newCheckpointId() + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, oldCurrent) + writeSnapshot(dataDir, oldPrevious) + writeSnapshot(dataDir, checkpointId, operationId) + writeState(dataDir, checkpointId, oldCurrent, operationId) + writeCheckpointOperation(dataDir, operationId, checkpointId, "pointer-complete", { + revision: baseRevision, + current: oldCurrent, + previous: oldPrevious, + }) + let injected = false + + await rejects( + reconcileCheckpointOperations(dataDir, { + [boundary]: () => { + if (injected) return + injected = true + throw new Error(`injected ${boundary}`) + }, + }), + /injected/, + ) + await reconcileCheckpointOperations(dataDir) + + strictEqual((await readCheckpointState(dataDir)).current, checkpointId) + ok(!existsSync(checkpointSnapshotDir(dataDir, oldPrevious)), boundary) + ok( + !existsSync(join(checkpointRoot(dataDir), "operations", `checkpoint-${operationId}`)), + boundary, + ) + }) + } + }) + + it("cleans an exactly completed retirement after the operation completion record", async () => { + for (const keepRetirementRecord of [true, false]) { + await withDataDir(async (dataDir) => { + const oldCurrent = newCheckpointId() + const oldPrevious = newCheckpointId() + const baseRevision = newCheckpointId() + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, oldCurrent) + writeSnapshot(dataDir, checkpointId, operationId) + writeState(dataDir, checkpointId, oldCurrent, operationId) + writeCheckpointOperation(dataDir, operationId, checkpointId, "retention-complete", { + revision: baseRevision, + current: oldCurrent, + previous: oldPrevious, + }) + const retirement = join(checkpointRoot(dataDir), "retiring", `retirement-${operationId}`) + const interruptedCleanup = `${retirement}.cleanup-${newCheckpointId()}` + if (keepRetirementRecord) { + mkdirSync(retirement, { recursive: true }) + const record = { + formatVersion: 1, + retirementId: operationId, + checkpointId: oldPrevious, + stateRevision: operationId, + } + writeFileSync(join(retirement, "intent.json"), `${JSON.stringify(record)}\n`) + writeFileSync(join(retirement, "complete.json"), `${JSON.stringify(record)}\n`) + } else { + mkdirSync(interruptedCleanup, { recursive: true }) + writeFileSync(join(interruptedCleanup, "preserve"), "completed cleanup debris") + } + + await reconcileCheckpointOperations(dataDir) + + ok(!existsSync(retirement)) + if (!keepRetirementRecord) { + strictEqual( + readFileSync(join(interruptedCleanup, "preserve"), "utf8"), + "completed cleanup debris", + ) + } + ok(!existsSync(join(checkpointRoot(dataDir), "operations", `checkpoint-${operationId}`))) + strictEqual((await readCheckpointState(dataDir)).current, checkpointId) + }) + } + }) + + it("retains current, previous, pinned, and malformed candidates; retires only proven safe", async () => { + await withDataDir(async (dataDir) => { + const current = newCheckpointId() + const previous = newCheckpointId() + const old = newCheckpointId() + writeSnapshot(dataDir, current) + writeSnapshot(dataDir, previous) + writeSnapshot(dataDir, old) + writeState(dataDir, current, previous) + const state = await readCheckpointState(dataDir) + + await retireCheckpointIfEligible(dataDir, current, state) + await retireCheckpointIfEligible(dataDir, previous, state) + ok(existsSync(checkpointSnapshotDir(dataDir, current))) + ok(existsSync(checkpointSnapshotDir(dataDir, previous))) + + const pinDir = join(checkpointRoot(dataDir), "pins", old) + mkdirSync(pinDir, { recursive: true }) + writeFileSync(join(pinDir, "pin.json"), "{}") + await retireCheckpointIfEligible(dataDir, old, state) + ok(existsSync(checkpointSnapshotDir(dataDir, old))) + + rmSync(pinDir, { recursive: true }) + await retireCheckpointIfEligible(dataDir, old, state) + ok(!existsSync(checkpointSnapshotDir(dataDir, old))) + + const malformed = newCheckpointId() + writeSnapshot(dataDir, malformed) + writeFileSync(join(checkpointSnapshotDir(dataDir, malformed), "manifest.json"), "{bad json") + await rejects(retireCheckpointIfEligible(dataDir, malformed, state)) + ok(existsSync(checkpointSnapshotDir(dataDir, malformed))) + }) + }) +}) + +describe("live-store reset safety", () => { + it("removes live chDB data and sibling markers while preserving the checkpoint registry", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId) + writeState(dataDir, checkpointId) + mkdirSync(join(dataDir, "store"), { recursive: true }) + mkdirSync(join(dataDir, "metadata"), { recursive: true }) + mkdirSync(join(dataDir, "tmp"), { recursive: true }) + writeFileSync(join(dataDir, "store", "part.bin"), "live") + writeFileSync(join(dataDir, "metadata", "table.sql"), "live") + writeFileSync(join(dataDir, "tmp", "scratch.bin"), "live") + writeFileSync(storeMarkerPath(dataDir), "{}") + writeFileSync(storeOpenMarkerPath(dataDir), "999\n") + + await Effect.runPromise(resetLiveStorePreservingCheckpoints(dataDir)) + + strictEqual((await readCheckpointState(dataDir)).current, checkpointId) + ok(existsSync(checkpointSnapshotDir(dataDir, checkpointId))) + ok(!existsSync(join(dataDir, "store"))) + ok(!existsSync(join(dataDir, "metadata"))) + ok(!existsSync(join(dataDir, "tmp"))) + ok(!existsSync(storeMarkerPath(dataDir))) + ok(!existsSync(storeOpenMarkerPath(dataDir))) + ok(!existsSync(resetTransactionPath(dataDir))) + }) + }) + + it("fails closed before deleting live data when checkpoint infrastructure is unsafe", async () => { + await withDataDir(async (dataDir) => { + const outside = join(dirname(dataDir), "outside-pins") + mkdirSync(outside) + mkdirSync(checkpointRoot(dataDir), { recursive: true }) + symlinkSync(outside, join(checkpointRoot(dataDir), "pins")) + mkdirSync(join(dataDir, "store"), { recursive: true }) + writeFileSync(join(dataDir, "store", "preserve.bin"), "live") + + await rejects(Effect.runPromise(resetLiveStorePreservingCheckpoints(dataDir)), /real directory/) + strictEqual(readFileSync(join(dataDir, "store", "preserve.bin"), "utf8"), "live") + }) + }) + + it("preserves and reports unknown data-directory entries before any deletion", async () => { + await withDataDir(async (dataDir) => { + mkdirSync(join(dataDir, "store"), { recursive: true }) + writeFileSync(join(dataDir, "store", "preserve.bin"), "live") + writeFileSync(join(dataDir, "user-owned.txt"), "preserve") + + await rejects( + Effect.runPromise(resetLiveStorePreservingCheckpoints(dataDir)), + /unrecognized data-directory entries were preserved/, + ) + + strictEqual(readFileSync(join(dataDir, "store", "preserve.bin"), "utf8"), "live") + strictEqual(readFileSync(join(dataDir, "user-owned.txt"), "utf8"), "preserve") + ok(!existsSync(resetTransactionPath(dataDir))) + }) + }) + + it("reconciles interruption at every reset deletion, marker, and journal boundary", async () => { + const boundaries: ReadonlyArray = [ + "afterResetIntent", + "afterResetEntryRemoval", + "afterResetLiveClearedRecord", + "afterResetStoreMarkerRemoval", + "afterResetOpenMarkerRemoval", + "afterResetMarkersClearedRecord", + "afterResetTransactionRemoval", + ] + for (const boundary of boundaries) { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + writeSnapshot(dataDir, checkpointId) + writeState(dataDir, checkpointId) + for (const entry of ["data", "metadata", "store", "tmp"]) { + mkdirSync(join(dataDir, entry), { recursive: true }) + writeFileSync(join(dataDir, entry, "live.bin"), "live") + } + writeFileSync(storeMarkerPath(dataDir), "{}") + writeFileSync(storeOpenMarkerPath(dataDir), "999\n") + let injected = false + const faults = { + [boundary]: () => { + if (injected) return + injected = true + throw new Error(`injected ${boundary}`) + }, + } as RestoreRecoveryFaults + + await rejects( + Effect.runPromise(resetLiveStorePreservingCheckpoints(dataDir, faults)), + /injected/, + ) + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + + for (const entry of ["data", "metadata", "store", "tmp"]) { + ok(!existsSync(join(dataDir, entry)), `${boundary}: ${entry}`) + } + strictEqual((await readCheckpointState(dataDir)).current, checkpointId) + ok(!existsSync(storeMarkerPath(dataDir)), boundary) + ok(!existsSync(storeOpenMarkerPath(dataDir)), boundary) + ok(!existsSync(resetTransactionPath(dataDir)), boundary) + }) + } + }) + + it("rejects malformed or escaping reset journals without mutation", async () => { + await withDataDir(async (dataDir) => { + const outside = join(dirname(dataDir), "outside-reset") + mkdirSync(outside) + writeFileSync(join(outside, "preserve"), "outside") + writeFileSync( + resetTransactionPath(dataDir), + `${JSON.stringify({ + formatVersion: 1, + operationId: newCheckpointId(), + dataDir, + targets: ["../outside-reset"], + phase: "intent", + createdAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ) + + await rejects(Effect.runPromise(reconcileCheckpointRecovery(dataDir)), /unsafe reset/) + strictEqual(readFileSync(join(outside, "preserve"), "utf8"), "outside") + ok(existsSync(resetTransactionPath(dataDir))) + }) + }) +}) + +describe("live restore transaction reconciliation", () => { + it("is a no-op when no transaction or restore debris exists", async () => { + await withDataDir(async (dataDir) => { + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + ok(existsSync(dataDir)) + }) + }) + + it("preserves an interrupted pre-ready restore and leaves the old live store selected", async () => { + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + const quarantineId = newCheckpointId() + writeFileSync(join(dataDir, "old-live"), "old") + writeRestoreTransaction(dataDir, operationId, checkpointId, quarantineId, "intent") + mkdirSync(restoreDataPath(dataDir, operationId), { recursive: true }) + writeFileSync(join(restoreDataPath(dataDir, operationId), "partial"), "partial") + + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + + ok(existsSync(join(dataDir, "old-live"))) + ok(!existsSync(restoreRootPath(dataDir, operationId))) + ok(!existsSync(restoreTransactionPath(dataDir))) + const siblingNames = readdirSync(dirname(dataDir)) + ok( + siblingNames.some((name) => + name.startsWith(`${basename(dataDir)}.restore-${operationId}.quarantine-`), + ), + ) + ok( + siblingNames.some((name) => + name.startsWith(`${basename(dataDir)}.restore-transaction.json.quarantine-`), + ), + ) + }) + }) + + it("resumes from restore-ready through quarantine, swap, durable markers, and completion", async () => { + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + const quarantineId = newCheckpointId() + const quarantine = restoreQuarantinePath(dataDir, operationId, quarantineId) + writeFileSync(join(dataDir, "old-live"), "old") + writeFileSync(storeOpenMarkerPath(dataDir), "999\n") + writeRestoreReady(dataDir, operationId, checkpointId) + writeFileSync(join(restoreDataPath(dataDir, operationId), "new-live"), "new") + writeRestoreTransaction(dataDir, operationId, checkpointId, quarantineId, "restore-ready") + + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + + ok(existsSync(join(dataDir, "new-live"))) + ok(existsSync(join(quarantine, "old-live"))) + ok(existsSync(storeMarkerPath(dataDir))) + ok(!existsSync(storeOpenMarkerPath(dataDir))) + ok(!existsSync(restoreTransactionPath(dataDir))) + ok(!existsSync(restoreRootPath(dataDir, operationId))) + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + }) + }) + + it("infers the recorded rename boundary from exact topology and completes idempotently", async () => { + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + const quarantineId = newCheckpointId() + const quarantine = restoreQuarantinePath(dataDir, operationId, quarantineId) + writeFileSync(join(dataDir, "old-live"), "old") + writeRestoreReady(dataDir, operationId, checkpointId) + writeFileSync(join(restoreDataPath(dataDir, operationId), "new-live"), "new") + writeRestoreTransaction(dataDir, operationId, checkpointId, quarantineId, "restore-ready") + renameSync(dataDir, quarantine) + + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + + ok(existsSync(join(dataDir, "new-live"))) + ok(existsSync(join(quarantine, "old-live"))) + ok(!existsSync(restoreTransactionPath(dataDir))) + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + }) + }) + + it("converges after interruption at every live-swap, marker, and cleanup boundary", async () => { + const boundaries: ReadonlyArray = [ + "afterLiveQuarantineRename", + "afterOldQuarantinedRecord", + "afterRestoredLiveRename", + "afterNewLiveRecord", + "afterStoreMarkerWrite", + "afterOpenMarkerRemoval", + "afterMarkersCommittedRecord", + "afterReadyMarkerRemoval", + "afterRestoreRootRemoval", + ] + for (const boundary of boundaries) { + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + const quarantineId = newCheckpointId() + const quarantine = restoreQuarantinePath(dataDir, operationId, quarantineId) + writeFileSync(join(dataDir, "old-live"), "old") + writeFileSync(storeOpenMarkerPath(dataDir), "999\n") + writeRestoreReady(dataDir, operationId, checkpointId) + writeFileSync(join(restoreDataPath(dataDir, operationId), "new-live"), "new") + writeRestoreTransaction(dataDir, operationId, checkpointId, quarantineId, "restore-ready") + const faults = { + [boundary]: () => { + throw new Error(`injected ${boundary}`) + }, + } as RestoreRecoveryFaults + + await rejects(Effect.runPromise(reconcileCheckpointRecovery(dataDir, faults)), /injected/) + await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) + + ok(existsSync(join(dataDir, "new-live")), boundary) + ok(existsSync(join(quarantine, "old-live")), boundary) + ok(existsSync(storeMarkerPath(dataDir)), boundary) + ok(!existsSync(storeOpenMarkerPath(dataDir)), boundary) + ok(!existsSync(restoreTransactionPath(dataDir)), boundary) + ok(!existsSync(restoreRootPath(dataDir, operationId)), boundary) + }) + } + }) + + it("fails closed on malformed or unrecorded restore state without deleting it", async () => { + await withDataDir(async (dataDir) => { + writeFileSync(restoreTransactionPath(dataDir), "{bad json") + await rejects(Effect.runPromise(reconcileCheckpointRecovery(dataDir))) + ok(existsSync(restoreTransactionPath(dataDir))) + rmSync(restoreTransactionPath(dataDir)) + + const debris = `${dataDir}.restore-${newCheckpointId()}` + mkdirSync(debris) + await rejects( + Effect.runPromise(reconcileCheckpointRecovery(dataDir)), + /without a valid transaction/, + ) + ok(existsSync(debris)) + }) + }) + + it("rejects symlinked transaction and restore topology without touching targets", async () => { + await withDataDir(async (dataDir) => { + const outside = join(dirname(dataDir), "outside-transaction") + writeFileSync(outside, "{}") + symlinkSync(outside, restoreTransactionPath(dataDir)) + await rejects(Effect.runPromise(reconcileCheckpointRecovery(dataDir)), /real file/) + strictEqual(readFileSync(outside, "utf8"), "{}") + }) + await withDataDir(async (dataDir) => { + const operationId = newCheckpointId() + const checkpointId = newCheckpointId() + const quarantineId = newCheckpointId() + const outside = join(dirname(dataDir), "outside-restore") + mkdirSync(outside) + writeFileSync(join(outside, "sentinel"), "preserve") + writeRestoreTransaction(dataDir, operationId, checkpointId, quarantineId, "intent") + symlinkSync(outside, restoreRootPath(dataDir, operationId)) + + await rejects(Effect.runPromise(reconcileCheckpointRecovery(dataDir)), /real directory/) + strictEqual(readFileSync(join(outside, "sentinel"), "utf8"), "preserve") + }) + }) +}) + +describe("backup configuration classification", () => { + it("classifies only backup-specific errors", () => { + ok( + isMissingBackupConfigurationError( + new LocalQueryError(500, "INVALID_CONFIG_PARAMETER: backups.allowed_disk is not set"), + ), + ) + ok( + isMissingBackupConfigurationError( + new LocalQueryError(500, "Disk default is not allowed for backups"), + ), + ) + ok(!isMissingBackupConfigurationError(new LocalQueryError(500, "INVALID_CONFIG_PARAMETER"))) + ok(!isMissingBackupConfigurationError(new Error("UNKNOWN_TABLE"))) + ok(!isMissingBackupConfigurationError(new Error("connection refused"))) + }) +}) + +describe("durable filesystem primitives", () => { + it("atomically replaces a file and syncs a directory on this platform", async () => { + await withDataDir(async (dataDir) => { + const path = join(dataDir, "state.json") + await durableWrite(path, "old\n") + await durableWrite(path, "new\n") + strictEqual(readFileSync(path, "utf8"), "new\n") + strictEqual(lstatSync(path).mode & 0o777, 0o600) + await syncDirectory(dataDir) + }) + }) + + it("leaves the old destination intact when injected before file sync or rename", async () => { + await withDataDir(async (dataDir) => { + const path = join(dataDir, "state.json") + await durableWrite(path, "old\n") + await rejects( + durableWrite(path, "new\n", { + beforeFileSync: () => { + throw new Error("sync fault") + }, + }), + /sync fault/, + ) + strictEqual(readFileSync(path, "utf8"), "old\n") + await rejects( + durableWrite(path, "new\n", { + beforeRename: () => { + throw new Error("rename fault") + }, + }), + /rename fault/, + ) + strictEqual(readFileSync(path, "utf8"), "old\n") + strictEqual(readFileSync(path, "utf8"), "old\n", "fault must not partially publish new bytes") + }) + }) + + it("does not treat descriptor/type errors as unsupported directory sync", () => { + ok(isUnsupportedDirectorySyncError({ code: "EINVAL" })) + ok(isUnsupportedDirectorySyncError({ code: "ENOTSUP" })) + ok(!isUnsupportedDirectorySyncError({ code: "EBADF" })) + ok(!isUnsupportedDirectorySyncError({ code: "EISDIR" })) + }) + + it("refuses symlinks while syncing a checkpoint tree", async () => { + await withDataDir(async (dataDir) => { + const outside = join(dirname(dataDir), "outside.txt") + writeFileSync(outside, "outside") + symlinkSync(outside, join(dataDir, "link")) + await rejects(syncTree(dataDir), /non-file checkpoint entry/) + await syncTree(dataDir, { allowSymlinks: true }) + ok(existsSync(outside)) + }) + }) +}) + +const throwsMessage = (run: () => unknown, expected: RegExp): void => { + try { + run() + throw new Error("expected function to throw") + } catch (error) { + match(error instanceof Error ? error.message : String(error), expected) + } +} diff --git a/apps/cli/test/merge-injection-probe.ts b/apps/cli/test/merge-injection-probe.ts new file mode 100644 index 000000000..35f841d20 --- /dev/null +++ b/apps/cli/test/merge-injection-probe.ts @@ -0,0 +1,130 @@ +// Adversarial merge-injection probe for the static-snapshot part-interval plan. +// +// Reproduces the exact cross-check corruption scenario: two parts for the same +// hour, with an OPTIMIZE TABLE ... FINAL injected between shard exports via the +// afterShardValidated hook. With SYSTEM STOP MERGES in effect, the OPTIMIZE +// must be blocked (or harmless), and the exported shards must contain the exact +// source row set with no duplicates or omissions. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/merge-injection-probe.ts + +import { rmSync, mkdirSync } from "node:fs" +import { join } from "node:path" +import { Chdb } from "../src/server/chdb" +import { exportSignalShards } from "../src/server/archives/export" +import { archiveSignal } from "../src/server/archives/signals" + +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, SpanId String, + ParentSpanId String, TraceState String, SpanName LowCardinality(String), + SpanKind LowCardinality(String), ServiceName LowCardinality(String), + StatusCode LowCardinality(String), StatusMessage String +) ENGINE = MergeTree ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)); +` + +const OUT = "/tmp/merge-injection-probe-out" +rmSync(OUT, { recursive: true, force: true }) +mkdirSync(OUT, { recursive: true }) + +const db = Chdb.open({ dataDir: "/tmp/merge-injection-probe-db", schemaSql: SCHEMA, bootstrapSchema: true }) +rmSync("/tmp/merge-injection-probe-db", { recursive: true, force: true }) +// Re-open after clearing (bootstrapSchema creates the schema on open). +const db2 = Chdb.open({ dataDir: "/tmp/merge-injection-probe-db", schemaSql: SCHEMA, bootstrapSchema: true }) + +// Insert two batches at the same UTC hour → two parts. +// Batch 1: IDs t4-t7. Batch 2: IDs t0-t3. (Out-of-order, like the cross-check.) +db2.exec( + "INSERT INTO traces SELECT 'local', toDateTime64('2026-06-29 12:00:00.000', 9, 'UTC'), 't'||toString(number+4), 's'||toString(number+4), '', '', 'probe', 'Server', 'probe', 'Ok', '' FROM numbers(4)", +) +db2.exec( + "INSERT INTO traces SELECT 'local', toDateTime64('2026-06-29 12:00:00.000', 9, 'UTC'), 't'||toString(number), 's'||toString(number), '', '', 'probe', 'Server', 'probe', 'Ok', '' FROM numbers(4)", +) + +// Verify two parts exist. +const partsResult = db2.query( + "SELECT count(DISTINCT _part) AS n FROM traces WHERE toDate(Timestamp,'UTC')='2026-06-29' AND toHour(Timestamp,'UTC')=12", + "JSONEachRow", +) +console.log("parts before export:", partsResult.trim()) + +// The source set of IDs (sorted) that MUST appear in the archive. +const sourceIds = db2 + .query( + "SELECT TraceId FROM traces WHERE toDate(Timestamp,'UTC')='2026-06-29' AND toHour(Timestamp,'UTC')=12 ORDER BY TraceId", + "JSONEachRow", + ) + .trim() + .split("\n") + .map((l) => JSON.parse(l).TraceId as string) + .sort() +console.log("source IDs:", sourceIds.join(",")) + +// Export with an OPTIMIZE injected between shards. +let optimizeFired = false +let optimizeError: string | null = null +const shardsDir = join(OUT, "shards") +mkdirSync(shardsDir, { recursive: true }) + +try { + const written = exportSignalShards(db2, archiveSignal("traces"), "2026-06-29", shardsDir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + afterShardValidated: (d, signal) => { + // Inject an OPTIMIZE TABLE FINAL between shards. + optimizeFired = true + try { + d.exec(`OPTIMIZE TABLE ${signal.name} FINAL`) + console.log(" OPTIMIZE fired (not blocked by STOP MERGES)") + } catch (e) { + optimizeError = e instanceof Error ? e.message : String(e) + console.log(" OPTIMIZE blocked by STOP MERGES:", optimizeError?.slice(0, 80)) + } + }, + }) + console.log("shards written:", written.length) + console.log("optimize fired:", optimizeFired, "blocked:", optimizeError !== null) + + // Read back all shard IDs via chDB file(). + const allIds: string[] = [] + for (const shard of written) { + const rows = db2.query( + `SELECT TraceId FROM file('${shard.path}', Parquet) ORDER BY TraceId`, + "JSONEachRow", + ) + for (const line of rows.trim().split("\n")) { + if (line.trim()) allIds.push(JSON.parse(line).TraceId) + } + } + allIds.sort() + console.log("archived IDs:", allIds.join(",")) + + // The corruption check: [5,6,7,8,5,6,7,8] under the old OFFSET approach. + const sourceStr = sourceIds.join(",") + const archivedStr = allIds.join(",") + if (sourceStr !== archivedStr) { + console.error(`CORRUPTION: source={${sourceStr}} archived={${archivedStr}}`) + process.exit(1) + } + if (allIds.length !== 8) { + console.error(`COUNT MISMATCH: expected 8, got ${allIds.length}`) + process.exit(1) + } + const distinct = new Set(allIds).size + if (distinct !== 8) { + console.error(`DUPLICATES: ${allIds.length} rows but only ${distinct} distinct`) + process.exit(1) + } + console.log("PASS: exact ID match [t0-t7], no duplicates, no omissions") +} catch (e) { + console.error("EXPORT FAILED:", e instanceof Error ? e.message : String(e)) + process.exit(1) +} finally { + db2.close() + db.close() + rmSync(OUT, { recursive: true, force: true }) + rmSync("/tmp/merge-injection-probe-db", { recursive: true, force: true }) +} diff --git a/apps/cli/test/native-archive-adversarial-probe.sh b/apps/cli/test/native-archive-adversarial-probe.sh new file mode 100755 index 000000000..c659640f5 --- /dev/null +++ b/apps/cli/test/native-archive-adversarial-probe.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Native adversarial probe runner for archive export. +# +# Invokes ONLY committed, repository-relative probe sources under apps/cli/test/ +# probes/. Every probe uses an owned mkdtemp dir and consistent exit semantics: +# nonzero when corruption is ACCEPTED (the bug is present), zero when corruption +# is correctly REJECTED. This runner reports the verdict per probe. +# +# See apps/cli/test/archive-adversarial-matrix.md for the invariant matrix. +# +# Usage: apps/cli/test/native-archive-adversarial-probe.sh [libchdb-path] + +set -uo pipefail + +BUNDLE="${1:?usage: $0 [libchdb-path]}" +LIBCHDB="${2:-$BUNDLE/libchdb.so}" +export MAPLE_LIBCHDB="$LIBCHDB" + +cd "$(dirname "$0")/../../.." # apps/cli/test/x.sh -> apps/cli/test -> apps/cli -> apps -> repo root +PROBE_DIR="apps/cli/test/probes" + +pass=0 +fail=0 +declare -a FAILURES=() + +# run_probe --data-dir --archive-dir \ +// --scratch-root --range --signal [--block-ms ] +// +// Exit semantics for the worker itself (NOT the harness verdict): +// - If the boundary is reached: writes the marker, blocks, then is SIGKILLed. +// - If createArchiveGeneration throws before the boundary: exits 2 with the +// error, so the harness can distinguish "boundary never reached" (a bug) +// from "crashed at the boundary" (expected). + +import { writeFileSync, existsSync, mkdirSync } from "node:fs" +import { join } from "node:path" +import { createArchiveGeneration, type ArchiveGenerationFaults } from "../../src/server/archives/generation" +import { resolveArchiveTuning } from "../../src/server/archives/config" + +interface Args { + boundary: string + markerDir: string + dataDir: string + archiveDir: string + scratchRoot: string + rangeDate: string + signal: string + blockMs: number +} + +const parseArgs = (argv: string[]): Args => { + const get = (name: string): string => { + const i = argv.indexOf(`--${name}`) + const v = i >= 0 ? argv[i + 1] : undefined + if (!v) throw new Error(`missing --${name}`) + return v + } + return { + boundary: get("boundary"), + markerDir: get("marker-dir"), + dataDir: get("data-dir"), + archiveDir: get("archive-dir"), + scratchRoot: get("scratch-root"), + rangeDate: get("range-date"), + signal: get("signal"), + blockMs: Number(argv[argv.indexOf("--block-ms") + 1] ?? "120000"), + } +} + +// Each boundary maps to the fault seam that fires AT that boundary (before or +// during the durable write that defines it). The seam writes the pause marker +// and then blocks. A seam that returns normally lets the operation continue; +// blocking models a crash mid-boundary. +const BLOCK = (ms: number): Promise => + new Promise((resolve) => { + // An "unref" timer would let the process exit; we WANT to block until + // killed, so keep the timer referenced. Cap at blockMs as a safety so a + // misconfigured harness doesn't hang forever if the parent never kills. + setTimeout(resolve, ms) + }) + +const writeMarker = (markerDir: string, boundary: string): void => { + mkdirSync(markerDir, { recursive: true }) + writeFileSync(join(markerDir, "paused"), `${boundary}\n${process.pid}\n${new Date().toISOString()}\n`) +} + +const buildFaults = (args: Args): ArchiveGenerationFaults => { + const { boundary, markerDir, blockMs } = args + // The seam fires at the boundary, writes the marker, then blocks. + const at = (): void => { + writeMarker(markerDir, boundary) + } + // After blocking is released (should only happen on misconfiguration), throw + // so the operation does not silently complete past the intended crash point. + const blockThenFail = async (): Promise => { + at() + await BLOCK(blockMs) + throw new Error(`crash-worker block expired at boundary ${boundary} without SIGKILL`) + } + const blockSyncThenFail = (): void => { + at() + const waitBuffer = new Int32Array(new SharedArrayBuffer(4)) + Atomics.wait(waitBuffer, 0, 0, blockMs) + throw new Error(`crash-worker block expired at boundary ${boundary} without SIGKILL`) + } + switch (boundary) { + // Pre-boundary seams: fire BEFORE the durable write, block (crash during). + case "before-intent-durable": + return { beforeIntentDurable: blockThenFail } + case "before-pin-acquired": + return { beforePinAcquired: blockThenFail } + case "before-scratch-allocated": + return { beforeScratchAllocated: blockThenFail } + case "before-restore": + // This is the authoritative pre-restore boundary: scratch exists and + // the journal records it, but restore has not begun. There is no honest + // callback from inside chDB's synchronous RESTORE command. + return { beforeScratchAllocated: blockThenFail } + case "after-restore": + return { afterScratchRestored: blockThenFail } + case "after-building-created": + return { afterBuildingCreated: blockThenFail } + case "after-first-shard": + return { afterFirstDurableShard: blockSyncThenFail } + case "after-validation-complete": + return { afterValidationComplete: blockThenFail } + case "before-manifest-durable": + return { beforeManifestDurable: blockThenFail } + case "after-manifest-durable": + return { afterManifestWritten: blockThenFail } + case "after-promoted": + return { afterGenerationRenamed: blockThenFail } + case "before-pointer-update": + return { beforeActivePointerUpdated: blockThenFail } + case "after-pointer": + return { afterGenerationPromoted: blockThenFail } + case "after-catalog": + return { afterCatalogAppended: blockThenFail } + case "pin-removed-before-journal": + return { afterPinRemovedBeforeJournal: blockThenFail } + case "after-pin-released": + return { afterPinReleased: blockThenFail } + case "before-scratch-removed": + return { beforeScratchRemoved: blockThenFail } + case "before-operation-archived": + return { beforeOperationArchived: blockThenFail } + default: + throw new Error(`unknown crash boundary: ${boundary}`) + } +} + +const main = async (): Promise => { + const args = parseArgs(process.argv.slice(2)) + const tuning = resolveArchiveTuning({ + archiveDir: args.archiveDir, + scratchRoot: args.scratchRoot, + dataDir: args.dataDir, + // The harness inserts three rows and one row per shard forces a genuine + // pause after shard 1 while later shards do not yet exist. + maxShardRows: 1, + maxShardBytes: 256 * 1024 * 1024, + rowGroupRows: 1, + }) + const faults = buildFaults(args) + try { + await createArchiveGeneration( + args.dataDir, + args.archiveDir, + args.signal, + args.rangeDate, + tuning, + "current", + faults, + ) + // If the operation completed despite the seam (e.g. boundary seam fired + // but block expired and the throw was swallowed), signal the harness that + // NO crash happened at the expected boundary. + if (!existsSync(join(args.markerDir, "paused"))) { + console.error(`crash-worker: completed without reaching boundary ${args.boundary}`) + process.exit(3) + } + console.error( + `crash-worker: boundary ${args.boundary} reached but operation completed (block expired?)`, + ) + process.exit(4) + } catch (error) { + const msg = error instanceof Error ? error.message : String(error) + // exit 2 = boundary never reached (the generation threw before it). + if (!existsSync(join(args.markerDir, "paused"))) { + console.error(`crash-worker: threw before boundary ${args.boundary}: ${msg}`) + process.exit(2) + } + // A throw after the marker means the block expired then threw — unexpected. + console.error(`crash-worker: threw after marker at ${args.boundary}: ${msg}`) + process.exit(5) + } +} + +void main().catch((error) => { + console.error(`crash-worker fatal: ${error instanceof Error ? error.message : String(error)}`) + process.exit(6) +}) diff --git a/apps/cli/test/probes/archive-gc-worker.ts b/apps/cli/test/probes/archive-gc-worker.ts new file mode 100644 index 000000000..a6453cc0d --- /dev/null +++ b/apps/cli/test/probes/archive-gc-worker.ts @@ -0,0 +1,117 @@ +// Crash-injection child worker for archive garbage collection (Gate 3b). +// +// Committed TEST SEAM, not production. The parent SIGKILL harness spawns this +// worker, which runs runArchiveGc with an onTargetCollected callback that, after +// the FIRST target is collected, writes a durable "paused" marker and blocks — +// modeling a real SIGKILL DURING tombstone removal while another target remains. +// SIGKILL does not run any finally, so the post-crash topology (one target +// gone, one remaining, journal mid-progress) is exactly what a real crash leaves. +// +// Usage: +// MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-gc-worker.ts \ +// --boundary --marker-dir --data-dir --archive-dir \ +// --scratch-root --keep [--block-ms ] + +import { writeFileSync, existsSync, mkdirSync } from "node:fs" +import { runArchiveGc } from "../../src/server/archives/gc" + +interface Args { + boundary: string + markerDir: string + dataDir: string + archiveDir: string + scratchRoot: string + keep: number + blockMs: number +} + +const parseArgs = (argv: string[]): Args => { + const get = (name: string): string => { + const i = argv.indexOf(`--${name}`) + const v = i >= 0 ? argv[i + 1] : undefined + if (!v) throw new Error(`missing --${name}`) + return v + } + return { + boundary: get("boundary"), + markerDir: get("marker-dir"), + dataDir: get("data-dir"), + archiveDir: get("archive-dir"), + scratchRoot: get("scratch-root"), + keep: Number(get("keep")), + blockMs: Number(argv[argv.indexOf("--block-ms") + 1] ?? "120000"), + } +} + +const writeMarker = (markerDir: string, boundary: string): void => { + mkdirSync(markerDir, { recursive: true }) + writeFileSync(join(markerDir, "paused"), `${boundary}\n${process.pid}\n${new Date().toISOString()}\n`) +} + +const join = (require("node:path") as typeof import("node:path")).join + +const BLOCK = (ms: number): Promise => + new Promise((resolve) => { + setTimeout(resolve, ms) + }) + +const main = async (): Promise => { + const args = parseArgs(process.argv.slice(2)) + let firstRenameSeamed = false + // Map the named boundary to the corresponding GC fault seam. Each seam, when + // it fires for the relevant boundary, writes the paused marker and blocks — + // modeling a real SIGKILL at that exact intra-boundary point. The seams are + // AWAITED, so the worker blocks deterministically at the boundary. + const blockAtBoundary = async (boundary: string): Promise => { + writeMarker(args.markerDir, boundary) + await BLOCK(args.blockMs) + throw new Error(`gc-worker block expired at ${boundary} without SIGKILL`) + } + const boundaryFaults = (() => { + switch (args.boundary) { + case "after-intent-durable": + return { afterIntentDurable: async () => blockAtBoundary("after-intent-durable") } + case "after-first-rename": + return { + afterFirstTargetRenamed: async () => { + if (!firstRenameSeamed) { + firstRenameSeamed = true + await blockAtBoundary("after-first-rename") + } + }, + } + case "nonfinal-progress": + // The authoritative boundary that exposes the premature-complete + // defect: fire AFTER a NONFINAL target's durable gc-collecting + // progress write (index < total-1), while another target remains. + return { + afterTargetProgress: async (index: number, total: number) => { + if (index < total - 1) await blockAtBoundary("nonfinal-progress") + }, + } + case "after-all-removals": + return { afterAllRemovals: async () => blockAtBoundary("after-all-removals") } + case "after-catalog": + return { afterCatalogRebuilt: async () => blockAtBoundary("after-catalog") } + default: + throw new Error(`unknown gc crash boundary: ${args.boundary}`) + } + })() + await runArchiveGc({ + dataDir: args.dataDir, + archiveDir: args.archiveDir, + scratchRoot: args.scratchRoot, + keep: args.keep, + dryRun: false, + faults: boundaryFaults, + }) + // If we reach here, the worker completed without being SIGKILLed at the seam. + console.error(`gc-worker: completed boundary ${args.boundary} without SIGKILL (block expired?)`) + process.exit(4) +} + +void main().catch((error) => { + console.error(`gc-worker fatal: ${error instanceof Error ? error.message : String(error)}`) + process.exit(6) +}) +void existsSync diff --git a/apps/cli/test/probes/archive-probe-byte-heterogeneous.ts b/apps/cli/test/probes/archive-probe-byte-heterogeneous.ts new file mode 100644 index 000000000..15dd45bd5 --- /dev/null +++ b/apps/cli/test/probes/archive-probe-byte-heterogeneous.ts @@ -0,0 +1,70 @@ +// RED-BASELINE probe (round 5): heterogeneous byte widths must refine, not +// abort. 300 narrow ordered rows followed by 10 wide (60 KiB high-entropy) rows +// under a 128 KiB byte bound: a sample-based plan using the first 256 rows +// underestimates and then aborts on the wide tail. Authoritative refinement must +// split the wide region until every shard meets both bounds. +// Contract: exit 0 (PASS) when every shard's actual uncompressed size <= the +// bound; exit nonzero (FAIL) when export aborts with "recalibrate"/"exceeds". +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-byte-heterogeneous.ts + +import { mkdirSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe, firstRow } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, SpanId String, Body String +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, SpanId);` + +const h = ArchiveProbe.create("byte-heterogeneous") +mkdirSync(h.outDir, { recursive: true }) +const shardsDir = join(h.outDir, "shards") +mkdirSync(shardsDir, { recursive: true }) + +const BYTE_BOUND = 128 * 1024 + +try { + const db = h.openDb(SCHEMA) + // 300 narrow rows first (so the 256-row probe samples only narrow data), + // then 10 wide high-entropy rows. + db.exec( + `INSERT INTO traces SELECT 'org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'n'||toString(number), 'sn'||toString(number), 'x' FROM numbers(300)`, + ) + db.exec( + `INSERT INTO traces SELECT 'org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'w'||toString(number), 'sw'||toString(number), randomString(60000) FROM numbers(10)`, + ) + + const written = exportSignalShards(db, archiveSignal("traces"), "2026-06-29", shardsDir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: BYTE_BOUND, + }) + // Every shard's actual uncompressed size must be <= the bound. + let worst = 0 + for (const s of written) { + const u = Number( + firstRow( + db.query( + `SELECT total_uncompressed_size u FROM file('${s.path}', ParquetMetadata)`, + "JSONEachRow", + ), + )?.u ?? 0, + ) + if (u > worst) worst = u + if (u > BYTE_BOUND) h.fail(`shard ${s.name} uncompressed ${u} > bound ${BYTE_BOUND}`) + } + h.ok( + `heterogeneous widths refined: ${written.length} shards, each <= ${BYTE_BOUND} bytes (worst ${worst})`, + ) +} catch (e) { + const msg = e instanceof Error ? e.message : String(e) + if (/exceeds|maxShardBytes|recalibrate/i.test(msg)) { + h.fail(`heterogeneous export aborted instead of refining: ${msg.slice(0, 160)}`) + } + h.fail(msg) +} diff --git a/apps/cli/test/probes/archive-probe-byte-single-row.ts b/apps/cli/test/probes/archive-probe-byte-single-row.ts new file mode 100644 index 000000000..426fab007 --- /dev/null +++ b/apps/cli/test/probes/archive-probe-byte-single-row.ts @@ -0,0 +1,52 @@ +// Probe: a single row that genuinely exceeds maxShardBytes must fail DISTINCTLY +// with a "single row exceeds maxShardBytes" message, not a generic +// "recalibrate". This is the only impassable case. Contract: exit 0 (PASS) when +// export fails with the distinct single-row message; exit nonzero (FAIL) when it +// aborts with a generic message or (worse) succeeds. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-byte-single-row.ts + +import { mkdirSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, SpanId String, Body String +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, SpanId);` + +const h = ArchiveProbe.create("byte-single-row") +mkdirSync(h.outDir, { recursive: true }) +const shardsDir = join(h.outDir, "shards") +mkdirSync(shardsDir, { recursive: true }) + +// One row whose uncompressed size (~2 MiB) far exceeds a 128 KiB bound. +const BYTE_BOUND = 128 * 1024 + +try { + const db = h.openDb(SCHEMA) + db.exec( + `INSERT INTO traces SELECT 'org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 't0', 's0', randomString(2097152)`, + ) + + try { + exportSignalShards(db, archiveSignal("traces"), "2026-06-29", shardsDir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: BYTE_BOUND, + }) + h.fail("single oversized row was archived (should have failed distinctly)") + } catch (e) { + const msg = e instanceof Error ? e.message : String(e) + if (/single row exceeds maxShardBytes/i.test(msg)) { + h.ok(`distinct single-row failure: ${msg.slice(0, 120)}`) + } + h.fail(`oversized single row failed with a non-distinct message: ${msg.slice(0, 160)}`) + } +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-byte-uniform.ts b/apps/cli/test/probes/archive-probe-byte-uniform.ts new file mode 100644 index 000000000..7fdd20f54 --- /dev/null +++ b/apps/cli/test/probes/archive-probe-byte-uniform.ts @@ -0,0 +1,64 @@ +// Probe: uniform wide high-entropy rows split by the byte bound, each shard <= +// the bound, exact source set. This is the round-4 case that already passed; +// relocated hermetic. Contract: exit 0 (PASS) when uniform wide rows split into +// multiple byte-bounded shards with the exact source set. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-byte-uniform.ts + +import { mkdirSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe, firstRow, readRows } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, SpanId String, Body String +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, SpanId);` + +const h = ArchiveProbe.create("byte-uniform") +mkdirSync(h.outDir, { recursive: true }) +const shardsDir = join(h.outDir, "shards") +mkdirSync(shardsDir, { recursive: true }) + +const BYTE_BOUND = 512 * 1024 + +try { + const db = h.openDb(SCHEMA) + db.exec( + `INSERT INTO traces SELECT 'org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 't'||toString(number), 's'||toString(number), randomString(60000) FROM numbers(30)`, + ) + + const written = exportSignalShards(db, archiveSignal("traces"), "2026-06-29", shardsDir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: BYTE_BOUND, + }) + if (written.length <= 1) + h.fail(`byte-aware planner produced only ${written.length} shard for a ~1.8 MiB hour at 512 KiB`) + for (const s of written) { + const u = Number( + firstRow( + db.query( + `SELECT total_uncompressed_size u FROM file('${s.path}', ParquetMetadata)`, + "JSONEachRow", + ), + )?.u ?? 0, + ) + if (u > BYTE_BOUND) h.fail(`shard ${s.name} uncompressed ${u} > ${BYTE_BOUND}`) + } + const archived = readRows(db.query(`SELECT TraceId FROM traces ORDER BY TraceId`, "JSONEachRow")) + .map((r) => String(r.TraceId)) + .sort() + const got: string[] = [] + for (const s of written) + for (const r of readRows(db.query(`SELECT TraceId FROM file('${s.path}', Parquet)`, "JSONEachRow"))) + got.push(String(r.TraceId)) + got.sort() + if (got.join(",") !== archived.join(",")) h.fail(`source != archived`) + h.ok(`uniform wide rows split into ${written.length} byte-bounded shards, exact set`) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-complex-alter.ts b/apps/cli/test/probes/archive-probe-complex-alter.ts new file mode 100644 index 000000000..7bbc4d941 --- /dev/null +++ b/apps/cli/test/probes/archive-probe-complex-alter.ts @@ -0,0 +1,57 @@ +// Probe: an altered complex value with identical count and time extrema must +// change the digest. The round-4 per-column sum is value-sensitive for a single +// column change (this passed in round 4); the round-5 multiset must preserve it. +// Contract: exit 0 (PASS) when the altered export's digest DIFFERS. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-complex-alter.ts + +import { mkdirSync, rmSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, + SpanAttributes Map(LowCardinality(String), String) +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, TraceId);` + +const h = ArchiveProbe.create("complex-alter") +mkdirSync(h.outDir, { recursive: true }) + +const exportOnce = (db: ReturnType, suffix: string): string => { + const dir = join(h.outDir, `shards-${suffix}`) + mkdirSync(dir, { recursive: true }) + const written = exportSignalShards(db, archiveSignal("traces"), "2026-06-29", dir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + }) + rmSync(dir, { recursive: true, force: true }) + if (written.length !== 1) h.fail(`expected 1 shard for ${suffix}, got ${written.length}`) + return written[0]!.complexDigest +} + +try { + const dbA = h.openDb(SCHEMA, "db-a") + dbA.exec(`INSERT INTO traces VALUES + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'tid1', map('http.method','GET','http.route','/')), + ('org1', toDateTime64('2026-06-29 12:01:00', 9, 'UTC'), 'tid2', map('http.method','POST'))`) + const origDigest = exportOnce(dbA, "orig") + dbA.close() + + const dbB = h.openDb(SCHEMA, "db-b") + dbB.exec(`INSERT INTO traces VALUES + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'tid1', map('http.method','GETTTTTTT','http.route','/')), + ('org1', toDateTime64('2026-06-29 12:01:00', 9, 'UTC'), 'tid2', map('http.method','POST'))`) + const altDigest = exportOnce(dbB, "alt") + dbB.close() + + if (origDigest === altDigest) h.fail(`altered complex value produced same digest (${origDigest})`) + h.ok(`altered complex value detected: ${origDigest} != ${altDigest}`) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-digest-column-swap.ts b/apps/cli/test/probes/archive-probe-digest-column-swap.ts new file mode 100644 index 000000000..88495a68c --- /dev/null +++ b/apps/cli/test/probes/archive-probe-digest-column-swap.ts @@ -0,0 +1,84 @@ +// RED-BASELINE probe (round 5): same-typed column SWAP must change the digest. +// +// This faithfully reproduces the reviewer's round-4 CRITICAL finding: the +// round-4 digest is `sum(toUInt64(cityHash64(c_i)))` across columns, which is +// COMMUTATIVE across columns. Swapping two same-typed Map columns +// (SpanAttributes <-> ResourceAttributes) — exchanging which VALUES sit under +// which column name — preserves the sum exactly while corrupting every row. +// Count and time extrema are also preserved, so no other check catches it. +// +// NOTE the distinction the prior (buggy) probe got wrong: this is a COLUMN swap +// (column A's values move under column B's name and vice versa), NOT a value +// swap within one column. A value swap IS detected by a per-column hash; a +// column swap is NOT, because the per-column hashes are summed independently. +// +// Contract: exit 0 (PASS) when the column-swapped export's digest DIFFERS from +// the original's; exit nonzero (FAIL) when equal (the commutative bug). +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-digest-column-swap.ts + +import { mkdirSync, rmSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +// Two Map columns of the SAME type — the precondition for the commutative +// collision. (logs/traces/metrics all carry several Map(LowCardinality(String), +// String) columns, so this is a production-realistic shape.) +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, SpanId String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String) +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, SpanId);` + +const h = ArchiveProbe.create("digest-column-swap") +mkdirSync(h.outDir, { recursive: true }) + +const exportOnce = (db: ReturnType, suffix: string): string => { + const dir = join(h.outDir, `shards-${suffix}`) + mkdirSync(dir, { recursive: true }) + const written = exportSignalShards(db, archiveSignal("traces"), "2026-06-29", dir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + }) + rmSync(dir, { recursive: true, force: true }) + if (written.length !== 1) h.fail(`expected 1 shard for ${suffix}, got ${written.length}`) + return written[0]!.complexDigest +} + +try { + // Original: value X under SpanAttributes, value Y under ResourceAttributes. + const dbA = h.openDb(SCHEMA, "db-a") + dbA.exec(`INSERT INTO traces VALUES + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'tid1', 's1', map('k','X'), map('k','Y')), + ('org1', toDateTime64('2026-06-29 12:01:00', 9, 'UTC'), 'tid2', 's2', map('k','X2'), map('k','Y2'))`) + const origDigest = exportOnce(dbA, "orig") + dbA.close() + + // Column swap: the SAME value SETS, but X and Y are EXCHANGED between the two + // same-typed Map columns. Span now holds Y, Resource now holds X. Count and + // time extrema are identical; the per-column value sets are identical; only + // the column<->value BINDING changed. A commutative per-column-sum digest + // (cityHash64(c_i) summed across i) cannot distinguish this from the original. + // This is the reviewer's exact CRITICAL finding. + const dbB = h.openDb(SCHEMA, "db-b") + dbB.exec(`INSERT INTO traces VALUES + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'tid1', 's1', map('k','Y'), map('k','X')), + ('org1', toDateTime64('2026-06-29 12:01:00', 9, 'UTC'), 'tid2', 's2', map('k','Y2'), map('k','X2'))`) + const swapDigest = exportOnce(dbB, "swap") + dbB.close() + + if (origDigest === swapDigest) { + h.fail( + `COLUMN-SWAP ACCEPTED: original digest == swapped digest (${origDigest}); the digest is commutative across same-typed columns and cannot detect a column<->value exchange`, + ) + } + h.ok(`column swap detected: ${origDigest} != ${swapDigest}`) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-digest-dup-drop.ts b/apps/cli/test/probes/archive-probe-digest-dup-drop.ts new file mode 100644 index 000000000..ba71680be --- /dev/null +++ b/apps/cli/test/probes/archive-probe-digest-dup-drop.ts @@ -0,0 +1,65 @@ +// RED-BASELINE probe (round 5): duplicate-one/drop-another (equal-count pair) +// must be detected. Duplicate row A and drop row B where the total count is +// unchanged and the time extrema are unchanged. A multiset digest that +// preserves duplicates detects this; a count-only or sum digest does not. +// Contract: exit 0 (PASS) when the dup/drop export's digest DIFFERS. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-digest-dup-drop.ts + +import { mkdirSync, rmSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, SpanId String, + SpanAttributes Map(LowCardinality(String), String) +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, SpanId);` + +const h = ArchiveProbe.create("digest-dup-drop") +mkdirSync(h.outDir, { recursive: true }) + +const exportOnce = (db: ReturnType, suffix: string): string => { + const dir = join(h.outDir, `shards-${suffix}`) + mkdirSync(dir, { recursive: true }) + const written = exportSignalShards(db, archiveSignal("traces"), "2026-06-29", dir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + }) + rmSync(dir, { recursive: true, force: true }) + if (written.length !== 1) h.fail(`expected 1 shard for ${suffix}, got ${written.length}`) + return written[0]!.complexDigest +} + +try { + // Original: two distinct rows. + const dbA = h.openDb(SCHEMA, "db-a") + dbA.exec(`INSERT INTO traces VALUES + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'tid1', 's1', map('k','AAA')), + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'tid2', 's2', map('k','BBB'))`) + const origDigest = exportOnce(dbA, "orig") + dbA.close() + + // Dup/drop: duplicate tid1 (AAA) twice and drop tid2 (BBB). Count stays 2, + // time extrema stay identical. A duplicate-preserving multiset digest differs. + const dbB = h.openDb(SCHEMA, "db-b") + dbB.exec(`INSERT INTO traces VALUES + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'tid1', 's1', map('k','AAA')), + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'tid1', 's1', map('k','AAA'))`) + const dupDigest = exportOnce(dbB, "dup") + dbB.close() + + if (origDigest === dupDigest) { + h.fail( + `DUP/DROP ACCEPTED: original digest == dup/drop digest (${origDigest}); the digest cannot detect a duplicate-one/drop-another equal-count transform`, + ) + } + h.ok(`dup/drop detected: ${origDigest} != ${dupDigest}`) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-digest-row-swap.ts b/apps/cli/test/probes/archive-probe-digest-row-swap.ts new file mode 100644 index 000000000..8011f0497 --- /dev/null +++ b/apps/cli/test/probes/archive-probe-digest-row-swap.ts @@ -0,0 +1,65 @@ +// RED-BASELINE probe (round 5): cross-row value reassociation must be detected. +// Move one row's map value to another row (and vice versa), preserving the row +// count and the time extrema exactly. A row-commutative digest (sum of per-row +// hashes) cannot detect this. Contract: exit 0 (PASS) when the reassociated +// export's digest DIFFERS; exit nonzero (FAIL) when equal (the bug). +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-digest-row-swap.ts + +import { mkdirSync, rmSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, SpanId String, + SpanAttributes Map(LowCardinality(String), String) +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, SpanId);` + +const h = ArchiveProbe.create("digest-row-swap") +mkdirSync(h.outDir, { recursive: true }) + +const exportOnce = (db: ReturnType, suffix: string): string => { + const dir = join(h.outDir, `shards-${suffix}`) + mkdirSync(dir, { recursive: true }) + const written = exportSignalShards(db, archiveSignal("traces"), "2026-06-29", dir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + }) + rmSync(dir, { recursive: true, force: true }) + if (written.length !== 1) h.fail(`expected 1 shard for ${suffix}, got ${written.length}`) + return written[0]!.complexDigest +} + +try { + // Original: two rows with distinct map values, same hour. + const dbA = h.openDb(SCHEMA, "db-a") + dbA.exec(`INSERT INTO traces VALUES + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'tid1', 's1', map('k','AAA')), + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'tid2', 's2', map('k','BBB'))`) + const origDigest = exportOnce(dbA, "orig") + dbA.close() + + // Reassociated: swap the map VALUES between the two rows. Count and time + // extrema are identical; only the row<->value association changed. + const dbB = h.openDb(SCHEMA, "db-b") + dbB.exec(`INSERT INTO traces VALUES + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'tid1', 's1', map('k','BBB')), + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'tid2', 's2', map('k','AAA'))`) + const swapDigest = exportOnce(dbB, "swap") + dbB.close() + + if (origDigest === swapDigest) { + h.fail( + `ROW-VALUE-SWAP ACCEPTED: original digest == swapped digest (${origDigest}); the digest cannot detect cross-row value reassociation`, + ) + } + h.ok(`cross-row value reassociation detected: ${origDigest} != ${swapDigest}`) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-duckdb-oracle.ts b/apps/cli/test/probes/archive-probe-duckdb-oracle.ts new file mode 100644 index 000000000..d661e293c --- /dev/null +++ b/apps/cli/test/probes/archive-probe-duckdb-oracle.ts @@ -0,0 +1,138 @@ +// INDEPENDENT-ORACLE probe (round 5, Step 8): compare canonical source truth +// against the SAME data read back from the archived Parquet via DuckDB — an +// independent reader, NOT the export code's own digest. This closes the +// "tests confirm what you intended" failure: it asks DuckDB whether the archived +// Parquet holds the exact source values (count, NULLs, arrays, nanosecond +// timestamps), rather than checking the export code against itself. +// +// Contract: exit 0 (PASS) when every oracle matches; exit nonzero otherwise. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-duckdb-oracle.ts +// (duckdb must be on PATH) + +import { mkdirSync } from "node:fs" +import { join } from "node:path" +import { execSync } from "node:child_process" +import { ArchiveProbe, readRows } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +const SCHEMA = `CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, + SpanAttributes Map(LowCardinality(String), String), + EventsName Array(LowCardinality(String)), + EventsTimestamp Array(DateTime64(9)) +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, TraceId); +CREATE TABLE IF NOT EXISTS default.metrics_histogram ( + OrgId LowCardinality(String), TimeUnix DateTime64(9), MetricName LowCardinality(String), + Count UInt64, BucketCounts Array(UInt64), Min Nullable(Float64), Max Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (OrgId, MetricName);` + +const h = ArchiveProbe.create("duckdb-oracle") +mkdirSync(h.outDir, { recursive: true }) + +/** Run a scalar/CSV DuckDB query over a list of parquet paths. */ +const duckScalar = (paths: string[], expr: string): string => { + const list = paths.map((p) => `'${p}'`).join(",") + return execSync( + `duckdb -csv -noheader -c "SELECT ${expr} FROM read_parquet([${list}], union_by_name=true)"`, + { + encoding: "utf8", + }, + ).trim() +} + +try { + const db = h.openDb(SCHEMA) + db.exec(`INSERT INTO traces VALUES + ('org1', toDateTime64('2026-06-29 12:00:00.123456789', 9, 'UTC'), 't1', + map('http.method','GET','http.route','/'), array('start','finish'), + array(toDateTime64('2026-06-29 12:00:00.1', 9, 'UTC'), toDateTime64('2026-06-29 12:00:00.2', 9, 'UTC'))), + ('org1', toDateTime64('2026-06-29 12:01:00', 9, 'UTC'), 't2', + map('http.method','POST'), array('end'), array(toDateTime64('2026-06-29 12:01:00', 9, 'UTC')));`) + db.exec(`INSERT INTO metrics_histogram VALUES + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'lat', 3, [1,1,1], 1.0, 5.0), + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'lat', 2, [1,1], NULL, NULL);`) + + const tDir = join(h.outDir, "traces") + mkdirSync(tDir, { recursive: true }) + const hDir = join(h.outDir, "hist") + mkdirSync(hDir, { recursive: true }) + const tShards = exportSignalShards(db, archiveSignal("traces"), "2026-06-29", tDir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + }) + const hShards = exportSignalShards(db, archiveSignal("metrics_histogram"), "2026-06-29", hDir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + }) + + const tPaths = tShards.map((s) => s.path) + const hPaths = hShards.map((s) => s.path) + + // ORACLE 1 — traces row count via DuckDB equals source. + const srcTraces = Number(readRows(db.query(`SELECT count() c FROM traces`, "JSONEachRow"))[0]!.c) + const duckTraces = Number(duckScalar(tPaths, "count()")) + if (duckTraces !== srcTraces) h.fail(`DuckDB traces count ${duckTraces} != source ${srcTraces}`) + + // ORACLE 2 — traces nanosecond timestamp fidelity via DuckDB. The high- + // precision 12:00:00.123456789 row must round-trip to the same epoch + // MICROSECONDS — a timezone-stable cross-tool comparison (DuckDB renders the + // value in the host timezone, but epoch_us is invariant). + const srcNano = BigInt( + String( + readRows( + db.query( + `SELECT toString(toUnixTimestamp64Nano(toDateTime64(Timestamp, 9))) n FROM traces WHERE TraceId='t1'`, + "JSONEachRow", + ), + )[0]!.n, + ), + ) + const srcMicro = srcNano / 1000n + // epoch_us on the RAW TIMESTAMPTZ column (no CAST — a CAST strips the tz and + // makes epoch_us host-tz-dependent). DuckDB reads DateTime64(9) as TIMESTAMP + // WITH TIME ZONE, so epoch_us on the raw column is UTC-stable. + const duckMicro = BigInt( + execSync( + `duckdb -csv -noheader -c "SELECT epoch_us(Timestamp) FROM read_parquet([${tPaths.map((p) => `'${p}'`).join(",")}], union_by_name=true) WHERE TraceId='t1'"`, + { encoding: "utf8" }, + ).trim(), + ) + if (duckMicro !== srcMicro) { + h.fail(`DuckDB timestamp micros ${duckMicro} != source ${srcMicro} for the high-precision t1 row`) + } + + // ORACLE 3 — histogram NULL fidelity: DuckDB sees the NULL Min/Max row. + const srcNull = Number( + readRows(db.query(`SELECT count() c FROM metrics_histogram WHERE isNull(Min)`, "JSONEachRow"))[0]!.c, + ) + const duckNull = Number(duckScalar(hPaths, `count() FILTER (WHERE Min IS NULL)`)) + if (duckNull !== srcNull) h.fail(`DuckDB NULL Min count ${duckNull} != source ${srcNull}`) + + // ORACLE 4 — histogram array fidelity: BucketCounts [1,1,1] preserved. + const srcBuckets = String( + readRows(db.query(`SELECT BucketCounts FROM metrics_histogram WHERE Count=3`, "JSONEachRow"))[0]! + .BucketCounts, + ) + const duckBuckets = execSync( + `duckdb -csv -noheader -c "SELECT BucketCounts FROM read_parquet([${hPaths.map((p) => `'${p}'`).join(",")}], union_by_name=true) WHERE Count=3"`, + { encoding: "utf8" }, + ).trim() + // DuckDB renders arrays as [1,1,1] or {1,1,1}; both contain three 1s. + const srcOnes = (srcBuckets.match(/1/g) || []).length + const duckOnes = (duckBuckets.match(/1/g) || []).length + if (srcOnes !== duckOnes) + h.fail(`DuckDB BucketCounts ones ${duckOnes} != source ${srcOnes} (duck=${duckBuckets})`) + + h.ok( + `DuckDB oracle: traces count+nanos (${duckTraces}), histogram NULL Min (${duckNull}) and BucketCounts (${srcOnes} ones) all match source`, + ) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-merge-freeze-leak.ts b/apps/cli/test/probes/archive-probe-merge-freeze-leak.ts new file mode 100644 index 000000000..1e74f0ef5 --- /dev/null +++ b/apps/cli/test/probes/archive-probe-merge-freeze-leak.ts @@ -0,0 +1,67 @@ +// Adversarial probe: merge-freeze leak on mid-export failure. A failure after +// SYSTEM STOP MERGES must still restart merges. Contract: exit 0 (PASS) when +// merges are confirmed restarted (a later OPTIMIZE is NOT rejected with code +// 236) after a forced mid-export failure. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-merge-freeze-leak.ts + +import { mkdirSync, writeFileSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, SpanId String +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, SpanId);` + +const h = ArchiveProbe.create("merge-freeze-leak") +mkdirSync(h.outDir, { recursive: true }) +const shardsDir = join(h.outDir, "shards") +mkdirSync(shardsDir, { recursive: true }) + +try { + const db = h.openDb(SCHEMA) + db.exec( + `INSERT INTO traces SELECT 'org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 't'||toString(number), 's'||toString(number) FROM numbers(4)`, + ) + db.exec( + `INSERT INTO traces SELECT 'org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 't'||toString(number+4), 's'||toString(number+4) FROM numbers(4)`, + ) + + // Pre-create the first shard path the planner will target, forcing a mid- + // export "already exists" throw AFTER STOP MERGES has fired. + writeFileSync(join(shardsDir, "12-0000.parquet"), "poison") + + let threw = false + try { + exportSignalShards(db, archiveSignal("traces"), "2026-06-29", shardsDir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 2, + maxShardBytes: 256 * 1024 * 1024, + }) + } catch { + threw = true + } + if (!threw) h.fail("export did not throw on pre-existing shard") + + // Critical assertion: merges must have been RESTARTED. An OPTIMIZE now must + // NOT be rejected with code 236. + let optimizeErr = "" + try { + db.exec( + `INSERT INTO traces VALUES ('org1', toDateTime64('2026-06-29 13:00:00', 9, 'UTC'), 'x1', 'x1')`, + ) + db.exec(`OPTIMIZE TABLE traces FINAL`) + } catch (e) { + optimizeErr = e instanceof Error ? e.message : String(e) + } + if (optimizeErr.includes("236")) + h.fail(`merges remained stopped after failure: ${optimizeErr.slice(0, 120)}`) + h.ok(`merges restarted after mid-export failure (optimize error: "${optimizeErr.slice(0, 60)}")`) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-merge-injection.ts b/apps/cli/test/probes/archive-probe-merge-injection.ts new file mode 100644 index 000000000..85bb12b63 --- /dev/null +++ b/apps/cli/test/probes/archive-probe-merge-injection.ts @@ -0,0 +1,70 @@ +// Adversarial probe: an OPTIMIZE TABLE FINAL injected between shard exports is +// blocked by SYSTEM STOP MERGES, and the exported shards contain the exact +// source set. Contract: exit 0 (PASS) when OPTIMIZE is blocked (code 236) every +// time and the archived IDs exactly equal the source IDs. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-merge-injection.ts + +import { mkdirSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe, readRows } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, SpanId String +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, SpanId);` + +const h = ArchiveProbe.create("merge-injection") +mkdirSync(h.outDir, { recursive: true }) +const shardsDir = join(h.outDir, "shards") +mkdirSync(shardsDir, { recursive: true }) + +try { + const db = h.openDb(SCHEMA) + db.exec( + `INSERT INTO traces SELECT 'org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 't'||toString(number+4), 's'||toString(number+4) FROM numbers(4)`, + ) + db.exec( + `INSERT INTO traces SELECT 'org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 't'||toString(number), 's'||toString(number) FROM numbers(4)`, + ) + + const sourceSet = readRows( + db.query( + `SELECT TraceId FROM traces WHERE toHour(Timestamp,'UTC')=12 ORDER BY TraceId`, + "JSONEachRow", + ), + ) + .map((r) => String(r.TraceId)) + .sort() + + let blocked = 0 + let leaked = 0 + const written = exportSignalShards(db, archiveSignal("traces"), "2026-06-29", shardsDir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 2, + maxShardBytes: 256 * 1024 * 1024, + afterShardValidated: (d, signal) => { + try { + d.exec(`OPTIMIZE TABLE ${signal.name} FINAL`) + leaked++ + } catch { + blocked++ + } + }, + }) + if (leaked > 0) h.fail(`OPTIMIZE was NOT blocked ${leaked} time(s)`) + const archived: string[] = [] + for (const s of written) { + for (const r of readRows(db.query(`SELECT TraceId FROM file('${s.path}', Parquet)`, "JSONEachRow"))) + archived.push(String(r.TraceId)) + } + archived.sort() + if (archived.join(",") !== sourceSet.join(",")) h.fail(`source [${sourceSet}] != archived [${archived}]`) + h.ok(`injected OPTIMIZE blocked ${blocked}x; exact set archived [${archived}] (${written.length} shards)`) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-mixed-hour.ts b/apps/cli/test/probes/archive-probe-mixed-hour.ts new file mode 100644 index 000000000..888be9d4e --- /dev/null +++ b/apps/cli/test/probes/archive-probe-mixed-hour.ts @@ -0,0 +1,81 @@ +// Adversarial probe: mixed-hour non-contiguous matching offsets must archive +// exactly. The round-3 part planner assumed filtered offsets were contiguous and +// selected cross-hour rows. Contract: exit 0 (PASS) when the hour-12 rows +// {tid0,tid9} (offsets 0 and 9, with hour-13 at offsets 1..8) archive exactly +// with no cross-hour bleed and the full day's source set is archived once. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-mixed-hour.ts + +import { mkdirSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe, firstRow, readRows } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, SpanId String +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, SpanId);` + +const h = ArchiveProbe.create("mixed-hour") +mkdirSync(h.outDir, { recursive: true }) +const shardsDir = join(h.outDir, "shards") +mkdirSync(shardsDir, { recursive: true }) + +try { + const db = h.openDb(SCHEMA) + // ORDER BY (OrgId, SpanId): hour-12 occupies non-contiguous offsets 0 and 9, + // hour-13 fills 1..8. + db.exec(`INSERT INTO traces SELECT 'org1', + multiIf(toUInt64(SpanId) IN (0,9), + toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), + toDateTime64('2026-06-29 13:00:00', 9, 'UTC')) AS Timestamp, + 'tid' || SpanId, SpanId + FROM (SELECT toString(number) AS SpanId FROM numbers(10))`) + + const written = exportSignalShards(db, archiveSignal("traces"), "2026-06-29", shardsDir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + }) + + // Every source row across the whole day must be archived exactly once, and + // each shard must contain only its own hour (no cross-hour bleed). + const sourceSet = readRows( + db.query( + `SELECT TraceId FROM traces WHERE toDate(Timestamp,'UTC')='2026-06-29' ORDER BY TraceId`, + "JSONEachRow", + ), + ) + .map((r) => String(r.TraceId)) + .sort() + const archived: Array<{ id: string; hour: number }> = [] + for (const s of written) { + const shardHour = Number(s.name.slice(0, 2)) + for (const r of readRows( + db.query( + `SELECT TraceId, toHour(Timestamp,'UTC') h FROM file('${s.path}', Parquet)`, + "JSONEachRow", + ), + )) { + if (Number(r.h) !== shardHour) + h.fail(`shard ${s.name} (hour ${shardHour}) contains hour-${r.h} row ${r.TraceId}`) + archived.push({ id: String(r.TraceId), hour: Number(r.h) }) + } + } + const archivedSet = archived.map((a) => a.id).sort() + if (archivedSet.join(",") !== sourceSet.join(",")) + h.fail(`source [${sourceSet}] != archived [${archivedSet}]`) + if (new Set(archivedSet).size !== archivedSet.length) h.fail(`duplicate rows in archive`) + + const hour12 = written.find((s) => s.name.startsWith("12-")) + if (!hour12) h.fail("no hour-12 shard produced") + void firstRow + h.ok( + `mixed-hour non-contiguous offsets archived exactly; hour-12 in ${hour12!.name}, ${written.length} shards`, + ) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-multipart.ts b/apps/cli/test/probes/archive-probe-multipart.ts new file mode 100644 index 000000000..7559d407d --- /dev/null +++ b/apps/cli/test/probes/archive-probe-multipart.ts @@ -0,0 +1,60 @@ +// Adversarial probe: multiple parts in one hour (out-of-order inserts) archive +// the exact source set with no duplicates or omissions. Contract: exit 0 (PASS) +// when 8 rows across 2 parts archive as the exact source set. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-multipart.ts + +import { mkdirSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe, readRows } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, SpanId String +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, SpanId);` + +const h = ArchiveProbe.create("multipart") +mkdirSync(h.outDir, { recursive: true }) +const shardsDir = join(h.outDir, "shards") +mkdirSync(shardsDir, { recursive: true }) + +try { + const db = h.openDb(SCHEMA) + db.exec( + `INSERT INTO traces SELECT 'org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 't'||toString(number+4), 's'||toString(number+4) FROM numbers(4)`, + ) + db.exec( + `INSERT INTO traces SELECT 'org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 't'||toString(number), 's'||toString(number) FROM numbers(4)`, + ) + + const sourceSet = readRows( + db.query( + `SELECT TraceId FROM traces WHERE toHour(Timestamp,'UTC')=12 ORDER BY TraceId`, + "JSONEachRow", + ), + ) + .map((r) => String(r.TraceId)) + .sort() + + const written = exportSignalShards(db, archiveSignal("traces"), "2026-06-29", shardsDir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + }) + const archived: string[] = [] + for (const s of written) { + for (const r of readRows(db.query(`SELECT TraceId FROM file('${s.path}', Parquet)`, "JSONEachRow"))) + archived.push(String(r.TraceId)) + } + archived.sort() + if (archived.join(",") !== sourceSet.join(",")) h.fail(`source [${sourceSet}] != archived [${archived}]`) + if (archived.length !== 8) h.fail(`expected 8 rows got ${archived.length}`) + if (new Set(archived).size !== 8) h.fail(`duplicates among ${archived}`) + h.ok(`multi-part hour archived exact set [${archived}] (${written.length} shards)`) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-null-digest.ts b/apps/cli/test/probes/archive-probe-null-digest.ts new file mode 100644 index 000000000..4945c3e88 --- /dev/null +++ b/apps/cli/test/probes/archive-probe-null-digest.ts @@ -0,0 +1,44 @@ +// Probe: a NULL in any column (histogram Min/Max) must NOT collapse the digest +// to empty/NULL. Round 4 handled this with per-column sentinels; round 5 must +// preserve it. Contract: exit 0 (PASS) when the export succeeds with a non-empty +// digest for a row containing NULL columns. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-null-digest.ts + +import { mkdirSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +const SCHEMA = ` +CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.metrics_histogram ( + OrgId LowCardinality(String), TimeUnix DateTime64(9), MetricName LowCardinality(String), + Count UInt64, Sum Float64, BucketCounts Array(UInt64), ExplicitBounds Array(Float64), + Flags UInt32, Min Nullable(Float64), Max Nullable(Float64), AggregationTemporality Int32 +) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (OrgId, MetricName);` + +const h = ArchiveProbe.create("null-digest") +mkdirSync(h.outDir, { recursive: true }) +const shardsDir = join(h.outDir, "shards") +mkdirSync(shardsDir, { recursive: true }) + +try { + const db = h.openDb(SCHEMA) + // Both NULL Min/Max (the round-4 collapse case). + db.exec(`INSERT INTO metrics_histogram (OrgId, TimeUnix, MetricName, Count, Sum, BucketCounts, ExplicitBounds, Flags, Min, Max, AggregationTemporality) VALUES + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'lat', 1, 1.0, [1], [0.0], 0, NULL, NULL, 0)`) + + const written = exportSignalShards(db, archiveSignal("metrics_histogram"), "2026-06-29", shardsDir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + }) + const digest = written[0]!.complexDigest + if (!digest || digest.length === 0) h.fail(`NULL columns collapsed digest to empty: '${digest}'`) + h.ok(`NULL columns did not collapse digest: ${digest}`) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-null-flag-binding.ts b/apps/cli/test/probes/archive-probe-null-flag-binding.ts new file mode 100644 index 000000000..f8989fd84 --- /dev/null +++ b/apps/cli/test/probes/archive-probe-null-flag-binding.ts @@ -0,0 +1,68 @@ +// RED→GREEN probe (round-5 repair): NULL must NOT collide with a real string +// value. The round-5-repair digest used a sentinel string for NULL; a real +// Nullable(String) value equal to that sentinel produced the same digest as an +// actual NULL (verified collision). The documented invariant is an EXPLICIT +// isNull flag, so NULL-ness is never conflated with a value. +// +// Contract: exit 0 (PASS) when a NULL and the sentinel string produce DIFFERENT +// digests; exit nonzero (FAIL) when they collide. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-null-flag-binding.ts + +import { mkdirSync, rmSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +// A traces-shaped table with an added Nullable(String) column to expose the +// NULL-vs-sentinel-string collision generically. +const SCHEMA = `CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.traces ( + OrgId LowCardinality(String), Timestamp DateTime64(9), TraceId String, SpanId String, + Note Nullable(String) +) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (OrgId, SpanId);` + +const h = ArchiveProbe.create("null-flag-binding") +mkdirSync(h.outDir, { recursive: true }) + +const exportOnce = (db: ReturnType, suffix: string, noteExpr: string): string => { + const dir = join(h.outDir, `shards-${suffix}`) + mkdirSync(dir, { recursive: true }) + db.exec( + `INSERT INTO traces (OrgId, Timestamp, TraceId, SpanId, Note) ` + + `VALUES ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 't1', 's1', ${noteExpr})`, + ) + const written = exportSignalShards(db, archiveSignal("traces"), "2026-06-29", dir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + }) + rmSync(dir, { recursive: true, force: true }) + if (written.length !== 1) h.fail(`expected 1 shard for ${suffix}, got ${written.length}`) + return written[0]!.complexDigest +} + +try { + // NULL Note. + const dbA = h.openDb(SCHEMA, "db-a") + const digestNull = exportOnce(dbA, "null", "NULL") + dbA.close() + + // A real string value that, under a sentinel-only digest, would collide with + // NULL. Use a value that a sentinel might be (and that '\x00NULL' represents). + const dbB = h.openDb(SCHEMA, "db-b") + const digestString = exportOnce(dbB, "string", "'\\x00NULL'") + dbB.close() + + if (digestNull === digestString) { + h.fail( + `NULL collided with a real Nullable(String) value: digest ${digestNull}; ` + + `the digest must bind an explicit isNull flag, not a sentinel string`, + ) + } + h.ok(`explicit NULL flag distinguishes NULL from a sentinel string: ${digestNull} != ${digestString}`) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-null-value-sensitivity.ts b/apps/cli/test/probes/archive-probe-null-value-sensitivity.ts new file mode 100644 index 000000000..70b586d02 --- /dev/null +++ b/apps/cli/test/probes/archive-probe-null-value-sensitivity.ts @@ -0,0 +1,71 @@ +// RED→GREEN probe (round 5): a row containing NULL nullable columns must STILL +// be value-sensitive in its NON-NULL columns. This is the reviewer's round-5 +// finding: round-5's `isNull(c), c` passed a bare nullable value into +// cityHash64, and chDB returns NULL if ANY argument is NULL — collapsing the +// ENTIRE per-row hash. Two metrics_histogram datasets with NULL Min/Max but +// different Count/Sum/BucketCounts produced the identical digest. +// +// Contract: exit 0 (PASS) when the two datasets produce DIFFERENT digests; +// exit nonzero (FAIL) when they are equal (the bare-NULL collapse). +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-null-value-sensitivity.ts + +import { mkdirSync, rmSync } from "node:fs" +import { join } from "node:path" +import { ArchiveProbe } from "../archive-probe-helpers" +import { archiveSignal } from "../../src/server/archives/signals" +import { exportSignalShards } from "../../src/server/archives/export" + +// metrics_histogram carries Nullable(Float64) Min/Max, NULL when unset — the +// exact production shape that defeated the round-5 digest. +const SCHEMA = `CREATE DATABASE IF NOT EXISTS default; +CREATE TABLE IF NOT EXISTS default.metrics_histogram ( + OrgId LowCardinality(String), TimeUnix DateTime64(9), MetricName LowCardinality(String), + Count UInt64, Sum Float64, BucketCounts Array(UInt64), ExplicitBounds Array(Float64), + Flags UInt32, Min Nullable(Float64), Max Nullable(Float64), AggregationTemporality Int32 +) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (OrgId, MetricName);` + +const h = ArchiveProbe.create("null-value-sensitivity") +mkdirSync(h.outDir, { recursive: true }) + +const exportOnce = (db: ReturnType, suffix: string): string => { + const dir = join(h.outDir, `shards-${suffix}`) + mkdirSync(dir, { recursive: true }) + const written = exportSignalShards(db, archiveSignal("metrics_histogram"), "2026-06-29", dir, { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + }) + rmSync(dir, { recursive: true, force: true }) + if (written.length !== 1) h.fail(`expected 1 shard for ${suffix}, got ${written.length}`) + return written[0]!.complexDigest +} + +try { + // Dataset A: NULL Min/Max, Count=3, Sum=1.0, buckets [1,1,1]. + const dbA = h.openDb(SCHEMA, "db-a") + dbA.exec(`INSERT INTO metrics_histogram (OrgId, TimeUnix, MetricName, Count, Sum, BucketCounts, ExplicitBounds, Flags, Min, Max, AggregationTemporality) VALUES + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'lat', 3, 1.0, [1,1,1], [0.0,1.0,5.0], 0, NULL, NULL, 0)`) + const digestA = exportOnce(dbA, "A") + dbA.close() + + // Dataset B: SAME schema, SAME row count (1), SAME event-time extrema, NULL + // Min/Max — but DIFFERENT non-null values (Count=99, Sum=42.0, buckets [9,9,9]). + // A value-sensitive digest must distinguish this from A. + const dbB = h.openDb(SCHEMA, "db-b") + dbB.exec(`INSERT INTO metrics_histogram (OrgId, TimeUnix, MetricName, Count, Sum, BucketCounts, ExplicitBounds, Flags, Min, Max, AggregationTemporality) VALUES + ('org1', toDateTime64('2026-06-29 12:00:00', 9, 'UTC'), 'lat', 99, 42.0, [9,9,9], [9.0,10.0,50.0], 0, NULL, NULL, 0)`) + const digestB = exportOnce(dbB, "B") + dbB.close() + + if (digestA === digestB) { + h.fail( + `NULL-bearing rows with different non-null values produced identical digest ${digestA}; ` + + `the digest passes a bare nullable value into cityHash64 and collapses the row hash`, + ) + } + h.ok(`NULL-bearing rows remain value-sensitive: digest A=${digestA} != B=${digestB}`) +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-schema-substitution.ts b/apps/cli/test/probes/archive-probe-schema-substitution.ts new file mode 100644 index 000000000..983deb144 --- /dev/null +++ b/apps/cli/test/probes/archive-probe-schema-substitution.ts @@ -0,0 +1,60 @@ +// Adversarial probe: schema substitution rejection. The reviewer's round-3 +// attack: source Array(UInt64) vs injected Array(String) Parquet reopened +// schema. compareSchema must reject it. Also confirms a valid round-trip schema +// is accepted. Contract: exit 0 (PASS) when the substitution is rejected AND the +// valid schema is accepted; exit nonzero if either check is wrong. +// +// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-schema-substitution.ts + +import { ArchiveProbe } from "../archive-probe-helpers" +import { compareSchema } from "../../src/server/archives/export" + +const h = ArchiveProbe.create("schema-substitution") + +const source = [ + { name: "OrgId", type: "LowCardinality(String)" }, + { name: "TraceId", type: "String" }, + { name: "BucketCounts", type: "Array(UInt64)" }, + { name: "Bounds", type: "Array(Float64)" }, + { name: "Attributes", type: "Map(LowCardinality(String), String)" }, + { name: "Timestamp", type: "DateTime64(9)" }, + { name: "Min", type: "Nullable(Float64)" }, +] + +const parquetFrom = (overrides: Record) => + source.map((c) => ({ name: c.name, type: overrides[c.name] ?? c.type })) + +try { + // 1. Valid round-trip must be accepted. + const valid: Record = { + OrgId: "String", + BucketCounts: "Array(UInt64)", + Bounds: "Array(Float64)", + Attributes: "Map(String, String)", + Timestamp: "DateTime64(9, 'UTC')", + Min: "Nullable(Float64)", + } + compareSchema(source, parquetFrom(valid), "") // must not throw + + // 2. Array(UInt64) -> Array(String) must be rejected. + let rejected = false + try { + compareSchema(source, parquetFrom({ ...valid, BucketCounts: "Array(String)" }), "") + } catch { + rejected = true + } + if (!rejected) h.fail("source Array(UInt64) accepted against injected Array(String)") + + // 3. Map value-type substitution must be rejected. + let mapRejected = false + try { + compareSchema(source, parquetFrom({ ...valid, Attributes: "Map(String, Int64)" }), "") + } catch { + mapRejected = true + } + if (!mapRejected) h.fail("Map value-type substitution accepted") + + h.ok("Array(UInt64)!=Array(String) and Map value-type substitution both rejected; valid schema accepted") +} catch (e) { + h.fail(e instanceof Error ? e.message : String(e)) +} diff --git a/apps/cli/test/probes/archive-probe-timezone-bound.ts b/apps/cli/test/probes/archive-probe-timezone-bound.ts new file mode 100644 index 000000000..f0e395cd5 --- /dev/null +++ b/apps/cli/test/probes/archive-probe-timezone-bound.ts @@ -0,0 +1,79 @@ +// RED-BASELINE probe (round 5): a valid late-UTC shard bound (23:30 UTC) must be +// accepted regardless of host timezone. Round 4 persists timezone-less chDB +// strings and parses them with Date.parse, which is host-timezone-dependent; a +// valid 23:30 UTC bound is rejected as next-day under America/New_York. +// Contract: exit 0 (PASS) when the manifest parser ACCEPTS the valid 23:30 UTC +// bound; exit nonzero (FAIL) when it rejects it. +// +// This probe parses a manifest shard record directly (no chDB needed) under +// TZ=America/New_York to reproduce the host-timezone dependence deterministically. +// +// Run: TZ=America/New_York MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-timezone-bound.ts + +import { ArchiveProbe } from "../archive-probe-helpers" +import { parseArchiveGenerationManifest } from "../../src/server/archives/manifest" +import { randomUUID } from "node:crypto" +import { CHDB_VERSION, MAPLE_VERSION } from "../../src/version" +import { SCHEMA_FINGERPRINT } from "../../src/server/serve" + +const h = ArchiveProbe.create("timezone-bound") + +// Force the host timezone to a non-UTC zone so any host-tz-dependent parse is +// exposed. The probe runner sets TZ; assert it here too. +if (process.env.TZ !== "America/New_York") { + process.env.TZ = "America/New_York" +} + +const manifest = { + formatVersion: 3, + generationId: randomUUID(), + signal: "traces", + rangeStart: "2026-06-29", + // 23:30 UTC is a VALID bound within the 2026-06-29 sealed range. + rangeEndExclusive: "2026-06-30T00:00:00.000Z", + checkpointId: randomUUID(), + checkpointManifestFingerprint: "fp", + createdAt: "2026-06-29T12:00:00.000Z", + mapleVersion: MAPLE_VERSION, + chdbVersion: CHDB_VERSION, + schemaFingerprint: SCHEMA_FINGERPRINT, + sourceRowCount: 1, + archivedRowCount: 1, + tuning: { + writerThreads: 1, + rowGroupRows: 10_000, + maxShardRows: 500_000, + maxShardBytes: 256 * 1024 * 1024, + targetChunkBytes: 1024 * 1024 * 1024, + minFreeSpaceReserve: 512 * 1024 * 1024, + }, + tuningConfig: null, + shards: [ + { + name: "23-0000.parquet", + rowCount: 1, + // 2026-06-29 23:30:00 UTC as epoch nanoseconds. Computed from the UTC + // instant, so parsing it is host-timezone-independent by construction. + minEventTimeUnixNano: `${BigInt(Date.parse("2026-06-29T23:30:00.000Z")) * 1_000_000n}`, + maxEventTimeUnixNano: `${BigInt(Date.parse("2026-06-29T23:30:00.000Z")) * 1_000_000n}`, + sha256: "a".repeat(64), + bytes: 4096, + columns: ["Timestamp"], + complexDigest: "123456789", + complexDigestAlgorithm: "cityhash64-multiset-v3", + }, + ], +} + +try { + parseArchiveGenerationManifest(manifest, "traces", "2026-06-29") + h.ok("valid 23:30 UTC bound accepted under America/New_York") +} catch (e) { + const msg = e instanceof Error ? e.message : String(e) + if (/outside sealed range/i.test(msg)) { + h.fail( + `VALID_UTC_2330_REJECTED under ${process.env.TZ}: ${msg.slice(0, 160)} (host-timezone-dependent parse)`, + ) + } + h.fail(msg) +} diff --git a/apps/cli/test/probes/archive-reconcile-worker.ts b/apps/cli/test/probes/archive-reconcile-worker.ts new file mode 100644 index 000000000..cf6e83494 --- /dev/null +++ b/apps/cli/test/probes/archive-reconcile-worker.ts @@ -0,0 +1,46 @@ +// Reconciliation-only worker for the archive generation lifecycle (Gate 3). +// +// Invokes reconcileArchiveGeneration WITHOUT starting a new export. This models +// the pure crash-recovery path: after a crash, an operator (or the next +// operation's preamble) reconciles the interrupted operation to its exact +// intended state, then a SEPARATE later operation may proceed. Used by the +// native crash-recovery harness to verify recovery independently of a fresh +// export. +// +// Usage: +// MAPLE_LIBCHDB=/libchdb.so \ +// bun apps/cli/test/probes/archive-reconcile-worker.ts \ +// --data-dir --archive-dir +// +// Exit semantics: +// 0 = reconciliation converged (or no active operation existed) +// 1 = reconciliation threw (ambiguous/corrupt state — surfaced to the harness) + +import { randomUUID } from "node:crypto" +import { withMaintenanceLock } from "../../src/server/checkpoints" +import { reconcileArchiveGeneration } from "../../src/server/archives/generation" + +const get = (name: string): string => { + const i = process.argv.indexOf(`--${name}`) + const v = i >= 0 ? process.argv[i + 1] : undefined + if (!v) { + console.error(`reconcile-worker: missing --${name}`) + process.exit(2) + } + return v +} + +const dataDir = get("data-dir") +const archiveDir = get("archive-dir") +const scratchRoot = get("scratch-root") + +withMaintenanceLock(dataDir, randomUUID(), () => reconcileArchiveGeneration(dataDir, archiveDir, scratchRoot)) + .then(() => { + console.log("reconcile: converged") + process.exit(0) + }) + .catch((error) => { + const msg = error instanceof Error ? error.message : String(error) + console.error(`reconcile: FAILED ${msg}`) + process.exit(1) + }) diff --git a/apps/cli/test/probes/calibration-validation-compare.ts b/apps/cli/test/probes/calibration-validation-compare.ts new file mode 100644 index 000000000..969981fd6 --- /dev/null +++ b/apps/cli/test/probes/calibration-validation-compare.ts @@ -0,0 +1,134 @@ +// Calibration validation comparison helper for the native acceptance probe. +// +// Reads a config document and an observed-metrics JSON (produced by a REAL +// calibrate-run trial on held-out data, measured under /usr/bin/time), builds +// the typed six-metric predicted-vs-observed comparison via the PRODUCTION +// comparePredictedObserved function, and emits a CalibrationValidationReport. +// +// The trial is LIKE-FOR-LIKE with the calibration: both run an export sample +// through the same shared writer, so RSS, wall time, throughput, compression, +// physical bytes, and temp disk are directly comparable. The tolerances are +// therefore meaningful acceptance bands, not vacuous. +// +// Usage: bun calibration-validation-compare.ts > report.json + +import { readFileSync } from "node:fs" +import { + comparePredictedObserved, + HELD_OUT_TOLERANCES, + type CandidateMetrics, + type CalibrationValidationReport, +} from "../../src/server/archives/calibrate" + +const configPath = process.argv[2] +const observedPath = process.argv[3] +const genId = process.argv[4] +const signal = process.argv[5] +const rows = Number(process.argv[6]) +const shards = Number(process.argv[7]) + +if (!configPath || !observedPath) { + console.error( + "usage: calibration-validation-compare.ts ", + ) + process.exit(2) +} + +const config = JSON.parse(readFileSync(configPath, "utf8")) as { + selected: { + candidate: { + writerThreads: number + rowGroupRows: number + maxShardRows: number + maxShardBytes: number + } + worstCase: CandidateMetrics + } | null + results: Array<{ + candidate: { + writerThreads: number + rowGroupRows: number + maxShardRows: number + maxShardBytes: number + } + signal: string + metrics: CandidateMetrics | null + ok: boolean + }> +} +const observed = JSON.parse(readFileSync(observedPath, "utf8")) as CandidateMetrics + +if (!config.selected) { + console.error("config has no selected candidate; cannot compare") + process.exit(2) +} + +const sameCandidate = ( + left: (typeof config.results)[number]["candidate"], + right: NonNullable["candidate"], +): boolean => + left.writerThreads === right.writerThreads && + left.rowGroupRows === right.rowGroupRows && + left.maxShardRows === right.maxShardRows && + left.maxShardBytes === right.maxShardBytes + +// Compare the held-out logs trial with the selected candidate's TRAINING logs +// result. Using selected.worstCase here would compare one signal with a +// synthetic aggregate whose individual maxima/minimum may come from six +// different signals, which is not like-for-like. +const predictedResult = config.results.find( + (result) => + result.signal === signal && + result.ok && + result.metrics !== null && + sameCandidate(result.candidate, config.selected!.candidate), +) +if (!predictedResult?.metrics) { + console.error(`config has no successful ${signal} training result for the selected candidate`) + process.exit(2) +} +const predicted = predictedResult.metrics +// Canonical held-out policy (the same constant the calibrator and loader use), +// with hybrid size-scaling. This independent trial is disjoint from training +// but the same size; its measured logical-byte ratio still accounts for row +// shape differences. Throughput/compression are compared directly; RSS/temp- +// disk are absolute peaks. Every tolerance is a relative delta below 1.0; +// throughput's comparator uses +// `observed >= predicted * (1-t)`, so a tolerance >= 1 would be vacuous. +if ( + !Number.isFinite(predicted.logicalBytes) || + predicted.logicalBytes <= 0 || + !Number.isFinite(observed.logicalBytes) || + observed.logicalBytes <= 0 +) { + console.error("training and observed logicalBytes must both be finite and positive") + process.exit(2) +} +const scaleRatio = observed.logicalBytes / predicted.logicalBytes +const comparison = comparePredictedObserved(predicted, observed, HELD_OUT_TOLERANCES, { + ratio: scaleRatio, + metrics: new Set(["wallMs", "physicalBytes"]), +}) + +const report: CalibrationValidationReport = { + formatVersion: 1, + configSha256: "", // filled by the caller (shell computes the SHA) + configName: "", + trial: { generationId: genId, signal, rangeStart: "", archivedRowCount: rows, shardCount: shards }, + comparison, + measuredAt: new Date().toISOString(), +} + +// Emit the comparison summary to stderr for the probe log, the report to stdout. +for (const c of comparison.comparisons) { + process.stderr.write( + ` ${c.metric.padEnd(28)} predicted=${c.predicted} observed=${c.observed} ` + + `delta=${c.relativeDelta.toFixed(3)} tol=${c.tolerance} ${c.withinTolerance ? "OK" : "FAIL"}\n`, + ) +} +process.stderr.write(` verdict: ${comparison.passed ? "PASS" : "FAIL"}\n`) + +// Emit the report as JSON to stdout (the caller reads it). +process.stdout.write(`${JSON.stringify(report, null, 2)}\n`) + +if (!comparison.passed) process.exit(1) diff --git a/apps/cli/test/server-args.test.ts b/apps/cli/test/server-args.test.ts new file mode 100644 index 000000000..8e6eeeb10 --- /dev/null +++ b/apps/cli/test/server-args.test.ts @@ -0,0 +1,48 @@ +import { describe, it } from "@effect/vitest" +import { deepStrictEqual, strictEqual } from "node:assert" +import { buildDetachedChildArgs, type DirtyStorePolicy } from "../src/commands/server-args" + +describe("buildDetachedChildArgs", () => { + for (const policy of ["wipe", "fail", "restore-checkpoint"] satisfies DirtyStorePolicy[]) { + it(`forwards ${policy} exactly once`, () => { + const args = buildDetachedChildArgs({ + entry: "/repo/apps/cli/src/bin.ts", + port: 4318, + dataDir: "/tmp/maple data", + offline: true, + chdbConfigFile: "/tmp/backup config.xml", + onDirtyStore: policy, + }) + deepStrictEqual(args, [ + "/repo/apps/cli/src/bin.ts", + "start", + "--port", + "4318", + "--data-dir", + "/tmp/maple data", + "--on-dirty-store", + policy, + "--chdb-config-file", + "/tmp/backup config.xml", + "--offline", + ]) + strictEqual(args.filter((arg) => arg === "--on-dirty-store").length, 1) + strictEqual(args.includes("--background"), false) + strictEqual(args.includes("-d"), false) + }) + } + + it("omits the virtual compiled entrypoint and optional flags", () => { + deepStrictEqual( + buildDetachedChildArgs({ + entry: "/$bunfs/root/maple", + port: 4418, + dataDir: "/data", + offline: false, + chdbConfigFile: undefined, + onDirtyStore: "fail", + }), + ["start", "--port", "4418", "--data-dir", "/data", "--on-dirty-store", "fail"], + ) + }) +}) diff --git a/apps/landing/src/content/docs/local-mode/cli-reference.md b/apps/landing/src/content/docs/local-mode/cli-reference.md index fc3655e37..0c536b852 100644 --- a/apps/landing/src/content/docs/local-mode/cli-reference.md +++ b/apps/landing/src/content/docs/local-mode/cli-reference.md @@ -40,13 +40,15 @@ Local mode only. `maple start` is the long-lived process that owns the embedded Start the local ingest + query server (embedded ClickHouse via chDB). -| Flag | Default | Description | -| -------------------- | --------------- | ----------------------------------------------------------------------------------- | -| `--port ` | `4318` | Port for OTLP/HTTP ingest, the query API, and the bundled UI | -| `--data-dir ` | `~/.maple/data` | Embedded ClickHouse data directory | -| `--offline` | `false` | Serve the UI bundled in this binary (from `127.0.0.1`) instead of `local.maple.dev` | -| `--background`, `-d` | `false` | Run detached (logs to `~/.maple/maple.log`); stop with `maple stop` | -| `--reset` | `false` | Wipe the existing store before starting — use after an incompatible upgrade | +| Flag | Default | Description | +| --------------------------------------------------- | --------------- | ----------------------------------------------------------------------------------- | +| `--port ` | `4318` | Port for OTLP/HTTP ingest, the query API, and the bundled UI | +| `--data-dir ` | `~/.maple/data` | Embedded ClickHouse data directory | +| `--chdb-config-file ` | | Optional ClickHouse config file passed to embedded chDB | +| `--offline` | `false` | Serve the UI bundled in this binary (from `127.0.0.1`) instead of `local.maple.dev` | +| `--background`, `-d` | `false` | Run detached (logs to `~/.maple/maple.log`); stop with `maple stop` | +| `--reset` | `false` | Wipe live chDB data while preserving checkpoints | +| `--on-dirty-store ` | `fail` | Recovery policy when the store was not cleanly closed | ```bash maple start # foreground, UI from local.maple.dev @@ -54,6 +56,20 @@ maple start --offline # foreground, bundled UI, no internet needed maple start -d --port 4400 # detached on a custom port ``` +Detached startup forwards the selected `--on-dirty-store` policy unchanged to +the foreground child. Before any reset, compatibility check, dirty-store +decision, or data-directory creation, startup reconciles a recorded reset or +checkpoint-restore transaction. Ambiguous, malformed, or conflicting +transaction state fails closed and prints the preserved paths. + +The default dirty-store policy is `fail`, so an unclean shutdown never silently +deletes telemetry. Choose `restore-checkpoint` to recover the selected +checkpoint, or explicitly choose `wipe` to discard only live chDB data. +Checkpoint snapshots, pins, operation evidence, and quarantine state under +`/backups` are preserved by both `--reset` and explicit wipe. +Schema-incompatible stores also fail closed until an operator explicitly +resets live data. + ### `maple stop` Stop a running `maple start` server (reads the PID file beside the data dir). @@ -64,13 +80,140 @@ Stop a running `maple start` server (reads the PID file beside the data dir). ### `maple reset` -Delete the local chDB store so the next `maple start` bootstraps fresh. Refuses to run while a server still owns the store. +Delete live chDB data so the next `maple start` bootstraps fresh. The checkpoint +registry under `/backups` is preserved. Refuses to run while a server +still owns the store. + +Reset is journaled beside the data directory and removes only the chDB-owned +top-level directories produced by the bundled native build (`data`, `metadata`, +`store`, and `tmp`). If any other entry exists, reset preserves everything and +fails with the unrecognized paths so an operator can inspect them. Startup +finishes an interrupted recorded reset before it evaluates store compatibility +or cleanliness. | Flag | Default | Description | | ------------------- | --------------- | ---------------------------- | -| `--data-dir ` | `~/.maple/data` | Store to delete | +| `--data-dir ` | `~/.maple/data` | Store whose live data clears | | `--yes`, `-y` | `false` | Skip the confirmation prompt | +### `maple checkpoint` + +Create and validate a restorable checkpoint of the local chDB store. The running +server must have been started with a chDB config that allows ClickHouse backups: + +```xml + + + default + backups + + +``` + +```bash +maple start --chdb-config-file ./chdb-backups.xml +maple checkpoint +``` + +Every completed checkpoint receives an immutable UUID and is written under: + +```text +/backups/ + state.json + snapshots// + backup/ + manifest.json + operations/ + pins/ + quarantine/ + retiring/ +``` + +`state.json` is the only authority for the selected `current` and `previous` +IDs. Maple writes and syncs a strict versioned manifest only after restoring +the native backup into one sacrificial chDB and validating all six raw telemetry +tables. It then selects the snapshot with a synced atomic state-file +replacement. A third checkpoint retires the old previous snapshot only when it +is complete, compatible, unreferenced, and unpinned; uncertain state is +preserved. + +The earlier unreleased `backups/{building,current,previous}` preview layout is +not inferred or deleted. If it is present without a valid new state pointer, +checkpoint commands fail closed and report the paths for operator inspection. +Missing, malformed, incompatible, incomplete, or symlinked checkpoint state is +also rejected rather than guessed. + +### `maple restore` + +Restore the local chDB store from the last promoted checkpoint. Refuses to run +while a server still owns the store. The existing store is moved aside for +quarantine rather than deleted. + +| Flag | Default | Description | +| ------------------------ | ---------------- | ----------------------------------- | +| `--data-dir ` | `~/.maple/data` | Store to restore | +| `--checkpoint-id ` | selected current | Restore one immutable checkpoint ID | +| `--yes`, `-y` | `false` | Skip the confirmation prompt | + +Restore uses a collision-resistant working path and quarantine, and records a +durable sibling transaction before changing the live directory. Reconciliation +can resume the recorded quarantine, live swap, and marker-update boundaries +idempotently. The displaced live store is never deleted. Unrecorded or +mismatched restore-like paths fail closed without mutation. + +```bash +maple restore --yes +maple restore --checkpoint-id 01234567-89ab-4cde-8fab-0123456789ab --yes +``` + +Checkpoint and restore operations share one maintenance lock. A live owner is +reported as busy; uncertain ownership is preserved and blocks destructive +maintenance. + +## Archive commands + +Local mode only. Export sealed UTC-day ranges of the six raw telemetry tables +from immutable checkpoints into portable Parquet, queryable independently with +DuckDB. See [Local telemetry archives](/docs/local-telemetry-archives) for the +full architecture, calibration, and off-happy-path reference. + +### `maple archive create ` + +Seal one UTC day of one signal into a validated Parquet generation from a +checkpoint. Resolves and pins the checkpoint (default: current), restores it to +sacrificial scratch, exports bounded Parquet shards, validates row counts and +checksums, atomically selects the generation, and releases the pin. The live +store is never opened for export. + +| Argument / Flag | Description | +| ----------------- | --------------------------------------------------------------------------------------------------------- | +| `` | UTC day to seal, `YYYY-MM-DD` | +| `` | `logs`, `traces`, `metrics_sum`, `metrics_gauge`, `metrics_histogram`, or `metrics_exponential_histogram` | +| `--data-dir` | Live chDB data directory (default: `~/.maple/data`) | +| `--archive-dir` | Archive root (default: `~/.maple/archive`) | +| `--scratch-root` | Restored-checkpoint scratch root (default: `~/.maple/scratch`) | +| `--checkpoint-id` | Archive from a specific checkpoint instead of the selected current | + +A late-arrival re-export creates a new generation that supersedes the old one; +the previous generation is retained but excluded from active listings and query +paths. + +### `maple archive list` + +Report active archive generations. Superseded generations are retained on disk +but never listed. + +| Flag | Description | +| ------------------------------- | ------------------------------------------------------------------------------------------ | +| `--archive-dir` | Archive root (default: `~/.maple/archive`) | +| `--output summary\|paths\|json` | `summary` (default), `paths` (machine-readable active Parquet paths for DuckDB), or `json` | +| `--signal ` | Required with `--output paths`; the signal whose active paths to emit | + +### `maple archive rebuild ` + +Rebuild a signal's `catalog.jsonl` from the authoritative generation manifests, +recovering from a truncated or missing catalog without rescanning Parquet bytes. + ## Services ### `maple services` @@ -329,7 +472,7 @@ The on-disk config at `~/.maple/config.json` stores `apiUrl`, `token`, `orgId`, **`maple is already running (PID …)`.** A server already owns this data dir. Stop it with `maple stop`, or start a second instance on another port and data dir: `maple start --port 4400 --data-dir ~/.maple/data-2`. -**Incompatible store after an upgrade.** If a new binary refuses to open an older store (`the local store … is incompatible`), wipe it with `maple reset`, or start fresh in one step with `maple start --reset`. +**Incompatible store after an upgrade.** If a new binary refuses to open an older store (`the local store … is incompatible`), explicitly clear live data with `maple reset --yes`, or start fresh in one step with `maple start --reset`. Both preserve the checkpoint registry; incompatible checkpoints remain preserved and fail closed until deliberately handled. **Browser asks to "access devices on your local network" (or CORS errors).** The default dashboard at `local.maple.dev` is a public origin reaching your loopback server, which trips Chrome's Private Network Access gate. Run `maple start --offline` to serve the dashboard same-origin from `127.0.0.1` — no prompt, no internet needed. diff --git a/docs/local-telemetry-archives.md b/docs/local-telemetry-archives.md new file mode 100644 index 000000000..83d31df7e --- /dev/null +++ b/docs/local-telemetry-archives.md @@ -0,0 +1,750 @@ +# Local telemetry archives + +Maple's embedded chDB store is a bounded **hot store**: it retains recent +telemetry (logs and traces for 30 days, metrics for 90 days) for fast local +querying. Local telemetry archives extend Maple with **long-term, portable +Parquet storage** exported from immutable checkpoints, queryable independently +with DuckDB — without reloading history into the live store or running a second +always-on database. + +This document is the operator and architecture guide for local archives. It +covers the model, the happy path, every major off-happy-path outcome, the +independent query path, the full tuning configuration reference, calibration, +and the directory and manifest layouts. + +## What archives are (and are not) + +**Are:** + +- Immutable Parquet exports of the six raw telemetry tables from a validated + checkpoint. +- Sealed by fixed UTC day and signal, one generation at a time. +- Independently queryable with DuckDB; portable across machines. +- Crash-safe: an interrupted archive leaves the live store untouched and the + archive in a recoverable state. + +**Are not:** + +- A live export endpoint. v1 archives are created explicitly by an operator. +- Automatic hot-store pruning. Existing chDB TTLs govern the hot store; archives + do not delete from it. +- Archive rehydration into the Maple UI. Historical data is queried in DuckDB, + not reloaded into the dashboard. +- A second always-running database. Archives are files; DuckDB opens them on + demand. + +## The six signals + +Archives export exactly the six **raw** telemetry tables. Aggregation and +materialized-view tables are deliberately excluded: they are rebuildable from +raw telemetry and would balloon archive volume without preserving any fact the +raw tables do not already carry. + +| Signal (table / directory name) | Event-time column used for the UTC-day range | +| ------------------------------- | -------------------------------------------- | +| `logs` | `TimestampTime` | +| `traces` | `Timestamp` | +| `metrics_sum` | `TimeUnix` | +| `metrics_gauge` | `TimeUnix` | +| `metrics_histogram` | `TimeUnix` | +| `metrics_exponential_histogram` | `TimeUnix` | + +Each signal drives fixed half-open UTC-day range semantics. Production queries +implement that range with UTC `toDate(...) = ` and per-hour +`toHour(...)` predicates, equivalent to `eventTime >= start AND eventTime < end` +for valid timestamps. + +## Architecture + +```text +Live Maple store (chDB) Archive volume (operator-configured, SEPARATE from data/) + data/ / + backups/ logs/ + state.json 2026-06-01/ + snapshots// active.json ← atomic active pointer (formatVersion 1) + backup/ generations// + manifest.json manifest.json ← generation manifest (formatVersion 3) + pins//.json shards/HH-NNNN.parquet ← one or more shards per hour + operations/active/ catalog.jsonl ← canonical rebuildable JSONL index + quarantine/ traces/ ... + retiring/ building// (in-progress; owned temp output) + quarantine/ + building-/ (retained pre-publication debris) + calibration/ (calibration ownership + samples) + recovery.json + samples// + operations/ + active/archive-/ + intent.json + tombstones// (GC only) + completed/archive-/ + intent.json +``` + +The archive volume is an operator-configured directory that **must be separate** +from the live data directory. The `assertArchiveRootSeparate` check refuses to +archive into (or beneath) the live store. + +### Why checkpoint-restored scratch, not a live copy + +The only proven safe source for an archive is a native chDB checkpoint restored +into sacrificial scratch. A raw copy of the live data directory is unsafe: it +captures an inconsistent on-disk state, may include half-written merges, and +races concurrent ingest. A checkpoint, by contrast, is a validated, consistent +snapshot. Archive export restores one checkpoint into a private scratch chDB +(reusing the same scoped instance that checkpoint validation uses), exports from +it, then removes the scratch. The live store is never opened for export. + +A consequence: archive export holds the **maintenance lock** so it cannot overlap +checkpoint creation, restore, or reset. This is by design — the two operations +share one sacrificial chDB and must serialize. + +### Why generations supersede instead of deduplicating by TraceId + +There is no universal deduplication key across the six raw tables. `TraceId` is +shared by many spans, may be absent from logs, and does not exist on metrics. An +archive therefore seals a fixed UTC-day range into an immutable **generation**. +Late-arriving telemetry for an already-sealed day creates a **new generation** +that structurally supersedes the old one. The `active.json` pointer atomically +selects the new generation; the old generation is retained on disk but never +returned to listings or queries. This avoids scanning all generations to dedup +and makes each generation independently reproducible. + +### Separation of logical chunks, physical shards, and row groups + +Three distinct units, all configurable and calibratable: + +- A **logical chunk** is a provisioning target (the `targetChunkBytes` tuning + value); it is not a hard limit. +- A **physical shard** is one Parquet file, bounded by `maxShardRows` and + `maxShardBytes`. In v1, each shard covers one UTC hour within the sealed day; + if a single hour exceeds `maxShardBytes` uncompressed, it is recursively + bisected at the physical `_part_offset` boundary. A single row that exceeds the + byte bound is a distinct failure (raise `maxShardBytes` or recalibrate). +- A **Parquet row group** is the unit of compression and parallel decode inside a + shard, sized by `rowGroupRows`. + +## Pinning and the maintenance lock + +Archive export holds Maple's **maintenance lock** so it cannot overlap checkpoint +creation, restore, or reset. Inside the lock, it acquires a **persistent pin** +on the source checkpoint so retention cannot delete the snapshot between +resolution and export. A stale pin (e.g. from a crashed archive that never +released it) safely over-retains data rather than risking deletion. The pin is +released after the generation is durable. Calibration pins use the purpose +`archive-calibrate:` so they are unambiguous and operation-scoped. + +## Commands + +`maple archive` has six operator-facing subcommands (`create`, `list`, +`rebuild`, `reconcile`, `gc`, `calibrate`) plus the internal +`calibrate-session` and `calibrate-run` commands used by calibration and its +fault probes. There are no short flags anywhere in this command tree. Root +flags fall back to `~/.maple` defaults when omitted. + +| Flag | Default | +| ---------------- | ------------------ | +| `--data-dir` | `~/.maple/data` | +| `--archive-dir` | `~/.maple/archive` | +| `--scratch-root` | `~/.maple/scratch` | + +### `maple archive create ` + +Seal one UTC day of one signal into a validated Parquet generation. + +```sh +maple archive create 2026-06-01 traces \ + --data-dir ~/.maple/data \ + --archive-dir /Volumes/External/maple-archive \ + --scratch-root /Volumes/External/maple-scratch \ + --config ./maple-archive-config.json +``` + +- ``: the UTC day to seal, as `YYYY-MM-DD` (validated; impossible + calendar dates like `2026-02-31` are rejected). +- ``: one of the six signal names (positional, not a flag). +- `--checkpoint-id`: archive from a specific checkpoint instead of `current`. +- `--archive-dir` / `--scratch-root` / `--data-dir`: override the defaults. +- `--config`: load tuning overrides from a versioned calibration config document + (see [Tuning configuration](#tuning-configuration)). The config's SHA-256 + identity is recorded in the generation manifest. The strict config schema has + no root override fields; roots always come from the CLI/defaults. + +The command resolves and pins the checkpoint, restores it to scratch, exports +bounded Parquet shards, validates row counts and checksums, publishes the +generation manifest, atomically selects it, canonically rebuilds the catalog, +releases the pin, and removes the owned scratch. + +**Tuning precedence:** `--config` effective values override compiled tuning +defaults. `archive create` exposes no per-knob CLI tuning flags in v1; its root +flags are separate and remain authoritative. + +### `maple archive list` + +Report active generations: + +```sh +maple archive list --archive-dir /Volumes/External/maple-archive +maple archive list --output paths --signal traces # machine-readable paths +maple archive list --output json # full JSON +``` + +`--output` modes (`summary` is default, only `list` has this flag): + +- `summary`: one line per active generation: signal, range, rows, shards, + short generation id. +- `paths`: a single comma-separated, double-quoted list of the active + generation's Parquet shard paths (excluding superseded generations), ready for + DuckDB's `read_parquet`. Requires `--signal`. +- `json`: the full `listActiveGenerations` object, pretty-printed. + +`list` verifies every shard's actual SHA-256 and byte size against the manifest +before returning it; a tampered shard fails closed (the affected range surfaces +in `errors`, other ranges still list). Only the active generation is exposed. + +### `maple archive rebuild ` + +Rebuild a signal's `catalog.jsonl` from the authoritative generation manifests, +recovering from a truncated or missing catalog without rescanning Parquet bytes. +`` is positional. + +### `maple archive reconcile` + +Reconcile an interrupted `create` or `gc` operation to its intended state +**without a fresh export**. Flags: the three root flags plus `--dry-run`. + +- `--dry-run`: report the decision and the archive root without mutating + anything. +- Apply: execute the decision function's verdict. + +The decision is one of: `NoOp` (nothing active), `FailClosed` (unsafe state — +zero mutation, exits non-zero), `CreateVerifyComplete`, `CreateAbortPrepublication`, +`CreateFinishPublication`, `GcVerifyComplete`, or `GcResume`. A subsequent +`create` also runs this reconciliation automatically as its first step. + +### `maple archive gc` + +Reclaim superseded archive generations, retaining the newest N per signal/range. +This is the **only** archive operation that deletes published generations. + +```sh +maple archive gc --archive-dir /Volumes/External/maple-archive --keep 1 +maple archive gc --keep 0 --dry-run # preview reclaiming all superseded +``` + +- `--keep` (default `1`, `>= 0`): generations to retain per signal/range beyond + the active one. `--keep 0` reclaims all superseded generations. +- `--dry-run`: plan only, no mutation. If an operation is active in + `operations/active/`, the dry run reports the blocker and reclaims nothing. + +GC is conservative to a fault: it verifies every generation's manifest and shard +checksums up front, excludes any signal/range whose catalog is not provably +reconstructable or whose active pointer is missing, deletes by tombstone-rename +(never in-place recursive delete), persists progress after every target, and +proves terminal invariants before retiring the journal. + +### `maple archive calibrate ` + +Calibrate archive tuning by running a candidate matrix against a pinned +checkpoint across **all six signals**. + +```sh +maple archive calibrate 2026-06-01 \ + --archive-dir /Volumes/External/maple-archive \ + --memory-budget 536870912 --time-budget 60000 \ + --write-config ./maple-archive-config.json +``` + +Flags (defaults shown): + +| Flag | Default | Meaning | +| ------------------------- | --------------------- | -------------------------------------------------- | +| `--checkpoint-id` | `current` | Source checkpoint | +| `--memory-budget` | `536870912` (512 MiB) | Per-candidate RSS ceiling | +| `--time-budget` | `60000` (ms) | Total matrix deadline | +| `--sample-rows` | `10000` | Rows sampled per signal (training window `[0, N)`) | +| `--max-candidate-wall-ms` | `30000` (ms) | Per-candidate wall ceiling | +| `--min-throughput` | `0` (B/s) | Throughput floor (0 disables) | +| `--max-temp-disk` | `2147483648` (2 GiB) | Temporary disk ceiling | +| `--free-space-reserve` | `536870912` (512 MiB) | Required free space on the archive volume | +| `--safety-margin-milli` | `1100` (→ 1.1×) | Margin applied inside each ceiling (thousandths) | +| `--write-config` | none | Write the recommended config document to this path | + +The calibrator spawns each candidate as a child process under `/usr/bin/time` +(for independent peak-RSS measurement) inside its own process group with a +wall-clock and temporary-disk watchdog. It selects the candidate with the lowest +worst-case peak RSS (tie-broken by wall) that passes every signal's ceiling, +then validates the selection on a **disjoint held-out window** +`[sampleRows, 3*sampleRows)` through the same real writer: `N` training rows +followed by `2N` held-out rows. Confidence is `high` only when a candidate is +selected and every signal actually produced the complete requested training +and held-out cardinality; otherwise it is `low` with `selected: null` and no +config is written. See [Calibration](#calibration). + +`maple archive calibrate-session --action open|close` is an internal recovery +and probe command. `open` reconciles an older session, resolves one checkpoint, +acquires its operation-scoped pin, and prints the operation/checkpoint identity +required by `calibrate-run`. `close` invokes the authoritative reconciler, +removing only the derived sample/scratch paths and exact session pin. Ordinary +operators should use `calibrate`, which owns this lifecycle automatically. +After all measurements finish, `calibrate` reconciles the session and releases +the source pin before publishing the config. A deterministic +`post-session-release` crash probe proves that interruption in this gap writes +no config and leaves no pin, recovery record, sample, or scratch debris; the +operator must rerun calibration. + +## The happy path: fresh checkpoint through DuckDB investigation + +1. Ingest telemetry into the running Maple store. +2. `maple checkpoint` to create a validated checkpoint. +3. (Optional) `maple archive calibrate --write-config cfg.json` to tune for + your hardware, then use `--config cfg.json` on `create`. +4. `maple archive create 2026-06-01 traces` (and the other five signals). +5. `maple archive list --output paths --signal traces` to get the Parquet paths. +6. Query in DuckDB: + +```sh +duckdb -c "SELECT ServiceName, count(*) FROM read_parquet(['/path/to/00.parquet', ...], union_by_name=true) GROUP BY ServiceName" +``` + +## DuckDB queries + +Archives are portable Parquet. Use `read_parquet` with the active paths from +`maple archive list --output paths`. `union_by_name=true` NULL-fills columns +added between generations; without it, a schema mismatch fails closed. + +```sql +-- Logs by service containing a keyword +SELECT ServiceName, min(Timestamp), max(Timestamp), count(*) +FROM read_parquet(, union_by_name=true) +WHERE Body ILIKE '%timeout%' +GROUP BY ServiceName; + +-- Traces with p99 duration by service +SELECT ServiceName, count(*), quantile_cont(Duration, 0.99) +FROM read_parquet(, union_by_name=true) +WHERE StatusCode = 'Error' +GROUP BY ServiceName; + +-- Sum metric maxima +SELECT ServiceName, MetricName, max(Value) +FROM read_parquet(, union_by_name=true) +GROUP BY ServiceName, MetricName; +``` + +### Memory limits and spill storage + +For large archive ranges, constrain DuckDB's memory and direct spills to the +archive volume: + +```sql +PRAGMA memory_limit='2GB'; +PRAGMA temp_directory='/Volumes/External/duckdb-spill'; +``` + +## Tuning configuration + +The tuning knobs are centralized, documented, and overridable. Defaults are the +measured research baselines — **not universal constants**. A deployment should +calibrate against its checkpoint, archive volume, chDB version, and memory budget +with `maple archive calibrate`. + +### Fields, defaults, and validation + +| Field | Type | Default | Constraint | +| --------------------- | ------ | --------------------- | ------------------------------------------ | +| `writerThreads` | number | `1` | positive integer, `<= 32` | +| `rowGroupRows` | number | `10000` | positive integer, `<= maxShardRows` | +| `maxShardRows` | number | `500000` | positive integer | +| `maxShardBytes` | number | `268435456` (256 MiB) | positive integer, `>= rowGroupRows * 1024` | +| `targetChunkBytes` | number | `1073741824` (1 GiB) | positive integer, `> minFreeSpaceReserve` | +| `minFreeSpaceReserve` | number | `536870912` (512 MiB) | positive integer, `< targetChunkBytes` | + +There is no clamping: any out-of-bounds value or unsafe combination fails closed +with an explicit error. `archiveDir` and `scratchRoot` have no defaults in the +tuning block; they are always resolved from the CLI/defaults. + +- `writerThreads` → chDB `max_threads` (Parquet writer thread count). +- `rowGroupRows` → `output_format_parquet_row_group_size`. +- `maxShardRows` / `maxShardBytes` → physical shard split bounds. +- `targetChunkBytes` → provisioning hint (not a hard limit). +- `minFreeSpaceReserve` → enforced free-space headroom at operation time. + +Every generation manifest records the effective tuning values (the six knobs +above), so a generation is reproducible and deployment drift is visible. + +### The calibration config document + +`maple archive calibrate --write-config ` writes a **versioned calibration +config document** (`formatVersion: 3`, mode `0o600`) with strict, exact-key +schema. It is a complete evidence record, not just the numbers, and the loader +re-derives every aggregate from the recorded evidence rather than trusting it. +Top-level keys (all required; unknown keys rejected): + +| Key | Contents | +| ----------------------- | ---------------------------------------------------------------------- | +| `formatVersion` | `3` | +| `checkpoint` | `{ checkpointId, manifestFingerprint }` — the single source snapshot | +| `candidateMatrix` | The exact four-candidate matrix evaluated | +| `requiredSignals` | The exact six-signal set | +| `budget` | The full `CalibrationBudget` the run used (see below) | +| `selected` | `{ candidate, worstCase }` for the chosen candidate (always present) | +| `confidence` | `"high"` (a loadable recommendation is always high-confidence) | +| `heldOut` | Selected held-out evidence, scaling inputs, and six comparisons | +| `heldOutAttempts` | Every attempt, including rejected results and recomputed comparisons | +| `samplePolicy` | The disjoint training/held-out window contract (sizes + windows) | +| `environment` | Maple/chDB version, schema fingerprint, CPU, memory, archive-volume id | +| `results` | Per-signal, per-candidate evidence, each with a `sample` scope | +| `effective` | The six effective tuning knobs (what `--config` applies) | +| `derivation` | How `minFreeSpaceReserve`/`targetChunkBytes` are derived | +| `safetyMargin` | The margin applied inside each ceiling | +| `recalibrationTriggers` | The six events that should prompt recalibration | +| `measuredAt` | Canonical UTC ISO-8601 timestamp | +| `note` | Human-readable summary | + +Each `results` entry carries a `sample` scope — `{ checkpointId, +checkpointManifestFingerprint, rangeDate, role, startRow, requestedRows, +rowCount }` — binding that measurement to one immutable checkpoint/range and an +exact ordered-row window. Every training sample is `role: "training"`, +`startRow: 0`; every held-out sample is `role: "held-out"`, `startRow: +sampleRows`. The loader proves all scopes share one checkpoint/range, that the +two windows are disjoint, and that actual `rowCount` equals `requestedRows` +(`N` for training and `2N` for held-out). A short source window is +unrepresentative and cannot produce a loadable recommendation. + +`heldOut` and every complete `heldOutAttempts` entry persist a descriptive +aggregate `worstCase` plus a `signalComparisons` array — one entry per signal in +canonical order. Each entry carries its own `scaleRatio`, six metric comparison +records (adjusted prediction, observation, tolerance, relative delta, pass/fail), +and a per-signal `passed` flag; raw metrics are not duplicated (the loader +re-derives them from the training/held-out results by candidate + signal). The +loader recomputes these values; the document cannot choose its own ratio, +prediction, tolerance, or signal pairing. + +Format 3 treats resource costs directionally: a lower observed RSS, wall time, +compression ratio, physical-byte count, or temporary-disk peak is safe; only a +regression beyond tolerance fails. Write operations emit only format 3. For +upgrade compatibility, the loader also accepts a format-2 document only when +its *entire* held-out evidence matches one coherent historical policy: either +the original symmetric comparison or the brief directional format-2 form. It +rejects a document that mixes those representations across the selected +evidence and attempts. + +`environment.archiveVolume` records `{ fsid, type, archiveDir }` so a config is +bound to the volume it was measured on, and `archive create --config` enforces +that identity (plus the host environment) before exporting. `recalibrationTriggers` +is exactly: + +1. Maple version change +2. chDB version change +3. Schema fingerprint change +4. Hardware change (CPU count, memory, storage speed) +5. Archive-volume replacement or filesystem change +6. Material telemetry-shape change (row width, cardinality, signal mix) + +A document containing only `formatVersion` + `effective` is **rejected** — all +evidence fields are required, so a config cannot be hand-edited into existence. +The loader recomputes the worst cases, the held-out comparisons, the tuning +derivation, and the sample scopes, and rejects any field that does not match +(for example a forged tolerance, a redefined derivation, or a scope bound to the +wrong checkpoint). + +### How `--config` loads + +`loadTuningConfig` opens the file with defense-in-depth against tampering and +TOCTOU: + +1. `lstat` first — refuse if not a regular file (rejects symlinks/devices). +2. Size cap: `16 MiB` (`MAX_CONFIG_BYTES`). +3. `open` with `O_NOFOLLOW` — the kernel refuses a symlink at the final path. +4. `fstat` the fd — refuse if not a regular file. +5. **fd-identity check** — the opened fd's `dev`/`ino` must equal the pre-`lstat` + `dev`/`ino` (detects a swap between lstat and open). +6. Bounded read to exactly the fd's size; SHA-256 is computed over those exact + bytes. + +The result is a `TuningConfigIdentity` bound into the manifest: + +```jsonc +{ "formatVersion": 3, "configName": "maple-archive-config.json", "sha256": "<64 hex>" } +``` + +`configName` is the file basename (validated `^[A-Za-z0-9._-]+$`); `sha256` is +the content hash. A generation thus records exactly which config produced it. +(The manifest stores this as an opaque, hash-bound identity, so it can describe +both legacy v1 and verified v2 config documents; only the loader refuses v1 for +new writes. It also records and accepts format-3 directional config documents.) + +## Calibration + +Calibration measures how archive export behaves on your hardware and recommends +the candidate that meets your resource budget with the most headroom. It is the +recommended way to set tuning; the defaults are a research baseline only. + +### The candidate matrix + +Four fixed candidates are evaluated, each across **all six signals**: + +| Candidate | `writerThreads` | `rowGroupRows` | `maxShardRows` | `maxShardBytes` | +| --------- | --------------- | -------------- | -------------- | --------------- | +| 1 | 1 | 10 000 | 500 000 | 256 MiB | +| 2 | 1 | 5 000 | 250 000 | 128 MiB | +| 3 | 2 | 10 000 | 500 000 | 256 MiB | +| 4 | 1 | 20 000 | 1 000 000 | 512 MiB | + +### Worst-case aggregation and selection + +For each candidate, per-signal metrics are aggregated into a single worst case: +**MAX** of every cost metric (`logicalBytes`, `physicalBytes`, +`compressionRatio`, `peakTempDiskBytes`, `peakRssBytes`, `wallMs`, `rowCount`) +and **MIN** of `writeThroughputBytesPerSec` (the slowest signal is the worst +case). A candidate is eligible only if **every** signal individually meets the +ceilings. Selection is best-first by lowest worst-case peak RSS, tie-broken by +lowest wall. + +### Margin inside each ceiling + +The safety margin is applied **inside** each ceiling, not to the result: + +- **RSS:** `peakRssBytes * margin > memoryBudget` → fail +- **Wall:** `wallMs > maxCandidateWallMs` → fail (hard ceiling, no margin) +- **Throughput** (only if `minThroughputBytesPerSec > 0`): + `writeThroughputBytesPerSec / margin < minThroughputBytesPerSec` → fail +- **Temp disk:** `peakTempDiskBytes * margin > maxTempDiskBytes` → fail + +So `safetyMargin` 1.1 reserves 10% headroom under the declared budget for RSS, +throughput, and temp disk. + +### Held-out validation + +The selected candidate is re-measured on a **larger, disjoint** row window +through the same shared writer. Training covered ordered rows `[0, sampleRows)`; +held-out covers `[sampleRows, sampleRows + 2*sampleRows)`, equivalently +`[N, 3N)` — strictly larger than training and non-overlapping. Both requested +windows and observed cardinalities are recorded in every result's `sample` +scope and in `samplePolicy`; the loader requires actual `N`/`2N` rows. + +The comparison is **per-signal and like-for-like**: each signal's held-out +result is paired with the same candidate's TRAINING result for that signal, and +the comparison uses that signal's own +`scaleRatio = heldOut.logicalBytes / training.logicalBytes`. Cross-signal +aggregate extrema never decide acceptance (they are recorded only as a +descriptive `worstCase` summary). Each signal's entry in `signalComparisons` +records its own `scaleRatio`, its six metric comparisons, and a per-signal +`passed` flag. The attempt passes only when **all six signals** pass. The fixed +canonical tolerances (`< 1.0` for every metric) apply per metric, per signal: + +| Metric | Comparison | +| ---------------------------- | --------------------------------------------- | +| `peakRssBytes` | absolute peak, two-sided | +| `wallMs` | training prediction × `scaleRatio`, two-sided | +| `writeThroughputBytesPerSec` | direct; higher observed is better | +| `compressionRatio` | direct, two-sided | +| `physicalBytes` | training prediction × `scaleRatio`, two-sided | +| `peakTempDiskBytes` | absolute peak, two-sided | + +The loader rejects document-selected tolerances and independently recomputes +each signal's ratio, adjusted predictions, relative deltas, and per-signal pass +result by re-pairing the recorded training and held-out results by exact +candidate + signal identity. Training and held-out `logicalBytes` must both be +strictly positive for every paired signal (an undefined ratio makes the attempt +incomplete, never silently ratio 1). + +A candidate that fails held-out (any signal fails) is **rejected** and the next +eligible candidate is tried; every attempt is recorded in `heldOutAttempts`. A +complete attempt records six `signalComparisons` entries (even when it fails); +an incomplete or over-budget attempt records `signalComparisons: []`, +`worstCase: null`, `passed: false`. + +### When calibration does not recommend + +- **No candidate meets the ceilings across all six signals** → no + recommendation; the command exits non-zero with a clear note and **no config + is written**. +- **No eligible candidate passes held-out**, or the data is + small/unrepresentative (a signal's training row count below `sampleRows`) → + the command exits non-zero with `selected: null` and **no config is written**. + The CLI throws before writing a config when there is no recommendation. +- **An impossible resource budget** is not a special case: it composes from the + above — every candidate is rejected and no config is written, with no change + to existing configuration and no temporary data left behind. + +The calibrator never redefines the operator's goals to make a candidate pass, +and a config cannot redefine its tolerances, derivation, or sample scope. + +## Manifest, pointer, and catalog formats + +### Generation manifest (`manifest.json`, formatVersion 3) + +One per generation at +`///generations//manifest.json`. Fields: + +| Field | Type | Notes | +| ---------------------------------------------------- | --------------- | ------------------------------------------------------- | +| `formatVersion` | `3` | Readers reject v2/v1 fail-closed (re-export to migrate) | +| `generationId` | string (UUIDv4) | | +| `signal` | string | | +| `rangeStart` | string | `YYYY-MM-DD` | +| `rangeEndExclusive` | string | ISO, the next UTC midnight | +| `checkpointId` | string | Source checkpoint | +| `checkpointManifestFingerprint` | string | `id:createdAt:backupBytes` of the source checkpoint | +| `createdAt` | string | ISO | +| `mapleVersion` / `chdbVersion` / `schemaFingerprint` | string | | +| `sourceRowCount` / `archivedRowCount` | number | Must be equal; `Σ shard.rowCount == archivedRowCount` | +| `tuning` | object | The six effective knobs | +| `tuningConfig` | object \| null | `{ formatVersion, configName, sha256 }` or null | +| `shards` | array | One `ArchiveShardRecord` per shard | + +Each `shard` entry: `name` (e.g. `00-0000.parquet`), `rowCount`, +`minEventTimeUnixNano` / `maxEventTimeUnixNano` (epoch-nanosecond decimal +strings), `sha256`, `bytes`, `columns`, `complexDigest`, and +`complexDigestAlgorithm`. Cross-field invariants (unique names, row-count sums, +source == archived) are enforced. + +**Format-version history.** v1 used timezone-dependent time evidence and a +per-column-sum digest; v2 moved to UTC epoch-nanosecond strings and a multiset +digest but carried a bare `tuningConfigName`; **v3** replaces that with the +SHA-256-bound structured `tuningConfig` identity. v3 readers reject v2/v1 +fail-closed and preserve the files — older archives must be re-exported, not +migrated in place. + +### Active pointer (`active.json`, formatVersion 1) + +One per `///active.json`: `{ formatVersion: 1, +generationId, signal, rangeStart, selectedAt }`. The signal and range are bound +to the enclosing directory (mismatch fails closed). It is replaced atomically to +select a new generation. + +### Catalog (`catalog.jsonl`) + +One per signal at `//catalog.jsonl`. Each line is a JSON +object: `{ generationId, signal, rangeStart, checkpointId, archivedRowCount, +shardCount, createdAt, formatVersion: 1 }`. The catalog is a canonical, +rebuildable index: create, GC, and `archive rebuild` durably rewrite it from the +authoritative manifests. `assertCatalogExact` proves the result byte-for-byte +without rescanning Parquet. + +## Recovery and reconciliation + +Create and GC persist durable ownership/intent records **before** mutation and +retire them **only after** proving terminal state. Calibration has its own +recovery records. Catalog rebuild uses a durable atomic rewrite rather than an +operation journal. A single pure decision function (`decideReconciliation`) is +the sole branch logic for create/GC recovery. + +### The decision function + +Given an inspection of the on-disk state, it returns one of: + +- `NoOp` — nothing active. +- `FailClosed` — an unsafe/impossible topology (e.g. both building and final + state present, a published generation with no manifest, a final generation + before the manifest-written phase, an aborted operation still active). Zero + mutation; exits non-zero. +- `CreateVerifyComplete` — a create reached `complete`; verify terminal + invariants only. +- `CreateAbortPrepublication` — an interrupted create that had not published; + move its owned building dir into retained quarantine, remove exact owned + scratch, and release its exact pin. +- `CreateFinishPublication` — an interrupted create that **had** published; + re-select the pointer and rebuild the catalog. +- `GcResume` — resume collecting a frozen GC target set. +- `GcVerifyComplete` — a GC reached `complete`; prove terminal invariants and + retire the journal. + +A phase label is never proof: the decision and the terminal checks re-read +reality from disk. Reconciliation runs inside the maintenance lock, and a +subsequent `create` runs it automatically as its first step, so most +interruptions heal without an explicit operator action. + +### Calibration recovery + +Calibration has its own durable record at +`/calibration/recovery.json` (`formatVersion: 1`), naming owned +paths **derived from the operation id** (`calibrate-` scratch, +`calibration/samples/` archive, and the pin at +`pinFilePath(dataDir, checkpointId, pinId)` with purpose +`archive-calibrate:`). Because the paths are derived, a crash +between pin creation and the phase advance (when the record still shows +`pinPath: null`) still releases the exact pin. A checkpoint-fingerprint +mismatch fails closed and preserves the record. Reconciliation removes the owned +dirs only after classifying them as real directories, and clears the record only +after the pin is confirmed released and both dirs confirmed absent — otherwise +it preserves the record for retry. + +### GC recovery + +GC persists the **non-terminal** `gc-collecting` phase after every target +(including the last); `complete` is written only after catalog rebuild and +`assertCatalogExact`. Collection is by tombstone-rename (`generations/` → +`operations/active/archive-/tombstones/`) then removal, never in-place +recursive delete. A read-only preflight classifies every frozen target into +**prefix** (already collected), **current** (the documented crash topologies), +and **suffix** (must still be untouched); an out-of-order suffix mutation is +`impossible` and fails closed. Resume finishes a half-removed tombstone or +idempotently confirms an already-absent target. + +## Off-happy-path outcomes + +| Outcome | What happens | +| ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Unavailable checkpoint** | `archive create` fails closed; the live store is untouched. No generation is written. | +| **Incompatible checkpoint** (wrong chDB/schema version) | The checkpoint resolver rejects it; no export runs. | +| **Stale pin** | A crashed archive's pin over-retains the checkpoint snapshot safely. Re-running archive create succeeds; the pin from the failed run can be inspected under `backups/pins/`. | +| **Interrupted restore** | The restored scratch remains journal-owned until the next `create` or `archive reconcile` removes the exact owned path. The live store is never modified. | +| **Partial shard** | Row limits are planned into separate shards and byte-overflow candidates are recursively bisected. Only one matching row that still exceeds `maxShardBytes` fails distinctly. | +| **Validation mismatch** (source vs archived row count) | The generation is not promoted. Reconciliation moves owned building output into retained quarantine, clears exact scratch/pin state, and leaves the active pointer unchanged. | +| **Full or disconnected archive volume** | Free-space preflight fails before any export. No scratch is created. | +| **Pointer or catalog corruption** | Summary mode omits malformed ranges; JSON exposes their errors, while paths mode fails closed for the requested signal. `archive rebuild` atomically replaces only the catalog. | +| **Late telemetry** | A new generation supersedes; the old generation is retained but excluded from active paths. | +| **Supersession** | Same as late telemetry: the newest generation becomes active; superseded ones remain on disk until `archive gc` reclaims them. | +| **Interrupted create** | Reconciles automatically on the next `create`, or via `archive reconcile`. Pre-publication output moves to retained quarantine; post-publication repairs pointer and catalog. | +| **Interrupted GC** | Resumes the frozen target set; a half-removed tombstone is finished, an already-absent target is confirmed. Out-of-order mutation fails closed. | +| **Interrupted calibration** | The derived-pin and owned-dir reconciliation releases the pin and removes the sample; the record is preserved until cleanup is proven. | +| **Insufficient memory budget** | Calibration reports `low` confidence (or no recommendation) rather than presenting synthetic precision. | +| **Failed calibration** | No config is written; temporary calibration output is cleaned up. Existing configuration is unchanged. | + +### What failures leave untouched vs. require action + +- **Live store untouched by every archive failure.** Export reads only from + restored scratch. GC never touches the live store either. +- **Recoverable or retained debris:** create reconciliation releases exact + scratch/pin ownership but retains pre-publication building evidence under + `quarantine/building-`. Unrelated stale pins are safely + over-retained. +- **Requires reconciliation:** an interrupted `create` after publication + (pointer/catalog may be inconsistent until reconcile re-selects/rebuilds) and + an interrupted GC (frozen target set resumed). +- **Operator intervention:** a `FailClosed` reconciliation (impossible topology + or suspected corruption), a persistently corrupt active pointer, or a shard + that repeatedly exceeds bounds requires manual inspection. `archive reconcile +--dry-run` reports the verdict without mutating. + +## Capacity and resource model + +For a 4 GiB hot-store target, live store plus current and previous checkpoints is +roughly 3x the live footprint. Checkpoint creation, scratch restore, and archive +building can temporarily raise aggregate working storage toward 4–5x. That is +an aggregate across volumes, not a free-space requirement for one disk. +Checkpoint validation and archive export share **one** sacrificial chDB, so +archive export does not add a second concurrent `f(4)` memory term. + +The archive volume grows with retained historical ranges. Use volume-specific +free-space measurements in deployment. Create requires +`minFreeSpaceReserve + targetChunkBytes` on the archive filesystem; calibration +children require `freeSpaceReserve + 4 * maxShardBytes`. GC lets you bound growth +by reclaiming superseded generations. + +> **Capacity caveat:** The research baselines were measured on one macOS ARM64 +> machine with one synthetic data distribution. CPU count, RAM, storage speed, +> row width, cardinality, and compression ratio vary. Operators should +> calibrate their deployment. + +## Non-goals (v1) + +- No live export endpoint. +- No automatic hot-store pruning. +- No archive rehydration into the Maple UI. +- No always-running twin database. +- No automatic archive scheduling (start manual; add scheduling only after + repeated successful runs and measured checkpoint pause). diff --git a/scripts/build-local-binary.sh b/scripts/build-local-binary.sh index 33811d20a..06e79e25f 100755 --- a/scripts/build-local-binary.sh +++ b/scripts/build-local-binary.sh @@ -3,16 +3,17 @@ # executable plus libchdb. No Rust/cargo involved. # # Pipeline: -# 1. Build the lightweight SPA (`apps/local-ui` → its `dist/`). -# 2. Inline that dist into apps/cli/src/server/ui-embed.gen.ts so +# 1. Build the workspace packages consumed through gitignored `dist/` paths. +# 2. Build the lightweight SPA (`apps/local-ui` → its `dist/`). +# 3. Inline that dist into apps/cli/src/server/ui-embed.gen.ts so # `bun build --compile` bakes the SPA into the binary. -# 3. Compile apps/cli (the CLI + the OTLP-ingest/query server) into a single +# 4. Compile apps/cli (the CLI + the OTLP-ingest/query server) into a single # executable with `bun build --compile`. The schema artifacts and SPA are # embedded; the OTLP encoders run in-process; chDB is reached via bun:ffi. -# 4. Download libchdb (v26.1.0, matching what we test against) for the host +# 5. Download libchdb (v26.1.0, matching what we test against) for the host # platform and place it beside the binary. At runtime `maple` dlopens the # sibling libchdb (resolved relative to its own path) — no rpath tricks. -# 5. Restore the committed ui-embed.gen.ts stub so the tree stays clean. +# 6. Restore the committed ui-embed.gen.ts stub so the tree stays clean. # # The distributable is a 2-file bundle: `maple` + `libchdb.so`. Keep them in the # same directory. @@ -33,15 +34,14 @@ MAPLE_BUILD_VERSION="${MAPLE_BUILD_VERSION:-$(git -C "$REPO_ROOT" describe --tag mkdir -p "$OUT_DIR" -# apps/cli imports `@maple-dev/effect-sdk/server`, which is published from its -# built `dist/` (tsdown) — and `lib/effect-sdk/dist` is gitignored. A fresh -# checkout / CI only runs `bun install`, so the dist won't exist and -# `bun build --compile` fails to resolve the import. Build it first. -echo "==> Building @maple-dev/effect-sdk (telemetry SDK consumed by the CLI)" -bun --filter @maple-dev/effect-sdk build +# Several workspace packages are published from gitignored `dist/` directories. +# A fresh checkout / CI only runs `bun install`, so build the repository's +# canonical prerequisite set before local-ui or the CLI resolves those imports. +echo "==> Building workspace prerequisites" +bun run alchemy:build-deps echo "==> Building local-ui SPA" -bun --filter @maple/local-ui build +bun run --filter @maple/local-ui build echo "==> Inlining SPA into ui-embed.gen.ts" restore_stub() { git -C "$REPO_ROOT" checkout -- "$UI_EMBED" 2>/dev/null || true; }