From 8909dd6960ae6433459e0ce75d58f5ba0c191141 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 4 Jul 2026 00:50:56 +0200 Subject: [PATCH] chore(bench): sort-key spike scaffolding (ClickStack Phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measurement tooling to decide whether a time-bucketed primary key can replace the two-stage list-query cutoff (read_in_order). Ships NO production change. - spike/sortkey-variants.ts: candidate sort keys (t1/t2/t3, l1/l2/l3, OrgId always first) + shadow-table / backfill / read_in_order-probe SQL builders. - spike/emit-spike-sql.ts: dependency-free emitter (setup | backfill | probe | drop) — runs with bare bun in a fresh worktree. - spike/SPIKE-SORTKEY.md: runbook + explicit promote/reject decision criteria. Drives the A/B via `bench suite --table-map` against a dedicated ClickHouse. Tests: 10 spike + 11 bench. Co-Authored-By: Claude Fable 5 --- apps/api/scripts/spike/SPIKE-SORTKEY.md | 71 +++++++++ apps/api/scripts/spike/emit-spike-sql.ts | 120 ++++++++++++++ .../scripts/spike/sortkey-variants.test.ts | 92 +++++++++++ apps/api/scripts/spike/sortkey-variants.ts | 147 ++++++++++++++++++ 4 files changed, 430 insertions(+) create mode 100644 apps/api/scripts/spike/SPIKE-SORTKEY.md create mode 100644 apps/api/scripts/spike/emit-spike-sql.ts create mode 100644 apps/api/scripts/spike/sortkey-variants.test.ts create mode 100644 apps/api/scripts/spike/sortkey-variants.ts diff --git a/apps/api/scripts/spike/SPIKE-SORTKEY.md b/apps/api/scripts/spike/SPIKE-SORTKEY.md new file mode 100644 index 000000000..2d562f5a8 --- /dev/null +++ b/apps/api/scripts/spike/SPIKE-SORTKEY.md @@ -0,0 +1,71 @@ +# Sort-key / `read_in_order` spike (ClickStack Phase 3) + +**Question:** ClickStack changed their primary key to `(toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp)`, activating `optimize_read_in_order` for `ORDER BY Timestamp DESC LIMIT N`. Maple's `traces`/`logs` sort keys are `(OrgId, ServiceName, …)`, so the list queries use a **two-stage cutoff workaround** (`logsListQuery` / `tracesListQuery`). Would a time-bucketed key let us **delete that workaround** without regressing service-scoped reads, attribute-bloom pruning, or compression? + +This is a **measure-then-decide** spike. It ships **no production change** by itself — it builds throwaway shadow tables, backfills one busy org, and A/Bs them with the committed benchmark suite. + +## Prerequisites + +- A **dedicated ClickHouse** (docker `clickhouse-server`, or a staging cluster) reachable via `CLICKHOUSE_URL`. **Not** a Tinybird branch — branches share prod compute, so wall-time is noisy; lean on `read rows` / `read bytes`, which are deterministic. +- The base `traces` / `logs` tables populated with real data for the target org (restore a partition, or replicate from prod into the dedicated CH). +- One **busy org id** and its **7-day window**. + +## Candidate sort keys + +Defined in [`sortkey-variants.ts`](sortkey-variants.ts). OrgId stays first in every candidate (tenant scoping is enforced on every query). + +| Shadow | ORDER BY | Trade-off | +| --- | --- | --- | +| `traces_t1` / `logs_l1` | current key (rebuilt) | **control** — compare against a like-aged table, never the aged prod one | +| `traces_t2` / `logs_l2` | `(OrgId, toStartOfFiveMinutes(ts), ServiceName, ts)` | time bucket **+** service locality | +| `traces_t3` / `logs_l3` | `(OrgId, ts)` | pure time-first — the read_in_order upper bound | + +## Procedure + +```bash +export CLICKHOUSE_URL=http://localhost:8123 # dedicated CH +ORG=org_xxxxx ; END="2026-01-08 00:00:00" ; DAYS=7 +CH() { curl -sS "$CLICKHOUSE_URL" --data-binary @- ; } # or clickhouse-client -mn < + +# 1. Create shadow tables (CREATE ... AS copies columns + skip indexes). +bun run scripts/spike/emit-spike-sql.ts --tables all --mode setup | CH + +# 2. Backfill 7 days of one org, one INSERT per day per table. +bun run scripts/spike/emit-spike-sql.ts --org $ORG --days $DAYS --end "$END" --mode backfill | CH + +# 3. Confirm read_in_order actually engaged for each candidate (EXPLAIN). +bun run scripts/spike/emit-spike-sql.ts --org $ORG --mode probe | CH +# Look for ReadFromMergeTree WITHOUT a separate Sorting step on t2/t3/l2/l3 +# (and its presence on t1/l1). EXPLAIN PIPELINE via `bench inspect` also shows it. + +# 4. A/B the benchmark suite: control vs each candidate. Same org + window. +bun bench:suite --org $ORG --since ${DAYS}d --trace-id --service \ + --table-map "traces=traces_t1,logs=logs_l1" --out .bench/ctrl.json +bun bench:suite --org $ORG --since ${DAYS}d --trace-id --service \ + --table-map "traces=traces_t2,logs=logs_l2" --out .bench/t2.json +bun bench:run .bench/ctrl.json --out .bench/resCtrl.json +bun bench:run .bench/t2.json --out .bench/resT2.json +bun bench:compare .bench/resCtrl.json .bench/resT2.json + +# 5. (Repeat 4 for t3/l3.) Tear down. +bun run scripts/spike/emit-spike-sql.ts --tables all --mode drop | CH +``` + +To test whether the **two-stage workaround can be deleted**, also hand-run a single-stage `ORDER BY DESC LIMIT N` on the candidate table and compare its `read rows` / wall-time to the two-stage form on the control — that difference is the whole point. + +## Decision criteria (promote `t2`/`l2` only if ALL hold) + +- **List win:** list-group p50 −40% **or** `read_rows` −80% vs control, **and** the single-stage form ≤ the two-stage-on-control form. +- **Service-scoped raw reads** (`service_span_search`, `top_operations`, MV-disqualified timeseries) regress ≤ **+25% p50** / **+50% read_rows**. +- **Attribute-bloom pruning** (`traces_list_attr_equals`) regresses ≤ **+25% read_rows** (the item/attr blooms must still skip). +- **Compression:** compressed size (`system.parts`) ≤ **+15%**. +- **MV-backed cases** (facets, autocomplete) stay flat — they don't read the raw tables, so they're a sanity check that the harness is measuring what we think. + +If `t3`/`l3` (pure time-first) wins big but `t2`/`l2` doesn't, the service-locality cost is real and the decision is a genuine trade-off — capture the numbers and escalate rather than auto-promoting. + +## If it's a go (separate production PR) + +- Tinybird: change `sortingKey` in `datasources.ts` (needs-verification: tb deploy semantics for a key change = full table rewrite). +- BYO: a `000X` migration that rebuilds + `RENAME`s (immutable sort key, like migration 0004's aggregate rebuild). +- Delete the two-stage cutoff in `logsListQuery` / `tracesListQuery` if the single-stage form proved competitive. +- `logs.TimestampTime` elimination is a **further** scoped change (partition key + Rust insert mappings + MV + `rawLogsTimeRange`); `logs_l3` measures whether it's worth it. diff --git a/apps/api/scripts/spike/emit-spike-sql.ts b/apps/api/scripts/spike/emit-spike-sql.ts new file mode 100644 index 000000000..df0c79b9b --- /dev/null +++ b/apps/api/scripts/spike/emit-spike-sql.ts @@ -0,0 +1,120 @@ +#!/usr/bin/env bun +// --------------------------------------------------------------------------- +// spike/emit-spike-sql.ts — emit the sort-key spike SQL to stdout +// +// Dependency-free (plain bun + process.argv) so it runs in a fresh worktree +// without `bun install`. Pipe the output at a DEDICATED ClickHouse (docker or +// staging), never a Tinybird branch — see spike/SPIKE-SORTKEY.md. +// +// bun run scripts/spike/emit-spike-sql.ts --org org_x --days 7 --mode setup > setup.sql +// bun run scripts/spike/emit-spike-sql.ts --org org_x --days 7 --mode backfill > backfill.sql +// bun run scripts/spike/emit-spike-sql.ts --org org_x --mode probe > probe.sql +// bun run scripts/spike/emit-spike-sql.ts --mode drop > drop.sql +// --------------------------------------------------------------------------- + +import { + ALL_VARIANTS, + LOGS_VARIANTS, + TRACES_VARIANTS, + backfillSQL, + dropSQL, + readInOrderProbeSQL, + shadowTableDDL, + type SortKeyVariant, +} from "./sortkey-variants" + +type Mode = "setup" | "backfill" | "probe" | "drop" +type TableSet = "traces" | "logs" | "all" + +interface Args { + org: string + days: number + end: string // ClickHouse datetime "YYYY-MM-DD HH:MM:SS" + tables: TableSet + mode: Mode +} + +const usage = `Usage: bun run scripts/spike/emit-spike-sql.ts [flags] + --org Org to backfill/probe (required for backfill and probe) + --days Backfill window in days (default 7) + --end
Window end, "YYYY-MM-DD HH:MM:SS" UTC (default: now) + --tables traces | logs | all (default all) + --mode setup | backfill | probe | drop (default setup)` + +const pad = (n: number): string => String(n).padStart(2, "0") +const fmt = (d: Date): string => + `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ` + + `${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}` + +function parseArgs(argv: readonly string[]): Args { + const get = (flag: string): string | undefined => { + const i = argv.indexOf(flag) + return i >= 0 && i + 1 < argv.length ? argv[i + 1] : undefined + } + const mode = (get("--mode") ?? "setup") as Mode + const tables = (get("--tables") ?? "all") as TableSet + const days = Number(get("--days") ?? "7") + const end = get("--end") ?? fmt(new Date()) + const org = get("--org") ?? "" + if (!["setup", "backfill", "probe", "drop"].includes(mode)) throw new Error(`bad --mode "${mode}"\n${usage}`) + if (!["traces", "logs", "all"].includes(tables)) throw new Error(`bad --tables "${tables}"\n${usage}`) + if (!Number.isFinite(days) || days <= 0) throw new Error(`bad --days "${days}"\n${usage}`) + if ((mode === "backfill" || mode === "probe") && org === "") throw new Error(`--org is required for --mode ${mode}\n${usage}`) + return { org, days, end, tables, mode } +} + +function variantsFor(tables: TableSet): readonly SortKeyVariant[] { + if (tables === "traces") return TRACES_VARIANTS + if (tables === "logs") return LOGS_VARIANTS + return ALL_VARIANTS +} + +/** Per-day windows [end - (i+1)d, end - i*d) for i in 0..days-1, oldest last. */ +export function dayWindows(end: string, days: number): Array<{ start: string; end: string }> { + const endMs = Date.parse(`${end.replace(" ", "T")}Z`) + const dayMs = 86_400_000 + const windows: Array<{ start: string; end: string }> = [] + for (let i = 0; i < days; i++) { + const winEnd = new Date(endMs - i * dayMs) + const winStart = new Date(endMs - (i + 1) * dayMs) + windows.push({ start: fmt(winStart), end: fmt(winEnd) }) + } + return windows +} + +export function emitSpikeSql(args: Args): string { + const variants = variantsFor(args.tables) + const header = `-- sort-key spike (${args.mode}) — tables=${args.tables}${args.org ? ` org=${args.org}` : ""}` + const blocks: string[] = [header, ""] + + if (args.mode === "setup") { + for (const v of variants) blocks.push(`-- ${v.name}: ${v.label}`, `${shadowTableDDL(v)};`, "") + } else if (args.mode === "backfill") { + const windows = dayWindows(args.end, args.days) + blocks.push(`-- ${args.days}-day backfill, one INSERT per day per table (oldest last)`, "") + for (const v of variants) { + blocks.push(`-- ${v.name}`) + for (const w of windows) { + blocks.push(`${backfillSQL(v, { orgId: args.org, startInclusive: w.start, endExclusive: w.end })};`) + } + blocks.push("") + } + } else if (args.mode === "probe") { + blocks.push("-- read_in_order probe: does ORDER BY DESC read straight from the sort key?", "") + for (const v of variants) blocks.push(`-- ${v.name}: ${v.label}`, `${readInOrderProbeSQL(v, { orgId: args.org })};`, "") + } else { + for (const v of variants) blocks.push(`${dropSQL(v)};`) + } + + return blocks.join("\n") +} + +if (import.meta.main) { + try { + const args = parseArgs(process.argv.slice(2)) + process.stdout.write(emitSpikeSql(args) + "\n") + } catch (err) { + process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`) + process.exit(1) + } +} diff --git a/apps/api/scripts/spike/sortkey-variants.test.ts b/apps/api/scripts/spike/sortkey-variants.test.ts new file mode 100644 index 000000000..bf0b2f731 --- /dev/null +++ b/apps/api/scripts/spike/sortkey-variants.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest" +import { + backfillSQL, + dropSQL, + LOGS_VARIANTS, + readInOrderProbeSQL, + shadowTableDDL, + TRACES_VARIANTS, +} from "./sortkey-variants" +import { dayWindows, emitSpikeSql } from "./emit-spike-sql" + +const t2 = TRACES_VARIANTS.find((v) => v.name === "traces_t2")! +const l1 = LOGS_VARIANTS.find((v) => v.name === "logs_l1")! + +describe("shadowTableDDL", () => { + it("copies the base schema via AS and overrides the sort key", () => { + const ddl = shadowTableDDL(t2) + expect(ddl).toContain("CREATE TABLE IF NOT EXISTS traces_t2 AS traces") + expect(ddl).toContain("PARTITION BY toDate(Timestamp)") + expect(ddl).toContain( + "ORDER BY (OrgId, toStartOfFiveMinutes(toDateTime(Timestamp)), ServiceName, toDateTime(Timestamp))", + ) + // No TTL on throwaway shadow tables. + expect(ddl).not.toContain("TTL") + }) + + it("keeps OrgId first in every candidate (multi-tenant scoping)", () => { + for (const v of [...TRACES_VARIANTS, ...LOGS_VARIANTS]) { + expect(v.orderBy.startsWith("(OrgId,"), `${v.name} must start with OrgId`).toBe(true) + } + }) +}) + +describe("backfillSQL", () => { + it("windows on the variant's ts column and escapes the org id", () => { + const sql = backfillSQL(t2, { orgId: "org_x", startInclusive: "2026-01-07 00:00:00", endExclusive: "2026-01-08 00:00:00" }) + expect(sql).toContain("INSERT INTO traces_t2") + expect(sql).toContain("SELECT * FROM traces") + expect(sql).toContain("OrgId = 'org_x'") + expect(sql).toContain("Timestamp >= toDateTime('2026-01-07 00:00:00')") + expect(sql).toContain("Timestamp < toDateTime('2026-01-08 00:00:00')") + }) + + it("uses TimestampTime for logs variants", () => { + const sql = backfillSQL(l1, { orgId: "o", startInclusive: "2026-01-07 00:00:00", endExclusive: "2026-01-08 00:00:00" }) + expect(sql).toContain("TimestampTime >= toDateTime(") + }) +}) + +describe("readInOrderProbeSQL / dropSQL", () => { + it("probes ORDER BY DESC with EXPLAIN indexes", () => { + const sql = readInOrderProbeSQL(t2, { orgId: "org_x", limit: 25 }) + expect(sql).toContain("EXPLAIN actions = 1, indexes = 1") + expect(sql).toContain("ORDER BY Timestamp DESC") + expect(sql).toContain("LIMIT 25") + }) + + it("drops shadow tables idempotently", () => { + expect(dropSQL(t2)).toBe("DROP TABLE IF EXISTS traces_t2") + }) +}) + +describe("dayWindows", () => { + it("produces N contiguous 1-day windows, most-recent first", () => { + const windows = dayWindows("2026-01-08 00:00:00", 3) + expect(windows).toEqual([ + { start: "2026-01-07 00:00:00", end: "2026-01-08 00:00:00" }, + { start: "2026-01-06 00:00:00", end: "2026-01-07 00:00:00" }, + { start: "2026-01-05 00:00:00", end: "2026-01-06 00:00:00" }, + ]) + }) +}) + +describe("emitSpikeSql", () => { + it("setup emits a CREATE per variant", () => { + const sql = emitSpikeSql({ org: "", days: 7, end: "2026-01-08 00:00:00", tables: "traces", mode: "setup" }) + expect(sql).toContain("CREATE TABLE IF NOT EXISTS traces_t1 AS traces") + expect(sql).toContain("CREATE TABLE IF NOT EXISTS traces_t2 AS traces") + expect(sql).toContain("CREATE TABLE IF NOT EXISTS traces_t3 AS traces") + }) + + it("backfill emits one INSERT per day per variant", () => { + const sql = emitSpikeSql({ org: "org_x", days: 2, end: "2026-01-08 00:00:00", tables: "traces", mode: "backfill" }) + // 3 traces variants × 2 days = 6 INSERTs. + expect(sql.match(/INSERT INTO/g)).toHaveLength(6) + }) + + it("drop emits one DROP per variant across all tables", () => { + const sql = emitSpikeSql({ org: "", days: 7, end: "2026-01-08 00:00:00", tables: "all", mode: "drop" }) + expect(sql.match(/DROP TABLE IF EXISTS/g)).toHaveLength(6) + }) +}) diff --git a/apps/api/scripts/spike/sortkey-variants.ts b/apps/api/scripts/spike/sortkey-variants.ts new file mode 100644 index 000000000..35027a1cd --- /dev/null +++ b/apps/api/scripts/spike/sortkey-variants.ts @@ -0,0 +1,147 @@ +// --------------------------------------------------------------------------- +// spike/sortkey-variants.ts — shadow-table definitions for the sort-key spike +// +// Phase 3 of the ClickStack work: measure whether a time-bucketed primary key +// activates `optimize_read_in_order` for `ORDER BY DESC LIMIT N` list +// queries (which today use a two-stage cutoff workaround because the sort key +// is `(OrgId, ServiceName, …)`), without regressing service-scoped reads, +// attribute-bloom pruning, or compression. +// +// This is measurement scaffolding, NOT a production change: it builds throwaway +// shadow tables with candidate sort keys, backfills a bounded window of one +// org, and hands off to the `bench suite --table-map` A/B harness. Everything +// here is a pure SQL-string builder (no IO, no deps) so it's unit-testable and +// runnable with bare `bun`. +// --------------------------------------------------------------------------- + +export interface SortKeyVariant { + /** Shadow table name, e.g. `traces_t2`. */ + readonly name: string + /** Production source table the shadow copies from and backfills out of. */ + readonly base: string + /** Human-readable description of what this sort key trades off. */ + readonly label: string + /** PARTITION BY expression (kept identical to the base table). */ + readonly partitionBy: string + /** Candidate ORDER BY (the thing under test). */ + readonly orderBy: string + /** Column used to window the backfill (Timestamp for traces, TimestampTime for logs). */ + readonly tsColumn: string +} + +// Traces base: PARTITION BY toDate(Timestamp), +// ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)). +export const TRACES_VARIANTS: readonly SortKeyVariant[] = [ + { + name: "traces_t1", + base: "traces", + label: "control — current key, freshly rebuilt (compare like-aged, not vs the aged prod table)", + partitionBy: "toDate(Timestamp)", + orderBy: "(OrgId, ServiceName, SpanName, toDateTime(Timestamp))", + tsColumn: "Timestamp", + }, + { + name: "traces_t2", + base: "traces", + label: "5-min time bucket + service locality (OrgId stays first for tenant scoping)", + partitionBy: "toDate(Timestamp)", + orderBy: "(OrgId, toStartOfFiveMinutes(toDateTime(Timestamp)), ServiceName, toDateTime(Timestamp))", + tsColumn: "Timestamp", + }, + { + name: "traces_t3", + base: "traces", + label: "pure time-first (max read_in_order, no service locality) — the upper bound", + partitionBy: "toDate(Timestamp)", + orderBy: "(OrgId, toDateTime(Timestamp))", + tsColumn: "Timestamp", + }, +] + +// Logs base: PARTITION BY toDate(TimestampTime), +// ORDER BY (OrgId, ServiceName, TimestampTime, Timestamp). +export const LOGS_VARIANTS: readonly SortKeyVariant[] = [ + { + name: "logs_l1", + base: "logs", + label: "control — current key", + partitionBy: "toDate(TimestampTime)", + orderBy: "(OrgId, ServiceName, TimestampTime, Timestamp)", + tsColumn: "TimestampTime", + }, + { + name: "logs_l2", + base: "logs", + label: "5-min time bucket + service locality", + partitionBy: "toDate(TimestampTime)", + orderBy: "(OrgId, toStartOfFiveMinutes(TimestampTime), ServiceName, Timestamp)", + tsColumn: "TimestampTime", + }, + { + name: "logs_l3", + base: "logs", + label: "pure time-first — the upper bound (also informs the TimestampTime-elimination question)", + partitionBy: "toDate(TimestampTime)", + orderBy: "(OrgId, TimestampTime, Timestamp)", + tsColumn: "TimestampTime", + }, +] + +export const ALL_VARIANTS: readonly SortKeyVariant[] = [...TRACES_VARIANTS, ...LOGS_VARIANTS] + +/** Single-quote-escape a string literal for inline SQL (org ids are trusted internal ids, but be safe). */ +const lit = (s: string): string => `'${s.replace(/'/g, "\\'")}'` + +/** + * `CREATE TABLE AS ` copies the full column structure AND skip + * indexes from the base, so we only override the engine clause to swap the sort + * key. No TTL — shadow tables are throwaway; drop them after the spike. + */ +export function shadowTableDDL(v: SortKeyVariant): string { + return ( + `CREATE TABLE IF NOT EXISTS ${v.name} AS ${v.base}\n` + + `ENGINE = MergeTree\n` + + `PARTITION BY ${v.partitionBy}\n` + + `ORDER BY ${v.orderBy}` + ) +} + +/** + * One day's chunk of backfill. `SELECT *` works because the shadow shares the + * base's exact column list (via `AS`). Windowed on the sort/partition ts column + * so the apply engine (or a shell loop) can split a wide range into per-day + * INSERTs — a single INSERT…SELECT over a busy org's week is too heavy. + */ +export function backfillSQL( + v: SortKeyVariant, + opts: { orgId: string; startInclusive: string; endExclusive: string }, +): string { + return ( + `INSERT INTO ${v.name}\n` + + `SELECT * FROM ${v.base}\n` + + `WHERE OrgId = ${lit(opts.orgId)}\n` + + ` AND ${v.tsColumn} >= toDateTime(${lit(opts.startInclusive)})\n` + + ` AND ${v.tsColumn} < toDateTime(${lit(opts.endExclusive)})` + ) +} + +export function dropSQL(v: SortKeyVariant): string { + return `DROP TABLE IF EXISTS ${v.name}` +} + +/** + * EXPLAIN a `ORDER BY DESC LIMIT N` list query against a shadow table to + * see whether `optimize_read_in_order` engaged (look for `ReadFromMergeTree` + * with a sort description / no separate sorting step). This is the question + * that decides whether the two-stage cutoff workaround can be deleted. + */ +export function readInOrderProbeSQL(v: SortKeyVariant, opts: { orgId: string; limit?: number }): string { + const limit = opts.limit ?? 50 + return ( + `EXPLAIN actions = 1, indexes = 1\n` + + `SELECT * FROM ${v.name}\n` + + `WHERE OrgId = ${lit(opts.orgId)}\n` + + `ORDER BY ${v.tsColumn} DESC\n` + + `LIMIT ${limit}` + ) +}