From d12a2223ebc2bcdc308a98a8181a95d0aa56f432 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 4 Jul 2026 00:42:59 +0200 Subject: [PATCH 1/7] perf(query-engine): ClickStack-style skip indexes + token-aware log search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply optimizations from ClickHouse's "Making ClickStack 5x faster" to Maple's telemetry read path: - logs.Body gains a tokenbf_v1(32768,3,0) index on lower(Body); body search AND-s an interior-token `hasToken(lower(Body), t)` pre-filter ahead of the substring ILIKE. Only separator-bounded, ASCII, interior tokens qualify, so the result set is byte-for-byte unchanged — the index just prunes granules. - logs gains the four attribute-map bloom filters that traces already had. - traces gains ClickStack "Items" bloom indexes over arrayMap((k,v)->concat(k,'=',v), ...) so attribute-equality filters prune via has(items,'k=v') (AND-ed with the exact map equality; enabled only on the raw traces table, not trace_detail_spans or metrics). - Index expressions live in one shared module (index-exprs.ts) imported by the datasource, the migration, and the query predicate so they can't drift. New committed benchmark suite (bench suite): 13 cases compiled from the real query-engine DSL, plus `--table-map` for A/B against shadow tables and `compare --max-regression-pct` as a CI regression gate. BYO migration 0005 adds every index with ADD INDEX (no MATERIALIZE — new parts only; TTL churn backfills), bumping SCHEMA_VERSION to 5. Tests: builder 72, domain 174, query-engine 727, apps/api bench 11; repo typecheck 24/24. Pre-merge gates (need a live warehouse): - Tinybird `tb --cloud deploy --check` on a branch to confirm managed Tinybird accepts tokenbf_v1 + the arrayMap expression index as an in-place ADD INDEX. - Live `EXPLAIN indexes=1` granule-drop check on a dev ClickHouse. Co-Authored-By: Claude Fable 5 --- apps/api/package.json | 1 + apps/api/scripts/BENCH.md | 51 +- apps/api/scripts/bench-queries.ts | 192 +++++- apps/api/scripts/bench/suite.test.ts | 54 ++ apps/api/scripts/bench/suite.ts | 153 +++++ apps/api/scripts/bench/table-map.test.ts | 48 ++ apps/api/scripts/bench/table-map.ts | 50 ++ apps/api/vitest.config.ts | 2 +- apps/cli/src/server/schema/local-schema.sql | 17 +- apps/ingest/src/clickhouse_insert_mappings.rs | 4 +- lib/clickhouse-builder/src/ch/expr.ts | 2 + .../src/ch/functions/array.ts | 8 + .../src/ch/functions/index.ts | 3 +- .../src/ch/functions/string.ts | 10 +- lib/clickhouse-builder/src/ch/index.ts | 3 + packages/domain/package.json | 1 + .../migrations/0005_body_and_attr_indexes.ts | 49 ++ .../src/clickhouse/migrations/index.test.ts | 20 +- .../domain/src/clickhouse/migrations/index.ts | 2 + .../domain/src/generated/clickhouse-schema.ts | 8 +- .../generated/tinybird-project-manifest.ts | 584 ++++++++---------- packages/domain/src/tinybird/datasources.ts | 55 ++ packages/domain/src/tinybird/index-exprs.ts | 33 + .../src/ch/queries/body-search.test.ts | 64 ++ .../src/ch/queries/body-search.ts | 80 +++ packages/query-engine/src/ch/queries/logs.ts | 5 +- .../src/ch/queries/query-helpers.ts | 9 +- .../src/ch/queries/traces.test.ts | 73 +++ .../query-engine/src/ch/queries/traces.ts | 13 +- packages/query-engine/src/traces-shared.ts | 37 +- 30 files changed, 1277 insertions(+), 354 deletions(-) create mode 100644 apps/api/scripts/bench/suite.test.ts create mode 100644 apps/api/scripts/bench/suite.ts create mode 100644 apps/api/scripts/bench/table-map.test.ts create mode 100644 apps/api/scripts/bench/table-map.ts create mode 100644 packages/domain/src/clickhouse/migrations/0005_body_and_attr_indexes.ts create mode 100644 packages/domain/src/tinybird/index-exprs.ts create mode 100644 packages/query-engine/src/ch/queries/body-search.test.ts create mode 100644 packages/query-engine/src/ch/queries/body-search.ts diff --git a/apps/api/package.json b/apps/api/package.json index 4c28e7eef..92ce3c992 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -20,6 +20,7 @@ "tinybird:build": "tinybird build", "tinybird:deploy": "tinybird deploy", "bench:fetch": "bun run scripts/bench-queries.ts fetch", + "bench:suite": "bun run scripts/bench-queries.ts suite", "bench:run": "bun run scripts/bench-queries.ts run", "bench:inspect": "bun run scripts/bench-queries.ts inspect", "bench:compare": "bun run scripts/bench-queries.ts compare", diff --git a/apps/api/scripts/BENCH.md b/apps/api/scripts/BENCH.md index cae7d8a58..d1601b1eb 100644 --- a/apps/api/scripts/BENCH.md +++ b/apps/api/scripts/BENCH.md @@ -15,18 +15,65 @@ bun bench:fetch [--context name] [--profile name] [--since 24h] [--top 20] [--out path] [--org id] # mine recent db.query.text spans from prod traces → JSON +bun bench:suite [--org id] [--since 24h] [--trace-id id] [--service name] + [--search phrase] [--attr-key k] [--attr-value v] + [--attr-scope span] [--table-map from=to,...] [--out path] + # compile the committed DSL benchmark suite → JSON (same + # shape as `fetch`, so `run`/`inspect`/`compare` replay it) + bun bench:run [--runs 5] [--warmup 1] [--out path] # replay each query N times and report aggregated stats bun bench:inspect # run EXPLAIN and EXPLAIN PIPELINE for each query -bun bench:compare - # diff two run outputs (p95 wall, read bytes, memory) +bun bench:compare [--max-regression-pct N] + # diff two run outputs (p95 wall, read bytes, memory); + # exits non-zero if any case's p95 regresses beyond N% ``` Output JSONs land in `apps/api/scripts/.bench/` by default (gitignored). +## Two ways to get a query set + +- **`fetch`** mines whatever prod actually ran for a context label — great for + chasing a specific slow query, but not repeatable (the fingerprints drift as + traffic changes). +- **`suite`** compiles a **committed, repeatable** set of ~13 representative + cases (needle-in-haystack body search, trace point lookup, attribute + equality/contains filters, list scans, facets, autocomplete, plus canaries) + straight from the `@maple/query-engine` DSL, so the benchmark runs the exact + SQL production emits. Definitions live in + [scripts/bench/suite.ts](bench/suite.ts). Pass `--trace-id` / `--service` / + `--attr-key` / `--attr-value` that exist in the target window for + representative timing — the two lookup cases fall back to zero-row + placeholders (with a warning) otherwise. + +## A/B: schema-change spike (e.g. sort-key experiment) + +`suite --table-map` rewrites `FROM`/`JOIN` table names in the compiled SQL so +you can benchmark the same queries against shadow tables without recompiling. +Rules are whole-identifier and longest-first, so `traces=traces_t2` never +touches `trace_detail_spans` or `traces_aggregates_hourly`. + +``` +# control vs candidate tables, same window/org +bun bench:suite --org org_x --trace-id --service \ + --out .bench/suiteA.json +bun bench:suite --org org_x --trace-id --service \ + --table-map "traces=traces_t2,logs=logs_l2" --out .bench/suiteB.json + +bun bench:run .bench/suiteA.json --out .bench/resA.json # CLICKHOUSE_URL = dedicated CH +bun bench:run .bench/suiteB.json --out .bench/resB.json +bun bench:compare .bench/resA.json .bench/resB.json --max-regression-pct 25 +``` + +Run A/B against a **dedicated ClickHouse** (docker/staging), not a Tinybird +branch — Tinybird branches share prod compute, so wall-time is noisy (lean on +`read rows`/`read bytes`, which are deterministic). The `inspect` command's +`EXPLAIN PIPELINE` tells you whether `optimize_read_in_order` actually engaged +for the candidate sort key. + ## Implementation Built on Effect v4 end-to-end: diff --git a/apps/api/scripts/bench-queries.ts b/apps/api/scripts/bench-queries.ts index 76ac76eba..180d459a3 100644 --- a/apps/api/scripts/bench-queries.ts +++ b/apps/api/scripts/bench-queries.ts @@ -39,6 +39,8 @@ import { import { Argument, Command, Flag } from "effect/unstable/cli" import { BunRuntime, BunServices } from "@effect/platform-bun" import { CH } from "@maple/query-engine" +import { buildSuite } from "./bench/suite" +import { parseTableMap, remapTables } from "./bench/table-map" // --------------------------------------------------------------------------- // Errors @@ -86,6 +88,13 @@ class InvalidDurationError extends Schema.TaggedErrorClass }, ) {} +class RegressionThresholdError extends Schema.TaggedErrorClass()( + "@maple/api/scripts/bench-queries/RegressionThresholdError", + { + message: Schema.String, + }, +) {} + // --------------------------------------------------------------------------- // Internal data shapes (typed JSON; not branded — local dev tool) // --------------------------------------------------------------------------- @@ -598,7 +607,7 @@ const fetchHandler = Effect.fn("bench.fetch")(function* (config: FetchConfig) { () => `apps/api/scripts/.bench/queries-${timestampSlug()}.json`, ) const output: FetchOutput = { - fetchedAt: now.toISOString(), + fetchedAt: new Date(nowMs).toISOString(), source: host, criteria: { orgId, @@ -638,6 +647,108 @@ const fetchHandler = Effect.fn("bench.fetch")(function* (config: FetchConfig) { yield* Console.log(`\nWrote ${outputPath}`) }) +// Non-empty sentinels so the point-lookup and service-scoped cases keep their +// query SHAPE (right table + filter) even when the user doesn't pass real +// values. They match zero rows, so timing for those two cases is only +// representative once real values are supplied — the handler warns when unset. +const SUITE_PLACEHOLDER_TRACE_ID = "REPLACE_WITH_REAL_TRACE_ID" +const SUITE_PLACEHOLDER_SERVICE = "REPLACE_WITH_REAL_SERVICE" + +interface SuiteConfig { + readonly org: Option.Option + readonly since: string + readonly traceId: string + readonly service: string + readonly search: string + readonly attrKey: string + readonly attrValue: string + readonly attrScope: string + readonly tableMap: Option.Option + readonly out: Option.Option +} + +// Compile the committed suite into a `fetch`-shaped JSON file so the existing +// `run` / `inspect` / `compare` commands replay it unchanged. For the sort-key +// A/B spike, run this twice with different `--table-map` targets and diff the +// two `run` outputs. +const suiteHandler = Effect.fn("bench.suite")(function* (config: SuiteConfig) { + const sinceMs = yield* parseRelativeDuration(config.since) + const nowMs = yield* Clock.currentTimeMillis + const startTime = formatCHDateTime(new Date(nowMs - sinceMs)) + const endTime = formatCHDateTime(new Date(nowMs)) + const orgId = Option.getOrElse(config.org, () => "internal") + + const tableMap = yield* Effect.try({ + try: () => + parseTableMap(Option.match(config.tableMap, { onNone: () => [], onSome: (s) => s.split(",") })), + catch: (cause) => new MissingConfigError({ what: "--table-map", message: String(cause) }), + }) + + const cases = buildSuite({ + traceId: config.traceId, + serviceName: config.service, + searchTerm: config.search, + attrKey: config.attrKey, + attrValue: config.attrValue, + attrScope: config.attrScope, + }) + + const samples: ReadonlyArray = cases.map((c) => ({ + fingerprint: c.name, + context: c.name, + profile: c.profile, + sampleSql: remapTables(c.compile({ orgId, startTime, endTime }), tableMap), + sampleCount: 0, + p50DurationMs: 0, + p95DurationMs: 0, + p99DurationMs: 0, + maxDurationMs: 0, + })) + + const remapNote = Option.match(config.tableMap, { + onNone: () => "", + onSome: (s) => ` table-map: ${s}`, + }) + const source = `suite (org=${orgId}, window=${config.since})${remapNote}` + const outputPath = Option.getOrElse( + config.out, + () => `apps/api/scripts/.bench/suite-${timestampSlug()}.json`, + ) + const output: FetchOutput = { + fetchedAt: new Date(nowMs).toISOString(), + source, + criteria: { orgId, startTime, endTime, topN: cases.length }, + samples, + } + yield* writeJsonFile(outputPath, output) + + yield* Console.log( + renderTable( + `Benchmark suite (${cases.length} cases)`, + [ + { header: "case", width: 28 }, + { header: "profile", width: 12 }, + { header: "description", width: 56 }, + ], + cases.map((c) => [c.name, c.profile, c.description]), + ), + ) + yield* Console.log(` org: ${orgId} window: ${startTime} → ${endTime} (${config.since})${remapNote}`) + + const placeholders: string[] = [] + if (config.traceId === SUITE_PLACEHOLDER_TRACE_ID) placeholders.push("--trace-id (trace_point_lookup)") + if (config.service === SUITE_PLACEHOLDER_SERVICE) placeholders.push("--service (service_span_search)") + if (placeholders.length > 0) { + yield* Console.log( + ` ⚠ using placeholder ${placeholders.join(" and ")} — those cases match 0 rows; ` + + "pass real values for representative timing.", + ) + } + + yield* Console.log(`\nWrote ${outputPath}`) + yield* Console.log(` Replay: bun run scripts/bench-queries.ts run ${outputPath}`) +}) + const stripTrailingSemi = (sql: string) => sql.replace(/;\s*$/, "") const failedResult = (sample: Sample, message: string): SampleResult => ({ @@ -871,6 +982,7 @@ const inspectHandler = Effect.fn("bench.inspect")(function* (config: { readonly const compareHandler = Effect.fn("bench.compare")(function* (config: { readonly baseline: string readonly candidate: string + readonly maxRegressionPct: Option.Option }) { const a = yield* readJsonFile(config.baseline) const b = yield* readJsonFile(config.candidate) @@ -915,6 +1027,33 @@ const compareHandler = Effect.fn("bench.compare")(function* (config: { rows, ), ) + + // Optional CI gate: fail if any case's candidate p95 regresses beyond the + // threshold vs the baseline. Cases missing from either side are ignored. + if (Option.isSome(config.maxRegressionPct)) { + const limit = config.maxRegressionPct.value + const regressions = a.results.flatMap((aResult) => { + const bResult = bByFp.get(aResult.fingerprint) + if (!bResult) return [] + const base = aResult.aggregates.p95WallMs + const cand = bResult.aggregates.p95WallMs + if (!Number.isFinite(base) || !Number.isFinite(cand) || base <= 0) return [] + const pct = ((cand - base) / base) * 100 + return pct > limit ? [{ name: aResult.context || aResult.fingerprint, pct }] : [] + }) + if (regressions.length > 0) { + const detail = regressions + .map((r) => ` ${r.name}: +${r.pct.toFixed(1)}% p95 (limit +${limit}%)`) + .join("\n") + yield* Console.log(`\n${regressions.length} case(s) regressed beyond +${limit}%:\n${detail}`) + return yield* Effect.fail( + new RegressionThresholdError({ + message: `${regressions.length} case(s) exceeded the +${limit}% p95 regression threshold`, + }), + ) + } + yield* Console.log(`\nNo case regressed beyond +${limit}% p95.`) + } }) // --------------------------------------------------------------------------- @@ -968,13 +1107,62 @@ const compareCommand = Command.make( { baseline: Argument.string("baseline").pipe(Argument.withDescription("Baseline results JSON")), candidate: Argument.string("candidate").pipe(Argument.withDescription("Candidate results JSON")), + maxRegressionPct: Flag.integer("max-regression-pct").pipe( + Flag.withDescription("Exit non-zero if any case's p95 regresses beyond this percent"), + Flag.optional, + ), }, compareHandler, ).pipe(Command.withDescription("Diff two run outputs (p95 wall, read bytes, memory)")) +const suiteCommand = Command.make( + "suite", + { + org: Flag.string("org").pipe(Flag.withDescription("Org id to scope the queries (default: internal)"), Flag.optional), + since: Flag.string("since").pipe( + Flag.withDescription("Time window the queries scan, e.g. 24h or 7d"), + Flag.withDefault("24h"), + ), + traceId: Flag.string("trace-id").pipe( + Flag.withDescription("Existing trace id for the point-lookup case (pass a real one for meaningful timing)"), + Flag.withDefault(SUITE_PLACEHOLDER_TRACE_ID), + ), + service: Flag.string("service").pipe( + Flag.withDescription("Service name for the service-scoped canary (pass a real one for meaningful timing)"), + Flag.withDefault(SUITE_PLACEHOLDER_SERVICE), + ), + search: Flag.string("search").pipe( + Flag.withDescription("Body-search phrase (an interior token engages the pre-filter, e.g. 'user login failed')"), + Flag.withDefault("user login failed"), + ), + attrKey: Flag.string("attr-key").pipe( + Flag.withDescription("Attribute key for the equality/contains filter cases"), + Flag.withDefault("http.request.method"), + ), + attrValue: Flag.string("attr-value").pipe( + Flag.withDescription("Attribute value for the equality/contains filter cases"), + Flag.withDefault("GET"), + ), + attrScope: Flag.string("attr-scope").pipe( + Flag.withDescription("Attribute scope for the autocomplete case (span|resource|log)"), + Flag.withDefault("span"), + ), + tableMap: Flag.string("table-map").pipe( + Flag.withDescription("Comma-separated FROM/JOIN rewrites for A/B, e.g. traces=traces_t2,logs=logs_l2"), + Flag.optional, + ), + out: Flag.string("out").pipe(Flag.withDescription("Output JSON path"), Flag.optional), + }, + suiteHandler, +).pipe( + Command.withDescription( + "Compile the committed DSL benchmark suite into a fetch-shaped JSON for `run`/`compare`", + ), +) + const rootCommand = Command.make("bench-queries").pipe( Command.withDescription("Measure ClickHouse query performance"), - Command.withSubcommands([fetchCommand, runCommand, inspectCommand, compareCommand]), + Command.withSubcommands([fetchCommand, suiteCommand, runCommand, inspectCommand, compareCommand]), ) const BenchLive = Layer.mergeAll(ClickHouse.layer, Tinybird.layer) diff --git a/apps/api/scripts/bench/suite.test.ts b/apps/api/scripts/bench/suite.test.ts new file mode 100644 index 000000000..be0973901 --- /dev/null +++ b/apps/api/scripts/bench/suite.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest" +import { buildSuite, type SuiteInputs, type SuiteParams } from "./suite" + +const inputs: SuiteInputs = { + traceId: "trace_abc123", + serviceName: "maple-api", + searchTerm: "user login failed", + attrKey: "http.request.method", + attrValue: "GET", + attrScope: "span", +} + +const params: SuiteParams = { + orgId: "org_test", + startTime: "2024-01-01 00:00:00", + endTime: "2024-01-02 00:00:00", +} + +describe("buildSuite", () => { + const cases = buildSuite(inputs) + + it("every case compiles to non-empty SQL scoped to the org", () => { + for (const c of cases) { + const sql = c.compile(params) + expect(sql.length, `${c.name} produced empty SQL`).toBeGreaterThan(0) + // Every query must be OrgId-scoped (the warehouse executor enforces this). + expect(sql, `${c.name} missing OrgId scope`).toContain("OrgId = 'org_test'") + } + }) + + it("has unique case names", () => { + const names = cases.map((c) => c.name) + expect(new Set(names).size).toBe(names.length) + }) + + it("the body-search case exercises the token index and the ILIKE tail", () => { + const sql = cases.find((c) => c.name === "logs_body_search")!.compile(params) + // "user login failed" → only the interior token "login" is index-safe. + expect(sql).toContain("hasToken(lower(Body), 'login')") + expect(sql).toContain("Body ILIKE '%user login failed%'") + }) + + it("the attribute-equality case exercises the items index", () => { + const sql = cases.find((c) => c.name === "traces_list_attr_equals")!.compile(params) + expect(sql).toContain("has(arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes)") + expect(sql).toContain("'http.request.method=GET'") + }) + + it("the point-lookup case reads the trace-detail projection", () => { + const sql = cases.find((c) => c.name === "trace_point_lookup")!.compile(params) + expect(sql).toContain("FROM trace_detail_spans") + expect(sql).toContain("TraceId = 'trace_abc123'") + }) +}) diff --git a/apps/api/scripts/bench/suite.ts b/apps/api/scripts/bench/suite.ts new file mode 100644 index 000000000..3007d7be0 --- /dev/null +++ b/apps/api/scripts/bench/suite.ts @@ -0,0 +1,153 @@ +// --------------------------------------------------------------------------- +// bench/suite.ts — committed, repeatable query benchmark suite +// +// Each case compiles a query from the SAME `@maple/query-engine` DSL that +// production runs, so the benchmark measures the exact SQL the app emits (not a +// hand-written approximation). Cases mirror ClickStack's representative +// patterns: needle-in-haystack body search, trace point lookup, attribute +// equality/contains filters, top-N list scans, facets, and autocomplete, plus a +// couple of service-scoped canaries to catch regressions from schema changes. +// +// The suite compiles to plain SQL strings, which the `bench` CLI writes into the +// same JSON shape `fetch` produces — so `run` / `inspect` / `compare` replay it +// unchanged. Keep this pure (no IO) so it stays unit-testable. +// --------------------------------------------------------------------------- + +import { CH } from "@maple/query-engine" + +/** Params substituted at compile time (the `param.*` placeholders every query shares). */ +export interface SuiteParams { + readonly orgId: string + readonly startTime: string + readonly endTime: string +} + +/** Literal inputs baked into query construction (a real trace id, a real attribute, …). */ +export interface SuiteInputs { + /** Existing trace id for the point-lookup case (bloom + trace_detail_spans path). */ + readonly traceId: string + /** A service present in the window, for the service-scoped canary. */ + readonly serviceName: string + /** Multi-word phrase so the body-search token pre-filter actually engages. */ + readonly searchTerm: string + /** Attribute key/value that occurs in the window (drives the items-index case). */ + readonly attrKey: string + readonly attrValue: string + /** Attribute scope for the autocomplete case ("span" | "resource" | "log"). */ + readonly attrScope: string +} + +export interface SuiteCase { + readonly name: string + readonly profile: string + readonly description: string + /** Compile this case's SQL for the given window/org. */ + readonly compile: (params: SuiteParams) => string +} + +/** + * Build the benchmark suite. `inputs` supplies the literal values (trace id, + * attribute, search phrase) that must exist in the target window for the query + * to touch real data — the CLI resolves sensible defaults or takes them from + * flags. + */ +export function buildSuite(inputs: SuiteInputs): ReadonlyArray { + const q = (query: Parameters[0], p: SuiteParams): string => CH.compile(query, p).sql + const u = (union: Parameters[0], p: SuiteParams): string => + CH.compileUnion(union, p).sql + + return [ + { + name: "logs_body_search", + profile: "list", + description: "Needle-in-haystack body search — exercises idx_body_tokens + ILIKE tail", + compile: (p) => q(CH.logsListQuery({ search: inputs.searchTerm, limit: 50 }), p), + }, + { + name: "logs_count_search", + profile: "list", + description: "count(*) over a body search — pure granule-pruning win from the token index", + compile: (p) => q(CH.logsCountQuery({ search: inputs.searchTerm }), p), + }, + { + name: "trace_point_lookup", + profile: "list", + description: "All spans of one trace — trace_detail_spans (OrgId,TraceId,SpanId) sort key", + compile: (p) => q(CH.spanSearchQuery({ traceId: inputs.traceId, limit: 100 }), p), + }, + { + name: "traces_list", + profile: "list", + description: "Traces list ORDER BY Timestamp DESC LIMIT N (two-stage cutoff)", + compile: (p) => q(CH.tracesListQuery({ limit: 50 }), p), + }, + { + name: "traces_list_attr_equals", + profile: "list", + description: "Traces list filtered by attribute equality — exercises idx_span_attr_items", + compile: (p) => + q( + CH.tracesListQuery({ + attributeFilters: [{ key: inputs.attrKey, value: inputs.attrValue, mode: "equals" }], + limit: 50, + }), + p, + ), + }, + { + name: "traces_list_attr_contains", + profile: "list", + description: "Traces list filtered by attribute substring — no items index (positionCaseInsensitive)", + compile: (p) => + q( + CH.tracesListQuery({ + attributeFilters: [{ key: inputs.attrKey, value: inputs.attrValue, mode: "contains" }], + limit: 50, + }), + p, + ), + }, + { + name: "logs_list", + profile: "list", + description: "Logs list ORDER BY Timestamp DESC LIMIT N (two-stage cutoff)", + compile: (p) => q(CH.logsListQuery({ limit: 50 }), p), + }, + { + name: "traces_facets", + profile: "unbounded", + description: "Trace facet breakdown (multi-branch UNION ALL)", + compile: (p) => u(CH.tracesFacetsQuery({}), p), + }, + { + name: "services_facets", + profile: "unbounded", + description: "Service facet breakdown (4-way UNION ALL over service_overview_spans)", + compile: (p) => u(CH.servicesFacetsQuery(), p), + }, + { + name: "logs_facets", + profile: "unbounded", + description: "Log facet breakdown (UNION over logs_aggregates_hourly)", + compile: (p) => u(CH.logsFacetsQuery({}), p), + }, + { + name: "attribute_keys_autocomplete", + profile: "discovery", + description: "Attribute-key autocomplete over the attribute_keys_hourly MV", + compile: (p) => q(CH.attributeKeysQuery({ scope: inputs.attrScope, limit: 200 }), p), + }, + { + name: "top_operations", + profile: "aggregation", + description: "Top operations by count — aggregation canary (should stay flat)", + compile: (p) => q(CH.topOperationsQuery({ metric: "count", limit: 20 }), p), + }, + { + name: "service_span_search", + profile: "list", + description: "Service-scoped raw span scan — sort-key-locality canary for the PK spike", + compile: (p) => q(CH.spanSearchQuery({ serviceName: inputs.serviceName, limit: 50 }), p), + }, + ] +} diff --git a/apps/api/scripts/bench/table-map.test.ts b/apps/api/scripts/bench/table-map.test.ts new file mode 100644 index 000000000..193870885 --- /dev/null +++ b/apps/api/scripts/bench/table-map.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest" +import { parseTableMap, remapTables } from "./table-map" + +describe("parseTableMap", () => { + it("parses from=to entries and skips blanks", () => { + const map = parseTableMap(["traces=traces_t2", " logs = logs_l2 ", ""]) + expect(map.get("traces")).toBe("traces_t2") + expect(map.get("logs")).toBe("logs_l2") + expect(map.size).toBe(2) + }) + + it("rejects malformed entries", () => { + expect(() => parseTableMap(["traces"])).toThrow() + expect(() => parseTableMap(["=traces_t2"])).toThrow() + expect(() => parseTableMap(["traces="])).toThrow() + }) +}) + +describe("remapTables", () => { + const map = parseTableMap(["traces=traces_t2", "logs=logs_l2"]) + + it("rewrites FROM and JOIN table positions", () => { + expect(remapTables("SELECT * FROM traces WHERE OrgId = 'x'", map)).toBe( + "SELECT * FROM traces_t2 WHERE OrgId = 'x'", + ) + expect(remapTables("SELECT * FROM a JOIN logs ON a.id = logs.id", map)).toContain("JOIN logs_l2 ON") + }) + + it("does NOT touch longer table names that share a prefix", () => { + // `traces` rule must not clobber trace_detail_spans / traces_aggregates_hourly. + expect(remapTables("SELECT * FROM trace_detail_spans", map)).toBe("SELECT * FROM trace_detail_spans") + expect(remapTables("SELECT * FROM traces_aggregates_hourly", map)).toBe( + "SELECT * FROM traces_aggregates_hourly", + ) + }) + + it("does NOT rewrite the name outside a FROM/JOIN position", () => { + // A column or literal that happens to equal a table name is left alone. + expect(remapTables("SELECT traces FROM x WHERE note = 'traces'", map)).toBe( + "SELECT traces FROM x WHERE note = 'traces'", + ) + }) + + it("is a no-op for an empty map", () => { + const sql = "SELECT * FROM traces" + expect(remapTables(sql, parseTableMap([]))).toBe(sql) + }) +}) diff --git a/apps/api/scripts/bench/table-map.ts b/apps/api/scripts/bench/table-map.ts new file mode 100644 index 000000000..3561d3303 --- /dev/null +++ b/apps/api/scripts/bench/table-map.ts @@ -0,0 +1,50 @@ +// --------------------------------------------------------------------------- +// bench/table-map.ts — rewrite table identifiers in compiled SQL for A/B runs +// +// The sort-key spike compares the same suite against shadow tables (e.g. +// `traces` vs `traces_t2`). Rather than recompile the DSL against alternate +// table names, we rewrite `FROM ` / `JOIN ` in the already-compiled +// SQL. Rules apply longest-source-name-first and are anchored on whole +// identifiers so a `traces` rule never touches `trace_detail_spans` or +// `traces_aggregates_hourly`. +// --------------------------------------------------------------------------- + +export type TableMap = ReadonlyMap + +const escapeRegExp = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + +/** + * Parse `["traces=traces_t2", "logs=logs_l2"]` (or a single comma-joined string + * split by the caller) into a source→target map. Throws on a malformed entry. + */ +export function parseTableMap(entries: ReadonlyArray): TableMap { + const map = new Map() + for (const raw of entries) { + const entry = raw.trim() + if (entry === "") continue + const eq = entry.indexOf("=") + if (eq <= 0 || eq === entry.length - 1) { + throw new Error(`Invalid --table-map entry "${raw}" (expected from=to, e.g. traces=traces_t2)`) + } + map.set(entry.slice(0, eq).trim(), entry.slice(eq + 1).trim()) + } + return map +} + +/** + * Rewrite table identifiers in `sql`. Only rewrites the table position (directly + * after `FROM` / `JOIN`), matching the whole identifier, so string literals or + * column names that happen to share a table's name are untouched. Rules are + * applied longest-source-first to avoid a short name shadowing a longer one. + */ +export function remapTables(sql: string, map: TableMap): string { + if (map.size === 0) return sql + const names = [...map.keys()].sort((a, b) => b.length - a.length) + let out = sql + for (const name of names) { + const target = map.get(name)! + const re = new RegExp(`(\\b(?:FROM|JOIN)\\s+)${escapeRegExp(name)}\\b`, "g") + out = out.replace(re, `$1${target}`) + } + return out +} diff --git a/apps/api/vitest.config.ts b/apps/api/vitest.config.ts index 81ef9aaec..a2b2b316e 100644 --- a/apps/api/vitest.config.ts +++ b/apps/api/vitest.config.ts @@ -14,7 +14,7 @@ export default defineConfig({ }, test: { environment: "node", - include: ["src/**/*.test.ts"], + include: ["src/**/*.test.ts", "scripts/**/*.test.ts"], // Generous timeouts: the DB-backed suites boot a fresh PGlite (WASM) per // test and some retry tests run real exponential backoff. Under CI's // parallel `turbo test`, CPU starvation stretches these past the 5s diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 3f13c0bb5..9e42b98d6 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: d58ce4a83d3ad3f3a29b9bb972272b757547ae793c050194354454634f3abccd +-- projectRevision: a8d56c30f4f7c8998db16ca014680db16a82f69143adbeb054ccc47d718b222b CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -130,7 +130,12 @@ CREATE TABLE IF NOT EXISTS logs ( ScopeVersion String, ScopeAttributes Map(LowCardinality(String), String), LogAttributes Map(LowCardinality(String), String), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_body_tokens lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimestampTime) @@ -651,7 +656,9 @@ CREATE TABLE IF NOT EXISTS traces ( INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes), mapValues(SpanAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) @@ -1073,7 +1080,7 @@ SELECT OrgId, toStartOfHour(toDateTime(Timestamp)) AS Hour, ServiceName, - SpanAttributes['db.system.name'] AS DbSystem, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, ResourceAttributes['deployment.environment'] AS DeploymentEnv, count() AS CallCount, countIf(StatusCode = 'Error') AS ErrorCount, @@ -1084,7 +1091,7 @@ SELECT sum(SampleRate) AS SampleRateSum FROM traces WHERE SpanKind IN ('Client', 'Producer') - AND SpanAttributes['db.system.name'] != '' + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' AND ServiceName != '' GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv; diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index d6c8f675c..2d280d5ca 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,12 +1,12 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "d58ce4a83d3ad3f3a29b9bb972272b757547ae793c050194354454634f3abccd"; +pub const PROJECT_REVISION: &str = "a8d56c30f4f7c8998db16ca014680db16a82f69143adbeb054ccc47d718b222b"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse // clickHouseSchemaVersion. -pub const SCHEMA_VERSION: &str = "4"; +pub const SCHEMA_VERSION: &str = "5"; pub const ORG_PLACEHOLDER: &str = "__ORG__"; #[derive(Debug)] diff --git a/lib/clickhouse-builder/src/ch/expr.ts b/lib/clickhouse-builder/src/ch/expr.ts index ab773a727..de953d7f9 100644 --- a/lib/clickhouse-builder/src/ch/expr.ts +++ b/lib/clickhouse-builder/src/ch/expr.ts @@ -266,6 +266,7 @@ export { left_, length_, lower_, + hasToken, replaceOne, extract_, concat, @@ -294,6 +295,7 @@ export { arrayOf, arrayStringConcat, arrayFilter, + has, mapContains, mapGet, mapKeys, diff --git a/lib/clickhouse-builder/src/ch/functions/array.ts b/lib/clickhouse-builder/src/ch/functions/array.ts index 4f9aff505..6f2bdbe0d 100644 --- a/lib/clickhouse-builder/src/ch/functions/array.ts +++ b/lib/clickhouse-builder/src/ch/functions/array.ts @@ -1,4 +1,5 @@ import { makeExpr } from "../expr" +import { defineCondFn } from "../define-fn" import { raw, str, compile } from "../../sql/sql-fragment" import type { Expr } from "../expr" @@ -29,3 +30,10 @@ export function arrayStringConcat( export function arrayFilter(fn: string, arr: Expr): Expr { return makeExpr(raw(`arrayFilter(${fn}, ${compile(arr.toFragment())})`)) } + +/** + * `has(arr, elem)` — true when `elem` is an element of `arr`. Used with a + * `bloom_filter` skip index over a concatenated `key=value` items expression so + * attribute-equality filters prune granules (ClickStack "Items" pattern). + */ +export const has = defineCondFn<[Expr>, Expr]>("has") diff --git a/lib/clickhouse-builder/src/ch/functions/index.ts b/lib/clickhouse-builder/src/ch/functions/index.ts index 5c8337b7e..b944d5d11 100644 --- a/lib/clickhouse-builder/src/ch/functions/index.ts +++ b/lib/clickhouse-builder/src/ch/functions/index.ts @@ -27,6 +27,7 @@ export { left_, length_, lower_, + hasToken, replaceOne, extract_, concat, @@ -58,7 +59,7 @@ export { export { if_, multiIf, coalesce, nullIf } from "./conditional" -export { arrayOf, arrayStringConcat, arrayFilter } from "./array" +export { arrayOf, arrayStringConcat, arrayFilter, has } from "./array" export { mapContains, mapGet, mapKeys, mapValues, mapLiteral } from "./map" diff --git a/lib/clickhouse-builder/src/ch/functions/string.ts b/lib/clickhouse-builder/src/ch/functions/string.ts index 313361578..0fc419c80 100644 --- a/lib/clickhouse-builder/src/ch/functions/string.ts +++ b/lib/clickhouse-builder/src/ch/functions/string.ts @@ -1,4 +1,4 @@ -import { defineFn, compileFnCall } from "../define-fn" +import { defineFn, defineCondFn, compileFnCall } from "../define-fn" import type { Expr } from "../expr" // --------------------------------------------------------------------------- @@ -13,6 +13,14 @@ export const positionCaseInsensitive = defineFn<[Expr, Expr], nu ) export const left_ = defineFn<[Expr, Expr], string>("left") +/** + * `hasToken(haystack, token)` — true when `token` occurs as a whole token of + * `haystack` (separators are non-alphanumeric ASCII). Backed by `tokenbf_v1` + * skip indexes for granule pruning. Whole-token semantics ≠ substring `ILIKE`; + * see `body-search.ts` in the query engine for the safe-token derivation. + */ +export const hasToken = defineCondFn<[Expr, Expr]>("hasToken") + // --------------------------------------------------------------------------- // Mixed Expr + literal args (compileFnCall wrappers) // --------------------------------------------------------------------------- diff --git a/lib/clickhouse-builder/src/ch/index.ts b/lib/clickhouse-builder/src/ch/index.ts index 1cf23845c..7386245d7 100644 --- a/lib/clickhouse-builder/src/ch/index.ts +++ b/lib/clickhouse-builder/src/ch/index.ts @@ -76,6 +76,8 @@ export { position_ as position, left_ as left, length_ as length, + lower_ as lower, + hasToken, replaceOne, extract_ as extract, concat, @@ -107,6 +109,7 @@ export { arrayOf, arrayStringConcat, arrayFilter, + has, // Map mapContains, mapGet, diff --git a/packages/domain/package.json b/packages/domain/package.json index 5d6e88bea..c6b2b899a 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -15,6 +15,7 @@ "./warehouse-queries": "./src/warehouse-queries.ts", "./tinybird": "./src/tinybird/index.ts", "./tinybird/db-query-shape-sql": "./src/tinybird/db-query-shape-sql.ts", + "./tinybird/index-exprs": "./src/tinybird/index-exprs.ts", "./clickhouse": "./src/clickhouse/index.ts", "./where-clause": "./src/where-clause.ts", "./http/observability": "./src/http/observability.ts", diff --git a/packages/domain/src/clickhouse/migrations/0005_body_and_attr_indexes.ts b/packages/domain/src/clickhouse/migrations/0005_body_and_attr_indexes.ts new file mode 100644 index 000000000..8d9e95f74 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0005_body_and_attr_indexes.ts @@ -0,0 +1,49 @@ +import { attrItemsExpr, BODY_TOKEN_INDEX_EXPR } from "../../tinybird/index-exprs" + +/** + * Migration 0005 — full-text body index + attribute skip indexes. + * + * Adds ClickStack-style data-skipping indexes (see "Making ClickStack 5x + * faster"): + * + * - `logs.idx_body_tokens` — `tokenbf_v1` over `lower(Body)` so the query + * engine's `hasToken(lower(Body), )` pre-filter prunes granules + * ahead of the substring `Body ILIKE '%…%'`. + * - `logs.idx_log_*` — bloom filters over `LogAttributes`/`ResourceAttributes` + * keys and values, mirroring the ones `traces` already has. + * - `traces.idx_*_attr_items` — bloom filters over the concatenated + * `key=value` "Items" expression so attribute-equality filters prune via + * `has(, 'k=v')`. + * + * The index expressions are imported from the same {@link attrItemsExpr} / + * {@link BODY_TOKEN_INDEX_EXPR} helpers the datasource definitions and the + * query-engine predicates use — the three must stay byte-for-byte identical or + * ClickHouse silently stops applying the index. + * + * Deliberately **no `MATERIALIZE INDEX`**: materializing over an existing + * multi-billion-row part set exceeds the Cloudflare Worker subrequest budget + * (same reason migration 0004 adds its `set` index without materializing). The + * indexes apply to new parts immediately; with the 30-day TTL the whole table + * is covered within a retention cycle as old parts roll off. An operator who + * wants immediate coverage can run, partition by partition: + * + * ALTER TABLE logs MATERIALIZE INDEX idx_body_tokens IN PARTITION '' + * ALTER TABLE traces MATERIALIZE INDEX idx_span_attr_items IN PARTITION '' + * + * All statements are `IF NOT EXISTS`, so they're no-ops on a fresh instance + * whose migration-0001 snapshot already created the indexes at CREATE TABLE. + */ +export const migration_0005_body_and_attr_indexes = { + version: 5, + description: + "Add tokenbf body index + logs attribute blooms + traces attribute 'items' blooms for search/filter granule pruning", + statements: [ + `ALTER TABLE logs ADD INDEX IF NOT EXISTS idx_body_tokens ${BODY_TOKEN_INDEX_EXPR} TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1`, + "ALTER TABLE logs ADD INDEX IF NOT EXISTS idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1", + "ALTER TABLE logs ADD INDEX IF NOT EXISTS idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1", + "ALTER TABLE logs ADD INDEX IF NOT EXISTS idx_log_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1", + "ALTER TABLE logs ADD INDEX IF NOT EXISTS idx_log_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1", + `ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_span_attr_items ${attrItemsExpr("SpanAttributes")} TYPE bloom_filter(0.01) GRANULARITY 1`, + `ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_resource_attr_items ${attrItemsExpr("ResourceAttributes")} TYPE bloom_filter(0.01) GRANULARITY 1`, + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 89e2c6697..9941cf883 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest" import { isBackfill, renderStatementFull, type BackfillSpec } from "../backfill" import { migration_0004_service_namespace_projections } from "./0004_service_namespace_projections" +import { migration_0005_body_and_attr_indexes } from "./0005_body_and_attr_indexes" import { migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -14,9 +15,22 @@ const renderedSql = migration_0004_service_namespace_projections.statements .join("\n\n") describe("ClickHouse migrations", () => { - it("keeps service.namespace migration ordered after the previous deltas", () => { - expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4]) - expect(migrations.at(-1)).toBe(migration_0004_service_namespace_projections) + it("keeps migrations ordered with the body/attr-index delta last", () => { + expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5]) + expect(migrations.at(-1)).toBe(migration_0005_body_and_attr_indexes) + }) + + it("adds body + attribute skip indexes as new-parts-only ADD INDEX statements", () => { + const sql = migration_0005_body_and_attr_indexes.statements.join("\n") + expect(sql).toContain( + "ALTER TABLE logs ADD INDEX IF NOT EXISTS idx_body_tokens lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1", + ) + expect(sql).toContain( + "ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_span_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes), mapValues(SpanAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1", + ) + // Must NOT materialize — that would exceed the Worker subrequest budget on + // billion-row parts; indexes apply to new parts and backfill via TTL churn. + expect(sql).not.toContain("MATERIALIZE INDEX") }) it("rebuilds namespace-aware log aggregates and recreates affected materialized views", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 226f6a772..1e0a6044f 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -3,6 +3,7 @@ import { migration_0001_initial } from "./0001_initial" import { migration_0002_service_map_edges_rollup } from "./0002_service_map_edges_rollup" import { migration_0003_error_events_label } from "./0003_error_events_label" import { migration_0004_service_namespace_projections } from "./0004_service_namespace_projections" +import { migration_0005_body_and_attr_indexes } from "./0005_body_and_attr_indexes" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -36,6 +37,7 @@ export const migrations: ReadonlyArray = [ migration_0002_service_map_edges_rollup, migration_0003_error_events_label, migration_0004_service_namespace_projections, + migration_0005_body_and_attr_indexes, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 312d93ccc..23bc1673c 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "d58ce4a83d3ad3f3a29b9bb972272b757547ae793c050194354454634f3abccd" as const +export const projectRevision = "a8d56c30f4f7c8998db16ca014680db16a82f69143adbeb054ccc47d718b222b" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 90 DAY", @@ -10,7 +10,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS error_events (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, FingerprintHash, Timestamp)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_events_by_time (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, FingerprintHash)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n StatusMessage String,\n Duration UInt64,\n DeploymentEnv LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, Timestamp)\nTTL Timestamp + INTERVAL 90 DAY", - "CREATE TABLE IF NOT EXISTS logs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TimestampTime DateTime,\n TraceId String,\n SpanId String,\n TraceFlags UInt8,\n SeverityText LowCardinality(String),\n SeverityNumber UInt8,\n ServiceName LowCardinality(String),\n Body String,\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n LogAttributes Map(LowCardinality(String), String),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimestampTime)\nORDER BY (OrgId, ServiceName, TimestampTime, Timestamp)\nTTL toDate(TimestampTime) + INTERVAL 30 DAY", + "CREATE TABLE IF NOT EXISTS logs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TimestampTime DateTime,\n TraceId String,\n SpanId String,\n TraceFlags UInt8,\n SeverityText LowCardinality(String),\n SeverityNumber UInt8,\n ServiceName LowCardinality(String),\n Body String,\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n LogAttributes Map(LowCardinality(String), String),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_body_tokens lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,\n INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_log_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_log_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimestampTime)\nORDER BY (OrgId, ServiceName, TimestampTime, Timestamp)\nTTL toDate(TimestampTime) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS logs_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SeverityText LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n Count SimpleAggregateFunction(sum, UInt64),\n SizeBytes SimpleAggregateFunction(sum, UInt64),\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metric_catalog (\n OrgId LowCardinality(String),\n Hour DateTime,\n MetricType LowCardinality(String),\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription SimpleAggregateFunction(anyLast, String),\n MetricUnit SimpleAggregateFunction(anyLast, String),\n IsMonotonic SimpleAggregateFunction(anyLast, UInt8),\n DataPointCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour)\nTTL Hour + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metrics_exponential_histogram (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Count UInt64,\n Sum Float64,\n Scale Int32,\n ZeroCount UInt64,\n PositiveOffset Int32,\n PositiveBucketCounts Array(UInt64),\n NegativeOffset Int32,\n NegativeBucketCounts Array(UInt64),\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)),\n Flags UInt32,\n Min Nullable(Float64),\n Max Nullable(Float64),\n AggregationTemporality Int32\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", @@ -33,7 +33,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n SpanKind LowCardinality(String),\n AttrFingerprint UInt64,\n ResourceFingerprint UInt64,\n StartTimeUnix DateTime64(9),\n LastValue AggregateFunction(argMax, Float64, DateTime64(9))\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS trace_detail_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String))\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, TraceId, SpanId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS trace_list_mv (\n OrgId LowCardinality(String),\n TraceId String,\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n SpanName String,\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n HttpMethod LowCardinality(String),\n HttpRoute String,\n HttpStatusCode LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n HasError UInt8,\n TraceState String,\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL Timestamp + INTERVAL 30 DAY", - "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", + "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes), mapValues(SpanAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'", @@ -49,7 +49,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '',\n if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR SpanAttributes['messaging.destination'] != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanAttributes['db.system.name'] AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n coalesce(\n nullIf(SpanAttributes['db.query.fingerprint'], ''),\n nullIf(SpanAttributes['db.statement.fingerprint'], ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\\'[^\\']*\\'', '?'), '\\\\bin\\\\s*\\\\([^)]*\\\\)', 'in (?)'), '[0-9]+(\\\\.[0-9]+)?', '?'), '\\\\s+', ' '), '^\\\\s+|\\\\s+$', ''))), ''), ''),\n toString(cityHash64(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n)))\n) AS QueryKey,\n any(substring(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n), 1, 220)) AS QueryLabel,\n any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(SampleRate) AS EstimatedCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv, QueryKey", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer')", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 99e020eb0..0e89cc4d8 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,336 +1,272 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "5e5eac5fd0008ab8726225d6f6182c1855be4d8620aedea83b4f83d0b78d8278" as const +export const projectRevision = "a8d56c30f4f7c8998db16ca014680db16a82f69143adbeb054ccc47d718b222b" as const export const datasources = [ - { - name: "alert_checks", - content: - 'DESCRIPTION >\n One row per alert rule evaluation. Durable audit trail of checks: status, observed value, threshold, sample count, incident linkage.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.OrgId`,\n RuleId String `json:$.RuleId`,\n GroupKey String `json:$.GroupKey`,\n Timestamp DateTime64(3) `json:$.Timestamp`,\n Status LowCardinality(String) `json:$.Status`,\n SignalType LowCardinality(String) `json:$.SignalType`,\n Comparator LowCardinality(String) `json:$.Comparator`,\n Threshold Float64 `json:$.Threshold`,\n ObservedValue Nullable(Float64) `json:$.ObservedValue`,\n SampleCount UInt32 `json:$.SampleCount`,\n WindowMinutes UInt16 `json:$.WindowMinutes`,\n WindowStart DateTime64(3) `json:$.WindowStart`,\n WindowEnd DateTime64(3) `json:$.WindowEnd`,\n ConsecutiveBreaches UInt16 `json:$.ConsecutiveBreaches`,\n ConsecutiveHealthy UInt16 `json:$.ConsecutiveHealthy`,\n IncidentId Nullable(String) `json:$.IncidentId`,\n IncidentTransition LowCardinality(String) `json:$.IncidentTransition`,\n EvaluationDurationMs UInt32 `json:$.EvaluationDurationMs`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, RuleId, GroupKey, Timestamp"\nENGINE_TTL "toDate(Timestamp) + INTERVAL 90 DAY"', - }, - { - name: "attribute_keys_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated attribute keys with hourly usage counts from traces, logs, and metrics.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, AttributeScope, Hour, AttributeKey"\nENGINE_TTL "Hour + INTERVAL 90 DAY"', - }, - { - name: "attribute_values_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated attribute values with hourly usage counts from trace span and resource attributes.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeValue String,\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, AttributeScope, AttributeKey, Hour, AttributeValue"\nENGINE_TTL "Hour + INTERVAL 90 DAY"', - }, - { - name: "error_events", - content: - 'DESCRIPTION >\n Per-error-occurrence rows for the triageable-errors system. Unwraps OTel exception events and computes a stable FingerprintHash for grouping into issues. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT \'__unset__\',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, FingerprintHash, Timestamp"\nENGINE_TTL "Timestamp + INTERVAL 90 DAY"', - }, - { - name: "error_events_by_time", - content: - 'DESCRIPTION >\n Time-ordered sibling of error_events (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans (errorIssuesScan tick + dashboard error queries). Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT \'__unset__\',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, Timestamp, FingerprintHash"\nENGINE_TTL "Timestamp + INTERVAL 90 DAY"', - }, - { - name: "error_spans", - content: - 'DESCRIPTION >\n Pre-materialized error spans for the errors page. Pre-filters to StatusCode=\'Error\' and pre-extracts deployment.environment. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT \'__unset__\',\n ServiceName LowCardinality(String),\n StatusMessage String,\n Duration UInt64,\n DeploymentEnv LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, ServiceName, Timestamp"\nENGINE_TTL "Timestamp + INTERVAL 90 DAY"', - }, - { - name: "logs", - content: - 'DESCRIPTION >\n This is a table that contains the logs from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n TimestampTime DateTime `json:$.timestamp`,\n TraceId String `json:$.trace_id`,\n SpanId String `json:$.span_id`,\n TraceFlags UInt8 `json:$.flags`,\n SeverityText LowCardinality(String) `json:$.severity_text`,\n SeverityNumber UInt8 `json:$.severity_number`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n Body String `json:$.body`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n LogAttributes Map(LowCardinality(String), String) `json:$.log_attributes`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(TimestampTime)"\nENGINE_SORTING_KEY "OrgId, ServiceName, TimestampTime, Timestamp"\nENGINE_TTL "toDate(TimestampTime) + INTERVAL 30 DAY"\n\nINDEXES >\n idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1', - }, - { - name: "logs_aggregates_hourly", - content: - 'DESCRIPTION >\n Hourly pre-aggregated log counts and sizes by service × severity × deployment env. AggregatingMergeTree.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SeverityText LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n Count SimpleAggregateFunction(sum, UInt64),\n SizeBytes SimpleAggregateFunction(sum, UInt64),\n ServiceNamespace LowCardinality(String)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"\n\nINDEXES >\n idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n\nFORWARD_QUERY >\n SELECT\n OrgId, Hour, ServiceName, SeverityText, DeploymentEnv,\n Count, SizeBytes,\n ServiceNamespace', - }, - { - name: "metric_catalog", - content: - 'DESCRIPTION >\n Hourly catalog of distinct metrics (name/type/service) with datapoint counts and first/last-seen. AggregatingMergeTree MV target; powers the Metrics page discovery queries.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n MetricType LowCardinality(String),\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription SimpleAggregateFunction(anyLast, String),\n MetricUnit SimpleAggregateFunction(anyLast, String),\n IsMonotonic SimpleAggregateFunction(anyLast, UInt8),\n DataPointCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, MetricType, ServiceName, MetricName, Hour"\nENGINE_TTL "Hour + INTERVAL 90 DAY"', - }, - { - name: "metrics_exponential_histogram", - content: - 'DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Count UInt64 `json:$.count`,\n Sum Float64 `json:$.sum`,\n Scale Int32 `json:$.scale`,\n ZeroCount UInt64 `json:$.zero_count`,\n PositiveOffset Int32 `json:$.positive_offset`,\n PositiveBucketCounts Array(UInt64) `json:$.positive_bucket_counts[:]`,\n NegativeOffset Int32 `json:$.negative_offset`,\n NegativeBucketCounts Array(UInt64) `json:$.negative_bucket_counts[:]`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n Flags UInt32 `json:$.flags`,\n Min Nullable(Float64) `json:$.min`,\n Max Nullable(Float64) `json:$.max`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(TimeUnix)"\nENGINE_SORTING_KEY "OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)"\nENGINE_TTL "toDate(TimeUnix) + INTERVAL 90 DAY"', - }, - { - name: "metrics_gauge", - content: - 'DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Value Float64 `json:$.value`,\n Flags UInt32 `json:$.flags`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(TimeUnix)"\nENGINE_SORTING_KEY "OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)"\nENGINE_TTL "toDate(TimeUnix) + INTERVAL 90 DAY"', - }, - { - name: "metrics_histogram", - content: - 'DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Count UInt64 `json:$.count`,\n Sum Float64 `json:$.sum`,\n BucketCounts Array(UInt64) `json:$.bucket_counts[:]`,\n ExplicitBounds Array(Float64) `json:$.explicit_bounds[:]`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n Flags UInt32 `json:$.flags`,\n Min Nullable(Float64) `json:$.min`,\n Max Nullable(Float64) `json:$.max`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(TimeUnix)"\nENGINE_SORTING_KEY "OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)"\nENGINE_TTL "toDate(TimeUnix) + INTERVAL 90 DAY"', - }, - { - name: "metrics_sum", - content: - 'DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Value Float64 `json:$.value`,\n Flags UInt32 `json:$.flags`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`,\n IsMonotonic Bool `json:$.is_monotonic`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(TimeUnix)"\nENGINE_SORTING_KEY "OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)"\nENGINE_TTL "toDate(TimeUnix) + INTERVAL 90 DAY"', - }, - { - name: "service_address_resolutions_hourly", - content: - 'DESCRIPTION >\n Resolved (sourceService, parent.server.address) → resolved targetService facts emitted by the ServiceMapRollupService rollup. Used to anti-join internal-service overlap out of the external-edges query.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n ParentServerAddress String,\n ResolvedTargetService LowCardinality(String),\n DeploymentEnv LowCardinality(String)\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"', - }, - { - name: "service_external_edges_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated hourly service-to-external-target edges (http / messaging / rpc) for the service-detail Dependencies tab. Captures Client/Producer spans WITHOUT db.system.name. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n TargetType LowCardinality(String),\n TargetSystem LowCardinality(String),\n TargetName String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"', - }, - { - name: "service_map_children", - content: - 'DESCRIPTION >\n Server/Consumer spans with ParentSpanId for efficient service map child-side JOIN lookups. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, TraceId, ParentSpanId, Timestamp"\nENGINE_TTL "Timestamp + INTERVAL 30 DAY"', - }, - { - name: "service_map_db_edges_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated hourly service-to-database edges (one row per service/db.system.name) for the service map\'s database-node query. Uses AggregatingMergeTree for incremental aggregation. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DbSystem LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampledSpanCount SimpleAggregateFunction(sum, UInt64),\n UnsampledSpanCount SimpleAggregateFunction(sum, UInt64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, DeploymentEnv, ServiceName, DbSystem"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"\n\nFORWARD_QUERY >\n SELECT *', - }, - { - name: "service_map_db_query_shapes_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated hourly database query shapes (one row per service/db.system/query-shape) for the service map\'s database detail panel. Uses AggregatingMergeTree with a sample-weighted t-digest state for true p50/p95. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DbSystem LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n QueryKey String,\n QueryLabel SimpleAggregateFunction(any, String),\n SampleStatement SimpleAggregateFunction(any, String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n EstimatedCount SimpleAggregateFunction(sum, Float64),\n EstimatedErrorCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSumMs SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, QueryKey"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"\n\nFORWARD_QUERY >\n SELECT *', - }, - { - name: "service_map_edges_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated hourly service-to-service edges for the service map. Uses AggregatingMergeTree for incremental aggregation. Populated by the scheduled ServiceMapRollupService rollup (one write per completed hour).\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n TargetService String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampledSpanCount SimpleAggregateFunction(sum, UInt64),\n UnsampledSpanCount SimpleAggregateFunction(sum, UInt64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, DeploymentEnv, SourceService, TargetService"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"', - }, - { - name: "service_map_spans", - content: - 'DESCRIPTION >\n Lightweight projection of traces for service map JOIN queries. Pre-extracts deployment.environment from Map columns. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, TraceId, SpanId, Timestamp"\nENGINE_TTL "Timestamp + INTERVAL 30 DAY"', - }, - { - name: "service_overview_spans", - content: - 'DESCRIPTION >\n Lightweight projection of service entry point spans (Server/Consumer + root) for service overview queries. Pre-extracts deployment attributes from ResourceAttributes. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String),\n CommitSha LowCardinality(String),\n SampleRate Float64 DEFAULT 1,\n ServiceNamespace LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, ServiceName, Timestamp"\nENGINE_TTL "Timestamp + INTERVAL 30 DAY"\n\nINDEXES >\n idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4', - }, - { - name: "service_platforms_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated hourly per-service platform/runtime attributes (k8s, cloud, faas) for the service map\'s hosting-icon resolver. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, DeploymentEnv"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"\n\nFORWARD_QUERY >\n SELECT\n OrgId, Hour, ServiceName, DeploymentEnv,\n K8sCluster, K8sPodName, K8sDeploymentName,\n K8sStatefulSetName,\n K8sDaemonSetName,\n K8sNamespaceName,\n CloudPlatform, CloudProvider, FaasName, MapleSdkType, ProcessRuntimeName, SpanCount', - }, - { - name: "service_usage", - content: - 'DESCRIPTION >\n Aggregated usage statistics per service per hour. Uses SummingMergeTree for efficient incremental updates from multiple materialized views.\n\nSCHEMA >\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64\n\nENGINE "SummingMergeTree"\nENGINE_SORTING_KEY "OrgId, ServiceName, Hour"\nENGINE_TTL "Hour + INTERVAL 365 DAY"\n\nFORWARD_QUERY >\n SELECT *', - }, - { - name: "session_events", - content: - "DESCRIPTION >\n Distilled structured session events (navigation, click, input, console, network, error) captured client-side and ingested via POST /v1/sessionEvents. Powers in-session search, replay panels, and agent transcripts.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Seq UInt32 `json:$.seq` DEFAULT 0,\n Type LowCardinality(String) `json:$.type`,\n Url String `json:$.url` DEFAULT '',\n TraceId String `json:$.trace_id` DEFAULT '',\n Level LowCardinality(String) `json:$.level` DEFAULT '',\n Message String `json:$.message` DEFAULT '',\n TargetSelector String `json:$.target_selector` DEFAULT '',\n TargetText String `json:$.target_text` DEFAULT '',\n NetMethod LowCardinality(String) `json:$.net_method` DEFAULT '',\n NetUrl String `json:$.net_url` DEFAULT '',\n NetStatus UInt16 `json:$.net_status` DEFAULT 0,\n NetDurationMs UInt32 `json:$.net_duration_ms` DEFAULT 0,\n ErrorStack String `json:$.error_stack` DEFAULT '',\n Attributes Map(LowCardinality(String), String) `json:$.attributes`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, SessionId, Timestamp, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"", - }, - { - name: "session_replay_events", - content: - 'DESCRIPTION >\n Session replay rrweb events (one row per chunk, payload included). The ingest gateway gunzips the chunk and stores the event-array JSON in `Events`. Playback reads directly from ClickHouse — no R2.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n ChunkSeq UInt32 `json:$.chunk_seq`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n DurationMs UInt32 `json:$.duration_ms` DEFAULT 0,\n EventCount UInt32 `json:$.event_count` DEFAULT 0,\n ByteSize UInt32 `json:$.byte_size` DEFAULT 0,\n Events String `json:$.events`,\n IsCheckpoint UInt8 `json:$.is_checkpoint` DEFAULT 0\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, SessionId, ChunkSeq"\nENGINE_TTL "toDate(Timestamp) + INTERVAL 30 DAY"', - }, - { - name: "session_replays", - content: - 'DESCRIPTION >\n Per-session browser replay metadata (one row per session). Ingested directly from the @maple-dev/browser SDK via POST /v1/sessionReplays/meta. Event payloads live inline in session_replay_events; this holds only queryable metadata. ReplacingMergeTree(Version) for start/end upsert.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n StartTime DateTime64(9) `json:$.start_time`,\n EndTime Nullable(DateTime64(9)) `json:$.end_time`,\n DurationMs Nullable(UInt32) `json:$.duration_ms`,\n Status LowCardinality(String) `json:$.status`,\n UserId String `json:$.user_id`,\n UrlInitial String `json:$.url_initial`,\n UserAgent String `json:$.user_agent`,\n BrowserName LowCardinality(String) `json:$.browser_name`,\n OsName LowCardinality(String) `json:$.os_name`,\n DeviceType LowCardinality(String) `json:$.device_type`,\n Country LowCardinality(String) `json:$.country` DEFAULT \'\',\n ServiceName LowCardinality(String) `json:$.service_name`,\n PageViews UInt32 `json:$.page_views` DEFAULT 0,\n ClickCount UInt32 `json:$.click_count` DEFAULT 0,\n ErrorCount UInt32 `json:$.error_count` DEFAULT 0,\n TraceIds Array(String) `json:$.trace_ids[:]` DEFAULT [],\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n Version UInt32 `json:$.version`\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "toDate(StartTime)"\nENGINE_SORTING_KEY "OrgId, SessionId"\nENGINE_TTL "toDate(StartTime) + INTERVAL 30 DAY"\nENGINE_VER "Version"', - }, - { - name: "span_metrics_calls_hourly", - content: - 'DESCRIPTION >\n Hourly per-series last-value (argMax) rollup of the span-metrics calls counter. AggregatingMergeTree MV target powering sampling-aware throughput without scanning raw metrics_sum.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n SpanKind LowCardinality(String),\n AttrFingerprint UInt64,\n ResourceFingerprint UInt64,\n StartTimeUnix DateTime64(9),\n LastValue AggregateFunction(argMax, Float64, DateTime64(9))\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"', - }, - { - name: "trace_detail_spans", - content: - 'DESCRIPTION >\n All spans for a trace, sorted by TraceId for fast detail lookups. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String))\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, TraceId, SpanId"\nENGINE_TTL "toDate(Timestamp) + INTERVAL 30 DAY"', - }, - { - name: "trace_list_mv", - content: - 'DESCRIPTION >\n Pre-materialized root spans for the trace list view. Extracts HTTP attributes and normalizes span names at write time. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n TraceId String,\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n SpanName String,\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n HttpMethod LowCardinality(String),\n HttpRoute String,\n HttpStatusCode LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n HasError UInt8,\n TraceState String,\n ServiceNamespace LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, Timestamp, TraceId"\nENGINE_TTL "Timestamp + INTERVAL 30 DAY"\n\nINDEXES >\n idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4', - }, - { - name: "traces", - content: - "DESCRIPTION >\n A table that contains trace data from OpenTelemetry in Tinybird format.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n Timestamp DateTime64(9) `json:$.start_time`,\n TraceId String `json:$.trace_id`,\n SpanId String `json:$.span_id`,\n ParentSpanId String `json:$.parent_span_id`,\n TraceState String `json:$.trace_state`,\n SpanName LowCardinality(String) `json:$.span_name`,\n SpanKind LowCardinality(String) `json:$.span_kind`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n Duration UInt64 `json:$.duration` DEFAULT 0,\n StatusCode LowCardinality(String) `json:$.status_code`,\n StatusMessage String `json:$.status_message`,\n SpanAttributes Map(LowCardinality(String), String) `json:$.span_attributes`,\n EventsTimestamp Array(DateTime64(9)) `json:$.events_timestamp[:]`,\n EventsName Array(LowCardinality(String)) `json:$.events_name[:]`,\n EventsAttributes Array(Map(LowCardinality(String), String)) `json:$.events_attributes[:]`,\n LinksTraceId Array(String) `json:$.links_trace_id[:]`,\n LinksSpanId Array(String) `json:$.links_span_id[:]`,\n LinksTraceState Array(String) `json:$.links_trace_state[:]`,\n LinksAttributes Array(Map(LowCardinality(String), String)) `json:$.links_attributes[:]`,\n SampleRate Float64 `json:$.SampleRate` DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 `json:$.IsEntryPoint` DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, SpanName, toDateTime(Timestamp)\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1", - }, - { - name: "traces_aggregates_hourly", - content: - 'DESCRIPTION >\n Hourly pre-aggregated trace metrics with sampling-weighted state columns. Generalized MV target for timeseries/breakdown/service-overview queries. AggregatingMergeTree.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"', - }, + { + "name": "alert_checks", + "content": "DESCRIPTION >\n One row per alert rule evaluation. Durable audit trail of checks: status, observed value, threshold, sample count, incident linkage.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.OrgId`,\n RuleId String `json:$.RuleId`,\n GroupKey String `json:$.GroupKey`,\n Timestamp DateTime64(3) `json:$.Timestamp`,\n Status LowCardinality(String) `json:$.Status`,\n SignalType LowCardinality(String) `json:$.SignalType`,\n Comparator LowCardinality(String) `json:$.Comparator`,\n Threshold Float64 `json:$.Threshold`,\n ObservedValue Nullable(Float64) `json:$.ObservedValue`,\n SampleCount UInt32 `json:$.SampleCount`,\n WindowMinutes UInt16 `json:$.WindowMinutes`,\n WindowStart DateTime64(3) `json:$.WindowStart`,\n WindowEnd DateTime64(3) `json:$.WindowEnd`,\n ConsecutiveBreaches UInt16 `json:$.ConsecutiveBreaches`,\n ConsecutiveHealthy UInt16 `json:$.ConsecutiveHealthy`,\n IncidentId Nullable(String) `json:$.IncidentId`,\n IncidentTransition LowCardinality(String) `json:$.IncidentTransition`,\n EvaluationDurationMs UInt32 `json:$.EvaluationDurationMs`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, RuleId, GroupKey, Timestamp\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 90 DAY\"" + }, + { + "name": "attribute_keys_hourly", + "content": "DESCRIPTION >\n Pre-aggregated attribute keys with hourly usage counts from traces, logs, and metrics.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, AttributeScope, Hour, AttributeKey\"\nENGINE_TTL \"Hour + INTERVAL 90 DAY\"" + }, + { + "name": "attribute_values_hourly", + "content": "DESCRIPTION >\n Pre-aggregated attribute values with hourly usage counts from trace span and resource attributes.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeValue String,\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, AttributeScope, AttributeKey, Hour, AttributeValue\"\nENGINE_TTL \"Hour + INTERVAL 90 DAY\"" + }, + { + "name": "error_events", + "content": "DESCRIPTION >\n Per-error-occurrence rows for the triageable-errors system. Unwraps OTel exception events and computes a stable FingerprintHash for grouping into issues. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, FingerprintHash, Timestamp\"\nENGINE_TTL \"Timestamp + INTERVAL 90 DAY\"" + }, + { + "name": "error_events_by_time", + "content": "DESCRIPTION >\n Time-ordered sibling of error_events (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans (errorIssuesScan tick + dashboard error queries). Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, FingerprintHash\"\nENGINE_TTL \"Timestamp + INTERVAL 90 DAY\"" + }, + { + "name": "error_spans", + "content": "DESCRIPTION >\n Pre-materialized error spans for the errors page. Pre-filters to StatusCode='Error' and pre-extracts deployment.environment. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n StatusMessage String,\n Duration UInt64,\n DeploymentEnv LowCardinality(String)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, Timestamp\"\nENGINE_TTL \"Timestamp + INTERVAL 90 DAY\"" + }, + { + "name": "logs", + "content": "DESCRIPTION >\n This is a table that contains the logs from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n TimestampTime DateTime `json:$.timestamp`,\n TraceId String `json:$.trace_id`,\n SpanId String `json:$.span_id`,\n TraceFlags UInt8 `json:$.flags`,\n SeverityText LowCardinality(String) `json:$.severity_text`,\n SeverityNumber UInt8 `json:$.severity_number`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n Body String `json:$.body`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n LogAttributes Map(LowCardinality(String), String) `json:$.log_attributes`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(TimestampTime)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, TimestampTime, Timestamp\"\nENGINE_TTL \"toDate(TimestampTime) + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1\n idx_body_tokens lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1\n idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_log_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_log_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1" + }, + { + "name": "logs_aggregates_hourly", + "content": "DESCRIPTION >\n Hourly pre-aggregated log counts and sizes by service × severity × deployment env. AggregatingMergeTree.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SeverityText LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n Count SimpleAggregateFunction(sum, UInt64),\n SizeBytes SimpleAggregateFunction(sum, UInt64),\n ServiceNamespace LowCardinality(String)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"\n\nINDEXES >\n idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n\nFORWARD_QUERY >\n SELECT\n OrgId, Hour, ServiceName, SeverityText, DeploymentEnv,\n Count, SizeBytes,\n ServiceNamespace" + }, + { + "name": "metric_catalog", + "content": "DESCRIPTION >\n Hourly catalog of distinct metrics (name/type/service) with datapoint counts and first/last-seen. AggregatingMergeTree MV target; powers the Metrics page discovery queries.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n MetricType LowCardinality(String),\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription SimpleAggregateFunction(anyLast, String),\n MetricUnit SimpleAggregateFunction(anyLast, String),\n IsMonotonic SimpleAggregateFunction(anyLast, UInt8),\n DataPointCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, MetricType, ServiceName, MetricName, Hour\"\nENGINE_TTL \"Hour + INTERVAL 90 DAY\"" + }, + { + "name": "metrics_exponential_histogram", + "content": "DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Count UInt64 `json:$.count`,\n Sum Float64 `json:$.sum`,\n Scale Int32 `json:$.scale`,\n ZeroCount UInt64 `json:$.zero_count`,\n PositiveOffset Int32 `json:$.positive_offset`,\n PositiveBucketCounts Array(UInt64) `json:$.positive_bucket_counts[:]`,\n NegativeOffset Int32 `json:$.negative_offset`,\n NegativeBucketCounts Array(UInt64) `json:$.negative_bucket_counts[:]`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n Flags UInt32 `json:$.flags`,\n Min Nullable(Float64) `json:$.min`,\n Max Nullable(Float64) `json:$.max`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(TimeUnix)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)\"\nENGINE_TTL \"toDate(TimeUnix) + INTERVAL 90 DAY\"" + }, + { + "name": "metrics_gauge", + "content": "DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Value Float64 `json:$.value`,\n Flags UInt32 `json:$.flags`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(TimeUnix)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)\"\nENGINE_TTL \"toDate(TimeUnix) + INTERVAL 90 DAY\"" + }, + { + "name": "metrics_histogram", + "content": "DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Count UInt64 `json:$.count`,\n Sum Float64 `json:$.sum`,\n BucketCounts Array(UInt64) `json:$.bucket_counts[:]`,\n ExplicitBounds Array(Float64) `json:$.explicit_bounds[:]`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n Flags UInt32 `json:$.flags`,\n Min Nullable(Float64) `json:$.min`,\n Max Nullable(Float64) `json:$.max`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(TimeUnix)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)\"\nENGINE_TTL \"toDate(TimeUnix) + INTERVAL 90 DAY\"" + }, + { + "name": "metrics_sum", + "content": "DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Value Float64 `json:$.value`,\n Flags UInt32 `json:$.flags`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`,\n IsMonotonic Bool `json:$.is_monotonic`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(TimeUnix)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)\"\nENGINE_TTL \"toDate(TimeUnix) + INTERVAL 90 DAY\"" + }, + { + "name": "service_address_resolutions_hourly", + "content": "DESCRIPTION >\n Resolved (sourceService, parent.server.address) → resolved targetService facts emitted by the ServiceMapRollupService rollup. Used to anti-join internal-service overlap out of the external-edges query.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n ParentServerAddress String,\n ResolvedTargetService LowCardinality(String),\n DeploymentEnv LowCardinality(String)\n\nENGINE \"ReplacingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"" + }, + { + "name": "service_external_edges_hourly", + "content": "DESCRIPTION >\n Pre-aggregated hourly service-to-external-target edges (http / messaging / rpc) for the service-detail Dependencies tab. Captures Client/Producer spans WITHOUT db.system.name. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n TargetType LowCardinality(String),\n TargetSystem LowCardinality(String),\n TargetName String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"" + }, + { + "name": "service_map_children", + "content": "DESCRIPTION >\n Server/Consumer spans with ParentSpanId for efficient service map child-side JOIN lookups. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, TraceId, ParentSpanId, Timestamp\"\nENGINE_TTL \"Timestamp + INTERVAL 30 DAY\"" + }, + { + "name": "service_map_db_edges_hourly", + "content": "DESCRIPTION >\n Pre-aggregated hourly service-to-database edges (one row per service/db.system.name) for the service map's database-node query. Uses AggregatingMergeTree for incremental aggregation. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DbSystem LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampledSpanCount SimpleAggregateFunction(sum, UInt64),\n UnsampledSpanCount SimpleAggregateFunction(sum, UInt64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, DeploymentEnv, ServiceName, DbSystem\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"\n\nFORWARD_QUERY >\n SELECT *" + }, + { + "name": "service_map_db_query_shapes_hourly", + "content": "DESCRIPTION >\n Pre-aggregated hourly database query shapes (one row per service/db.system/query-shape) for the service map's database detail panel. Uses AggregatingMergeTree with a sample-weighted t-digest state for true p50/p95. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DbSystem LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n QueryKey String,\n QueryLabel SimpleAggregateFunction(any, String),\n SampleStatement SimpleAggregateFunction(any, String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n EstimatedCount SimpleAggregateFunction(sum, Float64),\n EstimatedErrorCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSumMs SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, QueryKey\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"\n\nFORWARD_QUERY >\n SELECT *" + }, + { + "name": "service_map_edges_hourly", + "content": "DESCRIPTION >\n Pre-aggregated hourly service-to-service edges for the service map. Uses AggregatingMergeTree for incremental aggregation. Populated by the scheduled ServiceMapRollupService rollup (one write per completed hour).\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n TargetService String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampledSpanCount SimpleAggregateFunction(sum, UInt64),\n UnsampledSpanCount SimpleAggregateFunction(sum, UInt64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, DeploymentEnv, SourceService, TargetService\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"" + }, + { + "name": "service_map_spans", + "content": "DESCRIPTION >\n Lightweight projection of traces for service map JOIN queries. Pre-extracts deployment.environment from Map columns. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, TraceId, SpanId, Timestamp\"\nENGINE_TTL \"Timestamp + INTERVAL 30 DAY\"" + }, + { + "name": "service_overview_spans", + "content": "DESCRIPTION >\n Lightweight projection of service entry point spans (Server/Consumer + root) for service overview queries. Pre-extracts deployment attributes from ResourceAttributes. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String),\n CommitSha LowCardinality(String),\n SampleRate Float64 DEFAULT 1,\n ServiceNamespace LowCardinality(String)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, Timestamp\"\nENGINE_TTL \"Timestamp + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4" + }, + { + "name": "service_platforms_hourly", + "content": "DESCRIPTION >\n Pre-aggregated hourly per-service platform/runtime attributes (k8s, cloud, faas) for the service map's hosting-icon resolver. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, ServiceName, DeploymentEnv\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"\n\nFORWARD_QUERY >\n SELECT\n OrgId, Hour, ServiceName, DeploymentEnv,\n K8sCluster, K8sPodName, K8sDeploymentName,\n K8sStatefulSetName,\n K8sDaemonSetName,\n K8sNamespaceName,\n CloudPlatform, CloudProvider, FaasName, MapleSdkType, ProcessRuntimeName, SpanCount" + }, + { + "name": "service_usage", + "content": "DESCRIPTION >\n Aggregated usage statistics per service per hour. Uses SummingMergeTree for efficient incremental updates from multiple materialized views.\n\nSCHEMA >\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64\n\nENGINE \"SummingMergeTree\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, Hour\"\nENGINE_TTL \"Hour + INTERVAL 365 DAY\"\n\nFORWARD_QUERY >\n SELECT *" + }, + { + "name": "session_events", + "content": "DESCRIPTION >\n Distilled structured session events (navigation, click, input, console, network, error) captured client-side and ingested via POST /v1/sessionEvents. Powers in-session search, replay panels, and agent transcripts.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Seq UInt32 `json:$.seq` DEFAULT 0,\n Type LowCardinality(String) `json:$.type`,\n Url String `json:$.url` DEFAULT '',\n TraceId String `json:$.trace_id` DEFAULT '',\n Level LowCardinality(String) `json:$.level` DEFAULT '',\n Message String `json:$.message` DEFAULT '',\n TargetSelector String `json:$.target_selector` DEFAULT '',\n TargetText String `json:$.target_text` DEFAULT '',\n NetMethod LowCardinality(String) `json:$.net_method` DEFAULT '',\n NetUrl String `json:$.net_url` DEFAULT '',\n NetStatus UInt16 `json:$.net_status` DEFAULT 0,\n NetDurationMs UInt32 `json:$.net_duration_ms` DEFAULT 0,\n ErrorStack String `json:$.error_stack` DEFAULT '',\n Attributes Map(LowCardinality(String), String) `json:$.attributes`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, SessionId, Timestamp, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"" + }, + { + "name": "session_replay_events", + "content": "DESCRIPTION >\n Session replay rrweb events (one row per chunk, payload included). The ingest gateway gunzips the chunk and stores the event-array JSON in `Events`. Playback reads directly from ClickHouse — no R2.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n ChunkSeq UInt32 `json:$.chunk_seq`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n DurationMs UInt32 `json:$.duration_ms` DEFAULT 0,\n EventCount UInt32 `json:$.event_count` DEFAULT 0,\n ByteSize UInt32 `json:$.byte_size` DEFAULT 0,\n Events String `json:$.events`,\n IsCheckpoint UInt8 `json:$.is_checkpoint` DEFAULT 0\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, SessionId, ChunkSeq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"" + }, + { + "name": "session_replays", + "content": "DESCRIPTION >\n Per-session browser replay metadata (one row per session). Ingested directly from the @maple-dev/browser SDK via POST /v1/sessionReplays/meta. Event payloads live inline in session_replay_events; this holds only queryable metadata. ReplacingMergeTree(Version) for start/end upsert.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n StartTime DateTime64(9) `json:$.start_time`,\n EndTime Nullable(DateTime64(9)) `json:$.end_time`,\n DurationMs Nullable(UInt32) `json:$.duration_ms`,\n Status LowCardinality(String) `json:$.status`,\n UserId String `json:$.user_id`,\n UrlInitial String `json:$.url_initial`,\n UserAgent String `json:$.user_agent`,\n BrowserName LowCardinality(String) `json:$.browser_name`,\n OsName LowCardinality(String) `json:$.os_name`,\n DeviceType LowCardinality(String) `json:$.device_type`,\n Country LowCardinality(String) `json:$.country` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name`,\n PageViews UInt32 `json:$.page_views` DEFAULT 0,\n ClickCount UInt32 `json:$.click_count` DEFAULT 0,\n ErrorCount UInt32 `json:$.error_count` DEFAULT 0,\n TraceIds Array(String) `json:$.trace_ids[:]` DEFAULT [],\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n Version UInt32 `json:$.version`\n\nENGINE \"ReplacingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(StartTime)\"\nENGINE_SORTING_KEY \"OrgId, SessionId\"\nENGINE_TTL \"toDate(StartTime) + INTERVAL 30 DAY\"\nENGINE_VER \"Version\"" + }, + { + "name": "span_metrics_calls_hourly", + "content": "DESCRIPTION >\n Hourly per-series last-value (argMax) rollup of the span-metrics calls counter. AggregatingMergeTree MV target powering sampling-aware throughput without scanning raw metrics_sum.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n SpanKind LowCardinality(String),\n AttrFingerprint UInt64,\n ResourceFingerprint UInt64,\n StartTimeUnix DateTime64(9),\n LastValue AggregateFunction(argMax, Float64, DateTime64(9))\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"" + }, + { + "name": "trace_detail_spans", + "content": "DESCRIPTION >\n All spans for a trace, sorted by TraceId for fast detail lookups. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String))\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, TraceId, SpanId\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"" + }, + { + "name": "trace_list_mv", + "content": "DESCRIPTION >\n Pre-materialized root spans for the trace list view. Extracts HTTP attributes and normalizes span names at write time. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n TraceId String,\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n SpanName String,\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n HttpMethod LowCardinality(String),\n HttpRoute String,\n HttpStatusCode LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n HasError UInt8,\n TraceState String,\n ServiceNamespace LowCardinality(String)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, TraceId\"\nENGINE_TTL \"Timestamp + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4" + }, + { + "name": "traces", + "content": "DESCRIPTION >\n A table that contains trace data from OpenTelemetry in Tinybird format.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n Timestamp DateTime64(9) `json:$.start_time`,\n TraceId String `json:$.trace_id`,\n SpanId String `json:$.span_id`,\n ParentSpanId String `json:$.parent_span_id`,\n TraceState String `json:$.trace_state`,\n SpanName LowCardinality(String) `json:$.span_name`,\n SpanKind LowCardinality(String) `json:$.span_kind`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n Duration UInt64 `json:$.duration` DEFAULT 0,\n StatusCode LowCardinality(String) `json:$.status_code`,\n StatusMessage String `json:$.status_message`,\n SpanAttributes Map(LowCardinality(String), String) `json:$.span_attributes`,\n EventsTimestamp Array(DateTime64(9)) `json:$.events_timestamp[:]`,\n EventsName Array(LowCardinality(String)) `json:$.events_name[:]`,\n EventsAttributes Array(Map(LowCardinality(String), String)) `json:$.events_attributes[:]`,\n LinksTraceId Array(String) `json:$.links_trace_id[:]`,\n LinksSpanId Array(String) `json:$.links_span_id[:]`,\n LinksTraceState Array(String) `json:$.links_trace_state[:]`,\n LinksAttributes Array(Map(LowCardinality(String), String)) `json:$.links_attributes[:]`,\n SampleRate Float64 `json:$.SampleRate` DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 `json:$.IsEntryPoint` DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, SpanName, toDateTime(Timestamp)\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes), mapValues(SpanAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1" + }, + { + "name": "traces_aggregates_hourly", + "content": "DESCRIPTION >\n Hourly pre-aggregated trace metrics with sampling-weighted state columns. Generalized MV target for timeseries/breakdown/service-overview queries. AggregatingMergeTree.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"" + } ] as const export const pipes = [ - { - name: "error_events_by_time_mv", - content: - "DESCRIPTION >\n Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.\n\nNODE error_events_by_time_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_events_by_time", - }, - { - name: "error_events_mv", - content: - "DESCRIPTION >\n Materializes per-occurrence error events from traces. Unwraps the first OTel exception event and computes a cityHash64 FingerprintHash for issue grouping.\n\nNODE error_events_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_events", - }, - { - name: "error_spans_mv", - content: - "DESCRIPTION >\n Materializes error spans from traces. Pre-filters to StatusCode='Error' and pre-extracts deployment.environment.\n\nNODE error_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n StatusMessage,\n Duration,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_spans", - }, - { - name: "log_attribute_keys_mv", - content: - "DESCRIPTION >\n Aggregates log attribute keys from logs hourly.\n\nNODE log_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(LogAttributes)) AS AttributeKey,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n WHERE LogAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly", - }, - { - name: "log_attribute_values_mv", - content: - "DESCRIPTION >\n Aggregates log attribute values from logs hourly.\n\nNODE log_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n ARRAY JOIN\n mapKeys(LogAttributes) AS AttributeKey,\n mapValues(LogAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", - }, - { - name: "logs_aggregates_hourly_mv", - content: - "DESCRIPTION >\n Pre-aggregates logs hourly by service × severity × deployment env. Drop-in for severity-distribution and log-volume queries.\n\nNODE logs_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(TimestampTime) AS Hour,\n ServiceName,\n SeverityText,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS Count,\n sum(length(Body) + 200) AS SizeBytes,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM logs\n GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace\n\nTYPE MATERIALIZED\nDATASOURCE logs_aggregates_hourly", - }, - { - name: "metric_attribute_keys_mv", - content: - "DESCRIPTION >\n Aggregates metric attribute keys from metrics_sum hourly.\n\nNODE metric_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n arrayJoin(mapKeys(Attributes)) AS AttributeKey,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n WHERE Attributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly", - }, - { - name: "metric_attribute_values_mv", - content: - "DESCRIPTION >\n Aggregates metric attribute values from metrics_sum hourly.\n\nNODE metric_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n ARRAY JOIN\n mapKeys(Attributes) AS AttributeKey,\n mapValues(Attributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", - }, - { - name: "metric_catalog_exp_histogram_mv", - content: - "DESCRIPTION >\n Hourly rollup of distinct exponential histogram metrics into metric_catalog.\n\nNODE metric_catalog_exp_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'exponential_histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_exponential_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog", - }, - { - name: "metric_catalog_gauge_mv", - content: - "DESCRIPTION >\n Hourly rollup of distinct gauge metrics into metric_catalog.\n\nNODE metric_catalog_gauge_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'gauge' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_gauge\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog", - }, - { - name: "metric_catalog_histogram_mv", - content: - "DESCRIPTION >\n Hourly rollup of distinct histogram metrics into metric_catalog.\n\nNODE metric_catalog_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog", - }, - { - name: "metric_catalog_sum_mv", - content: - "DESCRIPTION >\n Hourly rollup of distinct sum metrics into metric_catalog.\n\nNODE metric_catalog_sum_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog", - }, - { - name: "service_external_edges_hourly_mv", - content: - "DESCRIPTION >\n Pre-aggregates Client/Producer spans without db.system.name into hourly service-to-external-target edges (http / messaging / rpc) for the service-detail Dependencies tab.\n\nNODE service_external_edges_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '',\n if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR SpanAttributes['messaging.destination'] != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''\n\nTYPE MATERIALIZED\nDATASOURCE service_external_edges_hourly", - }, - { - name: "service_map_children_mv", - content: - "DESCRIPTION >\n Populates service_map_children with Server/Consumer spans that have a parent for efficient JOIN lookups.\n\nNODE service_map_children_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''\n\nTYPE MATERIALIZED\nDATASOURCE service_map_children", - }, - { - name: "service_map_db_edges_hourly_mv", - content: - "DESCRIPTION >\n Pre-aggregates Client/Producer spans with db.system.name into hourly service-to-database edge buckets for fast service map db-node queries.\n\nNODE service_map_db_edges_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_map_db_edges_hourly", - }, - { - name: "service_map_db_query_shapes_hourly_mv", - content: - "DESCRIPTION >\n Pre-aggregates Client/Producer DB spans into hourly query-shape buckets (normalized fingerprint + label + sample-weighted t-digest) for the service map's database detail panel.\n\nNODE service_map_db_query_shapes_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n coalesce(\n nullIf(SpanAttributes['db.query.fingerprint'], ''),\n nullIf(SpanAttributes['db.statement.fingerprint'], ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\\'[^\\']*\\'', '?'), '\\\\bin\\\\s*\\\\([^)]*\\\\)', 'in (?)'), '[0-9]+(\\\\.[0-9]+)?', '?'), '\\\\s+', ' '), '^\\\\s+|\\\\s+$', ''))), ''), ''),\n toString(cityHash64(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n )))\n ) AS QueryKey,\n any(substring(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n ), 1, 220)) AS QueryLabel,\n any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(SampleRate) AS EstimatedCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv, QueryKey\n\nTYPE MATERIALIZED\nDATASOURCE service_map_db_query_shapes_hourly", - }, - { - name: "service_map_spans_mv", - content: - "DESCRIPTION >\n Materialized view projecting trace spans needed for service dependency map. Extracts deployment.environment from Map columns at write time.\n\nNODE service_map_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer')\n\nTYPE MATERIALIZED\nDATASOURCE service_map_spans", - }, - { - name: "service_overview_spans_mv", - content: - "DESCRIPTION >\n Materialized view projecting service entry point spans (Server/Consumer + root) for service overview queries. Pre-extracts deployment attributes from ResourceAttributes at write time.\n\nNODE service_overview_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE service_overview_spans", - }, - { - name: "service_platforms_hourly_mv", - content: - "DESCRIPTION >\n Pre-aggregates per-service hosting-platform resource attributes (k8s.*, cloud.*, faas.*) into hourly buckets for the service map's runtime-icon resolver.\n\nNODE service_platforms_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_platforms_hourly", - }, - { - name: "service_usage_logs_mv", - content: - "DESCRIPTION >\n Materialized view to aggregate log usage statistics per service per hour\n\nNODE service_usage_logs_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(TimestampTime) AS Hour,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM logs\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", - }, - { - name: "service_usage_metrics_exp_histogram_mv", - content: - "DESCRIPTION >\n Materialized view to aggregate exponential histogram metric usage statistics per service per hour\n\nNODE service_usage_metrics_exp_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n count() AS ExpHistogramMetricCount,\n count() * 300 AS ExpHistogramMetricSizeBytes\n FROM metrics_exponential_histogram\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", - }, - { - name: "service_usage_metrics_gauge_mv", - content: - "DESCRIPTION >\n Materialized view to aggregate gauge metric usage statistics per service per hour\n\nNODE service_usage_metrics_gauge_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n count() AS GaugeMetricCount,\n count() * 150 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_gauge\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", - }, - { - name: "service_usage_metrics_histogram_mv", - content: - "DESCRIPTION >\n Materialized view to aggregate histogram metric usage statistics per service per hour\n\nNODE service_usage_metrics_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n count() AS HistogramMetricCount,\n count() * 250 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_histogram\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", - }, - { - name: "service_usage_metrics_sum_mv", - content: - "DESCRIPTION >\n Materialized view to aggregate sum metric usage statistics per service per hour\n\nNODE service_usage_metrics_sum_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n count() AS SumMetricCount,\n count() * 150 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_sum\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", - }, - { - name: "service_usage_traces_mv", - content: - "DESCRIPTION >\n Materialized view to aggregate trace/span usage statistics per service per hour\n\nNODE service_usage_traces_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS TraceCount,\n sum(length(SpanName) + 300) AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM traces\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", - }, - { - name: "span_metrics_calls_hourly_mv", - content: - "DESCRIPTION >\n Hourly per-series argMax(value) rollup of the span-metrics calls counter into span_metrics_calls_hourly.\n\nNODE span_metrics_calls_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n ServiceName,\n MetricName,\n Attributes['span.kind'] AS SpanKind,\n cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint,\n cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint,\n StartTimeUnix,\n argMaxState(Value, TimeUnix) AS LastValue\n FROM metrics_sum\n WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic\n GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix\n\nTYPE MATERIALIZED\nDATASOURCE span_metrics_calls_hourly", - }, - { - name: "trace_detail_spans_mv", - content: - "DESCRIPTION >\n Populates trace_detail_spans with all spans re-sorted by TraceId for fast detail lookups\n\nNODE trace_detail_spans_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n SpanName,\n SpanKind,\n ServiceName,\n Duration,\n StatusCode,\n StatusMessage,\n SpanAttributes,\n ResourceAttributes,\n EventsTimestamp,\n EventsName,\n EventsAttributes\n FROM traces\n\nTYPE MATERIALIZED\nDATASOURCE trace_detail_spans", - }, - { - name: "trace_list_mv_mv", - content: - "DESCRIPTION >\n Populates trace_list_mv from root spans with pre-extracted HTTP attributes and normalized span names.\n\nNODE trace_list_mv_node\nSQL >\n SELECT\n OrgId,\n TraceId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n if(\n (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS'))\n AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''),\n concat(\n if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName),\n ' ',\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])\n ),\n SpanName\n ) AS SpanName,\n SpanKind,\n Duration,\n StatusCode,\n if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod,\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute,\n if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n toUInt8(\n StatusCode = 'Error'\n OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500)\n OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500)\n ) AS HasError,\n TraceState,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE trace_list_mv", - }, - { - name: "trace_resource_attribute_keys_mv", - content: - "DESCRIPTION >\n Aggregates resource attribute keys from traces hourly.\n\nNODE trace_resource_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE ResourceAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly", - }, - { - name: "trace_resource_attribute_values_mv", - content: - "DESCRIPTION >\n Aggregates resource attribute values from traces hourly.\n\nNODE trace_resource_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(ResourceAttributes) AS AttributeKey,\n mapValues(ResourceAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", - }, - { - name: "trace_span_attribute_keys_mv", - content: - "DESCRIPTION >\n Aggregates span attribute keys from traces hourly.\n\nNODE trace_span_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE SpanAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly", - }, - { - name: "trace_span_attribute_values_mv", - content: - "DESCRIPTION >\n Aggregates span attribute values from traces hourly.\n\nNODE trace_span_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(SpanAttributes) AS AttributeKey,\n mapValues(SpanAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", - }, - { - name: "traces_aggregates_hourly_mv", - content: - "DESCRIPTION >\n Pre-aggregates spans hourly with sample-weighted state columns (count, duration sum, t-digest quantiles, error count). Sample-correct from day one via SampleRate materialized column on traces.\n\nNODE traces_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanName,\n SpanKind,\n StatusCode,\n IsEntryPoint,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n sum(SampleRate) AS WeightedCount,\n sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum,\n sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount,\n quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles,\n min(Duration) AS DurationMin,\n max(Duration) AS DurationMax\n FROM traces\n GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE traces_aggregates_hourly", - }, + { + "name": "error_events_by_time_mv", + "content": "DESCRIPTION >\n Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.\n\nNODE error_events_by_time_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_events_by_time" + }, + { + "name": "error_events_mv", + "content": "DESCRIPTION >\n Materializes per-occurrence error events from traces. Unwraps the first OTel exception event and computes a cityHash64 FingerprintHash for issue grouping.\n\nNODE error_events_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_events" + }, + { + "name": "error_spans_mv", + "content": "DESCRIPTION >\n Materializes error spans from traces. Pre-filters to StatusCode='Error' and pre-extracts deployment.environment.\n\nNODE error_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n StatusMessage,\n Duration,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_spans" + }, + { + "name": "log_attribute_keys_mv", + "content": "DESCRIPTION >\n Aggregates log attribute keys from logs hourly.\n\nNODE log_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(LogAttributes)) AS AttributeKey,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n WHERE LogAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly" + }, + { + "name": "log_attribute_values_mv", + "content": "DESCRIPTION >\n Aggregates log attribute values from logs hourly.\n\nNODE log_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n ARRAY JOIN\n mapKeys(LogAttributes) AS AttributeKey,\n mapValues(LogAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly" + }, + { + "name": "logs_aggregates_hourly_mv", + "content": "DESCRIPTION >\n Pre-aggregates logs hourly by service × severity × deployment env. Drop-in for severity-distribution and log-volume queries.\n\nNODE logs_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(TimestampTime) AS Hour,\n ServiceName,\n SeverityText,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS Count,\n sum(length(Body) + 200) AS SizeBytes,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM logs\n GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace\n\nTYPE MATERIALIZED\nDATASOURCE logs_aggregates_hourly" + }, + { + "name": "metric_attribute_keys_mv", + "content": "DESCRIPTION >\n Aggregates metric attribute keys from metrics_sum hourly.\n\nNODE metric_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n arrayJoin(mapKeys(Attributes)) AS AttributeKey,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n WHERE Attributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly" + }, + { + "name": "metric_attribute_values_mv", + "content": "DESCRIPTION >\n Aggregates metric attribute values from metrics_sum hourly.\n\nNODE metric_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n ARRAY JOIN\n mapKeys(Attributes) AS AttributeKey,\n mapValues(Attributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly" + }, + { + "name": "metric_catalog_exp_histogram_mv", + "content": "DESCRIPTION >\n Hourly rollup of distinct exponential histogram metrics into metric_catalog.\n\nNODE metric_catalog_exp_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'exponential_histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_exponential_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog" + }, + { + "name": "metric_catalog_gauge_mv", + "content": "DESCRIPTION >\n Hourly rollup of distinct gauge metrics into metric_catalog.\n\nNODE metric_catalog_gauge_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'gauge' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_gauge\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog" + }, + { + "name": "metric_catalog_histogram_mv", + "content": "DESCRIPTION >\n Hourly rollup of distinct histogram metrics into metric_catalog.\n\nNODE metric_catalog_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog" + }, + { + "name": "metric_catalog_sum_mv", + "content": "DESCRIPTION >\n Hourly rollup of distinct sum metrics into metric_catalog.\n\nNODE metric_catalog_sum_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog" + }, + { + "name": "service_external_edges_hourly_mv", + "content": "DESCRIPTION >\n Pre-aggregates Client/Producer spans without db.system.name into hourly service-to-external-target edges (http / messaging / rpc) for the service-detail Dependencies tab.\n\nNODE service_external_edges_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '',\n if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR SpanAttributes['messaging.destination'] != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''\n\nTYPE MATERIALIZED\nDATASOURCE service_external_edges_hourly" + }, + { + "name": "service_map_children_mv", + "content": "DESCRIPTION >\n Populates service_map_children with Server/Consumer spans that have a parent for efficient JOIN lookups.\n\nNODE service_map_children_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''\n\nTYPE MATERIALIZED\nDATASOURCE service_map_children" + }, + { + "name": "service_map_db_edges_hourly_mv", + "content": "DESCRIPTION >\n Pre-aggregates Client/Producer spans with db.system.name into hourly service-to-database edge buckets for fast service map db-node queries.\n\nNODE service_map_db_edges_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_map_db_edges_hourly" + }, + { + "name": "service_map_db_query_shapes_hourly_mv", + "content": "DESCRIPTION >\n Pre-aggregates Client/Producer DB spans into hourly query-shape buckets (normalized fingerprint + label + sample-weighted t-digest) for the service map's database detail panel.\n\nNODE service_map_db_query_shapes_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n coalesce(\n nullIf(SpanAttributes['db.query.fingerprint'], ''),\n nullIf(SpanAttributes['db.statement.fingerprint'], ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\\'[^\\']*\\'', '?'), '\\\\bin\\\\s*\\\\([^)]*\\\\)', 'in (?)'), '[0-9]+(\\\\.[0-9]+)?', '?'), '\\\\s+', ' '), '^\\\\s+|\\\\s+$', ''))), ''), ''),\n toString(cityHash64(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n )))\n ) AS QueryKey,\n any(substring(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n ), 1, 220)) AS QueryLabel,\n any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(SampleRate) AS EstimatedCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv, QueryKey\n\nTYPE MATERIALIZED\nDATASOURCE service_map_db_query_shapes_hourly" + }, + { + "name": "service_map_spans_mv", + "content": "DESCRIPTION >\n Materialized view projecting trace spans needed for service dependency map. Extracts deployment.environment from Map columns at write time.\n\nNODE service_map_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer')\n\nTYPE MATERIALIZED\nDATASOURCE service_map_spans" + }, + { + "name": "service_overview_spans_mv", + "content": "DESCRIPTION >\n Materialized view projecting service entry point spans (Server/Consumer + root) for service overview queries. Pre-extracts deployment attributes from ResourceAttributes at write time.\n\nNODE service_overview_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE service_overview_spans" + }, + { + "name": "service_platforms_hourly_mv", + "content": "DESCRIPTION >\n Pre-aggregates per-service hosting-platform resource attributes (k8s.*, cloud.*, faas.*) into hourly buckets for the service map's runtime-icon resolver.\n\nNODE service_platforms_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_platforms_hourly" + }, + { + "name": "service_usage_logs_mv", + "content": "DESCRIPTION >\n Materialized view to aggregate log usage statistics per service per hour\n\nNODE service_usage_logs_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(TimestampTime) AS Hour,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM logs\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage" + }, + { + "name": "service_usage_metrics_exp_histogram_mv", + "content": "DESCRIPTION >\n Materialized view to aggregate exponential histogram metric usage statistics per service per hour\n\nNODE service_usage_metrics_exp_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n count() AS ExpHistogramMetricCount,\n count() * 300 AS ExpHistogramMetricSizeBytes\n FROM metrics_exponential_histogram\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage" + }, + { + "name": "service_usage_metrics_gauge_mv", + "content": "DESCRIPTION >\n Materialized view to aggregate gauge metric usage statistics per service per hour\n\nNODE service_usage_metrics_gauge_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n count() AS GaugeMetricCount,\n count() * 150 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_gauge\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage" + }, + { + "name": "service_usage_metrics_histogram_mv", + "content": "DESCRIPTION >\n Materialized view to aggregate histogram metric usage statistics per service per hour\n\nNODE service_usage_metrics_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n count() AS HistogramMetricCount,\n count() * 250 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_histogram\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage" + }, + { + "name": "service_usage_metrics_sum_mv", + "content": "DESCRIPTION >\n Materialized view to aggregate sum metric usage statistics per service per hour\n\nNODE service_usage_metrics_sum_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n count() AS SumMetricCount,\n count() * 150 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_sum\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage" + }, + { + "name": "service_usage_traces_mv", + "content": "DESCRIPTION >\n Materialized view to aggregate trace/span usage statistics per service per hour\n\nNODE service_usage_traces_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS TraceCount,\n sum(length(SpanName) + 300) AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM traces\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage" + }, + { + "name": "span_metrics_calls_hourly_mv", + "content": "DESCRIPTION >\n Hourly per-series argMax(value) rollup of the span-metrics calls counter into span_metrics_calls_hourly.\n\nNODE span_metrics_calls_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n ServiceName,\n MetricName,\n Attributes['span.kind'] AS SpanKind,\n cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint,\n cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint,\n StartTimeUnix,\n argMaxState(Value, TimeUnix) AS LastValue\n FROM metrics_sum\n WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic\n GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix\n\nTYPE MATERIALIZED\nDATASOURCE span_metrics_calls_hourly" + }, + { + "name": "trace_detail_spans_mv", + "content": "DESCRIPTION >\n Populates trace_detail_spans with all spans re-sorted by TraceId for fast detail lookups\n\nNODE trace_detail_spans_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n SpanName,\n SpanKind,\n ServiceName,\n Duration,\n StatusCode,\n StatusMessage,\n SpanAttributes,\n ResourceAttributes,\n EventsTimestamp,\n EventsName,\n EventsAttributes\n FROM traces\n\nTYPE MATERIALIZED\nDATASOURCE trace_detail_spans" + }, + { + "name": "trace_list_mv_mv", + "content": "DESCRIPTION >\n Populates trace_list_mv from root spans with pre-extracted HTTP attributes and normalized span names.\n\nNODE trace_list_mv_node\nSQL >\n SELECT\n OrgId,\n TraceId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n if(\n (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS'))\n AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''),\n concat(\n if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName),\n ' ',\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])\n ),\n SpanName\n ) AS SpanName,\n SpanKind,\n Duration,\n StatusCode,\n if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod,\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute,\n if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n toUInt8(\n StatusCode = 'Error'\n OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500)\n OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500)\n ) AS HasError,\n TraceState,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE trace_list_mv" + }, + { + "name": "trace_resource_attribute_keys_mv", + "content": "DESCRIPTION >\n Aggregates resource attribute keys from traces hourly.\n\nNODE trace_resource_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE ResourceAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly" + }, + { + "name": "trace_resource_attribute_values_mv", + "content": "DESCRIPTION >\n Aggregates resource attribute values from traces hourly.\n\nNODE trace_resource_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(ResourceAttributes) AS AttributeKey,\n mapValues(ResourceAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly" + }, + { + "name": "trace_span_attribute_keys_mv", + "content": "DESCRIPTION >\n Aggregates span attribute keys from traces hourly.\n\nNODE trace_span_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE SpanAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly" + }, + { + "name": "trace_span_attribute_values_mv", + "content": "DESCRIPTION >\n Aggregates span attribute values from traces hourly.\n\nNODE trace_span_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(SpanAttributes) AS AttributeKey,\n mapValues(SpanAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly" + }, + { + "name": "traces_aggregates_hourly_mv", + "content": "DESCRIPTION >\n Pre-aggregates spans hourly with sample-weighted state columns (count, duration sum, t-digest quantiles, error count). Sample-correct from day one via SampleRate materialized column on traces.\n\nNODE traces_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanName,\n SpanKind,\n StatusCode,\n IsEntryPoint,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n sum(SampleRate) AS WeightedCount,\n sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum,\n sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount,\n quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles,\n min(Duration) AS DurationMin,\n max(Duration) AS DurationMax\n FROM traces\n GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE traces_aggregates_hourly" + } ] as const export const tinybirdProjectManifest = { - projectRevision, - datasources, - pipes, + projectRevision, + datasources, + pipes, } as const diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index ca9e6ecb9..fe252941f 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -1,4 +1,5 @@ import { defineDatasource, t, engine, column, type InferRow } from "@tinybirdco/sdk" +import { attrItemsExpr, BODY_TOKEN_INDEX_EXPR } from "./index-exprs" /** * OpenTelemetry logs datasource @@ -48,6 +49,43 @@ export const logs = defineDatasource("logs", { type: "bloom_filter(0.01)", granularity: 1, }, + // Full-text token index on the log body. `Body ILIKE '%term%'` otherwise + // scans whole partitions (guard-railed by `LOGS_BODY_SEARCH_SETTINGS`); + // the query engine AND-s a `hasToken(lower(Body), )` pre-filter for + // interior tokens so this index prunes granules. tokenbf params match + // ClickStack: 32 KB filter, 3 hashes, seed 0. + { + name: "idx_body_tokens", + expr: BODY_TOKEN_INDEX_EXPR, + type: "tokenbf_v1(32768, 3, 0)", + granularity: 1, + }, + // Attribute-map blooms, mirroring `traces`. Enable granule skipping for + // `LogAttributes['k'] = 'v'` / key-existence filters on the logs table. + { + name: "idx_log_attr_keys", + expr: "mapKeys(LogAttributes)", + type: "bloom_filter(0.01)", + granularity: 1, + }, + { + name: "idx_log_attr_vals", + expr: "mapValues(LogAttributes)", + type: "bloom_filter(0.01)", + granularity: 1, + }, + { + name: "idx_log_resource_attr_keys", + expr: "mapKeys(ResourceAttributes)", + type: "bloom_filter(0.01)", + granularity: 1, + }, + { + name: "idx_log_resource_attr_vals", + expr: "mapValues(ResourceAttributes)", + type: "bloom_filter(0.01)", + granularity: 1, + }, ], engine: engine.mergeTree({ partitionKey: "toDate(TimestampTime)", @@ -199,6 +237,23 @@ export const traces = defineDatasource("traces", { type: "bloom_filter(0.01)", granularity: 1, }, + // ClickStack "Items" indexes: a bloom over concatenated `key=value` strings + // so an equality filter `SpanAttributes['k'] = 'v'` prunes granules via + // `has(, 'k=v')`. The mapKeys/mapValues blooms above only skip on + // key OR value independently; the items index skips on the exact pair. + // The `expr` MUST match `attrItemsExpr(...)` used by the query engine. + { + name: "idx_span_attr_items", + expr: attrItemsExpr("SpanAttributes"), + type: "bloom_filter(0.01)", + granularity: 1, + }, + { + name: "idx_resource_attr_items", + expr: attrItemsExpr("ResourceAttributes"), + type: "bloom_filter(0.01)", + granularity: 1, + }, ], engine: engine.mergeTree({ partitionKey: "toDate(Timestamp)", diff --git a/packages/domain/src/tinybird/index-exprs.ts b/packages/domain/src/tinybird/index-exprs.ts new file mode 100644 index 000000000..a3895a577 --- /dev/null +++ b/packages/domain/src/tinybird/index-exprs.ts @@ -0,0 +1,33 @@ +// --------------------------------------------------------------------------- +// Shared skip-index expressions +// +// These SQL expression strings are used in TWO places that MUST stay byte-for- +// byte identical, otherwise the index silently stops applying: +// +// 1. The `expr` of a data-skipping INDEX on a datasource (datasources.ts) and +// the equivalent `ALTER TABLE ... ADD INDEX` in a BYO-ClickHouse migration. +// 2. The query-engine predicate that references the same expression so +// ClickHouse's index analysis recognizes it (e.g. `has(, …)`). +// +// Keep the single source here and import it on both sides. +// --------------------------------------------------------------------------- + +/** + * ClickStack "Items" expression: concatenates each map entry into a single + * `key=value` string so an equality filter `Map['k'] = 'v'` can be pruned by a + * `bloom_filter` skip index via `has(, 'k=v')`. + * + * `mapKeys`/`mapValues` preserve element order, so `arrayMap` pairs them + * correctly. Applied to `SpanAttributes` / `ResourceAttributes` on `traces`. + */ +export function attrItemsExpr(mapName: string): string { + return `arrayMap((k, v) -> concat(k, '=', v), mapKeys(${mapName}), mapValues(${mapName}))` +} + +/** + * Body full-text token index expression. The index is on `lower(Body)` so a + * case-insensitive `hasToken(lower(Body), '')` pre-filter can + * prune granules ahead of the substring `ILIKE` (see the query engine's + * `body-search.ts`). + */ +export const BODY_TOKEN_INDEX_EXPR = "lower(Body)" diff --git a/packages/query-engine/src/ch/queries/body-search.test.ts b/packages/query-engine/src/ch/queries/body-search.test.ts new file mode 100644 index 000000000..6c234110d --- /dev/null +++ b/packages/query-engine/src/ch/queries/body-search.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest" +import { compileCH, from } from "@maple-dev/clickhouse-builder" +import { Logs } from "../tables" +import { bodySearchConditions, extractSafeSearchTokens } from "./body-search" + +// --------------------------------------------------------------------------- +// extractSafeSearchTokens — only interior, separator-bounded ASCII tokens +// --------------------------------------------------------------------------- + +describe("extractSafeSearchTokens", () => { + it.each([ + // A single word is one edge-to-edge run — no interior boundary, so unsafe + // (it could be a prefix/suffix of a longer token in Body). + ["exception", []], + ["timeout", []], + // Interior tokens are bounded on both sides by separators within the term. + ["error: user 42 timeout", ["user", "42"]], + ["GET /api/users failed", ["api", "users"]], + // Lowercased to match the `lower(Body)` index. + ["a FOO b", ["foo"]], + // Underscore is not treated as a separator (tokenizer-version-safe). + ["foo_bar baz qux", ["baz"]], + // Non-ASCII disables the pre-filter entirely (JS/CH lower() divergence). + ["café con leche", []], + // Dedup: the second "user" is dropped; edge tokens x/z excluded, y kept. + ["x user y user z", ["user", "y"]], + // Empty / whitespace. + ["", []], + [" ", []], + ])("extractSafeSearchTokens(%j) = %j", (term, expected) => { + expect(extractSafeSearchTokens(term)).toEqual(expected) + }) + + it("caps at 4 tokens", () => { + expect(extractSafeSearchTokens("a b c d e f g h").length).toBe(4) + }) +}) + +// --------------------------------------------------------------------------- +// bodySearchConditions — compiled SQL shape +// --------------------------------------------------------------------------- + +function whereSql(term: string): string { + const q = from(Logs) + .select(($) => ({ body: $.Body })) + .where(($) => bodySearchConditions($.Body, term)) + .format("JSON") + return compileCH(q, {}).sql +} + +describe("bodySearchConditions", () => { + it("emits hasToken pre-filters plus the substring ILIKE for a phrase", () => { + const sql = whereSql("error: user 42 timeout") + expect(sql).toContain("hasToken(lower(Body), 'user')") + expect(sql).toContain("hasToken(lower(Body), '42')") + expect(sql).toContain("Body ILIKE '%error: user 42 timeout%'") + }) + + it("degrades to ILIKE only for a single word (no safe token)", () => { + const sql = whereSql("exception") + expect(sql).not.toContain("hasToken") + expect(sql).toContain("Body ILIKE '%exception%'") + }) +}) diff --git a/packages/query-engine/src/ch/queries/body-search.ts b/packages/query-engine/src/ch/queries/body-search.ts new file mode 100644 index 000000000..3bc4b718f --- /dev/null +++ b/packages/query-engine/src/ch/queries/body-search.ts @@ -0,0 +1,80 @@ +// --------------------------------------------------------------------------- +// Log body search - token pre-filter + substring predicate +// +// `Body ILIKE '%term%'` is substring-matching and scans whole partitions. The +// `logs.idx_body_tokens` skip index (`tokenbf_v1` over `lower(Body)`) can prune +// granules, but only via whole-token `hasToken(...)` predicates. Whole-token +// semantics are NOT the same as substring semantics, so we can only AND a +// `hasToken` pre-filter for tokens that are *guaranteed* to appear as whole +// tokens whenever the substring matches - otherwise we would silently drop rows. +// +// The final `ILIKE` predicate always stays, so the user-visible result set is +// byte-for-byte unchanged; the token predicates are a pure granule-pruning +// accelerator that the query planner is free to satisfy from the index. +// --------------------------------------------------------------------------- + +import * as CH from "@maple-dev/clickhouse-builder/expr" + +/** At most this many token pre-filters (diminishing pruning, longer SQL). */ +const MAX_TOKENS = 4 + +const NON_ASCII = /[^\x00-\x7F]/ +const ALNUM_RUN = /[A-Za-z0-9]+/g + +/** + * Tokens of `term` that are safe to AND as a `hasToken(lower(Body), token)` + * pre-filter without changing the substring semantics of `Body ILIKE '%term%'`. + * + * Soundness: if `term` occurs as a substring of `Body`, an alphanumeric run of + * `term` is guaranteed to be a *whole* ClickHouse token of `Body` only when it + * is bounded on BOTH sides - within `term` itself - by a definite token + * separator. An edge run (touching the start/end of `term`) could be extended + * inside `Body` (`"conn"` inside `"connection"`), so it is unsafe. + * + * Conservative choices (each only ever drops an optimization, never a result): + * - ASCII-only: JS `String.toLowerCase()` and ClickHouse `lower()` agree on + * case folding for ASCII; outside ASCII they diverge, so bail entirely. + * - `_` is NOT treated as a separator. ClickHouse's tokenizer treatment of + * underscore is version-dependent; requiring boundaries in `[^A-Za-z0-9_]` + * keeps us correct whether or not `_` splits tokens. + */ +export function extractSafeSearchTokens(term: string): string[] { + if (NON_ASCII.test(term)) return [] + + const tokens: string[] = [] + const seen = new Set() + ALNUM_RUN.lastIndex = 0 + let match: RegExpExecArray | null + while ((match = ALNUM_RUN.exec(term)) !== null) { + const start = match.index + const end = start + match[0].length + // A maximal `[A-Za-z0-9]+` run's neighbours (if present) are non- + // alphanumeric. Require both neighbours to exist and to be a definite + // separator - i.e. present and not `_`. A missing neighbour = term edge. + const leftBounded = start > 0 && term[start - 1] !== "_" + const rightBounded = end < term.length && term[end] !== "_" + if (!leftBounded || !rightBounded) continue + + const token = match[0].toLowerCase() + if (seen.has(token)) continue + seen.add(token) + tokens.push(token) + if (tokens.length >= MAX_TOKENS) break + } + return tokens +} + +/** + * Conditions for a log body search: safe `hasToken` pre-filters (index-prunable) + * followed by the substring `ILIKE` that fixes the exact semantics. Spread into + * a query's `where` array. When no safe token exists (e.g. a single word) this + * degrades to just the `ILIKE`, i.e. today's behavior. + */ +export function bodySearchConditions(body: CH.Expr, term: string): CH.Condition[] { + const loweredBody = CH.lower_(body) + const conditions: CH.Condition[] = extractSafeSearchTokens(term).map((token) => + CH.hasToken(loweredBody, CH.lit(token)), + ) + conditions.push(body.ilike(`%${term}%`)) + return conditions +} diff --git a/packages/query-engine/src/ch/queries/logs.ts b/packages/query-engine/src/ch/queries/logs.ts index 797b80f4f..2b7b78720 100644 --- a/packages/query-engine/src/ch/queries/logs.ts +++ b/packages/query-engine/src/ch/queries/logs.ts @@ -12,6 +12,7 @@ import type { ColumnDefs } from "@maple-dev/clickhouse-builder/types" import * as T from "@maple-dev/clickhouse-builder/types" import { unionAll, type CHUnionQuery } from "@maple-dev/clickhouse-builder" import { Logs, LogsAggregatesHourly } from "../tables" +import { bodySearchConditions } from "./body-search" import { finalizeTimeseries } from "./series-cap" // --------------------------------------------------------------------------- @@ -356,7 +357,7 @@ export function logsCountQuery(opts: LogsQueryOpts): CHQuery $.SeverityText.eq(v)), CH.when(opts.traceId, (v: string) => $.TraceId.eq(v)), CH.when(opts.spanId, (v: string) => $.SpanId.eq(v)), - CH.when(opts.search, (v: string) => $.Body.ilike(`%${v}%`)), + ...(opts.search ? bodySearchConditions($.Body, opts.search) : []), environmentCondition($, opts), namespaceCondition($, opts), ]) @@ -451,7 +452,7 @@ export function logsListQuery(opts: LogsListOpts) { CH.when(opts.traceId, (v: string) => $.TraceId.eq(v)), CH.when(opts.spanId, (v: string) => $.SpanId.eq(v)), CH.when(opts.cursor, (v: string) => $.Timestamp.lt(v)), - CH.when(opts.search, (v: string) => $.Body.ilike(`%${v}%`)), + ...(opts.search ? bodySearchConditions($.Body, opts.search) : []), environmentCondition($, opts), namespaceCondition($, opts), ] diff --git a/packages/query-engine/src/ch/queries/query-helpers.ts b/packages/query-engine/src/ch/queries/query-helpers.ts index 3902475e1..4ef36b4b0 100644 --- a/packages/query-engine/src/ch/queries/query-helpers.ts +++ b/packages/query-engine/src/ch/queries/query-helpers.ts @@ -149,6 +149,11 @@ type TracesBaseWhereColumns = Pick< export function tracesBaseWhereConditions( $: ColumnAccessor, opts: TracesBaseWhereOpts, + config?: { + /** Emit `idx_*_attr_items` pre-filters for equality filters. Set only when + * reading the raw `traces` table, which carries those indexes. */ + itemsIndex?: boolean + }, ): Array { const mm = opts.matchModes const conditions: Array = [ @@ -215,12 +220,12 @@ export function tracesBaseWhereConditions( } if (opts.attributeFilters) { for (const af of opts.attributeFilters) { - conditions.push(buildAttrFilterCondition(af, "SpanAttributes")) + conditions.push(buildAttrFilterCondition(af, "SpanAttributes", { itemsIndex: config?.itemsIndex })) } } if (opts.resourceAttributeFilters) { for (const rf of opts.resourceAttributeFilters) { - conditions.push(buildAttrFilterCondition(rf, "ResourceAttributes")) + conditions.push(buildAttrFilterCondition(rf, "ResourceAttributes", { itemsIndex: config?.itemsIndex })) } } if (opts.excludedServiceNames?.length) { diff --git a/packages/query-engine/src/ch/queries/traces.test.ts b/packages/query-engine/src/ch/queries/traces.test.ts index 3f8e0516d..c51317189 100644 --- a/packages/query-engine/src/ch/queries/traces.test.ts +++ b/packages/query-engine/src/ch/queries/traces.test.ts @@ -128,6 +128,65 @@ describe("tracesListQuery", () => { }) }) +// --------------------------------------------------------------------------- +// tracesListQuery — ClickStack "Items" attribute-index pre-filter +// --------------------------------------------------------------------------- + +const SPAN_ITEMS = "has(arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes), mapValues(SpanAttributes))" +const RESOURCE_ITEMS = + "has(arrayMap((k, v) -> concat(k, '=', v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes))" + +describe("tracesListQuery attribute items pre-filter", () => { + it("adds a has(items,'k=v') pre-filter alongside the exact map equality", () => { + const q = tracesListQuery({ + attributeFilters: [{ key: "db.system", value: "postgres", mode: "equals" }], + }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain(`${SPAN_ITEMS}, 'db.system=postgres')`) + expect(sql).toContain("SpanAttributes['db.system'] = 'postgres'") + }) + + it("ORs the pre-filter across HTTP semconv alias spellings", () => { + const q = tracesListQuery({ + attributeFilters: [{ key: "http.method", value: "GET", mode: "equals" }], + }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain(`${SPAN_ITEMS}, 'http.method=GET')`) + expect(sql).toContain(`${SPAN_ITEMS}, 'http.request.method=GET')`) + }) + + it("uses the resource items index for resource attribute equality", () => { + const q = tracesListQuery({ + resourceAttributeFilters: [{ key: "deployment.environment", value: "prod", mode: "equals" }], + }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain(`${RESOURCE_ITEMS}, 'deployment.environment=prod')`) + }) + + it("does NOT emit the items pre-filter for contains, negated, or empty-value filters", () => { + const contains = compileCH( + tracesListQuery({ attributeFilters: [{ key: "db.system", value: "postgres", mode: "contains" }] }), + baseParams, + ).sql + expect(contains).not.toContain("has(arrayMap") + expect(contains).toContain("positionCaseInsensitive") + + const negated = compileCH( + tracesListQuery({ + attributeFilters: [{ key: "db.system", value: "postgres", mode: "equals", negated: true }], + }), + baseParams, + ).sql + expect(negated).not.toContain("has(arrayMap") + + const emptyValue = compileCH( + tracesListQuery({ attributeFilters: [{ key: "db.system", value: "", mode: "equals" }] }), + baseParams, + ).sql + expect(emptyValue).not.toContain("has(arrayMap") + }) +}) + // --------------------------------------------------------------------------- // tracesRootListQuery // --------------------------------------------------------------------------- @@ -274,4 +333,18 @@ describe("spanSearchQuery", () => { expect(sql).not.toContain("FROM trace_detail_spans") expect(sql).toContain("SpanName = 'GET /users'") }) + + it("emits the items pre-filter on the raw table but not on trace_detail_spans", () => { + const filter = { attributeFilters: [{ key: "db.system", value: "postgres", mode: "equals" as const }] } + + // Raw traces scan (no traceId) → has the items index → pre-filter emitted. + const raw = compileCH(spanSearchQuery(filter), baseParams).sql + expect(raw).toContain("has(arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes)") + + // trace_detail_spans (traceId set) lacks the items index and is already + // pruned by its sort key → no pre-filter, just the exact equality. + const detail = compileCH(spanSearchQuery({ ...filter, traceId: "trace_123" }), baseParams).sql + expect(detail).not.toContain("has(arrayMap") + expect(detail).toContain("SpanAttributes['db.system'] = 'postgres'") + }) }) diff --git a/packages/query-engine/src/ch/queries/traces.ts b/packages/query-engine/src/ch/queries/traces.ts index d1954d272..6b737af99 100644 --- a/packages/query-engine/src/ch/queries/traces.ts +++ b/packages/query-engine/src/ch/queries/traces.ts @@ -244,7 +244,8 @@ function buildWhereConditions( $: ColumnAccessor, opts: TracesQueryOpts, ): Array { - return tracesBaseWhereConditions($, opts) + // Reads the raw `traces` table → its `idx_*_attr_items` blooms can prune. + return tracesBaseWhereConditions($, opts, { itemsIndex: true }) } // --------------------------------------------------------------------------- @@ -691,6 +692,7 @@ function spanSearchFrom( opts: SpanSearchOpts, limit: number, offset: number, + itemsIndex: boolean, ) { const q = from(source) .select(($) => ({ @@ -706,7 +708,7 @@ function spanSearchFrom( timestamp: CH.toString_($.Timestamp), })) .where(($) => [ - ...tracesBaseWhereConditions($, opts), + ...tracesBaseWhereConditions($, opts, { itemsIndex }), CH.when(opts.traceId, (v: string) => $.TraceId.eq(v)), ]) .orderBy(["timestamp", "desc"]) @@ -720,11 +722,14 @@ export function spanSearchQuery(opts: SpanSearchOpts) { const limit = opts.limit ?? 20 const offset = opts.offset ?? 0 + // `trace_detail_spans` (the traceId path) has no `idx_*_attr_items` and is + // already pruned to a single trace by its (OrgId, TraceId, SpanId) sort key, + // so the pre-filter is only worthwhile on the raw `traces` scan. if (opts.traceId) { - return spanSearchFrom(TraceDetailSpans, opts, limit, offset) + return spanSearchFrom(TraceDetailSpans, opts, limit, offset, false) } - return spanSearchFrom(Traces, opts, limit, offset) + return spanSearchFrom(Traces, opts, limit, offset, true) } // --------------------------------------------------------------------------- diff --git a/packages/query-engine/src/traces-shared.ts b/packages/query-engine/src/traces-shared.ts index 8e721e1eb..4864483ae 100644 --- a/packages/query-engine/src/traces-shared.ts +++ b/packages/query-engine/src/traces-shared.ts @@ -43,6 +43,7 @@ export const TRACE_LIST_MV_RESOURCE_MAP: Record = { // --------------------------------------------------------------------------- import * as CH from "@maple-dev/clickhouse-builder/expr" +import { attrItemsExpr } from "@maple/domain/tinybird/index-exprs" // --------------------------------------------------------------------------- // HTTP semconv coalescing @@ -83,6 +84,23 @@ function anyMapContains(mapExpr: CH.Expr>, keys: readonly return cond } +/** + * ClickStack "Items" pre-filter: `has(, 'k0=v') OR has(, 'k1=v')` + * over the `bloom_filter`-indexed `arrayMap((k, v) -> concat(k, '=', v), …)` + * expression on `traces`. Implied by the exact map equality (`map[ki] = v` means + * the item `ki=v` is present), so AND-ing it never changes the result set — it + * only lets the index skip granules. The `attrItemsExpr` string MUST match the + * datasource index expression byte-for-byte (single source in @maple/domain). + */ +function attrItemsPrefilter(mapName: string, keys: readonly string[], value: string): CH.Condition { + const items = CH.rawExpr>(attrItemsExpr(mapName)) + let cond = CH.has(items, CH.lit(`${keys[0]}=${value}`)) + for (let i = 1; i < keys.length; i++) { + cond = cond.or(CH.has(items, CH.lit(`${keys[i]}=${value}`))) + } + return cond +} + /** * Rewrites an HTTP server span name to the display form used by the UI and by * `trace_list_mv.SpanName`: spanName `"http.server GET"` + route → `"GET /api/users"`. @@ -111,6 +129,16 @@ export function httpDisplaySpanName( export function buildAttrFilterCondition( af: AttributeFilter, mapName: "SpanAttributes" | "ResourceAttributes", + opts?: { + /** + * When true, an equality filter also emits a granule-prunable + * `has(, 'k=v')` pre-filter backed by the `idx_*_attr_items` bloom + * index. Enable only for callers reading the raw `traces` table (which + * carries those indexes); leave off for tables without them (e.g. metrics) + * to avoid the per-row `arrayMap` cost with no index to pay it back. + */ + itemsIndex?: boolean + }, ): CH.Condition { const mapExpr = CH.dynamicColumn>(mapName) // Span attributes renamed across OTel semconv versions match either spelling, @@ -139,7 +167,14 @@ export function buildAttrFilterCondition( return CH.toFloat64OrZero(colExpr).lte(Number(value)) } // equals (default) - return colExpr.eq(value) + const eq = colExpr.eq(value) + // The items pre-filter can't represent "missing/empty" (an empty value + // has no `k=` entry) and adds no index benefit to a negated predicate, so + // gate it on a non-empty value and the positive branch only. + if (opts?.itemsIndex && value !== "" && !af.negated) { + return attrItemsPrefilter(mapName, keys, value).and(eq) + } + return eq })() return af.negated ? CH.not(positive) : positive From d57014007b0d6b63e5b77cd3c306465d6c838c4b Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 4 Jul 2026 00:57:00 +0200 Subject: [PATCH 2/7] fix(domain): re-export index-exprs from tinybird barrel + regen local-inserts Two generated/barrel artifacts missed in the first commit: - tinybird/index.ts now re-exports ./index-exprs like its sibling modules. - local-inserts.json regenerated to the new schema revision (was stale, would have failed clickhouse:schema:check in CI). Co-Authored-By: Claude Fable 5 --- apps/cli/src/server/schema/local-inserts.json | 684 +++++++++--------- packages/domain/src/tinybird/index.ts | 3 + 2 files changed, 345 insertions(+), 342 deletions(-) diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 8b5516720..cba5117ad 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,344 +1,344 @@ { - "projectRevision": "d58ce4a83d3ad3f3a29b9bb972272b757547ae793c050194354454634f3abccd", - "orgPlaceholder": "__ORG__", - "datasources": { - "traces": { - "table": "traces", - "columns": [ - "OrgId", - "Timestamp", - "TraceId", - "SpanId", - "ParentSpanId", - "TraceState", - "SpanName", - "SpanKind", - "ServiceName", - "ResourceSchemaUrl", - "ResourceAttributes", - "ScopeSchemaUrl", - "ScopeName", - "ScopeVersion", - "ScopeAttributes", - "Duration", - "StatusCode", - "StatusMessage", - "SpanAttributes", - "EventsTimestamp", - "EventsName", - "EventsAttributes", - "LinksTraceId", - "LinksSpanId", - "LinksTraceState", - "LinksAttributes" - ], - "selects": [ - "__ORG__", - "start_time", - "trace_id", - "span_id", - "parent_span_id", - "trace_state", - "span_name", - "span_kind", - "service_name", - "resource_schema_url", - "resource_attributes", - "scope_schema_url", - "scope_name", - "scope_version", - "scope_attributes", - "duration", - "status_code", - "status_message", - "span_attributes", - "events_timestamp", - "events_name", - "events_attributes", - "links_trace_id", - "links_span_id", - "links_trace_state", - "links_attributes" - ], - "inputSchema": "start_time DateTime64(9), trace_id String, span_id String, parent_span_id String, trace_state String, span_name LowCardinality(String), span_kind LowCardinality(String), service_name LowCardinality(String), resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), duration UInt64, status_code LowCardinality(String), status_message String, span_attributes Map(LowCardinality(String), String), events_timestamp Array(DateTime64(9)), events_name Array(LowCardinality(String)), events_attributes Array(Map(LowCardinality(String), String)), links_trace_id Array(String), links_span_id Array(String), links_trace_state Array(String), links_attributes Array(Map(LowCardinality(String), String))" - }, - "logs": { - "table": "logs", - "columns": [ - "OrgId", - "Timestamp", - "TimestampTime", - "TraceId", - "SpanId", - "TraceFlags", - "SeverityText", - "SeverityNumber", - "ServiceName", - "Body", - "ResourceSchemaUrl", - "ResourceAttributes", - "ScopeSchemaUrl", - "ScopeName", - "ScopeVersion", - "ScopeAttributes", - "LogAttributes" - ], - "selects": [ - "__ORG__", - "timestamp", - "timestamp", - "trace_id", - "span_id", - "flags", - "severity_text", - "severity_number", - "service_name", - "body", - "resource_schema_url", - "resource_attributes", - "scope_schema_url", - "scope_name", - "scope_version", - "scope_attributes", - "log_attributes" - ], - "inputSchema": "timestamp DateTime64(9), trace_id String, span_id String, flags UInt8, severity_text LowCardinality(String), severity_number UInt8, service_name LowCardinality(String), body String, resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), log_attributes Map(LowCardinality(String), String)" - }, - "metrics_sum": { - "table": "metrics_sum", - "columns": [ - "OrgId", - "ResourceAttributes", - "ResourceSchemaUrl", - "ScopeName", - "ScopeVersion", - "ScopeAttributes", - "ScopeSchemaUrl", - "ServiceName", - "MetricName", - "MetricDescription", - "MetricUnit", - "Attributes", - "StartTimeUnix", - "TimeUnix", - "Value", - "Flags", - "ExemplarsTraceId", - "ExemplarsSpanId", - "ExemplarsTimestamp", - "ExemplarsValue", - "ExemplarsFilteredAttributes", - "AggregationTemporality", - "IsMonotonic" - ], - "selects": [ - "__ORG__", - "resource_attributes", - "resource_schema_url", - "scope_name", - "scope_version", - "scope_attributes", - "scope_schema_url", - "service_name", - "metric_name", - "metric_description", - "metric_unit", - "metric_attributes", - "start_timestamp", - "timestamp", - "value", - "flags", - "exemplars_trace_id", - "exemplars_span_id", - "exemplars_timestamp", - "exemplars_value", - "exemplars_filtered_attributes", - "aggregation_temporality", - "is_monotonic" - ], - "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), value Float64, flags UInt32, exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String)), aggregation_temporality Int32, is_monotonic Bool" - }, - "metrics_gauge": { - "table": "metrics_gauge", - "columns": [ - "OrgId", - "ResourceAttributes", - "ResourceSchemaUrl", - "ScopeName", - "ScopeVersion", - "ScopeAttributes", - "ScopeSchemaUrl", - "ServiceName", - "MetricName", - "MetricDescription", - "MetricUnit", - "Attributes", - "StartTimeUnix", - "TimeUnix", - "Value", - "Flags", - "ExemplarsTraceId", - "ExemplarsSpanId", - "ExemplarsTimestamp", - "ExemplarsValue", - "ExemplarsFilteredAttributes" - ], - "selects": [ - "__ORG__", - "resource_attributes", - "resource_schema_url", - "scope_name", - "scope_version", - "scope_attributes", - "scope_schema_url", - "service_name", - "metric_name", - "metric_description", - "metric_unit", - "metric_attributes", - "start_timestamp", - "timestamp", - "value", - "flags", - "exemplars_trace_id", - "exemplars_span_id", - "exemplars_timestamp", - "exemplars_value", - "exemplars_filtered_attributes" - ], - "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), value Float64, flags UInt32, exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String))" - }, - "metrics_histogram": { - "table": "metrics_histogram", - "columns": [ - "OrgId", - "ResourceAttributes", - "ResourceSchemaUrl", - "ScopeName", - "ScopeVersion", - "ScopeAttributes", - "ScopeSchemaUrl", - "ServiceName", - "MetricName", - "MetricDescription", - "MetricUnit", - "Attributes", - "StartTimeUnix", - "TimeUnix", - "Count", - "Sum", - "BucketCounts", - "ExplicitBounds", - "ExemplarsTraceId", - "ExemplarsSpanId", - "ExemplarsTimestamp", - "ExemplarsValue", - "ExemplarsFilteredAttributes", - "Flags", - "Min", - "Max", - "AggregationTemporality" - ], - "selects": [ - "__ORG__", - "resource_attributes", - "resource_schema_url", - "scope_name", - "scope_version", - "scope_attributes", - "scope_schema_url", - "service_name", - "metric_name", - "metric_description", - "metric_unit", - "metric_attributes", - "start_timestamp", - "timestamp", - "count", - "sum", - "bucket_counts", - "explicit_bounds", - "exemplars_trace_id", - "exemplars_span_id", - "exemplars_timestamp", - "exemplars_value", - "exemplars_filtered_attributes", - "flags", - "min", - "max", - "aggregation_temporality" - ], - "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), count UInt64, sum Float64, bucket_counts Array(UInt64), explicit_bounds Array(Float64), exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String)), flags UInt32, min Nullable(Float64), max Nullable(Float64), aggregation_temporality Int32" - }, - "metrics_exponential_histogram": { - "table": "metrics_exponential_histogram", - "columns": [ - "OrgId", - "ResourceAttributes", - "ResourceSchemaUrl", - "ScopeName", - "ScopeVersion", - "ScopeAttributes", - "ScopeSchemaUrl", - "ServiceName", - "MetricName", - "MetricDescription", - "MetricUnit", - "Attributes", - "StartTimeUnix", - "TimeUnix", - "Count", - "Sum", - "Scale", - "ZeroCount", - "PositiveOffset", - "PositiveBucketCounts", - "NegativeOffset", - "NegativeBucketCounts", - "ExemplarsTraceId", - "ExemplarsSpanId", - "ExemplarsTimestamp", - "ExemplarsValue", - "ExemplarsFilteredAttributes", - "Flags", - "Min", - "Max", - "AggregationTemporality" - ], - "selects": [ - "__ORG__", - "resource_attributes", - "resource_schema_url", - "scope_name", - "scope_version", - "scope_attributes", - "scope_schema_url", - "service_name", - "metric_name", - "metric_description", - "metric_unit", - "metric_attributes", - "start_timestamp", - "timestamp", - "count", - "sum", - "scale", - "zero_count", - "positive_offset", - "positive_bucket_counts", - "negative_offset", - "negative_bucket_counts", - "exemplars_trace_id", - "exemplars_span_id", - "exemplars_timestamp", - "exemplars_value", - "exemplars_filtered_attributes", - "flags", - "min", - "max", - "aggregation_temporality" - ], - "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), count UInt64, sum Float64, scale Int32, zero_count UInt64, positive_offset Int32, positive_bucket_counts Array(UInt64), negative_offset Int32, negative_bucket_counts Array(UInt64), exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String)), flags UInt32, min Nullable(Float64), max Nullable(Float64), aggregation_temporality Int32" - } - } + "projectRevision": "a8d56c30f4f7c8998db16ca014680db16a82f69143adbeb054ccc47d718b222b", + "orgPlaceholder": "__ORG__", + "datasources": { + "traces": { + "table": "traces", + "columns": [ + "OrgId", + "Timestamp", + "TraceId", + "SpanId", + "ParentSpanId", + "TraceState", + "SpanName", + "SpanKind", + "ServiceName", + "ResourceSchemaUrl", + "ResourceAttributes", + "ScopeSchemaUrl", + "ScopeName", + "ScopeVersion", + "ScopeAttributes", + "Duration", + "StatusCode", + "StatusMessage", + "SpanAttributes", + "EventsTimestamp", + "EventsName", + "EventsAttributes", + "LinksTraceId", + "LinksSpanId", + "LinksTraceState", + "LinksAttributes" + ], + "selects": [ + "__ORG__", + "start_time", + "trace_id", + "span_id", + "parent_span_id", + "trace_state", + "span_name", + "span_kind", + "service_name", + "resource_schema_url", + "resource_attributes", + "scope_schema_url", + "scope_name", + "scope_version", + "scope_attributes", + "duration", + "status_code", + "status_message", + "span_attributes", + "events_timestamp", + "events_name", + "events_attributes", + "links_trace_id", + "links_span_id", + "links_trace_state", + "links_attributes" + ], + "inputSchema": "start_time DateTime64(9), trace_id String, span_id String, parent_span_id String, trace_state String, span_name LowCardinality(String), span_kind LowCardinality(String), service_name LowCardinality(String), resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), duration UInt64, status_code LowCardinality(String), status_message String, span_attributes Map(LowCardinality(String), String), events_timestamp Array(DateTime64(9)), events_name Array(LowCardinality(String)), events_attributes Array(Map(LowCardinality(String), String)), links_trace_id Array(String), links_span_id Array(String), links_trace_state Array(String), links_attributes Array(Map(LowCardinality(String), String))" + }, + "logs": { + "table": "logs", + "columns": [ + "OrgId", + "Timestamp", + "TimestampTime", + "TraceId", + "SpanId", + "TraceFlags", + "SeverityText", + "SeverityNumber", + "ServiceName", + "Body", + "ResourceSchemaUrl", + "ResourceAttributes", + "ScopeSchemaUrl", + "ScopeName", + "ScopeVersion", + "ScopeAttributes", + "LogAttributes" + ], + "selects": [ + "__ORG__", + "timestamp", + "timestamp", + "trace_id", + "span_id", + "flags", + "severity_text", + "severity_number", + "service_name", + "body", + "resource_schema_url", + "resource_attributes", + "scope_schema_url", + "scope_name", + "scope_version", + "scope_attributes", + "log_attributes" + ], + "inputSchema": "timestamp DateTime64(9), trace_id String, span_id String, flags UInt8, severity_text LowCardinality(String), severity_number UInt8, service_name LowCardinality(String), body String, resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), log_attributes Map(LowCardinality(String), String)" + }, + "metrics_sum": { + "table": "metrics_sum", + "columns": [ + "OrgId", + "ResourceAttributes", + "ResourceSchemaUrl", + "ScopeName", + "ScopeVersion", + "ScopeAttributes", + "ScopeSchemaUrl", + "ServiceName", + "MetricName", + "MetricDescription", + "MetricUnit", + "Attributes", + "StartTimeUnix", + "TimeUnix", + "Value", + "Flags", + "ExemplarsTraceId", + "ExemplarsSpanId", + "ExemplarsTimestamp", + "ExemplarsValue", + "ExemplarsFilteredAttributes", + "AggregationTemporality", + "IsMonotonic" + ], + "selects": [ + "__ORG__", + "resource_attributes", + "resource_schema_url", + "scope_name", + "scope_version", + "scope_attributes", + "scope_schema_url", + "service_name", + "metric_name", + "metric_description", + "metric_unit", + "metric_attributes", + "start_timestamp", + "timestamp", + "value", + "flags", + "exemplars_trace_id", + "exemplars_span_id", + "exemplars_timestamp", + "exemplars_value", + "exemplars_filtered_attributes", + "aggregation_temporality", + "is_monotonic" + ], + "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), value Float64, flags UInt32, exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String)), aggregation_temporality Int32, is_monotonic Bool" + }, + "metrics_gauge": { + "table": "metrics_gauge", + "columns": [ + "OrgId", + "ResourceAttributes", + "ResourceSchemaUrl", + "ScopeName", + "ScopeVersion", + "ScopeAttributes", + "ScopeSchemaUrl", + "ServiceName", + "MetricName", + "MetricDescription", + "MetricUnit", + "Attributes", + "StartTimeUnix", + "TimeUnix", + "Value", + "Flags", + "ExemplarsTraceId", + "ExemplarsSpanId", + "ExemplarsTimestamp", + "ExemplarsValue", + "ExemplarsFilteredAttributes" + ], + "selects": [ + "__ORG__", + "resource_attributes", + "resource_schema_url", + "scope_name", + "scope_version", + "scope_attributes", + "scope_schema_url", + "service_name", + "metric_name", + "metric_description", + "metric_unit", + "metric_attributes", + "start_timestamp", + "timestamp", + "value", + "flags", + "exemplars_trace_id", + "exemplars_span_id", + "exemplars_timestamp", + "exemplars_value", + "exemplars_filtered_attributes" + ], + "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), value Float64, flags UInt32, exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String))" + }, + "metrics_histogram": { + "table": "metrics_histogram", + "columns": [ + "OrgId", + "ResourceAttributes", + "ResourceSchemaUrl", + "ScopeName", + "ScopeVersion", + "ScopeAttributes", + "ScopeSchemaUrl", + "ServiceName", + "MetricName", + "MetricDescription", + "MetricUnit", + "Attributes", + "StartTimeUnix", + "TimeUnix", + "Count", + "Sum", + "BucketCounts", + "ExplicitBounds", + "ExemplarsTraceId", + "ExemplarsSpanId", + "ExemplarsTimestamp", + "ExemplarsValue", + "ExemplarsFilteredAttributes", + "Flags", + "Min", + "Max", + "AggregationTemporality" + ], + "selects": [ + "__ORG__", + "resource_attributes", + "resource_schema_url", + "scope_name", + "scope_version", + "scope_attributes", + "scope_schema_url", + "service_name", + "metric_name", + "metric_description", + "metric_unit", + "metric_attributes", + "start_timestamp", + "timestamp", + "count", + "sum", + "bucket_counts", + "explicit_bounds", + "exemplars_trace_id", + "exemplars_span_id", + "exemplars_timestamp", + "exemplars_value", + "exemplars_filtered_attributes", + "flags", + "min", + "max", + "aggregation_temporality" + ], + "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), count UInt64, sum Float64, bucket_counts Array(UInt64), explicit_bounds Array(Float64), exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String)), flags UInt32, min Nullable(Float64), max Nullable(Float64), aggregation_temporality Int32" + }, + "metrics_exponential_histogram": { + "table": "metrics_exponential_histogram", + "columns": [ + "OrgId", + "ResourceAttributes", + "ResourceSchemaUrl", + "ScopeName", + "ScopeVersion", + "ScopeAttributes", + "ScopeSchemaUrl", + "ServiceName", + "MetricName", + "MetricDescription", + "MetricUnit", + "Attributes", + "StartTimeUnix", + "TimeUnix", + "Count", + "Sum", + "Scale", + "ZeroCount", + "PositiveOffset", + "PositiveBucketCounts", + "NegativeOffset", + "NegativeBucketCounts", + "ExemplarsTraceId", + "ExemplarsSpanId", + "ExemplarsTimestamp", + "ExemplarsValue", + "ExemplarsFilteredAttributes", + "Flags", + "Min", + "Max", + "AggregationTemporality" + ], + "selects": [ + "__ORG__", + "resource_attributes", + "resource_schema_url", + "scope_name", + "scope_version", + "scope_attributes", + "scope_schema_url", + "service_name", + "metric_name", + "metric_description", + "metric_unit", + "metric_attributes", + "start_timestamp", + "timestamp", + "count", + "sum", + "scale", + "zero_count", + "positive_offset", + "positive_bucket_counts", + "negative_offset", + "negative_bucket_counts", + "exemplars_trace_id", + "exemplars_span_id", + "exemplars_timestamp", + "exemplars_value", + "exemplars_filtered_attributes", + "flags", + "min", + "max", + "aggregation_temporality" + ], + "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), count UInt64, sum Float64, scale Int32, zero_count UInt64, positive_offset Int32, positive_bucket_counts Array(UInt64), negative_offset Int32, negative_bucket_counts Array(UInt64), exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String)), flags UInt32, min Nullable(Float64), max Nullable(Float64), aggregation_temporality Int32" + } + } } diff --git a/packages/domain/src/tinybird/index.ts b/packages/domain/src/tinybird/index.ts index 85a5e1767..ee0aa33a7 100644 --- a/packages/domain/src/tinybird/index.ts +++ b/packages/domain/src/tinybird/index.ts @@ -29,5 +29,8 @@ export * from "./materializations" // Export shared DB query-shape SQL fragments (label/key derivation) export * from "./db-query-shape-sql" +// Export shared skip-index expressions (kept in sync with query-engine predicates) +export * from "./index-exprs" + // Export TTL override helpers for BYO Tinybird raw retention export * from "./ttl-override" From 6fd754c2dbd62d34bd2279f93b9a2ea67185130e Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 15 Jul 2026 00:36:11 +0200 Subject: [PATCH 3/7] Refactor MCP tenant resolution and chat-flue binding --- alchemy.run.ts | 13 +- apps/api/alchemy.run.ts | 14 +- apps/api/src/app.ts | 21 +-- apps/api/src/internal-rpc.test.ts | 90 ++++++++++ apps/api/src/internal-rpc.ts | 61 +++++++ apps/api/src/mcp/__evals__/eval-runtime.ts | 24 ++- apps/api/src/mcp/__evals__/tools.ts | 2 +- apps/api/src/mcp/dispatcher.test.ts | 48 ++++++ apps/api/src/mcp/dispatcher.ts | 133 ++++++++++++++ .../src/mcp/lib/dashboard-mutations.test.ts | 48 ++---- apps/api/src/mcp/lib/query-warehouse.ts | 11 +- apps/api/src/mcp/server.ts | 162 +++--------------- .../routes/investigations-internal.http.ts | 39 ----- .../InternalServiceAuthorizationLayer.ts | 62 ------- apps/api/src/worker.ts | 119 +++++++++++-- apps/api/wrangler.jsonc | 1 + apps/chat-flue/.dev.vars.example | 2 +- apps/chat-flue/README.md | 6 +- apps/chat-flue/alchemy.run.ts | 9 +- apps/chat-flue/package.json | 2 + apps/chat-flue/src/agents/maple-chat.ts | 2 +- apps/chat-flue/src/lib/api-rpc.test.ts | 70 ++++++++ apps/chat-flue/src/lib/api-rpc.ts | 6 + apps/chat-flue/src/lib/auth.ts | 3 +- apps/chat-flue/src/lib/env.ts | 6 +- apps/chat-flue/src/lib/mcp.ts | 98 ++++++++--- apps/chat-flue/src/lib/submit-diagnosis.ts | 19 +- apps/chat-flue/src/lib/triage.test.ts | 30 +++- apps/chat-flue/wrangler.jsonc | 5 +- bun.lock | 2 + packages/domain/package.json | 1 + packages/domain/src/http/api.ts | 3 +- packages/domain/src/http/current-tenant.ts | 18 -- packages/domain/src/http/investigations.ts | 26 +-- packages/domain/src/index.ts | 1 + packages/domain/src/internal-rpc.ts | 74 ++++++++ 36 files changed, 789 insertions(+), 442 deletions(-) create mode 100644 apps/api/src/internal-rpc.test.ts create mode 100644 apps/api/src/internal-rpc.ts create mode 100644 apps/api/src/mcp/dispatcher.test.ts create mode 100644 apps/api/src/mcp/dispatcher.ts delete mode 100644 apps/api/src/routes/investigations-internal.http.ts delete mode 100644 apps/api/src/services/InternalServiceAuthorizationLayer.ts create mode 100644 apps/chat-flue/src/lib/api-rpc.test.ts create mode 100644 apps/chat-flue/src/lib/api-rpc.ts create mode 100644 packages/domain/src/internal-rpc.ts diff --git a/alchemy.run.ts b/alchemy.run.ts index 6b828e812..2dbbafed1 100644 --- a/alchemy.run.ts +++ b/alchemy.run.ts @@ -48,11 +48,14 @@ export default Alchemy.Stack( // back to a caller-supplied env var or the public Maple ingest endpoint. const ingestUrl = resolveUrl(domains.ingest, "VITE_INGEST_URL", "https://ingest.maple.dev") - // chat-flue deploys before api so api can service-bind the real worker (the - // v1 WorkerStub cycle-breaker is gone — chat-flue's api URL is now static). - const chatFlue = yield* createChatFlueWorker({ stage, domains, mapleApiUrl: apiUrl }) - - const { worker: api, db: mapleDb } = yield* createMapleApi({ stage, domains, chatFlue }) + // Deploy the API RPC surface before switching chat-flue to it. The reverse + // CHAT_FLUE binding is attached after chat deploys, which breaks the resource + // cycle without an HTTP fallback or a placeholder Worker. + const { worker: api, db: mapleDb } = yield* createMapleApi({ stage, domains }) + const chatFlue = yield* createChatFlueWorker({ stage, domains, mapleApiRpc: api }) + yield* api.bind("CHAT_FLUE", { + bindings: [{ type: "service", name: "CHAT_FLUE", service: chatFlue.workerName }], + }) // Standalone ElectricSQL shape-proxy worker (DB-free); its public origin is // baked into the web build (VITE_ELECTRIC_SYNC_URL). diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index 8d1214670..0e0bc7246 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -1,7 +1,9 @@ import path from "node:path" import * as Cloudflare from "alchemy/Cloudflare" +import type { Rpc } from "alchemy/Rpc" import * as Effect from "effect/Effect" import * as Redacted from "effect/Redacted" +import type { MapleApiRpcShape } from "@maple/domain/internal-rpc" import type { MapleDomains, MapleStage } from "@maple/infra/cloudflare" import { CLOUDFLARE_WORKER_PLACEMENT, @@ -32,11 +34,12 @@ const optionalSecret = (key: string): Record> export interface CreateMapleApiOptions { stage: MapleStage domains: MapleDomains - /** The chat-flue worker, service-bound as CHAT_FLUE (hosts the Flue `triage` workflow). */ - chatFlue: Cloudflare.Worker } -export const createMapleApi = ({ stage, domains, chatFlue }: CreateMapleApiOptions) => +/** Alchemy resource type carried across the chat-flue service binding. */ +export type MapleApiWorker = Cloudflare.Worker & Rpc + +export const createMapleApi = ({ stage, domains }: CreateMapleApiOptions) => Effect.gen(function* () { // MAPLE_DB Hyperdrive comes in two flavors: // @@ -115,7 +118,7 @@ export const createMapleApi = ({ stage, domains, chatFlue }: CreateMapleApiOptio name: resolveWorkerName("vcs-sync", stage), }) - const worker = yield* Cloudflare.Worker("api", { + const worker = (yield* Cloudflare.Worker("api", { name: resolveWorkerName("api", stage), main: path.join(import.meta.dirname, "src", "worker.ts"), compatibility: { date: "2026-04-08", flags: ["nodejs_compat"] }, @@ -134,7 +137,6 @@ export const createMapleApi = ({ stage, domains, chatFlue }: CreateMapleApiOptio VCS_SYNC_QUEUE: vcsSyncQueue, CLICKHOUSE_SCHEMA_APPLY_WORKFLOW: schemaApplyWorkflow, AI_TRIAGE_WORKFLOW: aiTriageWorkflow, - CHAT_FLUE: chatFlue, EMAIL: Cloudflare.Email.SendEmail("email", { allowedSenderAddresses: ["notifications@noreply.maple.dev"], }), @@ -198,7 +200,7 @@ export const createMapleApi = ({ stage, domains, chatFlue }: CreateMapleApiOptio ...optionalPlain("PLANETSCALE_OAUTH_TOKEN_INFO_URL"), ...optionalPlain("MAPLE_PLANETSCALE_API_BASE_URL"), }, - }) + })) as MapleApiWorker if (hyperdriveRefId) { // v1 `HyperdriveRef` equivalent: bind the dashboard-managed config by ID diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index d9673cda1..81fb190fb 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -16,7 +16,6 @@ import { HttpDemoLive } from "./routes/demo.http" import { HttpDigestLive } from "./routes/digest.http" import { HttpIntegrationsLive, IntegrationsCallbackRouter } from "./routes/integrations.http" import { HttpInvestigationsLive } from "./routes/investigations.http" -import { HttpInvestigationsInternalLive } from "./routes/investigations-internal.http" import { HttpIngestAttributeMappingsLive } from "./routes/ingest-attribute-mappings.http" import { HttpIngestKeysLive } from "./routes/ingest-keys.http" import { HttpObservabilityLive } from "./routes/observability.http" @@ -45,7 +44,6 @@ import { NotificationDispatcher } from "./services/NotificationDispatcher" import { ApiKeysService } from "./services/ApiKeysService" import { AuthService } from "./services/AuthService" import { ApiAuthorizationLayer } from "./services/ApiAuthorizationLayer" -import { InternalServiceAuthorizationLayer } from "./services/InternalServiceAuthorizationLayer" import { CloudflareAnalyticsService } from "./services/CloudflareAnalyticsService" import { CloudflareOAuthService } from "./services/CloudflareOAuthService" import { DashboardPersistenceService } from "./services/DashboardPersistenceService" @@ -240,15 +238,7 @@ export const MainLive = Layer.mergeAll( const ApiRoutes = HttpApiBuilder.layer(MapleApi).pipe( Layer.provide(HttpAuthPublicLive), Layer.provide(HttpAuthLive), - Layer.provide( - Layer.mergeAll( - HttpAiTriageLive, - HttpAnomaliesLive, - HttpChatLive, - HttpInvestigationsLive, - HttpInvestigationsInternalLive, - ), - ), + Layer.provide(Layer.mergeAll(HttpAiTriageLive, HttpAnomaliesLive, HttpChatLive, HttpInvestigationsLive)), Layer.provide(HttpApiKeysLive), Layer.provide(Layer.mergeAll(HttpBillingLive, HttpBillingPublicLive)), Layer.provide(HttpAlertsLive), @@ -304,15 +294,6 @@ export const ApiAuthLive = ApiAuthorizationLayer.pipe( Layer.provideMerge(Env.layer), ) -// Internal-service-token middleware (the chat-flue `submit_diagnosis` write). -// Mirrors ApiAuthLive; AuthService/ApiKeysService/Env are layer-memoized, so this -// shares instances with MainLive rather than constructing duplicates. -export const InternalServiceAuthLive = InternalServiceAuthorizationLayer.pipe( - Layer.provideMerge(ApiKeysService.layer), - Layer.provideMerge(AuthService.layer), - Layer.provideMerge(Env.layer), -) - // The OTLP tracer/logger is constructed once at worker module scope and // provided to the same runtime as the routes. This shared layer only installs // the `TracerDisabledWhen` filter, which is a ServiceMap.Reference read by diff --git a/apps/api/src/internal-rpc.test.ts b/apps/api/src/internal-rpc.test.ts new file mode 100644 index 000000000..180cf49bf --- /dev/null +++ b/apps/api/src/internal-rpc.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest" +import { Effect } from "effect" +import type { InternalRpcInvalidInputError } from "@maple/domain/internal-rpc" +import { callMcpToolRpc, submitDiagnosisRpc } from "./internal-rpc" +import { InvestigationService, type InvestigationServiceShape } from "./services/InvestigationService" + +const investigationId = "00000000-0000-4000-8000-000000000001" +const report = { + summary: "Checkout latency doubled after deploy.", + suspectedCause: "Connection pool regression", + severityAssessment: "high", + affectedScope: "checkout-api", + evidence: [ + { + traceIds: ["trace-1"], + logPatterns: ["pool exhausted"], + relatedServices: ["payments"], + note: "The failing traces share the same pool exhaustion event.", + }, + ], + suggestedActions: ["Roll back the deploy"], + confidence: "high", +} as const + +const unusedInvestigationService: InvestigationServiceShape = { + listInvestigations: () => Effect.die("unused"), + getInvestigation: () => Effect.die("unused"), + createInvestigation: () => Effect.die("unused"), + updateStatus: () => Effect.die("unused"), + submitDiagnosis: () => Effect.die("unused"), +} + +describe("internal RPC boundary", () => { + it("rejects invalid org IDs before MCP dispatch", async () => { + const error = await Effect.runPromise( + Effect.flip( + callMcpToolRpc({ orgId: " ", name: "inspect_trace", input: {} }) as Effect.Effect< + never, + InternalRpcInvalidInputError, + never + >, + ), + ) + expect(error._tag).toBe("@maple/internal-rpc/InvalidInputError") + expect(error.method).toBe("callMcpTool") + }) + + it("rejects invalid investigation IDs and model-produced reports", async () => { + for (const input of [ + { orgId: "org_1", investigationId: "not-a-uuid", report }, + { orgId: "org_1", investigationId, report: { summary: "incomplete" } }, + ]) { + const error = await Effect.runPromise( + Effect.flip( + submitDiagnosisRpc(input).pipe( + Effect.provideService(InvestigationService, unusedInvestigationService), + ), + ), + ) + expect(error._tag).toBe("@maple/internal-rpc/InvalidInputError") + if (error._tag !== "@maple/internal-rpc/InvalidInputError") { + throw new Error(`Expected invalid input, received ${error._tag}`) + } + expect(error.method).toBe("submitDiagnosis") + } + }) + + it("submits a decoded diagnosis to the org-scoped service", async () => { + const calls: Array<{ orgId: string; investigationId: string; summary: string }> = [] + const expected = { id: investigationId, status: "diagnosed" } as never + const service: InvestigationServiceShape = { + ...unusedInvestigationService, + submitDiagnosis: (orgId, id, request) => + Effect.sync(() => { + calls.push({ orgId, investigationId: id, summary: request.report.summary }) + return expected + }), + } + + const result = await Effect.runPromise( + submitDiagnosisRpc({ orgId: "org_1", investigationId, report }).pipe( + Effect.provideService(InvestigationService, service), + ), + ) + expect(result).toBe(expected) + expect(calls).toEqual([ + { orgId: "org_1", investigationId, summary: "Checkout latency doubled after deploy." }, + ]) + }) +}) diff --git a/apps/api/src/internal-rpc.ts b/apps/api/src/internal-rpc.ts new file mode 100644 index 000000000..43905edd2 --- /dev/null +++ b/apps/api/src/internal-rpc.ts @@ -0,0 +1,61 @@ +import { + CallMcpToolRpcRequest, + InternalRpcInvalidInputError, + SubmitDiagnosisRpcRequest, +} from "@maple/domain/internal-rpc" +import { SubmitDiagnosisRequest } from "@maple/domain/http" +import { UserId } from "@maple/domain/primitives" +import { Effect, Schema } from "effect" +import type { TenantContext } from "./lib/tenant-context" +import { callMcpTool, listMcpTools } from "./mcp/dispatcher" +import { CurrentMcpTenant } from "./mcp/lib/query-warehouse" +import { InvestigationService } from "./services/InvestigationService" + +const internalServiceUserId = Schema.decodeUnknownSync(UserId)("internal-service") + +const invalidInput = (method: "callMcpTool" | "submitDiagnosis") => (error: { message: string }) => + new InternalRpcInvalidInputError({ method, message: error.message }) + +const decodeCallMcpTool = (input: unknown) => + Schema.decodeUnknownEffect(CallMcpToolRpcRequest)(input).pipe( + Effect.mapError(invalidInput("callMcpTool")), + ) + +const decodeSubmitDiagnosis = (input: unknown) => + Schema.decodeUnknownEffect(SubmitDiagnosisRpcRequest)(input).pipe( + Effect.mapError(invalidInput("submitDiagnosis")), + ) + +const makeInternalTenant = (orgId: CallMcpToolRpcRequest["orgId"]): TenantContext => ({ + orgId, + userId: internalServiceUserId, + roles: [], + authMode: "self_hosted", +}) + +export const listMcpToolsRpc = listMcpTools.pipe(Effect.withSpan("InternalRpc.listMcpTools")) + +export const callMcpToolRpc = (input: unknown) => + decodeCallMcpTool(input).pipe( + Effect.flatMap((request) => + callMcpTool(request.name, request.input).pipe( + Effect.provideService(CurrentMcpTenant, makeInternalTenant(request.orgId)), + ), + ), + Effect.withSpan("InternalRpc.callMcpTool"), + ) + +export const submitDiagnosisRpc = (input: unknown) => + Effect.gen(function* () { + const request = yield* decodeSubmitDiagnosis(input) + yield* Effect.annotateCurrentSpan({ + "maple.org_id": request.orgId, + "maple.investigation.id": request.investigationId, + }) + const investigations = yield* InvestigationService + return yield* investigations.submitDiagnosis( + request.orgId, + request.investigationId, + new SubmitDiagnosisRequest({ report: request.report }), + ) + }).pipe(Effect.withSpan("InternalRpc.submitDiagnosis")) diff --git a/apps/api/src/mcp/__evals__/eval-runtime.ts b/apps/api/src/mcp/__evals__/eval-runtime.ts index 66ecfbcc9..5437ce209 100644 --- a/apps/api/src/mcp/__evals__/eval-runtime.ts +++ b/apps/api/src/mcp/__evals__/eval-runtime.ts @@ -1,10 +1,11 @@ import { ConfigProvider, Effect, Layer, ManagedRuntime, Schema } from "effect" -import { HttpServerRequest } from "effect/unstable/http" +import { OrgId, UserId } from "@maple/domain/http" import { MainLive } from "@/app" import { Env } from "@/lib/Env" import { WorkerEnvironment } from "@/lib/WorkerEnvironment" import { createTestDb } from "@/lib/test-pglite" import { mapleToolDefinitions } from "@/mcp/tools/registry" +import { CurrentMcpTenant } from "@/mcp/lib/query-warehouse" import { FIXTURES } from "./utils" const INTERNAL_TOKEN = "eval-internal-token" @@ -36,8 +37,8 @@ export interface EvalRuntime { * test config (mirrors apps/api `getMapleAgentSetup`/buildSetup, swapping Hyperdrive→PGlite). * The warehouse client must be faked separately via `installFakeWarehouse` — * this runtime uses the REAL WarehouseQueryService. The returned `requestLayer` - * carries an internal-service-token request so tool handlers resolve the tenant - * without exercising Clerk/API-key auth. + * carries the already-resolved MCP tenant, matching the post-auth dispatcher + * context shared by HTTP and RPC. */ export const makeEvalRuntime = (): EvalRuntime => { const testDb = createTestDb() @@ -57,17 +58,12 @@ export const makeEvalRuntime = (): EvalRuntime => { // eslint-disable-next-line @typescript-eslint/no-explicit-any const runtime = ManagedRuntime.make(layer as any) as ManagedRuntime.ManagedRuntime - const requestLayer = Layer.succeed( - HttpServerRequest.HttpServerRequest, - HttpServerRequest.fromWeb( - new Request("https://maple.eval/mcp", { - headers: { - Authorization: `Bearer maple_svc_${INTERNAL_TOKEN}`, - "X-Org-Id": FIXTURES.orgId, - }, - }), - ), - ) + const requestLayer = Layer.succeed(CurrentMcpTenant, { + orgId: Schema.decodeUnknownSync(OrgId)(FIXTURES.orgId), + userId: Schema.decodeUnknownSync(UserId)("internal-service"), + roles: [], + authMode: "self_hosted", + }) return { runtime, diff --git a/apps/api/src/mcp/__evals__/tools.ts b/apps/api/src/mcp/__evals__/tools.ts index 6335f3a02..f97e46320 100644 --- a/apps/api/src/mcp/__evals__/tools.ts +++ b/apps/api/src/mcp/__evals__/tools.ts @@ -22,7 +22,7 @@ export const buildPredictionToolSet = (): ToolSet => /** * Like `buildPredictionToolSet` but with a real `execute` that runs the tool * handler through the given runtime (which must provide the app services) and a - * request layer (which carries the tenant). Mirrors + * request layer (which carries the resolved tenant). Mirrors * apps/chat-agent/src/services/direct-tools.ts `createMapleAiTools`, but the * runtime is wired with a FAKE warehouse for full-execution evals. */ diff --git a/apps/api/src/mcp/dispatcher.test.ts b/apps/api/src/mcp/dispatcher.test.ts new file mode 100644 index 000000000..4d63ba995 --- /dev/null +++ b/apps/api/src/mcp/dispatcher.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest" +import { Effect } from "effect" +import type { InternalRpcToolNotFoundError } from "@maple/domain/internal-rpc" +import { callMcpTool, listMcpTools } from "./dispatcher" +import { mapleToolDefinitions, toInputSchema } from "./tools/registry" + +describe("MCP dispatcher", () => { + it("publishes the same names, descriptions, and schemas used by HTTP MCP", async () => { + const descriptors = await Effect.runPromise(listMcpTools) + expect(descriptors).toEqual( + mapleToolDefinitions.map((definition) => ({ + name: definition.name, + description: definition.description, + inputSchema: toInputSchema(definition.schema), + })), + ) + }) + + it("returns MCP validation feedback for invalid model tool input", async () => { + const result = await Effect.runPromise( + callMcpTool("inspect_trace", {}) as unknown as Effect.Effect< + { + readonly content: ReadonlyArray<{ readonly type: "text"; readonly text: string }> + readonly isError?: boolean + }, + never, + never + >, + ) + expect(result.isError).toBe(true) + expect(result.content[0]?.text).toContain("Invalid parameters") + expect(result.content[0]?.text).toContain("inspect_trace") + }) + + it("fails unknown RPC tool names with a typed error", async () => { + const error = await Effect.runPromise( + Effect.flip( + callMcpTool("not_a_maple_tool", {}) as Effect.Effect< + never, + InternalRpcToolNotFoundError, + never + >, + ), + ) + expect(error._tag).toBe("@maple/internal-rpc/ToolNotFoundError") + expect(error.name).toBe("not_a_maple_tool") + }) +}) diff --git a/apps/api/src/mcp/dispatcher.ts b/apps/api/src/mcp/dispatcher.ts new file mode 100644 index 000000000..822348b58 --- /dev/null +++ b/apps/api/src/mcp/dispatcher.ts @@ -0,0 +1,133 @@ +import { InternalRpcToolNotFoundError, type InternalMcpToolDescriptor } from "@maple/domain/internal-rpc" +import { Effect, Schema } from "effect" +import { mapleToolDefinitions, toInputSchema, type MapleToolDefinition } from "./tools/registry" +import type { McpToolResult } from "./tools/types" + +class McpDecodeError extends Schema.TaggedErrorClass()("@maple/mcp/decode-error", { + errorMessage: Schema.String, +}) {} + +const toErrorMessage = (error: unknown): string => { + if (error instanceof Error && "error" in error && error.error != null) { + const inner = error.error + return inner instanceof Error ? inner.message : String(inner) + } + if (error instanceof Error) return error.message + return String(error) +} + +const toDecodeErrorMessage = (definition: MapleToolDefinition, error: unknown): string => { + if (Schema.isSchemaError(error)) { + return `${String(error)}. Check the "${definition.name}" tool schema for valid parameter names and types.` + } + return String(error) +} + +const toolDescriptors: ReadonlyArray = mapleToolDefinitions.map((definition) => ({ + name: definition.name, + description: definition.description, + inputSchema: toInputSchema(definition.schema), +})) + +export const listMcpTools = Effect.succeed(toolDescriptors) + +/** Shared tool dispatcher for public MCP-over-HTTP and internal Worker RPC. */ +export const callMcpTool = Effect.fn("McpToolDispatcher.call")(function* (name: string, input: unknown) { + const definition = mapleToolDefinitions.find((candidate) => candidate.name === name) + if (!definition) { + return yield* new InternalRpcToolNotFoundError({ + name, + message: `Unknown MCP tool: ${name}`, + }) + } + + const execute = Effect.gen(function* () { + yield* Effect.annotateCurrentSpan({ tool: definition.name }) + const decoded = yield* Effect.try({ + try: () => Schema.decodeUnknownSync(definition.schema)(input), + catch: (error) => error, + }).pipe( + Effect.mapError( + (error) => + new McpDecodeError({ + errorMessage: toDecodeErrorMessage(definition, error), + }), + ), + ) + + return yield* definition.handler(decoded).pipe(Effect.tap(() => Effect.logInfo("Tool completed"))) + }) + + return yield* execute.pipe( + Effect.catchTag("@maple/mcp/decode-error", (error) => + Effect.logWarning("Invalid parameters").pipe( + Effect.annotateLogs({ error: error.errorMessage }), + Effect.as({ + isError: true, + content: [ + { + type: "text" as const, + text: `Invalid parameters: ${error.errorMessage}`, + }, + ], + } satisfies McpToolResult), + ), + ), + Effect.catchTags({ + "@maple/mcp/errors/McpQueryError": (error) => + Effect.logError(`Tool error: ${error.message}`).pipe( + Effect.annotateLogs({ errorTag: error._tag, pipe: error.pipeName }), + Effect.as({ + isError: true, + content: [{ type: "text", text: `${error._tag}: ${error.message}` }], + } satisfies McpToolResult), + ), + "@maple/mcp/errors/McpTenantError": (error) => + Effect.logError(`Tool error: ${error.message}`).pipe( + Effect.annotateLogs({ errorTag: error._tag }), + Effect.as({ + isError: true, + content: [{ type: "text", text: `${error._tag}: ${error.message}` }], + } satisfies McpToolResult), + ), + "@maple/mcp/errors/McpAuthMissingError": (error) => + Effect.logError(`Auth error: ${error.message}`).pipe( + Effect.annotateLogs({ errorTag: error._tag }), + Effect.as({ + isError: true, + content: [{ type: "text", text: `${error._tag}: ${error.message}` }], + } satisfies McpToolResult), + ), + "@maple/mcp/errors/McpAuthInvalidError": (error) => + Effect.logError(`Auth error: ${error.message}`).pipe( + Effect.annotateLogs({ errorTag: error._tag }), + Effect.as({ + isError: true, + content: [{ type: "text", text: `${error._tag}: ${error.message}` }], + } satisfies McpToolResult), + ), + "@maple/mcp/errors/McpInvalidTenantError": (error) => + Effect.logError(`Tenant validation error [${error.field}]: ${error.message}`).pipe( + Effect.annotateLogs({ errorTag: error._tag, field: error.field }), + Effect.as({ + isError: true, + content: [ + { + type: "text", + text: `${error._tag} (${error.field}): ${error.message}`, + }, + ], + } satisfies McpToolResult), + ), + }), + Effect.catchDefect((error) => + Effect.logError(`Tool defect: ${toErrorMessage(error)}`).pipe( + Effect.as({ + isError: true, + content: [{ type: "text", text: `Error: ${toErrorMessage(error)}` }], + } satisfies McpToolResult), + ), + ), + Effect.annotateLogs({ tool: definition.name }), + ) +}) diff --git a/apps/api/src/mcp/lib/dashboard-mutations.test.ts b/apps/api/src/mcp/lib/dashboard-mutations.test.ts index 114042a8c..454f24035 100644 --- a/apps/api/src/mcp/lib/dashboard-mutations.test.ts +++ b/apps/api/src/mcp/lib/dashboard-mutations.test.ts @@ -12,20 +12,12 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { ConfigProvider, Effect, Layer, Schema } from "effect" -import { HttpServerRequest } from "effect/unstable/http" -import { - DashboardDocument, - DashboardId, - IsoDateTimeString, - OrgId, - UserId, -} from "@maple/domain/http" +import { DashboardDocument, DashboardId, IsoDateTimeString, OrgId, UserId } from "@maple/domain/http" import { DashboardPersistenceService } from "@/services/DashboardPersistenceService" -import { AuthService } from "@/services/AuthService" -import { ApiKeysService } from "@/services/ApiKeysService" import { Env } from "@/lib/Env" import { cleanupTestDbs, createTestDb, type TestDb } from "@/lib/test-pglite" import { withDashboardMutation } from "./dashboard-mutations" +import { CurrentMcpTenant } from "./query-warehouse" import { registerUpdateDashboardTool } from "@/mcp/tools/update-dashboard" import type { McpToolError, McpToolRegistrar, McpToolResult } from "@/mcp/tools/types" @@ -33,9 +25,8 @@ const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) -// The dashboard-mutation tools resolve their tenant from the inbound HTTP -// request. We take the internal-service auth branch (a `maple_svc_` bearer + -// `x-org-id`), which only needs `Env`, so no API key / session plumbing. +// MCP transport authentication resolves the tenant before dispatch. Tool tests +// inject that already-resolved context directly, matching both HTTP and RPC. const INTERNAL_TOKEN = "test-internal-token" const ORG = "org_no_tags" @@ -55,33 +46,16 @@ const testConfig = () => }), ) -const requestLayer = Layer.succeed( - HttpServerRequest.HttpServerRequest, - HttpServerRequest.fromWeb( - new Request("http://api.localhost/mcp", { - method: "POST", - headers: { - authorization: `Bearer maple_svc_${INTERNAL_TOKEN}`, - "x-org-id": ORG, - }, - }), - ), -) - const makeLayer = (testDb: TestDb) => Layer.mergeAll( DashboardPersistenceService.layer, - AuthService.layer, - ApiKeysService.layer, - requestLayer, - ).pipe( - Layer.provide(testDb.layer), - // `provideMerge` so `Env` is both satisfied for the services above and - // exposed in the output — `withDashboardMutation` → `resolveTenant` reads - // `Env` directly from the outer context. - Layer.provideMerge(Env.layer), - Layer.provide(testConfig()), - ) + Layer.succeed(CurrentMcpTenant, { + orgId: Schema.decodeUnknownSync(OrgId)(ORG), + userId: Schema.decodeUnknownSync(UserId)("internal-service"), + roles: [], + authMode: "self_hosted", + }), + ).pipe(Layer.provide(testDb.layer), Layer.provideMerge(Env.layer), Layer.provide(testConfig())) const asDashboardId = Schema.decodeUnknownSync(DashboardId) const asIsoDateTimeString = Schema.decodeUnknownSync(IsoDateTimeString) diff --git a/apps/api/src/mcp/lib/query-warehouse.ts b/apps/api/src/mcp/lib/query-warehouse.ts index 35641425b..b0e33a53e 100644 --- a/apps/api/src/mcp/lib/query-warehouse.ts +++ b/apps/api/src/mcp/lib/query-warehouse.ts @@ -1,14 +1,19 @@ import { HttpServerRequest } from "effect/unstable/http" import type { WarehouseQueryName } from "@maple/domain" -import { Effect } from "effect" +import { Context, Effect } from "effect" import { resolveMcpTenantContext } from "@/mcp/lib/resolve-tenant" +import type { TenantContext } from "@/lib/tenant-context" import { toMcpQueryError } from "@/mcp/lib/map-warehouse-error" import { McpAuthMissingError } from "@/mcp/tools/types" import { WarehouseQueryService } from "@/lib/WarehouseQueryService" import { WarehouseExecutor } from "@maple/query-engine/observability" import { makeWarehouseExecutorFromTenant } from "@/lib/WarehouseQueryService" -export const resolveTenant = Effect.gen(function* () { +export class CurrentMcpTenant extends Context.Service()( + "@maple/api/mcp/CurrentMcpTenant", +) {} + +export const resolveHttpMcpTenant = Effect.gen(function* () { const req = yield* HttpServerRequest.HttpServerRequest const nativeReq = yield* HttpServerRequest.toWeb(req).pipe( Effect.mapError((e) => new McpAuthMissingError({ message: `Failed to read request: ${e.message}` })), @@ -16,6 +21,8 @@ export const resolveTenant = Effect.gen(function* () { return yield* resolveMcpTenantContext(nativeReq) }) +export const resolveTenant = CurrentMcpTenant + /** Infrastructure binding: resolves tenant and provides WarehouseExecutor layer. */ export const withTenantExecutor = (effect: Effect.Effect) => Effect.fn("withTenantExecutor")(function* () { diff --git a/apps/api/src/mcp/server.ts b/apps/api/src/mcp/server.ts index 6b1e08384..cd5252dab 100644 --- a/apps/api/src/mcp/server.ts +++ b/apps/api/src/mcp/server.ts @@ -1,21 +1,9 @@ import { McpSchema, McpServer as EffectMcpServer } from "effect/unstable/ai" -import { Effect, Layer, Schema, Context } from "effect" -import { mapleToolDefinitions, toInputSchema, type MapleToolDefinition } from "./tools/registry" +import { Context, Effect, Layer } from "effect" +import { callMcpTool, listMcpTools } from "./dispatcher" +import { CurrentMcpTenant, resolveHttpMcpTenant } from "./lib/query-warehouse" import type { McpToolResult } from "./tools/types" -class McpDecodeError extends Schema.TaggedErrorClass()("@maple/mcp/decode-error", { - errorMessage: Schema.String, -}) {} - -const toErrorMessage = (error: unknown): string => { - if (error instanceof Error && "error" in error && (error as any).error != null) { - const inner = (error as any).error - return inner instanceof Error ? inner.message : String(inner) - } - if (error instanceof Error) return error.message - return String(error) -} - const toCallToolResult = (result: McpToolResult): typeof McpSchema.CallToolResult.Type => new McpSchema.CallToolResult({ isError: result.isError === true ? true : undefined, @@ -25,141 +13,43 @@ const toCallToolResult = (result: McpToolResult): typeof McpSchema.CallToolResul })), }) -const toDecodeErrorMessage = (definition: MapleToolDefinition, error: unknown): string => { - if (Schema.isSchemaError(error)) { - return `${String(error)}. Check the "${definition.name}" tool schema for valid parameter names and types.` - } - return String(error) -} +const toBoundaryErrorResult = (error: { readonly _tag: string; readonly message: string }) => + toCallToolResult({ + isError: true, + content: [{ type: "text", text: `${error._tag}: ${error.message}` }], + }) +/** Public MCP transport backed by the same dispatcher as internal Worker RPC. */ export const McpToolsLive = Layer.effectDiscard( Effect.gen(function* () { const server = yield* EffectMcpServer.McpServer - yield* Effect.forEach(mapleToolDefinitions, (definition) => + const descriptors = yield* listMcpTools + yield* Effect.forEach(descriptors, (descriptor) => server.addTool({ tool: new McpSchema.Tool({ - name: definition.name, - description: definition.description, - inputSchema: toInputSchema(definition.schema), + name: descriptor.name, + description: descriptor.description, + inputSchema: descriptor.inputSchema, }), annotations: Context.empty(), - handle: Effect.fn("McpTool.handle")( - function* (payload) { - yield* Effect.annotateCurrentSpan({ tool: definition.name }) - const decoded = yield* Effect.try({ - try: () => Schema.decodeUnknownSync(definition.schema)(payload), - catch: (error) => error, - }).pipe( - Effect.mapError( - (error) => - new McpDecodeError({ - errorMessage: toDecodeErrorMessage(definition, error), - }), - ), - ) - - return yield* definition.handler(decoded).pipe( - Effect.tap(() => Effect.logInfo("Tool completed")), - Effect.map(toCallToolResult), - ) - }, - Effect.catchTag("@maple/mcp/decode-error", (error) => - Effect.logWarning("Invalid parameters").pipe( - Effect.annotateLogs({ error: error.errorMessage }), - Effect.as( - toCallToolResult({ - isError: true, - content: [ - { - type: "text", - text: `Invalid parameters: ${error.errorMessage}`, - }, - ], - }), + handle: (payload) => + resolveHttpMcpTenant.pipe( + Effect.flatMap((tenant) => + callMcpTool(descriptor.name, payload).pipe( + Effect.provideService(CurrentMcpTenant, tenant), ), ), - ), - Effect.catchTags({ - "@maple/mcp/errors/McpQueryError": (error) => - Effect.logError(`Tool error: ${error.message}`).pipe( - Effect.annotateLogs({ - errorTag: error._tag, - pipe: error.pipeName, - }), - Effect.as( - toCallToolResult({ - isError: true, - content: [ - { type: "text", text: `${error._tag}: ${error.message}` }, - ], - }), - ), - ), - "@maple/mcp/errors/McpTenantError": (error) => - Effect.logError(`Tool error: ${error.message}`).pipe( - Effect.annotateLogs({ errorTag: error._tag }), - Effect.as( - toCallToolResult({ - isError: true, - content: [ - { type: "text", text: `${error._tag}: ${error.message}` }, - ], - }), - ), - ), + Effect.map(toCallToolResult), + Effect.catchTags({ + "@maple/internal-rpc/ToolNotFoundError": (error) => + Effect.succeed(toBoundaryErrorResult(error)), "@maple/mcp/errors/McpAuthMissingError": (error) => - Effect.logError(`Auth error: ${error.message}`).pipe( - Effect.annotateLogs({ errorTag: error._tag }), - Effect.as( - toCallToolResult({ - isError: true, - content: [ - { type: "text", text: `${error._tag}: ${error.message}` }, - ], - }), - ), - ), + Effect.succeed(toBoundaryErrorResult(error)), "@maple/mcp/errors/McpAuthInvalidError": (error) => - Effect.logError(`Auth error: ${error.message}`).pipe( - Effect.annotateLogs({ errorTag: error._tag }), - Effect.as( - toCallToolResult({ - isError: true, - content: [ - { type: "text", text: `${error._tag}: ${error.message}` }, - ], - }), - ), - ), + Effect.succeed(toBoundaryErrorResult(error)), "@maple/mcp/errors/McpInvalidTenantError": (error) => - Effect.logError( - `Tenant validation error [${error.field}]: ${error.message}`, - ).pipe( - Effect.annotateLogs({ errorTag: error._tag, field: error.field }), - Effect.as( - toCallToolResult({ - isError: true, - content: [ - { - type: "text", - text: `${error._tag} (${error.field}): ${error.message}`, - }, - ], - }), - ), - ), + Effect.succeed(toBoundaryErrorResult(error)), }), - Effect.catchDefect((error) => - Effect.logError(`Tool defect: ${toErrorMessage(error)}`).pipe( - Effect.as( - toCallToolResult({ - isError: true, - content: [{ type: "text", text: `Error: ${toErrorMessage(error)}` }], - }), - ), - ), - ), - Effect.annotateLogs({ tool: definition.name }), ), }), ) diff --git a/apps/api/src/routes/investigations-internal.http.ts b/apps/api/src/routes/investigations-internal.http.ts deleted file mode 100644 index 2439521d0..000000000 --- a/apps/api/src/routes/investigations-internal.http.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { HttpApiBuilder } from "effect/unstable/httpapi" -import { CurrentTenant, MapleApi } from "@maple/domain/http" -import { Effect } from "effect" -import { InvestigationService } from "../services/InvestigationService" - -/** - * Internal `submit_diagnosis` write the chat-flue investigate agent posts once - * it finishes its diagnostic pass: - * - * POST /api/internal/investigations/:id/diagnosis - * - * Server-to-server, authed by the internal-service token via the - * `InternalServiceAuthorization` middleware (see InternalServiceAuthorizationLayer), - * which provides the same `CurrentTenant.Context` the Clerk-authed groups use — so - * a service caller can only write investigations in the org it resolves to. - * - * The framework owns the boilerplate: `:id`/payload decode (→ 400), auth (→ 401), - * and the declared `InvestigationNotFoundError`/`InvestigationPersistenceError` - * → 404/503 mapping (via their `httpApiStatus`). The handler is just the write. - */ -export const HttpInvestigationsInternalLive = HttpApiBuilder.group( - MapleApi, - "investigationsInternal", - (handlers) => - Effect.gen(function* () { - const service = yield* InvestigationService - - return handlers.handle("submitDiagnosis", ({ params, payload }) => - Effect.gen(function* () { - const tenant = yield* CurrentTenant.Context - yield* Effect.annotateCurrentSpan({ - "maple.org_id": tenant.orgId, - "maple.investigation.id": params.id, - }) - return yield* service.submitDiagnosis(tenant.orgId, params.id, payload) - }).pipe(Effect.withSpan("HttpInvestigationsInternal.submitDiagnosis")), - ) - }), -) diff --git a/apps/api/src/services/InternalServiceAuthorizationLayer.ts b/apps/api/src/services/InternalServiceAuthorizationLayer.ts deleted file mode 100644 index b91f8e1de..000000000 --- a/apps/api/src/services/InternalServiceAuthorizationLayer.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { HttpServerRequest } from "effect/unstable/http" -import { CurrentTenant, UnauthorizedError } from "@maple/domain/http" -import { Effect, Layer } from "effect" -import { resolveMcpTenantContext } from "../mcp/lib/resolve-tenant" -import { AuthService } from "./AuthService" -import { ApiKeysService } from "./ApiKeysService" -import { Env } from "../lib/Env" - -const messageOf = (error: unknown): string => - typeof error === "object" && error !== null && "message" in error && typeof error.message === "string" - ? error.message - : "Unauthorized" - -/** - * Implementation of the {@link CurrentTenant.InternalServiceAuthorization} - * middleware — the server-to-server counterpart of {@link ApiAuthorizationLayer}. - * - * Resolves the tenant from the internal-service bearer token (+ `x-org-id`) via - * the same {@link resolveMcpTenantContext} the MCP server uses, then provides the - * shared {@link CurrentTenant.Context} so internal HttpApi handlers read the org - * exactly like the Clerk-authed ones. Any resolution failure becomes a declared - * `UnauthorizedError` (→ 401), so the route never maps errors by hand. - */ -export const InternalServiceAuthorizationLayer = Layer.effect( - CurrentTenant.InternalServiceAuthorization, - Effect.gen(function* () { - const env = yield* Env - const apiKeys = yield* ApiKeysService - const auth = yield* AuthService - - return CurrentTenant.InternalServiceAuthorization.of({ - bearer: (httpEffect) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest - const webRequest = yield* HttpServerRequest.toWeb(request).pipe( - Effect.mapError(() => new UnauthorizedError({ message: "Failed to read request" })), - ) - - const tenant = yield* Effect.provideService( - Effect.provideService( - Effect.provideService(resolveMcpTenantContext(webRequest), Env, env), - ApiKeysService, - apiKeys, - ), - AuthService, - auth, - ).pipe(Effect.mapError((error) => new UnauthorizedError({ message: messageOf(error) }))) - - return yield* Effect.provideService( - httpEffect, - CurrentTenant.Context, - new CurrentTenant.TenantSchema({ - orgId: tenant.orgId, - userId: tenant.userId, - roles: tenant.roles, - authMode: tenant.authMode, - }), - ) - }), - }) - }), -) diff --git a/apps/api/src/worker.ts b/apps/api/src/worker.ts index e4fc06c8d..1b0b07353 100644 --- a/apps/api/src/worker.ts +++ b/apps/api/src/worker.ts @@ -1,8 +1,14 @@ import type { MessageBatch, ScheduledController } from "@cloudflare/workers-types" import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" import { ANTICIPATED_ERROR_TAGS } from "@maple/domain/anticipated-errors" -import { runScheduledEffect, WorkerConfigProviderLayer, WorkerEnvironment } from "@maple/effect-cloudflare" -import { Context, FileSystem, Layer, Path } from "effect" +import { + layerFromEnvRecord, + runScheduledEffect, + WorkerConfigProviderLayer, + WorkerEnvironment, +} from "@maple/effect-cloudflare" +import { WorkerEntrypoint } from "cloudflare:workers" +import { Cause, Context, Effect, FileSystem, Layer, ManagedRuntime, Path } from "effect" import { HttpMiddleware, HttpRouter } from "effect/unstable/http" import * as Etag from "effect/unstable/http/Etag" import * as HttpPlatform from "effect/unstable/http/HttpPlatform" @@ -79,14 +85,12 @@ const passThroughMiddleware: HttpMiddleware.HttpMiddleware = (httpApp) => httpAp // the top level near-empty; the cost moves to the first request, which runs // under the far larger per-request CPU budget. const buildHandler = async () => { - const { AllRoutes, ApiAuthLive, InternalServiceAuthLive, ApiObservabilityLive, MainLive } = - await import("./app") + const { AllRoutes, ApiAuthLive, ApiObservabilityLive, MainLive } = await import("./app") const { layerPg } = await import("./lib/DatabasePgLive") return HttpRouter.toWebHandler( AllRoutes.pipe( Layer.provideMerge(MainLive), Layer.provideMerge(ApiAuthLive), - Layer.provideMerge(InternalServiceAuthLive), Layer.provideMerge(ApiObservabilityLive), Layer.provideMerge(WorkerPlatformLive), Layer.provideMerge(layerPg), @@ -106,6 +110,74 @@ const buildHandler = async () => { let handlerPromise: ReturnType | undefined const getHandler = () => (handlerPromise ??= buildHandler()) +// RPC has no HttpApi request to construct the application services for it, so +// it gets a sibling isolate-wide ManagedRuntime. The heavy route/service graph +// stays behind a dynamic import, preserving the worker's startup-CPU budget. +const buildRpcRuntime = async (env: Record) => { + const { MainLive } = await import("./app") + const { layerPg } = await import("./lib/DatabasePgLive") + return ManagedRuntime.make( + MainLive.pipe( + Layer.provideMerge(WorkerPlatformLive), + Layer.provideMerge(layerPg), + Layer.provideMerge(layerFromEnvRecord(env)), + Layer.provideMerge(telemetry.layer), + Layer.provideMerge(WorkerConfigProviderLayer), + ), + ) +} + +let rpcRuntimePromise: ReturnType | undefined +const getRpcRuntime = (env: Record) => (rpcRuntimePromise ??= buildRpcRuntime(env)) + +type InternalRpcMethod = "listMcpTools" | "callMcpTool" | "submitDiagnosis" + +const ALCHEMY_RPC_ERROR_TAG = "~alchemy/rpc/error" as const + +// Alchemy's schemaless RPC error envelope is deliberately tiny. Keeping this +// encoder local avoids pulling its full Worker bridge into an already large API +// bundle; chat-flue's `toRpcAsync` decodes this exact public wire shape. +const encodeRpcError = (error: unknown): unknown => { + if (error == null || typeof error !== "object") return error + const object = error as Record + if (typeof object._tag === "string") { + const encoded = Object.fromEntries(Object.keys(object).map((key) => [key, object[key]])) + if (error instanceof Error && !("message" in encoded)) encoded.message = error.message + return encoded + } + if (error instanceof Error) { + return { name: error.name, message: error.message, stack: error.stack } + } + return error +} + +const runInternalRpc = async ( + method: InternalRpcMethod, + input: unknown, + env: Record, + ctx: ExecutionContext, +) => { + const [runtime, rpc] = await Promise.all([getRpcRuntime(env), import("./internal-rpc")]) + const program = + method === "listMcpTools" + ? rpc.listMcpToolsRpc + : method === "callMcpTool" + ? rpc.callMcpToolRpc(input) + : rpc.submitDiagnosisRpc(input) + const exit = await runtime.runPromiseExit(program as Effect.Effect) + ctx.waitUntil(flushTelemetry(env)) + if (exit._tag === "Success") return exit.value + const failure = exit.cause.reasons.find(Cause.isFailReason) + if (failure) { + return { + _tag: ALCHEMY_RPC_ERROR_TAG, + error: encodeRpcError(failure.error), + } + } + const defect = exit.cause.reasons.find(Cause.isDieReason) + throw defect?.defect ?? new Error("RPC method failed with an unexpected cause") +} + const isMcpPost = (request: Request): boolean => { if (request.method !== "POST") return false try { @@ -257,11 +329,34 @@ const handleScheduled = async (env: Record, ctx: ExecutionConte } } -export default { - fetch: (request: Request, env: Record, ctx: ExecutionContext) => - handle(request, env, ctx), - queue: (batch: MessageBatch, env: Record, ctx: ExecutionContext) => - handleQueue(batch, env, ctx), - scheduled: (_event: ScheduledController, env: Record, ctx: ExecutionContext) => - handleScheduled(env, ctx), +/** + * Class entrypoint keeps fetch/queue/cron intact while publishing Alchemy's + * schemaless RPC methods over a Cloudflare service binding. RPC failures are + * encoded with Alchemy's wire envelope so a plain Worker caller can recover tagged + * Effect errors via `toRpcAsync`. + */ +export default class MapleApiWorker extends WorkerEntrypoint> { + override fetch(request: Request): Promise { + return handle(request, this.env, this.ctx) + } + + override queue(batch: MessageBatch): Promise { + return handleQueue(batch, this.env, this.ctx) + } + + override scheduled(_event: ScheduledController): Promise { + return handleScheduled(this.env, this.ctx) + } + + listMcpTools() { + return runInternalRpc("listMcpTools", undefined, this.env, this.ctx) + } + + callMcpTool(input: unknown) { + return runInternalRpc("callMcpTool", input, this.env, this.ctx) + } + + submitDiagnosis(input: unknown) { + return runInternalRpc("submitDiagnosis", input, this.env, this.ctx) + } } diff --git a/apps/api/wrangler.jsonc b/apps/api/wrangler.jsonc index af677dd52..06be5ce8b 100644 --- a/apps/api/wrangler.jsonc +++ b/apps/api/wrangler.jsonc @@ -27,6 +27,7 @@ "id": "00000000000000000000000000000000", }, ], + "services": [{ "binding": "CHAT_FLUE", "service": "maple-chat-flue" }], // Cloudflare Email Service (Email Sending). `remote: true` routes sends through // the real Cloudflare sender even under local dev. (Mirrored in alchemy.run.ts // for deploy.) Pure `bun dev` (non-wrangler) has no EMAIL binding → sends skip. diff --git a/apps/chat-flue/.dev.vars.example b/apps/chat-flue/.dev.vars.example index 7d37b72c0..78f31e31c 100644 --- a/apps/chat-flue/.dev.vars.example +++ b/apps/chat-flue/.dev.vars.example @@ -1,7 +1,7 @@ # Copy to `.dev.vars` for `flue dev`. Do not commit `.dev.vars`. # Shared secret for Maple internal-service auth. Must match apps/api's -# INTERNAL_SERVICE_TOKEN — the Flue agent sends `Bearer maple_svc_`. +# INTERNAL_SERVICE_TOKEN — authenticates API → Flue private workflow routes. INTERNAL_SERVICE_TOKEN="dev-internal-service-token" # Optional Workers AI model override (provider id `cloudflare` + @cf model id). diff --git a/apps/chat-flue/README.md b/apps/chat-flue/README.md index 873e4e392..5fcbaaed3 100644 --- a/apps/chat-flue/README.md +++ b/apps/chat-flue/README.md @@ -11,8 +11,8 @@ the architecture before the full rebuild. See the plan at | File | Role | | ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `src/agents/maple-chat.ts` | The addressable chat agent. `export const route` exposes it at `POST/GET /agents/maple-chat/:id`; Workers AI model + `connectMcpServer` → Maple MCP tools (`mcp__maple__*`). | -| `src/lib/env.ts` | Worker bindings/vars (`AI`, `MAPLE_API_URL`, `INTERNAL_SERVICE_TOKEN`). | +| `src/agents/maple-chat.ts` | The addressable chat agent. `export const route` exposes it at `POST/GET /agents/maple-chat/:id`; Workers AI model + API Worker RPC → Maple MCP tools (`mcp__maple__*`). | +| `src/lib/env.ts` | Worker bindings/vars (`AI`, `MAPLE_API_RPC`, `INTERNAL_SERVICE_TOKEN`). | | `src/lib/org.ts` | Recovers `orgId` from the `":"` instance id. | | `flue.config.ts` / `wrangler.jsonc` | Flue + Cloudflare build config (Workers AI `AI` binding, DO migrations). | @@ -60,7 +60,7 @@ bun run connect # flue connect maple-chat local - [x] `bun run build` (`flue build --target cloudflare`) discovers `maple-chat` and generates a valid worker (AI binding + `FlueMapleChatAgent`/`FlueRegistry` DOs). - [x] `flue dev` boots the worker on the Cloudflare target. - [x] A prompt runs end-to-end on **Workers AI** (`@cf/moonshotai/kimi-k2.6`): `POST /agents/maple-chat/:?wait=result` → HTTP 200 with the model's reply (~1.5s). The binding path works; the legacy-parity kimi model is live. -- [ ] A prompt calls `mcp__maple__search_traces` against `apps/api` — **needs a `flue dev` restart** so it loads `.dev.vars` (`INTERNAL_SERVICE_TOKEN`), plus a running `apps/api` and a real org id. +- [ ] A prompt calls `mcp__maple__search_traces` through the `MAPLE_API_RPC` service binding, with a running `apps/api` and a real org id. - [ ] A browser page streams the agent's events via `@flue/sdk` (`agents.send` + `agents.stream`). - [ ] Tool-calling quality across the full tool set is acceptable on the chosen model. diff --git a/apps/chat-flue/alchemy.run.ts b/apps/chat-flue/alchemy.run.ts index c7017401a..5d9f00d48 100644 --- a/apps/chat-flue/alchemy.run.ts +++ b/apps/chat-flue/alchemy.run.ts @@ -10,6 +10,7 @@ import { resolveDeploymentEnvironment, resolveWorkerName, } from "@maple/infra/cloudflare" +import type { MapleApiWorker } from "../api/alchemy.run.ts" const requireEnv = (key: string): string => { const value = process.env[key]?.trim() @@ -32,10 +33,10 @@ const optionalSecret = (key: string): Record> export interface CreateChatFlueWorkerOptions { stage: MapleStage domains: MapleDomains - mapleApiUrl: string + /** API worker deployed first, then bound privately for schemaless RPC. */ + mapleApiRpc: MapleApiWorker } - /** * Deploy the Flue chat worker (`apps/chat-flue`) via alchemy, consistent with * the rest of the stack. @@ -53,7 +54,7 @@ export interface CreateChatFlueWorkerOptions { * Manual fallback (Flue-native): `cd apps/chat-flue && bun run build && * wrangler deploy --config dist/maple_chat_flue/wrangler.json`. */ -export const createChatFlueWorker = ({ stage, domains, mapleApiUrl }: CreateChatFlueWorkerOptions) => +export const createChatFlueWorker = ({ stage, domains, mapleApiRpc }: CreateChatFlueWorkerOptions) => Effect.gen(function* () { // Flue generates the Worker entrypoint + DO classes; build before deploy. const build = yield* Command.Build("chat-flue-build", { @@ -109,7 +110,7 @@ export const createChatFlueWorker = ({ stage, domains, mapleApiUrl }: CreateChat FLUE_MAPLE_CHAT_AGENT: chatAgent, FLUE_TRIAGE_WORKFLOW: triageWorkflow, FLUE_REGISTRY: registry, - MAPLE_API_URL: mapleApiUrl, + MAPLE_API_RPC: mapleApiRpc, INTERNAL_SERVICE_TOKEN: Redacted.make(requireEnv("INTERNAL_SERVICE_TOKEN")), // OpenTelemetry → Maple ingest. Provide the internal-org ingest key so // chat-flue spans land beside `maple-api`; telemetry no-ops when unset. diff --git a/apps/chat-flue/package.json b/apps/chat-flue/package.json index a6d34b7b7..67f5c0bc9 100644 --- a/apps/chat-flue/package.json +++ b/apps/chat-flue/package.json @@ -14,6 +14,7 @@ "@clerk/backend": "^2.30.1", "@flue/opentelemetry": "1.0.0-beta.1", "@flue/runtime": "1.0.0-beta.2", + "@maple/domain": "workspace:*", "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.0", "@opentelemetry/exporter-trace-otlp-http": "^0.205.0", @@ -21,6 +22,7 @@ "@opentelemetry/sdk-trace-base": "^2.0.0", "@opentelemetry/semantic-conventions": "^1.36.0", "agents": "^0.14.1", + "alchemy": "2.0.0-beta.61", "hono": "^4.8.3", "valibot": "^1.1.0" }, diff --git a/apps/chat-flue/src/agents/maple-chat.ts b/apps/chat-flue/src/agents/maple-chat.ts index 9c614310f..3836ad681 100644 --- a/apps/chat-flue/src/agents/maple-chat.ts +++ b/apps/chat-flue/src/agents/maple-chat.ts @@ -78,7 +78,7 @@ export default createAgent(async (ctx) => { const instructions = buildSystemPrompt({ mode }) // Connect to Maple's MCP server (all tools). We tolerate connection failures so - // the agent still answers on Workers AI when apps/api or INTERNAL_SERVICE_TOKEN + // the agent still answers on Workers AI when the API RPC binding // isn't wired yet. The initializer runs per interaction and we don't `close()` // here because tool calls need the connection live for the whole turn — // connection lifecycle/pooling is a follow-up. diff --git a/apps/chat-flue/src/lib/api-rpc.test.ts b/apps/chat-flue/src/lib/api-rpc.test.ts new file mode 100644 index 000000000..11201b77b --- /dev/null +++ b/apps/chat-flue/src/lib/api-rpc.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest" +import type { ChatFlueEnv } from "./env.ts" +import { mapleApiRpc } from "./api-rpc.ts" +import { buildSubmitDiagnosisTool, DIAGNOSIS_STATUS } from "./submit-diagnosis.ts" + +const report = { + summary: "Checkout latency doubled after deploy.", + suspectedCause: "Connection pool regression", + severityAssessment: "high" as const, + affectedScope: "checkout-api", + evidence: [ + { + traceIds: ["trace-1"], + logPatterns: ["pool exhausted"], + relatedServices: ["payments"], + note: "Correlated evidence", + }, + ], + suggestedActions: ["Roll back"], + confidence: "high" as const, +} + +const envWithBinding = (binding: Record): ChatFlueEnv => ({ + AI: { run: async () => ({}) }, + MAPLE_API_RPC: binding as never, + INTERNAL_SERVICE_TOKEN: "workflow-token", +}) + +describe("Maple API Worker RPC", () => { + it("preserves tagged errors across the Alchemy RPC envelope", async () => { + const api = mapleApiRpc( + envWithBinding({ + callMcpTool: async () => ({ + _tag: "~alchemy/rpc/error", + error: { + _tag: "@maple/internal-rpc/InvalidInputError", + method: "callMcpTool", + message: "invalid org", + }, + }), + }), + ) + + await expect(api.callMcpTool({ orgId: " ", name: "x", input: {} })).rejects.toMatchObject({ + _tag: "@maple/internal-rpc/InvalidInputError", + method: "callMcpTool", + }) + }) + + it("submits a diagnosis over RPC and returns the existing render marker", async () => { + const calls: unknown[] = [] + const env = envWithBinding({ + submitDiagnosis: async (request: unknown) => { + calls.push(request) + return { id: "ignored-by-chat" } + }, + }) + const tool = buildSubmitDiagnosisTool(env, "org_1", "00000000-0000-4000-8000-000000000001") + const result = JSON.parse(await tool.execute(report)) + + expect(calls).toEqual([ + { + orgId: "org_1", + investigationId: "00000000-0000-4000-8000-000000000001", + report, + }, + ]) + expect(result).toEqual({ status: DIAGNOSIS_STATUS, report }) + }) +}) diff --git a/apps/chat-flue/src/lib/api-rpc.ts b/apps/chat-flue/src/lib/api-rpc.ts new file mode 100644 index 000000000..202ed9f03 --- /dev/null +++ b/apps/chat-flue/src/lib/api-rpc.ts @@ -0,0 +1,6 @@ +import { toRpcAsync } from "alchemy/Cloudflare/Bridge" +import type { MapleApiRpcShape } from "@maple/domain/internal-rpc" +import type { ChatFlueEnv } from "./env.ts" + +/** Fresh per-invocation async facade over the API Worker service binding. */ +export const mapleApiRpc = (env: ChatFlueEnv) => toRpcAsync(env.MAPLE_API_RPC) diff --git a/apps/chat-flue/src/lib/auth.ts b/apps/chat-flue/src/lib/auth.ts index 2c94caf20..0e53e5c23 100644 --- a/apps/chat-flue/src/lib/auth.ts +++ b/apps/chat-flue/src/lib/auth.ts @@ -107,7 +107,8 @@ const SERVICE_TOKEN_PREFIX = "maple_svc_" /** * Verify an internal-service-token request (apps/api → chat-flue server-to-server, * e.g. the headless triage workflow). The bearer value must equal - * `maple_svc_` — the same scheme the MCP connection uses + * `maple_svc_` — the API uses this private route to start + * Flue workflows over its service binding. * (see lib/mcp.ts). Constant-time compared; fails closed when the token is unset. */ export const verifyInternalServiceToken = (request: Request, env: ChatFlueEnv): boolean => { diff --git a/apps/chat-flue/src/lib/env.ts b/apps/chat-flue/src/lib/env.ts index 795020759..0abd01595 100644 --- a/apps/chat-flue/src/lib/env.ts +++ b/apps/chat-flue/src/lib/env.ts @@ -4,9 +4,9 @@ import type { CloudflareAIBinding } from "@flue/runtime/cloudflare" export interface ChatFlueEnv { /** Workers AI binding. Backs the `cloudflare/*` model provider (env.AI.run). */ AI: CloudflareAIBinding - /** Base URL of the Maple API worker that hosts the MCP server (`/mcp`). */ - MAPLE_API_URL: string - /** Shared secret for Maple internal-service auth (`Bearer maple_svc_`). */ + /** Private service binding to the Maple API's schemaless RPC surface. */ + MAPLE_API_RPC: Service + /** Shared secret used to authenticate the API's private Flue workflow calls. */ INTERNAL_SERVICE_TOKEN: string /** Optional Workers AI model override, e.g. `cloudflare/@cf/meta/llama-3.3-70b-instruct-fp8-fast`. */ MAPLE_CHAT_MODEL?: string diff --git a/apps/chat-flue/src/lib/mcp.ts b/apps/chat-flue/src/lib/mcp.ts index a551a6560..5b16d3236 100644 --- a/apps/chat-flue/src/lib/mcp.ts +++ b/apps/chat-flue/src/lib/mcp.ts @@ -1,7 +1,9 @@ -import { connectMcpServer, type McpServerConnection } from "@flue/runtime" +import type { McpServerConnection, ToolDefinition } from "@flue/runtime" +import type { InternalMcpToolResult } from "@maple/domain/internal-rpc" import type { ChatFlueEnv } from "./env.ts" +import { mapleApiRpc } from "./api-rpc.ts" -/** Prefix `connectMcpServer` adapts MCP tool names with: `mcp____`. */ +/** Prefix retained for parity with Flue's MCP HTTP adapter. */ const MCP_PREFIX = "mcp__maple__" /** Strip the `mcp__maple__` prefix to recover the registry tool name. */ @@ -14,45 +16,89 @@ export const filterMcpTools = ( allowlist: ReadonlySet, ): T[] => tools.filter((tool) => allowlist.has(baseToolName(tool.name))) -/** Default MCP request timeout (ms): an unreachable endpoint fails fast rather than hanging the turn. */ +/** Default internal RPC timeout (ms): an unavailable API fails the turn promptly. */ export const MCP_DEFAULT_TIMEOUT_MS = 12_000 export interface ConnectMapleMcpOptions { /** If set, keep only tools whose base name is in this allowlist (e.g. the triage subset). */ allowlist?: ReadonlySet - /** MCP request timeout in ms. Defaults to {@link MCP_DEFAULT_TIMEOUT_MS}. */ + /** Worker RPC timeout in ms. Defaults to {@link MCP_DEFAULT_TIMEOUT_MS}. */ timeoutMs?: number } +const sanitizeToolNamePart = (value: string): string => + value.replace(/[^A-Za-z0-9_-]/g, "_").replace(/^_+|_+$/g, "") || "unnamed" + +const normalizeInputSchema = (schema: Record): Record => ({ + ...schema, + type: schema.type ?? "object", + properties: schema.properties ?? {}, +}) + +const formatResult = (result: InternalMcpToolResult): string => + result.content + .map((entry) => entry.text) + .filter(Boolean) + .join("\n\n") || "(MCP tool returned no content)" + +const withTimeout = async (promise: Promise, timeoutMs: number, operation: string): Promise => { + let timeout: ReturnType | undefined + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new Error(`${operation} timed out after ${timeoutMs}ms`)), + timeoutMs, + ) + }), + ]) + } finally { + if (timeout !== undefined) clearTimeout(timeout) + } +} + /** - * Connect to Maple's MCP server with internal-service auth. The tenant rides - * out-of-band in `x-org-id` (never trusted from model/tool output); the contract - * is apps/api/src/mcp/lib/resolve-tenant.ts. Tools arrive adapted as - * `mcp__maple__`; an optional `allowlist` narrows them by base name. - * - * The caller owns the returned connection's lifecycle and must `close()` it. + * Adapt API Worker RPC descriptors into the same ordinary Flue tools that the + * former streamable-HTTP MCP connection returned. The service binding itself + * authenticates the Worker-to-Worker hop; org scope is explicit and validated + * again by the API boundary. */ export const connectMapleMcp = async ( env: ChatFlueEnv, orgId: string, options: ConnectMapleMcpOptions = {}, ): Promise => { - // Fail fast rather than sending `Bearer maple_svc_` (empty): an empty token - // would match an empty server-side INTERNAL_SERVICE_TOKEN via the constant-time - // compare in apps/api resolve-tenant.ts. Callers already tolerate a throw. - const token = env.INTERNAL_SERVICE_TOKEN - if (!token) throw new Error("INTERNAL_SERVICE_TOKEN is not configured") - - const maple = await connectMcpServer("maple", { - url: new URL("/mcp", env.MAPLE_API_URL).toString(), - transport: "streamable-http", - headers: { - Authorization: `Bearer maple_svc_${token}`, - "x-org-id": orgId, - }, - timeoutMs: options.timeoutMs ?? MCP_DEFAULT_TIMEOUT_MS, + const timeoutMs = options.timeoutMs ?? MCP_DEFAULT_TIMEOUT_MS + const api = mapleApiRpc(env) + const descriptors = await withTimeout(api.listMcpTools(), timeoutMs, "listMcpTools") + const seen = new Set() + const tools: ToolDefinition[] = descriptors.map((descriptor) => { + const name = `${MCP_PREFIX}${sanitizeToolNamePart(descriptor.name)}` + if (seen.has(name)) throw new Error(`Maple RPC tools produced duplicate tool name "${name}"`) + seen.add(name) + + return { + name, + description: `MCP tool "${descriptor.name}" from server "maple". ${descriptor.description}`, + parameters: normalizeInputSchema(descriptor.inputSchema), + execute: async (input, signal) => { + if (signal?.aborted) throw new Error("Operation aborted") + const result = await withTimeout( + api.callMcpTool({ orgId, name: descriptor.name, input }), + timeoutMs, + `callMcpTool(${descriptor.name})`, + ) + const text = formatResult(result) + if (result.isError) throw new Error(text) + return text + }, + } }) - if (!options.allowlist) return maple - return { ...maple, tools: filterMcpTools(maple.tools, options.allowlist) } + return { + name: "maple", + tools: options.allowlist ? filterMcpTools(tools, options.allowlist) : tools, + close: async () => undefined, + } } diff --git a/apps/chat-flue/src/lib/submit-diagnosis.ts b/apps/chat-flue/src/lib/submit-diagnosis.ts index 1fb5c9027..9cee9896d 100644 --- a/apps/chat-flue/src/lib/submit-diagnosis.ts +++ b/apps/chat-flue/src/lib/submit-diagnosis.ts @@ -1,6 +1,7 @@ import type { ToolDefinition } from "@flue/runtime" import type { ChatFlueEnv } from "./env.ts" import { AiTriageResultSchema, type AiTriageResult } from "./triage-result.ts" +import { mapleApiRpc } from "./api-rpc.ts" /** Marker the `submit_diagnosis` tool returns; the web renders it as the report card. */ export const DIAGNOSIS_STATUS = "diagnosis" as const @@ -33,23 +34,7 @@ export const buildSubmitDiagnosisTool = ( "Record your structured diagnosis for THIS investigation. Call it exactly once, after you have gathered evidence, with your final assessment (summary, suspectedCause, severityAssessment, affectedScope, evidence, suggestedActions, confidence). It persists the report and renders it for the user. After calling it, stop unless the user asks a follow-up question.", parameters: AiTriageResultSchema, execute: async (report) => { - const url = new URL( - `/api/internal/investigations/${investigationId}/diagnosis`, - env.MAPLE_API_URL, - ).toString() - const res = await fetch(url, { - method: "POST", - headers: { - "content-type": "application/json", - Authorization: `Bearer maple_svc_${env.INTERNAL_SERVICE_TOKEN}`, - "x-org-id": orgId, - }, - body: JSON.stringify({ report }), - }) - if (!res.ok) { - const detail = await res.text().catch(() => "") - throw new Error(`submit_diagnosis failed (${res.status}): ${detail.slice(0, 200)}`) - } + await mapleApiRpc(env).submitDiagnosis({ orgId, investigationId, report }) return JSON.stringify({ status: DIAGNOSIS_STATUS, report } satisfies DiagnosisMarker) }, }) diff --git a/apps/chat-flue/src/lib/triage.test.ts b/apps/chat-flue/src/lib/triage.test.ts index ec4b2b89d..cf2e892aa 100644 --- a/apps/chat-flue/src/lib/triage.test.ts +++ b/apps/chat-flue/src/lib/triage.test.ts @@ -55,15 +55,35 @@ describe("mcp tool filtering", () => { expect(kept).toEqual(["mcp__maple__search_traces", "mcp__maple__find_errors"]) }) - it("connectMapleMcp fails fast (no empty bearer) when the token is unset", async () => { + it("connectMapleMcp discovers tools over the API Worker binding", async () => { + const calls: unknown[] = [] const env: ChatFlueEnv = { AI: { run: async () => ({}) }, - MAPLE_API_URL: "http://maple-test.invalid", + MAPLE_API_RPC: { + listMcpTools: async () => [ + { + name: "search_traces", + description: "Search traces", + inputSchema: { type: "object", properties: {} }, + }, + ], + callMcpTool: async (request: unknown) => { + calls.push(request) + return { content: [{ type: "text", text: "ok" }] } + }, + } as never, INTERNAL_SERVICE_TOKEN: "", } - await expect(connectMapleMcp(env, "org_1")).rejects.toThrow( - "INTERNAL_SERVICE_TOKEN is not configured", - ) + const connection = await connectMapleMcp(env, "org_1") + expect(connection.tools.map((tool) => tool.name)).toEqual(["mcp__maple__search_traces"]) + expect(await connection.tools[0]!.execute({ service: "checkout" })).toBe("ok") + expect(calls).toEqual([ + { + orgId: "org_1", + name: "search_traces", + input: { service: "checkout" }, + }, + ]) }) }) diff --git a/apps/chat-flue/wrangler.jsonc b/apps/chat-flue/wrangler.jsonc index 5e106caa9..f7430558d 100644 --- a/apps/chat-flue/wrangler.jsonc +++ b/apps/chat-flue/wrangler.jsonc @@ -21,10 +21,7 @@ "new_sqlite_classes": ["FlueRegistry", "FlueMapleChatAgent", "FlueTriageWorkflow"], }, ], - "vars": { - // Maple API worker that hosts the MCP server at `/mcp`. - "MAPLE_API_URL": "http://localhost:3472", - }, + "services": [{ "binding": "MAPLE_API_RPC", "service": "maple-api" }], // Workers Observability — `traces.enabled` is required for `tracing.enterSpan` // custom spans; the "maple" destination forwards logs + traces to Maple ingest. // The real deploy is Alchemy-owned (alchemy.run.ts sets this too); kept here in diff --git a/bun.lock b/bun.lock index 0296cd480..56bd1205c 100644 --- a/bun.lock +++ b/bun.lock @@ -80,6 +80,7 @@ "@clerk/backend": "^2.30.1", "@flue/opentelemetry": "1.0.0-beta.1", "@flue/runtime": "1.0.0-beta.2", + "@maple/domain": "workspace:*", "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^2.0.0", "@opentelemetry/exporter-trace-otlp-http": "^0.205.0", @@ -87,6 +88,7 @@ "@opentelemetry/sdk-trace-base": "^2.0.0", "@opentelemetry/semantic-conventions": "^1.36.0", "agents": "^0.14.1", + "alchemy": "2.0.0-beta.61", "hono": "^4.8.3", "valibot": "^1.1.0", }, diff --git a/packages/domain/package.json b/packages/domain/package.json index 5d6e88bea..72f47f1cb 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -7,6 +7,7 @@ "./anticipated-errors": "./src/anticipated-errors.ts", "./billing": "./src/billing.ts", "./http": "./src/http/index.ts", + "./internal-rpc": "./src/internal-rpc.ts", "./primitives": "./src/primitives.ts", "./query-engine": "./src/query-engine.ts", "./recommendations": "./src/recommendations.ts", diff --git a/packages/domain/src/http/api.ts b/packages/domain/src/http/api.ts index 0aea37099..cf2a37421 100644 --- a/packages/domain/src/http/api.ts +++ b/packages/domain/src/http/api.ts @@ -13,7 +13,7 @@ import { ErrorsApiGroup } from "./errors" import { IngestAttributeMappingsApiGroup } from "./ingest-attribute-mappings" import { IngestKeysApiGroup } from "./ingest-keys" import { IntegrationsApiGroup } from "./integrations" -import { InvestigationApiGroup, InvestigationsInternalApiGroup } from "./investigations" +import { InvestigationApiGroup } from "./investigations" import { ObservabilityApiGroup } from "./observability" import { OnboardingApiGroup } from "./onboarding" import { OrgClickHouseSettingsApiGroup } from "./org-clickhouse-settings" @@ -41,7 +41,6 @@ export class MapleApi extends HttpApi.make("MapleApi") .add(IngestKeysApiGroup) .add(IntegrationsApiGroup) .add(InvestigationApiGroup) - .add(InvestigationsInternalApiGroup) .add(ObservabilityApiGroup) .add(OnboardingApiGroup) .add(OrgClickHouseSettingsApiGroup) diff --git a/packages/domain/src/http/current-tenant.ts b/packages/domain/src/http/current-tenant.ts index aa834cd93..092054c4f 100644 --- a/packages/domain/src/http/current-tenant.ts +++ b/packages/domain/src/http/current-tenant.ts @@ -32,21 +32,3 @@ export class Authorization extends HttpApiMiddleware.Service< bearer: HttpApiSecurity.bearer, }, }) {} - -/** - * Server-to-server auth for internal endpoints (the chat-flue `submit_diagnosis` - * write, etc.). Verifies the internal-service bearer token rather than a Clerk - * session, but provides the same {@link Context} so handlers read the tenant - * uniformly. The implementation lives in `apps/api` (InternalServiceAuthorizationLayer). - */ -export class InternalServiceAuthorization extends HttpApiMiddleware.Service< - InternalServiceAuthorization, - { - provides: Context - } ->()("InternalServiceAuthorization", { - error: UnauthorizedError, - security: { - bearer: HttpApiSecurity.bearer, - }, -}) {} diff --git a/packages/domain/src/http/investigations.ts b/packages/domain/src/http/investigations.ts index bbd534072..2b2eed7f2 100644 --- a/packages/domain/src/http/investigations.ts +++ b/packages/domain/src/http/investigations.ts @@ -2,7 +2,7 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi" import { Schema } from "effect" import { ErrorIssueId, InvestigationId, IsoDateTimeString, UserId } from "../primitives" import { AiTriageIncidentKind, AiTriageResult } from "./ai-triage" -import { Authorization, InternalServiceAuthorization } from "./current-tenant" +import { Authorization } from "./current-tenant" import { IssueSeverity } from "./errors" // --------------------------------------------------------------------------- @@ -181,8 +181,8 @@ const InvestigationsListQuery = Schema.Struct({ }) // --------------------------------------------------------------------------- -// API group (user-facing; the internal `submit_diagnosis` write is a separate -// service-token-guarded router in apps/api, not part of this Clerk-authed group) +// API group (user-facing; diagnosis submission crosses the internal Worker RPC +// boundary and is not part of this Clerk-authenticated HTTP group) // --------------------------------------------------------------------------- export class InvestigationApiGroup extends HttpApiGroup.make("investigations") @@ -217,23 +217,3 @@ export class InvestigationApiGroup extends HttpApiGroup.make("investigations") ) .prefix("/api/investigations") .middleware(Authorization) {} - -// --------------------------------------------------------------------------- -// Internal API group (server-to-server). The chat-flue `submit_diagnosis` tool -// posts the structured report here once the investigate agent finishes its pass. -// Guarded by the internal-service token (not Clerk) via InternalServiceAuthorization; -// the framework handles param/payload decode (400), auth (401), and the declared -// error → status mapping (404/503) — no manual response wiring in the handler. -// --------------------------------------------------------------------------- - -export class InvestigationsInternalApiGroup extends HttpApiGroup.make("investigationsInternal") - .add( - HttpApiEndpoint.post("submitDiagnosis", "/:id/diagnosis", { - params: { id: InvestigationId }, - payload: SubmitDiagnosisRequest, - success: InvestigationDocument, - error: [InvestigationNotFoundError, InvestigationPersistenceError], - }), - ) - .prefix("/api/internal/investigations") - .middleware(InternalServiceAuthorization) {} diff --git a/packages/domain/src/index.ts b/packages/domain/src/index.ts index ae55d141b..942e860d3 100644 --- a/packages/domain/src/index.ts +++ b/packages/domain/src/index.ts @@ -1,4 +1,5 @@ export * from "./http" +export * from "./internal-rpc" export * from "./mcp-structured-types" export * from "./primitives" export * from "./query-engine" diff --git a/packages/domain/src/internal-rpc.ts b/packages/domain/src/internal-rpc.ts new file mode 100644 index 000000000..3cf8ec641 --- /dev/null +++ b/packages/domain/src/internal-rpc.ts @@ -0,0 +1,74 @@ +import type * as Effect from "effect/Effect" +import { Schema } from "effect" +import type { + InvestigationDocument, + InvestigationNotFoundError, + InvestigationPersistenceError, +} from "./http/investigations" +import { AiTriageResult } from "./http/ai-triage" +import { InvestigationId, OrgId } from "./primitives" + +const NonEmptyString = Schema.String.pipe(Schema.check(Schema.isMinLength(1), Schema.isTrimmed())) + +/** Runtime-validated arguments for an internal MCP tool call. */ +export class CallMcpToolRpcRequest extends Schema.Class("CallMcpToolRpcRequest")({ + orgId: OrgId, + name: NonEmptyString, + input: Schema.Unknown, +}) {} + +/** Runtime-validated structured diagnosis submitted by chat-flue. */ +export class SubmitDiagnosisRpcRequest extends Schema.Class( + "SubmitDiagnosisRpcRequest", +)({ + orgId: OrgId, + investigationId: InvestigationId, + report: AiTriageResult, +}) {} + +export interface InternalMcpToolDescriptor { + readonly name: string + readonly description: string + readonly inputSchema: Record +} + +export interface InternalMcpToolResult { + readonly content: ReadonlyArray<{ readonly type: "text"; readonly text: string }> + readonly isError?: boolean +} + +export class InternalRpcInvalidInputError extends Schema.TaggedErrorClass()( + "@maple/internal-rpc/InvalidInputError", + { + method: Schema.Literals(["callMcpTool", "submitDiagnosis"]), + message: Schema.String, + }, +) {} + +export class InternalRpcToolNotFoundError extends Schema.TaggedErrorClass()( + "@maple/internal-rpc/ToolNotFoundError", + { + name: Schema.String, + message: Schema.String, + }, +) {} + +/** + * Alchemy schemaless RPC shape exposed by the Maple API Worker. + * + * The method parameters intentionally remain `unknown`: Cloudflare RPC does + * structured cloning, not validation, so the implementation must decode each + * request with the schemas above before using it. + */ +export interface MapleApiRpcShape { + readonly listMcpTools: () => Effect.Effect> + readonly callMcpTool: ( + request: unknown, + ) => Effect.Effect + readonly submitDiagnosis: ( + request: unknown, + ) => Effect.Effect< + InvestigationDocument, + InternalRpcInvalidInputError | InvestigationNotFoundError | InvestigationPersistenceError + > +} From d1a12123e347a3093fedb8ff300054bd5445cfc6 Mon Sep 17 00:00:00 2001 From: Makisuo <31933546+Makisuo@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:47:54 +0200 Subject: [PATCH 4/7] Bootstrap shared production bindings and refine PlanetScale scrape detection (#210) * Bootstrap shared production bindings and tighten PlanetScale probes * fix * :3 --- alchemy.run.ts | 88 ++++- apps/alerting/alchemy.run.ts | 13 +- apps/api/alchemy.run.ts | 13 +- .../PlanetScaleConnectionService.test.ts | 100 +++++ .../services/PlanetScaleConnectionService.ts | 99 ++++- .../PlanetScaleDiscoveryService.test.ts | 40 +- .../services/PlanetScaleDiscoveryService.ts | 36 +- apps/api/src/services/ScrapeTargetsService.ts | 6 +- apps/chat-flue/alchemy.run.ts | 29 +- apps/scraper/src/ScrapeScheduler.test.ts | 52 ++- apps/scraper/src/ScrapeScheduler.ts | 62 ++- .../planetscale/planetscale-branch-table.tsx | 137 +++++++ .../planetscale-database-table.tsx | 255 +++++++++--- .../planetscale/planetscale-not-connected.tsx | 6 +- .../planetscale/planetscale-top-queries.tsx | 7 +- .../integrations/integration-catalog.tsx | 2 +- .../planetscale-integration-card.tsx | 51 +-- .../planetscale-metrics-health.tsx | 92 +++++ .../settings/scrape-targets-section.tsx | 372 +++--------------- .../src/routes/infra/planetscale/$dbName.tsx | 120 +++--- .../src/routes/infra/planetscale/index.tsx | 51 ++- 21 files changed, 1079 insertions(+), 552 deletions(-) create mode 100644 apps/web/src/components/infra/planetscale/planetscale-branch-table.tsx create mode 100644 apps/web/src/components/integrations/planetscale-metrics-health.tsx diff --git a/alchemy.run.ts b/alchemy.run.ts index 2dbbafed1..c9f3c6c02 100644 --- a/alchemy.run.ts +++ b/alchemy.run.ts @@ -1,6 +1,8 @@ import { appendFileSync } from "node:fs" import * as Alchemy from "alchemy" import * as Cloudflare from "alchemy/Cloudflare" +import * as Output from "alchemy/Output" +import * as RemovalPolicy from "alchemy/RemovalPolicy" import * as Effect from "effect/Effect" import { formatMapleStage, parseMapleStage, resolveMapleDomains } from "@maple/infra/cloudflare" import { createAlertingWorker } from "./apps/alerting/alchemy.run.ts" @@ -27,6 +29,70 @@ if (!process.env.CLOUDFLARE_ACCOUNT_ID && process.env.CLOUDFLARE_DEFAULT_ACCOUNT const resolveUrl = (domain: string | undefined, envKey: string, fallback = ""): string => domain ? `https://${domain}` : process.env[envKey]?.trim() || fallback +const requireEnv = (key: string): string => { + const value = process.env[key]?.trim() + if (!value) { + throw new Error(`Missing required deployment env: ${key}`) + } + return value +} + +const createProductionSharedResources = (stage: ReturnType) => + Effect.gen(function* () { + // Bootstrap these account/zone-wide resources in production first. Other + // stages keep their existing bindings until the production state has been + // deployed, after which they can safely resolve these logical IDs via ref(). + if (stage.kind !== "prd") { + return {} + } + + const emailRouting = yield* Cloudflare.Email.Routing("email-routing-maple-dev", { + zone: "maple.dev", + }).pipe(RemovalPolicy.retain()) + const sendingSubdomain = yield* Cloudflare.Email.SendingSubdomain("email-sending-noreply", { + zoneId: emailRouting.zoneId, + name: "noreply.maple.dev", + }).pipe(RemovalPolicy.retain()) + + const ingestEndpoint = (process.env.MAPLE_ENDPOINT?.trim() || "https://ingest.maple.dev").replace( + /\/+$/, + "", + ) + const headers = { authorization: `Bearer ${requireEnv("MAPLE_OTEL_INGEST_KEY")}` } + const tracesDestination = yield* Cloudflare.Workers.ObservabilityDestination( + "workers-observability-traces", + { + name: "maple-workers-traces", + url: `${ingestEndpoint}/v1/traces`, + headers, + logpushDataset: "opentelemetry-traces", + enabled: true, + }, + ).pipe(RemovalPolicy.retain()) + const logsDestination = yield* Cloudflare.Workers.ObservabilityDestination( + "workers-observability-logs", + { + name: "maple-workers-logs", + url: `${ingestEndpoint}/v1/logs`, + headers, + logpushDataset: "opentelemetry-logs", + enabled: true, + }, + ).pipe(RemovalPolicy.retain()) + + // The binding descriptor is lazy so enabling the sending subdomain is an + // explicit dependency of both Workers that send transactional email. + const emailBinding = sendingSubdomain.enabled.pipe( + Output.mapEffect(() => + Cloudflare.Email.SendEmail("email", { + allowedSenderAddresses: ["notifications@noreply.maple.dev"], + }), + ), + ) + + return { emailBinding, logsDestination, tracesDestination } + }) + export default Alchemy.Stack( "maple", { @@ -40,6 +106,7 @@ export default Alchemy.Stack( Effect.gen(function* () { const stage = parseMapleStage(yield* Alchemy.Stage) const domains = resolveMapleDomains(stage) + const shared = yield* createProductionSharedResources(stage) const apiUrl = resolveUrl(domains.api, "MAPLE_API_BASE_URL") const chatUrl = resolveUrl(domains.chat, "MAPLE_CHAT_BASE_URL") @@ -51,8 +118,18 @@ export default Alchemy.Stack( // Deploy the API RPC surface before switching chat-flue to it. The reverse // CHAT_FLUE binding is attached after chat deploys, which breaks the resource // cycle without an HTTP fallback or a placeholder Worker. - const { worker: api, db: mapleDb } = yield* createMapleApi({ stage, domains }) - const chatFlue = yield* createChatFlueWorker({ stage, domains, mapleApiRpc: api }) + const { worker: api, db: mapleDb } = yield* createMapleApi({ + stage, + domains, + emailBinding: shared.emailBinding, + }) + const chatFlue = yield* createChatFlueWorker({ + stage, + domains, + mapleApiRpc: api, + logsDestination: shared.logsDestination, + tracesDestination: shared.tracesDestination, + }) yield* api.bind("CHAT_FLUE", { bindings: [{ type: "service", name: "CHAT_FLUE", service: chatFlue.workerName }], }) @@ -74,7 +151,12 @@ export default Alchemy.Stack( const localUi = yield* createLocalUiWorker({ stage, domains }) - const alerting = yield* createAlertingWorker({ stage, domains, mapleDb }) + const alerting = yield* createAlertingWorker({ + stage, + domains, + mapleDb, + emailBinding: shared.emailBinding, + }) const summary = { stage: formatMapleStage(stage), diff --git a/apps/alerting/alchemy.run.ts b/apps/alerting/alchemy.run.ts index f9983d576..e9013bd93 100644 --- a/apps/alerting/alchemy.run.ts +++ b/apps/alerting/alchemy.run.ts @@ -1,4 +1,5 @@ import path from "node:path" +import type { Input } from "alchemy" import * as Cloudflare from "alchemy/Cloudflare" import * as Effect from "effect/Effect" import * as Redacted from "effect/Redacted" @@ -33,9 +34,11 @@ export interface CreateAlertingWorkerOptions { domains: MapleDomains /** Managed per-branch Hyperdrive from the api factory; undefined on ref stages (stg/prd). */ mapleDb: Cloudflare.Hyperdrive.Connection | undefined + /** Shared production binding derived from the retained sending subdomain. */ + emailBinding?: Input } -export const createAlertingWorker = ({ stage, mapleDb }: CreateAlertingWorkerOptions) => +export const createAlertingWorker = ({ stage, mapleDb, emailBinding }: CreateAlertingWorkerOptions) => Effect.gen(function* () { const hyperdriveRefId = resolveHyperdriveRefId(stage) // Cross-script binding to the AI triage Workflow hosted by the api worker — @@ -64,9 +67,11 @@ export const createAlertingWorker = ({ stage, mapleDb }: CreateAlertingWorkerOpt // Ref stages attach MAPLE_DB via worker.bind below. ...(mapleDb ? { MAPLE_DB: mapleDb } : {}), AI_TRIAGE_WORKFLOW: aiTriageWorkflow, - EMAIL: Cloudflare.Email.SendEmail("email", { - allowedSenderAddresses: ["notifications@noreply.maple.dev"], - }), + EMAIL: + emailBinding ?? + Cloudflare.Email.SendEmail("email", { + allowedSenderAddresses: ["notifications@noreply.maple.dev"], + }), TINYBIRD_HOST: requireEnv("TINYBIRD_HOST"), TINYBIRD_TOKEN: Redacted.make(requireEnv("TINYBIRD_TOKEN")), MAPLE_AUTH_MODE: process.env.MAPLE_AUTH_MODE?.trim() || "self_hosted", diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index 0e0bc7246..9afa5c934 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -1,4 +1,5 @@ import path from "node:path" +import type { Input } from "alchemy" import * as Cloudflare from "alchemy/Cloudflare" import type { Rpc } from "alchemy/Rpc" import * as Effect from "effect/Effect" @@ -34,12 +35,14 @@ const optionalSecret = (key: string): Record> export interface CreateMapleApiOptions { stage: MapleStage domains: MapleDomains + /** Shared production binding derived from the retained sending subdomain. */ + emailBinding?: Input } /** Alchemy resource type carried across the chat-flue service binding. */ export type MapleApiWorker = Cloudflare.Worker & Rpc -export const createMapleApi = ({ stage, domains }: CreateMapleApiOptions) => +export const createMapleApi = ({ stage, domains, emailBinding }: CreateMapleApiOptions) => Effect.gen(function* () { // MAPLE_DB Hyperdrive comes in two flavors: // @@ -137,9 +140,11 @@ export const createMapleApi = ({ stage, domains }: CreateMapleApiOptions) => VCS_SYNC_QUEUE: vcsSyncQueue, CLICKHOUSE_SCHEMA_APPLY_WORKFLOW: schemaApplyWorkflow, AI_TRIAGE_WORKFLOW: aiTriageWorkflow, - EMAIL: Cloudflare.Email.SendEmail("email", { - allowedSenderAddresses: ["notifications@noreply.maple.dev"], - }), + EMAIL: + emailBinding ?? + Cloudflare.Email.SendEmail("email", { + allowedSenderAddresses: ["notifications@noreply.maple.dev"], + }), TINYBIRD_HOST: requireEnv("TINYBIRD_HOST"), TINYBIRD_TOKEN: Redacted.make(requireEnv("TINYBIRD_TOKEN")), ...optionalPlain("CLICKHOUSE_URL"), diff --git a/apps/api/src/services/PlanetScaleConnectionService.test.ts b/apps/api/src/services/PlanetScaleConnectionService.test.ts index 075cae756..fbfbecf51 100644 --- a/apps/api/src/services/PlanetScaleConnectionService.test.ts +++ b/apps/api/src/services/PlanetScaleConnectionService.test.ts @@ -68,6 +68,17 @@ const stubPlanetScaleApi = (options?: { * unknown/bad service tokens) 403, only `token tok_good:*` passes. */ readonly denyBearerMetrics?: boolean + /** http_sd payload served by the control-plane SD endpoint (any auth). */ + readonly sdGroups?: ReadonlyArray<{ + targets: ReadonlyArray + labels?: Record + }> + /** + * Prod-faithful data-plane split: discovered metrics hosts (anything off + * api.planetscale.com) 403 everything but the service-token scheme — the + * behavior that made OAuth-auth'd scrapes fail while the SD probe passed. + */ + readonly denyDataPlaneBearer?: boolean }) => { const organizations = options?.organizations ?? [{ id: "psorg_1", name: "acme" }] const stub = (async (input: string | URL | Request, init?: RequestInit) => { @@ -92,6 +103,18 @@ const stubPlanetScaleApi = (options?: { if (denied) { return new Response("{}", { status: denied[1], headers: { "content-type": "application/json" } }) } + if (!requestUrl.includes("api.planetscale.com") && options?.denyDataPlaneBearer) { + const authorization = headers.get("authorization") ?? "" + return authorization.startsWith("token ") + ? new Response("up 1\n", { status: 200 }) + : new Response("forbidden", { status: 403 }) + } + if (options?.sdGroups && requestUrl.includes("api.planetscale.com") && requestUrl.includes("/metrics")) { + return new Response(JSON.stringify(options.sdGroups), { + status: 200, + headers: { "content-type": "application/json" }, + }) + } if (options?.denyBearerMetrics && requestUrl.includes("/metrics")) { const authorization = headers.get("authorization") ?? "" if (!authorization.startsWith("token tok_good:")) { @@ -313,6 +336,83 @@ describe("PlanetScaleConnectionService", () => { ) }) + it.effect("pauses metrics when the data plane rejects the bearer despite a passing SD probe", () => { + const testDb = createTestDb(trackedDbs) + const calls: Array<{ url: string; authorization: string | null }> = [] + // The prod bug: api.planetscale.com's SD endpoint 2xx'd the OAuth bearer, + // so the target auto-enabled — but the discovered metrics.psdb.cloud hosts + // only accept service tokens and 403'd every actual scrape. + const stub = stubPlanetScaleApi({ + calls, + denyDataPlaneBearer: true, + sdGroups: [ + { + targets: ["branch-1.metrics.psdb.cloud:443"], + labels: { __metrics_path__: "/metrics", planetscale_database_branch_id: "branch-1" }, + }, + ], + }) + + return Effect.gen(function* () { + const service = yield* PlanetScaleConnectionService + const orgId = asOrgId("org_1") + + yield* storeGrant(orgId) + const bound = yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + + // Binding succeeds, but scraping is paused until a service token arrives. + assert.isTrue(bound.connected) + assert.strictEqual(bound.metricsAuth, "missing") + assert.isFalse(bound.scrapeTarget!.enabled) + assert.isFalse(bound.detectedPermissions?.readMetricsEndpoints) + + // The verdict came from an actual data-plane scrape probe with the + // bearer. (Match on the host: fetch normalizes away the :443 port.) + assert.isTrue( + calls.some( + (call) => + call.url.includes("branch-1.metrics.psdb.cloud") && + call.authorization === "Bearer ps-access-token", + ), + ) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide(Layer.mergeAll(makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub))), + ) + }) + + it.effect("keeps oauth metrics enabled when the data-plane scrape accepts the bearer", () => { + const testDb = createTestDb(trackedDbs) + const calls: Array<{ url: string; authorization: string | null }> = [] + // Data-plane host answers 200 to the bearer (the stub's default) — the + // probe must not pause a working OAuth-auth'd target. + const stub = stubPlanetScaleApi({ + calls, + sdGroups: [ + { + targets: ["branch-1.metrics.psdb.cloud:443"], + labels: { __metrics_path__: "/metrics", planetscale_database_branch_id: "branch-1" }, + }, + ], + }) + + return Effect.gen(function* () { + const service = yield* PlanetScaleConnectionService + const orgId = asOrgId("org_1") + + yield* storeGrant(orgId) + const bound = yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + + assert.strictEqual(bound.metricsAuth, "oauth") + assert.isTrue(bound.scrapeTarget!.enabled) + assert.isTrue(bound.detectedPermissions?.readMetricsEndpoints) + assert.isTrue(calls.some((call) => call.url.includes("branch-1.metrics.psdb.cloud"))) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide(Layer.mergeAll(makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub))), + ) + }) + it.effect("classifies transient metrics probe failures as upstream errors", () => { const testDb = createTestDb(trackedDbs) const stub = stubPlanetScaleApi({ deny: { "/metrics": 503 } }) diff --git a/apps/api/src/services/PlanetScaleConnectionService.ts b/apps/api/src/services/PlanetScaleConnectionService.ts index fe2ccb0bd..f84570807 100644 --- a/apps/api/src/services/PlanetScaleConnectionService.ts +++ b/apps/api/src/services/PlanetScaleConnectionService.ts @@ -25,6 +25,7 @@ import { decryptAes256Gcm, encryptAes256Gcm, parseBase64Aes256GcmKey } from "../ import { Database } from "../lib/DatabaseLive" import { Env } from "../lib/Env" import { decodeDiscoveryConfig } from "./planetscale/discovery-config" +import { HttpSdResponse, subTargetsFromGroup } from "./PlanetScaleDiscoveryService" import { PlanetScaleOAuthService, planetScaleBearerHeader } from "./PlanetScaleOAuthService" import { ScrapeTargetsService } from "./ScrapeTargetsService" @@ -128,25 +129,25 @@ export class PlanetScaleConnectionService extends Context.Service< const apiBase = env.MAPLE_PLANETSCALE_API_BASE_URL.replace(/\/$/, "") /** - * GET a management-API path with the given Authorization header (an OAuth - * bearer or a service-token scheme). Returns the HTTP status; + * GET an absolute URL with the given Authorization header (an OAuth bearer + * or a service-token scheme). Returns the HTTP status and body text; * network-level failures surface as IntegrationsUpstreamError. */ - const probeStatus = Effect.fn("PlanetScaleConnectionService.probeStatus")(function* ( - path: string, + const probeUrl = Effect.fn("PlanetScaleConnectionService.probeUrl")(function* ( + url: string, authorization: string, ) { return yield* Effect.gen(function* () { - const request = HttpClientRequest.get(`${apiBase}${path}`).pipe( + const request = HttpClientRequest.get(url).pipe( HttpClientRequest.setHeaders({ Authorization: authorization, Accept: "application/json", }), ) const res = yield* httpClient.execute(request) - // Drain the body so the connection is released. - yield* res.text - return res.status + // Read (and thereby drain) the body so the connection is released. + const text = yield* res.text + return { status: res.status, text } }).pipe( Effect.mapError( (error) => @@ -166,6 +167,65 @@ export class PlanetScaleConnectionService extends Context.Service< ) }) + /** GET a management-API path; returns only the HTTP status. */ + const probeStatus = Effect.fn("PlanetScaleConnectionService.probeStatus")(function* ( + path: string, + authorization: string, + ) { + const response = yield* probeUrl(`${apiBase}${path}`, authorization) + return response.status + }) + + const decodeHttpSd = Schema.decodeUnknownEffect(Schema.fromJsonString(HttpSdResponse)) + + /** + * Whether the bearer can perform an ACTUAL branch-metrics scrape. The SD + * endpoint on api.planetscale.com accepting the bearer proves nothing + * about the data plane: the discovered hosts (metrics.psdb.cloud) only + * document service-token auth and reject OAuth bearers with 403. Probe one + * discovered endpoint with the bearer so `readMetricsEndpoints` reflects + * scraping, not listing. Inconclusive outcomes (no branches discovered + * yet, transport blip, undecodable payload) keep the control-plane answer + * instead of pausing a possibly-working target. + */ + const probeDataPlaneScrape = Effect.fn("PlanetScaleConnectionService.probeDataPlaneScrape")( + function* (organization: string, bearer: string) { + const org = encodeURIComponent(organization) + const outcome: "ok" | "rejected" | "inconclusive" = yield* Effect.gen(function* () { + const sd = yield* probeUrl(`${apiBase}/v1/organizations/${org}/metrics`, bearer) + if (sd.status === 401 || sd.status === 403) return "rejected" as const + if (sd.status < 200 || sd.status >= 300) return "inconclusive" as const + + const groups = yield* decodeHttpSd(sd.text).pipe( + Effect.catch(() => Effect.succeed(null)), + ) + if (groups === null) return "inconclusive" as const + + // subTargetsFromGroup already SSRF-validates the discovered URLs. + const first = groups.flatMap((group) => subTargetsFromGroup(group).ok)[0] + if (first === undefined) return "inconclusive" as const + + const scrape = yield* probeUrl(first.url, bearer) + return scrape.status >= 200 && scrape.status < 300 + ? ("ok" as const) + : ("rejected" as const) + }).pipe( + Effect.catchTag("@maple/http/errors/IntegrationsUpstreamError", (error) => + Effect.logWarning("PlanetScale data-plane scrape probe failed; keeping SD result").pipe( + Effect.annotateLogs({ organization, error: error.message }), + Effect.as("inconclusive" as const), + ), + ), + ) + if (outcome !== "ok") { + yield* Effect.logInfo("PlanetScale data-plane scrape probe outcome").pipe( + Effect.annotateLogs({ organization, outcome }), + ) + } + return outcome + }, + ) + const probePermissions = Effect.fn("PlanetScaleConnectionService.probePermissions")(function* ( organization: string, accessToken: string, @@ -181,9 +241,13 @@ export class PlanetScaleConnectionService extends Context.Service< { concurrency: 3 }, ) const ok = (status: number) => status >= 200 && status < 300 + // The SD listing passing is necessary but not sufficient for scraping — + // confirm against the data plane before declaring the bearer scrape-capable. + const readMetricsEndpoints = + ok(metricsStatus) && (yield* probeDataPlaneScrape(organization, bearer)) !== "rejected" const permissions: PlanetScaleDetectedPermissions = { readOrganization: ok(orgStatus), - readMetricsEndpoints: ok(metricsStatus), + readMetricsEndpoints, readDatabases: ok(databasesStatus), } return { permissions, orgStatus, metricsStatus } @@ -248,9 +312,9 @@ export class PlanetScaleConnectionService extends Context.Service< const discoveryConfig = target ? decodeDiscoveryConfig(target.discoveryConfigJson) : null // How scraping authenticates: a stored service token wins; grant-resolved // bearer auth counts only while the target is enabled (finalize disables - // it when the bearer probe failed — PlanetScale's metrics endpoints only - // document service-token auth); anything else means scraping is paused - // until a token is added. + // it unless the bearer passed an end-to-end data-plane scrape probe — + // PlanetScale's metrics endpoints only document service-token auth); + // anything else means scraping is paused until a token is added. const metricsAuth = target === null ? ("missing" as const) @@ -367,11 +431,12 @@ export class PlanetScaleConnectionService extends Context.Service< ) } - // Probe what the grant can do against this org. The metrics endpoints - // only document service-token auth, so a failing bearer probe does NOT - // block the binding — inventory/insights/webhooks work on the grant, and - // scraping stays paused until a service token is added via - // setMetricsToken (the card's follow-up step). + // Probe what the grant can do against this org (readMetricsEndpoints + // requires an actual data-plane scrape to pass — see probePermissions). + // The metrics endpoints only document service-token auth, so a failing + // bearer probe does NOT block the binding — inventory/insights/webhooks + // work on the grant, and scraping stays paused until a service token is + // added via setMetricsToken (the card's follow-up step). const { permissions } = yield* probePermissions(organization, accessToken) // Attribution comes from the grant, so finalize behaves identically when diff --git a/apps/api/src/services/PlanetScaleDiscoveryService.test.ts b/apps/api/src/services/PlanetScaleDiscoveryService.test.ts index dc7659b88..c0a41e15a 100644 --- a/apps/api/src/services/PlanetScaleDiscoveryService.test.ts +++ b/apps/api/src/services/PlanetScaleDiscoveryService.test.ts @@ -168,7 +168,7 @@ describe("PlanetScaleDiscoveryService", () => { // Prod hazard: an http_sd payload with several groups that carry no // `planetscale_database_branch_id`, so subTargetKey falls back to the - // shared host. Without dedup these become N rows with the SAME + // shared host+path. Without dedup these become N rows with the SAME // (id, subTargetKey) and the scraper forks a leaking loop fiber per row. const DUP_HOST_PAYLOAD = [ { targets: ["metrics.psdb.cloud:443"], labels: { planetscale_database: "mydb" } }, @@ -184,11 +184,47 @@ describe("PlanetScaleDiscoveryService", () => { ) assert.strictEqual(entries.length, 1) - assert.strictEqual(entries[0]?.subTargetKey, "metrics.psdb.cloud:443") + assert.strictEqual(entries[0]?.subTargetKey, "metrics.psdb.cloud:443/metrics") assert.strictEqual(entries[0]?.url, "https://metrics.psdb.cloud:443/metrics") }).pipe(Effect.provide(makeLayer(testDb))) }) + it.effect("keeps branch-id-less groups distinct when they differ only by metrics path", () => { + const testDb = createTestDb(trackedDbs) + const recorded: Array = [] + return Effect.gen(function* () { + const discovery = yield* PlanetScaleDiscoveryService + const row = yield* createPlanetScaleTargetRow("my-org") + + // Same host, distinct `__metrics_path__` per group, no branch-id label — + // a bare-host fallback key would collapse these to ONE endpoint and + // silently drop the other's metrics. + const PATH_ONLY_PAYLOAD = [ + { targets: ["metrics.psdb.cloud:443"], labels: { __metrics_path__: "/metrics/db-a" } }, + { targets: ["metrics.psdb.cloud:443"], labels: { __metrics_path__: "/metrics/db-b" } }, + ] + + const entries = yield* discovery.discover(row).pipe( + Effect.provideService( + FetchHttpClient.Fetch, + stubFetch(recorded, () => Response.json(PATH_ONLY_PAYLOAD)), + ), + ) + + assert.deepStrictEqual( + entries.map((entry) => entry.subTargetKey), + ["metrics.psdb.cloud:443/metrics/db-a", "metrics.psdb.cloud:443/metrics/db-b"], + ) + assert.deepStrictEqual( + entries.map((entry) => entry.url), + [ + "https://metrics.psdb.cloud:443/metrics/db-a", + "https://metrics.psdb.cloud:443/metrics/db-b", + ], + ) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + it.effect("caches discovery for the TTL and refreshes after it elapses", () => { const testDb = createTestDb(trackedDbs) const recorded: Array = [] diff --git a/apps/api/src/services/PlanetScaleDiscoveryService.ts b/apps/api/src/services/PlanetScaleDiscoveryService.ts index b936a9064..112a3c315 100644 --- a/apps/api/src/services/PlanetScaleDiscoveryService.ts +++ b/apps/api/src/services/PlanetScaleDiscoveryService.ts @@ -34,13 +34,13 @@ type ScrapeTargetRow = typeof scrapeTargets.$inferSelect export interface PlanetScaleSubTarget { /** Concrete per-branch scrape URL (`https://{host}{__metrics_path__}`). */ readonly url: string - /** Stable discriminator: `planetscale_database_branch_id` SD label, falling back to `host:port`. */ + /** Stable discriminator: `planetscale_database_branch_id` SD label, falling back to `host:port` + metrics path. */ readonly subTargetKey: string /** SD labels minus `__`-prefixed Prometheus meta labels. */ readonly labels: Record } -const HttpSdResponse = Schema.Array( +export const HttpSdResponse = Schema.Array( Schema.Struct({ targets: Schema.Array(Schema.String), labels: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)), @@ -74,7 +74,7 @@ const toUpstreamError = (message: string, status?: number) => new ScrapeTargetUpstreamError({ message, ...(status === undefined ? {} : { status }) }) /** Convert one http_sd group into sub-targets, dropping SSRF-invalid hosts. */ -const subTargetsFromGroup = (group: { +export const subTargetsFromGroup = (group: { readonly targets: ReadonlyArray readonly labels?: Record | undefined }): { readonly ok: Array; readonly dropped: Array } => { @@ -97,12 +97,15 @@ const subTargetsFromGroup = (group: { dropped.push(url) continue } + // The no-branch-id fallback keys on host + path (not bare host): groups + // that differ only by `__metrics_path__` are distinct endpoints, and a + // host-only key would silently collapse them away in dedupe. const subTargetKey = branchId && group.targets.length === 1 ? branchId : branchId ? `${branchId}:${hostPort}` - : hostPort + : `${hostPort}${path}` ok.push({ url, subTargetKey, labels }) } return { ok, dropped } @@ -112,9 +115,10 @@ const subTargetsFromGroup = (group: { * Collapse sub-targets sharing a `subTargetKey` (last wins). The scraper keys * one scrape-loop fiber per `(targetId, subTargetKey)`, so duplicate keys would * each fork a fiber that the scheduler can't track — a runaway scrape loop. - * Two entries with the same key resolve to the same logical endpoint, so - * collapsing them is lossless. Happens when an http_sd payload exposes several - * groups that fall back to the same host key (no `planetscale_database_branch_id`). + * Two entries with the same key resolve to the same scrape URL (the fallback + * key is host + path), so collapsing them is lossless. Happens when an http_sd + * payload exposes several groups that fall back to the same host+path key + * (no `planetscale_database_branch_id`). */ const dedupeBySubTargetKey = ( entries: ReadonlyArray, @@ -277,6 +281,24 @@ export class PlanetScaleDiscoveryService extends Context.Service< collected.push(...converted.ok) dropped.push(...converted.dropped) } + + // Track label drift: PlanetScale documents `planetscale_database_branch_id` + // on every group, and its absence forces the host+path fallback key (losing + // per-branch attribution). Log the label keys actually seen so a rename on + // their side is diagnosable from prod logs. + const missingBranchLabel = groups.filter( + (group) => (group.labels ?? {}).planetscale_database_branch_id === undefined, + ) + if (missingBranchLabel.length > 0) { + yield* Effect.logInfo("PlanetScale http_sd groups missing the branch-id label").pipe( + Effect.annotateLogs({ + scrapeTargetId: row.id, + groupsMissingLabel: missingBranchLabel.length, + groupsTotal: groups.length, + observedLabelKeys: Object.keys(missingBranchLabel[0]?.labels ?? {}).join(", "), + }), + ) + } if (dropped.length > 0) { yield* Effect.logWarning( "Dropped PlanetScale discovered targets failing URL validation", diff --git a/apps/api/src/services/ScrapeTargetsService.ts b/apps/api/src/services/ScrapeTargetsService.ts index c806abcfc..1853d4630 100644 --- a/apps/api/src/services/ScrapeTargetsService.ts +++ b/apps/api/src/services/ScrapeTargetsService.ts @@ -845,8 +845,10 @@ export class ScrapeTargetsService extends Context.Service entry.subTargetKey === subTargetKey) if (!match) { diff --git a/apps/chat-flue/alchemy.run.ts b/apps/chat-flue/alchemy.run.ts index 5d9f00d48..e0ddcffa2 100644 --- a/apps/chat-flue/alchemy.run.ts +++ b/apps/chat-flue/alchemy.run.ts @@ -35,6 +35,10 @@ export interface CreateChatFlueWorkerOptions { domains: MapleDomains /** API worker deployed first, then bound privately for schemaless RPC. */ mapleApiRpc: MapleApiWorker + /** Production-owned OTLP log destination; non-production temporarily uses the legacy slug. */ + logsDestination?: Cloudflare.Workers.ObservabilityDestination + /** Production-owned OTLP trace destination; non-production temporarily uses the legacy slug. */ + tracesDestination?: Cloudflare.Workers.ObservabilityDestination } /** @@ -54,7 +58,13 @@ export interface CreateChatFlueWorkerOptions { * Manual fallback (Flue-native): `cd apps/chat-flue && bun run build && * wrangler deploy --config dist/maple_chat_flue/wrangler.json`. */ -export const createChatFlueWorker = ({ stage, domains, mapleApiRpc }: CreateChatFlueWorkerOptions) => +export const createChatFlueWorker = ({ + stage, + domains, + mapleApiRpc, + logsDestination, + tracesDestination, +}: CreateChatFlueWorkerOptions) => Effect.gen(function* () { // Flue generates the Worker entrypoint + DO classes; build before deploy. const build = yield* Command.Build("chat-flue-build", { @@ -88,15 +98,18 @@ export const createChatFlueWorker = ({ stage, domains, mapleApiRpc }: CreateChat compatibility: { date: "2026-06-01", flags: ["nodejs_compat"] }, placement: CLOUDFLARE_WORKER_PLACEMENT, // Workers Observability. `traces.enabled` is required for the `tracing.enterSpan` - // custom spans in src/agents/maple-chat.ts to emit; the `"maple"` destination - // forwards both logs and traces over Cloudflare's native pipeline → Maple ingest - // (the same path the existing auto-spans use, unlike the @flue/opentelemetry - // export which doesn't flush reliably from DO isolates). `"maple"` is the - // account-level telemetry destination id. + // custom spans in src/agents/maple-chat.ts to emit. Production uses one + // Alchemy-owned destination per signal; non-production temporarily keeps + // the legacy account-level `"maple"` destination until the production + // resources have been bootstrapped and can be referenced from prd state. observability: { enabled: true, - logs: { enabled: true, invocationLogs: true, destinations: ["maple"] }, - traces: { enabled: true, destinations: ["maple"] }, + logs: { + enabled: true, + invocationLogs: true, + destinations: [logsDestination?.slug ?? "maple"], + }, + traces: { enabled: true, destinations: [tracesDestination?.slug ?? "maple"] }, }, url: true, domain: domains.chat, diff --git a/apps/scraper/src/ScrapeScheduler.test.ts b/apps/scraper/src/ScrapeScheduler.test.ts index 632f9770b..7ded4af3d 100644 --- a/apps/scraper/src/ScrapeScheduler.test.ts +++ b/apps/scraper/src/ScrapeScheduler.test.ts @@ -451,6 +451,24 @@ describe("ScrapeScheduler", () => { }), ) + it.effect("backs off a target whose credential is rejected (403) instead of hammering it", () => + Effect.gen(function* () { + // Prod scenario: PlanetScale's metrics.psdb.cloud rejects an OAuth + // bearer with 403 on every scrape — the loop must escalate its delay + // exactly like a rate limit, not retry every interval forever. + const harness = makeHarness([mkTarget(TARGET_A, 10)]) + harness.scrapeImpl = () => Effect.succeed(proxyResponse({ status: 403, body: "forbidden" })) + yield* startScheduler.pipe(Effect.provide(harnessLayer(harness))) + + yield* TestClock.adjust(Duration.seconds(60)) + + // Exponential backoff off a 10s base: scrapes at t=0, 10, 30 (next is + // t=70, outside the window). A fixed interval would have fired 7 times. + assert.strictEqual(harness.scrapeCalls.length, 3) + assert.include(harness.reportedResults[0]?.error ?? "", "HTTP 403") + }), + ) + it.effect("honors a longer Retry-After before the next scrape", () => Effect.gen(function* () { const harness = makeHarness([mkTarget(TARGET_A, 10)]) @@ -588,40 +606,58 @@ describe("sendResultsInChunks", () => { }) describe("nextScrapeDelayMs", () => { - const ok: ScrapeOutcome = { error: null, rateLimited: false, retryAfterMs: null } + const ok: ScrapeOutcome = { error: null, rateLimited: false, authFailed: false, retryAfterMs: null } const limited = (retryAfterMs: number | null = null): ScrapeOutcome => ({ error: "target returned HTTP 429", rateLimited: true, + authFailed: false, retryAfterMs, }) + const authRejected: ScrapeOutcome = { + error: "target returned HTTP 403", + rateLimited: false, + authFailed: true, + retryAfterMs: null, + } it("holds the base interval on a healthy scrape, ignoring the counter", () => { - assert.strictEqual(nextScrapeDelayMs({ baseMs: 5_000, outcome: ok, consecutiveRateLimits: 3 }), 5_000) + assert.strictEqual(nextScrapeDelayMs({ baseMs: 5_000, outcome: ok, consecutiveBackoffs: 3 }), 5_000) }) it("escalates exponentially while rate-limited", () => { - assert.strictEqual(nextScrapeDelayMs({ baseMs: 10_000, outcome: limited(), consecutiveRateLimits: 0 }), 10_000) - assert.strictEqual(nextScrapeDelayMs({ baseMs: 10_000, outcome: limited(), consecutiveRateLimits: 1 }), 20_000) - assert.strictEqual(nextScrapeDelayMs({ baseMs: 10_000, outcome: limited(), consecutiveRateLimits: 3 }), 80_000) + assert.strictEqual(nextScrapeDelayMs({ baseMs: 10_000, outcome: limited(), consecutiveBackoffs: 0 }), 10_000) + assert.strictEqual(nextScrapeDelayMs({ baseMs: 10_000, outcome: limited(), consecutiveBackoffs: 1 }), 20_000) + assert.strictEqual(nextScrapeDelayMs({ baseMs: 10_000, outcome: limited(), consecutiveBackoffs: 3 }), 80_000) + }) + + it("escalates exponentially on a rejected credential (401/403) too", () => { + assert.strictEqual( + nextScrapeDelayMs({ baseMs: 10_000, outcome: authRejected, consecutiveBackoffs: 1 }), + 20_000, + ) + assert.strictEqual( + nextScrapeDelayMs({ baseMs: 60_000, outcome: authRejected, consecutiveBackoffs: 5 }), + Duration.toMillis(Duration.minutes(5)), + ) }) it("caps the backoff at 5 minutes", () => { assert.strictEqual( - nextScrapeDelayMs({ baseMs: 60_000, outcome: limited(), consecutiveRateLimits: 5 }), + nextScrapeDelayMs({ baseMs: 60_000, outcome: limited(), consecutiveBackoffs: 5 }), Duration.toMillis(Duration.minutes(5)), ) }) it("honors Retry-After when it exceeds the exponential backoff", () => { assert.strictEqual( - nextScrapeDelayMs({ baseMs: 10_000, outcome: limited(120_000), consecutiveRateLimits: 0 }), + nextScrapeDelayMs({ baseMs: 10_000, outcome: limited(120_000), consecutiveBackoffs: 0 }), 120_000, ) }) it("prefers the exponential backoff when Retry-After is shorter", () => { assert.strictEqual( - nextScrapeDelayMs({ baseMs: 10_000, outcome: limited(5_000), consecutiveRateLimits: 2 }), + nextScrapeDelayMs({ baseMs: 10_000, outcome: limited(5_000), consecutiveBackoffs: 2 }), 40_000, ) }) diff --git a/apps/scraper/src/ScrapeScheduler.ts b/apps/scraper/src/ScrapeScheduler.ts index 38897eab1..d1d84cee4 100644 --- a/apps/scraper/src/ScrapeScheduler.ts +++ b/apps/scraper/src/ScrapeScheduler.ts @@ -41,31 +41,44 @@ export interface ScrapeOutcome { readonly samplesPostMetricRelabeling?: number /** Upstream signalled a rate limit (HTTP 429/503) — back off before retrying. */ readonly rateLimited: boolean + /** + * Upstream rejected the credential (HTTP 401/403) — back off like a rate + * limit: the failure won't clear until the org's auth is fixed, so retrying + * every interval just hammers the target (prod hit this with PlanetScale + * rejecting OAuth bearers on metrics.psdb.cloud every 60s). + */ + readonly authFailed: boolean /** Upstream `Retry-After` translated to ms, when present. */ readonly retryAfterMs: number | null } +/** A scrape outcome that must escalate the delay instead of holding cadence. */ +export const shouldBackOff = (outcome: ScrapeOutcome): boolean => + outcome.rateLimited || outcome.authFailed + /** * The target period before a target's next scrape. The happy path returns the * configured interval; the caller ({@link ScrapeScheduler}'s target loop) * subtracts the scrape's own elapsed time so the happy-path cadence stays - * start-to-start. A rate-limited scrape escalates exponentially — honoring - * `Retry-After` when it is longer — capped at {@link MAX_BACKOFF_MS} so the - * target keeps probing for recovery; that delay runs from scrape end. + * start-to-start. A rate-limited or auth-rejected scrape escalates + * exponentially — honoring `Retry-After` when it is longer — capped at + * {@link MAX_BACKOFF_MS} so the target keeps probing for recovery (an auth fix + * needs no restart: the credential is resolved server-side per scrape); that + * delay runs from scrape end. */ export const nextScrapeDelayMs = ({ baseMs, outcome, - consecutiveRateLimits, + consecutiveBackoffs, }: { readonly baseMs: number readonly outcome: ScrapeOutcome - readonly consecutiveRateLimits: number + readonly consecutiveBackoffs: number }): number => { - if (!outcome.rateLimited) return baseMs - // exponential is always >= baseMs (consecutiveRateLimits >= 0), so baseMs + if (!shouldBackOff(outcome)) return baseMs + // exponential is always >= baseMs (consecutiveBackoffs >= 0), so baseMs // never needs to be a floor here. - const exponential = baseMs * 2 ** consecutiveRateLimits + const exponential = baseMs * 2 ** consecutiveBackoffs const retryAfter = outcome.retryAfterMs ?? 0 return Math.min(MAX_BACKOFF_MS, Math.max(exponential, retryAfter)) } @@ -191,10 +204,12 @@ export class ScrapeScheduler extends Context.Service= 300) { // A non-2xx is a recorded failure, not an Effect error: only - // 429/503 trigger backoff, and we need the Retry-After hint. + // 429/503 (rate limit) and 401/403 (rejected credential) + // trigger backoff, and we need the Retry-After hint. return { error: `target returned HTTP ${response.status}`, rateLimited: response.status === 429 || response.status === 503, + authFailed: response.status === 401 || response.status === 403, retryAfterMs: response.retryAfterSeconds !== null ? response.retryAfterSeconds * 1000 @@ -240,6 +255,7 @@ export class ScrapeScheduler extends Context.Service({ error: error.message, rateLimited: false, + authFailed: false, retryAfterMs: null, }), ), @@ -254,6 +271,7 @@ export class ScrapeScheduler extends Context.Service({ error: Cause.pretty(Cause.die(defect)), rateLimited: false, + authFailed: false, retryAfterMs: null, }), ), @@ -285,35 +303,41 @@ export class ScrapeScheduler extends Context.Service { const baseMs = target.scrapeIntervalSeconds * 1000 - const loop = (consecutiveRateLimits: number): Effect.Effect => + const loop = (consecutiveBackoffs: number): Effect.Effect => Effect.gen(function* () { const startedAt = yield* Clock.currentTimeMillis const outcome = yield* scrapeOnce(target) const elapsedMs = (yield* Clock.currentTimeMillis) - startedAt - const delayMs = nextScrapeDelayMs({ baseMs, outcome, consecutiveRateLimits }) - if (outcome.rateLimited) { - yield* Effect.logWarning("Scrape rate-limited, backing off").pipe( + const backingOff = shouldBackOff(outcome) + const delayMs = nextScrapeDelayMs({ baseMs, outcome, consecutiveBackoffs }) + if (backingOff) { + yield* Effect.logWarning( + outcome.authFailed + ? "Scrape auth rejected, backing off" + : "Scrape rate-limited, backing off", + ).pipe( Effect.annotateLogs({ targetId: target.id, orgId: target.orgId, ...(target.subTargetKey ? { subTargetKey: target.subTargetKey } : {}), delayMs, retryAfterMs: outcome.retryAfterMs, - consecutiveRateLimits: consecutiveRateLimits + 1, + consecutiveBackoffs: consecutiveBackoffs + 1, }), ) } // Happy path: subtract the scrape's own elapsed time so cadence // stays start-to-start (matching the old Schedule.fixed). Backoff // runs the full delay from scrape end so Retry-After is honored. - const sleepMs = outcome.rateLimited ? delayMs : Math.max(0, delayMs - elapsedMs) + const sleepMs = backingOff ? delayMs : Math.max(0, delayMs - elapsedMs) yield* Effect.sleep(Duration.millis(sleepMs)) - return yield* loop(outcome.rateLimited ? consecutiveRateLimits + 1 : 0) + return yield* loop(backingOff ? consecutiveBackoffs + 1 : 0) }) // Plain targets have nothing to de-sync against; only stagger the // branches of a discovered (PlanetScale) target so they spread across diff --git a/apps/web/src/components/infra/planetscale/planetscale-branch-table.tsx b/apps/web/src/components/infra/planetscale/planetscale-branch-table.tsx new file mode 100644 index 000000000..3e1f91f97 --- /dev/null +++ b/apps/web/src/components/infra/planetscale/planetscale-branch-table.tsx @@ -0,0 +1,137 @@ +import { Badge } from "@maple/ui/components/ui/badge" +import { cn } from "@maple/ui/lib/utils" + +import type { PlanetScaleBranchStat } from "@/api/warehouse/service-map" +import { formatNumber } from "@/lib/format" +import { ColumnHead, TableShell, useTableSort } from "../primitives/data-table" +import { formatLag, lagClass, utilizationClass } from "./planetscale-database-table" + +type SortKey = "branch" | "connectionsAvg" | "cpuMaxPercent" | "memMaxPercent" | "replicaLagMaxSeconds" + +/** + * Per-branch health for one database. Branch flags (production/ready) come from + * the polled inventory; the metric columns from the window's rollups. + */ +export function PlanetScaleBranchTable({ + branches, + branchInfoByName, + waiting, +}: { + branches: ReadonlyArray + branchInfoByName: ReadonlyMap + waiting?: boolean +}) { + const { sorted, sortKey, sortDir, handleSort } = useTableSort( + branches, + { initialKey: "connectionsAvg", stringKeys: ["branch"] }, + ) + + return ( + + + label="Branch" + sortKey="branch" + currentKey={sortKey} + dir={sortDir} + onSort={handleSort} + width="flex-1 min-w-[200px]" + /> + + label="Connections" + sortKey="connectionsAvg" + currentKey={sortKey} + dir={sortDir} + onSort={handleSort} + align="right" + width="w-[96px]" + /> + + label="CPU (max)" + sortKey="cpuMaxPercent" + currentKey={sortKey} + dir={sortDir} + onSort={handleSort} + align="right" + width="w-[88px]" + /> + + label="Memory (max)" + sortKey="memMaxPercent" + currentKey={sortKey} + dir={sortDir} + onSort={handleSort} + align="right" + width="w-[104px]" + hidden="hidden md:flex" + /> + + label="Replica lag" + sortKey="replicaLagMaxSeconds" + currentKey={sortKey} + dir={sortDir} + onSort={handleSort} + align="right" + width="w-[88px]" + /> + + } + > + {sorted.map((row) => { + const info = branchInfoByName.get(row.branch) + return ( +
+
+ {row.branch} + {info?.production ? ( + + production + + ) : null} + {info !== undefined && !info.ready ? ( + + provisioning + + ) : null} +
+
+ {formatNumber(row.connectionsAvg)} +
+
+ {row.cpuMaxPercent.toFixed(0)}% +
+
+ {row.memMaxPercent.toFixed(0)}% +
+
+ {formatLag(row.replicaLagMaxSeconds)} +
+
+ ) + })} + + ) +} diff --git a/apps/web/src/components/infra/planetscale/planetscale-database-table.tsx b/apps/web/src/components/infra/planetscale/planetscale-database-table.tsx index f70fa0519..eddd565fd 100644 --- a/apps/web/src/components/infra/planetscale/planetscale-database-table.tsx +++ b/apps/web/src/components/infra/planetscale/planetscale-database-table.tsx @@ -7,101 +7,250 @@ import { cn } from "@maple/ui/lib/utils" import type { PlanetScaleDatabaseSummary } from "@maple/domain/http" import type { PlanetScaleDatabaseStat } from "@/api/warehouse/service-map" import { formatNumber } from "@/lib/format" +import { + ColumnHead, + MetaChip, + ROW_LINK_CLASS, + TableShell, + TableSkeleton, + useTableSort, +} from "../primitives/data-table" -export function PlanetScaleDatabaseTableLoading() { - return -} - -const formatLag = (seconds: number) => +export const formatLag = (seconds: number) => seconds >= 1 ? `${seconds.toFixed(1)}s` : `${Math.round(seconds * 1000)}ms` -function utilizationClass(percent: number): string | undefined { +export function utilizationClass(percent: number): string | undefined { if (percent > 80) return "text-severity-error" if (percent > 60) return "text-severity-warn" return undefined } -function lagClass(seconds: number): string | undefined { +export function lagClass(seconds: number): string | undefined { if (seconds > 10) return "text-severity-error" if (seconds > 1) return "text-severity-warn" return undefined } +/** States that indicate the database isn't serving normally — worth a badge. */ +export function abnormalState(state: string | null): string | null { + if (state === null) return null + const normalized = state.toLowerCase() + return normalized === "ready" || normalized === "active" ? null : normalized +} + +type SortKey = + | "name" + | "branchCount" + | "connectionsAvg" + | "cpuMaxPercent" + | "memMaxPercent" + | "replicaLagMaxSeconds" + +interface DatabaseRow { + id: string + name: string + kind: string + region: string | null + plan: string | null + state: string | null + branchCount: number + hasStats: boolean + connectionsAvg: number + cpuMaxPercent: number + memMaxPercent: number + replicaLagMaxSeconds: number +} + +// Databases with no metrics in the window sort below every real value. +const MISSING = Number.NEGATIVE_INFINITY + +const headerCells = (sort?: { + sortKey: SortKey + sortDir: "asc" | "desc" + handleSort: (k: SortKey) => void +}) => ( + <> + + label="Database" + sortKey={sort ? "name" : undefined} + currentKey={sort?.sortKey} + dir={sort?.sortDir} + onSort={sort?.handleSort} + width="flex-1 min-w-[220px]" + /> + + label="Branches" + sortKey={sort ? "branchCount" : undefined} + currentKey={sort?.sortKey} + dir={sort?.sortDir} + onSort={sort?.handleSort} + align="right" + width="w-[80px]" + hidden="hidden md:flex" + /> + + label="Connections" + sortKey={sort ? "connectionsAvg" : undefined} + currentKey={sort?.sortKey} + dir={sort?.sortDir} + onSort={sort?.handleSort} + align="right" + width="w-[96px]" + /> + + label="CPU (max)" + sortKey={sort ? "cpuMaxPercent" : undefined} + currentKey={sort?.sortKey} + dir={sort?.sortDir} + onSort={sort?.handleSort} + align="right" + width="w-[88px]" + /> + + label="Memory (max)" + sortKey={sort ? "memMaxPercent" : undefined} + currentKey={sort?.sortKey} + dir={sort?.sortDir} + onSort={sort?.handleSort} + align="right" + width="w-[104px]" + hidden="hidden md:flex" + /> + + label="Replica lag" + sortKey={sort ? "replicaLagMaxSeconds" : undefined} + currentKey={sort?.sortKey} + dir={sort?.sortDir} + onSort={sort?.handleSort} + align="right" + width="w-[88px]" + /> + +) + +export function PlanetScaleDatabaseTableLoading() { + return ( + ( + <> +
+ +
+ + + + + + + )} + /> + ) +} + /** * Fleet table: one row per database from the polled inventory, joined with the - * window's scraped-metric rollups. Databases with no metrics in the window - * (excluded branches, asleep) still render with muted dashes. + * window's metric rollups. Databases with no metrics in the window (excluded + * branches, asleep) still render with muted dashes and sort last. */ export function PlanetScaleDatabaseTable({ databases, statsByName, + waiting, }: { databases: ReadonlyArray statsByName: ReadonlyMap + waiting?: boolean }) { + const rows: DatabaseRow[] = databases.map((db) => { + const stats = statsByName.get(db.name.toLowerCase()) + return { + id: db.id, + name: db.name, + kind: db.kind, + region: db.region, + plan: db.plan, + state: db.state, + branchCount: db.branches.length, + hasStats: stats !== undefined, + connectionsAvg: stats?.connectionsAvg ?? MISSING, + cpuMaxPercent: stats?.cpuMaxPercent ?? MISSING, + memMaxPercent: stats?.memMaxPercent ?? MISSING, + replicaLagMaxSeconds: stats?.replicaLagMaxSeconds ?? MISSING, + } + }) + + const { sorted, sortKey, sortDir, handleSort } = useTableSort(rows, { + initialKey: "connectionsAvg", + stringKeys: ["name"], + }) + return ( -
-
- Database - Branches - Connections - CPU (max) - Memory (max) - Replica lag -
- {databases.map((db) => { - const stats = statsByName.get(db.name.toLowerCase()) + + {sorted.map((row) => { + const state = abnormalState(row.state) return ( - - {db.name} +
+ + {row.name} + - {db.kind === "postgresql" ? "Postgres" : "MySQL"} + {row.kind === "postgresql" ? "Postgres" : "MySQL"} - {db.region ? ( - - {db.region} - + {state !== null ? ( + + {state} + ) : null} - - - {db.branches.length} - - - {stats ? formatNumber(stats.connectionsAvg) : "—"} - - {row.region} : null} + {row.plan ? {row.plan} : null} +
+
+ {row.branchCount} +
+
+ {row.hasStats ? formatNumber(row.connectionsAvg) : "—"} +
+
- {stats ? `${stats.cpuMaxPercent.toFixed(0)}%` : "—"} - - +
- {stats ? `${stats.memMaxPercent.toFixed(0)}%` : "—"} - - +
- {stats ? formatLag(stats.replicaLagMaxSeconds) : "—"} - + {row.hasStats ? formatLag(row.replicaLagMaxSeconds) : "—"} +
) })} -
+ ) } diff --git a/apps/web/src/components/infra/planetscale/planetscale-not-connected.tsx b/apps/web/src/components/infra/planetscale/planetscale-not-connected.tsx index df189f06d..103a42522 100644 --- a/apps/web/src/components/infra/planetscale/planetscale-not-connected.tsx +++ b/apps/web/src/components/infra/planetscale/planetscale-not-connected.tsx @@ -21,9 +21,9 @@ export function PlanetScaleNotConnected() { Connect PlanetScale to see database health - Authorize your PlanetScale organization with one click and Maple continuously scrapes - every branch's metrics — connections, CPU, memory, replication lag — with no agent to - run. + Authorize your PlanetScale organization with one click and Maple tracks every + branch's health — connections, CPU, memory, replication lag — with nothing to + install. diff --git a/apps/web/src/components/infra/planetscale/planetscale-top-queries.tsx b/apps/web/src/components/infra/planetscale/planetscale-top-queries.tsx index bc128d285..6b0cada7d 100644 --- a/apps/web/src/components/infra/planetscale/planetscale-top-queries.tsx +++ b/apps/web/src/components/infra/planetscale/planetscale-top-queries.tsx @@ -6,7 +6,7 @@ import { cn } from "@maple/ui/lib/utils" import { Result } from "@/lib/effect-atom" import { useRefreshableAtomValue } from "@/hooks/use-refreshable-atom-value" import { planetscaleQueryInsightsResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" -import { formatLatency, formatNumber } from "@/lib/format" +import { formatLatency, formatNumber, formatRelativeTime } from "@/lib/format" /** Warehouse "YYYY-MM-DD HH:mm:ss" → epoch ms (values are UTC). */ const warehouseTimeToMs = (value: string): number => new Date(`${value.replace(" ", "T")}Z`).getTime() @@ -103,6 +103,11 @@ export function PlanetScaleTopQueries({ {formatNumber(row.rowsReadPerQuery)} rows read/query {row.statementType ? {row.statementType} : null} + {row.lastRunAt !== null ? ( + + last run {formatRelativeTime(new Date(row.lastRunAt).toISOString())} + + ) : null}
))} diff --git a/apps/web/src/components/integrations/integration-catalog.tsx b/apps/web/src/components/integrations/integration-catalog.tsx index ca61e1325..293802143 100644 --- a/apps/web/src/components/integrations/integration-catalog.tsx +++ b/apps/web/src/components/integrations/integration-catalog.tsx @@ -60,7 +60,7 @@ const CATALOG: ReadonlyArray = [ id: "planetscale", name: "PlanetScale", description: - "Authorize your organization with one click — Maple discovers and scrapes every database branch.", + "Authorize your organization with one click — Maple tracks every database branch automatically.", icon: PlanetScaleIcon, // PlanetScale's mark is monochrome — neutral wash that works in both themes. accent: "#8B8B8B", diff --git a/apps/web/src/components/integrations/planetscale-integration-card.tsx b/apps/web/src/components/integrations/planetscale-integration-card.tsx index 429fd5472..4cb239733 100644 --- a/apps/web/src/components/integrations/planetscale-integration-card.tsx +++ b/apps/web/src/components/integrations/planetscale-integration-card.tsx @@ -25,11 +25,10 @@ import { toast } from "sonner" import { CheckIcon, CircleWarningIcon, LoaderIcon, PlanetScaleIcon } from "@/components/icons" import { cn } from "@maple/ui/utils" import { Result, useAtomRefresh, useAtomSet, useAtomValue } from "@/lib/effect-atom" -import { formatRelativeTime } from "@/lib/format" import { MapleApiAtomClient } from "@/lib/services/common/atom-client" -import { ScrapeTargetsSection } from "@/components/settings/scrape-targets-section" import { IntegrationIconPlate, catalogEntry } from "./integration-catalog" import { IntegrationEmptyState } from "./integration-empty-state" +import { PlanetScaleMetricsHealth } from "./planetscale-metrics-health" const PLANETSCALE_ENTRY = catalogEntry("planetscale") @@ -43,9 +42,9 @@ const parsePatternList = (value: string): string[] => /** * First-class PlanetScale connection card: authorize Maple's OAuth application * in a popup, pick the PlanetScale organization (auto-bound when the grant - * reaches exactly one), and Maple provisions the managed branch-metrics scrape - * target, polls database inventory, and proxies query insights. The managed - * scrape target's per-branch health renders below via the shared list. + * reaches exactly one), and Maple collects branch metrics, polls database + * inventory, and proxies query insights automatically. Collection health shows + * as a single status row — the machinery stays out of the UI. */ export function PlanetScaleIntegrationCard() { const statusQuery = MapleApiAtomClient.query("integrations", "planetscaleStatus", { @@ -204,7 +203,7 @@ export function PlanetScaleIntegrationCard() { description="Authorize Maple in PlanetScale and databases, branches, query insights, and webhooks connect instantly. Branch metrics take one more paste: a read-only service token, since PlanetScale only exposes its metrics endpoints to service tokens." features={[ "Databases appear on the service map with live health", - "Branch metrics scraped automatically, no agent required", + "Branch metrics collected automatically — nothing to run", "Branch filters keep preview branches out", ]} footer="You'll authorize Maple in a PlanetScale popup; a read_metrics_endpoints service token completes metrics afterwards." @@ -249,22 +248,6 @@ export function PlanetScaleIntegrationCard() { )}

- {target ? ( -

- {target.lastScrapeAt ? ( - <> - Last scrape{" "} - {formatRelativeTime(new Date(target.lastScrapeAt).toISOString())} · - every {target.scrapeIntervalSeconds}s - - ) : ( - "First scrape starts within a minute." - )} - {target.excludeBranches.length > 0 ? ( - <> · excluding {target.excludeBranches.join(", ")} - ) : null} -

- ) : null} @@ -291,6 +274,10 @@ export function PlanetScaleIntegrationCard() { /> ) : null} + {status && target ? ( + + ) : null} + {missingDatabasesPermission ? (
@@ -306,32 +293,18 @@ export function PlanetScaleIntegrationCard() {
) : null} - {target?.lastScrapeError && status?.metricsAuth !== "missing" ? ( -
- - - Metrics collection degraded - - {target.lastScrapeError} - - -
- ) : null} - {/* The managed target's per-branch scrape health, via the shared scrape-target list. */} - - {/* Re-binding to another org the grant covers — finalize is an upsert. */} Change PlanetScale organization - Pick another organization the authorization covers. The managed scrape target - follows the new organization. + Pick another organization the authorization covers. Metrics collection follows + automatically. @@ -571,7 +544,7 @@ function PlanetScaleOrgPicker(props: { autoComplete="off" />

- Glob patterns for branches to skip — keeps preview branches from being scraped. + Glob patterns for branches to skip — keeps preview branches out of your metrics.

diff --git a/apps/web/src/components/integrations/planetscale-metrics-health.tsx b/apps/web/src/components/integrations/planetscale-metrics-health.tsx new file mode 100644 index 000000000..f4fc3dce3 --- /dev/null +++ b/apps/web/src/components/integrations/planetscale-metrics-health.tsx @@ -0,0 +1,92 @@ +import { useState } from "react" + +import { cn } from "@maple/ui/utils" + +import type { PlanetScaleScrapeTargetSummary } from "@maple/domain/http" +import { formatRelativeTime } from "@/lib/format" + +type HealthState = "degraded" | "waiting" | "stalled" | "healthy" + +/** + * Outcome-level health for the managed branch-metrics collection. Collection + * itself is fully automatic (Maple provisions and runs it), so the card shows a + * single status row instead of the underlying machinery: one dot, one label, + * and — only when degraded — the raw error behind a disclosure. + */ +export function PlanetScaleMetricsHealth({ + target, + metricsAuth, +}: { + target: PlanetScaleScrapeTargetSummary + metricsAuth: "oauth" | "service_token" | "missing" +}) { + const [detailsOpen, setDetailsOpen] = useState(false) + + // The token-setup step owns the missing-auth state — don't show two messages. + if (metricsAuth === "missing") return null + + const state: HealthState = + target.lastScrapeError !== null + ? "degraded" + : target.lastScrapeAt === null + ? "waiting" + : Date.now() - target.lastScrapeAt > 3 * target.scrapeIntervalSeconds * 1000 + ? "stalled" + : "healthy" + + const updatedAgo = + target.lastScrapeAt !== null + ? formatRelativeTime(new Date(target.lastScrapeAt).toISOString()) + : null + + return ( +
+
+ + + {state === "degraded" + ? "Metrics collection degraded" + : state === "stalled" + ? "Metrics collection stalled" + : state === "waiting" + ? "Waiting for first metrics" + : "Metrics"} + + + {state === "waiting" + ? "Branch metrics usually appear within a minute of connecting." + : state === "healthy" + ? `Updated ${updatedAgo}` + : updatedAgo !== null + ? `Last data ${updatedAgo}` + : null} + {state === "healthy" && target.excludeBranches.length > 0 ? ( + <> · excluding {target.excludeBranches.join(", ")} + ) : null} + + {state === "degraded" ? ( + + ) : null} +
+ {state === "degraded" && detailsOpen ? ( +

+ {target.lastScrapeError} +

+ ) : null} +
+ ) +} diff --git a/apps/web/src/components/settings/scrape-targets-section.tsx b/apps/web/src/components/settings/scrape-targets-section.tsx index 1579edb34..6a8f71698 100644 --- a/apps/web/src/components/settings/scrape-targets-section.tsx +++ b/apps/web/src/components/settings/scrape-targets-section.tsx @@ -10,7 +10,6 @@ import type { ScrapeTargetChecksListResponse, ScrapeTargetId, ScrapeTargetResponse, - ScrapeTargetType, } from "@maple/domain/http" import { useState, type KeyboardEvent, type ReactNode } from "react" import { Exit, Schema } from "effect" @@ -190,39 +189,21 @@ function scheduledStatus( } } -interface SourceCopy { - readonly description: string - readonly emptyTitle: string - readonly emptyDescription: string -} - -const SOURCE_COPY: Record<"all" | ScrapeTargetType, SourceCopy> = { - all: { - description: "Scrape Prometheus exporters and inspect scheduled scrape health.", - emptyTitle: "No scrape targets", - emptyDescription: "Add a Prometheus exporter endpoint to start scraping metrics.", - }, - prometheus: { - description: "Scrape Prometheus exporters and inspect scheduled scrape health.", - emptyTitle: "No scrape targets", - emptyDescription: "Add a Prometheus exporter endpoint to start scraping metrics.", - }, - planetscale: { - description: - "Connect PlanetScale organizations — Maple discovers and scrapes every database branch automatically.", - emptyTitle: "No PlanetScale organizations", - emptyDescription: "Connect an organization with a service token to start scraping branch metrics.", - }, -} +const COPY = { + description: "Scrape Prometheus exporters and inspect scheduled scrape health.", + emptyTitle: "No scrape targets", + emptyDescription: "Add a Prometheus exporter endpoint to start scraping metrics.", +} as const +/** + * Prometheus scrape-target manager. PlanetScale metrics collection is fully + * managed by its integration and never surfaces here — this section only + * lists and edits user-created prometheus targets. + */ export function ScrapeTargetsSection({ - sourceFilter, + sourceFilter = "prometheus", }: { - /** - * Scope this section to one target type (Integrations hub drill-ins): - * filters the list, presets the add dialog, and hides the source selector. - */ - sourceFilter?: ScrapeTargetType + sourceFilter?: "prometheus" } = {}) { const [dialogOpen, setDialogOpen] = useState(false) const [isSaving, setIsSaving] = useState(false) @@ -232,15 +213,9 @@ export function ScrapeTargetsSection({ const [selectedTargetId, setSelectedTargetId] = useState(null) const [editingTarget, setEditingTarget] = useState(null) - const [formTargetType, setFormTargetType] = useState("prometheus") const [formName, setFormName] = useState("") const [formServiceName, setFormServiceName] = useState("") const [formUrl, setFormUrl] = useState("") - const [formOrganization, setFormOrganization] = useState("") - const [formTokenId, setFormTokenId] = useState("") - const [formTokenSecret, setFormTokenSecret] = useState("") - const [formIncludeBranches, setFormIncludeBranches] = useState("") - const [formExcludeBranches, setFormExcludeBranches] = useState("") const [formInterval, setFormInterval] = useState("15") const [formAuthType, setFormAuthType] = useState("none") const [formAuthToken, setFormAuthToken] = useState("") @@ -269,20 +244,18 @@ export function ScrapeTargetsSection({ const targets = Result.builder(listResult) .onSuccess((response) => [...response.targets] as ScrapeTarget[]) .orElse(() => []) - .filter((target) => !sourceFilter || target.targetType === sourceFilter) + .filter((target) => target.targetType === sourceFilter) const selectedTarget = targets.find((target) => target.id === selectedTargetId) ?? null - const copy = SOURCE_COPY[sourceFilter ?? "all"] + const copy = COPY // When empty, the centered empty state owns the primary action — hide the toolbar row. const isEmpty = Result.isSuccess(listResult) && targets.length === 0 - const emptyEntry = sourceFilter ? catalogEntry(sourceFilter) : null + const emptyEntry = catalogEntry(sourceFilter) async function handleProbe(target: ScrapeTarget) { setProbingId(target.id) const result = await probeMutation({ params: { targetId: target.id }, - // Invalidate the list plus the PlanetScale card's status (which surfaces - // this target's lastScrapeError/enabled) instead of only refreshing locally. - reactivityKeys: ["scrapeTargets", "planetscaleIntegrationStatus"], + reactivityKeys: ["scrapeTargets"], }) if (Exit.isSuccess(result)) { if (result.value.success) { @@ -297,18 +270,11 @@ export function ScrapeTargetsSection({ } function openAddDialog() { - const targetType = sourceFilter ?? "prometheus" setEditingTarget(null) - setFormTargetType(targetType) setFormName("") setFormServiceName("") setFormUrl("") - setFormOrganization("") - setFormTokenId("") - setFormTokenSecret("") - setFormIncludeBranches("") - setFormExcludeBranches("") - setFormInterval(targetType === "planetscale" ? "30" : "15") + setFormInterval("15") setFormAuthType("none") setFormAuthToken("") setFormAuthUsername("") @@ -318,15 +284,9 @@ export function ScrapeTargetsSection({ function openEditDialog(target: ScrapeTarget) { setEditingTarget(target) - setFormTargetType(target.targetType) setFormName(target.name) setFormServiceName(target.serviceName ?? "") setFormUrl(target.url) - setFormOrganization(target.organization ?? "") - setFormTokenId("") - setFormTokenSecret("") - setFormIncludeBranches(target.includeBranches.join(", ")) - setFormExcludeBranches(target.excludeBranches.join(", ")) setFormInterval(String(target.scrapeIntervalSeconds)) setFormAuthType(target.authType) setFormAuthToken("") @@ -335,25 +295,7 @@ export function ScrapeTargetsSection({ setDialogOpen(true) } - function selectTargetType(type: ScrapeTargetType) { - setFormTargetType(type) - setFormInterval(type === "planetscale" ? "30" : "15") - } - - // Managed rows (provisioned by the PlanetScale integration) resolve auth from - // the org's OAuth grant — the edit dialog must not flip them back to "token" - // or ask for credentials. - const isManagedPlanetScaleAuth = editingTarget?.authType === "planetscale_oauth" - function buildAuthCredentials(): string | null { - if (formTargetType === "planetscale") { - if (isManagedPlanetScaleAuth) return null - if (!formTokenId.trim() || !formTokenSecret.trim()) return null - return JSON.stringify({ - tokenId: formTokenId.trim(), - tokenSecret: formTokenSecret.trim(), - }) - } if (formAuthType === "bearer") { if (!formAuthToken.trim()) return null return JSON.stringify({ token: formAuthToken.trim() }) @@ -368,35 +310,21 @@ export function ScrapeTargetsSection({ return null } - function parseBranchList(value: string): string[] { - return value - .split(",") - .map((entry) => entry.trim()) - .filter((entry) => entry.length > 0) - } - async function handleSave() { - const isPlanetScale = formTargetType === "planetscale" - if (!formName.trim() || (isPlanetScale ? !formOrganization.trim() : !formUrl.trim())) { - toast.error(isPlanetScale ? "Name and organization are required" : "Name and URL are required") + if (!formName.trim() || !formUrl.trim()) { + toast.error("Name and URL are required") return } let parsedInterval: ScrapeIntervalSeconds try { - parsedInterval = asScrapeIntervalSeconds( - Number.parseInt(formInterval, 10) || (isPlanetScale ? 30 : 15), - ) + parsedInterval = asScrapeIntervalSeconds(Number.parseInt(formInterval, 10) || 15) } catch { toast.error("Scrape interval must be an integer from 5 to 300 seconds") return } const authCredentials = buildAuthCredentials() - if (isPlanetScale && !editingTarget && authCredentials === null) { - toast.error("Service token ID and secret are required") - return - } setIsSaving(true) @@ -407,22 +335,11 @@ export function ScrapeTargetsSection({ name: formName.trim(), scrapeIntervalSeconds: parsedInterval, serviceName: formServiceName.trim() || null, - ...(isPlanetScale - ? { - organization: formOrganization.trim(), - authType: isManagedPlanetScaleAuth - ? ("planetscale_oauth" as const) - : ("token" as const), - includeBranches: parseBranchList(formIncludeBranches), - excludeBranches: parseBranchList(formExcludeBranches), - } - : { - url: formUrl.trim(), - authType: formAuthType, - }), + url: formUrl.trim(), + authType: formAuthType, ...(authCredentials !== null ? { authCredentials } : {}), }), - reactivityKeys: ["scrapeTargets", "planetscaleIntegrationStatus"], + reactivityKeys: ["scrapeTargets"], }) if (Exit.isSuccess(result)) { toast.success("Scrape target updated") @@ -436,21 +353,11 @@ export function ScrapeTargetsSection({ name: formName.trim(), scrapeIntervalSeconds: parsedInterval, serviceName: formServiceName.trim() || null, - ...(isPlanetScale - ? { - targetType: "planetscale" as const, - organization: formOrganization.trim(), - authType: "token" as const, - includeBranches: parseBranchList(formIncludeBranches), - excludeBranches: parseBranchList(formExcludeBranches), - } - : { - url: formUrl.trim(), - authType: formAuthType, - }), + url: formUrl.trim(), + authType: formAuthType, ...(authCredentials !== null ? { authCredentials } : {}), }), - reactivityKeys: ["scrapeTargets", "planetscaleIntegrationStatus"], + reactivityKeys: ["scrapeTargets"], }) if (Exit.isSuccess(result)) { toast.success("Scrape target created") @@ -467,7 +374,7 @@ export function ScrapeTargetsSection({ setDeleteConfirmTarget(null) const result = await deleteMutation({ params: { targetId }, - reactivityKeys: ["scrapeTargets", "planetscaleIntegrationStatus"], + reactivityKeys: ["scrapeTargets"], }) if (Exit.isSuccess(result)) { toast.success("Scrape target deleted") @@ -484,7 +391,7 @@ export function ScrapeTargetsSection({ payload: new UpdateScrapeTargetRequest({ enabled: !target.enabled, }), - reactivityKeys: ["scrapeTargets", "planetscaleIntegrationStatus"], + reactivityKeys: ["scrapeTargets"], }) if (!Exit.isSuccess(result)) { toast.error("Failed to update scrape target") @@ -541,7 +448,6 @@ export function ScrapeTargetsSection({ selected={target.id === selectedTarget?.id} toggling={togglingId === target.id} probing={probingId === target.id} - hideTypeBadge={sourceFilter === "planetscale"} onSelect={setSelectedTargetId} onProbe={handleProbe} onToggle={handleToggleEnabled} @@ -581,32 +487,10 @@ export function ScrapeTargetsSection({ {editingTarget ? "Update the scrape target configuration." - : formTargetType === "planetscale" - ? "Connect a PlanetScale organization. Maple discovers every database branch's metrics endpoint and scrapes them automatically." - : "Enter the URL of a Prometheus exporter endpoint. Maple will periodically scrape this endpoint for metrics."} + : "Enter the URL of a Prometheus exporter endpoint. Maple will periodically scrape this endpoint for metrics."}
- {!editingTarget && !sourceFilter && ( -
- - -
- )}
- {formTargetType === "prometheus" ? ( -
- - setFormUrl(e.target.value)} - /> -
- ) : ( - <> -
- - setFormOrganization(e.target.value)} - /> -

- Your PlanetScale organization name as it appears in the dashboard URL. -

-
- {isManagedPlanetScaleAuth ? ( -

- Authentication is managed by the PlanetScale integration — this target - scrapes with the connected organization's OAuth authorization, no - credentials to enter. -

- ) : ( - <> -
- - setFormTokenId(e.target.value)} - /> -
-
- - setFormTokenSecret(e.target.value)} - /> -

- Create a service token with the{" "} - read_metrics_endpoints{" "} - organization permission. -

-
- - )} -
- - setFormIncludeBranches(e.target.value)} - /> -

- Comma-separated branch globs. When set, only matching branches are - scraped. Leave blank to scrape all branches. -

-
-
- - setFormExcludeBranches(e.target.value)} - /> -

- Comma-separated branch globs to skip — e.g.{" "} - pr-* to avoid scraping PR-preview - branches (a common source of PlanetScale rate-limit 429s). -

-
- - )} +
+ + setFormUrl(e.target.value)} + /> +
setFormInterval(e.target.value)} />
- {formTargetType === "prometheus" && ( -
- - -
- )} - {formTargetType === "prometheus" && formAuthType === "bearer" && ( +
+ + +
+ {formAuthType === "bearer" && (
)} - {formTargetType === "prometheus" && formAuthType === "basic" && ( + {formAuthType === "basic" && ( <>
@@ -868,7 +664,6 @@ function ScrapeTargetRow({ selected, toggling, probing, - hideTypeBadge, onSelect, onProbe, onToggle, @@ -879,8 +674,6 @@ function ScrapeTargetRow({ selected: boolean toggling: boolean probing: boolean - /** The PlanetScale drill-in shows only planetscale targets — the badge is noise there. */ - hideTypeBadge?: boolean onSelect: (targetId: ScrapeTargetId) => void onProbe: (target: ScrapeTarget) => void onToggle: (target: ScrapeTarget) => void @@ -923,21 +716,6 @@ function ScrapeTargetRow({ {status.label} - {target.targetType === "planetscale" && !hideTypeBadge && ( - - PlanetScale - - )} - {target.managedBy && ( - - }> - Managed - - - Provisioned by the PlanetScale integration — edit it from the integration card. - - - )} {target.serviceName && ( {target.serviceName} @@ -950,11 +728,7 @@ function ScrapeTargetRow({ )}
- - {target.targetType === "planetscale" - ? (target.organization ?? hostnameFromUrl(target.url)) - : hostnameFromUrl(target.url)} - + {hostnameFromUrl(target.url)} {target.scrapeIntervalSeconds}s interval {status.detail} {target.lastScrapeAt && ( @@ -1182,25 +956,7 @@ function ScrapeTargetDetails({
- {target.targetType === "planetscale" ? ( - <> - - {target.includeBranches.length > 0 && ( - {target.includeBranches.join(", ")}} - /> - )} - {target.excludeBranches.length > 0 && ( - {target.excludeBranches.join(", ")}} - /> - )} - - ) : ( - - )} + - seconds >= 1 ? `${seconds.toFixed(1)}s` : `${Math.round(seconds * 1000)}ms` - function PlanetScaleDatabasePage() { const { dbName } = Route.useParams() const search = Route.useSearch() @@ -90,7 +92,7 @@ function PlanetScaleDatabasePage() {
@@ -98,6 +100,7 @@ function PlanetScaleDatabasePage() { {database.kind === "postgresql" ? "Postgres" : "MySQL / Vitess"} {database.region ? {database.region} : null} + {database.plan ? {database.plan} : null} {database.branches.length} branch {database.branches.length === 1 ? "" : "es"} @@ -153,6 +156,18 @@ function PlanetScaleDatabaseData({ return map }, [inventoryResult, database]) + // Query-insights branch switcher: ready branches only, production first. + // undefined = let the API resolve the production branch. + const [insightsBranch, setInsightsBranch] = useState(undefined) + const insightsBranches = useMemo( + () => + [...branchInfoByName.entries()] + .filter(([, info]) => info.ready) + .sort(([, a], [, b]) => Number(b.production) - Number(a.production)) + .map(([name]) => name), + [branchInfoByName], + ) + const buckets = Result.builder(timeseriesResult) .onSuccess((r) => r.buckets) .orElse(() => []) @@ -195,70 +210,43 @@ function PlanetScaleDatabaseData({ ) : branchStats.length > 0 ? (

Branches

-
-
- Branch - Connections - CPU (max) - Memory (max) - Replica lag -
- {branchStats.map((row) => { - const info = branchInfoByName.get(row.branch) - return ( -
- - {row.branch} - {info?.production ? ( - - production - - ) : null} - - - {formatNumber(row.connectionsAvg)} - - 80 - ? "text-severity-error" - : row.cpuMaxPercent > 60 - ? "text-severity-warn" - : undefined, - )} - > - {row.cpuMaxPercent.toFixed(0)}% - - - {row.memMaxPercent.toFixed(0)}% - - 10 - ? "text-severity-error" - : row.replicaLagMaxSeconds > 1 - ? "text-severity-warn" - : undefined, - )} - > - {formatLag(row.replicaLagMaxSeconds)} - -
- ) - })} -
+
) : null}
-

Top Queries (PlanetScale Insights)

+
+

+ Top Queries (PlanetScale Insights) +

+ {insightsBranches.length > 1 ? ( + + ) : null} +
{Result.builder(statusResult) .onInitial(() => ( @@ -88,7 +91,13 @@ function PlanetScalePage() { .onError((err) => ) .onSuccess((status) => { if (!status.connected) return - return + return ( + + ) }) .render()}
@@ -97,7 +106,19 @@ function PlanetScalePage() { ) } -function PlanetScaleData({ startTime, endTime }: { startTime: string; endTime: string }) { +// Inventory refreshes every few minutes — older than this and the database +// list is likely out of date (poller stalled, token revoked). +const INVENTORY_STALE_MS = 15 * 60 * 1000 + +function PlanetScaleData({ + startTime, + endTime, + lastInventoryError, +}: { + startTime: string + endTime: string + lastInventoryError: string | null +}) { const inventoryResult = useAtomValue( MapleApiAtomClient.query("integrations", "planetscaleDatabases", { reactivityKeys: ["planetscaleIntegrationStatus"], @@ -131,14 +152,26 @@ function PlanetScaleData({ startTime, endTime }: { startTime: string; endTime: s .onInitial(() => (
- +
)) .onError((err) => ) .onSuccess((inventory) => { const branchTotal = inventory.databases.reduce((sum, db) => sum + db.branches.length, 0) + const inventoryStale = + inventory.lastInventoryAt !== null && + Date.now() - inventory.lastInventoryAt > INVENTORY_STALE_MS return (
+ {lastInventoryError !== null || inventoryStale ? ( +

+ {lastInventoryError !== null + ? "Inventory refresh failing — the database list may be out of date." + : `Inventory last refreshed ${formatRelativeTime( + new Date(inventory.lastInventoryAt ?? 0).toISOString(), + )} — the database list may be out of date.`} +

+ ) : null} {Result.isFailure(statsResult) ? ( ) : ( - + )}
) From 634612ef1634eb4e4faad18961b7dc2a137a1bcc Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 15 Jul 2026 15:37:30 +0200 Subject: [PATCH 5/7] fix(deploy): drop prod email-routing bootstrap blocking prd deploys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared-bindings bootstrap in #210 added Email.Routing + SendingSubdomain (zone-level enable + DKIM/SPF DNS writes) that the prod CLOUDFLARE_API_TOKEN lacks scopes for, so every push-to-main prd deploy 403'd in the plan phase (POST /zones/{id}/email/routing/enable → Forbidden). Revert the email plumbing to the pre-#210 wiring: both api and alerting keep their plain Cloudflare.Email.SendEmail("email", ...) binding (unchanged from before, needs no extra token scope). The Workers ObservabilityDestination bootstrap stays — that permission is granted and its plan call now succeeds. Removes the emailBinding param from createMapleApi/createAlertingWorker and the now-unused Input/Output imports. Co-Authored-By: Claude Opus 4.8 --- alchemy.run.ts | 23 +---------------------- apps/alerting/alchemy.run.ts | 13 ++++--------- apps/api/alchemy.run.ts | 13 ++++--------- 3 files changed, 9 insertions(+), 40 deletions(-) diff --git a/alchemy.run.ts b/alchemy.run.ts index c9f3c6c02..da76af802 100644 --- a/alchemy.run.ts +++ b/alchemy.run.ts @@ -1,7 +1,6 @@ import { appendFileSync } from "node:fs" import * as Alchemy from "alchemy" import * as Cloudflare from "alchemy/Cloudflare" -import * as Output from "alchemy/Output" import * as RemovalPolicy from "alchemy/RemovalPolicy" import * as Effect from "effect/Effect" import { formatMapleStage, parseMapleStage, resolveMapleDomains } from "@maple/infra/cloudflare" @@ -46,14 +45,6 @@ const createProductionSharedResources = (stage: ReturnType - Cloudflare.Email.SendEmail("email", { - allowedSenderAddresses: ["notifications@noreply.maple.dev"], - }), - ), - ) - - return { emailBinding, logsDestination, tracesDestination } + return { logsDestination, tracesDestination } }) export default Alchemy.Stack( @@ -121,7 +102,6 @@ export default Alchemy.Stack( const { worker: api, db: mapleDb } = yield* createMapleApi({ stage, domains, - emailBinding: shared.emailBinding, }) const chatFlue = yield* createChatFlueWorker({ stage, @@ -155,7 +135,6 @@ export default Alchemy.Stack( stage, domains, mapleDb, - emailBinding: shared.emailBinding, }) const summary = { diff --git a/apps/alerting/alchemy.run.ts b/apps/alerting/alchemy.run.ts index e9013bd93..f9983d576 100644 --- a/apps/alerting/alchemy.run.ts +++ b/apps/alerting/alchemy.run.ts @@ -1,5 +1,4 @@ import path from "node:path" -import type { Input } from "alchemy" import * as Cloudflare from "alchemy/Cloudflare" import * as Effect from "effect/Effect" import * as Redacted from "effect/Redacted" @@ -34,11 +33,9 @@ export interface CreateAlertingWorkerOptions { domains: MapleDomains /** Managed per-branch Hyperdrive from the api factory; undefined on ref stages (stg/prd). */ mapleDb: Cloudflare.Hyperdrive.Connection | undefined - /** Shared production binding derived from the retained sending subdomain. */ - emailBinding?: Input } -export const createAlertingWorker = ({ stage, mapleDb, emailBinding }: CreateAlertingWorkerOptions) => +export const createAlertingWorker = ({ stage, mapleDb }: CreateAlertingWorkerOptions) => Effect.gen(function* () { const hyperdriveRefId = resolveHyperdriveRefId(stage) // Cross-script binding to the AI triage Workflow hosted by the api worker — @@ -67,11 +64,9 @@ export const createAlertingWorker = ({ stage, mapleDb, emailBinding }: CreateAle // Ref stages attach MAPLE_DB via worker.bind below. ...(mapleDb ? { MAPLE_DB: mapleDb } : {}), AI_TRIAGE_WORKFLOW: aiTriageWorkflow, - EMAIL: - emailBinding ?? - Cloudflare.Email.SendEmail("email", { - allowedSenderAddresses: ["notifications@noreply.maple.dev"], - }), + EMAIL: Cloudflare.Email.SendEmail("email", { + allowedSenderAddresses: ["notifications@noreply.maple.dev"], + }), TINYBIRD_HOST: requireEnv("TINYBIRD_HOST"), TINYBIRD_TOKEN: Redacted.make(requireEnv("TINYBIRD_TOKEN")), MAPLE_AUTH_MODE: process.env.MAPLE_AUTH_MODE?.trim() || "self_hosted", diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index 9afa5c934..0e0bc7246 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -1,5 +1,4 @@ import path from "node:path" -import type { Input } from "alchemy" import * as Cloudflare from "alchemy/Cloudflare" import type { Rpc } from "alchemy/Rpc" import * as Effect from "effect/Effect" @@ -35,14 +34,12 @@ const optionalSecret = (key: string): Record> export interface CreateMapleApiOptions { stage: MapleStage domains: MapleDomains - /** Shared production binding derived from the retained sending subdomain. */ - emailBinding?: Input } /** Alchemy resource type carried across the chat-flue service binding. */ export type MapleApiWorker = Cloudflare.Worker & Rpc -export const createMapleApi = ({ stage, domains, emailBinding }: CreateMapleApiOptions) => +export const createMapleApi = ({ stage, domains }: CreateMapleApiOptions) => Effect.gen(function* () { // MAPLE_DB Hyperdrive comes in two flavors: // @@ -140,11 +137,9 @@ export const createMapleApi = ({ stage, domains, emailBinding }: CreateMapleApiO VCS_SYNC_QUEUE: vcsSyncQueue, CLICKHOUSE_SCHEMA_APPLY_WORKFLOW: schemaApplyWorkflow, AI_TRIAGE_WORKFLOW: aiTriageWorkflow, - EMAIL: - emailBinding ?? - Cloudflare.Email.SendEmail("email", { - allowedSenderAddresses: ["notifications@noreply.maple.dev"], - }), + EMAIL: Cloudflare.Email.SendEmail("email", { + allowedSenderAddresses: ["notifications@noreply.maple.dev"], + }), TINYBIRD_HOST: requireEnv("TINYBIRD_HOST"), TINYBIRD_TOKEN: Redacted.make(requireEnv("TINYBIRD_TOKEN")), ...optionalPlain("CLICKHOUSE_URL"), From def2465f011e9b0b38d8de73e4baa08cae19ebec Mon Sep 17 00:00:00 2001 From: Makisuo <31933546+Makisuo@users.noreply.github.com> Date: Thu, 16 Jul 2026 00:12:14 +0200 Subject: [PATCH 6/7] fix(planetscale): forward signed data-plane auth params to branch scrapes (#212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PlanetScale authenticates the metrics data plane (metrics.psdb.cloud) with a signed, expiring URL — NOT the service token or OAuth bearer. Its Prometheus http_sd response mints per-branch `__param_sig`/`__param_exp` labels, which Prometheus convention promotes to `?sig=&exp=` query params on the scrape URL. The Authorization header (token/bearer) only auths the discovery listing on api.planetscale.com. `subTargetsFromGroup` stripped every `__`-prefixed label except `__scheme__`/`__metrics_path__`, dropping sig/exp and building an unsigned URL, so every branch scrape returned `403 invalid signature` (a 22h+ metrics outage in production). Reconnecting and rotating service tokens couldn't help because the token was never the data-plane credential. Fix: promote `__param_*` labels onto a new `PlanetScaleSubTarget.signedUrl` (base `url` stays param-free so the scraper's per-branch fiber identity + `instance` label don't churn as the signature rotates each 10-min refresh). `ScrapeTargetsService.scrapeForCollector` and `PlanetScaleConnectionService.probeDataPlaneScrape` now fetch `signedUrl`. Verified: the URL built via URLSearchParams is byte-identical to one proven to return HTTP 200 with live branch metrics. apps/api-only deploy; existing targets recover on the next scrape with no reconnect. Co-authored-by: Claude Opus 4.8 --- .../services/PlanetScaleConnectionService.ts | 19 +++---- .../PlanetScaleDiscoveryService.test.ts | 53 +++++++++++++++++++ .../services/PlanetScaleDiscoveryService.ts | 43 ++++++++++++--- apps/api/src/services/ScrapeTargetsService.ts | 13 ++--- 4 files changed, 105 insertions(+), 23 deletions(-) diff --git a/apps/api/src/services/PlanetScaleConnectionService.ts b/apps/api/src/services/PlanetScaleConnectionService.ts index f84570807..cb7c3fe9a 100644 --- a/apps/api/src/services/PlanetScaleConnectionService.ts +++ b/apps/api/src/services/PlanetScaleConnectionService.ts @@ -179,14 +179,14 @@ export class PlanetScaleConnectionService extends Context.Service< const decodeHttpSd = Schema.decodeUnknownEffect(Schema.fromJsonString(HttpSdResponse)) /** - * Whether the bearer can perform an ACTUAL branch-metrics scrape. The SD - * endpoint on api.planetscale.com accepting the bearer proves nothing - * about the data plane: the discovered hosts (metrics.psdb.cloud) only - * document service-token auth and reject OAuth bearers with 403. Probe one - * discovered endpoint with the bearer so `readMetricsEndpoints` reflects - * scraping, not listing. Inconclusive outcomes (no branches discovered - * yet, transport blip, undecodable payload) keep the control-plane answer - * instead of pausing a possibly-working target. + * Whether an ACTUAL branch-metrics scrape works. The SD endpoint on + * api.planetscale.com accepting the credential proves nothing about the + * data plane: metrics.psdb.cloud authenticates with the signed `?sig=&exp=` + * URL params minted in the SD response, so we probe a discovered endpoint + * via its `signedUrl` and let `readMetricsEndpoints` reflect scraping, not + * listing. Inconclusive outcomes (no branches discovered yet, transport + * blip, undecodable payload) keep the control-plane answer instead of + * pausing a possibly-working target. */ const probeDataPlaneScrape = Effect.fn("PlanetScaleConnectionService.probeDataPlaneScrape")( function* (organization: string, bearer: string) { @@ -205,7 +205,8 @@ export class PlanetScaleConnectionService extends Context.Service< const first = groups.flatMap((group) => subTargetsFromGroup(group).ok)[0] if (first === undefined) return "inconclusive" as const - const scrape = yield* probeUrl(first.url, bearer) + // signedUrl carries the `?sig=&exp=` params the data plane requires. + const scrape = yield* probeUrl(first.signedUrl, bearer) return scrape.status >= 200 && scrape.status < 300 ? ("ok" as const) : ("rejected" as const) diff --git a/apps/api/src/services/PlanetScaleDiscoveryService.test.ts b/apps/api/src/services/PlanetScaleDiscoveryService.test.ts index c0a41e15a..f04d3c946 100644 --- a/apps/api/src/services/PlanetScaleDiscoveryService.test.ts +++ b/apps/api/src/services/PlanetScaleDiscoveryService.test.ts @@ -225,6 +225,59 @@ describe("PlanetScaleDiscoveryService", () => { }).pipe(Effect.provide(makeLayer(testDb))) }) + it.effect("promotes __param_* meta labels to signed scrape-url query params", () => { + const testDb = createTestDb(trackedDbs) + const recorded: Array = [] + return Effect.gen(function* () { + const discovery = yield* PlanetScaleDiscoveryService + const row = yield* createPlanetScaleTargetRow("my-org") + + // PlanetScale authenticates the metrics data plane with a signed, expiring + // URL: the http_sd group carries `__param_sig`/`__param_exp`, which + // Prometheus promotes to `?sig=&exp=` on the scrape URL. Dropping them + // yields `403 invalid signature` on every scrape (the real prod outage). + const SIGNED_PAYLOAD = [ + { + targets: ["metrics.psdb.cloud"], + labels: { + __metrics_path__: "/metrics/branch/3a1nf2gvu9rf", + __scheme__: "https", + __param_sig: "abc-_signature123", + __param_exp: "1784238737", + planetscale_branch_name: "main", + planetscale_database_name: "mydb", + }, + }, + ] + + const entries = yield* discovery.discover(row).pipe( + Effect.provideService( + FetchHttpClient.Fetch, + stubFetch(recorded, () => Response.json(SIGNED_PAYLOAD)), + ), + ) + + assert.strictEqual(entries.length, 1) + // Base url + fiber-identity key stay param-free so identity is stable as + // PlanetScale rotates the signature each refresh. + assert.strictEqual(entries[0]?.url, "https://metrics.psdb.cloud/metrics/branch/3a1nf2gvu9rf") + assert.strictEqual( + entries[0]?.subTargetKey, + "metrics.psdb.cloud/metrics/branch/3a1nf2gvu9rf", + ) + // signedUrl carries the auth params the data plane actually verifies. + assert.strictEqual( + entries[0]?.signedUrl, + "https://metrics.psdb.cloud/metrics/branch/3a1nf2gvu9rf?sig=abc-_signature123&exp=1784238737", + ) + // The signed params must never leak into the metric labels. + assert.deepStrictEqual(entries[0]?.labels, { + planetscale_branch_name: "main", + planetscale_database_name: "mydb", + }) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + it.effect("caches discovery for the TTL and refreshes after it elapses", () => { const testDb = createTestDb(trackedDbs) const recorded: Array = [] diff --git a/apps/api/src/services/PlanetScaleDiscoveryService.ts b/apps/api/src/services/PlanetScaleDiscoveryService.ts index 112a3c315..85fd26cfa 100644 --- a/apps/api/src/services/PlanetScaleDiscoveryService.ts +++ b/apps/api/src/services/PlanetScaleDiscoveryService.ts @@ -20,10 +20,14 @@ type ScrapeTargetRow = typeof scrapeTargets.$inferSelect /** * Resolves PlanetScale `planetscale`-type scrape targets into their concrete * per-database-branch scrape endpoints via PlanetScale's Prometheus http_sd - * discovery API (`GET /v1/organizations/{org}/metrics`). Managed targets - * (`authType "planetscale_oauth"`) authenticate with the org's OAuth grant - * (`Authorization: Bearer …`); manual escape-hatch targets keep the service - * token scheme (`Authorization: token {ID}:{SECRET}`). + * discovery API (`GET /v1/organizations/{org}/metrics`). The Authorization + * header authenticates the DISCOVERY call only: managed targets + * (`authType "planetscale_oauth"`) use the org's OAuth grant + * (`Authorization: Bearer …`), manual escape-hatch targets the service-token + * scheme (`Authorization: token {ID}:{SECRET}`). The metrics DATA PLANE + * (`metrics.psdb.cloud`) does not use that header at all — it authenticates + * with a signed, expiring URL (`?sig=…&exp=…`) that the SD response mints per + * branch (see {@link subTargetsFromGroup} / {@link PlanetScaleSubTarget.signedUrl}). * * Discovery results are cached in-memory per target with a 10-minute TTL * (PlanetScale's documented refresh cadence). On refresh failure stale entries @@ -32,8 +36,20 @@ type ScrapeTargetRow = typeof scrapeTargets.$inferSelect */ export interface PlanetScaleSubTarget { - /** Concrete per-branch scrape URL (`https://{host}{__metrics_path__}`). */ + /** + * Per-branch scrape endpoint WITHOUT auth params (`https://{host}{__metrics_path__}`). + * Stable across discovery refreshes, so it's the scraper-facing target url + * (fiber identity + `instance` label). NOT fetched directly — see `signedUrl`. + */ readonly url: string + /** + * `url` plus PlanetScale's signed, expiring auth query params (`?sig=…&exp=…`), + * promoted from the http_sd `__param_*` meta labels. This is the URL that + * actually authenticates against the metrics data plane; fetching `url` + * without them returns `403 invalid signature`. Equals `url` when the SD group + * carried no `__param_*` labels. + */ + readonly signedUrl: string /** Stable discriminator: `planetscale_database_branch_id` SD label, falling back to `host:port` + metrics path. */ readonly subTargetKey: string /** SD labels minus `__`-prefixed Prometheus meta labels. */ @@ -81,10 +97,19 @@ export const subTargetsFromGroup = (group: { const sdLabels = group.labels ?? {} const scheme = sdLabels.__scheme__ ?? "https" const path = sdLabels.__metrics_path__ ?? "/metrics" + // PlanetScale authenticates the metrics data plane with a signed, expiring URL + // (`?sig=…&exp=…`) minted per branch in the http_sd response — NOT the discovery + // credential (the service token / OAuth bearer only auths the SD listing). + // Prometheus convention promotes `__param_` meta labels to `?=` + // query params on the scrape URL; forward them or every scrape returns + // `403 invalid signature`. const labels: Record = {} + const authParams = new URLSearchParams() for (const [key, value] of Object.entries(sdLabels)) { - if (!key.startsWith("__")) labels[key] = value + if (key.startsWith("__param_")) authParams.set(key.slice("__param_".length), value) + else if (!key.startsWith("__")) labels[key] = value } + const query = authParams.toString() const branchId = sdLabels.planetscale_database_branch_id const ok: Array = [] @@ -99,14 +124,16 @@ export const subTargetsFromGroup = (group: { } // The no-branch-id fallback keys on host + path (not bare host): groups // that differ only by `__metrics_path__` are distinct endpoints, and a - // host-only key would silently collapse them away in dedupe. + // host-only key would silently collapse them away in dedupe. The signed + // `?sig=&exp=` params are deliberately excluded from the key so per-branch + // fiber identity stays stable as PlanetScale rotates them each refresh. const subTargetKey = branchId && group.targets.length === 1 ? branchId : branchId ? `${branchId}:${hostPort}` : `${hostPort}${path}` - ok.push({ url, subTargetKey, labels }) + ok.push({ url, signedUrl: query ? `${url}?${query}` : url, subTargetKey, labels }) } return { ok, dropped } } diff --git a/apps/api/src/services/ScrapeTargetsService.ts b/apps/api/src/services/ScrapeTargetsService.ts index 1853d4630..bb4bb360c 100644 --- a/apps/api/src/services/ScrapeTargetsService.ts +++ b/apps/api/src/services/ScrapeTargetsService.ts @@ -844,11 +844,12 @@ export class ScrapeTargetsService extends Context.Service entry.subTargetKey === subTargetKey) if (!match) { @@ -859,7 +860,7 @@ export class ScrapeTargetsService extends Context.Service Date: Thu, 16 Jul 2026 00:34:35 +0200 Subject: [PATCH 7/7] feat: improve alert overview view --- .../components/alerts/alert-rule-chart.tsx | 45 ++++--- .../rule-detail/rule-diagnosis-panel.tsx | 58 ++++++++- apps/web/src/hooks/use-alert-rule-preview.ts | 9 +- apps/web/src/routes/alerts/$ruleId.tsx | 122 ++++++++++++++---- 4 files changed, 182 insertions(+), 52 deletions(-) diff --git a/apps/web/src/components/alerts/alert-rule-chart.tsx b/apps/web/src/components/alerts/alert-rule-chart.tsx index dafd85765..da8f67c0c 100644 --- a/apps/web/src/components/alerts/alert-rule-chart.tsx +++ b/apps/web/src/components/alerts/alert-rule-chart.tsx @@ -62,12 +62,12 @@ interface AlertRuleChartProps { } const SINGLE_KEY = "value" -const CHART_HEIGHT = 240 +const CHART_HEIGHT = 300 type ChartPoint = { t: number } & Record type SignalSource = "preview" | "checks" | "none" -const Y_AXIS_WIDTH = 62 -const PLOT_RIGHT = 8 +const Y_AXIS_WIDTH = 72 +const PLOT_RIGHT = 12 const RAIL_CELLS = 60 type RailStatus = "breached" | "error" | "skipped" | "healthy" | "empty" @@ -506,7 +506,7 @@ export function AlertRuleChart({ {isMultiSeries && ( )} + {/* Dashed threshold line(s) only — the labels live in the caption below + the plot so they can't clip at the right edge or collide with the + legend/series. */} {thresholdUpper != null && ( )} @@ -586,6 +577,28 @@ export function AlertRuleChart({
{chartArea} + {hasSignal && ( +
+ + + {thresholdUpper != null + ? "Threshold range " + : breachBelow + ? "Breach below " + : "Breach above "} + + {formatSignalValue(signalType, threshold)} + {thresholdUpper != null + ? ` – ${formatSignalValue(signalType, thresholdUpper)}` + : ""} + + +
+ )} + {hasSignal && error != null && source === "checks" && (

Live signal preview unavailable — showing recorded checks. diff --git a/apps/web/src/components/alerts/rule-detail/rule-diagnosis-panel.tsx b/apps/web/src/components/alerts/rule-detail/rule-diagnosis-panel.tsx index b27b75514..5b8cb076e 100644 --- a/apps/web/src/components/alerts/rule-detail/rule-diagnosis-panel.tsx +++ b/apps/web/src/components/alerts/rule-detail/rule-diagnosis-panel.tsx @@ -82,11 +82,23 @@ export function RuleDiagnosisPanel({ const [expanded, setExpanded] = useState(null) const isOpen = expanded ?? hasProblems + // When something's wrong, lead with the stage(s) that broke — the passing + // steps are noise you're not looking at. They fold behind a disclosure. + const problemStages = useMemo( + () => stages.filter((s) => s.status === "fail" || s.status === "warn"), + [stages], + ) + const [showAll, setShowAll] = useState(false) + const hasHiddenSteps = hasProblems && problemStages.length < stages.length + const visibleStages = showAll || !hasProblems ? stages : problemStages + const passingCount = useMemo(() => stages.filter((s) => s.status === "pass").length, [stages]) + const hiddenCount = stages.length - problemStages.length + return ( {isOpen && ( -

+
{groupKeys.length > 0 && (
Group @@ -116,17 +128,36 @@ export function RuleDiagnosisPanel({ />
)} -
    - {stages.map((stage) => ( +
      + {visibleStages.map((stage) => ( ))}
    + {hasHiddenSteps && ( + + )}
)} ) } +const EVIDENCE_CAP = 4 + function DiagnosisStageRow({ stage, onToggleEnabled, @@ -136,12 +167,16 @@ function DiagnosisStageRow({ }) { const problematic = stage.status === "fail" || stage.status === "warn" const [open, setOpen] = useState(null) + const [showAllEvidence, setShowAllEvidence] = useState(false) const showEvidence = (open ?? problematic) && stage.evidence.length > 0 + // A chatty stage shouldn't blow out the panel height — cap it, reveal on demand. + const evidence = showAllEvidence ? stage.evidence : stage.evidence.slice(0, EVIDENCE_CAP) + const hiddenEvidence = stage.evidence.length - evidence.length return (
  • {showEvidence && (
      - {stage.evidence.map((line) => ( + {evidence.map((line) => (
    • {line}
      • ))} + {hiddenEvidence > 0 && ( +
    • + +
      • + )}
    )}
    • diff --git a/apps/web/src/hooks/use-alert-rule-preview.ts b/apps/web/src/hooks/use-alert-rule-preview.ts index bd47d7b6e..9b83bf370 100644 --- a/apps/web/src/hooks/use-alert-rule-preview.ts +++ b/apps/web/src/hooks/use-alert-rule-preview.ts @@ -52,7 +52,7 @@ const isPreviewQueryReady = (form: RuleFormState): boolean => { * (rootSpansOnly, exclude-services), same no-data semantics. */ export function useAlertRulePreview( - form: RuleFormState, + form: RuleFormState | null, range?: { startTime: string; endTime: string }, ): AlertRulePreviewState { // Callers that own a page-level time window (the rule detail page) pass it in; @@ -65,7 +65,7 @@ export function useAlertRulePreview( const deferredForm = useDeferredValue(form) const payload = useMemo(() => { - if (!isPreviewQueryReady(deferredForm)) return null + if (deferredForm === null || !isPreviewQueryReady(deferredForm)) return null try { // The upsert schema requires a non-empty name; the preview doesn't care, // so substitute a placeholder while the user hasn't typed one yet. @@ -100,8 +100,9 @@ export function useAlertRulePreview( return useMemo(() => { if (!payload) { - // Unpreviewable state: surface the compile issue inline when there is one. - const issues = deriveRuleQueryIssues(deferredForm) + // No form yet (rule still resolving) is not an authoring error — stay clean. + // A present-but-unpreviewable form surfaces its compile issue inline. + const issues = deferredForm === null ? [] : deriveRuleQueryIssues(deferredForm) return { preview: null, previewLoading: false, previewError: issues[0] ?? null } } return Result.builder(result) diff --git a/apps/web/src/routes/alerts/$ruleId.tsx b/apps/web/src/routes/alerts/$ruleId.tsx index 46ca9df1b..1245fce0a 100644 --- a/apps/web/src/routes/alerts/$ruleId.tsx +++ b/apps/web/src/routes/alerts/$ruleId.tsx @@ -24,7 +24,6 @@ import { signalLabels, comparatorLabels, formatSignalValue, - defaultRuleForm, ruleToFormState, formatAlertDateTimeFull, formatAlertDuration, @@ -226,7 +225,9 @@ function RuleDetailContent() { [startTime, endTime], ) - const formState = useMemo(() => (rule ? ruleToFormState(rule) : defaultRuleForm()), [rule]) + // null until the rule resolves — keeps the chart from firing a throwaway preview + // for the default form (and flashing a wrong chart) before we know the real rule. + const formState = useMemo(() => (rule ? ruleToFormState(rule) : null), [rule]) const { preview, previewLoading, previewError } = useAlertRulePreview(formState, { startTime, endTime, @@ -244,9 +245,15 @@ function RuleDetailContent() { -
    - - + {/* Mirror the settled Overview rhythm so the first paint doesn't snap. */} +
    + +
    + + +
    + +
    ) @@ -422,7 +429,11 @@ function RuleDetailContent() { />
    - {overviewIncident ? ( + {/* Reserve the slot while incidents sync so the card doesn't pop in + and shove Configuration + Checks down once it resolves. */} + {Result.isInitial(incidentsResult) ? ( + + ) : overviewIncident ? (