Add minutely rollup for service operations

apps/api/src/routes/query-engine.http.ts

@@ -1511,8 +1511,11 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine",
 								startTime: payload.startTime,
 								endTime: payload.endTime,
 							}
+							// Keep production reads on the raw rollback path until the rollup has
+							// been deployed, backfilled under a write pause, and parity-verified on
+							// both managed Tinybird and every BYO ClickHouse backend.
 							const summaryCompiled = CH.compile(
-								CH.serviceOperationsSummaryQuery({
+								CH.serviceOperationsSummaryRawQuery({
 									serviceName: payload.serviceName,
 									environments: payload.environments,
 									limit: payload.limit,
@@ -1532,18 +1535,18 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine",
 							}
 
 							const spanNames = summaryRows.map((row) => String(row.spanName))
-							// Fallback bucket sizing mirrors the client's chart-grid density
-							// (~50 buckets), floored at 1 minute.
+							// The rollup is minute-grain, so every sparkline interval must be
+							// a whole-minute multiple. Nearest-minute rounding keeps ~50 points.
 							const windowSeconds = Math.max(
 								0,
 								(Date.parse(`${payload.endTime.replace(" ", "T")}Z`) -
 									Date.parse(`${payload.startTime.replace(" ", "T")}Z`)) /
 									1000,
 							)
-							const bucketSeconds =
-								payload.bucketSeconds ?? Math.max(60, Math.floor(windowSeconds / 50))
+							const requestedBucketSeconds = payload.bucketSeconds ?? windowSeconds / 50
+							const bucketSeconds = Math.max(1, Math.round(requestedBucketSeconds / 60)) * 60
 							const timeseriesCompiled = CH.compile(
-								CH.serviceOperationsTimeseriesQuery({
+								CH.serviceOperationsTimeseriesRawQuery({
 									serviceName: payload.serviceName,
 									environments: payload.environments,
 									spanNames,

apps/cli/src/server/schema/local-inserts.json

@@ -1,5 +1,5 @@
 {
-  "projectRevision": "ffade4bccc59af00fd33a561c4c919fd0229e0505f659d3242081f670f034a41",
+  "projectRevision": "c3f2be342187b6f6cf09010043e775b68f0901081a9b861803e55f5fe299e4c6",
   "orgPlaceholder": "__ORG__",
   "datasources": {
     "traces": {

apps/cli/src/server/schema/local-schema.sql

@@ -1,6 +1,6 @@
 -- This file is generated by scripts/generate-clickhouse-schema-sql.ts
 -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate.
--- projectRevision: ffade4bccc59af00fd33a561c4c919fd0229e0505f659d3242081f670f034a41
+-- projectRevision: c3f2be342187b6f6cf09010043e775b68f0901081a9b861803e55f5fe299e4c6
 
 CREATE TABLE IF NOT EXISTS alert_checks (
     OrgId LowCardinality(String),
@@ -431,6 +431,24 @@ PARTITION BY toDate(Timestamp)
 ORDER BY (OrgId, TraceId, SpanId, Timestamp)
 TTL Timestamp + INTERVAL 30 DAY;
 
+CREATE TABLE IF NOT EXISTS service_operations_minutely (
+    OrgId LowCardinality(String),
+    Minute DateTime,
+    ServiceName LowCardinality(String),
+    DeploymentEnv LowCardinality(String),
+    SpanName String,
+    SpanCount SimpleAggregateFunction(sum, UInt64),
+    EstimatedSpanCount SimpleAggregateFunction(sum, Float64),
+    ErrorCount SimpleAggregateFunction(sum, UInt64),
+    EstimatedErrorCount SimpleAggregateFunction(sum, Float64),
+    DurationSum SimpleAggregateFunction(sum, Float64),
+    DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64)
+)
+ENGINE = AggregatingMergeTree
+PARTITION BY toDate(Minute)
+ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName)
+TTL toDate(Minute) + INTERVAL 90 DAY;
+
 CREATE TABLE IF NOT EXISTS service_overview_spans (
     OrgId LowCardinality(String),
     Timestamp DateTime,
@@ -1153,6 +1171,22 @@ SELECT
         FROM traces
         WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer');
 
+CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS
+SELECT
+          OrgId,
+          toStartOfMinute(toDateTime(Timestamp)) AS Minute,
+          ServiceName,
+          ResourceAttributes['deployment.environment'] AS DeploymentEnv,
+          if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName,
+          count() AS SpanCount,
+          sum(SampleRate) AS EstimatedSpanCount,
+          countIf(StatusCode = 'Error') AS ErrorCount,
+          sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,
+          sum(toFloat64(Duration)) AS DurationSum,
+          quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles
+        FROM traces
+        GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName;
+
 CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS
 SELECT
           OrgId,

apps/ingest/src/clickhouse_insert_mappings.rs

@@ -1,12 +1,12 @@
 // This file is generated by scripts/generate-clickhouse-insert-mappings.ts
 // Do not edit manually.
 
-pub const PROJECT_REVISION: &str = "ffade4bccc59af00fd33a561c4c919fd0229e0505f659d3242081f670f034a41";
+pub const PROJECT_REVISION: &str = "c3f2be342187b6f6cf09010043e775b68f0901081a9b861803e55f5fe299e4c6";
 // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the
 // Tinybird-coupled PROJECT_REVISION. Compared against
 // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse
 // clickHouseSchemaVersion.
-pub const SCHEMA_VERSION: &str = "7";
+pub const SCHEMA_VERSION: &str = "8";
 pub const ORG_PLACEHOLDER: &str = "__ORG__";
 
 #[derive(Debug)]

apps/web/src/components/services/service-operations.test.ts

@@ -24,7 +24,13 @@ describe("windowSeconds", () => {
 
 describe("operationsBucketSeconds", () => {
 	it("targets ~50 buckets across the window", () => {
-		expect(operationsBucketSeconds("2024-01-01 00:00:00", "2024-01-01 12:00:00")).toBe(864)
+		expect(operationsBucketSeconds("2024-01-01 00:00:00", "2024-01-01 12:00:00")).toBe(840)
+	})
+
+	it("always rounds to a whole-minute multiple", () => {
+		const bucket = operationsBucketSeconds("2024-01-01 00:00:00", "2024-01-02 00:00:00")
+		expect(bucket).toBe(1740)
+		expect(bucket % 60).toBe(0)
 	})
 
 	it("floors at one minute for short windows", () => {
@@ -55,7 +61,7 @@ describe("serviceOperationsQueryInput", () => {
 			startTime: "2024-01-01 00:00:00",
 			endTime: "2024-01-01 12:00:00",
 			environments: ["production"],
-			bucketSeconds: 864,
+			bucketSeconds: 840,
 			limit: 25,
 		})
 	})

apps/web/src/components/services/service-operations.ts

@@ -1,7 +1,7 @@
 import type { GetServiceOperationsInput } from "@/api/warehouse/service-operations"
 import { normalizeTimestampInput } from "@/lib/timezone-format"
 
-/** Matches the chart grid's density: ~50 buckets across the window, ≥1 minute. */
+/** Matches the chart grid's density: ~50 whole-minute buckets, ≥1 minute. */
 export const OPERATIONS_SPARKLINE_BUCKETS = 50
 export const OPERATIONS_LIMIT = 25
 
@@ -13,7 +13,8 @@ export function windowSeconds(startTime: string, endTime: string): number {
 }
 
 export function operationsBucketSeconds(startTime: string, endTime: string): number {
-	return Math.max(60, Math.floor(windowSeconds(startTime, endTime) / OPERATIONS_SPARKLINE_BUCKETS))
+	const targetMinutes = windowSeconds(startTime, endTime) / OPERATIONS_SPARKLINE_BUCKETS / 60
+	return Math.max(1, Math.round(targetMinutes)) * 60
 }
 
 export function callsPerSecond(estimatedSpanCount: number, seconds: number): number {

packages/domain/package.json

@@ -17,6 +17,7 @@
 		"./warehouse-queries": "./src/warehouse-queries.ts",
 		"./tinybird": "./src/tinybird/index.ts",
 		"./tinybird/db-query-shape-sql": "./src/tinybird/db-query-shape-sql.ts",
+		"./tinybird/span-display-name": "./src/tinybird/span-display-name.ts",
 		"./clickhouse": "./src/clickhouse/index.ts",
 		"./where-clause": "./src/where-clause.ts",
 		"./http/observability": "./src/http/observability.ts",
@@ -27,6 +28,7 @@
 		"typecheck": "tsc --noEmit"
 	},
 	"dependencies": {
+		"@maple-dev/clickhouse-builder": "workspace:*",
 		"@tinybirdco/sdk": "catalog:tinybird",
 		"effect": "catalog:effect"
 	},

packages/domain/src/clickhouse/ddl-emitter.test.ts

@@ -100,6 +100,27 @@ describe("ClickHouse DDL emitter", () => {
 		const aggDdl = emitCreateTable(aggDs!, { engineFlavor: "ReplicatedMergeTree" })
 		expect(aggDdl).toContain("ENGINE = AggregatingMergeTree")
 	})
+
+	it("emits the service-operation rollup key, lifecycle, and MV projection", async () => {
+		const manifest = await buildTinybirdProjectManifest()
+		const rollup = manifest.datasources.find((ds) => ds.name === "service_operations_minutely")
+		const mv = manifest.pipes.find((pipe) => pipe.name === "service_operations_minutely_mv")
+		expect(rollup).toBeDefined()
+		expect(mv).toBeDefined()
+
+		const tableDdl = emitCreateTable(rollup!)
+		expect(tableDdl).toContain("ENGINE = AggregatingMergeTree")
+		expect(tableDdl).toContain("PARTITION BY toDate(Minute)")
+		expect(tableDdl).toContain("ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName)")
+		expect(tableDdl).toContain("TTL toDate(Minute) + INTERVAL 90 DAY")
+		expect(tableDdl).toContain("DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64)")
+
+		const mvDdl = emitCreateMaterializedView(mv!)
+		expect(mvDdl).toContain("FROM traces")
+		expect(mvDdl).toContain("toStartOfMinute(toDateTime(Timestamp)) AS Minute")
+		expect(mvDdl).toContain("http.route")
+		expect(mvDdl).toContain("GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName")
+	})
 })
 
 describe("extractColumnDefinition", () => {

packages/domain/src/clickhouse/migrations/0008_service_operations_minutely.ts

@@ -0,0 +1,84 @@
+import { NORMALIZED_SPAN_NAME_SQL } from "../../tinybird/span-display-name"
+import type { BackfillSpec } from "../backfill"
+
+/**
+ * Migration 0008 — minutely service-operation rollup.
+ *
+ * The migration installs the empty target and its live-write MV only. Historical
+ * backfill is deliberately a separate, chunkable operation: customer collectors
+ * can write directly to BYO ClickHouse during schema apply, so an automatic
+ * backfill/MV cutover cannot be gap-free. Rollout must pause writes, truncate the
+ * target, execute {@link serviceOperationsMinutelyBackfill}, verify parity, and
+ * then resume writes before enabling hybrid reads.
+ */
+export const serviceOperationsMinutelyBackfill: BackfillSpec = {
+	kind: "backfill",
+	target: "service_operations_minutely",
+	columns: [
+		"OrgId",
+		"Minute",
+		"ServiceName",
+		"DeploymentEnv",
+		"SpanName",
+		"SpanCount",
+		"EstimatedSpanCount",
+		"ErrorCount",
+		"EstimatedErrorCount",
+		"DurationSum",
+		"DurationQuantiles",
+	],
+	from: "traces",
+	tsColumn: "Timestamp",
+	select: `OrgId,
+  toStartOfMinute(toDateTime(Timestamp)) AS Minute,
+  ServiceName,
+  ResourceAttributes['deployment.environment'] AS DeploymentEnv,
+  ${NORMALIZED_SPAN_NAME_SQL} AS SpanName,
+  count() AS SpanCount,
+  sum(SampleRate) AS EstimatedSpanCount,
+  countIf(StatusCode = 'Error') AS ErrorCount,
+  sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,
+  sum(toFloat64(Duration)) AS DurationSum,
+  quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles`,
+	groupBy: "OrgId, Minute, ServiceName, DeploymentEnv, SpanName",
+}
+
+export const migration_0008_service_operations_minutely = {
+	version: 8,
+	description: "Add the minutely service-operation rollup and its live-write materialized view",
+	statements: [
+		"DROP VIEW IF EXISTS service_operations_minutely_mv",
+		`CREATE TABLE IF NOT EXISTS service_operations_minutely (
+  OrgId LowCardinality(String),
+  Minute DateTime,
+  ServiceName LowCardinality(String),
+  DeploymentEnv LowCardinality(String),
+  SpanName String,
+  SpanCount SimpleAggregateFunction(sum, UInt64),
+  EstimatedSpanCount SimpleAggregateFunction(sum, Float64),
+  ErrorCount SimpleAggregateFunction(sum, UInt64),
+  EstimatedErrorCount SimpleAggregateFunction(sum, Float64),
+  DurationSum SimpleAggregateFunction(sum, Float64),
+  DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64)
+)
+ENGINE = AggregatingMergeTree
+PARTITION BY toDate(Minute)
+ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName)
+TTL toDate(Minute) + INTERVAL 90 DAY`,
+		`CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS
+SELECT
+  OrgId,
+  toStartOfMinute(toDateTime(Timestamp)) AS Minute,
+  ServiceName,
+  ResourceAttributes['deployment.environment'] AS DeploymentEnv,
+  ${NORMALIZED_SPAN_NAME_SQL} AS SpanName,
+  count() AS SpanCount,
+  sum(SampleRate) AS EstimatedSpanCount,
+  countIf(StatusCode = 'Error') AS ErrorCount,
+  sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,
+  sum(toFloat64(Duration)) AS DurationSum,
+  quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles
+FROM traces
+GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName`,
+	],
+} as const

packages/domain/src/clickhouse/migrations/index.test.ts

@@ -3,7 +3,10 @@ import { isBackfill, renderStatementFull, type BackfillSpec } from "../backfill"
 import { migration_0004_service_namespace_projections } from "./0004_service_namespace_projections"
 import { migration_0005_alert_checks_error_columns } from "./0005_alert_checks_error_columns"
 import { migration_0006_db_edge_namespace } from "./0006_db_edge_namespace"
-import { migration_0007_db_namespace_hyperdrive } from "./0007_db_namespace_hyperdrive"
+import {
+	migration_0008_service_operations_minutely,
+	serviceOperationsMinutelyBackfill,
+} from "./0008_service_operations_minutely"
 import { migrations } from "./index"
 
 const backfills = migration_0004_service_namespace_projections.statements.filter(
@@ -18,8 +21,29 @@ const renderedSql = migration_0004_service_namespace_projections.statements
 
 describe("ClickHouse migrations", () => {
 	it("keeps migrations ordered by version", () => {
-		expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7])
-		expect(migrations.at(-1)).toBe(migration_0007_db_namespace_hyperdrive)
+		expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8])
+		expect(migrations.at(-1)).toBe(migration_0008_service_operations_minutely)
+	})
+
+	it("adds the service-operation rollup and exposes a coordinated chunkable backfill", () => {
+		const statements = migration_0008_service_operations_minutely.statements
+		const sql = statements.map((statement) => renderStatementFull(statement, "default")).join("\n\n")
+
+		expect(sql).toContain("PARTITION BY toDate(Minute)")
+		expect(sql).toContain("ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName)")
+		expect(sql).toContain("TTL toDate(Minute) + INTERVAL 90 DAY")
+		expect(sql).toContain("SpanName String")
+		expect(sql).toContain("quantilesTDigestState(0.5, 0.95)(Duration)")
+		expect(sql).toContain("http.route")
+		expect(sql).toContain("http.server %")
+		expect(statements.filter(isBackfill)).toHaveLength(0)
+		expect(sql).not.toContain("TRUNCATE TABLE service_operations_minutely")
+		expect(serviceOperationsMinutelyBackfill.target).toBe("service_operations_minutely")
+		expect(serviceOperationsMinutelyBackfill.from).toBe("traces")
+		expect(serviceOperationsMinutelyBackfill.tsColumn).toBe("Timestamp")
+		expect(serviceOperationsMinutelyBackfill.groupBy).toBe(
+			"OrgId, Minute, ServiceName, DeploymentEnv, SpanName",
+		)
 	})
 
 	it("adds alert_checks error columns as idempotent ALTERs", () => {

packages/domain/src/clickhouse/migrations/index.ts

@@ -6,6 +6,7 @@ import { migration_0004_service_namespace_projections } from "./0004_service_nam
 import { migration_0005_alert_checks_error_columns } from "./0005_alert_checks_error_columns"
 import { migration_0006_db_edge_namespace } from "./0006_db_edge_namespace"
 import { migration_0007_db_namespace_hyperdrive } from "./0007_db_namespace_hyperdrive"
+import { migration_0008_service_operations_minutely } from "./0008_service_operations_minutely"
 
 /**
  * A migration statement is either a raw SQL string (structural DDL) or a
@@ -42,6 +43,7 @@ export const migrations: ReadonlyArray<ClickHouseMigration> = [
 	migration_0005_alert_checks_error_columns,
 	migration_0006_db_edge_namespace,
 	migration_0007_db_namespace_hyperdrive,
+	migration_0008_service_operations_minutely,
 ] as const
 
 /** Highest migration `version` bundled — i.e. the schema level a fully-applied