diff --git a/.env.example b/.env.example index a98910cb3..86c22df3a 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,18 @@ # Tinybird TINYBIRD_HOST=http://localhost:7181 TINYBIRD_TOKEN=your-tinybird-token +# Required only when raw SQL is routed to Tinybird. These values mint scoped, +# per-org read JWTs and are intentionally independent from the API token. +# TINYBIRD_SIGNING_KEY=your-tinybird-jwt-signing-key +# TINYBIRD_WORKSPACE_ID=your-tinybird-workspace-id # ClickHouse CLICKHOUSE_URL=http://localhost:9000 +# tinybird is the default for compatibility with existing gateway deployments. +# Set clickhouse when CLICKHOUSE_URL points at a vanilla/self-managed server so +# raw SQL preserves CLICKHOUSE_PASSWORD instead of substituting a scoped JWT. +# Env-level vanilla ClickHouse raw SQL is allowed only in single-org self-hosted mode. +CLICKHOUSE_PROVIDER=clickhouse CLICKHOUSE_USER=maple CLICKHOUSE_DATABASE=default CLICKHOUSE_PASSWORD=maple @@ -175,4 +184,4 @@ ELECTRIC_URL=http://localhost:3473 # Flue chat model MAPLE_CHAT_MODEL=openrouter/z-ai/glm-5.2 -INTERNAL_SERVICE_TOKEN=dev-internal-service-token \ No newline at end of file +INTERNAL_SERVICE_TOKEN=dev-internal-service-token diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 715f128eb..e66ca955b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,6 +82,7 @@ These values align with `docker-compose.development.yml` and self-hosted auth: ```dotenv # Warehouse — route API queries to local ClickHouse instead of Tinybird Cloud CLICKHOUSE_URL=http://localhost:8123 +CLICKHOUSE_PROVIDER=clickhouse CLICKHOUSE_USER=maple CLICKHOUSE_PASSWORD=maple CLICKHOUSE_DATABASE=default @@ -186,7 +187,9 @@ Default URL: `http://localhost:3472` | `MAPLE_DEFAULT_ORG_ID` | yes | Default `default`; must match ingest org override | | `TINYBIRD_HOST` | yes | Placeholder OK when using `CLICKHOUSE_URL` | | `TINYBIRD_TOKEN` | yes | Placeholder OK when using `CLICKHOUSE_URL` | +| `TINYBIRD_SIGNING_KEY` / `TINYBIRD_WORKSPACE_ID` | with Tinybird raw SQL | Explicit JWT signing configuration; never derived from `TINYBIRD_TOKEN` | | `CLICKHOUSE_URL` | recommended | `http://localhost:8123` for local ClickHouse stack | +| `CLICKHOUSE_PROVIDER` | optional | `tinybird` (default) or `clickhouse`; set `clickhouse` for env-level vanilla/self-managed ClickHouse | | `CLICKHOUSE_USER` / `CLICKHOUSE_PASSWORD` / `CLICKHOUSE_DATABASE` | with CH | Match docker-compose (`maple` / `maple` / `default`) | | `INTERNAL_SERVICE_TOKEN` | recommended | Shared with scraper + chat-flue | | `SD_INTERNAL_TOKEN` | optional | Prometheus scraper internal API auth | diff --git a/apps/alerting/src/worker.ts b/apps/alerting/src/worker.ts index b0b9043ea..03cecea26 100644 --- a/apps/alerting/src/worker.ts +++ b/apps/alerting/src/worker.ts @@ -24,6 +24,7 @@ import { PlanetScaleService, QueryEngineService, ServiceMapRollupService, + TinybirdOrgTokenService, WarehouseQueryService, } from "@maple/api/alerting" import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" @@ -50,8 +51,10 @@ const buildLayer = (_env: Record) => { const OrgClickHouseSettingsLive = OrgClickHouseSettingsService.layer.pipe(Layer.provide(BaseLive)) + const TinybirdOrgTokenLive = TinybirdOrgTokenService.layer.pipe(Layer.provide(EnvLive)) + const WarehouseQueryServiceLive = WarehouseQueryService.layer.pipe( - Layer.provide(Layer.mergeAll(EnvLive, OrgClickHouseSettingsLive)), + Layer.provide(Layer.mergeAll(EnvLive, OrgClickHouseSettingsLive, TinybirdOrgTokenLive)), ) // EdgeCacheService's storage backend is injected via the CacheBackend port. diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index 0e0bc7246..9589db6e7 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -7,6 +7,7 @@ import type { MapleApiRpcShape } from "@maple/domain/internal-rpc" import type { MapleDomains, MapleStage } from "@maple/infra/cloudflare" import { CLOUDFLARE_WORKER_PLACEMENT, + formatMapleStage, resolveDeploymentEnvironment, resolveHyperdriveName, resolveHyperdriveRefId, @@ -137,12 +138,20 @@ export const createMapleApi = ({ stage, domains }: CreateMapleApiOptions) => VCS_SYNC_QUEUE: vcsSyncQueue, CLICKHOUSE_SCHEMA_APPLY_WORKFLOW: schemaApplyWorkflow, AI_TRIAGE_WORKFLOW: aiTriageWorkflow, + API_V2_RATE_LIMITER: Cloudflare.RateLimit("API_V2_RATE_LIMITER", { + namespaceId: 2026071801, + simple: { limit: 600, period: 60 }, + }), + API_V2_RATE_LIMIT_PARTITION: formatMapleStage(stage), EMAIL: Cloudflare.Email.SendEmail("email", { allowedSenderAddresses: ["notifications@noreply.maple.dev"], }), TINYBIRD_HOST: requireEnv("TINYBIRD_HOST"), TINYBIRD_TOKEN: Redacted.make(requireEnv("TINYBIRD_TOKEN")), + ...optionalSecret("TINYBIRD_SIGNING_KEY"), + ...optionalPlain("TINYBIRD_WORKSPACE_ID"), ...optionalPlain("CLICKHOUSE_URL"), + CLICKHOUSE_PROVIDER: process.env.CLICKHOUSE_PROVIDER?.trim() || "tinybird", ...optionalPlain("CLICKHOUSE_USER"), ...optionalPlain("CLICKHOUSE_DATABASE"), ...optionalSecret("CLICKHOUSE_PASSWORD"), diff --git a/apps/api/src/alerting.ts b/apps/api/src/alerting.ts index 469509c5f..3f7fe7102 100644 --- a/apps/api/src/alerting.ts +++ b/apps/api/src/alerting.ts @@ -25,5 +25,6 @@ export { PlanetScaleOAuthService } from "./services/PlanetScaleOAuthService" export { PlanetScaleService } from "./services/PlanetScaleService" export { QueryEngineService } from "./services/QueryEngineService" export { ServiceMapRollupService } from "./services/ServiceMapRollupService" +export { TinybirdOrgTokenService } from "./services/TinybirdOrgTokenService" export { WarehouseQueryService } from "./lib/WarehouseQueryService" export { WorkerEnvironment } from "./lib/WorkerEnvironment" diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 79b95126e..fe465348b 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -23,6 +23,14 @@ import { HttpV2OrganizationLive } from "./routes/v2/organization.http" import { HttpV2InstrumentationRecommendationsLive } from "./routes/v2/recommendations.http" import { HttpV2ScrapeTargetsLive } from "./routes/v2/scrape-targets.http" import { HttpV2SessionReplaysLive } from "./routes/v2/session-replays.http" +import { + HttpV2LogsLive, + HttpV2MetricsLive, + HttpV2QueryLive, + HttpV2ServiceMapLive, + HttpV2ServicesLive, + HttpV2TracesLive, +} from "./routes/v2/telemetry.http" import { V2SchemaErrorsLive } from "./routes/v2/error-envelope" import { HttpAuthLive, HttpAuthPublicLive } from "./routes/auth.http" import { HttpChatLive } from "./routes/chat.http" @@ -57,6 +65,7 @@ import { HazelOAuthService } from "./services/HazelOAuthService" import { InvestigationService } from "./services/InvestigationService" import { NotificationDispatcher } from "./services/NotificationDispatcher" import { ApiKeysService } from "./services/ApiKeysService" +import { ApiV2RateLimiter } from "./services/ApiV2RateLimiter" import { AuthService } from "./services/AuthService" import { ApiAuthorizationLayer } from "./services/ApiAuthorizationLayer" import { ApiAuthorizationV2Layer } from "./services/ApiAuthorizationV2Layer" @@ -72,10 +81,10 @@ import { Env } from "./lib/Env" import { IngestAttributeMappingService } from "./services/IngestAttributeMappingService" import { OrgIngestKeysService } from "./services/OrgIngestKeysService" import { OrgClickHouseSettingsService } from "./services/OrgClickHouseSettingsService" +import { TinybirdOrgTokenService } from "./services/TinybirdOrgTokenService" import { OrganizationService } from "./services/OrganizationService" import { QueryEngineService } from "./services/QueryEngineService" import { RecommendationIssueService } from "./services/RecommendationIssueService" -import { RawSqlChartService } from "@maple/query-engine/runtime" import { PlanetScaleConnectionService } from "./services/PlanetScaleConnectionService" import { PlanetScaleDiscoveryService } from "./services/PlanetScaleDiscoveryService" import { PlanetScaleOAuthService } from "./services/PlanetScaleOAuthService" @@ -91,6 +100,7 @@ import { VcsCommitService } from "./services/vcs/VcsCommitService" import { VcsProviderRegistry } from "./services/vcs/VcsProviderRegistry" import { VcsRepository } from "./services/vcs/VcsRepository" import { VcsSyncQueue } from "./services/vcs/VcsSyncQueue" +import { API_CORS_OPTIONS } from "./lib/api-cors" const HealthRouter = HttpRouter.use((router) => router.add("GET", "/health", HttpServerResponse.text("OK"))) @@ -133,6 +143,7 @@ const CoreServicesLive = Layer.mergeAll( OnboardingService.layer, OrgIngestKeysService.layer, OrgClickHouseSettingsService.layer, + TinybirdOrgTokenService.layer, OrganizationService.layer, PlanetScaleOAuthLive, PlanetScaleDiscoveryLive, @@ -253,7 +264,6 @@ export const MainLive = Layer.mergeAll( DigestServiceLive, DemoServiceLive, VcsServicesLive, - RawSqlChartService.layer, ) const ApiRoutes = HttpApiBuilder.layer(MapleApi).pipe( @@ -301,6 +311,12 @@ const ApiV2Routes = HttpApiBuilder.layer(MapleApiV2).pipe( HttpV2AnomaliesLive, HttpV2OrganizationLive, HttpV2SessionReplaysLive, + HttpV2TracesLive, + HttpV2LogsLive, + HttpV2MetricsLive, + HttpV2ServicesLive, + HttpV2ServiceMapLive, + HttpV2QueryLive, ), ), Layer.provide(V2SchemaErrorsLive), @@ -320,20 +336,10 @@ export const AllRoutes = Layer.mergeAll( McpGetFallback, DocsRoute, DocsV2Route, -).pipe( - Layer.provideMerge( - HttpRouter.cors({ - allowedOrigins: ["*"], - allowedMethods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], - allowedHeaders: ["*"], - // The ElectricSQL shape proxy (and its electric-* exposed headers) moved - // to the standalone `apps/electric-sync` worker. - exposedHeaders: ["Mcp-Session-Id"], - }), - ), -) +).pipe(Layer.provideMerge(HttpRouter.cors(API_CORS_OPTIONS))) export const ApiAuthLive = Layer.mergeAll(ApiAuthorizationLayer, ApiAuthorizationV2Layer).pipe( + Layer.provideMerge(ApiV2RateLimiter.layer), Layer.provideMerge(ApiKeysService.layer), Layer.provideMerge(Env.layer), ) diff --git a/apps/api/src/lib/Env.ts b/apps/api/src/lib/Env.ts index d7aed71de..9a06eff8f 100644 --- a/apps/api/src/lib/Env.ts +++ b/apps/api/src/lib/Env.ts @@ -11,7 +11,10 @@ export interface EnvShape { readonly PORT: number readonly TINYBIRD_HOST: string readonly TINYBIRD_TOKEN: Redacted.Redacted + readonly TINYBIRD_SIGNING_KEY: Option.Option> + readonly TINYBIRD_WORKSPACE_ID: Option.Option readonly CLICKHOUSE_URL: Option.Option + readonly CLICKHOUSE_PROVIDER: string readonly CLICKHOUSE_USER: string readonly CLICKHOUSE_PASSWORD: Option.Option> readonly CLICKHOUSE_DATABASE: string @@ -82,7 +85,10 @@ const envConfig = Config.all({ PORT: portConfig, TINYBIRD_HOST: Config.string("TINYBIRD_HOST"), TINYBIRD_TOKEN: Config.redacted("TINYBIRD_TOKEN"), + TINYBIRD_SIGNING_KEY: optionalRedacted("TINYBIRD_SIGNING_KEY"), + TINYBIRD_WORKSPACE_ID: optionalString("TINYBIRD_WORKSPACE_ID"), CLICKHOUSE_URL: optionalString("CLICKHOUSE_URL"), + CLICKHOUSE_PROVIDER: stringWithDefault("CLICKHOUSE_PROVIDER", "tinybird"), CLICKHOUSE_USER: stringWithDefault("CLICKHOUSE_USER", "default"), CLICKHOUSE_PASSWORD: optionalRedacted("CLICKHOUSE_PASSWORD"), CLICKHOUSE_DATABASE: stringWithDefault("CLICKHOUSE_DATABASE", "default"), @@ -212,6 +218,14 @@ const makeEnv = Effect.gen(function* () { return yield* Effect.die(new EnvValidationError({ message: "MAPLE_DEFAULT_ORG_ID cannot be empty" })) } + if (env.CLICKHOUSE_PROVIDER !== "clickhouse" && env.CLICKHOUSE_PROVIDER !== "tinybird") { + return yield* Effect.die( + new EnvValidationError({ + message: "CLICKHOUSE_PROVIDER must be either 'clickhouse' or 'tinybird'", + }), + ) + } + const authMode = env.MAPLE_AUTH_MODE.toLowerCase() if (authMode !== "clerk" && Option.isNone(env.MAPLE_ROOT_PASSWORD)) { diff --git a/apps/api/src/lib/WarehouseQueryService.clickhouse.e2e.test.ts b/apps/api/src/lib/WarehouseQueryService.clickhouse.e2e.test.ts new file mode 100644 index 000000000..15d1076f8 --- /dev/null +++ b/apps/api/src/lib/WarehouseQueryService.clickhouse.e2e.test.ts @@ -0,0 +1,208 @@ +import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" +import { spawn } from "node:child_process" +import { ConfigProvider, Effect, Layer, Schema } from "effect" +import { OrgId, RawSqlValidationError, UserId } from "@maple/domain/http" +import { prepareRawSql } from "@maple/query-engine/runtime" +import { OrgClickHouseSettingsService } from "../services/OrgClickHouseSettingsService" +import { TinybirdOrgTokenService } from "../services/TinybirdOrgTokenService" +import type { TenantContext } from "../services/AuthService" +import { Env } from "./Env" +import { cleanupTestDbs, createTestDb, type TestDb } from "./test-pglite" +import { WarehouseQueryService } from "./WarehouseQueryService" + +const enabled = process.env.CLICKHOUSE_E2E === "1" +const clickhouseUrl = process.env.CLICKHOUSE_E2E_URL ?? "http://127.0.0.1:8123" +const clickhouseUser = process.env.CLICKHOUSE_E2E_USER ?? "maple" +const clickhousePassword = process.env.CLICKHOUSE_E2E_PASSWORD ?? "maple" +const database = `maple_raw_sql_e2e_${Date.now()}_${Math.random().toString(36).slice(2, 8)}` +const orgId = "org_raw_sql_e2e" +const repoRoot = new URL("../../../..", import.meta.url).pathname + +const clickhouseExec = async (sql: string, targetDatabase = "default"): Promise => { + const response = await fetch( + `${clickhouseUrl.replace(/\/$/, "")}/?database=${encodeURIComponent(targetDatabase)}`, + { + method: "POST", + redirect: "manual", + headers: { + "Content-Type": "text/plain", + "X-ClickHouse-User": clickhouseUser, + "X-ClickHouse-Key": clickhousePassword, + "X-ClickHouse-Database": targetDatabase, + }, + body: sql, + }, + ) + const body = await response.text() + if (!response.ok) throw new Error(`ClickHouse ${response.status}: ${body.slice(0, 500)}`) + return body +} + +const applyRealMigrations = async (): Promise => { + const child = spawn( + "bun", + [ + "run", + "--cwd", + "packages/clickhouse-cli", + "start", + "apply", + `--url=${clickhouseUrl}`, + `--user=${clickhouseUser}`, + `--password=${clickhousePassword}`, + `--database=${database}`, + ], + { cwd: repoRoot, stdio: ["ignore", "pipe", "pipe"] }, + ) + let stdout = "" + let stderr = "" + child.stdout.setEncoding("utf8").on("data", (chunk: string) => { + stdout += chunk + }) + child.stderr.setEncoding("utf8").on("data", (chunk: string) => { + stderr += chunk + }) + const exitCode = await new Promise((resolve, reject) => { + child.once("error", reject) + child.once("close", (code) => resolve(code ?? 1)) + }) + if (exitCode !== 0) throw new Error(`Migration CLI failed (${exitCode}): ${stderr || stdout}`) +} + +const trackedDbs: TestDb[] = [] +const asOrgId = Schema.decodeUnknownSync(OrgId) +const asUserId = Schema.decodeUnknownSync(UserId) +const tenant: TenantContext = { + orgId: asOrgId(orgId), + userId: asUserId("user_raw_sql_e2e"), + roles: [], + authMode: "self_hosted", +} + +const buildLayer = () => { + const testDb = createTestDb(trackedDbs) + const configLive = ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3472", + TINYBIRD_HOST: "http://127.0.0.1:7181", + TINYBIRD_TOKEN: "unused-for-vanilla-clickhouse", + CLICKHOUSE_URL: clickhouseUrl, + CLICKHOUSE_PROVIDER: "clickhouse", + CLICKHOUSE_USER: clickhouseUser, + CLICKHOUSE_PASSWORD: clickhousePassword, + CLICKHOUSE_DATABASE: database, + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: orgId, + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 9).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "raw-sql-e2e-lookup-key", + MAPLE_INGEST_PUBLIC_URL: "http://127.0.0.1:3474", + MAPLE_APP_BASE_URL: "http://127.0.0.1:3471", + }), + ) + const envLive = Env.layer.pipe(Layer.provide(configLive)) + const orgSettingsLive = OrgClickHouseSettingsService.layer.pipe( + Layer.provide(Layer.mergeAll(envLive, testDb.layer)), + ) + const tokensLive = TinybirdOrgTokenService.layer.pipe(Layer.provide(envLive)) + return WarehouseQueryService.layer.pipe( + Layer.provide(Layer.mergeAll(envLive, orgSettingsLive, tokensLive)), + ) +} + +const expand = (sql: string) => + prepareRawSql({ + sql, + orgId, + startTime: "2026-01-01 00:00:00", + endTime: "2026-01-01 01:00:00", + granularitySeconds: 60, + workload: "interactive", + }) + +describe.skipIf(!enabled)("WarehouseQueryService ClickHouse raw-SQL E2E", () => { + beforeAll(async () => { + await clickhouseExec(`CREATE DATABASE ${database}`) + await applyRealMigrations() + await clickhouseExec( + `INSERT INTO traces + (OrgId, Timestamp, TraceId, SpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode) + VALUES + ('${orgId}', now64(9), 'trace-a', 'span-a', 'GET /a', 'Server', 'api', 1000, 'Ok'), + ('${orgId}', now64(9), 'trace-b', 'span-b', 'GET /b', 'Server', 'worker', 2000, 'Error')`, + database, + ) + }, 120_000) + + afterAll(async () => { + await cleanupTestDbs(trackedDbs) + await clickhouseExec(`DROP DATABASE IF EXISTS ${database}`) + }, 30_000) + + it.effect( + "uses the configured ClickHouse password for raw SQL", + () => { + const layer = buildLayer() + return Effect.gen(function* () { + const query = yield* expand( + "SELECT TraceId, ServiceName FROM traces WHERE $__orgFilter ORDER BY TraceId", + ) + const rows = yield* WarehouseQueryService.use((service) => + service.rawSqlQuery(tenant, query.sql, { + profile: "rawInteractive", + context: "clickhouse.e2e.fixture", + }), + ) + assert.deepStrictEqual(rows, [ + { TraceId: "trace-a", ServiceName: "api" }, + { TraceId: "trace-b", ServiceName: "worker" }, + ]) + }).pipe(Effect.provide(layer)) + }, + 120_000, + ) + + it.effect( + "accepts exactly 1,000 rows and aborts on the 1,001st streamed row", + () => { + const layer = buildLayer() + return Effect.gen(function* () { + const exact = yield* expand( + `SELECT number, '${orgId}' AS OrgId FROM numbers(1000) WHERE $__orgFilter`, + ) + const rows = yield* WarehouseQueryService.use((service) => + service.rawSqlQuery(tenant, exact.sql), + ) + assert.strictEqual(rows.length, 1000) + + const overflow = yield* expand( + `SELECT number, '${orgId}' AS OrgId FROM numbers(50000) WHERE $__orgFilter LIMIT 50000`, + ) + const error = yield* Effect.flip( + WarehouseQueryService.use((service) => service.rawSqlQuery(tenant, overflow.sql)), + ) + assert.instanceOf(error, RawSqlValidationError) + assert.strictEqual(error.code, "ResourceLimit") + }).pipe(Effect.provide(layer)) + }, + 120_000, + ) + + it.effect( + "aborts a streamed JSONEachRow response above five megabytes", + () => { + const layer = buildLayer() + return Effect.gen(function* () { + const query = yield* expand( + `SELECT number, repeat('x', 10000) AS payload, '${orgId}' AS OrgId FROM numbers(600) WHERE $__orgFilter`, + ) + const error = yield* Effect.flip( + WarehouseQueryService.use((service) => service.rawSqlQuery(tenant, query.sql)), + ) + assert.instanceOf(error, RawSqlValidationError) + assert.match(error.message, /5000000 encoded bytes/) + }).pipe(Effect.provide(layer)) + }, + 120_000, + ) +}) diff --git a/apps/api/src/lib/WarehouseQueryService.test.ts b/apps/api/src/lib/WarehouseQueryService.test.ts index b8db9f76c..bb0b46135 100644 --- a/apps/api/src/lib/WarehouseQueryService.test.ts +++ b/apps/api/src/lib/WarehouseQueryService.test.ts @@ -2,15 +2,21 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { Cause, ConfigProvider, Effect, Exit, Layer, Option, Schema } from "effect" import { WarehouseQueryError, + WarehouseConfigError, + MAX_RAW_SQL_RESULT_BYTES, WarehouseSchemaDriftError, WarehouseUpstreamError, OrgId, UserId, } from "@maple/domain/http" import { unsafeCompiledQuery } from "@maple/query-engine/ch" -import { makeWarehouseExecutor } from "@maple/query-engine/execution" +import { makeWarehouseExecutor, type ResolvedWarehouseConfig } from "@maple/query-engine/execution" import { __testables, WarehouseQueryService } from "./WarehouseQueryService" -import { OrgClickHouseSettingsService } from "../services/OrgClickHouseSettingsService" +import { + OrgClickHouseSettingsService, + type OrgClickHouseSettingsServiceShape, +} from "../services/OrgClickHouseSettingsService" +import { TinybirdOrgTokenService } from "../services/TinybirdOrgTokenService" import type { TenantContext } from "../services/AuthService" import { Env } from "./Env" import { cleanupTestDbs, createTestDb, type TestDb } from "./test-pglite" @@ -22,12 +28,18 @@ afterEach(async () => { await cleanupTestDbs(trackedDbs) }) -const makeConfig = (extra: Record = {}) => +const makeConfig = (extra: Record = {}, includeTinybirdSigning = true) => ConfigProvider.layer( ConfigProvider.fromUnknown({ PORT: "3472", TINYBIRD_HOST: "https://maple-managed.tinybird.co", TINYBIRD_TOKEN: "managed-token", + ...(includeTinybirdSigning + ? { + TINYBIRD_SIGNING_KEY: "test-signing-key", + TINYBIRD_WORKSPACE_ID: "test-workspace", + } + : {}), MAPLE_AUTH_MODE: "self_hosted", MAPLE_ROOT_PASSWORD: "test-root-password", MAPLE_DEFAULT_ORG_ID: "default", @@ -39,14 +51,17 @@ const makeConfig = (extra: Record = {}) => }), ) -const buildLayer = (testDb: TestDb, extra: Record = {}) => { - const configLive = makeConfig(extra) +const buildLayer = (testDb: TestDb, extra: Record = {}, includeTinybirdSigning = true) => { + const configLive = makeConfig(extra, includeTinybirdSigning) const envLive = Env.layer.pipe(Layer.provide(configLive)) const databaseLive = testDb.layer const orgSettingsLive = OrgClickHouseSettingsService.layer.pipe( Layer.provide(Layer.mergeAll(envLive, databaseLive)), ) - return WarehouseQueryService.layer.pipe(Layer.provide(Layer.mergeAll(envLive, orgSettingsLive))) + const tinybirdTokenLive = TinybirdOrgTokenService.layer.pipe(Layer.provide(envLive)) + return WarehouseQueryService.layer.pipe( + Layer.provide(Layer.mergeAll(envLive, orgSettingsLive, tinybirdTokenLive)), + ) } const getError = (exit: Exit.Exit): unknown => { @@ -70,6 +85,219 @@ const makeTenant = (): TenantContext => ({ const transient503 = () => new Error("HTTP status 503 service temporarily unavailable") +const decodeJwtPayload = (token: string): Record => + JSON.parse(Buffer.from(token.split(".")[1] ?? "", "base64url").toString("utf8")) as Record< + string, + unknown + > + +describe("WarehouseQueryService raw-SQL provider routing", () => { + it.effect("substitutes a scoped JWT for the Tinybird SDK token", () => { + let captured: ResolvedWarehouseConfig | undefined + let responseLimits: { readonly maxRows: number; readonly maxBytes: number } | undefined + __testables.setClientFactory((config) => { + captured = config + return { + sql: async (_sql, options) => { + responseLimits = options?.responseLimits + return { data: [] } + }, + insert: async () => {}, + } + }) + const layer = buildLayer(createTestDb(trackedDbs)) + + return Effect.gen(function* () { + yield* WarehouseQueryService.use((service) => + service.rawSqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'"), + ) + assert.strictEqual(captured?._tag, "tinybird") + if (captured?._tag !== "tinybird") throw new Error("expected Tinybird config") + assert.strictEqual(captured.provider, "tinybird") + assert.notStrictEqual(captured.token, "managed-token") + assert.strictEqual(decodeJwtPayload(captured.token).workspace_id, "test-workspace") + assert.deepStrictEqual(responseLimits, { maxRows: 1000, maxBytes: 5_000_000 }) + }).pipe(Effect.provide(layer)) + }) + + it.effect("defaults an env-level ClickHouse gateway to Tinybird and substitutes a scoped JWT", () => { + let captured: ResolvedWarehouseConfig | undefined + __testables.setClientFactory((config) => { + captured = config + return { sql: async () => ({ data: [] }), insert: async () => {} } + }) + const layer = buildLayer(createTestDb(trackedDbs), { + CLICKHOUSE_URL: "https://gateway.tinybird.example", + CLICKHOUSE_PASSWORD: "gateway-admin-token", + }) + + return Effect.gen(function* () { + yield* WarehouseQueryService.use((service) => + service.rawSqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'"), + ) + assert.strictEqual(captured?._tag, "clickhouse") + if (captured?._tag !== "clickhouse") throw new Error("expected gateway config") + assert.strictEqual(captured.provider, "tinybird") + assert.notStrictEqual(captured.password, "gateway-admin-token") + assert.strictEqual(decodeJwtPayload(captured.password).workspace_id, "test-workspace") + }).pipe(Effect.provide(layer)) + }) + + it.effect("preserves env-level vanilla ClickHouse credentials for raw SQL", () => { + let captured: ResolvedWarehouseConfig | undefined + __testables.setClientFactory((config) => { + captured = config + return { sql: async () => ({ data: [] }), insert: async () => {} } + }) + const layer = buildLayer(createTestDb(trackedDbs), { + CLICKHOUSE_URL: "https://clickhouse.example", + CLICKHOUSE_PROVIDER: "clickhouse", + CLICKHOUSE_PASSWORD: "original-clickhouse-password", + }) + + return Effect.gen(function* () { + yield* WarehouseQueryService.use((service) => + service.rawSqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'"), + ) + assert.strictEqual(captured?._tag, "clickhouse") + if (captured?._tag !== "clickhouse") throw new Error("expected ClickHouse config") + assert.strictEqual(captured.provider, "clickhouse") + assert.strictEqual(captured.password, "original-clickhouse-password") + }).pipe(Effect.provide(layer)) + }) + + it.effect("fails closed for env-level vanilla ClickHouse outside self-hosted mode", () => { + let constructed = false + __testables.setClientFactory(() => { + constructed = true + return { sql: async () => ({ data: [] }), insert: async () => {} } + }) + const layer = buildLayer(createTestDb(trackedDbs), { + CLICKHOUSE_URL: "https://clickhouse.example", + CLICKHOUSE_PROVIDER: "clickhouse", + CLICKHOUSE_PASSWORD: "shared-password", + MAPLE_AUTH_MODE: "clerk", + CLERK_SECRET_KEY: "sk_test_raw_sql", + }) + + return Effect.gen(function* () { + const error = yield* Effect.flip( + WarehouseQueryService.use((service) => + service.rawSqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'"), + ), + ) + assert.instanceOf(error, WarehouseConfigError) + assert.include(error.message, "single-org self-hosted mode") + assert.isFalse(constructed) + }).pipe(Effect.provide(layer)) + }) + + it.effect("preserves per-org ClickHouse override credentials for raw SQL", () => { + let captured: ResolvedWarehouseConfig | undefined + __testables.setClientFactory((config) => { + captured = config + return { sql: async () => ({ data: [] }), insert: async () => {} } + }) + // BYO credentials are already tenant-isolated and must not require the + // managed Tinybird JWT signing configuration. + const configLive = makeConfig({}, false) + const envLive = Env.layer.pipe(Layer.provide(configLive)) + const tokenLive = TinybirdOrgTokenService.layer.pipe(Layer.provide(envLive)) + const orgSettingsLive = Layer.succeed(OrgClickHouseSettingsService, { + resolveRuntimeConfig: () => + Effect.succeed( + Option.some({ + backend: "clickhouse" as const, + url: "https://byo.example", + user: "byo-user", + password: "byo-password", + database: "maple", + }), + ), + } as unknown as OrgClickHouseSettingsServiceShape) + const layer = WarehouseQueryService.layer.pipe( + Layer.provide(Layer.mergeAll(envLive, tokenLive, orgSettingsLive)), + ) + + return Effect.gen(function* () { + yield* WarehouseQueryService.use((service) => + service.rawSqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'"), + ) + assert.strictEqual(captured?._tag, "clickhouse") + if (captured?._tag !== "clickhouse") throw new Error("expected ClickHouse config") + assert.strictEqual(captured.provider, "clickhouse") + assert.strictEqual(captured.password, "byo-password") + }).pipe(Effect.provide(layer)) + }) + + it.effect("maps missing Tinybird signing configuration to WarehouseConfigError", () => { + __testables.setClientFactory(() => ({ + sql: async () => ({ data: [] }), + insert: async () => {}, + })) + const layer = buildLayer(createTestDb(trackedDbs), {}, false) + + return Effect.gen(function* () { + const exit = yield* WarehouseQueryService.use((service) => + service.rawSqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'"), + ).pipe(Effect.exit) + const failure = getError(exit) + assert.instanceOf(failure, WarehouseConfigError) + assert.include((failure as WarehouseConfigError).message, "TINYBIRD_SIGNING_KEY") + assert.notInclude((failure as WarehouseConfigError).message, "managed-token") + }).pipe(Effect.provide(layer)) + }) + + it.effect("rejects env-level URL userinfo before constructing a client", () => { + let constructed = false + __testables.setClientFactory(() => { + constructed = true + return { sql: async () => ({ data: [] }), insert: async () => {} } + }) + const layer = buildLayer(createTestDb(trackedDbs), { + CLICKHOUSE_URL: "https://user:secret@clickhouse.example", + CLICKHOUSE_PROVIDER: "clickhouse", + }) + + return Effect.gen(function* () { + const exit = yield* WarehouseQueryService.use((service) => + service.rawSqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'"), + ).pipe(Effect.exit) + assert.instanceOf(getError(exit), WarehouseConfigError) + assert.isFalse(constructed) + }).pipe(Effect.provide(layer)) + }) +}) + +describe("bounded Tinybird response fetch", () => { + it("accepts an exact-boundary response and aborts one byte over", async () => { + const realFetch = globalThis.fetch + try { + globalThis.fetch = (async () => + new Response(new Uint8Array(MAX_RAW_SQL_RESULT_BYTES))) as typeof fetch + const exact = await __testables.boundedResponseFetch(MAX_RAW_SQL_RESULT_BYTES)( + "https://api.tinybird.example/v0/sql", + ) + assert.strictEqual((await exact.arrayBuffer()).byteLength, MAX_RAW_SQL_RESULT_BYTES) + + globalThis.fetch = (async () => + new Response(new Uint8Array(MAX_RAW_SQL_RESULT_BYTES + 1))) as typeof fetch + let thrown: unknown + try { + await __testables.boundedResponseFetch(MAX_RAW_SQL_RESULT_BYTES)( + "https://api.tinybird.example/v0/sql", + ) + } catch (error) { + thrown = error + } + assert.instanceOf(thrown, Error) + assert.match((thrown as Error).message, /5000000 encoded bytes/) + } finally { + globalThis.fetch = realFetch + } + }) +}) + describe("WarehouseQueryService.sqlQuery retry on transient upstream failures", () => { // Runs under it.live: the retry schedule uses real exponential backoff // delays, so the default TestClock would stall the retries. @@ -381,6 +609,7 @@ describe("createClickHouseSqlClient.insert is disabled (ClickHouse is read-only) // read-only query gateway ("Only SELECT or DESCRIBE … Got: InsertQuery"). const chConfig = { _tag: "clickhouse" as const, + provider: "clickhouse" as const, url: "https://ch.example.com", username: "u", password: "p", @@ -414,7 +643,12 @@ describe("createClickHouseSqlClient.insert is disabled (ClickHouse is read-only) describe("createTinybirdSdkSqlClient.insert wire framing (the production insert path)", () => { // Inserts in the cloud only need to work on Tinybird. This pins that path so a // future change can't silently break ingest into the managed pipeline. - const tbConfig = { _tag: "tinybird" as const, host: "https://api.tinybird.co", token: "tok_123" } + const tbConfig = { + _tag: "tinybird" as const, + provider: "tinybird" as const, + host: "https://api.tinybird.co", + token: "tok_123", + } it("POSTs raw ndjson rows to the Tinybird Events API (/v0/events?name=)", async () => { const captured: Array<{ @@ -478,16 +712,22 @@ describe("ingest routes writes to the managed pipeline, not a per-org read overr const clickhouseReadOverride = { config: { _tag: "clickhouse" as const, + provider: "clickhouse" as const, url: "https://byo-clickhouse.example.com", username: "u", password: "p", database: "d", }, - source: "org_override" as const, + clientCacheKey: "read:org_test", } const tinybirdManaged = { - config: { _tag: "tinybird" as const, host: "https://managed.tinybird.co", token: "tok" }, - source: "managed" as const, + config: { + _tag: "tinybird" as const, + provider: "tinybird" as const, + host: "https://managed.tinybird.co", + token: "tok", + }, + clientCacheKey: "write:managed", } it.effect("ingest uses resolveIngestConfig (Tinybird) while reads use resolveConfig (override)", () => { @@ -503,6 +743,8 @@ describe("ingest routes writes to the managed pipeline, not a per-org read overr }, }), resolveConfig: () => Effect.succeed(clickhouseReadOverride), + resolveRawSqlConfig: () => + Effect.succeed({ ...clickhouseReadOverride, clientCacheKey: "raw:org_test" }), resolveIngestConfig: () => Effect.succeed(tinybirdManaged), }) const tenant = makeTenant() @@ -528,6 +770,7 @@ describe("ingest routes writes to the managed pipeline, not a per-org read overr }, }), resolveConfig: () => Effect.succeed(tinybirdManaged), + resolveRawSqlConfig: () => Effect.succeed({ ...tinybirdManaged, clientCacheKey: "raw:org_test" }), }) const tenant = makeTenant() diff --git a/apps/api/src/lib/WarehouseQueryService.tinybird.e2e.test.ts b/apps/api/src/lib/WarehouseQueryService.tinybird.e2e.test.ts new file mode 100644 index 000000000..c705bac32 --- /dev/null +++ b/apps/api/src/lib/WarehouseQueryService.tinybird.e2e.test.ts @@ -0,0 +1,195 @@ +import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" +import { mintOrgReadJwt } from "./tinybird-jwt" + +const enabled = process.env.TINYBIRD_LOCAL_E2E === "1" +const apiBase = (process.env.TINYBIRD_LOCAL_E2E_URL ?? "http://127.0.0.1:7181").replace(/\/$/, "") +const gatewayBase = (process.env.TINYBIRD_LOCAL_E2E_GATEWAY_URL ?? "http://127.0.0.1:7182").replace(/\/$/, "") +const workspaceName = `RawSqlE2E_${Date.now()}_${Math.random().toString(36).slice(2, 8)}` +const datasource = "raw_sql_e2e" + +interface LocalTokens { + readonly admin_token: string + readonly user_token: string + readonly workspace_admin_token: string +} + +interface LocalWorkspace { + readonly id: string + readonly name: string + readonly token: string +} + +let tokens: LocalTokens | undefined +let workspace: LocalWorkspace | undefined +let workspaceSigningKey: string | undefined + +const responseJson = async (response: Response): Promise => { + const text = await response.text() + if (!response.ok) throw new Error(`Tinybird ${response.status}: ${text.slice(0, 500)}`) + return JSON.parse(text) as T +} + +const listWorkspaces = async (adminToken: string): Promise> => { + const response = await fetch( + `${apiBase}/v1/user/workspaces?with_organization=true&token=${encodeURIComponent(adminToken)}`, + ) + const body = await responseJson<{ readonly workspaces: ReadonlyArray }>(response) + return body.workspaces +} + +const createWorkspace = async (localTokens: LocalTokens): Promise => { + const response = await fetch(`${apiBase}/v1/workspaces`, { + method: "POST", + headers: { + Authorization: `Bearer ${localTokens.user_token}`, + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ name: workspaceName }), + }) + await responseJson>(response) + const created = (await listWorkspaces(localTokens.admin_token)).find( + (candidate) => candidate.name === workspaceName, + ) + if (!created) throw new Error("Tinybird Local workspace was not returned after creation") + return created +} + +const buildDatasource = async (target: LocalWorkspace): Promise => { + const datafile = [ + "SCHEMA >", + " OrgId String `json:$.OrgId`,", + " value UInt64 `json:$.value`", + 'ENGINE "MergeTree"', + 'ENGINE_SORTING_KEY "OrgId"', + ].join("\n") + const form = new FormData() + form.append("data_project://", new Blob([datafile], { type: "text/plain" }), `${datasource}.datasource`) + const response = await fetch(`${apiBase}/v1/build`, { + method: "POST", + headers: { Authorization: `Bearer ${target.token}` }, + body: form, + }) + const body = await responseJson<{ readonly result: string; readonly error?: string }>(response) + if (body.result !== "success" && body.result !== "no_changes") { + throw new Error(`Tinybird local build failed: ${body.error ?? body.result}`) + } +} + +const ingestFixtures = async (target: LocalWorkspace): Promise => { + const response = await fetch(`${apiBase}/v0/events?name=${datasource}&wait=true`, { + method: "POST", + headers: { + Authorization: `Bearer ${target.token}`, + "Content-Type": "application/x-ndjson", + }, + body: ['{"OrgId":"org_a","value":1}', '{"OrgId":"org_b","value":2}'].join("\n"), + }) + await responseJson>(response) +} + +const loadWorkspaceSigningKey = async (target: LocalWorkspace): Promise => { + const response = await fetch(`${apiBase}/v0/tokens/`, { + headers: { Authorization: `Bearer ${target.token}` }, + }) + const body = await responseJson<{ + readonly tokens: ReadonlyArray<{ + readonly name: string + readonly token: string + readonly scopes: ReadonlyArray<{ readonly type: string }> + }> + }>(response) + const adminToken = body.tokens.find( + (candidate) => + candidate.name === "workspace admin token" && + candidate.scopes.some((scope) => scope.type === "ADMIN"), + ) + if (!adminToken) throw new Error("Tinybird Local did not return a workspace admin token") + return adminToken.token +} + +const scopedToken = (target: LocalWorkspace, signingKey: string): string => + mintOrgReadJwt({ + signingKey, + workspaceId: target.id, + orgId: "org_a", + datasourceNames: [datasource], + nowSeconds: Math.floor(Date.now() / 1000), + ttlSeconds: 600, + }) + +describe.skipIf(!enabled)("Tinybird Local raw-SQL JWT E2E", () => { + beforeAll(async () => { + tokens = await responseJson(await fetch(`${apiBase}/tokens`)) + workspace = await createWorkspace(tokens) + workspaceSigningKey = await loadWorkspaceSigningKey(workspace) + await buildDatasource(workspace) + await ingestFixtures(workspace) + }, 120_000) + + afterAll(async () => { + if (tokens === undefined || workspace === undefined) return + const response = await fetch( + `${apiBase}/v1/workspaces/${workspace.id}?hard_delete_confirmation=yes`, + { + method: "DELETE", + headers: { Authorization: `Bearer ${tokens.user_token}` }, + }, + ) + if (!response.ok) throw new Error(`Failed to delete Tinybird E2E workspace (${response.status})`) + }, 30_000) + + it("enforces the org filter through /v0/sql across predicates, UNIONs, and subqueries", async () => { + if (workspace === undefined || workspaceSigningKey === undefined) { + throw new Error("workspace not initialized") + } + for (const query of [ + `SELECT OrgId, value FROM ${datasource} WHERE OrgId = 'org_a' OR 1=1 ORDER BY OrgId`, + `SELECT OrgId, value FROM ${datasource} WHERE OrgId = 'org_a' UNION ALL SELECT OrgId, value FROM ${datasource} WHERE OrgId = 'org_b'`, + `SELECT OrgId, value FROM (SELECT OrgId, value FROM ${datasource}) ORDER BY OrgId`, + ]) { + const response = await fetch(`${apiBase}/v0/sql`, { + method: "POST", + headers: { + Authorization: `Bearer ${scopedToken(workspace, workspaceSigningKey)}`, + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ q: `${query} FORMAT JSON` }), + }) + const body = await responseJson<{ + readonly data: ReadonlyArray> + }>(response) + assert.deepStrictEqual(body.data, [{ OrgId: "org_a", value: 1 }]) + } + }) + + it("enforces the same isolation through the ClickHouse-compatible gateway", async () => { + if (workspace === undefined || workspaceSigningKey === undefined) { + throw new Error("workspace not initialized") + } + for (const query of [ + `SELECT OrgId, value FROM ${datasource} WHERE OrgId = 'org_a' OR 1=1 ORDER BY OrgId`, + `SELECT OrgId, value FROM ${datasource} WHERE OrgId = 'org_a' UNION ALL SELECT OrgId, value FROM ${datasource} WHERE OrgId = 'org_b'`, + `SELECT OrgId, value FROM (SELECT OrgId, value FROM ${datasource}) ORDER BY OrgId`, + ]) { + const response = await fetch(`${gatewayBase}/?database=${encodeURIComponent(workspace.name)}`, { + method: "POST", + headers: { + "X-ClickHouse-Key": scopedToken(workspace, workspaceSigningKey), + "X-ClickHouse-Database": workspace.name, + "Content-Type": "text/plain", + }, + body: `${query} FORMAT JSONEachRow`, + }) + const text = await response.text() + if (!response.ok) { + throw new Error(`Tinybird gateway ${response.status}: ${text.slice(0, 500)}`) + } + const rows = text + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record) + assert.deepStrictEqual(rows, [{ OrgId: "org_a", value: 1 }]) + } + }) +}) diff --git a/apps/api/src/lib/WarehouseQueryService.ts b/apps/api/src/lib/WarehouseQueryService.ts index 074759778..94e80bf0c 100644 --- a/apps/api/src/lib/WarehouseQueryService.ts +++ b/apps/api/src/lib/WarehouseQueryService.ts @@ -1,10 +1,11 @@ import { createClient as createClickHouseClient } from "@clickhouse/client-web" import { Tinybird } from "@tinybirdco/sdk" import { Context, Effect, Layer, Option, Redacted } from "effect" -import type { WarehouseQueryRequest } from "@maple/domain/http" +import { WarehouseConfigError, type WarehouseQueryRequest } from "@maple/domain/http" import { makeWarehouseExecutor, toWarehouseQueryError, + WarehouseResponseLimitError, type ResolvedWarehouseConfig, type SqlQueryOptions, type WarehouseExecutorDeps, @@ -16,6 +17,7 @@ import { WarehouseExecutor } from "@maple/query-engine/observability" import { Env } from "./Env" import type { TenantContext } from "../services/AuthService" import { OrgClickHouseSettingsService } from "../services/OrgClickHouseSettingsService" +import { TinybirdOrgTokenService } from "../services/TinybirdOrgTokenService" // --------------------------------------------------------------------------- // WarehouseQueryService — the API's managed-warehouse executor. @@ -42,12 +44,47 @@ const createClickHouseSqlClient = (config: ClickHouseConfig): WarehouseSqlClient database: config.database, }) return { - sql: async (sql: string) => { + sql: async (sql: string, options) => { const resultSet = await client.query({ query: sql, format: "JSONEachRow", }) - const data = await resultSet.json>() + const limits = options?.responseLimits + if (limits === undefined) { + const data = await resultSet.json>() + return { data } + } + + const data: Array> = [] + const encoder = new TextEncoder() + let encodedBytes = 0 + const reader = resultSet.stream().getReader() + try { + while (true) { + const chunk = await reader.read() + if (chunk.done) break + for (const row of chunk.value) { + encodedBytes += encoder.encode(row.text).byteLength + 1 + if (encodedBytes > limits.maxBytes) { + await reader.cancel().catch(() => undefined) + throw new WarehouseResponseLimitError({ + kind: "bytes", + message: `Raw SQL results may contain at most ${limits.maxBytes} encoded bytes`, + }) + } + data.push(row.json>()) + if (data.length > limits.maxRows) { + await reader.cancel().catch(() => undefined) + throw new WarehouseResponseLimitError({ + kind: "rows", + message: `Raw SQL results may contain at most ${limits.maxRows} rows`, + }) + } + } + } + } finally { + reader.releaseLock() + } return { data } }, insert: async (_datasource, _rows) => { @@ -69,18 +106,83 @@ const createClickHouseSqlClient = (config: ClickHouseConfig): WarehouseSqlClient const isEmptyJsonBodyError = (error: unknown): boolean => error instanceof SyntaxError && /unexpected end of json input/i.test(error.message) +const boundedResponseFetch = + (maxBytes: number): typeof fetch => + async (input, init) => { + const response = await fetch(input, init) + if (response.body === null) return response + + const reader = response.body.getReader() + const chunks: Uint8Array[] = [] + let totalBytes = 0 + try { + while (true) { + const chunk = await reader.read() + if (chunk.done) break + totalBytes += chunk.value.byteLength + if (totalBytes > maxBytes) { + await reader.cancel().catch(() => undefined) + throw new WarehouseResponseLimitError({ + kind: "bytes", + message: `Raw SQL results may contain at most ${maxBytes} encoded bytes`, + }) + } + chunks.push(chunk.value) + } + } finally { + reader.releaseLock() + } + + const body = new Uint8Array(totalBytes) + let offset = 0 + for (const chunk of chunks) { + body.set(chunk, offset) + offset += chunk.byteLength + } + return new Response(body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }) + } + const createTinybirdSdkSqlClient = (config: TinybirdConfig): WarehouseSqlClient => { - const client = new Tinybird({ - baseUrl: config.host, - token: config.token, - datasources: {}, - pipes: {}, - devMode: false, - }) + const makeClient = (fetchAdapter?: typeof fetch) => + new Tinybird({ + baseUrl: config.host, + token: config.token, + datasources: {}, + pipes: {}, + devMode: false, + ...(fetchAdapter === undefined ? {} : { fetch: fetchAdapter }), + }) + const client = makeClient() + const boundedClients = new Map() return { - sql: async (sql: string) => { + sql: async (sql: string, options) => { try { - return await client.sql(sql) + // Tinybird Cloud currently defaults /v0/sql to JSON, while Tinybird Local + // defaults to tab-separated output. The SDK always calls response.json(), so + // make the expected wire format explicit for both environments. + const jsonSql = `${sql.trimEnd().replace(/;$/, "")}\nFORMAT JSON` + const limits = options?.responseLimits + // The SDK normally buffers through response.json(). Raw execution gets + // a fetch adapter that aborts before constructing an oversized Response. + let queryClient = client + if (limits !== undefined) { + queryClient = + boundedClients.get(limits.maxBytes) ?? + makeClient(boundedResponseFetch(limits.maxBytes)) + boundedClients.set(limits.maxBytes, queryClient) + } + const result = await queryClient.sql>(jsonSql) + if (limits !== undefined && result.data.length > limits.maxRows) { + throw new WarehouseResponseLimitError({ + kind: "rows", + message: `Raw SQL results may contain at most ${limits.maxRows} rows`, + }) + } + return { data: result.data } } catch (error) { // Empty 2xx body ⇒ zero rows. Return an empty result set so the caller's no-data path // runs instead of surfacing a spurious WarehouseClientError. @@ -120,6 +222,7 @@ export class WarehouseQueryService extends Context.Service< make: Effect.gen(function* () { const env = yield* Env const orgClickHouseSettings = yield* OrgClickHouseSettingsService + const orgTokens = yield* TinybirdOrgTokenService /** * Resolve the upstream config for this tenant's queries. @@ -135,19 +238,37 @@ export class WarehouseQueryService extends Context.Service< // here — and the read-path fallback when an org has no BYO override. const resolveManagedConfig = Effect.fn("WarehouseQueryService.resolveManagedConfig")(function* () { if (Option.isSome(env.CLICKHOUSE_URL)) { + const configuredUrl = env.CLICKHOUSE_URL.value + const clickhouseUrl = yield* Effect.try({ + try: () => new URL(configuredUrl), + catch: () => + new WarehouseConfigError({ + pipeName: "resolveManagedConfig", + message: "CLICKHOUSE_URL is invalid", + }), + }) + if (clickhouseUrl.username.length > 0 || clickhouseUrl.password.length > 0) { + return yield* new WarehouseConfigError({ + pipeName: "resolveManagedConfig", + message: "CLICKHOUSE_URL must not contain embedded credentials", + }) + } yield* Effect.annotateCurrentSpan("db.client", "clickhouse") + const provider: "tinybird" | "clickhouse" = + env.CLICKHOUSE_PROVIDER === "tinybird" ? "tinybird" : "clickhouse" return { config: { _tag: "clickhouse" as const, - url: env.CLICKHOUSE_URL.value, + provider, + url: clickhouseUrl.toString().replace(/\/$/, ""), username: env.CLICKHOUSE_USER, password: Option.match(env.CLICKHOUSE_PASSWORD, { - onNone: () => Redacted.value(env.TINYBIRD_TOKEN), + onNone: () => (provider === "tinybird" ? Redacted.value(env.TINYBIRD_TOKEN) : ""), onSome: Redacted.value, }), database: env.CLICKHOUSE_DATABASE, }, - source: "managed" as const, + clientCacheKey: "read:managed", } } @@ -155,10 +276,11 @@ export class WarehouseQueryService extends Context.Service< return { config: { _tag: "tinybird" as const, + provider: "tinybird" as const, host: env.TINYBIRD_HOST, token: Redacted.value(env.TINYBIRD_TOKEN), }, - source: "managed" as const, + clientCacheKey: "read:managed", } }) @@ -180,12 +302,13 @@ export class WarehouseQueryService extends Context.Service< return { config: { _tag: "clickhouse" as const, + provider: "clickhouse" as const, url: override.value.url, username: override.value.user, password: override.value.password, database: override.value.database, }, - source: "org_override" as const, + clientCacheKey: `read:${tenant.orgId}`, } } @@ -193,6 +316,52 @@ export class WarehouseQueryService extends Context.Service< return yield* resolveManagedConfig() }) + const resolveRawSqlConfig: WarehouseExecutorDeps["resolveRawSqlConfig"] = Effect.fn( + "WarehouseQueryService.resolveRawSqlConfig", + )(function* (tenant, label) { + const resolved = yield* resolveConfig(tenant, label) + const clientCacheKey = `raw:${tenant.orgId}` + + // Per-org BYO ClickHouse credentials already isolate the warehouse. + if (resolved.clientCacheKey !== "read:managed") { + return { ...resolved, clientCacheKey } + } + + // Shared Tinybird is isolated with a datasource-scoped JWT. The same token + // works through both the SDK and Tinybird's ClickHouse-compatible gateway. + if (resolved.config.provider === "tinybird") { + const jwt = yield* orgTokens.getOrgReadToken(tenant.orgId).pipe( + Effect.mapError( + (error) => + new WarehouseConfigError({ + pipeName: label, + message: error.message, + cause: error, + }), + ), + ) + yield* Effect.annotateCurrentSpan("maple.tinybird.token.scope", "org_jwt") + return { + config: + resolved.config._tag === "tinybird" + ? { ...resolved.config, token: jwt } + : { ...resolved.config, password: jwt }, + clientCacheKey, + } + } + + // A shared vanilla ClickHouse credential has no database-enforced OrgId + // scope. It is safe only in Maple's single-org self-hosted deployment mode. + if (env.MAPLE_AUTH_MODE.toLowerCase() !== "self_hosted") { + return yield* new WarehouseConfigError({ + pipeName: label, + message: + "Raw SQL on managed vanilla ClickHouse is available only in single-org self-hosted mode", + }) + } + return { ...resolved, clientCacheKey } + }) + /** * Write-path config. Inserts (demo seed, service-map rollups, alert checks) * ALWAYS go to the Tinybird ingest pipeline (Events API) — never ClickHouse. @@ -211,26 +380,28 @@ export class WarehouseQueryService extends Context.Service< * it unconditionally (TINYBIRD_HOST/TOKEN are required env, always present). * Routing writes anywhere else broke demo-seed onboarding. */ - const resolveIngestConfig: WarehouseExecutorDeps["resolveConfig"] = Effect.fn( + const resolveIngestConfig: NonNullable = Effect.fn( "WarehouseQueryService.resolveIngestConfig", )(function* (tenant, _label) { yield* Effect.annotateCurrentSpan("orgId", tenant.orgId) yield* Effect.annotateCurrentSpan("clientSource", "managed") - yield* Effect.annotateCurrentSpan("ingest.routing", "tinybird") + yield* Effect.annotateCurrentSpan("query.routing", "ingest") yield* Effect.annotateCurrentSpan("db.client", "tinybird-sdk") return { config: { _tag: "tinybird" as const, + provider: "tinybird" as const, host: env.TINYBIRD_HOST, token: Redacted.value(env.TINYBIRD_TOKEN), }, - source: "managed" as const, + clientCacheKey: "write:managed", } }) return makeWarehouseExecutor({ createClient: (config) => sqlClientFactory(config), resolveConfig, + resolveRawSqlConfig, resolveIngestConfig, }) }), @@ -279,5 +450,6 @@ export const __testables = { }, createClickHouseSqlClient, createTinybirdSdkSqlClient, + boundedResponseFetch, isEmptyJsonBodyError, } diff --git a/apps/api/src/lib/api-cors.ts b/apps/api/src/lib/api-cors.ts new file mode 100644 index 000000000..816f97891 --- /dev/null +++ b/apps/api/src/lib/api-cors.ts @@ -0,0 +1,8 @@ +export const API_CORS_OPTIONS = { + allowedOrigins: ["*"], + allowedMethods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], + allowedHeaders: ["*"], + // The ElectricSQL shape proxy (and its electric-* exposed headers) moved + // to the standalone `apps/electric-sync` worker. + exposedHeaders: ["Mcp-Session-Id", "Retry-After"], +} diff --git a/apps/api/src/lib/tinybird-jwt.test.ts b/apps/api/src/lib/tinybird-jwt.test.ts new file mode 100644 index 000000000..1944ae76b --- /dev/null +++ b/apps/api/src/lib/tinybird-jwt.test.ts @@ -0,0 +1,57 @@ +import { createHmac } from "node:crypto" +import { assert, describe, it } from "@effect/vitest" +import { mintOrgReadJwt } from "./tinybird-jwt" + +const decodePart = (part: string): unknown => JSON.parse(Buffer.from(part, "base64url").toString("utf8")) + +const SIGNING_KEY = "explicit-test-signing-key" + +describe("mintOrgReadJwt", () => { + it("produces a well-formed HS256 JWT with the right header, exp, and scopes", () => { + const jwt = mintOrgReadJwt({ + signingKey: SIGNING_KEY, + workspaceId: "ws-uuid-123", + orgId: "org_abc", + datasourceNames: ["traces", "logs"], + nowSeconds: 1_000, + ttlSeconds: 600, + }) + + const [header, payload, signature] = jwt.split(".") + assert.deepStrictEqual(decodePart(header), { alg: "HS256", typ: "JWT" }) + + const decoded = decodePart(payload) as { + workspace_id: string + name: string + exp: number + scopes: ReadonlyArray<{ type: string; resource: string; filter: string }> + } + assert.strictEqual(decoded.workspace_id, "ws-uuid-123") + assert.strictEqual(decoded.name, "maple-raw-sql") + assert.strictEqual(decoded.exp, 1_600) + assert.deepStrictEqual(decoded.scopes, [ + { type: "DATASOURCES:READ", resource: "traces", filter: "OrgId = 'org_abc'" }, + { type: "DATASOURCES:READ", resource: "logs", filter: "OrgId = 'org_abc'" }, + ]) + + // Signature verifies independently against the explicit signing key. + const expected = createHmac("sha256", SIGNING_KEY).update(`${header}.${payload}`).digest("base64url") + assert.strictEqual(signature, expected) + }) + + it("escapes single quotes in the org id to prevent filter injection", () => { + const jwt = mintOrgReadJwt({ + signingKey: SIGNING_KEY, + workspaceId: "ws-uuid-123", + orgId: "org_' OR 1=1 --", + datasourceNames: ["traces"], + nowSeconds: 0, + ttlSeconds: 60, + }) + const decoded = decodePart(jwt.split(".")[1]) as { + scopes: ReadonlyArray<{ filter: string }> + } + // The embedded quote is backslash-escaped, so the ClickHouse literal stays closed. + assert.strictEqual(decoded.scopes[0].filter, "OrgId = 'org_\\' OR 1=1 --'") + }) +}) diff --git a/apps/api/src/lib/tinybird-jwt.ts b/apps/api/src/lib/tinybird-jwt.ts new file mode 100644 index 000000000..db39d3ee4 --- /dev/null +++ b/apps/api/src/lib/tinybird-jwt.ts @@ -0,0 +1,68 @@ +import { createHmac } from "node:crypto" +import { escapeClickHouseString } from "@maple/query-engine/sql" + +// --------------------------------------------------------------------------- +// Tinybird per-org read JWT minting. +// +// Tinybird "Forward" enforces row-level security via self-signed JWTs: a token +// carrying a `DATASOURCES:READ` scope with a `filter` has that filter ANDed onto +// every scan of the datasource server-side — inside subqueries, every UNION arm, +// both JOIN sides — regardless of the user's SQL. Verified live against the +// `/v0/sql` Query API: `WHERE 1=1` / `OR 1=1` / subqueries / direct other-org +// queries cannot escape the filter, and any datasource NOT listed in the token is +// denied outright (deny-by-default). +// +// We mint these locally (HMAC-SHA256 signed with the workspace admin token as the +// secret) rather than via the SDK's `createJWT`, which is a network round-trip. +// --------------------------------------------------------------------------- + +/** Standard Tinybird JWT scope entry. */ +export interface TinybirdJwtScope { + readonly type: "DATASOURCES:READ" + readonly resource: string + readonly filter: string +} + +export interface MintOrgReadJwtInput { + /** The explicitly configured HMAC signing secret. */ + readonly signingKey: string + /** The explicitly configured Tinybird workspace id. */ + readonly workspaceId: string + /** The org to scope the token to; embedded into every datasource filter. */ + readonly orgId: string + /** Every datasource the token may read. Each gets an `OrgId = ''` filter. */ + readonly datasourceNames: ReadonlyArray + /** Current time in whole seconds (Unix epoch). */ + readonly nowSeconds: number + /** Token lifetime in seconds. */ + readonly ttlSeconds: number +} + +const base64url = (input: string | Buffer): string => + (Buffer.isBuffer(input) ? input : Buffer.from(input, "utf8")).toString("base64url") + +/** + * Mint a per-org Tinybird read JWT scoped to `datasourceNames`, each filtered to + * `OrgId = ''`. HS256, signed with the workspace admin token. + */ +export function mintOrgReadJwt(input: MintOrgReadJwtInput): string { + const orgLiteral = `OrgId = '${escapeClickHouseString(input.orgId)}'` + const scopes: ReadonlyArray = input.datasourceNames.map((resource) => ({ + type: "DATASOURCES:READ", + resource, + filter: orgLiteral, + })) + + const header = base64url(JSON.stringify({ alg: "HS256", typ: "JWT" })) + const payload = base64url( + JSON.stringify({ + workspace_id: input.workspaceId, + name: "maple-raw-sql", + exp: input.nowSeconds + input.ttlSeconds, + scopes, + }), + ) + const signingInput = `${header}.${payload}` + const signature = base64url(createHmac("sha256", input.signingKey).update(signingInput).digest()) + return `${signingInput}.${signature}` +} diff --git a/apps/api/src/mcp/lib/warehouse-catalog.test.ts b/apps/api/src/lib/warehouse-catalog.test.ts similarity index 100% rename from apps/api/src/mcp/lib/warehouse-catalog.test.ts rename to apps/api/src/lib/warehouse-catalog.test.ts diff --git a/apps/api/src/mcp/lib/warehouse-catalog.ts b/apps/api/src/lib/warehouse-catalog.ts similarity index 87% rename from apps/api/src/mcp/lib/warehouse-catalog.ts rename to apps/api/src/lib/warehouse-catalog.ts index 98f808812..0ae5ee9d2 100644 --- a/apps/api/src/mcp/lib/warehouse-catalog.ts +++ b/apps/api/src/lib/warehouse-catalog.ts @@ -3,10 +3,9 @@ import * as Datasources from "@maple/domain/tinybird" // --------------------------------------------------------------------------- // Live introspection of the `defineDatasource` exports in -// packages/domain/src/tinybird/datasources.ts. Used by the MCP -// `describe_warehouse_tables` tool so agents discover real tables/columns at -// call time rather than relying on a hand-maintained list inside the tool -// description. +// packages/domain/src/tinybird/datasources.ts. This neutral catalog powers +// both raw-SQL datasource scoping and the MCP discovery tool, keeping the +// security allowlist and agent-visible schema derived from the same source. // // Type strings (`String`, `DateTime64(9)`, `LowCardinality(String)`, // `Map(LowCardinality(String), String)`) come straight from the SDK's @@ -94,6 +93,20 @@ export function listWarehouseTables(): ReadonlyArray { .sort((a, b) => a.name.localeCompare(b.name)) } +/** + * Datasource names that carry an `OrgId` column — the allowlist used to scope a + * per-org raw-SQL read JWT. Derived live from the `defineDatasource` exports, so + * a new telemetry datasource is covered automatically; any datasource WITHOUT an + * `OrgId` column is excluded, making it unqueryable in raw SQL (fail-closed) + * rather than leaking cross-tenant rows. + */ +export function listOrgScopedDatasourceNames(): ReadonlyArray { + return collectDatasources() + .filter((ds) => "OrgId" in ds._schema) + .map((ds) => ds._name) + .sort((a, b) => a.localeCompare(b)) +} + export function describeWarehouseTable(name: string): TableInfo | null { const ds = collectDatasources().find((d) => d._name === name) if (!ds) return null diff --git a/apps/api/src/mcp/lib/inspect-widget.ts b/apps/api/src/mcp/lib/inspect-widget.ts index b3a444d3a..3fd704ea9 100644 --- a/apps/api/src/mcp/lib/inspect-widget.ts +++ b/apps/api/src/mcp/lib/inspect-widget.ts @@ -346,17 +346,7 @@ const inspectRawSqlWidget = Effect.fn("inspectRawSqlWidget")(function* ( Effect.catchTag("@maple/http/errors/RawSqlValidationError", (error) => Effect.succeed({ ok: false as const, error: `${error.code}: ${error.message}` }), ), - Effect.catchTags({ - "@maple/http/errors/WarehouseQueryError": (e) => Effect.succeed({ ok: false as const, error: e.message }), - "@maple/http/errors/WarehouseUpstreamError": (e) => Effect.succeed({ ok: false as const, error: e.message }), - "@maple/http/errors/WarehouseAuthError": (e) => Effect.succeed({ ok: false as const, error: e.message }), - "@maple/http/errors/WarehouseConfigError": (e) => Effect.succeed({ ok: false as const, error: e.message }), - "@maple/http/errors/WarehouseClientError": (e) => Effect.succeed({ ok: false as const, error: e.message }), - "@maple/http/errors/WarehouseSchemaDriftError": (e) => Effect.succeed({ ok: false as const, error: e.message }), - "@maple/http/errors/WarehouseQuotaExceededError": (e) => - Effect.succeed({ ok: false as const, error: e.message }), - "@maple/http/errors/WarehouseValidationError": (e) => Effect.succeed({ ok: false as const, error: e.message }), - }), + Effect.catch((error) => Effect.succeed({ ok: false as const, error: error.message })), ) if (!result.ok) { diff --git a/apps/api/src/mcp/lib/resolve-tenant.ts b/apps/api/src/mcp/lib/resolve-tenant.ts index 86c53642d..cfdbaae9a 100644 --- a/apps/api/src/mcp/lib/resolve-tenant.ts +++ b/apps/api/src/mcp/lib/resolve-tenant.ts @@ -123,6 +123,18 @@ export const resolveMcpTenantContext = Effect.fn("resolveMcpTenantContext")(func ) if (Option.isSome(apiKeyResolved)) { + if (apiKeyResolved.value.kind !== "mcp") { + return yield* new McpAuthInvalidError({ + message: "This API key is only valid for the HTTP API", + reason: "key_kind", + }) + } + if (apiKeyResolved.value.scopes !== null) { + return yield* new McpAuthInvalidError({ + message: "Restricted API keys are not supported by the MCP server", + reason: "insufficient_scope", + }) + } const validOrgId = yield* decodeOrgId(apiKeyResolved.value.orgId).pipe( Effect.mapError( (e) => diff --git a/apps/api/src/mcp/lib/run-raw-sql.test.ts b/apps/api/src/mcp/lib/run-raw-sql.test.ts index 6a9656b42..ac5f3914c 100644 --- a/apps/api/src/mcp/lib/run-raw-sql.test.ts +++ b/apps/api/src/mcp/lib/run-raw-sql.test.ts @@ -1,5 +1,6 @@ import { assert, describe, it } from "@effect/vitest" import { Effect, Layer } from "effect" +import { MAX_RAW_SQL_RESULT_ROWS } from "@maple/domain/http" import { runRawSql, autoBucketSeconds } from "./run-raw-sql" import { WarehouseQueryService, type WarehouseQueryServiceShape } from "@/lib/WarehouseQueryService" import type { TenantContext } from "@/lib/tenant-context" @@ -8,11 +9,19 @@ const tenant = { orgId: "org_test" } as TenantContext const makeStub = ( rows: ReadonlyArray>, - captured?: { sql?: string }, + captured?: { sql?: string; profile?: string; context?: string }, ): WarehouseQueryServiceShape => ({ - sqlQuery: (_t: unknown, sql: string) => { - if (captured) captured.sql = sql + rawSqlQuery: ( + _t: unknown, + sql: string, + options?: { readonly profile?: string; readonly context?: string }, + ) => { + if (captured) { + captured.sql = sql + captured.profile = options?.profile + captured.context = options?.context + } return Effect.succeed(rows) }, }) as unknown as WarehouseQueryServiceShape @@ -24,7 +33,11 @@ const range = { startTime: "2026-04-01 00:00:00", endTime: "2026-04-01 01:00:00" describe("runRawSql", () => { it.effect("expands macros and returns rows + column metadata", () => Effect.gen(function* () { - const captured: { sql?: string } = {} + const captured: { + sql?: string + profile?: string + context?: string + } = {} const result = yield* runRawSql({ tenant, sql: "SELECT ServiceName, count() AS c FROM traces WHERE $__orgFilter GROUP BY ServiceName", @@ -34,6 +47,8 @@ describe("runRawSql", () => { // $__orgFilter expanded to the scoped predicate before execution. assert.include(captured.sql ?? "", "OrgId = 'org_test'") + assert.strictEqual(captured.profile, "rawInteractive") + assert.strictEqual(captured.context, "mcp.run_sql") assert.strictEqual(result.rowCount, 1) assert.deepStrictEqual([...result.columns], ["ServiceName", "c"]) }), @@ -68,6 +83,21 @@ describe("runRawSql", () => { assert.isTrue(exit._tag === "Failure") }), ) + + it.effect("rejects results above the hard row cap", () => + Effect.gen(function* () { + const rows = Array.from({ length: MAX_RAW_SQL_RESULT_ROWS + 1 }, (_, value) => ({ value })) + const exit = yield* runRawSql({ + tenant, + sql: "SELECT value FROM traces WHERE $__orgFilter", + ...range, + granularitySeconds: 60, + }).pipe(Effect.provide(provide(makeStub(rows))), Effect.exit) + + assert.isTrue(exit._tag === "Failure") + assert.include(JSON.stringify(exit), "ResourceLimit") + }), + ) }) describe("autoBucketSeconds", () => { diff --git a/apps/api/src/mcp/lib/run-raw-sql.ts b/apps/api/src/mcp/lib/run-raw-sql.ts index 8c7c9db57..5cf83d4ec 100644 --- a/apps/api/src/mcp/lib/run-raw-sql.ts +++ b/apps/api/src/mcp/lib/run-raw-sql.ts @@ -1,5 +1,7 @@ import { Effect } from "effect" -import { makeExpandMacros } from "@maple/query-engine/runtime" +import type { RawSqlValidationError } from "@maple/domain/http" +import type { WarehouseSqlError } from "@maple/query-engine/execution" +import { makeExecuteRawSql, type ExecuteRawSqlResult } from "@maple/query-engine/runtime" import { WarehouseQueryService } from "@/lib/WarehouseQueryService" import type { TenantContext } from "@/lib/tenant-context" @@ -30,46 +32,29 @@ export interface RunRawSqlInput { readonly granularitySeconds: number } -export interface RunRawSqlResult { - readonly rows: ReadonlyArray> - readonly columns: ReadonlyArray - readonly rowCount: number - readonly expandedSql: string - readonly granularitySeconds: number -} +export type RunRawSqlResult = ExecuteRawSqlResult /** * Expand the raw-SQL macros (`$__orgFilter`, `$__timeFilter(col)`, …) with the * full safety pass (required org filter, DDL/DML deny-list, single-statement, - * auto-LIMIT) and run the result through `WarehouseQueryService.sqlQuery`, + * auto-LIMIT) and run the result through `WarehouseQueryService.rawSqlQuery`, * returning the rows plus column/row metadata. Shared by the `run_sql` MCP tool * and `inspect_chart_data`'s raw_sql_chart branch so both honor the identical * guardrails. Fails with `RawSqlValidationError` (macro/safety) or a * `WarehouseError` (execution); callers surface these to the agent. */ export const runRawSql = Effect.fn("runRawSql")(function* (input: RunRawSqlInput) { - const expanded = yield* makeExpandMacros({ + const warehouse = yield* WarehouseQueryService + const executeRawSql = makeExecuteRawSql( + warehouse, + ) + return yield* executeRawSql(input.tenant, { sql: input.sql, orgId: input.tenant.orgId, startTime: input.startTime, endTime: input.endTime, granularitySeconds: input.granularitySeconds, - }) - - const warehouse = yield* WarehouseQueryService - const rows = yield* warehouse.sqlQuery(input.tenant, expanded.sql, { - profile: "list", + workload: "interactive", context: "mcp.run_sql", }) - - const records = rows as ReadonlyArray> - const columns = records.length > 0 ? Object.keys(records[0]) : [] - - return { - rows: records, - columns, - rowCount: records.length, - expandedSql: expanded.sql, - granularitySeconds: expanded.granularitySeconds, - } satisfies RunRawSqlResult }) diff --git a/apps/api/src/mcp/resources/instructions.ts b/apps/api/src/mcp/resources/instructions.ts index 128f25d32..68473eca6 100644 --- a/apps/api/src/mcp/resources/instructions.ts +++ b/apps/api/src/mcp/resources/instructions.ts @@ -107,7 +107,7 @@ The mutation tools now reject clauses the engine can't honor BEFORE persisting ( When you pass \`sql\` to \`add_dashboard_widget\` (or build a widget with \`dataSource.endpoint: "raw_sql_chart"\`), you author ClickHouse SQL directly. The server expands macros and runs the SQL through the warehouse. Use this path when the structured query builder can't express what you need (window functions, multi-step CTEs, unusual aggregations, joins). ### Macros — what gets substituted -- \`$__orgFilter\` → \`OrgId = ''\` — **REQUIRED**; without it the request is rejected before execution. Org isolation depends on this macro appearing in the SQL. +- \`$__orgFilter\` → \`OrgId = ''\` — **REQUIRED** for sorting-key pruning and defense in depth. Tenant isolation is also enforced by scoped warehouse credentials. - \`$__timeFilter(Column)\` → \`Column >= toDateTime('') AND Column <= toDateTime('')\`. \`Column\` must be a bare identifier (letters/digits/underscores/dots) — no expressions. **Prefer this over \`$__startTime\`/\`$__endTime\`** for WHERE clauses. - \`$__startTime\` / \`$__endTime\` → \`toDateTime('…')\` literals. Use when you need the bound inline somewhere other than a WHERE comparison. - \`$__interval_s\` → integer bucket size in seconds. Resolved from \`granularity_seconds\` (or auto-derived from the dashboard time range when omitted). **Only interpolate this if your SQL actually buckets time** — otherwise \`granularity_seconds\` is a no-op. @@ -115,7 +115,7 @@ When you pass \`sql\` to \`add_dashboard_widget\` (or build a widget with \`data ### Safety rules (server-enforced) - One statement only. Multiple statements separated by \`;\` are rejected. - Deny-listed keywords (INSERT, UPDATE, DELETE, DROP, ALTER, TRUNCATE, RENAME, ATTACH, DETACH, CREATE, GRANT, REVOKE, OPTIMIZE, SYSTEM, KILL) trigger a \`DisallowedStatement\` error. Comments and string literals are masked first so a SELECT containing the word "drop" in a string is fine. -- \`LIMIT 10000\` is auto-appended when no LIMIT clause is present. +- Every query is wrapped in an outer \`LIMIT 1001\`; the extra row is an overflow sentinel for the public 1,000-row cap. ### Tables — discover at call time @@ -222,6 +222,6 @@ The renderer is opinionated about column names. Get these wrong and the chart sh - **Pie missing \`name\` column** → renderer can't label slices. - **Timeseries with no DateTime in the first row** → reshape skips and you get raw rows; the chart looks empty. Put the bucket column first OR alias it \`bucket\`. - **\`Map\` lookup on missing key** returns empty string, not NULL — use \`SpanAttributes['k'] != ''\` not \`IS NOT NULL\`. -- **High-cardinality groupBy** without LIMIT → server appends \`LIMIT 10000\` but the chart still struggles. Always add an explicit \`LIMIT\` for pie/table/heatmap.`, +- **High-cardinality groupBy** without LIMIT → the server's outer 1,000-row cap protects the response, but the chart can still struggle. Add a tighter explicit \`LIMIT\` for pie/table/heatmap.`, ), }) diff --git a/apps/api/src/mcp/tools/create-alert-rule.ts b/apps/api/src/mcp/tools/create-alert-rule.ts index bccb2eea8..048ff33ef 100644 --- a/apps/api/src/mcp/tools/create-alert-rule.ts +++ b/apps/api/src/mcp/tools/create-alert-rule.ts @@ -169,12 +169,15 @@ function buildAlertRuleRequest( Match.when("raw_query", (): { error: string } | { draft?: unknown } => { if (!params.raw_query_sql) { return { - error: "signal_type=raw_query requires raw_query_sql: ClickHouse SQL returning a numeric `value` column (optional `group`, `samples` columns). Must reference $__orgFilter and may use $__timeFilter(col), $__startTime, $__endTime, $__interval_s.", + error: "signal_type=raw_query requires raw_query_sql: ClickHouse SQL returning a numeric `value` column (optional `group`, `samples` columns). Must reference $__orgFilter and $__timeFilter(col); may also use $__startTime, $__endTime, and $__interval_s.", } } if (!params.raw_query_sql.includes("$__orgFilter")) { return { error: "raw_query_sql must reference $__orgFilter for org scoping" } } + if (!params.raw_query_sql.includes("$__timeFilter(")) { + return { error: "raw_query_sql must reference $__timeFilter(...) to bound alert reads" } + } return {} }), Match.orElse((): { error: string } | { draft?: unknown } => ({})), @@ -302,7 +305,7 @@ export function registerCreateAlertRuleTool(server: McpToolRegistrar) { "JSON string of a query-builder draft (required when signal_type=builder_query). Same shape as dashboard custom-query widgets: { id, name, dataSource, aggregation, whereClause, groupBy, ... }.", ), raw_query_sql: optionalStringParam( - "ClickHouse SQL returning a numeric `value` column, optional `group`/`samples` columns (required when signal_type=raw_query). Must reference $__orgFilter; supports $__timeFilter(col), $__startTime, $__endTime, $__interval_s.", + "ClickHouse SQL returning a numeric `value` column, optional `group`/`samples` columns (required when signal_type=raw_query). Must reference $__orgFilter and $__timeFilter(col); supports $__startTime, $__endTime, and $__interval_s.", ), raw_query_reducer: optionalStringParam( "How to collapse raw_query result rows into one value: identity, sum, avg, min, max (default: identity).", diff --git a/apps/api/src/mcp/tools/describe-warehouse-tables.ts b/apps/api/src/mcp/tools/describe-warehouse-tables.ts index b7914021f..9fa5a15ab 100644 --- a/apps/api/src/mcp/tools/describe-warehouse-tables.ts +++ b/apps/api/src/mcp/tools/describe-warehouse-tables.ts @@ -1,6 +1,6 @@ import { optionalStringParam, type McpToolRegistrar } from "./types" import { Effect, Schema } from "effect" -import { describeWarehouseTable, listWarehouseTables } from "../lib/warehouse-catalog" +import { describeWarehouseTable, listWarehouseTables } from "../../lib/warehouse-catalog" const TOOL = "describe_warehouse_tables" diff --git a/apps/api/src/mcp/tools/run-sql.ts b/apps/api/src/mcp/tools/run-sql.ts index 8e6b366cf..a0c72266f 100644 --- a/apps/api/src/mcp/tools/run-sql.ts +++ b/apps/api/src/mcp/tools/run-sql.ts @@ -11,7 +11,7 @@ import { resolveTimeRange } from "../lib/time" import { autoBucketSeconds, runRawSql } from "../lib/run-raw-sql" import { createDualContent } from "../lib/structured-output" import { formatTable, truncate } from "../lib/format" -import { warehouseToMcpHandlers } from "../lib/map-warehouse-error" +import { toMcpQueryError } from "../lib/map-warehouse-error" // Rows returned to the model are capped so a wide/long result doesn't blow the // context. The full count is always reported via meta.rowCount. @@ -24,7 +24,7 @@ const runSqlSchema = Schema.Struct({ "Optional macros: `$__timeFilter(Column)` (expands to `Column >= AND Column <= `), " + "`$__startTime`, `$__endTime`, `$__interval_s` (bucket width in seconds for toStartOfInterval). " + "Only a single SELECT is allowed; DDL/DML keywords (INSERT, DROP, ALTER, …) are rejected. " + - "A `LIMIT 10000` is appended automatically if you omit one. " + + "An outer 1,000-row result cap is always enforced. " + "Use describe_warehouse_tables to discover table/column names.", ), start_time: optionalStringParam("Start time (YYYY-MM-DD HH:mm:ss UTC). Defaults to 1 hour ago."), @@ -83,7 +83,7 @@ export function registerRunSqlTool(server: McpToolRegistrar) { }), ), // Execution failures (CH syntax/schema/quota) surface the warehouse message so the agent can fix the SQL. - Effect.catchTags(warehouseToMcpHandlers("run_sql")), + Effect.mapError(toMcpQueryError("run_sql")), ) if (!outcome.ok) return outcome.result diff --git a/apps/api/src/mcp/tools/update-alert-rule.ts b/apps/api/src/mcp/tools/update-alert-rule.ts index cd06572e7..b13275b70 100644 --- a/apps/api/src/mcp/tools/update-alert-rule.ts +++ b/apps/api/src/mcp/tools/update-alert-rule.ts @@ -187,7 +187,7 @@ export function registerUpdateAlertRuleTool(server: McpToolRegistrar) { "JSON string of a query-builder draft (for signal_type=builder_query).", ), raw_query_sql: optionalStringParam( - "ClickHouse SQL returning a numeric `value` column (for signal_type=raw_query). Must reference $__orgFilter.", + "ClickHouse SQL returning a numeric `value` column (for signal_type=raw_query). Must reference $__orgFilter and $__timeFilter(col).", ), raw_query_reducer: optionalStringParam( "How to collapse raw_query result rows into one value: identity, sum, avg, min, max.", diff --git a/apps/api/src/routes/query-engine.http.ts b/apps/api/src/routes/query-engine.http.ts index 5a0822956..7f91c26d5 100644 --- a/apps/api/src/routes/query-engine.http.ts +++ b/apps/api/src/routes/query-engine.http.ts @@ -6,6 +6,7 @@ import { QueryEngineExecutionError, QueryEngineValidationError, RawSqlExecuteResponse, + type RawSqlValidationError, SpanHierarchyResponse, SpanDetailResponse, ErrorsByTypeResponse, @@ -68,7 +69,7 @@ import { } from "@maple/domain/http" import { Clock, Effect, Match, Option, Schema } from "effect" import { QueryEngineService } from "../services/QueryEngineService" -import { RawSqlChartService } from "@maple/query-engine/runtime" +import { makeExecuteRawSql } from "@maple/query-engine/runtime" import { WarehouseQueryService } from "../lib/WarehouseQueryService" import { traceCacheTtlSeconds } from "../lib/trace-detail-cache" import { @@ -78,6 +79,7 @@ import { parseWarehouseDateTime, } from "@maple/query-engine" import { LOGS_BODY_SEARCH_SETTINGS } from "@maple/query-engine/profiles" +import type { ExecutionTenant, WarehouseSqlError } from "@maple/query-engine/execution" import { buildBreakdownQuerySpec, buildTimeseriesQuerySpec } from "@maple/query-engine/query-builder" // `warehouse.sqlQuery` fails with the warehouse error union (distinct tagged @@ -122,7 +124,9 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", Effect.gen(function* () { const queryEngine = yield* QueryEngineService const warehouse = yield* WarehouseQueryService - const rawSqlChart = yield* RawSqlChartService + const executeRawSql = makeExecuteRawSql( + warehouse, + ) return handlers .handle("execute", ({ payload }) => @@ -156,19 +160,19 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", warehouse .compiledQueryFirst( tenant, - CH.compile( - CH.traceTimeProbeQuery({ traceId: payload.traceId, narrowByTime }), - narrowByTime - ? { - orgId: tenant.orgId, - startTime: formatWarehouseDateTime(nowMs - PROBE_RECENT_WINDOW_MS), - } - : { orgId: tenant.orgId }, - ), - { - profile: "discovery", - context: narrowByTime ? "spanHierarchyProbeRecent" : "spanHierarchyProbe", - }, + CH.compile( + CH.traceTimeProbeQuery({ traceId: payload.traceId, narrowByTime }), + narrowByTime + ? { + orgId: tenant.orgId, + startTime: formatWarehouseDateTime(nowMs - PROBE_RECENT_WINDOW_MS), + } + : { orgId: tenant.orgId }, + ), + { + profile: "discovery", + context: narrowByTime ? "spanHierarchyProbeRecent" : "spanHierarchyProbe", + }, ) .pipe(Effect.map(Option.getOrNull)), "spanHierarchy probe failed", @@ -2638,33 +2642,25 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", const autoBucketSeconds = computeAutoBucketSeconds(payload.startTime, payload.endTime) const granularitySeconds = payload.granularitySeconds ?? autoBucketSeconds - const expanded = yield* rawSqlChart.expandMacros({ - sql: payload.sql, - orgId: tenant.orgId, - startTime: payload.startTime, - endTime: payload.endTime, - granularitySeconds, - }) - - const profile: "aggregation" | "list" = - payload.displayType === "table" ? "list" : "aggregation" - const rows = yield* mapExecError( - warehouse.sqlQuery(tenant, expanded.sql, { - profile, + const result = yield* mapExecError( + executeRawSql(tenant, { + sql: payload.sql, + orgId: tenant.orgId, + startTime: payload.startTime, + endTime: payload.endTime, + granularitySeconds, + workload: "interactive", context: "rawSql", }), "rawSql query failed", ) - const records = rows - const columns = records.length > 0 ? Object.keys(records[0]) : [] - return new RawSqlExecuteResponse({ - data: records, + data: result.rows, meta: { - rowCount: records.length, - columns, - granularitySeconds: expanded.granularitySeconds, + rowCount: result.rowCount, + columns: result.columns, + granularitySeconds: result.granularitySeconds, }, }) }), diff --git a/apps/api/src/routes/v2/alert-rules.http.ts b/apps/api/src/routes/v2/alert-rules.http.ts index 97eb4e947..1acdbf207 100644 --- a/apps/api/src/routes/v2/alert-rules.http.ts +++ b/apps/api/src/routes/v2/alert-rules.http.ts @@ -177,12 +177,33 @@ const mergeUpsertRequest = ( patch: V2AlertRuleUpdateParams, ): Effect.Effect => Effect.gen(function* () { + const signalType = patch.signal_type ?? doc.signalType const queryBuilderDraft = - patch.query_builder_draft === undefined - ? doc.queryBuilderDraft + signalType !== "builder_query" + ? null + : patch.query_builder_draft === undefined + ? doc.signalType === "builder_query" + ? doc.queryBuilderDraft + : null : patch.query_builder_draft === null ? null : yield* decodeDraft(patch.query_builder_draft) + const rawQuerySql = + signalType !== "raw_query" + ? null + : patch.raw_query_sql !== undefined + ? patch.raw_query_sql + : doc.signalType === "raw_query" + ? doc.rawQuerySql + : null + const rawQueryReducer = + signalType !== "raw_query" + ? null + : patch.raw_query_reducer !== undefined + ? patch.raw_query_reducer + : doc.signalType === "raw_query" + ? doc.rawQueryReducer + : null return new AlertRuleUpsertRequest({ name: patch.name ?? doc.name, notes: patch.notes !== undefined ? patch.notes : doc.notes, @@ -196,7 +217,7 @@ const mergeUpsertRequest = ( excludeServiceNames: patch.exclude_service_names ?? doc.excludeServiceNames, tags: patch.tags ?? doc.tags, groupBy: patch.group_by !== undefined ? patch.group_by : doc.groupBy, - signalType: patch.signal_type ?? doc.signalType, + signalType, comparator: patch.comparator ?? doc.comparator, threshold: patch.threshold ?? doc.threshold, thresholdUpper: patch.threshold_upper !== undefined ? patch.threshold_upper : doc.thresholdUpper, @@ -213,9 +234,8 @@ const mergeUpsertRequest = ( apdexThresholdMs: patch.apdex_threshold_ms !== undefined ? patch.apdex_threshold_ms : doc.apdexThresholdMs, queryBuilderDraft, - rawQuerySql: patch.raw_query_sql !== undefined ? patch.raw_query_sql : doc.rawQuerySql, - rawQueryReducer: - patch.raw_query_reducer !== undefined ? patch.raw_query_reducer : doc.rawQueryReducer, + rawQuerySql, + rawQueryReducer, destinationIds: patch.destination_ids ?? doc.destinationIds, }) }) diff --git a/apps/api/src/routes/v2/alerts.http.test.ts b/apps/api/src/routes/v2/alerts.http.test.ts index acdfc4966..da19b9d91 100644 --- a/apps/api/src/routes/v2/alerts.http.test.ts +++ b/apps/api/src/routes/v2/alerts.http.test.ts @@ -20,7 +20,12 @@ import { HazelOAuthService } from "../../services/HazelOAuthService" import { OrgMembersService } from "../../services/OrgMembersService" import { QueryEngineService } from "../../services/QueryEngineService" import { V2SchemaErrorsLive } from "./error-envelope" -import { AllV2GroupLayersLive, ConfigResourceServiceStubsLayer } from "./v2-test-support" +import { + AllV2GroupLayersLive, + ApiV2RateLimiterAllowAllLayer, + ConfigResourceServiceStubsLayer, + TelemetryServiceStubsLayer, +} from "./v2-test-support" const createdDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(createdDbs)) @@ -47,6 +52,7 @@ const testConfig = () => const warehouseStub: WarehouseQueryServiceShape = { query: () => Effect.die(new Error("unexpected warehouse pipe query")), sqlQuery: () => Effect.succeed([]), + rawSqlQuery: () => Effect.succeed([]), compiledQuery: (_tenant, compiled) => compiled.decodeRows([]).pipe(Effect.orDie), compiledQueryFirst: () => Effect.die(new Error("unexpected compiled query")), ingest: () => Effect.void, @@ -106,8 +112,10 @@ const makeHarness = (warehouseService: WarehouseQueryServiceShape = warehouseStu const routes = HttpApiBuilder.layer(MapleApiV2).pipe( Layer.provide(AllV2GroupLayersLive), Layer.provide(ConfigResourceServiceStubsLayer), + Layer.provide(TelemetryServiceStubsLayer), Layer.provide(V2SchemaErrorsLive), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) const { handler, dispose: disposeHandler } = HttpRouter.toWebHandler(routes, { @@ -252,6 +260,94 @@ describe("v2 alerts over HTTP", () => { await harness.dispose() }) + it("clears stored raw SQL when a rule switches to a structured signal", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(["alerts:write"]) + const destination = await harness.request("POST", "/v2/alerts/destinations", key.secret, { + type: "webhook", + name: "Raw transition hook", + url: "https://example.com/hooks/raw-transition", + }) + expect(destination.status).toBe(200) + + const created = await harness.request("POST", "/v2/alerts/rules", key.secret, { + name: "Raw transition rule", + severity: "warning", + signal_type: "raw_query", + raw_query_sql: + "SELECT count() AS value FROM traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", + raw_query_reducer: "max", + comparator: "gt", + threshold: 10, + window_minutes: 5, + destination_ids: [destination.body.id], + }) + expect(created.status, JSON.stringify(created.body)).toBe(200) + expect(created.body.raw_query_sql).toContain("$__orgFilter") + + const patched = await harness.request( + "PATCH", + `/v2/alerts/rules/${created.body.id}`, + key.secret, + { signal_type: "error_rate" }, + ) + expect(patched.status, JSON.stringify(patched.body)).toBe(200) + expect(patched.body.signal_type).toBe("error_rate") + expect(patched.body.raw_query_sql).toBeNull() + expect(patched.body.raw_query_reducer).toBeNull() + await harness.dispose() + }) + + it("rejects hidden raw SQL on structured rules", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(["alerts:write"]) + const response = await harness.request("POST", "/v2/alerts/rules", key.secret, { + name: "Disguised raw rule", + severity: "warning", + signal_type: "error_rate", + raw_query_sql: "SELECT count() AS value FROM traces WHERE $__orgFilter", + comparator: "gt", + threshold: 0.1, + window_minutes: 5, + destination_ids: [], + }) + expect(response.status).toBe(400) + expect(response.body.error).toMatchObject({ type: "invalid_request_error" }) + await harness.dispose() + }) + + it("blocks raw SQL preview for alerts:read keys without querying the warehouse", async () => { + let warehouseCalls = 0 + const harness = makeHarness({ + ...warehouseStub, + sqlQuery: () => { + warehouseCalls += 1 + return Effect.succeed([{ value: 42 }]) + }, + }) + const key = await harness.bootstrapKey(["alerts:read"]) + const response = await harness.request("POST", "/v2/alerts/rules/preview", key.secret, { + rule: { + name: "Raw preview", + severity: "warning", + signal_type: "raw_query", + raw_query_sql: + "SELECT count() AS value FROM traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", + raw_query_reducer: "max", + comparator: "gt", + threshold: 10, + window_minutes: 5, + destination_ids: [], + }, + start_time: "2026-01-01T00:00:00.000Z", + end_time: "2026-01-01T00:30:00.000Z", + }) + expect(response.status).toBe(400) + expect(JSON.stringify(response.body)).toContain("cannot be previewed") + expect(warehouseCalls).toBe(0) + await harness.dispose() + }) + it("paginates alert checks beyond the former 2,000-row window", async () => { const checkRows = Array.from({ length: 2_005 }, (_, index) => { const timestamp = new Date(Date.UTC(2026, 0, 1) + (2_005 - index) * 1_000).toISOString() diff --git a/apps/api/src/routes/v2/api-keys.http.test.ts b/apps/api/src/routes/v2/api-keys.http.test.ts index e08b9a47d..57c28f5d3 100644 --- a/apps/api/src/routes/v2/api-keys.http.test.ts +++ b/apps/api/src/routes/v2/api-keys.http.test.ts @@ -4,17 +4,20 @@ import { HttpRouter } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" import { OrgId, UserId } from "@maple/domain/http" import { MapleApiV2 } from "@maple/domain/http/v2" +import { API_CORS_OPTIONS } from "../../lib/api-cors" import { Env } from "../../lib/Env" import { cleanupTestDbs, createTestDb, type TestDb } from "../../lib/test-pglite" import { ApiKeysService } from "../../services/ApiKeysService" import { AuthService } from "../../services/AuthService" import { DashboardPersistenceService } from "../../services/DashboardPersistenceService" import { ApiAuthorizationV2Layer } from "../../services/ApiAuthorizationV2Layer" +import { ApiV2RateLimiter, type ApiV2RateLimiterShape } from "../../services/ApiV2RateLimiter" import { V2SchemaErrorsLive } from "./error-envelope" import { AlertsServiceStubLayer, AllV2GroupLayersLive, ConfigResourceServiceStubsLayer, + TelemetryServiceStubsLayer, } from "./v2-test-support" /** @@ -42,7 +45,9 @@ const testConfig = () => }), ) -const makeHarness = () => { +const makeHarness = ( + checkRateLimit: ApiV2RateLimiterShape["check"] = () => Effect.succeed("allowed" as const), +) => { const testDb = createTestDb(createdDbs) const envLive = Env.layer.pipe(Layer.provide(testConfig())) const servicesLive = Layer.mergeAll( @@ -56,8 +61,11 @@ const makeHarness = () => { Layer.provide(V2SchemaErrorsLive), Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), + Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(Layer.succeed(ApiV2RateLimiter, { check: checkRateLimit })), Layer.provideMerge(servicesLive), + Layer.provideMerge(HttpRouter.cors(API_CORS_OPTIONS)), ) const { handler, dispose: disposeHandler } = HttpRouter.toWebHandler(routes, { @@ -69,7 +77,7 @@ const makeHarness = () => { const request = async ( method: string, path: string, - options: { token?: string; body?: unknown } = {}, + options: { token?: string; body?: unknown; origin?: string } = {}, ) => { const response = await handler( new Request(`http://maple.test${path}`, { @@ -77,32 +85,47 @@ const makeHarness = () => { headers: { ...(options.token !== undefined ? { authorization: `Bearer ${options.token}` } : {}), ...(options.body !== undefined ? { "content-type": "application/json" } : {}), + ...(options.origin !== undefined ? { origin: options.origin } : {}), }, body: options.body !== undefined ? JSON.stringify(options.body) : undefined, }), Context.empty() as never, ) const text = await response.text() - return { status: response.status, body: text.length > 0 ? JSON.parse(text) : null } + return { + status: response.status, + headers: response.headers, + body: text.length > 0 ? JSON.parse(text) : null, + } } const ORG = Schema.decodeUnknownSync(OrgId)("org_e2e") const USER = Schema.decodeUnknownSync(UserId)("user_e2e") - const bootstrapKey = (scopes?: ReadonlyArray) => + const bootstrapKey = (scopes?: ReadonlyArray, kind: "standard" | "mcp" = "standard") => runtime.runPromise( Effect.gen(function* () { const service = yield* ApiKeysService return yield* service.create(ORG, USER, { name: scopes === undefined ? "root-key" : `scoped:${scopes.join(",")}`, scopes, + kind, }) }), ) + const bootstrapSession = () => + runtime.runPromise( + Effect.gen(function* () { + const service = yield* AuthService + return (yield* service.loginSelfHosted("test-root-password")).token + }), + ) + return { request, bootstrapKey, + bootstrapSession, closeDatabase: () => testDb.close(), dispose: async () => { await disposeHandler() @@ -131,10 +154,87 @@ describe("v2 api_keys over HTTP", () => { }) it("rejects missing credentials with a 401 envelope", async () => { - const harness = makeHarness() + let rateLimitChecks = 0 + const harness = makeHarness(() => { + rateLimitChecks += 1 + return Effect.succeed("allowed") + }) const { status, body } = await harness.request("GET", "/v2/api_keys") expect(status).toBe(401) expect(body.error.type).toBe("authentication_error") + expect(rateLimitChecks).toBe(0) + await harness.dispose() + }) + + it("rejects MCP-only keys on the HTTP API", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(undefined, "mcp") + const { status, body } = await harness.request("GET", "/v2/api_keys", { token: key.secret }) + expect(status).toBe(401) + expect(body.error).toMatchObject({ + type: "authentication_error", + code: "invalid_credentials", + }) + await harness.dispose() + }) + + it("returns the rate-limit envelope and Retry-After before executing the handler", async () => { + const harness = makeHarness(() => Effect.succeed("limited")) + const key = await harness.bootstrapKey() + + // OrganizationService is deliberately an inert stub in this harness. A 429 + // proves authorization stopped before the route handler could execute it. + const response = await harness.request("GET", "/v2/organization", { + token: key.secret, + origin: "https://app.maple.dev", + }) + expect(response.status).toBe(429) + expect(response.body).toEqual({ + error: { + type: "rate_limit_error", + code: "rate_limited", + message: "Too many requests. Retry after 60 seconds.", + }, + }) + expect(response.headers.get("retry-after")).toBe("60") + expect(response.headers.get("access-control-expose-headers")).toContain("Retry-After") + await harness.dispose() + }) + + it("bypasses API-key rate limiting for session tokens", async () => { + let rateLimitChecks = 0 + const harness = makeHarness(() => { + rateLimitChecks += 1 + return Effect.succeed("limited") + }) + const sessionToken = await harness.bootstrapSession() + const response = await harness.request("GET", "/v2/api_keys", { token: sessionToken }) + expect(response.status).toBe(200) + expect(rateLimitChecks).toBe(0) + await harness.dispose() + }) + + it("allows requests when the limiter fails open", async () => { + const harness = makeHarness(() => Effect.succeed("failed_open")) + const key = await harness.bootstrapKey() + const response = await harness.request("GET", "/v2/api_keys", { token: key.secret }) + expect(response.status).toBe(200) + await harness.dispose() + }) + + it("counts valid-key requests that are later rejected for insufficient scope", async () => { + let rateLimitChecks = 0 + const harness = makeHarness(() => { + rateLimitChecks += 1 + return Effect.succeed("allowed") + }) + const readOnly = await harness.bootstrapKey(["api_keys:read"]) + const response = await harness.request("POST", "/v2/api_keys", { + token: readOnly.secret, + body: { name: "nope" }, + }) + expect(response.status).toBe(403) + expect(rateLimitChecks).toBe(1) await harness.dispose() }) diff --git a/apps/api/src/routes/v2/config-resources.http.test.ts b/apps/api/src/routes/v2/config-resources.http.test.ts index fe8d98726..36ea923f4 100644 --- a/apps/api/src/routes/v2/config-resources.http.test.ts +++ b/apps/api/src/routes/v2/config-resources.http.test.ts @@ -19,7 +19,13 @@ import { PlanetScaleOAuthService } from "../../services/PlanetScaleOAuthService" import { RecommendationIssueService } from "../../services/RecommendationIssueService" import { ScrapeTargetsService } from "../../services/ScrapeTargetsService" import { V2SchemaErrorsLive } from "./error-envelope" -import { AlertsServiceStubLayer, AllV2GroupLayersLive, Phase1ResourceStubsLayer } from "./v2-test-support" +import { + AlertsServiceStubLayer, + AllV2GroupLayersLive, + ApiV2RateLimiterAllowAllLayer, + Phase1ResourceStubsLayer, + TelemetryServiceStubsLayer, +} from "./v2-test-support" /** * End-to-end HTTP tests for the v2 config-resource bundle (attribute_mappings, @@ -52,6 +58,7 @@ const die = () => Effect.die(new Error("not available in this test harness")) const warehouseStub: WarehouseQueryServiceShape = { query: () => Effect.die(new Error("unexpected warehouse pipe query")), sqlQuery: () => Effect.succeed([]), + rawSqlQuery: () => Effect.succeed([]), compiledQuery: (_tenant, compiled) => compiled.decodeRows([]).pipe(Effect.orDie), compiledQueryFirst: () => Effect.die(new Error("unexpected compiled query")), ingest: () => Effect.void, @@ -98,9 +105,11 @@ const makeHarness = () => { Layer.provide(V2SchemaErrorsLive), Layer.provide(AlertsServiceStubLayer), Layer.provide(Phase1ResourceStubsLayer), + Layer.provide(TelemetryServiceStubsLayer), // session_replays (in AllV2GroupLayersLive) needs the warehouse at the routes level. Layer.provide(warehouseLive), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/dashboards.http.test.ts b/apps/api/src/routes/v2/dashboards.http.test.ts index 498b11f2a..87dbec423 100644 --- a/apps/api/src/routes/v2/dashboards.http.test.ts +++ b/apps/api/src/routes/v2/dashboards.http.test.ts @@ -14,7 +14,9 @@ import { V2SchemaErrorsLive } from "./error-envelope" import { AlertsServiceStubLayer, AllV2GroupLayersLive, + ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, + TelemetryServiceStubsLayer, } from "./v2-test-support" const createdDbs: TestDb[] = [] @@ -50,7 +52,9 @@ const makeHarness = () => { Layer.provide(V2SchemaErrorsLive), Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), + Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) const { handler, dispose: disposeHandler } = HttpRouter.toWebHandler(routes, { diff --git a/apps/api/src/routes/v2/phase1-resources.http.test.ts b/apps/api/src/routes/v2/phase1-resources.http.test.ts index f7956d2c5..f4f6f8b43 100644 --- a/apps/api/src/routes/v2/phase1-resources.http.test.ts +++ b/apps/api/src/routes/v2/phase1-resources.http.test.ts @@ -40,7 +40,9 @@ import { V2SchemaErrorsLive } from "./error-envelope" import { AlertsServiceStubLayer, AllV2GroupLayersLive, + ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, + TelemetryServiceStubsLayer, } from "./v2-test-support" /** @@ -203,6 +205,7 @@ const die = () => Effect.die(new Error("not exercised in this test harness")) const warehouseStub: WarehouseQueryServiceShape = { query: die, sqlQuery: () => Effect.succeed([]), + rawSqlQuery: () => Effect.succeed([]), compiledQuery: (_tenant, compiled) => compiled.decodeRows([]).pipe(Effect.orDie), compiledQueryFirst: () => Effect.succeed(Option.none()), ingest: () => Effect.void, @@ -343,7 +346,9 @@ const makeHarness = (warehouseService: WarehouseQueryServiceShape = warehouseStu Layer.provide(V2SchemaErrorsLive), Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), + Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/telemetry.http.test.ts b/apps/api/src/routes/v2/telemetry.http.test.ts new file mode 100644 index 000000000..4cc278498 --- /dev/null +++ b/apps/api/src/routes/v2/telemetry.http.test.ts @@ -0,0 +1,497 @@ +import { afterEach, describe, expect, it } from "@effect/vitest" +import { OrgId, UserId, WarehouseQueryError } from "@maple/domain/http" +import { MapleApiV2 } from "@maple/domain/http/v2" +import { QueryEngineExecuteResponse, type QueryEngineExecuteRequest } from "@maple/query-engine" +import { ConfigProvider, Context, Effect, Layer, ManagedRuntime, Option, Schema } from "effect" +import { HttpRouter } from "effect/unstable/http" +import { HttpApiBuilder } from "effect/unstable/httpapi" +import { Env } from "../../lib/Env" +import { cleanupTestDbs, createTestDb, type TestDb } from "../../lib/test-pglite" +import { WarehouseQueryService, type WarehouseQueryServiceShape } from "../../lib/WarehouseQueryService" +import { ApiAuthorizationV2Layer } from "../../services/ApiAuthorizationV2Layer" +import { ApiKeysService } from "../../services/ApiKeysService" +import { AuthService } from "../../services/AuthService" +import { DashboardPersistenceService } from "../../services/DashboardPersistenceService" +import { QueryEngineService, type QueryEngineServiceShape } from "../../services/QueryEngineService" +import { V2SchemaErrorsLive } from "./error-envelope" +import { + AlertsServiceStubLayer, + AllV2GroupLayersLive, + ApiV2RateLimiterAllowAllLayer, + ConfigResourceServiceStubsLayer, +} from "./v2-test-support" + +const TRACE_ID = "7f3a4b5c6d7e8f901234567890abcdef" +const SPAN_ID = "0123456789abcdef" +const START = "2026-07-15T12:00:00.000Z" +const END = "2026-07-15T13:00:00.000Z" +const windowQuery = `start_time=${encodeURIComponent(START)}&end_time=${encodeURIComponent(END)}` + +const createdDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(createdDbs)) + +const testConfig = () => + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3490", + MCP_PORT: "3491", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 1).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + INTERNAL_SERVICE_TOKEN: "test-internal-token", + }), + ) + +const logRow = { + timestamp: "2026-07-15 12:00:01.123", + severityText: "ERROR", + severityNumber: 17, + serviceName: "api", + body: "checkout failed", + traceId: TRACE_ID, + spanId: SPAN_ID, + recordIdentity: "00112233445566778899AABBCCDDEEFF", + logAttributes: JSON.stringify({ "error.type": "Timeout" }), + resourceAttributes: JSON.stringify({ "service.namespace": "checkout" }), +} + +const hierarchyRow = { + traceId: TRACE_ID, + spanId: SPAN_ID, + parentSpanId: "", + spanName: "GET /checkout", + serviceName: "api", + spanKind: "Server", + durationMs: 42.5, + startTime: "2026-07-15 12:00:00.000", + statusCode: "Error", + statusMessage: "timeout", + spanAttributes: JSON.stringify({ "http.route": "/checkout" }), + resourceAttributes: JSON.stringify({ "deployment.environment.name": "production" }), + relationship: "related", +} + +const rowsForSql = (sql: string): ReadonlyArray> => { + if (sql.includes("FROM trace_list_mv")) { + const rows = [ + { + traceId: TRACE_ID, + startTime: "2026-07-15 12:00:00", + durationMs: 42.5, + rootSpanName: "GET /checkout", + rootSpanKind: "Server", + rootServiceName: "api", + statusCode: "Error", + hasError: 1, + deploymentEnvironment: "production", + serviceNamespace: "checkout", + httpMethod: "GET", + httpRoute: "/checkout", + httpStatusCode: "500", + }, + { + traceId: "6f3a4b5c6d7e8f901234567890abcdef", + startTime: "2026-07-15 11:59:00", + durationMs: 10, + rootSpanName: "GET /health", + rootSpanKind: "Server", + rootServiceName: "api", + statusCode: "Ok", + hasError: 0, + deploymentEnvironment: "production", + serviceNamespace: "checkout", + httpMethod: "GET", + httpRoute: "/health", + httpStatusCode: "200", + }, + ] + return sql.includes("TraceId <") ? rows.slice(1) : rows + } + if (sql.includes("FROM trace_detail_spans") && sql.includes("AS relationship")) return [hierarchyRow] + if (sql.includes("FROM trace_detail_spans") && sql.includes("toJSONString(SpanAttributes)")) { + return [ + { + ...hierarchyRow, + spanAttributes: JSON.stringify({ "http.route": "/checkout", "error.type": "Timeout" }), + resourceAttributes: JSON.stringify({ "service.name": "api" }), + }, + ] + } + if (sql.includes("FROM logs")) return [logRow] + if (sql.includes("FROM metric_catalog")) { + return [ + { + metricName: "http.server.duration", + metricType: "histogram", + serviceName: "api", + metricDescription: "HTTP server duration", + metricUnit: "ms", + dataPointCount: "42", + firstSeen: "2026-07-15 12:00:00", + lastSeen: "2026-07-15 12:59:00", + isMonotonic: "0", + }, + ] + } + if (sql.includes("FROM service_overview_spans")) { + return [ + { + serviceName: "api", + serviceNamespaces: ["checkout"], + deploymentEnvironments: ["production"], + spanCount: "10", + errorCount: "2", + estimatedErrorCount: "4", + estimatedSpanCount: "20", + p50LatencyMs: "10", + p95LatencyMs: "40", + p99LatencyMs: "50", + }, + ] + } + if (sql.includes("service_map_edges_hourly")) { + return [ + { + sourceService: "api", + targetService: "payments", + callCount: "10", + errorCount: "2", + avgDurationMs: "12.5", + p95DurationMs: "30", + estimatedSpanCount: "20", + }, + ] + } + return [] +} + +const warehouseStub: WarehouseQueryServiceShape = { + query: () => Effect.die(new Error("unexpected named query")), + sqlQuery: () => Effect.succeed([{ bucket: "2026-07-15 12:00:00", value: 1 }]), + compiledQuery: (_tenant, compiled) => compiled.decodeRows(rowsForSql(compiled.sql)), + compiledQueryFirst: (_tenant, compiled) => + compiled + .decodeRows(rowsForSql(compiled.sql)) + .pipe(Effect.map((rows) => Option.fromNullishOr(rows[0]))), + ingest: () => Effect.void, + asExecutor: () => { + throw new Error("not used") + }, +} + +const queryEngineStub = { + execute: () => + Effect.succeed( + new QueryEngineExecuteResponse({ + result: { + kind: "timeseries", + source: "metrics", + data: [{ bucket: "2026-07-15 12:00:00", series: { all: 42 } }], + }, + }), + ), + evaluate: () => Effect.die(new Error("not used")), + evaluateRawSql: () => Effect.die(new Error("not used")), + evaluateSeries: () => Effect.die(new Error("not used")), + cachedDirect: (_tenant, _route, _payload, effect) => effect, +} satisfies QueryEngineServiceShape + +const makeHarness = ( + warehouseService: WarehouseQueryServiceShape = warehouseStub, + queryEngineService: QueryEngineServiceShape = queryEngineStub, +) => { + const testDb = createTestDb(createdDbs) + const envLive = Env.layer.pipe(Layer.provide(testConfig())) + const servicesLive = Layer.mergeAll( + ApiKeysService.layer, + AuthService.layer, + DashboardPersistenceService.layer, + ).pipe(Layer.provideMerge(Layer.mergeAll(envLive, testDb.layer))) + const telemetryLive = Layer.mergeAll( + Layer.succeed(WarehouseQueryService, warehouseService), + Layer.succeed(QueryEngineService, queryEngineService), + ) + const routes = HttpApiBuilder.layer(MapleApiV2).pipe( + Layer.provide(AllV2GroupLayersLive), + Layer.provide(telemetryLive), + Layer.provide(V2SchemaErrorsLive), + Layer.provide(AlertsServiceStubLayer), + Layer.provide(ConfigResourceServiceStubsLayer), + Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), + Layer.provideMerge(servicesLive), + ) + const { handler, dispose: disposeHandler } = HttpRouter.toWebHandler(routes, { disableLogger: true }) + const runtime = ManagedRuntime.make(servicesLive) + const org = Schema.decodeUnknownSync(OrgId)("org_telemetry_e2e") + const user = Schema.decodeUnknownSync(UserId)("user_telemetry_e2e") + const bootstrapKey = (scopes?: ReadonlyArray) => + runtime.runPromise( + Effect.gen(function* () { + const service = yield* ApiKeysService + return yield* service.create(org, user, { name: "telemetry-test", scopes }) + }), + ) + const request = async (method: string, path: string, token: string, body?: unknown) => { + const response = await handler( + new Request(`http://maple.test${path}`, { + method, + headers: { + authorization: `Bearer ${token}`, + ...(body ? { "content-type": "application/json" } : {}), + }, + body: body ? JSON.stringify(body) : undefined, + }), + Context.empty() as never, + ) + const text = await response.text() + return { status: response.status, body: text ? JSON.parse(text) : null } + } + return { + bootstrapKey, + request, + dispose: async () => { + await disposeHandler() + await runtime.dispose() + }, + } +} + +describe("v2 telemetry reads over HTTP", () => { + it("serves all trace, log, metric, service, service-map, and query reads", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey() + const bounds = { start_time: START, end_time: END } + + const traces = await harness.request("POST", "/v2/traces/search", key.secret, bounds) + expect(traces.status).toBe(200) + expect(traces.body.data[0]).toMatchObject({ object: "trace", id: TRACE_ID, has_error: true }) + + const trace = await harness.request("GET", `/v2/traces/${TRACE_ID}`, key.secret) + expect(trace.status).toBe(200) + expect(trace.body.spans).toHaveLength(1) + expect(trace.body.truncated).toBe(false) + + const span = await harness.request("GET", `/v2/traces/${TRACE_ID}/spans/${SPAN_ID}`, key.secret) + expect(span.status).toBe(200) + expect(span.body.attributes["error.type"]).toBe("Timeout") + + const logs = await harness.request("POST", "/v2/logs/search", key.secret, bounds) + expect(logs.status).toBe(200) + expect(logs.body.data[0].id.startsWith("log_")).toBe(true) + const log = await harness.request("GET", `/v2/logs/${logs.body.data[0].id}`, key.secret) + expect(log.status).toBe(200) + expect(log.body.trace_id).toBe(TRACE_ID) + + const metrics = await harness.request("GET", `/v2/metrics?${windowQuery}`, key.secret) + expect(metrics.status).toBe(200) + expect(metrics.body.data[0].data_point_count).toBe(42) + + const metricSeries = await harness.request("POST", "/v2/metrics/timeseries", key.secret, { + ...bounds, + metric: "avg", + metric_name: "http.server.duration", + metric_type: "histogram", + }) + expect(metricSeries.status).toBe(200) + expect(metricSeries.body.timeseries[0].series.all).toBe(42) + + const services = await harness.request("GET", `/v2/services?${windowQuery}`, key.secret) + expect(services.status).toBe(200) + expect(services.body.data[0]).toMatchObject({ name: "api", has_sampling: true, span_count: 10 }) + const service = await harness.request("GET", `/v2/services/api?${windowQuery}`, key.secret) + expect(service.status).toBe(200) + + const serviceMap = await harness.request("GET", `/v2/service_map?${windowQuery}`, key.secret) + expect(serviceMap.status).toBe(200) + expect(serviceMap.body.edges[0]).toMatchObject({ source_service: "api", target_service: "payments" }) + + const structured = await harness.request("POST", "/v2/query", key.secret, { + ...bounds, + query: { + kind: "timeseries", + source: "metrics", + metric: "avg", + filters: { metric_name: "http.server.duration", metric_type: "histogram" }, + }, + }) + expect(structured.status).toBe(200) + + await harness.dispose() + }) + + it("requires windows, validates log IDs, and treats telemetry POSTs as scoped reads", async () => { + const harness = makeHarness() + const tracesKey = await harness.bootstrapKey(["traces:read"]) + const allowed = await harness.request("POST", "/v2/traces/search", tracesKey.secret, { + start_time: START, + end_time: END, + }) + expect(allowed.status).toBe(200) + const denied = await harness.request("POST", "/v2/logs/search", tracesKey.secret, { + start_time: START, + end_time: END, + }) + expect(denied.status).toBe(403) + const missingWindow = await harness.request("POST", "/v2/traces/search", tracesKey.secret, {}) + expect(missingWindow.status).toBe(400) + const firstPage = await harness.request("POST", "/v2/traces/search", tracesKey.secret, { + start_time: START, + end_time: END, + limit: 1, + }) + expect(firstPage.body.has_more).toBe(true) + const secondPage = await harness.request("POST", "/v2/traces/search", tracesKey.secret, { + start_time: START, + end_time: END, + limit: 1, + cursor: firstPage.body.next_cursor, + }) + expect(secondPage.body.data[0].id).toBe("6f3a4b5c6d7e8f901234567890abcdef") + const badCursor = await harness.request("POST", "/v2/traces/search", tracesKey.secret, { + start_time: START, + end_time: END, + cursor: "garbage", + }) + expect(badCursor.status).toBe(400) + const oversizedWindow = await harness.request("POST", "/v2/traces/search", tracesKey.secret, { + start_time: "2026-01-01T00:00:00.000Z", + end_time: "2026-02-02T00:00:00.000Z", + }) + expect(oversizedWindow.status).toBe(400) + expect(oversizedWindow.body.error.code).toBe("time_range_too_large") + const rootKey = await harness.bootstrapKey() + const malformedLog = await harness.request("GET", "/v2/logs/log_bad", rootKey.secret) + expect(malformedLog.status).toBe(400) + const rawSql = await harness.request("POST", "/v2/query", rootKey.secret, { + start_time: START, + end_time: END, + query: { kind: "raw_sql", sql: "SELECT * FROM traces WHERE $__orgFilter" }, + }) + expect(rawSql.status).toBe(400) + await harness.dispose() + }) + + it("preserves fractional bounds and reads complete traces by their sorting-key identity", async () => { + const observedSql: string[] = [] + const observingWarehouse: WarehouseQueryServiceShape = { + ...warehouseStub, + compiledQuery: (tenant, compiled, options) => { + observedSql.push(compiled.sql) + return warehouseStub.compiledQuery(tenant, compiled, options) + }, + compiledQueryFirst: (tenant, compiled, options) => { + observedSql.push(compiled.sql) + return warehouseStub.compiledQueryFirst(tenant, compiled, options) + }, + } + const harness = makeHarness(observingWarehouse) + const key = await harness.bootstrapKey() + + const logs = await harness.request("POST", "/v2/logs/search", key.secret, { + start_time: "2026-07-15T12:00:00.900Z", + end_time: "2026-07-15T12:00:01.100Z", + }) + expect(logs.status).toBe(200) + const logSql = observedSql.find((sql) => sql.includes("FROM logs")) + expect(logSql).toContain("'2026-07-15 12:00:00.900'") + expect(logSql).toContain("'2026-07-15 12:00:01.100'") + + observedSql.length = 0 + const trace = await harness.request("GET", `/v2/traces/${TRACE_ID}`, key.secret) + expect(trace.status).toBe(200) + const hierarchySql = observedSql.find((sql) => sql.includes("FROM trace_detail_spans")) + expect(hierarchySql).toContain(`TraceId = '${TRACE_ID}'`) + expect(hierarchySql).not.toContain("Timestamp >=") + expect(hierarchySql).not.toContain("Timestamp <=") + expect(hierarchySql).toContain("LIMIT 5001") + await harness.dispose() + }) + + it("maps public trace attribute grouping onto the validated internal query", async () => { + let observedRequest: QueryEngineExecuteRequest | undefined + const queryEngine: QueryEngineServiceShape = { + ...queryEngineStub, + execute: (_tenant, request) => { + observedRequest = request + return queryEngineStub.execute() + }, + } + const harness = makeHarness(warehouseStub, queryEngine) + const key = await harness.bootstrapKey() + const response = await harness.request("POST", "/v2/query", key.secret, { + start_time: START, + end_time: END, + query: { + kind: "timeseries", + source: "traces", + metric: "count", + group_by: ["attribute"], + filters: { group_by_attribute_keys: ["http.route"] }, + }, + }) + expect(response.status).toBe(200) + expect(observedRequest?.query).toMatchObject({ + seriesLimit: 50, + filters: { groupByAttributeKeys: ["http.route"] }, + }) + const excessiveLimit = await harness.request("POST", "/v2/query", key.secret, { + start_time: START, + end_time: END, + query: { + kind: "timeseries", + source: "traces", + metric: "count", + group_by: ["service"], + series_limit: 101, + }, + }) + expect(excessiveLimit.status).toBe(400) + await harness.dispose() + }) + + it("applies bounded ClickHouse settings to log body searches", async () => { + let observedOptions: Parameters[2] + const observingWarehouse: WarehouseQueryServiceShape = { + ...warehouseStub, + compiledQuery: (tenant, compiled, options) => { + observedOptions = options + return warehouseStub.compiledQuery(tenant, compiled, options) + }, + } + const harness = makeHarness(observingWarehouse) + const key = await harness.bootstrapKey(["logs:read"]) + const response = await harness.request("POST", "/v2/logs/search", key.secret, { + start_time: START, + end_time: END, + search: "checkout failed", + }) + expect(response.status).toBe(200) + expect(observedOptions?.settings).toMatchObject({ maxBlockSize: 512 }) + await harness.dispose() + }) + + it("sanitizes warehouse failures as operation-specific 503 errors", async () => { + const failure = new WarehouseQueryError({ + message: "SECRET_CLICKHOUSE_DIAGNOSTIC", + pipeName: "traceSummaries", + }) + const harness = makeHarness({ + ...warehouseStub, + compiledQuery: () => Effect.fail(failure), + }) + const key = await harness.bootstrapKey(["traces:read"]) + const response = await harness.request("POST", "/v2/traces/search", key.secret, { + start_time: START, + end_time: END, + }) + expect(response.status).toBe(503) + expect(response.body.error.code).toBe("trace_search_unavailable") + expect(JSON.stringify(response.body)).not.toContain("SECRET_CLICKHOUSE_DIAGNOSTIC") + await harness.dispose() + }) +}) diff --git a/apps/api/src/routes/v2/telemetry.http.ts b/apps/api/src/routes/v2/telemetry.http.ts new file mode 100644 index 000000000..5f7178535 --- /dev/null +++ b/apps/api/src/routes/v2/telemetry.http.ts @@ -0,0 +1,834 @@ +import { HttpApiBuilder } from "effect/unstable/httpapi" +import { CurrentTenant, MetricName, ServiceName, SpanId, TraceId } from "@maple/domain/http" +import { + MapleApiV2, + dependencyUnavailable, + invalidRequest, + paginateOffsetQuery, + resourceNotFound, + timestamp, + type Timestamp, + type V2Log, + type V2Metric, + type V2Service, + type V2ServiceMapEdge, + type V2Span, + type V2StructuredQueryResult, + type V2TraceSummary, +} from "@maple/domain/http/v2" +import { CH, QueryEngineExecuteRequest } from "@maple/query-engine" +import { LOGS_BODY_SEARCH_SETTINGS } from "@maple/query-engine/profiles" +import { MAX_QUERY_RANGE_SECONDS } from "@maple/query-engine/runtime" +import { Effect, Encoding, Option, Result, Schema } from "effect" +import { WarehouseQueryService } from "../../lib/WarehouseQueryService" +import { QueryEngineService } from "../../services/QueryEngineService" + +const decodeTraceId = Schema.decodeSync(TraceId) +const decodeSpanId = Schema.decodeSync(SpanId) +const decodeServiceName = Schema.decodeSync(ServiceName) +const decodeMetricName = Schema.decodeSync(MetricName) + +const metricCatalogRowSchema = Schema.Struct({ + metricName: Schema.String, + metricType: Schema.String, + serviceName: Schema.String, + metricDescription: Schema.String, + metricUnit: Schema.String, + dataPointCount: CH.CHNumber, + firstSeen: Schema.String, + lastSeen: Schema.String, + isMonotonic: Schema.Union([Schema.Boolean, CH.CHNumber]), +}) + +const serviceCatalogRowSchema = Schema.Struct({ + serviceName: Schema.String, + serviceNamespaces: Schema.Array(Schema.String), + deploymentEnvironments: Schema.Array(Schema.String), + spanCount: CH.CHNumber, + errorCount: CH.CHNumber, + estimatedErrorCount: CH.CHNumber, + estimatedSpanCount: CH.CHNumber, + p50LatencyMs: CH.CHNumber, + p95LatencyMs: CH.CHNumber, + p99LatencyMs: CH.CHNumber, +}) + +const PARTITION_HINT_RADIUS_MS = 60 * 60 * 1000 +const PUBLIC_TIMESERIES_DEFAULT_SERIES_LIMIT = 50 + +const mapWarehouseError = (operation: string) => () => dependencyUnavailable(`${operation}_unavailable`) + +const toWarehouseDateTime = (value: string, param: string) => { + const ms = Date.parse(value) + return Number.isNaN(ms) + ? Effect.fail(invalidRequest("parameter_invalid", `Invalid ISO-8601 timestamp for ${param}.`, param)) + : Effect.succeed(new Date(ms).toISOString().replace("T", " ").replace(/Z$/, "")) +} + +const parseWindow = (start: string, end: string) => + Effect.gen(function* () { + const startMs = Date.parse(start) + const endMs = Date.parse(end) + if (Number.isNaN(startMs) || Number.isNaN(endMs) || endMs <= startMs) { + return yield* Effect.fail( + invalidRequest("time_range_invalid", "end_time must be later than start_time.", "end_time"), + ) + } + const rangeSeconds = (endMs - startMs) / 1000 + if (rangeSeconds > MAX_QUERY_RANGE_SECONDS) { + return yield* Effect.fail( + invalidRequest( + "time_range_too_large", + "Telemetry queries support a maximum time range of 31 days.", + "start_time", + ), + ) + } + return { + startTime: yield* toWarehouseDateTime(start, "start_time"), + endTime: yield* toWarehouseDateTime(end, "end_time"), + rangeSeconds, + } + }) + +const chToIso = (value: string): Timestamp => { + const normalized = value.includes("T") ? value : value.replace(" ", "T") + const zoned = /[zZ]|[+-]\d\d:?\d\d$/.test(normalized) ? normalized : `${normalized}Z` + const ms = Date.parse(zoned) + return timestamp(Number.isNaN(ms) ? value : new Date(ms).toISOString()) +} + +const warehouseDate = (ms: number) => new Date(ms).toISOString().replace("T", " ").replace(/Z$/, "") +const partitionWindow = (value: string) => { + const ms = Date.parse(value.includes("T") ? value : `${value.replace(" ", "T")}Z`) + return { + startTime: warehouseDate(ms - PARTITION_HINT_RADIUS_MS), + endTime: warehouseDate(ms + PARTITION_HINT_RADIUS_MS), + } +} + +const parseStringRecord = (value: unknown): Record => { + if (typeof value !== "string") return {} + try { + const parsed = JSON.parse(value) as unknown + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return {} + return Object.fromEntries( + Object.entries(parsed).map(([key, entry]) => [ + key, + typeof entry === "string" ? entry : String(entry), + ]), + ) + } catch { + return {} + } +} + +type LogKey = readonly [timestamp: string, recordIdentity: string] +const compactTimestamp = (value: string) => { + const match = /^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(\.\d+)?$/.exec(value) + if (match === null) return value + const epochSeconds = Date.parse(`${match[1]}T${match[2]}Z`) / 1000 + return Number.isSafeInteger(epochSeconds) ? `~${epochSeconds.toString(36)}${match[3] ?? ""}` : value +} +const expandTimestamp = (value: string) => { + const match = /^~([0-9a-z]+)(\.\d+)?$/.exec(value) + if (match === null) return value + const epochSeconds = Number.parseInt(match[1]!, 36) + if (!Number.isSafeInteger(epochSeconds)) return value + const seconds = new Date(epochSeconds * 1000).toISOString().slice(0, 19).replace("T", " ") + return `${seconds}${match[2] ?? ""}` +} +const compactHexId = (value: string) => { + if (!/^(?:[0-9a-f]{16}|[0-9a-f]{32})$/i.test(value)) return value + const bytes = Uint8Array.from({ length: value.length / 2 }, (_, index) => + Number.parseInt(value.slice(index * 2, index * 2 + 2), 16), + ) + return `~${Encoding.encodeBase64Url(bytes)}` +} +const expandHexId = (value: string) => { + if (!value.startsWith("~")) return value + const decoded = Encoding.decodeBase64Url(value.slice(1)) + if (Result.isFailure(decoded)) throw new Error("invalid compact identifier") + return [...decoded.success].map((byte) => byte.toString(16).padStart(2, "0")).join("") +} +const logKey = (row: { timestamp: string; recordIdentity: string }) => + JSON.stringify([compactTimestamp(row.timestamp), compactHexId(row.recordIdentity)] satisfies LogKey) + +const parseLogKey = (value: string) => { + try { + const parsed = JSON.parse(value) as unknown + if ( + !Array.isArray(parsed) || + parsed.length !== 2 || + parsed.some((part) => typeof part !== "string") || + Number.isNaN(Date.parse(expandTimestamp(parsed[0] as string).replace(" ", "T") + "Z")) || + !/^[0-9A-F]{32}$/i.test(expandHexId(parsed[1] as string)) + ) { + throw new Error("invalid") + } + return Effect.succeed([ + expandTimestamp(parsed[0] as string), + expandHexId(parsed[1] as string).toUpperCase(), + ] as const) + } catch { + return Effect.fail(invalidRequest("log_id_invalid", "Malformed log ID.", "id")) + } +} + +const encodeKeysetCursor = (prefix: string, parts: ReadonlyArray) => + `${prefix}_${Encoding.encodeBase64Url(JSON.stringify(parts))}` + +const decodeKeysetCursor = (value: string | undefined, prefix: string, length: number) => { + if (value === undefined) return Effect.succeed | undefined>(undefined) + if (!value.startsWith(`${prefix}_`)) { + return Effect.fail(invalidRequest("parameter_invalid", "Invalid pagination cursor.", "cursor")) + } + const decoded = Encoding.decodeBase64UrlString(value.slice(prefix.length + 1)) + if (Result.isFailure(decoded)) { + return Effect.fail(invalidRequest("parameter_invalid", "Invalid pagination cursor.", "cursor")) + } + try { + const parts = JSON.parse(decoded.success) as unknown + return Array.isArray(parts) && + parts.length === length && + parts.every((part) => typeof part === "string") + ? Effect.succeed(parts as ReadonlyArray) + : Effect.fail(invalidRequest("parameter_invalid", "Invalid pagination cursor.", "cursor")) + } catch { + return Effect.fail(invalidRequest("parameter_invalid", "Invalid pagination cursor.", "cursor")) + } +} + +const toLog = (row: { + timestamp: string + severityText: string + severityNumber: number + serviceName: string + body: string + traceId: string + spanId: string + recordIdentity: string + logAttributes: string + resourceAttributes: string +}): V2Log => ({ + id: logKey(row), + object: "log", + timestamp: chToIso(row.timestamp), + severity_text: row.severityText, + severity_number: Number(row.severityNumber), + service_name: decodeServiceName(row.serviceName), + body: row.body, + trace_id: row.traceId ? decodeTraceId(row.traceId) : null, + span_id: row.spanId ? decodeSpanId(row.spanId) : null, + log_attributes: parseStringRecord(row.logAttributes), + resource_attributes: parseStringRecord(row.resourceAttributes), +}) + +const toTraceSummary = (row: { + traceId: string + startTime: string + durationMs: number + rootSpanName: string + rootSpanKind: string + rootServiceName: string + statusCode: string + hasError: number + deploymentEnvironment: string + serviceNamespace: string + httpMethod: string + httpRoute: string + httpStatusCode: string +}): V2TraceSummary => ({ + id: decodeTraceId(row.traceId), + object: "trace", + start_time: chToIso(row.startTime), + duration_ms: Number(row.durationMs), + root_span_name: row.rootSpanName, + root_span_kind: row.rootSpanKind, + root_service_name: row.rootServiceName, + status_code: row.statusCode, + has_error: Number(row.hasError) !== 0, + deployment_environment: row.deploymentEnvironment || null, + service_namespace: row.serviceNamespace || null, + http_method: row.httpMethod || null, + http_route: row.httpRoute || null, + http_status_code: row.httpStatusCode || null, +}) + +const toSpan = (row: { + traceId: string + spanId: string + parentSpanId: string + spanName: string + serviceName: string + spanKind: string + durationMs: number + startTime: string + statusCode: string + statusMessage: string + spanAttributes: string + resourceAttributes: string +}): V2Span => ({ + id: decodeSpanId(row.spanId), + object: "span", + trace_id: decodeTraceId(row.traceId), + parent_span_id: row.parentSpanId ? decodeSpanId(row.parentSpanId) : null, + name: row.spanName, + service_name: row.serviceName, + kind: row.spanKind, + start_time: chToIso(row.startTime), + duration_ms: Number(row.durationMs), + status_code: row.statusCode, + status_message: row.statusMessage || null, + attributes: parseStringRecord(row.spanAttributes), + resource_attributes: parseStringRecord(row.resourceAttributes), +}) + +const attributeFilters = ( + filters: + | ReadonlyArray<{ + key: string + value?: string + mode: string + negated?: boolean + }> + | undefined, +) => + filters?.map((filter) => ({ + key: filter.key, + ...(filter.value !== undefined ? { value: filter.value } : {}), + mode: filter.mode, + ...(filter.negated !== undefined ? { negated: filter.negated } : {}), + })) + +const toInternalQuery = (query: Record): Record => { + const mapped: Record = { + kind: query.kind, + source: query.source, + metric: query.metric, + groupBy: query.group_by, + bucketSeconds: query.bucket_seconds, + seriesLimit: + query.kind === "timeseries" + ? (query.series_limit ?? PUBLIC_TIMESERIES_DEFAULT_SERIES_LIMIT) + : undefined, + limit: query.limit, + apdexThresholdMs: query.apdex_threshold_ms, + } + const filters = query.filters + if (filters) { + mapped.filters = { + serviceName: filters.service_name, + spanName: filters.span_name, + rootSpansOnly: filters.root_spans_only, + errorsOnly: filters.errors_only, + environments: filters.environments, + namespaces: filters.namespaces, + minDurationMs: filters.min_duration_ms, + maxDurationMs: filters.max_duration_ms, + severity: filters.severity, + traceId: filters.trace_id, + search: filters.search, + metricName: filters.metric_name, + metricType: filters.metric_type, + groupByAttributeKey: filters.group_by_attribute_key, + groupByAttributeKeys: filters.group_by_attribute_keys, + groupByResourceAttributeKey: filters.group_by_resource_attribute_key, + attributeFilters: attributeFilters(filters.attribute_filters), + resourceAttributeFilters: attributeFilters(filters.resource_attribute_filters), + } + } + return Object.fromEntries(Object.entries(mapped).filter(([, value]) => value !== undefined)) +} + +const structuredResult = (result: { + kind: "timeseries" | "breakdown" + source: "traces" | "logs" | "metrics" + data: ReadonlyArray +}): V2StructuredQueryResult => ({ + object: "query_result", + kind: result.kind, + source: result.source, + timeseries: + result.kind === "timeseries" + ? (result.data as ReadonlyArray<{ bucket: string; series: Record }>).map( + (point) => ({ bucket: chToIso(point.bucket), series: point.series }), + ) + : [], + breakdown: result.kind === "breakdown" ? (result.data as V2StructuredQueryResult["breakdown"]) : [], +}) + +const queryError = (error: unknown) => { + const tag = typeof error === "object" && error !== null && "_tag" in error ? String(error._tag) : "" + return tag.includes("Validation") + ? invalidRequest("query_invalid", "The query specification is invalid.", "query") + : dependencyUnavailable("query_unavailable") +} + +const decodeQueryEngineRequest = (input: unknown) => + Schema.decodeUnknownEffect(QueryEngineExecuteRequest)(input).pipe( + Effect.mapError(() => + invalidRequest("query_invalid", "The query specification is invalid.", "query"), + ), + ) + +export const HttpV2TracesLive = HttpApiBuilder.group(MapleApiV2, "traces", (handlers) => + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + + const hierarchy = Effect.fn("HttpV2Traces.hierarchy")(function* ( + tenant: CurrentTenant.TenantSchema, + traceId: string, + ) { + const compiled = CH.compile( + CH.spanHierarchyQuery({ traceId, limit: CH.SPAN_HIERARCHY_MAX_SPANS + 1 }), + { + orgId: tenant.orgId, + }, + ) + return yield* warehouse + .compiledQuery(tenant, compiled, { + profile: "list", + context: "v2GetTrace", + }) + .pipe(Effect.mapError(mapWarehouseError("trace_query"))) + }) + + return handlers + .handle("search", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(payload.start_time, payload.end_time) + const limit = payload.limit ?? 20 + const cursorParts = yield* decodeKeysetCursor(payload.cursor, "trc", 2) + const compiled = CH.compile( + CH.traceSummariesQuery({ + serviceName: payload.service_name, + spanName: payload.span_name, + hasError: payload.has_error, + minDurationMs: payload.min_duration_ms, + maxDurationMs: payload.max_duration_ms, + httpMethod: payload.http_method, + httpStatusCode: payload.http_status_code, + deploymentEnv: payload.deployment_environment, + namespace: payload.service_namespace, + limit: limit + 1, + cursor: cursorParts + ? { timestamp: cursorParts[0]!, traceId: cursorParts[1]! } + : undefined, + }), + { orgId: tenant.orgId, ...window }, + ) + const rows = yield* warehouse + .compiledQuery(tenant, compiled, { profile: "list", context: "v2TraceSearch" }) + .pipe(Effect.mapError(mapWarehouseError("trace_search"))) + const dataRows = rows.slice(0, limit) + const last = dataRows.at(-1) + const hasMore = rows.length > limit + return { + object: "list" as const, + data: dataRows.map(toTraceSummary), + has_more: hasMore, + next_cursor: + hasMore && last + ? encodeKeysetCursor("trc", [last.startTime, last.traceId]) + : null, + } + }), + ) + .handle("retrieve", ({ params }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* hierarchy(tenant, params.trace_id) + if (rows.length === 0) return yield* resourceNotFound("trace", "No such trace.") + const truncated = rows.length > CH.SPAN_HIERARCHY_MAX_SPANS + const spans = rows.slice(0, CH.SPAN_HIERARCHY_MAX_SPANS).map(toSpan) + const startMs = Math.min(...spans.map((span) => Date.parse(span.start_time))) + const endMs = Math.max( + ...spans.map((span) => Date.parse(span.start_time) + span.duration_ms), + ) + return { + id: params.trace_id, + object: "trace" as const, + start_time: timestamp(new Date(startMs).toISOString()), + end_time: timestamp(new Date(endMs).toISOString()), + duration_ms: endMs - startMs, + span_count: spans.length, + service_count: new Set(spans.map((span) => span.service_name)).size, + truncated, + spans, + } + }), + ) + .handle("retrieveSpan", ({ params }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const detail = yield* warehouse + .compiledQueryFirst( + tenant, + CH.compile( + CH.spanDetailQuery({ + traceId: params.trace_id, + spanId: params.span_id, + }), + { orgId: tenant.orgId }, + ), + { profile: "discovery", context: "v2GetSpan" }, + ) + .pipe(Effect.mapError(mapWarehouseError("span_query")), Effect.map(Option.getOrNull)) + if (!detail) return yield* resourceNotFound("span", "No such span.") + return toSpan(detail) + }), + ) + }), +) + +export const HttpV2LogsLive = HttpApiBuilder.group(MapleApiV2, "logs", (handlers) => + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + return handlers + .handle("search", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(payload.start_time, payload.end_time) + const limit = payload.limit ?? 20 + const cursorParts = yield* decodeKeysetCursor(payload.cursor, "log", 5) + const compiled = CH.compile( + CH.logsListQuery({ + serviceName: payload.service_name, + severity: payload.severity, + minSeverity: payload.min_severity, + traceId: payload.trace_id, + spanId: payload.span_id, + search: payload.search, + environments: payload.deployment_environment + ? [payload.deployment_environment] + : undefined, + namespaces: payload.service_namespace ? [payload.service_namespace] : undefined, + limit: limit + 1, + cursorIdentity: cursorParts + ? { + timestamp: cursorParts[0]!, + serviceName: cursorParts[1]!, + traceId: cursorParts[2]!, + spanId: cursorParts[3]!, + recordIdentity: cursorParts[4]!, + } + : undefined, + }), + { orgId: tenant.orgId, ...window }, + ) + const rows = yield* warehouse + .compiledQuery(tenant, compiled, { + profile: "list", + context: "v2LogSearch", + settings: payload.search ? LOGS_BODY_SEARCH_SETTINGS : undefined, + }) + .pipe(Effect.mapError(mapWarehouseError("log_search"))) + const dataRows = rows.slice(0, limit) + const last = dataRows.at(-1) + const hasMore = rows.length > limit + return { + object: "list" as const, + data: dataRows.map(toLog), + has_more: hasMore, + next_cursor: + hasMore && last + ? encodeKeysetCursor("log", [ + last.timestamp, + last.serviceName, + last.traceId, + last.spanId, + last.recordIdentity, + ]) + : null, + } + }), + ) + .handle("retrieve", ({ params }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const [logTimestamp, recordIdentity] = yield* parseLogKey(params.id) + const compiled = CH.compile( + CH.getLogByKeyQuery({ + recordIdentity, + }), + { + orgId: tenant.orgId, + ...partitionWindow(logTimestamp), + timestamp: logTimestamp, + }, + ) + const row = yield* warehouse + .compiledQueryFirst(tenant, compiled, { + profile: "list", + context: "v2GetLog", + }) + .pipe(Effect.mapError(mapWarehouseError("log_query")), Effect.map(Option.getOrNull)) + if (!row) return yield* resourceNotFound("log", "No such log.") + return toLog(row) + }), + ) + }), +) + +export const HttpV2MetricsLive = HttpApiBuilder.group(MapleApiV2, "metrics", (handlers) => + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + const queryEngine = yield* QueryEngineService + return handlers + .handle("list", ({ query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(query.start_time, query.end_time) + const page = yield* paginateOffsetQuery(query, ({ limit, offset }) => { + const compiled = CH.compile( + CH.listMetricsQuery({ + serviceName: query.service_name, + metricType: query.metric_type, + search: query.search, + limit, + offset, + }), + { orgId: tenant.orgId, ...window }, + { rowSchema: metricCatalogRowSchema }, + ) + return warehouse + .compiledQuery(tenant, compiled, { + profile: "discovery", + context: "v2ListMetrics", + }) + .pipe( + Effect.mapError(mapWarehouseError("metric_catalog")), + Effect.map( + (rows): ReadonlyArray => + rows.map((row) => ({ + object: "metric", + name: decodeMetricName(row.metricName), + type: row.metricType, + service_name: row.serviceName, + description: row.metricDescription, + unit: row.metricUnit, + is_monotonic: Number(row.isMonotonic) !== 0, + data_point_count: Number(row.dataPointCount), + first_seen: chToIso(row.firstSeen), + last_seen: chToIso(row.lastSeen), + })), + ), + ) + }) + return { object: "list" as const, ...page } + }), + ) + .handle("timeseries", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(payload.start_time, payload.end_time) + const request = yield* decodeQueryEngineRequest({ + startTime: window.startTime, + endTime: window.endTime, + query: { + kind: "timeseries", + source: "metrics", + metric: payload.metric, + groupBy: payload.group_by, + bucketSeconds: payload.bucket_seconds, + seriesLimit: payload.series_limit ?? PUBLIC_TIMESERIES_DEFAULT_SERIES_LIMIT, + filters: { + metricName: payload.metric_name, + metricType: payload.metric_type, + serviceName: payload.service_name, + groupByAttributeKey: payload.group_by_attribute_key, + groupByResourceAttributeKey: payload.group_by_resource_attribute_key, + attributeFilters: attributeFilters(payload.attribute_filters), + resourceAttributeFilters: attributeFilters( + payload.resource_attribute_filters, + ), + }, + }, + }) + const response = yield* queryEngine + .execute(tenant, request) + .pipe(Effect.mapError(queryError)) + if (response.result.kind !== "timeseries") + return yield* Effect.fail(dependencyUnavailable("metric_query_unavailable")) + return structuredResult(response.result) + }), + ) + }), +) + +const toService = ( + row: { + serviceName: string + serviceNamespaces: readonly string[] + deploymentEnvironments: readonly string[] + spanCount: number + errorCount: number + estimatedErrorCount: number + estimatedSpanCount: number + p50LatencyMs: number + p95LatencyMs: number + p99LatencyMs: number + }, + rangeSeconds: number, +): V2Service => { + const spanCount = Number(row.spanCount) + const estimatedSpanCount = Number(row.estimatedSpanCount) + const estimatedErrorCount = Number(row.estimatedErrorCount) + return { + object: "service", + name: decodeServiceName(row.serviceName), + service_namespaces: [...row.serviceNamespaces], + deployment_environments: [...row.deploymentEnvironments], + throughput: estimatedSpanCount / rangeSeconds, + traced_throughput: spanCount / rangeSeconds, + span_count: spanCount, + error_count: Number(row.errorCount), + error_rate: estimatedSpanCount > 0 ? estimatedErrorCount / estimatedSpanCount : 0, + p50_latency_ms: Number(row.p50LatencyMs), + p95_latency_ms: Number(row.p95LatencyMs), + p99_latency_ms: Number(row.p99LatencyMs), + has_sampling: estimatedSpanCount > spanCount + 0.001, + sampling_weight: spanCount > 0 ? estimatedSpanCount / spanCount : 1, + } +} + +export const HttpV2ServicesLive = HttpApiBuilder.group(MapleApiV2, "services", (handlers) => + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + const execute = ( + tenant: CurrentTenant.TenantSchema, + window: { startTime: string; endTime: string; rangeSeconds: number }, + opts: Parameters[0], + ) => { + const compiled = CH.compile( + CH.serviceCatalogQuery(opts), + { orgId: tenant.orgId, ...window }, + { rowSchema: serviceCatalogRowSchema }, + ) + return warehouse + .compiledQuery(tenant, compiled, { + profile: "aggregation", + context: "v2ServiceCatalog", + }) + .pipe( + Effect.mapError(mapWarehouseError("service_query")), + Effect.map((rows) => rows.map((row) => toService(row, window.rangeSeconds))), + ) + } + return handlers + .handle("list", ({ query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(query.start_time, query.end_time) + const page = yield* paginateOffsetQuery(query, ({ limit, offset }) => + execute(tenant, window, { + deploymentEnvironment: query.deployment_environment, + serviceNamespace: query.service_namespace, + limit, + offset, + }), + ) + return { object: "list" as const, ...page } + }), + ) + .handle("retrieve", ({ params, query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(query.start_time, query.end_time) + const rows = yield* execute(tenant, window, { + serviceName: params.name, + limit: 1, + }) + if (!rows[0]) return yield* resourceNotFound("service", "No such service.") + return rows[0] + }), + ) + }), +) + +const toMapEdge = (row: { + sourceService: string + targetService: string + callCount: number + errorCount: number + avgDurationMs: number + p95DurationMs: number + estimatedSpanCount: number +}): V2ServiceMapEdge => { + const calls = Number(row.callCount) + const estimated = Number(row.estimatedSpanCount) + const errors = Number(row.errorCount) + return { + object: "service_map.edge", + source_service: row.sourceService, + target_service: row.targetService, + call_count: calls, + estimated_call_count: estimated, + error_count: errors, + error_rate: calls > 0 ? errors / calls : 0, + avg_duration_ms: Number(row.avgDurationMs), + max_duration_ms: Number(row.p95DurationMs), + has_sampling: estimated > calls + 0.001, + sampling_weight: calls > 0 ? estimated / calls : 1, + } +} + +export const HttpV2ServiceMapLive = HttpApiBuilder.group(MapleApiV2, "serviceMap", (handlers) => + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + return handlers.handle("retrieve", ({ query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(query.start_time, query.end_time) + const compiled = query.service_name + ? CH.compile( + CH.serviceDependenciesForServiceQuery({ + serviceName: query.service_name, + deploymentEnv: query.deployment_environment, + }), + { orgId: tenant.orgId, ...window }, + ) + : CH.serviceDependenciesSQL( + { deploymentEnv: query.deployment_environment }, + { orgId: tenant.orgId, ...window }, + ) + const rows = yield* warehouse + .compiledQuery(tenant, compiled, { + profile: "aggregation", + context: "v2ServiceMap", + }) + .pipe(Effect.mapError(mapWarehouseError("service_map_query"))) + return { + object: "service_map" as const, + start_time: timestamp(query.start_time), + end_time: timestamp(query.end_time), + edges: rows.map(toMapEdge), + } + }), + ) + }), +) + +export const HttpV2QueryLive = HttpApiBuilder.group(MapleApiV2, "query", (handlers) => + Effect.gen(function* () { + const queryEngine = yield* QueryEngineService + return handlers.handle("execute", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(payload.start_time, payload.end_time) + const request = yield* decodeQueryEngineRequest({ + startTime: window.startTime, + endTime: window.endTime, + query: toInternalQuery(payload.query), + }) + const response = yield* queryEngine.execute(tenant, request).pipe(Effect.mapError(queryError)) + if (response.result.kind !== "timeseries" && response.result.kind !== "breakdown") { + return yield* Effect.fail(dependencyUnavailable("query_unavailable")) + } + return structuredResult(response.result) + }), + ) + }), +) diff --git a/apps/api/src/routes/v2/v2-test-support.ts b/apps/api/src/routes/v2/v2-test-support.ts index 73531a336..d0e046ada 100644 --- a/apps/api/src/routes/v2/v2-test-support.ts +++ b/apps/api/src/routes/v2/v2-test-support.ts @@ -8,7 +8,9 @@ import { OrganizationService } from "../../services/OrganizationService" import { OrgIngestKeysService } from "../../services/OrgIngestKeysService" import { RecommendationIssueService } from "../../services/RecommendationIssueService" import { ScrapeTargetsService } from "../../services/ScrapeTargetsService" +import { ApiV2RateLimiter } from "../../services/ApiV2RateLimiter" import { WarehouseQueryService } from "../../lib/WarehouseQueryService" +import { QueryEngineService } from "../../services/QueryEngineService" import { HttpV2AlertDestinationsLive } from "./alert-destinations.http" import { HttpV2AlertIncidentsLive } from "./alert-incidents.http" import { HttpV2AlertRulesLive } from "./alert-rules.http" @@ -22,6 +24,14 @@ import { HttpV2OrganizationLive } from "./organization.http" import { HttpV2InstrumentationRecommendationsLive } from "./recommendations.http" import { HttpV2ScrapeTargetsLive } from "./scrape-targets.http" import { HttpV2SessionReplaysLive } from "./session-replays.http" +import { + HttpV2LogsLive, + HttpV2MetricsLive, + HttpV2QueryLive, + HttpV2ServiceMapLive, + HttpV2ServicesLive, + HttpV2TracesLive, +} from "./telemetry.http" /** * Test-only support for the v2 HTTP harnesses. `HttpApiBuilder.layer(MapleApiV2)` @@ -44,8 +54,18 @@ export const AllV2GroupLayersLive = Layer.mergeAll( HttpV2AnomaliesLive, HttpV2OrganizationLive, HttpV2SessionReplaysLive, + HttpV2TracesLive, + HttpV2LogsLive, + HttpV2MetricsLive, + HttpV2ServicesLive, + HttpV2ServiceMapLive, + HttpV2QueryLive, ) +export const ApiV2RateLimiterAllowAllLayer = Layer.succeed(ApiV2RateLimiter, { + check: () => Effect.succeed("allowed" as const), +}) + const die = () => Effect.die(new Error("AlertsService is not available in this test harness")) /** Synchronous stub for non-Effect-returning service methods (e.g. `asExecutor`). */ @@ -112,12 +132,23 @@ export const Phase1ResourceStubsLayer = Layer.mergeAll( export const WarehouseServiceStubLayer = Layer.succeed(WarehouseQueryService, { query: die, sqlQuery: die, + rawSqlQuery: die, compiledQuery: die, compiledQueryFirst: die, ingest: die, asExecutor: dieSync, }) +export const TelemetryServiceStubsLayer = Layer.mergeAll( + Layer.succeed(QueryEngineService, { + execute: die, + evaluate: die, + evaluateRawSql: die, + evaluateSeries: die, + cachedDirect: die, + }), +) + /** Inert config-resource services for harnesses that never touch those groups. */ export const ConfigResourceServiceStubsLayer = Layer.mergeAll( Layer.succeed(IngestAttributeMappingService, { diff --git a/apps/api/src/services/AlertsService.test.ts b/apps/api/src/services/AlertsService.test.ts index ff9bb947b..a2b215bb8 100644 --- a/apps/api/src/services/AlertsService.test.ts +++ b/apps/api/src/services/AlertsService.test.ts @@ -15,7 +15,13 @@ import { } from "@maple/domain/http" import type { WarehouseQueryServiceShape } from "../lib/WarehouseQueryService" import { WarehouseQueryService } from "../lib/WarehouseQueryService" -import { AlertRuntime, type AlertRuntimeShape, AlertsService, type AlertsServiceShape } from "./AlertsService" +import { + AlertRuntime, + type AlertRuntimeShape, + AlertsService, + type AlertsServiceShape, + interleaveAlertRulesByOrg, +} from "./AlertsService" import { BucketCacheService, EdgeCacheService } from "@maple/query-engine/caching" import { CacheBackendLive } from "../lib/CacheBackendLive" import { Env } from "../lib/Env" @@ -29,6 +35,23 @@ const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) +describe("interleaveAlertRulesByOrg", () => { + it("round-robins organizations while preserving per-org order", () => { + const rows = [ + { orgId: "a", id: "a1" }, + { orgId: "a", id: "a2" }, + { orgId: "a", id: "a3" }, + { orgId: "b", id: "b1" }, + { orgId: "b", id: "b2" }, + { orgId: "c", id: "c1" }, + ] + assert.deepStrictEqual( + interleaveAlertRulesByOrg(rows).map((row) => row.id), + ["a1", "b1", "c1", "a2", "b2", "a3"], + ) + }) +}) + const getError = (exit: Exit.Exit): unknown => { if (!Exit.isFailure(exit)) return undefined @@ -81,6 +104,7 @@ function makeWarehouseStub(state: { return { query: (_tenant, payload) => Effect.die(new Error(`Unexpected pipe ${payload.pipeName}`)), sqlQuery: sqlQueryStub, + rawSqlQuery: sqlQueryStub, compiledQuery: (_tenant, compiled) => sqlQueryStub().pipe(Effect.flatMap((rows) => compiled.decodeRows(rows).pipe(Effect.orDie))), compiledQueryFirst: (_tenant, compiled) => @@ -309,6 +333,54 @@ const insertDeliveryEventRow = async ( } describe("AlertsService", () => { + it.effect("caps active alert rules per organization", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const alerts = yield* AlertsService + const orgId = asOrgId("org_active_rule_cap") + const userId = asUserId("user_active_rule_cap") + const destination = yield* createWebhookDestination(alerts, orgId, userId) + const createRule = (index: number) => + alerts.createRule( + orgId, + userId, + adminRoles, + new AlertRuleUpsertRequest({ + name: `Capped rule ${index}`, + severity: "warning", + enabled: true, + signalType: "error_rate", + comparator: "gt", + threshold: 5, + windowMinutes: 5, + destinationIds: [destination.id], + }), + ) + yield* Effect.forEach( + Array.from({ length: 99 }, (_, index) => index), + createRule, + { concurrency: 1, discard: true }, + ) + + const exits = yield* Effect.all( + [createRule(99).pipe(Effect.exit), createRule(100).pipe(Effect.exit)], + { + concurrency: "unbounded", + }, + ) + assert.strictEqual(exits.filter(Exit.isSuccess).length, 1) + assert.strictEqual(exits.filter(Exit.isFailure).length, 1) + const error = getError(exits.find(Exit.isFailure)!) + assert.instanceOf(error, AlertValidationError) + assert.include((error as AlertValidationError).message, "at most 100 active alert rules") + const rules = yield* alerts.listRules(orgId) + assert.lengthOf( + rules.rules.filter((rule) => rule.enabled), + 100, + ) + }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub({}), { fetch: okFetch }))) + }) + it.effect("opens an incident after consecutive breaches and delivers the webhook notification", () => { const testDb = createTestDb(trackedDbs) const state = { @@ -2425,6 +2497,7 @@ describe("AlertsService evaluation error persistence", () => { return { query: () => Effect.die(new Error("Unexpected pipe query")), sqlQuery: sqlQueryStub, + rawSqlQuery: sqlQueryStub, compiledQuery: (_tenant, compiled) => sqlQueryStub().pipe(Effect.flatMap((rows) => compiled.decodeRows(rows).pipe(Effect.orDie))), compiledQueryFirst: (_tenant, compiled) => @@ -2652,7 +2725,7 @@ describe("AlertsService.previewRule", () => { }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub(state), { fetch: okFetch }))) }) - it.effect("replays raw-SQL rules per evaluation window", () => { + it.effect("rejects raw-SQL previews before warehouse execution", () => { const testDb = createTestDb(trackedDbs) const state = { rawQueryRows: [{ value: 42, samples: 20 }] } @@ -2666,7 +2739,8 @@ describe("AlertsService.previewRule", () => { severity: "warning", enabled: true, signalType: "raw_query", - rawQuerySql: "SELECT count() AS value FROM traces WHERE $__orgFilter", + rawQuerySql: + "SELECT count() AS value FROM traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", rawQueryReducer: "max", comparator: "gt", threshold: 10, @@ -2681,13 +2755,11 @@ describe("AlertsService.previewRule", () => { endTime: "2026-01-01T00:30:00.000Z", }) - const response = yield* alerts.previewRule(orgId, request) - - assert.lengthOf(response.series, 1) - const points = response.series[0]!.points - assert.lengthOf(points, 6) - assert.isTrue(points.every((point) => point.value === 42 && point.status === "breached")) - assert.lengthOf(response.wouldFire, 1) + const exit = yield* alerts.previewRule(orgId, request).pipe(Effect.exit) + assert.isTrue(Exit.isFailure(exit)) + const failure = getError(exit) + assert.instanceOf(failure, AlertValidationError) + assert.include((failure as AlertValidationError).message, "cannot be previewed") }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub(state), { fetch: okFetch }))) }) }) diff --git a/apps/api/src/services/AlertsService.ts b/apps/api/src/services/AlertsService.ts index 306bc1afb..2325ef9f4 100644 --- a/apps/api/src/services/AlertsService.ts +++ b/apps/api/src/services/AlertsService.ts @@ -8,6 +8,7 @@ import { } from "@maple/query-engine" import * as CH from "@maple/query-engine/ch" import { buildTimeseriesQuerySpec, resolveGroupBy } from "@maple/query-engine/query-builder" +import { prepareRawSql } from "@maple/query-engine/runtime" import { AlertComparator as AlertComparatorSchema, AlertDeliveryError, @@ -84,7 +85,7 @@ import { type AlertRuleRow, alertRuleStates, } from "@maple/db" -import { and, asc, desc, eq, inArray, isNotNull, isNull, lt, lte, ne, or } from "drizzle-orm" +import { and, asc, desc, eq, inArray, isNotNull, isNull, lt, lte, ne, or, sql } from "drizzle-orm" import { Array as Arr, Cause, @@ -335,7 +336,7 @@ const planGroupingTokens = ( } const isGroupedPlan = (plan: Schema.Schema.Type): boolean => - planGroupingTokens(plan) != null + plan.kind === "raw_sql" || planGroupingTokens(plan) != null const resolveServiceLinkName = ( rule: Pick, @@ -390,11 +391,32 @@ export class AlertRuntime extends Context.Reference("@maple/a const toIso = (value: Date | null | undefined): IsoDateTimeValue | null => value == null ? null : decodeIsoDateTimeStringSync(value.toISOString()) -// Caps on how many evaluation windows a rule preview replays. Spec plans cost -// one CH query regardless of bucket count; raw-SQL plans run one query per -// window, so they get a tighter cap. +// Cap on how many evaluation windows a structured rule preview replays. const MAX_PREVIEW_BUCKETS = 200 -const MAX_RAW_PREVIEW_WINDOWS = 60 +const MAX_ACTIVE_ALERT_RULES_PER_ORG = 100 + +/** Preserve each org's oldest-first order while preventing one org from monopolizing a tick. */ +export const interleaveAlertRulesByOrg = ( + rows: ReadonlyArray, +): ReadonlyArray => { + const queues = new Map() + for (const row of rows) { + const queue = queues.get(row.orgId) + if (queue) queue.push(row) + else queues.set(row.orgId, [row]) + } + + const fair: T[] = [] + let index = 0 + while (fair.length < rows.length) { + for (const queue of queues.values()) { + const row = queue[index] + if (row !== undefined) fair.push(row) + } + index += 1 + } + return fair +} // Tinybird DateTime64(3) wire format for alert_checks ingest: // "YYYY-MM-DD HH:MM:SS.SSS" (UTC, no timezone). @@ -827,10 +849,13 @@ const compileRulePlan = Effect.fn("AlertsService.compileRulePlan")(function* (ru const parseCompiledPlan = ( row: Pick< AlertRuleRow, - "querySpecJson" | "rawQuerySql" | "reducer" | "sampleCountStrategy" | "noDataBehavior" + "signalType" | "querySpecJson" | "rawQuerySql" | "reducer" | "sampleCountStrategy" | "noDataBehavior" >, ): Effect.Effect, AlertValidationError> => { - if (row.rawQuerySql != null) { + if (row.signalType === "raw_query") { + if (row.rawQuerySql == null) { + return Effect.fail(makeValidationError("Stored raw alert is missing its SQL query")) + } return Schema.decodeUnknownEffect(CompiledAlertQueryPlan)({ kind: "raw_sql", query: null, @@ -938,7 +963,7 @@ const rowToRuleDocument = ( row.metricAggregation != null ? decodeAlertMetricAggregationSync(row.metricAggregation) : null, apdexThresholdMs: row.apdexThresholdMs, queryBuilderDraft: parseStoredQueryBuilderDraft(row.queryBuilderDraftJson), - rawQuerySql: row.rawQuerySql ?? null, + rawQuerySql: row.signalType === "raw_query" ? (row.rawQuerySql ?? null) : null, rawQueryReducer: row.signalType === "raw_query" ? decodeQueryEngineAlertReducerSync(row.reducer) : null, destinationIds: destinationIds.map((id) => decodeAlertDestinationIdSync(id)), @@ -1306,6 +1331,7 @@ export class AlertsService extends Context.Service 0) { + details.push("serviceNames is not supported for raw_query alerts") + } + if (groupBy != null) { + details.push( + "groupBy is not supported for raw_query alerts; return a group column instead", + ) + } + } else { + if (normalizeOptionalString(request.rawQuerySql) != null) { + details.push("rawQuerySql is only supported for raw_query alerts") + } + if (request.rawQueryReducer != null) { + details.push("rawQueryReducer is only supported for raw_query alerts") } } const allowsMetricFields = request.signalType === "metric" @@ -1377,7 +1417,6 @@ export class AlertsService extends Context.Service 0) { details.push("groupBy is only supported when no service is specified") } @@ -1393,6 +1432,20 @@ export class AlertsService extends Context.Service 0) { return yield* Effect.fail(makeValidationError("Invalid alert rule", details)) } + if (request.signalType === "raw_query") { + yield* prepareRawSql({ + sql: request.rawQuerySql ?? "", + orgId, + startTime: "2000-01-01 00:00:00", + endTime: "2000-01-01 00:05:00", + granularitySeconds: 60, + workload: "alert", + }).pipe( + Effect.mapError((error) => + makeValidationError("Invalid raw SQL alert query", [error.message], error), + ), + ) + } const nowMs = yield* now const normalizedBase = { @@ -1421,8 +1474,12 @@ export class AlertsService extends Context.Service - db - .insert(alertRules) - .values({ - id: ruleId, - orgId, - ...ruleFields, - createdAt: new Date(timestamp), - createdBy: userId, - }) - .returning(txidColumn), - ) - : yield* dbExecute((db) => - db - .update(alertRules) - .set(ruleFields) - .where(and(eq(alertRules.orgId, orgId), eq(alertRules.id, existingId))) - .returning(txidColumn), - ) - const txid = readTxid(writeRows) + const writeResult = yield* dbExecute((db) => + db.transaction(async (tx) => { + // Serialize quota checks per organization. Without the transaction-scoped + // advisory lock, concurrent creates can both observe 99 active rows and + // commit the 100th and 101st rules. + await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${orgId}))`) + if (normalized.enabled) { + const activeRows = await tx + .select({ id: alertRules.id }) + .from(alertRules) + .where(and(eq(alertRules.orgId, orgId), eq(alertRules.enabled, true))) + const alreadyActive = + existingId != null && activeRows.some((row) => row.id === existingId) + if (!alreadyActive && activeRows.length >= MAX_ACTIVE_ALERT_RULES_PER_ORG) { + return { limitExceeded: true as const, writeRows: [] } + } + } + + const writeRows = + existingId == null + ? await tx + .insert(alertRules) + .values({ + id: ruleId, + orgId, + ...ruleFields, + createdAt: new Date(timestamp), + createdBy: userId, + }) + .returning(txidColumn) + : await tx + .update(alertRules) + .set(ruleFields) + .where( + and(eq(alertRules.orgId, orgId), eq(alertRules.id, existingId)), + ) + .returning(txidColumn) + return { limitExceeded: false as const, writeRows } + }), + ) + if (writeResult.limitExceeded) { + return yield* Effect.fail( + makeValidationError( + `Organizations may have at most ${MAX_ACTIVE_ALERT_RULES_PER_ORG} active alert rules`, + ), + ) + } + const txid = readTxid(writeResult.writeRows) const row = yield* requireRuleRow(orgId, ruleId) const destinationIds = safeParseStringArray(row.destinationIdsJson) @@ -2568,7 +2651,7 @@ export class AlertsService extends Context.Service { - const normalized = yield* normalizeRule(request.rule, { forPreview: true }) + const normalized = yield* normalizeRule(orgId, request.rule, { forPreview: true }) const plan = normalized.compiledPlan + if (plan.kind === "raw_sql") { + return yield* Effect.fail(makeValidationError("Raw SQL alerts cannot be previewed")) + } const windowMs = normalized.windowMinutes * 60_000 const requestedStartMs = Date.parse(request.startTime) @@ -2788,7 +2874,7 @@ export class AlertsService extends Context.Service 1) { - // Mirror the scheduler's multi-service mode: independent per-service - // plans, groupKey = service name. - yield* Effect.forEach( - normalized.serviceNames, - (svcName) => - Effect.gen(function* () { - const perServicePlan = yield* compileRulePlan({ - ...normalized, - serviceName: svcName, - }) - if ( - perServicePlan.query == null || - perServicePlan.sampleCountStrategy == null - ) { - return - } - const observations = yield* queryEngine - .evaluateSeries(systemTenant(orgId), { - startTime: toTinybirdDateTime(startMs), - endTime: toTinybirdDateTime(queryEndMs), - query: perServicePlan.query, - reducer: perServicePlan.reducer, - sampleCountStrategy: perServicePlan.sampleCountStrategy, - }) - .pipe(catchQueryEngineErrors) - for (const obs of observations) { - record(svcName, Date.parse(obs.bucket), { - value: obs.value, - sampleCount: obs.sampleCount, - hasData: obs.sampleCount > 0, - }) - } - }), - { concurrency: 5 }, - ) - } else { - const observations = yield* queryEngine - .evaluateSeries(systemTenant(orgId), { - startTime: toTinybirdDateTime(startMs), - endTime: toTinybirdDateTime(queryEndMs), - query: plan.query, - reducer: plan.reducer, - sampleCountStrategy: plan.sampleCountStrategy, - }) - .pipe(catchQueryEngineErrors) - const excludeSet = new Set(normalized.excludeServiceNames) - for (const obs of observations) { - if (excludeSet.has(obs.groupKey)) continue - record(obs.groupKey, Date.parse(obs.bucket), { - value: obs.value, - sampleCount: obs.sampleCount, - hasData: obs.sampleCount > 0, - }) - } - } - } else { - // Raw SQL: the reducer runs INSIDE one evaluation window, so replay - // the evaluator over each historical window (plus the trailing - // in-progress one). Bounded by MAX_RAW_PREVIEW_WINDOWS via the range - // clamp above. + if (plan.query == null || plan.sampleCountStrategy == null) { + return yield* Effect.fail( + makeValidationError("Compiled alert plan is missing its query spec"), + ) + } + if (normalized.serviceNames.length > 1) { + // Mirror the scheduler's multi-service mode: independent per-service + // plans, groupKey = service name. yield* Effect.forEach( - pointBuckets, - (bucketMs) => - queryEngine - .evaluateRawSql(systemTenant(orgId), { - startTime: toTinybirdDateTime(bucketMs), - endTime: toTinybirdDateTime(Math.min(bucketMs + windowMs, queryEndMs)), - sql: plan.rawSql ?? "", - reducer: plan.reducer, - windowMinutes: normalized.windowMinutes, + normalized.serviceNames, + (svcName) => + Effect.gen(function* () { + const perServicePlan = yield* compileRulePlan({ + ...normalized, + serviceName: svcName, }) - .pipe( - catchQueryEngineErrors, - Effect.map((groups) => { - for (const group of groups) { - record(group.groupKey, bucketMs, { - value: group.value, - sampleCount: group.sampleCount, - hasData: group.hasData, - }) - } - }), - ), - { concurrency: 4 }, + if ( + perServicePlan.query == null || + perServicePlan.sampleCountStrategy == null + ) { + return + } + const observations = yield* queryEngine + .evaluateSeries(systemTenant(orgId), { + startTime: toTinybirdDateTime(startMs), + endTime: toTinybirdDateTime(queryEndMs), + query: perServicePlan.query, + reducer: perServicePlan.reducer, + sampleCountStrategy: perServicePlan.sampleCountStrategy, + }) + .pipe(catchQueryEngineErrors) + for (const obs of observations) { + record(svcName, Date.parse(obs.bucket), { + value: obs.value, + sampleCount: obs.sampleCount, + hasData: obs.sampleCount > 0, + }) + } + }), + { concurrency: 5 }, ) + } else { + const observations = yield* queryEngine + .evaluateSeries(systemTenant(orgId), { + startTime: toTinybirdDateTime(startMs), + endTime: toTinybirdDateTime(queryEndMs), + query: plan.query, + reducer: plan.reducer, + sampleCountStrategy: plan.sampleCountStrategy, + }) + .pipe(catchQueryEngineErrors) + const excludeSet = new Set(normalized.excludeServiceNames) + for (const obs of observations) { + if (excludeSet.has(obs.groupKey)) continue + record(obs.groupKey, Date.parse(obs.bucket), { + value: obs.value, + sampleCount: obs.sampleCount, + hasData: obs.sampleCount > 0, + }) + } } // Ungrouped rules always observe *something* per tick ("all"), so chart @@ -4391,7 +4445,7 @@ export class AlertsService extends Context.Service Effect.gen(function* () { const timestamp = yield* now diff --git a/apps/api/src/services/ApiAuthorizationLayer.ts b/apps/api/src/services/ApiAuthorizationLayer.ts index acc57c3b5..994760b92 100644 --- a/apps/api/src/services/ApiAuthorizationLayer.ts +++ b/apps/api/src/services/ApiAuthorizationLayer.ts @@ -40,14 +40,25 @@ export const ApiAuthorizationLayer = Layer.effect( ) if (Option.isSome(apiKeyResolved)) { + const resolved = apiKeyResolved.value + if (resolved.kind !== "standard") { + return yield* new UnauthorizedError({ + message: "This API key is only valid for the MCP server", + }) + } + if (resolved.scopes !== null) { + return yield* new UnauthorizedError({ + message: "Restricted API keys must use the /v2 API", + }) + } yield* annotateAuthSpan("api_key", { - orgId: apiKeyResolved.value.orgId, - userId: apiKeyResolved.value.userId, - keyId: apiKeyResolved.value.keyId, + orgId: resolved.orgId, + userId: resolved.userId, + keyId: resolved.keyId, }) const tenant = new CurrentTenant.TenantSchema({ - orgId: apiKeyResolved.value.orgId, - userId: apiKeyResolved.value.userId, + orgId: resolved.orgId, + userId: resolved.userId, roles: apiKeyDefaultRoles, authMode: "self_hosted", }) diff --git a/apps/api/src/services/ApiAuthorizationV2Layer.ts b/apps/api/src/services/ApiAuthorizationV2Layer.ts index b9517c448..d1b216149 100644 --- a/apps/api/src/services/ApiAuthorizationV2Layer.ts +++ b/apps/api/src/services/ApiAuthorizationV2Layer.ts @@ -1,10 +1,11 @@ -import { HttpServerRequest } from "effect/unstable/http" +import { HttpEffect, HttpServerRequest, HttpServerResponse } from "effect/unstable/http" import { CurrentTenant, RoleName } from "@maple/domain/http" import { AuthorizationV2, authenticationError, dependencyUnavailable, permissionError, + rateLimited, requiredScopeForRequest, scopeAllows, } from "@maple/domain/http/v2" @@ -13,6 +14,11 @@ import { ApiKeysService } from "./ApiKeysService" import { makeResolveTenant } from "./AuthService" import { annotateAuthSpan } from "../lib/auth-span" import { Env } from "../lib/Env" +import { + API_V2_RATE_LIMIT_PERIOD_SECONDS, + API_V2_RATE_LIMIT_REQUESTS, + ApiV2RateLimiter, +} from "./ApiV2RateLimiter" const decodeRoleNameSync = Schema.decodeUnknownSync(RoleName) const apiKeyDefaultRoles = [decodeRoleNameSync("root")] as const @@ -42,6 +48,7 @@ export const ApiAuthorizationV2Layer = Layer.effect( Effect.gen(function* () { const env = yield* Env const apiKeys = yield* ApiKeysService + const rateLimiter = yield* ApiV2RateLimiter const resolveTenant = makeResolveTenant(env) return AuthorizationV2.of({ @@ -60,6 +67,14 @@ export const ApiAuthorizationV2Layer = Layer.effect( if (Option.isSome(apiKeyResolved)) { const resolved = apiKeyResolved.value + if (resolved.kind !== "standard") { + return yield* Effect.fail( + authenticationError( + "invalid_credentials", + "This API key is only valid for the MCP server.", + ), + ) + } // Attribute before the scope check so scope-rejected // requests are still counted as API-key traffic. @@ -69,6 +84,26 @@ export const ApiAuthorizationV2Layer = Layer.effect( keyId: resolved.keyId, }) + const rateLimitOutcome = yield* rateLimiter.check(resolved.keyId) + yield* Effect.annotateCurrentSpan({ + "maple.rate_limit.outcome": rateLimitOutcome, + "maple.rate_limit.limit": API_V2_RATE_LIMIT_REQUESTS, + "maple.rate_limit.period_seconds": API_V2_RATE_LIMIT_PERIOD_SECONDS, + }) + + if (rateLimitOutcome === "limited") { + yield* HttpEffect.appendPreResponseHandler((_request, response) => + Effect.succeed( + HttpServerResponse.setHeader( + response, + "Retry-After", + String(API_V2_RATE_LIMIT_PERIOD_SECONDS), + ), + ), + ) + return yield* Effect.fail(rateLimited()) + } + const required = requiredScopeForRequest(request.method, requestPath(request.url)) if (required !== null && !scopeAllows(resolved.scopes, required)) { return yield* Effect.fail( diff --git a/apps/api/src/services/ApiKeysService.scopes.test.ts b/apps/api/src/services/ApiKeysService.scopes.test.ts index c15fc5dbf..b9ad7ecd3 100644 --- a/apps/api/src/services/ApiKeysService.scopes.test.ts +++ b/apps/api/src/services/ApiKeysService.scopes.test.ts @@ -48,6 +48,7 @@ describe("ApiKeysService scopes", () => { expect(Option.isSome(resolved)).toBe(true) if (Option.isSome(resolved)) { expect(resolved.value.scopes).toEqual(["api_keys:read", "dashboards:write"]) + expect(resolved.value.kind).toBe("standard") } const fetched = yield* service.get(ORG, created.id) diff --git a/apps/api/src/services/ApiKeysService.test.ts b/apps/api/src/services/ApiKeysService.test.ts index fdbce01ba..67655d74c 100644 --- a/apps/api/src/services/ApiKeysService.test.ts +++ b/apps/api/src/services/ApiKeysService.test.ts @@ -80,6 +80,7 @@ describe("ApiKeysService.roll", () => { assert.isTrue(Option.isSome(resolvedNew)) if (Option.isSome(resolvedNew)) { assert.strictEqual(resolvedNew.value.keyId, rolled.id) + assert.strictEqual(resolvedNew.value.kind, "mcp") } assert.deepStrictEqual(resolvedOld, Option.none()) }).pipe(Effect.provide(makeLayer(testDb))) diff --git a/apps/api/src/services/ApiKeysService.ts b/apps/api/src/services/ApiKeysService.ts index dab3ef699..8857d785b 100644 --- a/apps/api/src/services/ApiKeysService.ts +++ b/apps/api/src/services/ApiKeysService.ts @@ -20,10 +20,11 @@ import { readTxid, txidColumn } from "../lib/electric-txid" import { Env } from "../lib/Env" import { dateToMs, msToDate } from "../lib/time" -interface ResolvedApiKey { +export interface ResolvedApiKey { readonly orgId: OrgId readonly userId: UserId readonly keyId: ApiKeyId + readonly kind: ApiKeyKind readonly metadataJson: string | null /** v2 scope strings; null = legacy full access. */ readonly scopes: ReadonlyArray | null @@ -277,6 +278,7 @@ export class ApiKeysService extends Context.Service()("@maple/ap orgId: decodeOrgIdSync(row.value.orgId), userId: decodeUserIdSync(row.value.createdBy), keyId: decodeApiKeyIdSync(row.value.id), + kind: row.value.kind, metadataJson: row.value.metadataJson == null ? null : JSON.stringify(row.value.metadataJson), scopes: row.value.scopes ?? null, } satisfies ResolvedApiKey) diff --git a/apps/api/src/services/ApiV2RateLimiter.test.ts b/apps/api/src/services/ApiV2RateLimiter.test.ts new file mode 100644 index 000000000..138c7e625 --- /dev/null +++ b/apps/api/src/services/ApiV2RateLimiter.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from "@effect/vitest" +import { ApiKeyId } from "@maple/domain/http" +import { Effect, Layer, Schema } from "effect" +import { WorkerEnvironment } from "../lib/WorkerEnvironment" +import { + API_V2_RATE_LIMIT_BINDING, + API_V2_RATE_LIMIT_PARTITION, + ApiV2RateLimiter, + makeApiV2RateLimitKey, +} from "./ApiV2RateLimiter" + +const KEY_A = Schema.decodeUnknownSync(ApiKeyId)("00000000-0000-4000-8000-000000000001") +const KEY_B = Schema.decodeUnknownSync(ApiKeyId)("00000000-0000-4000-8000-000000000002") + +const limiterLayer = (environment: Record) => + ApiV2RateLimiter.layer.pipe(Layer.provide(Layer.succeed(WorkerEnvironment, environment))) + +describe("ApiV2RateLimiter", () => { + it.effect("uses only the stage partition and internal API-key ID as the counter key", () => { + const keys: string[] = [] + const environment = { + [API_V2_RATE_LIMIT_PARTITION]: "stg", + [API_V2_RATE_LIMIT_BINDING]: { + limit: ({ key }: { key: string }) => { + keys.push(key) + return Promise.resolve({ success: true }) + }, + }, + } + + return Effect.gen(function* () { + const limiter = yield* ApiV2RateLimiter + expect(yield* limiter.check(KEY_A)).toBe("allowed") + expect(yield* limiter.check(KEY_B)).toBe("allowed") + expect(keys).toEqual([makeApiV2RateLimitKey("stg", KEY_A), makeApiV2RateLimitKey("stg", KEY_B)]) + expect(keys.join(" ")).not.toContain("maple_ak_") + }).pipe(Effect.provide(limiterLayer(environment))) + }) + + it.effect("isolates the same key across deployment stages", () => { + const observed: string[] = [] + const binding = { + limit: ({ key }: { key: string }) => { + observed.push(key) + return Promise.resolve({ success: true }) + }, + } + const run = (partition: string) => + Effect.gen(function* () { + const limiter = yield* ApiV2RateLimiter + return yield* limiter.check(KEY_A) + }).pipe( + Effect.provide( + limiterLayer({ + [API_V2_RATE_LIMIT_PARTITION]: partition, + [API_V2_RATE_LIMIT_BINDING]: binding, + }), + ), + ) + + return Effect.gen(function* () { + expect(yield* run("prd")).toBe("allowed") + expect(yield* run("stg")).toBe("allowed") + expect(observed).toEqual([ + makeApiV2RateLimitKey("prd", KEY_A), + makeApiV2RateLimitKey("stg", KEY_A), + ]) + }) + }) + + it.effect("returns limited when Cloudflare denies the key", () => + Effect.gen(function* () { + const limiter = yield* ApiV2RateLimiter + expect(yield* limiter.check(KEY_A)).toBe("limited") + }).pipe( + Effect.provide( + limiterLayer({ + [API_V2_RATE_LIMIT_PARTITION]: "prd", + [API_V2_RATE_LIMIT_BINDING]: { + limit: () => Promise.resolve({ success: false }), + }, + }), + ), + ), + ) + + it.effect("fails open when the binding or partition is unavailable", () => { + const run = (environment: Record) => + Effect.gen(function* () { + const limiter = yield* ApiV2RateLimiter + return yield* limiter.check(KEY_A) + }).pipe(Effect.provide(limiterLayer(environment))) + + return Effect.gen(function* () { + expect(yield* run({ [API_V2_RATE_LIMIT_PARTITION]: "prd" })).toBe("failed_open") + expect( + yield* run({ + [API_V2_RATE_LIMIT_BINDING]: { limit: () => Promise.resolve({ success: true }) }, + }), + ).toBe("failed_open") + }) + }) + + it.effect("fails open when the Cloudflare binding throws", () => + Effect.gen(function* () { + const limiter = yield* ApiV2RateLimiter + expect(yield* limiter.check(KEY_A)).toBe("failed_open") + }).pipe( + Effect.provide( + limiterLayer({ + [API_V2_RATE_LIMIT_PARTITION]: "prd", + [API_V2_RATE_LIMIT_BINDING]: { + limit: () => Promise.reject(new Error("binding unavailable")), + }, + }), + ), + ), + ) +}) diff --git a/apps/api/src/services/ApiV2RateLimiter.ts b/apps/api/src/services/ApiV2RateLimiter.ts new file mode 100644 index 000000000..9a2e397da --- /dev/null +++ b/apps/api/src/services/ApiV2RateLimiter.ts @@ -0,0 +1,92 @@ +import type { ApiKeyId } from "@maple/domain/http" +import { Context, Effect, Layer, Schema } from "effect" +import { WorkerEnvironment } from "../lib/WorkerEnvironment" + +export const API_V2_RATE_LIMIT_BINDING = "API_V2_RATE_LIMITER" +export const API_V2_RATE_LIMIT_PARTITION = "API_V2_RATE_LIMIT_PARTITION" +export const API_V2_RATE_LIMIT_REQUESTS = 600 +export const API_V2_RATE_LIMIT_PERIOD_SECONDS = 60 + +export type ApiV2RateLimitOutcome = "allowed" | "limited" | "failed_open" + +interface RateLimitBinding { + readonly limit: (options: { readonly key: string }) => Promise<{ readonly success: boolean }> +} + +export interface ApiV2RateLimiterShape { + readonly check: (keyId: ApiKeyId) => Effect.Effect +} + +class ApiV2RateLimiterBindingError extends Schema.TaggedErrorClass()( + "@maple/api/services/ApiV2RateLimiterBindingError", + { + message: Schema.String, + cause: Schema.Defect(), + }, +) {} + +const isRateLimitBinding = (value: unknown): value is RateLimitBinding => + typeof value === "object" && + value !== null && + "limit" in value && + typeof (value as { readonly limit?: unknown }).limit === "function" + +const readPartition = (environment: Record): string | undefined => { + const value = environment[API_V2_RATE_LIMIT_PARTITION] + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined +} + +export const makeApiV2RateLimitKey = (partition: string, keyId: ApiKeyId): string => + `${partition}:v2:${keyId}` + +const warnFailedOpen = (reason: "binding_missing" | "partition_missing" | "binding_error", cause?: unknown) => + Effect.logWarning("API v2 rate limiter unavailable; allowing request").pipe( + Effect.annotateLogs({ + "maple.rate_limit.outcome": "failed_open", + "maple.rate_limit.reason": reason, + ...(cause instanceof Error ? { "error.type": cause.name } : {}), + }), + ) + +export class ApiV2RateLimiter extends Context.Service()( + "@maple/api/services/ApiV2RateLimiter", + { + make: Effect.gen(function* () { + const environment = yield* WorkerEnvironment + + const check = Effect.fn("ApiV2RateLimiter.check")(function* (keyId: ApiKeyId) { + const binding = environment[API_V2_RATE_LIMIT_BINDING] + if (!isRateLimitBinding(binding)) { + yield* warnFailedOpen("binding_missing") + return "failed_open" as const + } + + const partition = readPartition(environment) + if (partition === undefined) { + yield* warnFailedOpen("partition_missing") + return "failed_open" as const + } + + return yield* Effect.tryPromise({ + try: () => binding.limit({ key: makeApiV2RateLimitKey(partition, keyId) }), + catch: (cause) => + new ApiV2RateLimiterBindingError({ + message: "Cloudflare rate-limit binding call failed", + cause, + }), + }).pipe( + Effect.map(({ success }) => (success ? ("allowed" as const) : ("limited" as const))), + Effect.catchTag("@maple/api/services/ApiV2RateLimiterBindingError", (error) => + warnFailedOpen("binding_error", error.cause).pipe( + Effect.as("failed_open"), + ), + ), + ) + }) + + return { check } satisfies ApiV2RateLimiterShape + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/api/src/services/DigestService.test.ts b/apps/api/src/services/DigestService.test.ts index 8e7bc9ab9..a843d95bc 100644 --- a/apps/api/src/services/DigestService.test.ts +++ b/apps/api/src/services/DigestService.test.ts @@ -55,6 +55,7 @@ const warehouseStub = Layer.succeed(WarehouseQueryService, { }), ), sqlQuery: () => Effect.die("sqlQuery not used by DigestService tests"), + rawSqlQuery: () => Effect.die("rawSqlQuery not used by DigestService tests"), compiledQuery: () => Effect.die("compiledQuery not used by DigestService tests"), compiledQueryFirst: () => Effect.die("compiledQueryFirst not used by DigestService tests"), ingest: () => Effect.die("ingest not used by DigestService tests"), diff --git a/apps/api/src/services/ErrorsService.test.ts b/apps/api/src/services/ErrorsService.test.ts index 8c46b7e67..2c36ea528 100644 --- a/apps/api/src/services/ErrorsService.test.ts +++ b/apps/api/src/services/ErrorsService.test.ts @@ -153,6 +153,7 @@ const makeWarehouseStub = ( ): WarehouseQueryServiceShape => ({ query: () => Effect.die(new Error("unexpected warehouse query")), sqlQuery: () => Effect.succeed([]), + rawSqlQuery: () => Effect.succeed([]), compiledQuery: (tenant: unknown, compiled: CompiledQuery, options?: SqlQueryOptions) => Effect.sync(() => { if (options?.context === "errorIssuesScan") { @@ -244,6 +245,7 @@ const makeGatingLayer = (opts: { const warehouseStub: WarehouseQueryServiceShape = { query: () => Effect.die(new Error("unexpected warehouse query")), sqlQuery: () => Effect.succeed([]), + rawSqlQuery: () => Effect.succeed([]), compiledQuery: (tenant: unknown, compiled: CompiledQuery, options?: SqlQueryOptions) => { if (options?.context) opts.profiles?.set(options.context, options.profile) if (options?.context === "errorActiveOrgsDiscovery") { diff --git a/apps/api/src/services/OrgClickHouseSettingsService.test.ts b/apps/api/src/services/OrgClickHouseSettingsService.test.ts index 0a7c89e4d..c7838250e 100644 --- a/apps/api/src/services/OrgClickHouseSettingsService.test.ts +++ b/apps/api/src/services/OrgClickHouseSettingsService.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it } from "@effect/vitest" import { OrgClickHouseSettingsUpstreamRejectedError, OrgClickHouseSettingsUpstreamUnavailableError, + OrgClickHouseSettingsValidationError, OrgId, RoleName, } from "@maple/domain/http" @@ -10,6 +11,7 @@ import { Cause, ConfigProvider, Effect, Exit, Layer, Option, Schema } from "effe import { FetchHttpClient } from "effect/unstable/http" import type { TableDiffEntry } from "@maple/domain/clickhouse" import { Env } from "../lib/Env" +import { encryptAes256Gcm } from "../lib/Crypto" import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "../lib/test-pglite" import { type ClickHouseExecConfig, @@ -17,6 +19,7 @@ import { isRetryableUpstream, OrgClickHouseSettingsService, shouldHealSchemaVersion, + validateClickHouseCredentialTransport, } from "./OrgClickHouseSettingsService" // `execClickHouse` runs through Effect's HttpClient. We inject a stub `fetch` via @@ -60,6 +63,35 @@ const unavailable = (statusCode: number | null) => const rejected = (statusCode: number | null) => new OrgClickHouseSettingsUpstreamRejectedError({ message: "x", statusCode }) +describe("validateClickHouseCredentialTransport", () => { + it.effect("rejects sending a ClickHouse password over plaintext HTTP", () => + Effect.gen(function* () { + const exit = yield* validateClickHouseCredentialTransport( + "http://clickhouse.example.test", + "secret", + ).pipe(Effect.exit) + expect(getError(exit)).toBeInstanceOf(OrgClickHouseSettingsValidationError) + }), + ) + + it.effect("allows HTTPS credentials and passwordless HTTP", () => + Effect.gen(function* () { + yield* validateClickHouseCredentialTransport("https://clickhouse.example.test", "secret") + yield* validateClickHouseCredentialTransport("http://clickhouse.example.test", "") + }), + ) + + it.effect("rejects URL-embedded userinfo even without a separate password", () => + Effect.gen(function* () { + const exit = yield* validateClickHouseCredentialTransport( + "https://user:secret@clickhouse.example.test", + "", + ).pipe(Effect.exit) + expect(getError(exit)).toBeInstanceOf(OrgClickHouseSettingsValidationError) + }), + ) +}) + describe("shouldHealSchemaVersion", () => { const REV = "019c3db4cf690e3748b302098cae4c9213d18c55355db9fc68ea44982c7a980a" const STALE = "4d5d918315933608d316aa8d6e6b57948f15a3fdca2fa6226aa271553f0b0520" @@ -119,6 +151,27 @@ describe("isRetryableUpstream", () => { }) describe("execClickHouse", () => { + it.live("uses manual redirects and rejects every 3xx without following it", () => + Effect.gen(function* () { + let redirectMode: RequestRedirect | undefined + let calls = 0 + const fetchImpl = (async (_input: RequestInfo | URL, init?: RequestInit) => { + calls += 1 + redirectMode = init?.redirect + return new Response("moved", { + status: 307, + headers: { Location: "https://attacker.example/steal" }, + }) + }) as typeof fetch + const exit = yield* run("SELECT 1", fetchImpl).pipe(Effect.exit) + const failure = getError(exit) + expect(failure).toBeInstanceOf(OrgClickHouseSettingsUpstreamRejectedError) + expect((failure as OrgClickHouseSettingsUpstreamRejectedError).statusCode).toBe(307) + expect(redirectMode).toBe("manual") + expect(calls).toBe(1) + }), + ) + it.live("maps a Cloudflare 524 to a clear, actionable message (and retries 52x)", () => Effect.gen(function* () { const { state, fetchImpl } = makeFetch(() => @@ -237,6 +290,20 @@ describe("resolveRuntimeConfig caching", () => { [orgId, chUrl], ) + const seedPasswordRow = async (db: TestDb, orgId: string, chUrl: string) => { + const encrypted = await Effect.runPromise( + encryptAes256Gcm("legacy-password", Buffer.alloc(32, 5), (message) => new Error(message)), + ) + await executeSql( + db, + `INSERT INTO org_clickhouse_settings + (org_id, ch_url, ch_user, ch_database, ch_password_ciphertext, ch_password_iv, + ch_password_tag, sync_status, created_at, updated_at, created_by, updated_by) + VALUES ($1, $2, 'default', 'maple', $3, $4, $5, 'connected', NOW(), NOW(), 'u', 'u')`, + [orgId, chUrl, encrypted.ciphertext, encrypted.iv, encrypted.tag], + ) + } + const expectSome = (o: Option.Option): A => { expect(Option.isSome(o)).toBe(true) return (o as Option.Some).value @@ -283,4 +350,28 @@ describe("resolveRuntimeConfig caching", () => { expect(Option.isNone(after)).toBe(true) }).pipe(Effect.provide(buildLayer(testDb))) }) + + it.effect("fails closed for a legacy passworded HTTP runtime row", () => { + const testDb = createTestDb(cacheTrackedDbs) + const orgId = "org_ch_legacy_http" + return Effect.gen(function* () { + yield* Effect.promise(() => seedPasswordRow(testDb, orgId, "http://clickhouse.example.test")) + const exit = yield* OrgClickHouseSettingsService.resolveRuntimeConfig(asOrgId(orgId)).pipe( + Effect.exit, + ) + expect(getError(exit)).toBeInstanceOf(OrgClickHouseSettingsValidationError) + }).pipe(Effect.provide(buildLayer(testDb))) + }) + + it.effect("fails closed for legacy URL userinfo during collector config generation", () => { + const testDb = createTestDb(cacheTrackedDbs) + const orgId = "org_ch_legacy_userinfo" + return Effect.gen(function* () { + yield* Effect.promise(() => seedRow(testDb, orgId, "https://user:secret@clickhouse.example.test")) + const exit = yield* OrgClickHouseSettingsService.collectorConfig(asOrgId(orgId), [ + asRole("org:admin"), + ]).pipe(Effect.exit) + expect(getError(exit)).toBeInstanceOf(OrgClickHouseSettingsValidationError) + }).pipe(Effect.provide(buildLayer(testDb))) + }) }) diff --git a/apps/api/src/services/OrgClickHouseSettingsService.ts b/apps/api/src/services/OrgClickHouseSettingsService.ts index be6a78542..24c116e99 100644 --- a/apps/api/src/services/OrgClickHouseSettingsService.ts +++ b/apps/api/src/services/OrgClickHouseSettingsService.ts @@ -179,7 +179,9 @@ export interface OrgClickHouseSettingsServiceShape { orgId: OrgId, ) => Effect.Effect< Option.Option, - OrgClickHouseSettingsPersistenceError | OrgClickHouseSettingsEncryptionError + | OrgClickHouseSettingsPersistenceError + | OrgClickHouseSettingsEncryptionError + | OrgClickHouseSettingsValidationError > /** * Whether the ingest gateway is currently routing this org's frames to its @@ -378,7 +380,16 @@ const quoteYaml = (value: string): string => `"${value.replace(/\\/g, "\\\\").re const normalizeHttpUrl = (raw: string): Effect.Effect => validateExternalUrl(raw).pipe( - Effect.map(() => raw.trim().replace(/\/+$/, "")), + Effect.flatMap((url) => + url.username.length > 0 || url.password.length > 0 + ? Effect.fail( + new OrgClickHouseSettingsValidationError({ + message: + "ClickHouse credentials must use the user/password fields, not URL userinfo", + }), + ) + : Effect.succeed(raw.trim().replace(/\/+$/, "")), + ), Effect.mapError( (error) => new OrgClickHouseSettingsValidationError({ @@ -387,6 +398,40 @@ const normalizeHttpUrl = (raw: string): Effect.Effect => + Effect.try({ + try: () => new URL(url), + catch: () => + new OrgClickHouseSettingsValidationError({ message: "Stored ClickHouse URL is invalid" }), + }).pipe( + Effect.flatMap((parsed) => { + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + return Effect.fail( + new OrgClickHouseSettingsValidationError({ + message: "ClickHouse URLs must use HTTP or HTTPS", + }), + ) + } + if (parsed.username.length > 0 || parsed.password.length > 0) { + return Effect.fail( + new OrgClickHouseSettingsValidationError({ + message: "ClickHouse credentials must not be embedded in the URL", + }), + ) + } + return password.length > 0 && parsed.protocol !== "https:" + ? Effect.fail( + new OrgClickHouseSettingsValidationError({ + message: "ClickHouse URLs must use HTTPS when a password is configured", + }), + ) + : Effect.void + }), + ) + const isOrgAdmin = (roles: ReadonlyArray) => roles.includes(ROOT_ROLE) || roles.includes(ORG_ADMIN_ROLE) @@ -534,6 +579,14 @@ const mapStatusToError = ( }), ) } + if (status >= 300 && status < 400) { + return Effect.fail( + new OrgClickHouseSettingsUpstreamRejectedError({ + message: `ClickHouse redirect responses are not allowed (${status})`, + statusCode: status, + }), + ) + } if (status >= 500) { return Effect.fail( new OrgClickHouseSettingsUpstreamUnavailableError({ @@ -556,7 +609,9 @@ export const execClickHouse = (config: ClickHouseExecConfig, sql: string) => const request = HttpClientRequest.post(buildClickHouseUrl(config), { headers: buildClickHouseHeaders(config), }).pipe(HttpClientRequest.bodyText(sql)) - const response = yield* client.execute(request) + const response = yield* client + .execute(request) + .pipe(Effect.provideService(FetchHttpClient.RequestInit, { redirect: "manual" })) const text = yield* response.text return { status: response.status, text } }).pipe( @@ -818,6 +873,7 @@ export class OrgClickHouseSettingsService extends Context.Service< } plainPassword = yield* decryptStoredPassword(existing) } + yield* validateClickHouseCredentialTransport(url, plainPassword) // Connect-and-validate: hit the cluster with `SELECT 1` so a typo'd // host or token surfaces here rather than after the user closes the @@ -896,12 +952,16 @@ export class OrgClickHouseSettingsService extends Context.Service< }) const loadConfigForRow = (row: ActiveRow) => - Effect.map(decryptStoredPassword(row), (password) => ({ - url: row.chUrl, - user: row.chUser, - password, - database: row.chDatabase, - })) + Effect.gen(function* () { + const password = yield* decryptStoredPassword(row) + yield* validateClickHouseCredentialTransport(row.chUrl, password) + return { + url: row.chUrl, + user: row.chUser, + password, + database: row.chDatabase, + } + }) const schemaDiff = Effect.fn("OrgClickHouseSettingsService.schemaDiff")(function* ( orgId: OrgId, @@ -1164,6 +1224,7 @@ export class OrgClickHouseSettingsService extends Context.Service< cached.schemaVersion !== clickHouseSchemaVersion, ) const password = yield* decryptStoredPassword(cached) + yield* validateClickHouseCredentialTransport(cached.chUrl, password) return Option.some({ backend: "clickhouse", url: cached.chUrl, @@ -1203,6 +1264,15 @@ export class OrgClickHouseSettingsService extends Context.Service< yield* Effect.annotateCurrentSpan("orgId", orgId) yield* requireAdmin(roles) const row = yield* requireActiveRow(orgId) + const password = yield* decryptStoredPassword(row).pipe( + Effect.mapError( + () => + new OrgClickHouseSettingsValidationError({ + message: "Stored ClickHouse credentials could not be decrypted", + }), + ), + ) + yield* validateClickHouseCredentialTransport(row.chUrl, password) const yaml = renderCollectorYaml({ orgId, endpoint: row.chUrl, diff --git a/apps/api/src/services/QueryEngineEvaluateCache.test.ts b/apps/api/src/services/QueryEngineEvaluateCache.test.ts index 0219e45fb..1dde49fd5 100644 --- a/apps/api/src/services/QueryEngineEvaluateCache.test.ts +++ b/apps/api/src/services/QueryEngineEvaluateCache.test.ts @@ -74,6 +74,7 @@ const countRequest = (reducer: QueryEngineEvaluateRequest["reducer"]): QueryEngi const evalStub = (rows: ReadonlyArray>) => ({ sqlQuery: () => Effect.succeed(rows as never), + rawSqlQuery: () => Effect.die(new Error("rawSqlQuery is not used by evaluate cache tests")), compiledQuery: (_tenant, compiled) => compiled.decodeRows(rows).pipe(Effect.orDie), }) satisfies Parameters[0] diff --git a/apps/api/src/services/QueryEngineService.test.ts b/apps/api/src/services/QueryEngineService.test.ts index 2a2dffdf6..69e64fa9b 100644 --- a/apps/api/src/services/QueryEngineService.test.ts +++ b/apps/api/src/services/QueryEngineService.test.ts @@ -2,6 +2,7 @@ import { describe, it } from "@effect/vitest" import { Effect, Exit, Option, Schema } from "effect" import { strict as nodeAssert } from "node:assert" import { MetricName, OrgId, ServiceName, UserId } from "@maple/domain" +import { RawSqlValidationError } from "@maple/domain/http" import type { QueryEngineEvaluateRequest, QueryEngineExecuteRequest, @@ -72,14 +73,18 @@ function makeTinybirdStub(overrides: Partial () => Effect.die(new Error(`Unexpected tinybird call in test: ${name}`)) const sqlQuery = overrides.sqlQuery ?? unexpected("sqlQuery") + const rawSqlQuery = overrides.rawSqlQuery ?? sqlQuery return { - sqlQuery, - compiledQuery: (tenant, compiled, options) => - sqlQuery(tenant, compiled.sql, options).pipe( - Effect.flatMap((rows) => compiled.decodeRows(rows).pipe(Effect.orDie)), - ), ...overrides, + sqlQuery, + rawSqlQuery, + compiledQuery: + overrides.compiledQuery ?? + ((tenant, compiled, options) => + sqlQuery(tenant, compiled.sql, options).pipe( + Effect.flatMap((rows) => compiled.decodeRows(rows).pipe(Effect.orDie)), + )), } satisfies Parameters[0] } @@ -995,23 +1000,60 @@ describe("makeQueryEngineEvaluate", () => { }) describe("makeQueryEngineEvaluateRawSql", () => { + it.effect("translates raw warehouse limits once into the alert validation contract", () => + Effect.gen(function* () { + const evaluateRawSql = makeQueryEngineEvaluateRawSql( + makeTinybirdStub({ + rawSqlQuery: () => + Effect.fail( + new RawSqlValidationError({ + code: "ResourceLimit", + message: "Raw SQL results may contain at most 5000000 encoded bytes", + }), + ), + }), + ) + + const exit = yield* evaluateRawSql(tenant, { + startTime: "2026-01-01 00:00:00", + endTime: "2026-01-01 00:05:00", + sql: "SELECT value FROM traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", + reducer: "identity", + windowMinutes: 5, + }).pipe(Effect.exit) + const failure = Option.getOrUndefined(Exit.findErrorOption(exit)) as + | { readonly _tag?: string; readonly details?: ReadonlyArray } + | undefined + + assert.strictEqual(failure?._tag, "@maple/http/errors/QueryEngineValidationError") + assert.deepStrictEqual(failure?.details, [ + "Raw SQL results may contain at most 5000000 encoded bytes", + ]) + }), + ) + it.effect("groups raw SQL rows by the `group` column and reduces with the configured reducer", () => Effect.gen(function* () { + let profile: string | undefined + let context: string | undefined const evaluateRawSql = makeQueryEngineEvaluateRawSql( makeTinybirdStub({ - sqlQuery: () => - Effect.succeed([ + rawSqlQuery: (_tenant, _sql, options) => { + profile = options?.profile + context = options?.context + return Effect.succeed([ { group: "checkout", value: 10, samples: 4 }, { group: "checkout", value: 30, samples: 6 }, { group: "payments", value: 5, samples: 2 }, - ]), + ]) + }, }), ) const response = yield* evaluateRawSql(tenant, { startTime: "2026-01-01 00:00:00", endTime: "2026-01-01 00:05:00", - sql: "SELECT group, value FROM otel_traces WHERE $__orgFilter", + sql: "SELECT group, value FROM otel_traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", reducer: "max", windowMinutes: 5, }) @@ -1023,6 +1065,34 @@ describe("makeQueryEngineEvaluateRawSql", () => { assert.strictEqual(byGroup.payments?.value, 5) assert.strictEqual(byGroup.payments?.sampleCount, 2) assert.strictEqual(byGroup.payments?.hasData, true) + assert.strictEqual(profile, "rawAlert") + assert.strictEqual(context, "alertRawQuery") + }), + ) + + it.effect("rejects invalid sample counts returned by raw SQL", () => + Effect.gen(function* () { + const evaluateRawSql = makeQueryEngineEvaluateRawSql( + makeTinybirdStub({ + sqlQuery: () => Effect.succeed([{ value: 1, samples: -1 }]), + }), + ) + + const exit = yield* evaluateRawSql(tenant, { + startTime: "2026-01-01 00:00:00", + endTime: "2026-01-01 00:05:00", + sql: "SELECT value, samples FROM otel_traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", + reducer: "identity", + windowMinutes: 5, + }).pipe(Effect.exit) + + assert.isTrue(Exit.isFailure(exit)) + const failure = Option.getOrUndefined(Exit.findErrorOption(exit)) as + | { details?: readonly string[] } + | undefined + assert.deepStrictEqual(failure?.details, [ + "Raw SQL alert samples must be finite and nonnegative.", + ]) }), ) @@ -1035,7 +1105,7 @@ describe("makeQueryEngineEvaluateRawSql", () => { const response = yield* evaluateRawSql(tenant, { startTime: "2026-01-01 00:00:00", endTime: "2026-01-01 00:05:00", - sql: "SELECT value FROM otel_traces WHERE $__orgFilter", + sql: "SELECT value FROM otel_traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", reducer: "identity", windowMinutes: 5, }) @@ -1058,7 +1128,7 @@ describe("makeQueryEngineEvaluateRawSql", () => { evaluateRawSql(tenant, { startTime: "2026-01-01 00:00:00", endTime: "2026-01-01 00:05:00", - sql: "SELECT bucket, errors FROM otel_traces WHERE $__orgFilter", + sql: "SELECT bucket, errors FROM otel_traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", reducer: "identity", windowMinutes: 5, }), diff --git a/apps/api/src/services/TinybirdOrgTokenService.test.ts b/apps/api/src/services/TinybirdOrgTokenService.test.ts new file mode 100644 index 000000000..df9094a1c --- /dev/null +++ b/apps/api/src/services/TinybirdOrgTokenService.test.ts @@ -0,0 +1,111 @@ +import { assert, describe, it } from "@effect/vitest" +import { ConfigProvider, Effect, Layer } from "effect" +import { OrgId } from "@maple/domain" +import { Schema } from "effect" +import { TestClock } from "effect/testing" +import { TinybirdOrgTokenService } from "./TinybirdOrgTokenService" +import { Env } from "@/lib/Env" + +const SIGNING_KEY = "explicit-test-signing-key" +const asOrgId = Schema.decodeUnknownSync(OrgId) + +const testConfig = (extra: Record = {}, includeSigning = true) => + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3478", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "api-token-is-not-the-signing-key", + ...(includeSigning + ? { + TINYBIRD_SIGNING_KEY: SIGNING_KEY, + TINYBIRD_WORKSPACE_ID: "ws-uuid-abc", + } + : {}), + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 1).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + ...extra, + }), + ) + +const layer = TinybirdOrgTokenService.layer.pipe(Layer.provide(Env.layer), Layer.provide(testConfig())) + +const decodePayload = (jwt: string) => + JSON.parse(Buffer.from(jwt.split(".")[1], "base64url").toString("utf8")) as { + workspace_id: string + exp: number + scopes: ReadonlyArray<{ resource: string; filter: string }> + } + +describe("TinybirdOrgTokenService", () => { + it.effect("mints a workspace-scoped token whose scopes are all filtered to the org", () => + Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const token = yield* svc.getOrgReadToken(asOrgId("org_a")) + const payload = decodePayload(token) + assert.strictEqual(payload.workspace_id, "ws-uuid-abc") + assert.isAbove(payload.scopes.length, 0) + assert.isTrue(payload.scopes.every((s) => s.filter === "OrgId = 'org_a'")) + }).pipe(Effect.provide(layer)), + ) + + it.effect("returns the cached token on a second call within its lifetime", () => + Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const first = yield* svc.getOrgReadToken(asOrgId("org_a")) + // Advance well within the (ttl - skew = 540s) window. + yield* TestClock.setTime(120_000) + const second = yield* svc.getOrgReadToken(asOrgId("org_a")) + assert.strictEqual(second, first) + }).pipe(Effect.provide(layer)), + ) + + it.effect("re-mints after the cached token nears expiry", () => + Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const first = yield* svc.getOrgReadToken(asOrgId("org_a")) + // Past the 540s re-mint deadline → new token (later exp). + yield* TestClock.setTime(600_000) + const second = yield* svc.getOrgReadToken(asOrgId("org_a")) + assert.notStrictEqual(second, first) + assert.isAbove(decodePayload(second).exp, decodePayload(first).exp) + }).pipe(Effect.provide(layer)), + ) + + it.effect("issues distinct tokens per org", () => + Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const a = yield* svc.getOrgReadToken(asOrgId("org_a")) + const b = yield* svc.getOrgReadToken(asOrgId("org_b")) + assert.notStrictEqual(a, b) + assert.isTrue(decodePayload(b).scopes.every((s) => s.filter === "OrgId = 'org_b'")) + }).pipe(Effect.provide(layer)), + ) + + it.effect("returns a typed error when signing configuration is missing", () => { + const missingLayer = TinybirdOrgTokenService.layer.pipe( + Layer.provide(Env.layer), + Layer.provide(testConfig({}, false)), + ) + return Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const error = yield* Effect.flip(svc.getOrgReadToken(asOrgId("org_a"))) + assert.strictEqual(error.reason, "MissingSigningKey") + assert.notInclude(error.message, "api-token-is-not-the-signing-key") + }).pipe(Effect.provide(missingLayer)) + }) + + it.effect("returns a typed error for an empty workspace id", () => { + const malformedLayer = TinybirdOrgTokenService.layer.pipe( + Layer.provide(Env.layer), + Layer.provide(testConfig({ TINYBIRD_WORKSPACE_ID: "" })), + ) + return Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const error = yield* Effect.flip(svc.getOrgReadToken(asOrgId("org_a"))) + assert.strictEqual(error.reason, "MissingWorkspaceId") + }).pipe(Effect.provide(malformedLayer)) + }) +}) diff --git a/apps/api/src/services/TinybirdOrgTokenService.ts b/apps/api/src/services/TinybirdOrgTokenService.ts new file mode 100644 index 000000000..ae10bba29 --- /dev/null +++ b/apps/api/src/services/TinybirdOrgTokenService.ts @@ -0,0 +1,105 @@ +import type { OrgId } from "@maple/domain" +import { Clock, Context, Effect, Layer, Option, Redacted, Schema } from "effect" +import { listOrgScopedDatasourceNames } from "../lib/warehouse-catalog" +import { mintOrgReadJwt } from "../lib/tinybird-jwt" +import { Env } from "../lib/Env" + +// --------------------------------------------------------------------------- +// TinybirdOrgTokenService — mints and caches per-org Tinybird read JWTs used to +// scope the raw-SQL path to a single org's rows (row-level security enforced by +// Tinybird server-side; see lib/tinybird-jwt.ts). +// +// A JWT is reused for its lifetime and re-minted on expiry. The cache entry +// expires SKEW seconds before the token itself, so a served token always has +// comfortably more life left than the executor's 30s client-cache TTL — a cached +// Tinybird client never outlives the JWT it was built with. +// --------------------------------------------------------------------------- + +/** Token lifetime. */ +const JWT_TTL_SECONDS = 600 +/** Re-mint this many seconds before true expiry (must exceed the executor's 30s client cache). */ +const JWT_REFRESH_SKEW_SECONDS = 60 + +export interface TinybirdOrgTokenServiceShape { + /** A Tinybird read JWT scoped to `orgId` across every OrgId-bearing datasource. */ + readonly getOrgReadToken: (orgId: OrgId) => Effect.Effect +} + +export class TinybirdOrgTokenError extends Schema.TaggedErrorClass()( + "@maple/api/services/TinybirdOrgTokenError", + { + reason: Schema.Literals(["MissingSigningKey", "MissingWorkspaceId", "MintFailed"]), + message: Schema.String, + }, +) {} + +export class TinybirdOrgTokenService extends Context.Service< + TinybirdOrgTokenService, + TinybirdOrgTokenServiceShape +>()("@maple/api/services/TinybirdOrgTokenService", { + make: Effect.gen(function* () { + const env = yield* Env + // The scope allowlist is static per deploy — compute it once. + const datasourceNames = listOrgScopedDatasourceNames() + + // Per-instance (per-isolate) cache. `expiresAt` is the re-mint deadline in ms. + const cache = new Map() + + const getOrgReadToken = Effect.fn("TinybirdOrgTokenService.getOrgReadToken")(function* ( + orgId: OrgId, + ) { + const nowMs = yield* Clock.currentTimeMillis + const cached = cache.get(orgId) + if (cached !== undefined && cached.expiresAt > nowMs) { + yield* Effect.annotateCurrentSpan("maple.tinybird.jwt.cache_hit", true) + return cached.token + } + yield* Effect.annotateCurrentSpan("maple.tinybird.jwt.cache_hit", false) + if (Option.isNone(env.TINYBIRD_SIGNING_KEY)) { + return yield* new TinybirdOrgTokenError({ + reason: "MissingSigningKey", + message: "TINYBIRD_SIGNING_KEY is required for Tinybird-scoped raw SQL", + }) + } + if (Option.isNone(env.TINYBIRD_WORKSPACE_ID) || env.TINYBIRD_WORKSPACE_ID.value.trim() === "") { + return yield* new TinybirdOrgTokenError({ + reason: "MissingWorkspaceId", + message: "TINYBIRD_WORKSPACE_ID is required for Tinybird-scoped raw SQL", + }) + } + const workspaceId = env.TINYBIRD_WORKSPACE_ID.value + const signingKey = Redacted.value(env.TINYBIRD_SIGNING_KEY.value) + if (signingKey.trim() === "") { + return yield* new TinybirdOrgTokenError({ + reason: "MissingSigningKey", + message: "TINYBIRD_SIGNING_KEY must not be empty", + }) + } + const token = yield* Effect.try({ + try: () => + mintOrgReadJwt({ + signingKey, + workspaceId, + orgId, + datasourceNames, + nowSeconds: Math.floor(nowMs / 1000), + ttlSeconds: JWT_TTL_SECONDS, + }), + catch: () => + new TinybirdOrgTokenError({ + reason: "MintFailed", + message: "Failed to mint the Tinybird org-scoped read token", + }), + }) + cache.set(orgId, { + token, + expiresAt: nowMs + (JWT_TTL_SECONDS - JWT_REFRESH_SKEW_SECONDS) * 1000, + }) + return token + }) + + return { getOrgReadToken } satisfies TinybirdOrgTokenServiceShape + }), +}) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts b/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts index 1767b1a9f..1b7a4a07a 100644 --- a/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts +++ b/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts @@ -88,8 +88,11 @@ async function exec(cfg: ChConfig, sql: string): Promise { "X-ClickHouse-Database": cfg.database, } if (cfg.password.length > 0) headers["X-ClickHouse-Key"] = cfg.password - const response = await fetch(url, { method: "POST", headers, body: sql }) + const response = await fetch(url, { method: "POST", headers, body: sql, redirect: "manual" }) const text = await response.text() + if (response.status >= 300 && response.status < 400) { + throw new Error(`ClickHouse redirect responses are not allowed (${response.status})`) + } if (!response.ok) { throw new Error(`ClickHouse ${response.status}: ${text.split("\n")[0]?.slice(0, 500) ?? ""}`) } @@ -134,11 +137,7 @@ const recordVersion = (cfg: ChConfig, version: number, description: string) => // --- config load + decrypt (imperative mirror of the service helper) -------- -const loadConfig = async ( - db: MaplePgClient, - orgId: string, - encryptionKey: Buffer, -): Promise => { +const loadConfig = async (db: MaplePgClient, orgId: string, encryptionKey: Buffer): Promise => { const rows = await db .select() .from(orgClickHouseSettings) @@ -159,6 +158,21 @@ const loadConfig = async ( decipher.final(), ]).toString("utf8") } + let parsedUrl: URL + try { + parsedUrl = new URL(row.chUrl) + } catch { + throw new Error("Stored ClickHouse URL is invalid") + } + if (parsedUrl.username.length > 0 || parsedUrl.password.length > 0) { + throw new Error("Stored ClickHouse credentials must not be embedded in the URL") + } + if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") { + throw new Error("Stored ClickHouse URL must use HTTP or HTTPS") + } + if (password.length > 0 && parsedUrl.protocol !== "https:") { + throw new Error("Stored ClickHouse URL must use HTTPS when a password is configured") + } return { url: row.chUrl, user: row.chUser, password, database: row.chDatabase } } diff --git a/apps/api/wrangler.jsonc b/apps/api/wrangler.jsonc index 06be5ce8b..349c16f67 100644 --- a/apps/api/wrangler.jsonc +++ b/apps/api/wrangler.jsonc @@ -7,6 +7,19 @@ "dev": { "port": 3472, }, + "vars": { + "API_V2_RATE_LIMIT_PARTITION": "local", + }, + "ratelimits": [ + { + "name": "API_V2_RATE_LIMITER", + "namespace_id": "2026071801", + "simple": { + "limit": 600, + "period": 60, + }, + }, + ], // Periodic VCS sync backstop every 12h — handler: worker.ts `scheduled`. (Mirrored in alchemy.run.ts for deploy.) "triggers": { "crons": ["0 */12 * * *"], diff --git a/apps/web/src/components/settings/api-key-create-payload.test.ts b/apps/web/src/components/settings/api-key-create-payload.test.ts index ef596dd45..213e08ea4 100644 --- a/apps/web/src/components/settings/api-key-create-payload.test.ts +++ b/apps/web/src/components/settings/api-key-create-payload.test.ts @@ -12,13 +12,17 @@ describe("buildApiKeyCreatePayload", () => { }) it("preserves the MCP kind and a non-empty description", () => { - const payload = buildApiKeyCreatePayload(" MCP key ", " Claude desktop ", "mcp") + const payload = buildApiKeyCreatePayload(" MCP key ", " Claude desktop ", "mcp", { + scopes: ["traces:read"], + }) expect(payload).toEqual({ name: "MCP key", description: "Claude desktop", kind: "mcp", }) + // MCP authentication requires an unscoped MCP-kind key. Ignore scopes at + // the payload boundary even if a caller accidentally supplies them. expect(() => Schema.encodeUnknownSync(V2ApiKeyCreateParams)(payload)).not.toThrow() }) diff --git a/apps/web/src/components/settings/api-key-create-payload.ts b/apps/web/src/components/settings/api-key-create-payload.ts index 29ac78d77..c86d6be75 100644 --- a/apps/web/src/components/settings/api-key-create-payload.ts +++ b/apps/web/src/components/settings/api-key-create-payload.ts @@ -18,10 +18,8 @@ export const buildApiKeyCreatePayload = ( name: name.trim(), ...(trimmedDescription.length > 0 ? { description: trimmedDescription } : {}), ...(kind !== undefined ? { kind } : {}), - ...(options.expiresInSeconds !== undefined - ? { expires_in_seconds: options.expiresInSeconds } - : {}), - ...(options.scopes !== undefined && options.scopes.length > 0 + ...(options.expiresInSeconds !== undefined ? { expires_in_seconds: options.expiresInSeconds } : {}), + ...(kind !== "mcp" && options.scopes !== undefined && options.scopes.length > 0 ? { scopes: options.scopes } : {}), } diff --git a/apps/web/src/components/settings/create-api-key-dialog.tsx b/apps/web/src/components/settings/create-api-key-dialog.tsx index c9d6ec242..d4d730db3 100644 --- a/apps/web/src/components/settings/create-api-key-dialog.tsx +++ b/apps/web/src/components/settings/create-api-key-dialog.tsx @@ -1,5 +1,5 @@ import { useAtomSet } from "@/lib/effect-atom" -import { useState } from "react" +import { useId, useState } from "react" import { Exit } from "effect" import type { ApiKeyKind } from "@maple/domain/http" import type { V2ApiKeyWithSecret, V2Scope } from "@maple/domain/http/v2" @@ -59,6 +59,12 @@ const SCOPE_FAMILIES = [ { id: "investigations", label: "Investigations" }, { id: "anomalies", label: "Anomalies" }, { id: "session_replays", label: "Session replays" }, + { id: "traces", label: "Traces" }, + { id: "logs", label: "Logs" }, + { id: "metrics", label: "Metrics" }, + { id: "services", label: "Services" }, + { id: "service_map", label: "Service map" }, + { id: "query", label: "Query" }, { id: "organization", label: "Organization" }, ] as const @@ -75,6 +81,8 @@ const scopesFromLevels = (levels: Record): Array => }) export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: CreateApiKeyDialogProps) { + const isMcp = kind === "mcp" + const accessLabelId = useId() const [newName, setNewName] = useState("") const [newDescription, setNewDescription] = useState("") const [expiration, setExpiration] = useState("never") @@ -88,8 +96,8 @@ export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: Crea mode: "promiseExit", }) - const restrictedScopes = accessMode === "restricted" ? scopesFromLevels(scopeLevels) : undefined - const missingScopes = accessMode === "restricted" && restrictedScopes?.length === 0 + const restrictedScopes = !isMcp && accessMode === "restricted" ? scopesFromLevels(scopeLevels) : undefined + const missingScopes = !isMcp && accessMode === "restricted" && restrictedScopes?.length === 0 const canCreate = newName.trim().length > 0 && !missingScopes && !isCreating async function handleCreate() { @@ -132,7 +140,7 @@ export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: Crea {createdKey ? ( <> - API key created + {isMcp ? "MCP key created" : "API key created"} Copy your API key now. You won't be able to see it again. @@ -161,9 +169,11 @@ export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: Crea ) : ( <> - Create API key + {isMcp ? "Create MCP key" : "Create API key"} - API keys are used to authenticate with the Maple API and MCP server. + {isMcp + ? "MCP keys authenticate clients with the Maple MCP server." + : "API keys authenticate clients with the Maple API."} @@ -215,65 +225,81 @@ export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: Crea -
- - { - const next = values[0] - if (next === "full" || next === "restricted") setAccessMode(next) - }} - variant="outline" - size="sm" - > - Full access - Restricted - - {accessMode === "restricted" ? ( -
- {SCOPE_FAMILIES.map((family) => ( -
- - {family.label} - - { - const next = values[0] - if ( - next === "none" || - next === "read" || - next === "write" - ) { - setScopeLevels((current) => ({ - ...current, - [family.id]: next, - })) - } - }} - variant="outline" - size="sm" - > - None - Read - Write - -
- ))} + {!isMcp && ( +
+ + { + const next = values[0] + if (next === "full" || next === "restricted") setAccessMode(next) + }} + variant="outline" + size="sm" + > + Full access + Restricted + + {accessMode === "restricted" ? ( +
+ {SCOPE_FAMILIES.map((family) => { + const familyLabelId = `${accessLabelId}-${family.id}` + return ( +
+ + {family.label} + + { + const next = values[0] + if ( + next === "none" || + next === "read" || + next === "write" + ) { + setScopeLevels((current) => ({ + ...current, + [family.id]: next, + })) + } + }} + variant="outline" + size="sm" + > + + None + + + Read + + + Write + + +
+ ) + })} +

+ Write includes read. Scopes are fixed at creation — roll the + key to change access. +

+
+ ) : (

- Write includes read. Scopes are fixed at creation — roll the key - to change access. + Full access to the organization's API.

-
- ) : ( -

- Full access to the organization's API. -

- )} -
+ )} +
+ )} diff --git a/apps/web/src/components/settings/mcp-section.tsx b/apps/web/src/components/settings/mcp-section.tsx index fd915a8bd..b517ed529 100644 --- a/apps/web/src/components/settings/mcp-section.tsx +++ b/apps/web/src/components/settings/mcp-section.tsx @@ -186,6 +186,7 @@ export function McpSection() { open={createDialogOpen} onOpenChange={setCreateDialogOpen} onCreated={(secret) => setCreatedSecret(secret)} + kind="mcp" /> ) diff --git a/docs/api-v2.md b/docs/api-v2.md index 03fdbb89f..f1d7f639a 100644 --- a/docs/api-v2.md +++ b/docs/api-v2.md @@ -34,7 +34,7 @@ POST /v2/traces/search complex reads are POST .../search Every v2 object has a prefixed public ID (`key_4CzLmR…`, `dash_…`, `alrt_…`). Public IDs are opaque; internally they are a reversible base58 encoding of the internal ID, computed at the API boundary (`packages/domain/src/http/v2/public-id.ts` — the prefix registry lives there and is the single source of truth). No database migration: rows keep their raw UUIDs / internal strings. -Prefixes: `key` (API key), `ingk` (ingest key), `dash` (dashboard), `dbv` (dashboard version), `dtpl` (dashboard template), `alrt` (alert rule), `dest` (alert destination), `inc` (alert incident), `einc` (error incident), `iss` (error issue), `inv` (investigation), `anom` (anomaly incident), `scrp` (scrape target), `rec` (recommendation), `amap` (attribute mapping), and `srep` (session replay); `evt` and `we` are reserved for events/webhooks. +Prefixes: `key` (API key), `ingk` (ingest key), `dash` (dashboard), `dbv` (dashboard version), `dtpl` (dashboard template), `alrt` (alert rule), `dest` (alert destination), `inc` (alert incident), `einc` (error incident), `iss` (error issue), `inv` (investigation), `anom` (anomaly incident), `scrp` (scrape target), `rec` (recommendation), `amap` (attribute mapping), `srep` (session replay), and `log` (synthetic log identity); `evt` and `we` are reserved for events/webhooks. Exception: Clerk-issued `org_…` / `user_…` IDs are already prefixed public IDs and pass through unchanged. @@ -112,9 +112,13 @@ Implementation: `packages/domain/src/http/v2/auth.ts` + `apps/api/src/services/A Mutating endpoints will accept an `Idempotency-Key` header. Replays within the retention window return the original response. Backed by a Postgres `idempotency_keys` table keyed by `(org_id, key)`. -### Rate limiting (Phase 4 — reserved) +### Rate limiting -Per-key rate limits will return `429` with the error envelope (`type: "rate_limit_error"`) and a `Retry-After` header. +API-key-authenticated requests share one budget per key across the entire `/v2` surface: **600 requests per 60 seconds**. The budget is partitioned by deployment stage, and rolling a key starts a fresh budget because the replacement has a new internal key ID. Valid-key requests count even when a later scope or role check rejects them; invalid credentials and dashboard session tokens do not use this budget. + +Exceeding the budget returns `429` with the standard error envelope (`type: "rate_limit_error"`, `code: "rate_limited"`) and `Retry-After: 60`. No remaining/reset headers are emitted because the backing Cloudflare binding returns only an allow/deny result. + +The limiter fails open: if its binding is missing or errors, Maple logs and traces `maple.rate_limit.outcome=failed_open` and continues the request. Cloudflare counters are local to the location serving the Worker and eventually consistent, so this is abuse protection rather than exact global quota accounting. ### Expansion — not supported @@ -140,14 +144,27 @@ Implemented in phases; the pilot (`api_keys`) ships first and proves every conve | `attribute_mappings` ✅ | CRUD | `ingestAttributeMappings` | | `session_replays` ✅ | `search`/retrieve + events/transcript/`for_trace` (reduced; `facets`/`trace-summaries` deferred) | `sessionReplays` | | `organization` 🟡 | retrieve (GET only shipped); update settings (incl. ClickHouse BYOC) + delete deferred | `organizations`, `orgClickHouseSettings` | -| `traces` | `POST /v2/traces/search`, `GET /v2/traces/{trace_id}`, `GET /v2/traces/{trace_id}/spans/{span_id}` | `queryEngine`, `observability` | -| `logs` | `POST /v2/logs/search`, `GET /v2/logs/{id}` | `queryEngine` | -| `metrics` | `GET /v2/metrics`, `POST /v2/metrics/timeseries` | `queryEngine` | -| `services` | `GET /v2/services`, `GET /v2/services/{name}`, `GET /v2/service_map` | `queryEngine` | -| `query` | `POST /v2/query` — query-builder execution; raw SQL org-gated | `queryEngine` | +| `traces` ✅ | `POST /v2/traces/search`, `GET /v2/traces/{trace_id}`, `GET /v2/traces/{trace_id}/spans/{span_id}` | `queryEngine`, `observability` | +| `logs` ✅ | `POST /v2/logs/search`, `GET /v2/logs/{id}` | `queryEngine` | +| `metrics` ✅ | `GET /v2/metrics`, `POST /v2/metrics/timeseries` | `queryEngine` | +| `services` ✅ | `GET /v2/services`, `GET /v2/services/{name}` | `queryEngine` | +| `service_map` ✅ | `GET /v2/service_map` | `queryEngine` | +| `query` ✅ | `POST /v2/query` — structured telemetry queries | `queryEngine` | The long tail of ~40 query-engine RPC endpoints (facets, infra hosts/pods/nodes/workloads, Cloudflare/PlanetScale infra) starts in the internal RPC tier and is promoted into `/v2` individually as shapes stabilize. +### Telemetry reads + +Phase 2 exposes traces, logs, metrics, services, the service map, and the common query engine. Collection and aggregation operations require explicit ISO-8601 UTC `start_time` and `end_time`; `end_time` must be later than `start_time`, and ranges are capped at 31 days. Direct trace/span retrieval uses the `(OrgId, TraceId, SpanId)` sorting-key prefix so it returns the complete retained trace without a correctness-limiting timestamp window. Direct log retrieval derives its narrow partition window from the timestamp embedded in the log ID. + +Trace IDs, span IDs, metric names, and service names remain their native OpenTelemetry identifiers. Logs have no native record ID, so search results return a deterministic `log_…` ID containing a compact timestamp and a complete-record fingerprint. Malformed log IDs return `log_id_invalid`; records outside retention return `log_not_found`. + +All telemetry lists use the standard cursor envelope with a default limit of 20 and maximum of 100. Trace results are root-span summaries ordered by start time and trace ID. Log ordering includes timestamp plus the complete composite identity. Services are grouped by service name across namespaces and deployment environments. The service map includes only service-to-service edges in this version; database, external, and infrastructure nodes remain internal APIs. + +`POST /v2/query` accepts `timeseries` and `breakdown` specifications for trace, log, and metric sources using snake_case fields. Responses use the common `query_result` envelope. Raw SQL is intentionally unavailable through the public API because tenant isolation must not depend on inspecting caller-authored SQL text. Warehouse diagnostics are never returned to callers. + +Telemetry scope families are `traces`, `logs`, `metrics`, `services`, `service_map`, and `query`. Search, timeseries, and query POSTs are read-only operations for scope enforcement, so `:read` is sufficient. + Not in v2: org membership and invitations (delegated to Clerk; revisit if/when a members API is needed). ### Optional ElectricSQL `txid` metadata @@ -158,9 +175,9 @@ The dashboard can reconcile optimistic writes against ElectricSQL synced shapes - **Phase 0 (this change)** — conventions doc, v2 primitives (`public-id`, `envelopes`, `errors`, `auth`), scoped API keys (schema + service + enforcement), `MapleApiV2` shell mounted at `/v2` with Scalar docs at `/v2/docs`, pilot resource `api_keys` end-to-end with tests. - **Phase 1 — core resources**: dashboards, alerts, error issues, scrape targets, ingest keys, attribute mappings, investigations, anomalies, recommendations, organization, session replays. Thin handler adapters over existing services; `txid` preserved. -- **Phase 2 — telemetry reads**: traces/logs/metrics/services/service_map/query over `QueryEngineService`. +- **Phase 2 ✅ — telemetry reads**: traces/logs/metrics/services/service_map/query over `QueryEngineService`. - **Phase 3 — internal RPC tier + dashboard migration**: `RpcGroup` contracts in `packages/domain/src/rpc/` served at `/rpc`; dashboard gets a `MapleApiV2AtomClient` (same wiring as `apps/web/src/lib/services/common/atom-client.ts`, pointed at `MapleApiV2`) plus an `RpcClient`; migrate group-by-group, deleting v1 groups as they empty. (Note: the billing-scoped 401 retry in `atom-client.ts` must follow billing to its RPC home.) -- **Phase 4 — hardening**: `Idempotency-Key`, per-key rate limiting (Workers rate-limit binding), `Maple-Version` header enforcement. +- **Phase 4 — hardening**: `Idempotency-Key`, `Maple-Version` header enforcement. Per-key rate limiting is implemented. - **Phase 5 — events & webhooks**: `evt_` event objects, `GET /v2/events`, `/v2/webhook_endpoints` CRUD, HMAC-signed deliveries (`Maple-Signature`) via an outbox drained by the alerting worker. ## Adding a v2 resource (checklist) diff --git a/docs/self-hosted-clickhouse.md b/docs/self-hosted-clickhouse.md index 830d8d10a..89259fcbc 100644 --- a/docs/self-hosted-clickhouse.md +++ b/docs/self-hosted-clickhouse.md @@ -24,6 +24,25 @@ Self-managed Maple is a **per-org BYO** feature. Each org configures their own b The Maple deployment itself still uses the env-level `TINYBIRD_HOST` / `TINYBIRD_TOKEN` for any org without a BYO row. API query routing does not require new env vars for ClickHouse-BYO; D1-backed direct ingest does require `MAPLE_INGEST_KEY_ENCRYPTION_KEY` so the ingest gateway can decrypt stored ClickHouse passwords. +Env-level `CLICKHOUSE_URL` defaults to Tinybird's ClickHouse-compatible gateway for +compatibility with existing deployments; raw SQL substitutes a per-org JWT and +removes Tinybird-restricted query settings. For a vanilla/self-managed server, set +`CLICKHOUSE_PROVIDER=clickhouse`; Maple then preserves `CLICKHOUSE_PASSWORD` for raw +SQL. Tinybird raw SQL also requires explicit `TINYBIRD_SIGNING_KEY` and +`TINYBIRD_WORKSPACE_ID` values; Maple never derives either from the API token. + +Env-level vanilla ClickHouse raw SQL is enabled only when `MAPLE_AUTH_MODE=self_hosted`, +where the deployment is single-org. Hosted multi-org deployments fail closed unless +they use Tinybird's scoped JWT path or per-org BYO credentials. + +For user-authored SQL, configure the runtime ClickHouse account as SELECT-only on the +Maple database. Apply schema migrations with a separate administrative account, and +enforce server-side limits such as maximum execution time, memory, rows, and bytes +read. Maple's client-side caps are a final response boundary, not a substitute for a +least-privilege role and server-side resource controls. See +[ClickHouse's guidance for agent-authored queries](https://clickhouse.com/blog/how-to-set-up-clickhouse-for-agentic-analytics) +for the corresponding database-side setup. + ### Routing precedence For any given query the API resolves the upstream in this order: diff --git a/packages/domain/src/http/alerts.test.ts b/packages/domain/src/http/alerts.test.ts index dd59d1f19..f642c6464 100644 --- a/packages/domain/src/http/alerts.test.ts +++ b/packages/domain/src/http/alerts.test.ts @@ -182,6 +182,7 @@ describe("AlertNotificationTemplate", () => { describe("AlertRuleUpsertRequest notificationTemplate", () => { const decode = Schema.decodeUnknownSync(AlertRuleUpsertRequest) + const decodeExit = Schema.decodeUnknownExit(AlertRuleUpsertRequest) const baseRule = { name: "Checkout errors", @@ -205,4 +206,8 @@ describe("AlertRuleUpsertRequest notificationTemplate", () => { expect(decode({ ...baseRule, notificationTemplate: null }).notificationTemplate).toBeNull() expect(decode(baseRule).notificationTemplate).toBeUndefined() }) + + it("rejects alert windows longer than 24 hours", () => { + expect(Exit.isFailure(decodeExit({ ...baseRule, windowMinutes: 24 * 60 + 1 }))).toBe(true) + }) }) diff --git a/packages/domain/src/http/alerts.ts b/packages/domain/src/http/alerts.ts index cd1439499..1ced0fa80 100644 --- a/packages/domain/src/http/alerts.ts +++ b/packages/domain/src/http/alerts.ts @@ -157,6 +157,11 @@ const NonNegativeInt = Schema.Number.pipe(Schema.check(Schema.isInt(), Schema.is const PositiveFloat = Schema.Number.pipe(Schema.check(Schema.isFinite(), Schema.isGreaterThan(0))) +export const MAX_ALERT_WINDOW_MINUTES = 24 * 60 +export const AlertWindowMinutes = PositiveInt.pipe( + Schema.check(Schema.isLessThanOrEqualTo(MAX_ALERT_WINDOW_MINUTES)), +) + export class SlackAlertDestinationConfig extends Schema.Class( "SlackAlertDestinationConfig", )({ @@ -461,7 +466,7 @@ export class AlertRuleDocument extends Schema.Class("AlertRul comparator: AlertComparator, threshold: Schema.Number, thresholdUpper: Schema.NullOr(Schema.Number), - windowMinutes: PositiveInt, + windowMinutes: AlertWindowMinutes, minimumSampleCount: NonNegativeInt, consecutiveBreachesRequired: PositiveInt, consecutiveHealthyRequired: PositiveInt, @@ -505,7 +510,7 @@ export class AlertRuleUpsertRequest extends Schema.Class comparator: AlertComparator, threshold: Schema.Number, thresholdUpper: Schema.optionalKey(Schema.NullOr(Schema.Number)), - windowMinutes: PositiveInt, + windowMinutes: AlertWindowMinutes, minimumSampleCount: Schema.optionalKey(NonNegativeInt), consecutiveBreachesRequired: Schema.optionalKey(PositiveInt), consecutiveHealthyRequired: Schema.optionalKey(PositiveInt), diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 02114a005..c5616db4c 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -759,9 +759,9 @@ export class ListLogsResponse extends Schema.Class("ListLogsRe }) {} // Exact-match lookup of one log by its composite key (logs have no primary id). -// `timestamp` is the raw ClickHouse DateTime64 string and carries sub-second -// precision (`YYYY-MM-DD HH:mm:ss.fffffffff`), so it is a plain string rather -// than `TinybirdDateTime` (which only matches second-level precision). +// `timestamp` is the raw ClickHouse DateTime64 string. It remains a plain +// string because older stored rows and upstream drivers can vary their +// fractional-second rendering. export class GetLogRequest extends Schema.Class("GetLogRequest")({ timestamp: Schema.String, serviceName: ServiceName, @@ -1366,12 +1366,21 @@ export const RawSqlDisplayType = Schema.Literals([ ]) export type RawSqlDisplayType = Schema.Schema.Type +export const MAX_RAW_SQL_LENGTH = 32_768 +export const MAX_RAW_SQL_RESULT_ROWS = 1_000 +export const MAX_RAW_SQL_RESULT_BYTES = 5_000_000 +export const MAX_RAW_SQL_CELL_LENGTH = 64_000 +export const MAX_RAW_SQL_ALERT_GROUPS = 100 +export const MAX_RAW_SQL_GROUP_KEY_LENGTH = 256 + export class RawSqlExecuteRequest extends Schema.Class("RawSqlExecuteRequest")({ - sql: Schema.String, + sql: Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(MAX_RAW_SQL_LENGTH)), displayType: RawSqlDisplayType, startTime: TinybirdDateTime, endTime: TinybirdDateTime, - granularitySeconds: Schema.optional(Schema.Number), + granularitySeconds: Schema.optional( + Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0)), + ), }) {} export class RawSqlExecuteResponse extends Schema.Class("RawSqlExecuteResponse")({ @@ -1392,6 +1401,7 @@ export class RawSqlValidationError extends Schema.TaggedErrorClass): Record => { + for (const pathItem of Object.values(spec.paths ?? {})) { + if (typeof pathItem !== "object" || pathItem === null) continue + for (const method of HTTP_OPERATION_METHODS) { + const operation = (pathItem as Record)[method] + if (typeof operation !== "object" || operation === null) continue + const response = operation.responses?.["429"] + if (typeof response !== "object" || response === null) continue + response.headers = { + ...response.headers, + "Retry-After": { + description: "Seconds to wait before retrying the request.", + schema: { type: "integer", minimum: 1 }, + example: 60, + }, + } + } + } + return spec +} + /** * The Maple v2 public API (see docs/api-v2.md). * @@ -40,6 +72,12 @@ export class MapleApiV2 extends HttpApi.make("MapleApiV2") .add(V2AnomaliesApiGroup) .add(V2OrganizationApiGroup) .add(V2SessionReplaysApiGroup) + .add(V2TracesApiGroup) + .add(V2LogsApiGroup) + .add(V2MetricsApiGroup) + .add(V2ServicesApiGroup) + .add(V2ServiceMapApiGroup) + .add(V2QueryApiGroup) .middleware(V2UnexpectedErrors) .annotateMerge( OpenApi.annotations({ @@ -63,20 +101,23 @@ export class MapleApiV2 extends HttpApi.make("MapleApiV2") // `info.contact` and top-level `externalDocs` have no dedicated annotation // key (they are not in `OpenAPISpecInfo`), so inject them via the api-level // spec transform, which receives the whole generated document. - transform: (spec) => ({ - ...spec, - info: { - ...spec.info, - contact: { - name: "Maple Support", - url: "https://maple.dev", - email: "support@maple.dev", + transform: (spec) => { + const withRateLimitHeaders = addRateLimitResponseHeaders(spec) + return { + ...withRateLimitHeaders, + info: { + ...withRateLimitHeaders.info, + contact: { + name: "Maple Support", + url: "https://maple.dev", + email: "support@maple.dev", + }, }, - }, - externalDocs: { - url: "https://api.maple.dev/v2/docs", - description: "Interactive Maple API reference", - }, - }), + externalDocs: { + url: "https://api.maple.dev/v2/docs", + description: "Interactive Maple API reference", + }, + } + }, }), ) {} diff --git a/packages/domain/src/http/v2/auth.ts b/packages/domain/src/http/v2/auth.ts index 1c102d58c..dc0d617ac 100644 --- a/packages/domain/src/http/v2/auth.ts +++ b/packages/domain/src/http/v2/auth.ts @@ -6,6 +6,7 @@ import { V2AuthenticationError, V2InvalidRequestError, V2PermissionError, + V2RateLimitError, V2ServiceUnavailableError, } from "./errors" @@ -23,7 +24,7 @@ export class AuthorizationV2 extends HttpApiMiddleware.Service< provides: Context } >()("AuthorizationV2", { - error: [V2AuthenticationError, V2PermissionError, V2ServiceUnavailableError], + error: [V2AuthenticationError, V2PermissionError, V2RateLimitError, V2ServiceUnavailableError], security: { bearer: HttpApiSecurity.bearer.pipe( HttpApiSecurity.annotateMerge( @@ -77,6 +78,10 @@ const READ_ONLY_POST_PATHS = new Set([ "/v2/session_replays/search", "/v2/session_replays/for_trace", "/v2/alerts/rules/preview", + "/v2/traces/search", + "/v2/logs/search", + "/v2/metrics/timeseries", + "/v2/query", ]) /** diff --git a/packages/domain/src/http/v2/errors.ts b/packages/domain/src/http/v2/errors.ts index 28c8dd30e..34315fe59 100644 --- a/packages/domain/src/http/v2/errors.ts +++ b/packages/domain/src/http/v2/errors.ts @@ -247,6 +247,15 @@ export const resourceNotFound = (resource: string, message: string, param = "id" export const conflict = (code: string, message: string) => new V2ConflictError({ error: { type: "conflict_error", code, message } }) +export const rateLimited = () => + new V2RateLimitError({ + error: { + type: "rate_limit_error", + code: "rate_limited", + message: "Too many requests. Retry after 60 seconds.", + }, + }) + export const upstreamError = (code: string, message: string) => new V2UpstreamError({ error: { type: "api_error", code, message } }) diff --git a/packages/domain/src/http/v2/index.ts b/packages/domain/src/http/v2/index.ts index 6c637681f..5f929c5f4 100644 --- a/packages/domain/src/http/v2/index.ts +++ b/packages/domain/src/http/v2/index.ts @@ -17,3 +17,4 @@ export * from "./recommendations" export * from "./resource-ids" export * from "./scrape-targets" export * from "./session-replays" +export * from "./telemetry" diff --git a/packages/domain/src/http/v2/openapi.test.ts b/packages/domain/src/http/v2/openapi.test.ts index be2c97af8..0138dac8c 100644 --- a/packages/domain/src/http/v2/openapi.test.ts +++ b/packages/domain/src/http/v2/openapi.test.ts @@ -87,13 +87,20 @@ describe("MapleApiV2 OpenAPI", () => { "GET /v2/instrumentation/recommendations", "GET /v2/investigations", "GET /v2/investigations/{id}", + "GET /v2/logs/{id}", + "GET /v2/metrics", "GET /v2/organization", "GET /v2/scrape_targets", "GET /v2/scrape_targets/{id}", "GET /v2/scrape_targets/{id}/checks", + "GET /v2/service_map", + "GET /v2/services", + "GET /v2/services/{name}", "GET /v2/session_replays/{id}", "GET /v2/session_replays/{id}/events", "GET /v2/session_replays/{id}/transcript", + "GET /v2/traces/{trace_id}", + "GET /v2/traces/{trace_id}/spans/{span_id}", "PATCH /v2/alerts/destinations/{id}", "PATCH /v2/alerts/rules/{id}", "PATCH /v2/anomalies/settings", @@ -119,10 +126,14 @@ describe("MapleApiV2 OpenAPI", () => { "POST /v2/instrumentation/recommendations/{id}/reopen", "POST /v2/investigations", "POST /v2/investigations/{id}/status", + "POST /v2/logs/search", + "POST /v2/metrics/timeseries", + "POST /v2/query", "POST /v2/scrape_targets", "POST /v2/scrape_targets/{id}/probe", "POST /v2/session_replays/for_trace", "POST /v2/session_replays/search", + "POST /v2/traces/search", "PUT /v2/anomalies/incidents/{id}/issue", ]) }) @@ -161,12 +172,18 @@ describe("MapleApiV2 OpenAPI", () => { expect(op.description.length).toBeGreaterThan(20) expect(op.tags).toHaveLength(1) expect(op.security).toEqual([{ bearer: [] }]) - for (const status of ["400", "401", "403", "500"]) { + for (const status of ["400", "401", "403", "429", "500"]) { expect( op.responses[status], `${method.toUpperCase()} ${path} declares ${status}`, ).toBeDefined() } + const rateLimitResponse = op.responses["429"] as Record + expect(rateLimitResponse.headers?.["Retry-After"]).toMatchObject({ + description: expect.any(String), + schema: { type: "integer", minimum: 1 }, + example: 60, + }) } }) @@ -188,6 +205,7 @@ describe("MapleApiV2 OpenAPI", () => { "AuthenticationError", "PermissionError", "NotFoundError", + "RateLimitError", "ServiceUnavailableError", ]), ) diff --git a/packages/domain/src/http/v2/public-id.ts b/packages/domain/src/http/v2/public-id.ts index 4f0284ddd..efed158b4 100644 --- a/packages/domain/src/http/v2/public-id.ts +++ b/packages/domain/src/http/v2/public-id.ts @@ -35,6 +35,8 @@ export const PublicIdPrefixes = { ingestKey: "ingk", attributeMapping: "amap", sessionReplay: "srep", + /** Synthetic identity for logs, which have no native OTel record id. */ + log: "log", /** Reserved for the future events/webhooks system. */ event: "evt", webhookEndpoint: "we", diff --git a/packages/domain/src/http/v2/telemetry.ts b/packages/domain/src/http/v2/telemetry.ts new file mode 100644 index 000000000..03228ea08 --- /dev/null +++ b/packages/domain/src/http/v2/telemetry.ts @@ -0,0 +1,641 @@ +import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/unstable/httpapi" +import { Schema } from "effect" +import { MetricName, ServiceName, SpanId, TraceId } from "../../primitives" +import { AuthorizationV2, V2SchemaErrors } from "./auth" +import { ListOf, ListQuery, Timestamp } from "./envelopes" +import { V2InvalidRequestError, V2NotFoundError, V2ServiceUnavailableError } from "./errors" +import { PublicId, PublicIdPrefixes } from "./public-id" + +const wireExample = (example: object): A => example as A +const commonErrors = [V2InvalidRequestError, V2ServiceUnavailableError] as const +const PositiveInteger = Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)) +const PositiveFinite = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0)) +const NonNegativeFinite = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThanOrEqualTo(0)) +const BreakdownLimit = PositiveInteger.check(Schema.isLessThanOrEqualTo(100)) +const TimeseriesSeriesLimit = PositiveInteger.check(Schema.isLessThanOrEqualTo(100)) + +export const LogPublicId = PublicId(PublicIdPrefixes.log, Schema.String).annotate({ + identifier: "LogId", + title: "Log ID", +}) + +const JsonStringRecord = Schema.Record(Schema.String, Schema.String) +const RequiredTimeRange = { + start_time: Timestamp.annotate({ + description: "Inclusive query-window start.", + }), + end_time: Timestamp.annotate({ description: "Inclusive query-window end." }), +} as const +const PostPagination = { + limit: Schema.optionalKey( + Schema.Number.check(Schema.isInt(), Schema.isBetween({ minimum: 1, maximum: 100 })), + ), + cursor: Schema.optionalKey(Schema.String), +} as const + +export const V2TelemetryWindowQuery = Schema.Struct({ + ...RequiredTimeRange, +}).annotate({ + identifier: "TelemetryWindowQuery", + title: "Telemetry time window", +}) + +export const V2TraceSummary = Schema.Struct({ + id: TraceId, + object: Schema.Literal("trace"), + start_time: Timestamp, + duration_ms: Schema.Number, + root_span_name: Schema.String, + root_span_kind: Schema.String, + root_service_name: Schema.String, + status_code: Schema.String, + has_error: Schema.Boolean, + deployment_environment: Schema.NullOr(Schema.String), + service_namespace: Schema.NullOr(Schema.String), + http_method: Schema.NullOr(Schema.String), + http_route: Schema.NullOr(Schema.String), + http_status_code: Schema.NullOr(Schema.String), +}).annotate({ + identifier: "TraceSummary", + title: "Trace summary", + description: "A root-based summary returned by trace search.", + examples: [ + wireExample({ + id: "7f3a4b5c6d7e8f901234567890abcdef", + object: "trace", + start_time: "2026-07-15T12:00:00.000Z", + duration_ms: 42.5, + root_span_name: "GET /checkout", + root_span_kind: "Server", + root_service_name: "api", + status_code: "Ok", + has_error: false, + deployment_environment: "production", + service_namespace: "checkout", + http_method: "GET", + http_route: "/checkout", + http_status_code: "200", + }), + ], +}) +export type V2TraceSummary = Schema.Schema.Type + +export const V2Span = Schema.Struct({ + id: SpanId, + object: Schema.Literal("span"), + trace_id: TraceId, + parent_span_id: Schema.NullOr(SpanId), + name: Schema.String, + service_name: Schema.String, + kind: Schema.String, + start_time: Timestamp, + duration_ms: Schema.Number, + status_code: Schema.String, + status_message: Schema.NullOr(Schema.String), + attributes: JsonStringRecord, + resource_attributes: JsonStringRecord, +}).annotate({ + identifier: "Span", + title: "Span", + description: "An OpenTelemetry span.", +}) +export type V2Span = Schema.Schema.Type + +export const V2Trace = Schema.Struct({ + id: TraceId, + object: Schema.Literal("trace"), + start_time: Timestamp, + end_time: Timestamp, + duration_ms: Schema.Number, + span_count: Schema.Number, + service_count: Schema.Number, + truncated: Schema.Boolean, + spans: Schema.Array(V2Span), +}).annotate({ + identifier: "Trace", + title: "Trace", + description: "A trace and its spans.", +}) +export type V2Trace = Schema.Schema.Type + +export const V2TraceSearchParams = Schema.Struct({ + ...RequiredTimeRange, + ...PostPagination, + service_name: Schema.optionalKey(ServiceName), + span_name: Schema.optionalKey(Schema.String), + has_error: Schema.optionalKey(Schema.Boolean), + min_duration_ms: Schema.optionalKey(NonNegativeFinite), + max_duration_ms: Schema.optionalKey(NonNegativeFinite), + http_method: Schema.optionalKey(Schema.String), + http_status_code: Schema.optionalKey(Schema.String), + deployment_environment: Schema.optionalKey(Schema.String), + service_namespace: Schema.optionalKey(Schema.String), +}).annotate({ + identifier: "TraceSearchParams", + title: "Trace search parameters", +}) + +const TraceList = ListOf(V2TraceSummary).annotate({ + identifier: "TraceList", + title: "Trace list", +}) + +export class V2TracesApiGroup extends HttpApiGroup.make("traces") + .add( + HttpApiEndpoint.post("search", "/search", { + payload: V2TraceSearchParams, + success: TraceList, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "searchTraces", + summary: "Search traces", + description: "Searches root traces in an explicit time window. Requires `traces:read`.", + }), + ), + ) + .add( + HttpApiEndpoint.get("retrieve", "/:trace_id", { + params: { trace_id: TraceId }, + success: V2Trace, + error: [...commonErrors, V2NotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "getTrace", + summary: "Retrieve a trace", + description: + "Returns a trace and its flat parent-linked span collection. Requires `traces:read`.", + }), + ), + ) + .add( + HttpApiEndpoint.get("retrieveSpan", "/:trace_id/spans/:span_id", { + params: { trace_id: TraceId, span_id: SpanId }, + success: V2Span, + error: [...commonErrors, V2NotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "getSpan", + summary: "Retrieve a span", + description: "Returns one span with complete attribute maps. Requires `traces:read`.", + }), + ), + ) + .prefix("/v2/traces") + .middleware(AuthorizationV2) + .middleware(V2SchemaErrors) + .annotateMerge( + OpenApi.annotations({ + title: "Traces", + description: "Search and inspect distributed traces.", + }), + ) {} + +export const V2Log = Schema.Struct({ + id: LogPublicId, + object: Schema.Literal("log"), + timestamp: Timestamp, + severity_text: Schema.String, + severity_number: Schema.Number, + service_name: Schema.String, + body: Schema.String, + trace_id: Schema.NullOr(TraceId), + span_id: Schema.NullOr(SpanId), + log_attributes: JsonStringRecord, + resource_attributes: JsonStringRecord, +}).annotate({ + identifier: "Log", + title: "Log", + description: "An OpenTelemetry log record.", +}) +export type V2Log = Schema.Schema.Type + +export const V2LogSearchParams = Schema.Struct({ + ...RequiredTimeRange, + ...PostPagination, + service_name: Schema.optionalKey(ServiceName), + severity: Schema.optionalKey(Schema.String), + min_severity: Schema.optionalKey( + Schema.Number.check(Schema.isInt(), Schema.isBetween({ minimum: 0, maximum: 255 })), + ), + trace_id: Schema.optionalKey(TraceId), + span_id: Schema.optionalKey(SpanId), + search: Schema.optionalKey(Schema.String), + deployment_environment: Schema.optionalKey(Schema.String), + service_namespace: Schema.optionalKey(Schema.String), +}).annotate({ identifier: "LogSearchParams", title: "Log search parameters" }) +const LogList = ListOf(V2Log).annotate({ + identifier: "LogList", + title: "Log list", +}) + +export class V2LogsApiGroup extends HttpApiGroup.make("logs") + .add( + HttpApiEndpoint.post("search", "/search", { + payload: V2LogSearchParams, + success: LogList, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "searchLogs", + summary: "Search logs", + description: "Searches logs in an explicit time window. Requires `logs:read`.", + }), + ), + ) + .add( + HttpApiEndpoint.get("retrieve", "/:id", { + params: { id: LogPublicId }, + success: V2Log, + error: [...commonErrors, V2NotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "getLog", + summary: "Retrieve a log", + description: "Returns a log by its opaque `log_…` ID. Requires `logs:read`.", + }), + ), + ) + .prefix("/v2/logs") + .middleware(AuthorizationV2) + .middleware(V2SchemaErrors) + .annotateMerge( + OpenApi.annotations({ + title: "Logs", + description: "Search and retrieve OpenTelemetry logs.", + }), + ) {} + +export const V2Metric = Schema.Struct({ + object: Schema.Literal("metric"), + name: MetricName, + type: Schema.String, + service_name: Schema.String, + description: Schema.String, + unit: Schema.String, + is_monotonic: Schema.Boolean, + data_point_count: Schema.Number, + first_seen: Timestamp, + last_seen: Timestamp, +}).annotate({ + identifier: "Metric", + title: "Metric", + description: "A metric catalog entry.", +}) +export type V2Metric = Schema.Schema.Type + +export const V2MetricListQuery = Schema.Struct({ + ...V2TelemetryWindowQuery.fields, + ...ListQuery.fields, + service_name: Schema.optional(ServiceName), + metric_type: Schema.optional(Schema.String), + search: Schema.optional(Schema.String), +}).annotate({ identifier: "MetricListQuery", title: "Metric list query" }) + +export const V2AttributeFilter = Schema.Struct({ + key: Schema.String, + value: Schema.optionalKey(Schema.String), + mode: Schema.Literals(["equals", "exists", "gt", "gte", "lt", "lte", "contains"]), + negated: Schema.optionalKey(Schema.Boolean), +}).annotate({ identifier: "AttributeFilter", title: "Attribute filter" }) + +const MetricsMetric = Schema.Literals(["avg", "sum", "min", "max", "count", "rate", "increase"]) +const MetricType = Schema.Literals(["sum", "gauge", "histogram", "exponential_histogram"]) +const MetricsFilters = { + metric_name: MetricName, + metric_type: MetricType, + service_name: Schema.optionalKey(ServiceName), + group_by_attribute_key: Schema.optionalKey(Schema.String), + group_by_resource_attribute_key: Schema.optionalKey(Schema.String), + attribute_filters: Schema.optionalKey(Schema.Array(V2AttributeFilter)), + resource_attribute_filters: Schema.optionalKey(Schema.Array(V2AttributeFilter)), +} as const + +export const V2MetricsTimeseriesParams = Schema.Struct({ + ...RequiredTimeRange, + metric: MetricsMetric, + ...MetricsFilters, + group_by: Schema.optionalKey( + Schema.Array(Schema.Literals(["service", "attribute", "resource_attribute", "none"])), + ), + bucket_seconds: Schema.optionalKey(Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0))), + series_limit: Schema.optionalKey(TimeseriesSeriesLimit), +}).annotate({ + identifier: "MetricsTimeseriesParams", + title: "Metrics timeseries parameters", +}) + +export const V2TimeseriesPoint = Schema.Struct({ + bucket: Timestamp, + series: Schema.Record(Schema.String, Schema.Number), +}).annotate({ identifier: "TimeseriesPoint", title: "Timeseries point" }) +export const V2BreakdownItem = Schema.Struct({ + name: Schema.String, + value: Schema.Number, +}).annotate({ identifier: "BreakdownItem", title: "Breakdown item" }) +export const V2StructuredQueryResult = Schema.Struct({ + object: Schema.Literal("query_result"), + kind: Schema.Literals(["timeseries", "breakdown"]), + source: Schema.Literals(["traces", "logs", "metrics"]), + timeseries: Schema.Array(V2TimeseriesPoint), + breakdown: Schema.Array(V2BreakdownItem), +}).annotate({ + identifier: "StructuredQueryResult", + title: "Structured query result", +}) +export type V2StructuredQueryResult = Schema.Schema.Type + +const MetricList = ListOf(V2Metric).annotate({ + identifier: "MetricList", + title: "Metric list", +}) +export class V2MetricsApiGroup extends HttpApiGroup.make("metrics") + .add( + HttpApiEndpoint.get("list", "/", { + query: V2MetricListQuery, + success: MetricList, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "listMetrics", + summary: "List metrics", + description: + "Lists metric catalog entries in an explicit time window. Requires `metrics:read`.", + }), + ), + ) + .add( + HttpApiEndpoint.post("timeseries", "/timeseries", { + payload: V2MetricsTimeseriesParams, + success: V2StructuredQueryResult, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "queryMetricsTimeseries", + summary: "Query metric timeseries", + description: "Executes one typed metric timeseries query. Requires `metrics:read`.", + }), + ), + ) + .prefix("/v2/metrics") + .middleware(AuthorizationV2) + .middleware(V2SchemaErrors) + .annotateMerge( + OpenApi.annotations({ + title: "Metrics", + description: "Browse and query OpenTelemetry metrics.", + }), + ) {} + +export const V2Service = Schema.Struct({ + object: Schema.Literal("service"), + name: ServiceName, + service_namespaces: Schema.Array(Schema.String), + deployment_environments: Schema.Array(Schema.String), + throughput: Schema.Number, + traced_throughput: Schema.Number, + span_count: Schema.Number, + error_count: Schema.Number, + error_rate: Schema.Number, + p50_latency_ms: Schema.Number, + p95_latency_ms: Schema.Number, + p99_latency_ms: Schema.Number, + has_sampling: Schema.Boolean, + sampling_weight: Schema.Number, +}).annotate({ + identifier: "Service", + title: "Service", + description: "An observed service aggregated by service name.", +}) +export type V2Service = Schema.Schema.Type + +export const V2ServiceListQuery = Schema.Struct({ + ...V2TelemetryWindowQuery.fields, + ...ListQuery.fields, + deployment_environment: Schema.optional(Schema.String), + service_namespace: Schema.optional(Schema.String), +}).annotate({ identifier: "ServiceListQuery", title: "Service list query" }) + +export const V2ServiceMapEdge = Schema.Struct({ + object: Schema.Literal("service_map.edge"), + source_service: Schema.String, + target_service: Schema.String, + call_count: Schema.Number, + estimated_call_count: Schema.Number, + error_count: Schema.Number, + error_rate: Schema.Number, + avg_duration_ms: Schema.Number, + max_duration_ms: Schema.Number, + has_sampling: Schema.Boolean, + sampling_weight: Schema.Number, +}).annotate({ identifier: "ServiceMapEdge", title: "Service map edge" }) +export type V2ServiceMapEdge = Schema.Schema.Type +export const V2ServiceMap = Schema.Struct({ + object: Schema.Literal("service_map"), + start_time: Timestamp, + end_time: Timestamp, + edges: Schema.Array(V2ServiceMapEdge), +}).annotate({ identifier: "ServiceMap", title: "Service map" }) +export type V2ServiceMap = Schema.Schema.Type +export const V2ServiceMapQuery = Schema.Struct({ + ...V2TelemetryWindowQuery.fields, + service_name: Schema.optional(ServiceName), + deployment_environment: Schema.optional(Schema.String), +}).annotate({ identifier: "ServiceMapQuery", title: "Service map query" }) +const ServiceList = ListOf(V2Service).annotate({ + identifier: "ServiceList", + title: "Service list", +}) + +export class V2ServicesApiGroup extends HttpApiGroup.make("services") + .add( + HttpApiEndpoint.get("list", "/", { + query: V2ServiceListQuery, + success: ServiceList, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "listServices", + summary: "List services", + description: + "Lists services aggregated by name in an explicit time window. Requires `services:read`.", + }), + ), + ) + .add( + HttpApiEndpoint.get("retrieve", "/:name", { + params: { name: ServiceName }, + query: V2TelemetryWindowQuery, + success: V2Service, + error: [...commonErrors, V2NotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "getService", + summary: "Retrieve a service", + description: + "Returns one service aggregated across environments and namespaces. Requires `services:read`.", + }), + ), + ) + .prefix("/v2/services") + .middleware(AuthorizationV2) + .middleware(V2SchemaErrors) + .annotateMerge( + OpenApi.annotations({ + title: "Services", + description: "Observed service health summaries.", + }), + ) {} + +export class V2ServiceMapApiGroup extends HttpApiGroup.make("serviceMap") + .add( + HttpApiEndpoint.get("retrieve", "/", { + query: V2ServiceMapQuery, + success: V2ServiceMap, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "getServiceMap", + summary: "Retrieve the service map", + description: + "Returns service-to-service dependencies in an explicit time window. Requires `service_map:read`.", + }), + ), + ) + .prefix("/v2/service_map") + .middleware(AuthorizationV2) + .middleware(V2SchemaErrors) + .annotateMerge( + OpenApi.annotations({ + title: "Service Map", + description: "Service-to-service topology.", + }), + ) {} + +const TracesMetric = Schema.Literals([ + "count", + "avg_duration", + "p50_duration", + "p95_duration", + "p99_duration", + "error_rate", + "apdex", +]) +const TraceFilters = Schema.Struct({ + service_name: Schema.optionalKey(ServiceName), + span_name: Schema.optionalKey(Schema.String), + root_spans_only: Schema.optionalKey(Schema.Boolean), + errors_only: Schema.optionalKey(Schema.Boolean), + environments: Schema.optionalKey(Schema.Array(Schema.String)), + namespaces: Schema.optionalKey(Schema.Array(Schema.String)), + min_duration_ms: Schema.optionalKey(NonNegativeFinite), + max_duration_ms: Schema.optionalKey(NonNegativeFinite), + group_by_attribute_keys: Schema.optionalKey(Schema.Array(Schema.String)), + attribute_filters: Schema.optionalKey(Schema.Array(V2AttributeFilter)), + resource_attribute_filters: Schema.optionalKey(Schema.Array(V2AttributeFilter)), +}) +const LogFilters = Schema.Struct({ + service_name: Schema.optionalKey(ServiceName), + severity: Schema.optionalKey(Schema.String), + trace_id: Schema.optionalKey(TraceId), + search: Schema.optionalKey(Schema.String), + environments: Schema.optionalKey(Schema.Array(Schema.String)), + namespaces: Schema.optionalKey(Schema.Array(Schema.String)), +}) +const MetricsFiltersSchema = Schema.Struct(MetricsFilters) + +const V2TracesTimeseriesSpec = Schema.Struct({ + kind: Schema.Literal("timeseries"), + source: Schema.Literal("traces"), + metric: TracesMetric, + group_by: Schema.optionalKey( + Schema.Array( + Schema.Literals(["service", "span_name", "status_code", "http_method", "attribute", "none"]), + ), + ), + filters: Schema.optionalKey(TraceFilters), + bucket_seconds: Schema.optionalKey(PositiveInteger), + series_limit: Schema.optionalKey(TimeseriesSeriesLimit), + apdex_threshold_ms: Schema.optionalKey(PositiveFinite), +}) +const V2LogsTimeseriesSpec = Schema.Struct({ + kind: Schema.Literal("timeseries"), + source: Schema.Literal("logs"), + metric: Schema.Literal("count"), + group_by: Schema.optionalKey(Schema.Array(Schema.Literals(["service", "severity", "none"]))), + filters: Schema.optionalKey(LogFilters), + bucket_seconds: Schema.optionalKey(PositiveInteger), + series_limit: Schema.optionalKey(TimeseriesSeriesLimit), +}) +const V2MetricsTimeseriesSpec = Schema.Struct({ + kind: Schema.Literal("timeseries"), + source: Schema.Literal("metrics"), + metric: MetricsMetric, + group_by: Schema.optionalKey( + Schema.Array(Schema.Literals(["service", "attribute", "resource_attribute", "none"])), + ), + filters: MetricsFiltersSchema, + bucket_seconds: Schema.optionalKey(PositiveInteger), + series_limit: Schema.optionalKey(TimeseriesSeriesLimit), +}) +const V2TracesBreakdownSpec = Schema.Struct({ + kind: Schema.Literal("breakdown"), + source: Schema.Literal("traces"), + metric: TracesMetric, + group_by: Schema.Literals(["service", "span_name", "status_code", "http_method", "attribute"]), + filters: Schema.optionalKey(TraceFilters), + limit: Schema.optionalKey(BreakdownLimit), + apdex_threshold_ms: Schema.optionalKey(PositiveFinite), +}) +const V2LogsBreakdownSpec = Schema.Struct({ + kind: Schema.Literal("breakdown"), + source: Schema.Literal("logs"), + metric: Schema.Literal("count"), + group_by: Schema.Literals(["service", "severity"]), + filters: Schema.optionalKey(LogFilters), + limit: Schema.optionalKey(BreakdownLimit), +}) +const V2MetricsBreakdownSpec = Schema.Struct({ + kind: Schema.Literal("breakdown"), + source: Schema.Literal("metrics"), + metric: Schema.Literals(["avg", "sum", "count"]), + group_by: Schema.Literals(["service", "attribute", "resource_attribute"]), + filters: MetricsFiltersSchema, + limit: Schema.optionalKey(BreakdownLimit), +}) +export const V2QuerySpec = Schema.Union([ + V2TracesTimeseriesSpec, + V2LogsTimeseriesSpec, + V2MetricsTimeseriesSpec, + V2TracesBreakdownSpec, + V2LogsBreakdownSpec, + V2MetricsBreakdownSpec, +]).annotate({ identifier: "QuerySpec", title: "Query specification" }) +export type V2QuerySpec = Schema.Schema.Type +export const V2QueryParams = Schema.Struct({ + ...RequiredTimeRange, + query: V2QuerySpec, +}).annotate({ identifier: "QueryParams", title: "Query parameters" }) +export class V2QueryApiGroup extends HttpApiGroup.make("query") + .add( + HttpApiEndpoint.post("execute", "/", { + payload: V2QueryParams, + success: V2StructuredQueryResult, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "executeQuery", + summary: "Execute a telemetry query", + description: "Executes a typed telemetry query. Requires `query:read`.", + }), + ), + ) + .prefix("/v2/query") + .middleware(AuthorizationV2) + .middleware(V2SchemaErrors) + .annotateMerge( + OpenApi.annotations({ + title: "Query", + description: "Structured telemetry query execution.", + }), + ) {} diff --git a/packages/domain/src/http/v2/v2-contract.test.ts b/packages/domain/src/http/v2/v2-contract.test.ts index 1912b35f7..fb7abff9e 100644 --- a/packages/domain/src/http/v2/v2-contract.test.ts +++ b/packages/domain/src/http/v2/v2-contract.test.ts @@ -15,8 +15,9 @@ import { paginateOffsetQuery, Timestamp, } from "./envelopes" -import { notFound, permissionError, V2NotFoundError } from "./errors" +import { notFound, permissionError, rateLimited, V2NotFoundError, V2RateLimitError } from "./errors" import { encodePublicId } from "./public-id" +import { LogPublicId, V2QueryParams } from "./telemetry" const UUID = "0f8fad5b-d9cb-469f-a165-70867728950e" @@ -320,6 +321,16 @@ describe("v2 error envelope", () => { it("permissionError has type permission_error", () => { expect(permissionError("insufficient_scope", "nope").error.type).toBe("permission_error") }) + + it("rateLimited has the stable public 429 envelope", () => { + expect(Schema.encodeSync(V2RateLimitError)(rateLimited())).toEqual({ + error: { + type: "rate_limit_error", + code: "rate_limited", + message: "Too many requests. Retry after 60 seconds.", + }, + }) + }) }) describe("scopes", () => { @@ -368,6 +379,14 @@ describe("scopes", () => { family: "session_replays", access: "read", }) + for (const [path, family] of [ + ["/v2/traces/search", "traces"], + ["/v2/logs/search", "logs"], + ["/v2/metrics/timeseries", "metrics"], + ["/v2/query", "query"], + ] as const) { + expect(requiredScopeForRequest("POST", path)).toEqual({ family, access: "read" }) + } expect(requiredScopeForRequest("GET", "/api/api-keys")).toBeNull() }) @@ -398,6 +417,57 @@ describe("scopes", () => { }) }) +describe("telemetry contracts", () => { + it("round-trips the synthetic composite log ID", () => { + const internal = JSON.stringify([ + "2026-07-15 12:00:00.123", + "00112233445566778899AABBCCDDEEFF", + ]) + const wire = Schema.encodeSync(LogPublicId)(internal) + expect(wire.startsWith("log_")).toBe(true) + expect(Schema.decodeSync(LogPublicId)(wire)).toBe(internal) + }) + + it("decodes structured queries with trace attribute grouping", () => { + const structured = Schema.decodeUnknownSync(V2QueryParams)({ + start_time: "2026-07-15T00:00:00.000Z", + end_time: "2026-07-15T01:00:00.000Z", + query: { + kind: "timeseries", + source: "traces", + metric: "count", + group_by: ["attribute"], + filters: { group_by_attribute_keys: ["http.route"] }, + }, + }) + expect(structured.query.kind).toBe("timeseries") + }) + + it("rejects raw SQL and invalid query budgets at the HTTP boundary", () => { + const base = { + start_time: "2026-07-15T00:00:00.000Z", + end_time: "2026-07-15T01:00:00.000Z", + } + expect(() => + Schema.decodeUnknownSync(V2QueryParams)({ + ...base, + query: { kind: "raw_sql", sql: "SELECT 1 WHERE $__orgFilter" }, + }), + ).toThrow() + expect(() => + Schema.decodeUnknownSync(V2QueryParams)({ + ...base, + query: { + kind: "timeseries", + source: "logs", + metric: "count", + bucket_seconds: 0, + }, + }), + ).toThrow() + }) +}) + describe("list pagination", () => { const items = Array.from({ length: 45 }, (_, index) => index) diff --git a/packages/domain/src/query-engine.ts b/packages/domain/src/query-engine.ts index 2f5cf06b8..72df0d5f0 100644 --- a/packages/domain/src/query-engine.ts +++ b/packages/domain/src/query-engine.ts @@ -10,12 +10,12 @@ import { TraceId, } from "./primitives" -const dateTimePattern = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/ +const dateTimePattern = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d{1,9})?$/ export const TinybirdDateTime = Schema.String.check(Schema.isPattern(dateTimePattern)).pipe( Schema.annotate({ identifier: "TinybirdDateTime", - description: "Date time string in YYYY-MM-DD HH:mm:ss format", + description: "Date time string in YYYY-MM-DD HH:mm:ss[.fffffffff] format", }), ) @@ -155,6 +155,7 @@ export const MetricsTimeseriesQuery = Schema.Struct({ ), filters: MetricsFilters, bucketSeconds: Schema.optional(Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0))), + seriesLimit: Schema.optional(Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0))), }) export type MetricsTimeseriesQuery = Schema.Schema.Type diff --git a/packages/query-engine/src/ch/ch.test.ts b/packages/query-engine/src/ch/ch.test.ts index 87d664d4e..586a268c3 100644 --- a/packages/query-engine/src/ch/ch.test.ts +++ b/packages/query-engine/src/ch/ch.test.ts @@ -1238,6 +1238,8 @@ describe("converted queries", () => { expect(sql).toContain("TraceId = 'abc123'") expect(sql).toContain("SpanId = 'span1'") expect(sql).toContain("OrgId = 'org_1'") + expect(sql).toContain("ParentSpanId AS parentSpanId") + expect(sql).toContain("Duration / 1000000 AS durationMs") expect(sql).toContain("LIMIT 1") }) diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 5206a4dd6..f0f8339bf 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -28,6 +28,7 @@ export { tracesBreakdownQuery, tracesListQuery, tracesRootListQuery, + traceSummariesQuery, slowTracesQuery, spanSearchQuery, type TracesTimeseriesOpts, @@ -38,6 +39,8 @@ export { type TracesBreakdownOutput, type TracesListOutput, type TracesRootListOutput, + type TraceSummariesOpts, + type TraceSummaryOutput, type SlowTracesOpts, type SlowTracesOutput, type SpanSearchOpts, @@ -141,6 +144,7 @@ export { // Queries — Services export { serviceOverviewQuery, + serviceCatalogQuery, serviceHealthBaselineQuery, serviceReleasesTimelineQuery, serviceEnvironmentsQuery, @@ -150,6 +154,8 @@ export { servicesFacetsQuery, type ServiceOverviewOpts, type ServiceOverviewOutput, + type ServiceCatalogOpts, + type ServiceCatalogOutput, type ServiceHealthBaselineOpts, type ServiceHealthBaselineOutput, type ServiceReleasesTimelineOpts, @@ -169,6 +175,7 @@ export { errorsByTypeQuery, errorsTimeseriesQuery, spanHierarchyQuery, + SPAN_HIERARCHY_MAX_SPANS, spanDetailQuery, traceTimeProbeQuery, tracesDurationStatsQuery, diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index 4502796a8..bf88acdfa 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -181,11 +181,13 @@ const TREE_RESOURCE_ATTR_KEYS = ["deployment.environment", "deployment.commit_sh * keeps the earliest spans (ORDER BY StartTime ASC) so the root and its subtree * stay connected. */ -const SPAN_HIERARCHY_MAX_SPANS = 5_000 +export const SPAN_HIERARCHY_MAX_SPANS = 5_000 export interface SpanHierarchyOpts { traceId: string spanId?: string + /** Override the default cap for callers that fetch one sentinel row. */ + limit?: number /** * When true, the generated SQL adds `Timestamp BETWEEN startTime AND endTime` * filters using parameter placeholders. Callers must then pass `startTime` @@ -258,7 +260,7 @@ export function spanHierarchyQuery(opts: SpanHierarchyOpts) { // ORDER BY + LIMIT bounds pathological traces — the earliest spans keep // the root subtree connected. buildSpanTree (web) re-sorts children anyway. .orderBy(["startTime", "asc"]) - .limit(SPAN_HIERARCHY_MAX_SPANS) + .limit(opts.limit ?? SPAN_HIERARCHY_MAX_SPANS) .format("JSON") ) } @@ -281,6 +283,14 @@ export interface SpanDetailOpts { export interface SpanDetailOutput { readonly traceId: string readonly spanId: string + readonly parentSpanId: string + readonly spanName: string + readonly serviceName: string + readonly spanKind: string + readonly durationMs: number + readonly startTime: string + readonly statusCode: string + readonly statusMessage: string readonly spanAttributes: string readonly resourceAttributes: string } @@ -296,6 +306,18 @@ export function spanDetailQuery(opts: SpanDetailOpts) { .select(($) => ({ traceId: $.TraceId, spanId: $.SpanId, + parentSpanId: $.ParentSpanId, + spanName: httpDisplaySpanName( + $.SpanName, + $.SpanAttributes.get("http.route"), + $.SpanAttributes.get("url.path"), + ), + serviceName: $.ServiceName, + spanKind: $.SpanKind, + durationMs: $.Duration.div(1000000), + startTime: $.Timestamp, + statusCode: $.StatusCode, + statusMessage: $.StatusMessage, spanAttributes: CH.toJSONString($.SpanAttributes), resourceAttributes: CH.toJSONString($.ResourceAttributes), })) @@ -577,7 +599,8 @@ export function tracesFacetsQuery(opts: TracesFacetsOpts): CHUnionQuery makeFacetQuery("SpanName", "spanName", ($) => $.SpanName.neq(""), 20), httpMethod: () => makeFacetQuery("HttpMethod", "httpMethod", ($) => $.HttpMethod.neq(""), 20), httpStatus: () => makeFacetQuery("HttpStatusCode", "httpStatus", ($) => $.HttpStatusCode.neq(""), 20), - deploymentEnv: () => makeFacetQuery("DeploymentEnv", "deploymentEnv", ($) => $.DeploymentEnv.neq(""), 20), + deploymentEnv: () => + makeFacetQuery("DeploymentEnv", "deploymentEnv", ($) => $.DeploymentEnv.neq(""), 20), serviceNamespace: () => makeFacetQuery("ServiceNamespace", "serviceNamespace", ($) => $.ServiceNamespace.neq(""), 20), } diff --git a/packages/query-engine/src/ch/queries/logs.test.ts b/packages/query-engine/src/ch/queries/logs.test.ts index 6a0374303..9a0ce9efc 100644 --- a/packages/query-engine/src/ch/queries/logs.test.ts +++ b/packages/query-engine/src/ch/queries/logs.test.ts @@ -276,6 +276,8 @@ describe("logsListQuery", () => { expect(sql).toContain("Body AS body") expect(sql).toContain("TraceId AS traceId") expect(sql).toContain("SpanId AS spanId") + expect(sql).toContain("hex(MD5(toJSONString(tuple(") + expect(sql).toContain("AS recordIdentity") expect(sql).toContain("toJSONString(LogAttributes) AS logAttributes") expect(sql).toContain("toJSONString(ResourceAttributes) AS resourceAttributes") expect(sql).toContain("ORDER BY timestamp DESC") @@ -289,6 +291,31 @@ describe("logsListQuery", () => { expect(sql).toContain("Timestamp < '2024-01-01T12:00:00'") }) + it("uses the full composite identity for deterministic keyset continuation", () => { + const { sql } = compileCH( + logsListQuery({ + limit: 21, + cursorIdentity: { + timestamp: "2024-01-01 12:00:00.123", + serviceName: "api", + traceId: "trace123", + spanId: "span456", + recordIdentity: "00112233445566778899AABBCCDDEEFF", + }, + }), + baseParams, + ) + expect(sql).toContain( + "ORDER BY timestamp DESC, serviceName ASC, traceId ASC, spanId ASC, recordIdentity ASC", + ) + expect(sql).toContain("Timestamp < '2024-01-01 12:00:00.123'") + expect(sql).toContain("ServiceName > 'api'") + expect(sql).toContain("TraceId > 'trace123'") + expect(sql).toContain("SpanId > 'span456'") + expect(sql).toContain("hex(MD5(toJSONString(tuple(") + expect(sql).toContain("> '00112233445566778899AABBCCDDEEFF'") + }) + it("applies custom limit", () => { const q = logsListQuery({ limit: 100 }) const { sql } = compileCH(q, baseParams) @@ -376,10 +403,16 @@ describe("getLogByKeyQuery", () => { }) it("applies optional traceId and spanId filters", () => { - const q = getLogByKeyQuery({ serviceName: "api", traceId: "trace123", spanId: "span456" }) + const q = getLogByKeyQuery({ + serviceName: "api", + traceId: "trace123", + spanId: "span456", + recordIdentity: "00112233445566778899AABBCCDDEEFF", + }) const { sql } = compileCH(q, keyParams) expect(sql).toContain("TraceId = 'trace123'") expect(sql).toContain("SpanId = 'span456'") + expect(sql).toContain("= '00112233445566778899AABBCCDDEEFF'") }) it("omits traceId and spanId filters when not provided", () => { @@ -480,7 +513,10 @@ describe("logsFacetsQuery", () => { }) it("facet scoping follows the raw-`logs` fallback path", () => { - const q = logsFacetsQuery({ environments: ["prod"], matchModes: { deploymentEnv: "contains" } }, "severity") + const q = logsFacetsQuery( + { environments: ["prod"], matchModes: { deploymentEnv: "contains" } }, + "severity", + ) const { sql } = compileUnion(q, baseParams) expect(sql).not.toContain("UNION ALL") expect(sql).toContain("FROM logs") diff --git a/packages/query-engine/src/ch/queries/logs.ts b/packages/query-engine/src/ch/queries/logs.ts index 797b80f4f..86c52033d 100644 --- a/packages/query-engine/src/ch/queries/logs.ts +++ b/packages/query-engine/src/ch/queries/logs.ts @@ -4,7 +4,7 @@ // DSL-based query definitions for logs timeseries and breakdown. // --------------------------------------------------------------------------- -import { compileCH } from "@maple-dev/clickhouse-builder" +import { compileCH, compileFnCall } from "@maple-dev/clickhouse-builder" import * as CH from "@maple-dev/clickhouse-builder/expr" import { param } from "@maple-dev/clickhouse-builder" import { from, fromUnion, type CHQuery, type ColumnAccessor } from "@maple-dev/clickhouse-builder" @@ -32,6 +32,29 @@ interface LogsQueryOpts { } } +/** Stable identity for log records that do not carry a native OTel record ID. */ +const logRecordIdentity = ($: ColumnAccessor): CH.Expr => { + const record = compileFnCall( + "tuple", + $.Timestamp, + $.TraceId, + $.SpanId, + $.TraceFlags, + $.SeverityText, + $.SeverityNumber, + $.ServiceName, + $.Body, + $.ResourceSchemaUrl, + $.ResourceAttributes, + $.ScopeSchemaUrl, + $.ScopeName, + $.ScopeVersion, + $.ScopeAttributes, + $.LogAttributes, + ) + return compileFnCall("hex", compileFnCall("MD5", CH.toJSONString(record))) +} + function environmentCondition( $: ColumnAccessor, opts: LogsQueryOpts, @@ -408,6 +431,14 @@ export interface LogsListOpts extends LogsQueryOpts { minSeverity?: number cursor?: string limit?: number + offset?: number + cursorIdentity?: { + timestamp: string + serviceName: string + traceId: string + spanId: string + recordIdentity: string + } } export interface LogsListOutput { @@ -418,6 +449,7 @@ export interface LogsListOutput { readonly body: string readonly traceId: string readonly spanId: string + readonly recordIdentity: string readonly logAttributes: string readonly resourceAttributes: string } @@ -438,6 +470,7 @@ export interface LogsListOutput { */ export function logsListQuery(opts: LogsListOpts) { const limit = opts.limit ?? 50 + const offset = opts.offset ?? 0 const baseWhere = ($: ColumnAccessor): Array => [ $.OrgId.eq(param.string("orgId")), @@ -451,6 +484,25 @@ export function logsListQuery(opts: LogsListOpts) { CH.when(opts.traceId, (v: string) => $.TraceId.eq(v)), CH.when(opts.spanId, (v: string) => $.SpanId.eq(v)), CH.when(opts.cursor, (v: string) => $.Timestamp.lt(v)), + opts.cursorIdentity + ? $.Timestamp.lt(opts.cursorIdentity.timestamp).or( + $.Timestamp.eq(opts.cursorIdentity.timestamp).and( + $.ServiceName.gt(opts.cursorIdentity.serviceName).or( + $.ServiceName.eq(opts.cursorIdentity.serviceName).and( + $.TraceId.gt(opts.cursorIdentity.traceId).or( + $.TraceId.eq(opts.cursorIdentity.traceId).and( + $.SpanId.gt(opts.cursorIdentity.spanId).or( + $.SpanId.eq(opts.cursorIdentity.spanId).and( + logRecordIdentity($).gt(opts.cursorIdentity.recordIdentity), + ), + ), + ), + ), + ), + ), + ), + ) + : undefined, CH.when(opts.search, (v: string) => $.Body.ilike(`%${v}%`)), environmentCondition($, opts), namespaceCondition($, opts), @@ -462,12 +514,12 @@ export function logsListQuery(opts: LogsListOpts) { .select(($) => ({ ts: $.Timestamp })) .where(baseWhere) .orderBy(["ts", "desc"]) - .limit(limit) + .limit(limit + offset) const cutoffSql = compileCH(cutoffInner, {}, { skipFormat: true }).sql const cutoff = CH.rawExpr(`(SELECT min(ts) FROM (${cutoffSql}))`) // Stage 2: heavy columns read only for rows at/after the cutoff timestamp. - return from(Logs) + let query = from(Logs) .select(($) => ({ timestamp: $.Timestamp, severityText: $.SeverityText, @@ -476,28 +528,38 @@ export function logsListQuery(opts: LogsListOpts) { body: $.Body, traceId: $.TraceId, spanId: $.SpanId, + recordIdentity: logRecordIdentity($), logAttributes: CH.toJSONString($.LogAttributes), resourceAttributes: CH.toJSONString($.ResourceAttributes), })) .where(($) => [...baseWhere($), $.Timestamp.gte(cutoff)]) - .orderBy(["timestamp", "desc"]) + .orderBy( + ["timestamp", "desc"], + ["serviceName", "asc"], + ["traceId", "asc"], + ["spanId", "asc"], + ["recordIdentity", "asc"], + ) .limit(limit) .format("JSON") + + if (offset > 0) query = query.offset(offset) + return query } // --------------------------------------------------------------------------- // Single log lookup (exact-match by composite key) // -// Logs have no primary id; a row is identified by Timestamp + ServiceName -// (+ TraceId / SpanId when present). `Timestamp` is DateTime64 (sub-second), -// so the pair is effectively unique per service. Used by the shareable -// `/logs/$logId` detail page. +// Logs have no native primary id; the public identity combines the indexed +// timestamp with a deterministic hash of the complete stored record. Used by +// the shareable `/logs/$logId` detail page. // --------------------------------------------------------------------------- export interface LogByKeyOpts { - serviceName: string + serviceName?: string traceId?: string spanId?: string + recordIdentity?: string } export function getLogByKeyQuery(opts: LogByKeyOpts) { @@ -510,6 +572,7 @@ export function getLogByKeyQuery(opts: LogByKeyOpts) { body: $.Body, traceId: $.TraceId, spanId: $.SpanId, + recordIdentity: logRecordIdentity($), logAttributes: CH.toJSONString($.LogAttributes), resourceAttributes: CH.toJSONString($.ResourceAttributes), })) @@ -520,9 +583,10 @@ export function getLogByKeyQuery(opts: LogByKeyOpts) { $.TimestampTime.gte(param.dateTime("startTime")), $.TimestampTime.lte(param.dateTime("endTime")), $.Timestamp.eq(param.dateTime("timestamp")), - $.ServiceName.eq(opts.serviceName), + CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), CH.when(opts.traceId, (v: string) => $.TraceId.eq(v)), CH.when(opts.spanId, (v: string) => $.SpanId.eq(v)), + CH.when(opts.recordIdentity, (v: string) => logRecordIdentity($).eq(v)), ]) .limit(1) .format("JSON") diff --git a/packages/query-engine/src/ch/queries/metrics.test.ts b/packages/query-engine/src/ch/queries/metrics.test.ts index 42d76b700..1d0ee52f5 100644 --- a/packages/query-engine/src/ch/queries/metrics.test.ts +++ b/packages/query-engine/src/ch/queries/metrics.test.ts @@ -67,6 +67,17 @@ describe("metricsTimeseriesQuery", () => { expect(sql).toContain("GROUP BY bucket, serviceName, attributeValue") }) + it("caps grouped value timeseries before returning the long tail", () => { + const q = metricsTimeseriesQuery({ + metricType: "sum", + groupBy: ["service"], + seriesLimit: 7, + }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("WITH __series_base AS") + expect(sql).toContain("LIMIT 7") + }) + it("applies serviceName filter", () => { const q = metricsTimeseriesQuery({ metricType: "sum", serviceName: "api" }) const { sql } = compileCH(q, baseParams) @@ -111,6 +122,13 @@ describe("metricsTimeseriesQuery", () => { // --------------------------------------------------------------------------- describe("metricsTimeseriesRateQuery", () => { + it("caps grouped rate timeseries before returning the long tail", () => { + const q = metricsTimeseriesRateQuery({ groupBy: ["service"], seriesLimit: 7 }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain("WITH __series_base AS") + expect(sql).toContain("LIMIT 7") + }) + it("compiles CTE-based rate query", () => { const q = metricsTimeseriesRateQuery({}) const { sql } = compileCH(q, baseParams) @@ -376,7 +394,7 @@ describe("listMetricsQuery", () => { expect(sql).not.toContain("UNION ALL") expect(sql).toContain("FROM metric_catalog") expect(sql).toContain("GROUP BY metricName, metricType, serviceName") - expect(sql).toContain("ORDER BY lastSeen DESC") + expect(sql).toContain("ORDER BY lastSeen DESC, metricName ASC, metricType ASC, serviceName ASC") expect(sql).toContain("LIMIT 100") // start bound floored to the hour expect(sql).toContain("toStartOfInterval") diff --git a/packages/query-engine/src/ch/queries/metrics.ts b/packages/query-engine/src/ch/queries/metrics.ts index 3647dac7f..ff363da80 100644 --- a/packages/query-engine/src/ch/queries/metrics.ts +++ b/packages/query-engine/src/ch/queries/metrics.ts @@ -15,6 +15,7 @@ import { MetricsSum, MetricCatalog, SpanMetricsCallsHourly } from "../tables" import { compileCH } from "@maple-dev/clickhouse-builder" import { resolveMetricTable, metricsSelectExprs } from "./query-helpers" import { buildAttrFilterCondition } from "../../traces-shared" +import { finalizeTimeseries } from "./series-cap" /** * WHERE conditions for `resourceAttributeFilters` — predicates on the @@ -41,6 +42,8 @@ interface MetricsQueryOpts { attributeKey?: string attributeValue?: string resourceAttributeFilters?: readonly AttributeFilter[] + groupBy?: readonly string[] + seriesLimit?: number } export interface MetricsTimeseriesOpts extends MetricsQueryOpts {} @@ -49,6 +52,7 @@ export interface MetricsTimeseriesOutput { readonly bucket: string readonly serviceName: string readonly attributeValue: string + readonly groupName: string readonly avgValue: number readonly minValue: number readonly maxValue: number @@ -56,6 +60,18 @@ export interface MetricsTimeseriesOutput { readonly dataPointCount: number } +const metricsTimeseriesColumns = { + bucket: T.dateTime, + serviceName: T.string, + attributeValue: T.string, + groupName: T.string, + avgValue: T.float64, + minValue: T.float64, + maxValue: T.float64, + sumValue: T.float64, + dataPointCount: T.uint64, +} + // --------------------------------------------------------------------------- // Timeseries query — handles all 4 metric types // --------------------------------------------------------------------------- @@ -75,6 +91,12 @@ export function metricsTimeseriesQuery(opts: MetricsTimeseriesOpts) { : opts.groupByAttributeKey ? $.Attributes.get(opts.groupByAttributeKey) : CH.lit(""), + groupName: + opts.groupByAttributeKey || opts.groupByResourceAttributeKey + ? opts.groupByResourceAttributeKey + ? $.ResourceAttributes.get(opts.groupByResourceAttributeKey) + : $.Attributes.get(opts.groupByAttributeKey!) + : $.ServiceName, ...metricsSelectExprs($, isHistogram), })) .where(($) => [ @@ -87,13 +109,13 @@ export function metricsTimeseriesQuery(opts: MetricsTimeseriesOpts) { ...resourceFilterConditions(opts.resourceAttributeFilters), ]) - return ( + const inner = ( opts.groupByAttributeKey || opts.groupByResourceAttributeKey ? q.groupBy("bucket", "serviceName", "attributeValue") : q.groupBy("bucket", "serviceName") - ) - .orderBy(["bucket", "asc"]) - .format("JSON") + ).orderBy(["bucket", "asc"]) + + return finalizeTimeseries(inner, metricsTimeseriesColumns, "dataPointCount", opts) } // --------------------------------------------------------------------------- @@ -114,17 +136,30 @@ export interface MetricsRateTimeseriesOpts { attributeKey?: string attributeValue?: string resourceAttributeFilters?: readonly AttributeFilter[] + groupBy?: readonly string[] + seriesLimit?: number } export interface MetricsRateTimeseriesOutput { readonly bucket: string readonly serviceName: string readonly attributeValue: string + readonly groupName: string readonly rateValue: number readonly increaseValue: number readonly dataPointCount: number } +const metricsRateTimeseriesColumns = { + bucket: T.dateTime, + serviceName: T.string, + attributeValue: T.string, + groupName: T.string, + rateValue: T.float64, + increaseValue: T.float64, + dataPointCount: T.uint64, +} + const SPAN_METRICS_CALLS_NAMES = new Set(["span.metrics.calls", "calls"]) function canUseSpanMetricsCallsHourly(opts: MetricsRateTimeseriesOpts): boolean { @@ -249,19 +284,20 @@ function metricsTimeseriesRateFromSpanMetricsCallsHourly( bucket: CH.toStartOfInterval($.Hour, param.int("bucketSeconds")), serviceName: $.ServiceName, attributeValue: opts.groupByAttributeKey === "span.kind" ? $.SpanKind : CH.lit(""), + groupName: opts.groupByAttributeKey === "span.kind" ? $.SpanKind : $.ServiceName, rateValue: CH.sumIf($.delta.div(param.int("bucketSeconds")), $.delta.gte(0)), increaseValue: CH.sumIf($.delta, $.delta.gte(0)), dataPointCount: CH.count(), })) .where(($) => [$.Hour.gte(bucket), $.Hour.lte(endBucket)]) - return ( + const inner = ( opts.groupByAttributeKey === "span.kind" ? q.groupBy("bucket", "serviceName", "attributeValue") : q.groupBy("bucket", "serviceName") - ) - .orderBy(["bucket", "asc"]) - .format("JSON") + ).orderBy(["bucket", "asc"]) + + return finalizeTimeseries(inner, metricsRateTimeseriesColumns, "dataPointCount", opts) } export function metricsTimeseriesRateQuery( @@ -358,19 +394,25 @@ export function metricsTimeseriesRateQuery( : opts.groupByAttributeKey ? $.Attributes.get(opts.groupByAttributeKey) : CH.lit(""), + groupName: + opts.groupByAttributeKey || opts.groupByResourceAttributeKey + ? opts.groupByResourceAttributeKey + ? $.resourceAttributeValue + : $.Attributes.get(opts.groupByAttributeKey!) + : $.ServiceName, rateValue: CH.sumIf($.delta.div($.time_delta), $.delta.gte(0).and($.time_delta.gt(0))), increaseValue: CH.sumIf($.delta, $.delta.gte(0)), dataPointCount: CH.count(), })) .where(($) => [$.TimeUnix.gte(param.dateTime("startTime"))]) - return ( + const inner = ( opts.groupByAttributeKey || opts.groupByResourceAttributeKey ? q.groupBy("bucket", "serviceName", "attributeValue") : q.groupBy("bucket", "serviceName") - ) - .orderBy(["bucket", "asc"]) - .format("JSON") + ).orderBy(["bucket", "asc"]) + + return finalizeTimeseries(inner, metricsRateTimeseriesColumns, "dataPointCount", opts) } // --------------------------------------------------------------------------- @@ -522,7 +564,7 @@ export function listMetricsQuery(opts: ListMetricsOpts) { CH.when(opts.search, (v: string) => $.MetricName.ilike(`%${v}%`)), ]) .groupBy("metricName", "metricType", "serviceName") - .orderBy(["lastSeen", "desc"]) + .orderBy(["lastSeen", "desc"], ["metricName", "asc"], ["metricType", "asc"], ["serviceName", "asc"]) .limit(opts.limit ?? 100) .offset(opts.offset ?? 0) .format("JSON") diff --git a/packages/query-engine/src/ch/queries/service-map.test.ts b/packages/query-engine/src/ch/queries/service-map.test.ts index 95e94d631..8b6aa45bf 100644 --- a/packages/query-engine/src/ch/queries/service-map.test.ts +++ b/packages/query-engine/src/ch/queries/service-map.test.ts @@ -220,6 +220,21 @@ describe("serviceMapResolutionsRollupSQL", () => { // --------------------------------------------------------------------------- describe("serviceDependenciesSQL", () => { + it("reads partial boundary hours from raw spans without widening the start", () => { + const { sql } = serviceDependenciesSQL( + {}, + { + orgId: "org_1", + startTime: "2024-01-01 10:15:00", + endTime: "2024-01-01 12:30:00", + }, + ) + expect(sql).toContain("Timestamp >= toDateTime('2024-01-01 10:15:00')") + expect(sql).toContain("addHours(toStartOfHour(toDateTime('2024-01-01 10:15:00')), 1)") + expect(sql).toContain("greatest(least(") + expect(sql).not.toContain("Timestamp >= toStartOfHour(toDateTime('2024-01-01 10:15:00'))") + }) + it.effect("decodes service dependency rows with numeric strings from ClickHouse JSON", () => Effect.gen(function* () { const compiled = serviceDependenciesSQL({ deploymentEnv: "production" }, baseParams) @@ -252,6 +267,17 @@ describe("serviceDependenciesSQL", () => { }) describe("serviceDependenciesForServiceQuery", () => { + it("keeps partial start and end hours outside the hourly rollup", () => { + const { sql } = compileCH(serviceDependenciesForServiceQuery({ serviceName: "artifacts-api" }), { + orgId: "org_1", + startTime: "2024-01-01 10:15:00", + endTime: "2024-01-01 12:30:00", + }) + expect(sql).toContain("Timestamp >= toDateTime('2024-01-01 10:15:00')") + expect(sql).toContain("addHours(toStartOfHour(toDateTime('2024-01-01 10:15:00')), 1)") + expect(sql).toContain("greatest(least(") + }) + it("filters SourceService on the hourly branch", () => { const { sql } = compileCH( serviceDependenciesForServiceQuery({ serviceName: "artifacts-api" }), @@ -561,7 +587,11 @@ describe("service-map database query summaries", () => { // regression here silently widens the panel to every database of the system. // (Sub-hour timeseries is raw-only, covered separately below.) const namespaceCoalesce = `${DB_NAMESPACE_ATTR_SQL} = 'orders'` - const timeseries = serviceDbQueryTimeseriesSQL({ ...params, dbNamespace: "orders", bucketSeconds: 3600 }).sql + const timeseries = serviceDbQueryTimeseriesSQL({ + ...params, + dbNamespace: "orders", + bucketSeconds: 3600, + }).sql const topQueries = serviceDbTopQueriesSQL({ ...params, dbNamespace: "orders" }).sql for (const sql of [timeseries, topQueries]) { expect(sql).toContain(`${SEALED_NAMESPACE_COLLAPSE} = 'orders'`) // sealed rollup branch diff --git a/packages/query-engine/src/ch/queries/service-map.ts b/packages/query-engine/src/ch/queries/service-map.ts index 90418500b..76a83b6d8 100644 --- a/packages/query-engine/src/ch/queries/service-map.ts +++ b/packages/query-engine/src/ch/queries/service-map.ts @@ -46,6 +46,9 @@ import { CHNumber } from "../schema" // they're niche and only this builder uses them; promote later if reused. const _toFloat64 = defineFn<[CH.Expr], number>("toFloat64") const _matchRegex = defineCondFn<[CH.Expr, string]>("match") +const _leastDateTime = defineFn<[CH.Expr, CH.Expr], string>("least") +const _greatestDateTime = defineFn<[CH.Expr, CH.Expr], string>("greatest") +const _addHours = defineFn<[CH.Expr, CH.Expr], string>("addHours") // --------------------------------------------------------------------------- // Service dependencies @@ -182,10 +185,16 @@ export function serviceDependenciesSQL( // instead of `avg(avgDurationMs)` (averaging averages, which ignores // the relative call counts of each branch). // - // Time ranges are split so the two branches don't double-count the - // in-progress hour: the hourly rollup covers complete hourly buckets - // strictly before `toStartOfHour(endTime)`, the live topology join scans - // only from there to `endTime`. + // Only complete interior hours may come from the hourly rollup. The partial + // start and end hours are read from raw spans so neither edge includes data + // outside the requested half-open window. `startEdgeEnd` also collapses a + // same-hour request into one raw branch; the end branch is then empty. + const startDateTime = `toDateTime('${esc(params.startTime)}')` + const endDateTime = `toDateTime('${esc(params.endTime)}')` + const startHour = `toStartOfHour(${startDateTime})` + const endHour = `toStartOfHour(${endDateTime})` + const startEdgeEnd = `least(${endDateTime}, if(${startDateTime} = ${startHour}, ${startDateTime}, addHours(${startHour}, 1)))` + const endEdgeStart = `greatest(${startEdgeEnd}, ${endHour})` const completedHourEdges = `SELECT SourceService AS sourceService, TargetService AS targetService, @@ -196,8 +205,8 @@ export function serviceDependenciesSQL( sum(if(SampleRateSum > 0, SampleRateSum, toFloat64(CallCount))) AS bucketEstimatedSpanCount FROM service_map_edges_hourly WHERE OrgId = '${esc(params.orgId)}' - AND Hour >= toStartOfHour(toDateTime('${esc(params.startTime)}')) - AND Hour < toStartOfHour(toDateTime('${esc(params.endTime)}')) + AND Hour >= ${startEdgeEnd} + AND Hour < ${endHour} ${envFilter} GROUP BY sourceService, targetService` @@ -205,7 +214,7 @@ export function serviceDependenciesSQL( // yet sealed this hour into `service_map_edges_hourly`. Reuses the exact // SQL the rollup runs (`serviceMapEdgeJoinSQL`) so the two stay in lockstep, // then re-aggregates dropping `Hour` into the `bucket*` shape. - const joinEdges = `SELECT + const joinEdges = (startExpr: string, endExpr: string) => `SELECT SourceService AS sourceService, TargetService AS targetService, sum(CallCount) AS bucketCallCount, @@ -216,8 +225,8 @@ export function serviceDependenciesSQL( FROM ( ${serviceMapEdgeJoinSQL({ orgId: params.orgId, - startExpr: `toStartOfHour(toDateTime('${esc(params.endTime)}'))`, - endExpr: `toDateTime('${esc(params.endTime)}')`, + startExpr, + endExpr, deploymentEnv: opts.deploymentEnv, })} ) @@ -234,7 +243,9 @@ export function serviceDependenciesSQL( FROM ( ${completedHourEdges} UNION ALL - ${joinEdges} + ${joinEdges(startDateTime, startEdgeEnd)} + UNION ALL + ${joinEdges(endEdgeStart, endDateTime)} ) GROUP BY sourceService, targetService ORDER BY callCount DESC @@ -279,7 +290,19 @@ export function serviceDependenciesForServiceQuery(opts: ServiceDependenciesForS const envFilterMv = (deploymentEnv: CH.Expr) => opts.deploymentEnv ? deploymentEnv.eq(opts.deploymentEnv) : undefined - // Hourly branch — sealed buckets from the cross-span rollup. + const startDateTime = CH.toDateTime(param.dateTime("startTime")) + const endDateTime = CH.toDateTime(param.dateTime("endTime")) + const startHour = CH.toStartOfHour(startDateTime) + const endHour = CH.toStartOfHour(endDateTime) + const firstHourBoundary = CH.if_( + startDateTime.eq(startHour), + startDateTime, + _addHours(startHour, CH.lit(1)), + ) + const startEdgeEnd = _leastDateTime(endDateTime, firstHourBoundary) + const endEdgeStart = _greatestDateTime(startEdgeEnd, endHour) + + // Hourly branch — only sealed, complete buckets inside the requested window. const hourlyBranch = from(ServiceMapEdgesHourly) .select(($) => ({ sourceService: $.SourceService, @@ -298,85 +321,90 @@ export function serviceDependenciesForServiceQuery(opts: ServiceDependenciesForS .where(($) => [ $.OrgId.eq(param.string("orgId")), $.SourceService.eq(opts.serviceName), - $.Hour.gte(CH.toStartOfHour(CH.toDateTime(param.dateTime("startTime")))), - $.Hour.lt(CH.toStartOfHour(CH.toDateTime(param.dateTime("endTime")))), + $.Hour.gte(startEdgeEnd), + $.Hour.lt(endHour), envFilterMv($.DeploymentEnv), ]) .groupBy("sourceService", "targetService") - // Live topology JOIN for the in-progress hour only — the rollup hasn't - // sealed this hour yet. Mirrors the all-orgs `serviceMapEdgeJoinSQL` - // helper used by the rollup, but pushes `ServiceName = ?` into the parent - // subquery so the JOIN's left side shrinks to one service's outbound spans. - const parentSpans = from(ServiceMapSpans) - .select(($) => ({ - TraceId: $.TraceId, - SpanId: $.SpanId, - ServiceName: $.ServiceName, - Timestamp: $.Timestamp, - })) - .where(($) => [ - $.SpanKind.in_("Client", "Producer"), - $.Timestamp.gte(CH.toStartOfHour(CH.toDateTime(param.dateTime("endTime")))), - $.Timestamp.lt(param.dateTime("endTime")), - $.OrgId.eq(param.string("orgId")), - $.ServiceName.eq(opts.serviceName), - envFilterMv($.DeploymentEnv), - ]) - - const childSpans = from(ServiceMapChildren) - .select(($) => ({ - TraceId: $.TraceId, - ParentSpanId: $.ParentSpanId, - ServiceName: $.ServiceName, - Duration: $.Duration, - StatusCode: $.StatusCode, - TraceState: $.TraceState, - })) - .where(($) => [ - $.Timestamp.gte(CH.toStartOfHour(CH.toDateTime(param.dateTime("endTime")))), - $.Timestamp.lt(param.dateTime("endTime")), - $.OrgId.eq(param.string("orgId")), - envFilterMv($.DeploymentEnv), - ]) - - // Sample-weight expression: `1 / (1 - acceptanceProbability)` for spans - // carrying a `th:` TraceState threshold, else `1.0` (unsampled). The bit - // math is intentionally raw — these CH functions (reinterpret/unhex/ - // rightPad/pow/reverse/greatest) aren't worth promoting to first-class - // DSL helpers for a single call site. - const sampleWeightExpr = CH.multiIf( - [ + // Raw topology JOIN for one partial-hour edge. Called for both the start and + // end edges; empty intervals naturally return no rows. + const liveJoinBranch = (rangeStart: CH.Expr, rangeEnd: CH.Expr) => { + const parentSpans = from(ServiceMapSpans) + .select(($) => ({ + TraceId: $.TraceId, + SpanId: $.SpanId, + ServiceName: $.ServiceName, + Timestamp: $.Timestamp, + })) + .where(($) => [ + $.SpanKind.in_("Client", "Producer"), + $.Timestamp.gte(rangeStart), + $.Timestamp.lt(rangeEnd), + $.OrgId.eq(param.string("orgId")), + $.ServiceName.eq(opts.serviceName), + envFilterMv($.DeploymentEnv), + ]) + + const childSpans = from(ServiceMapChildren) + .select(($) => ({ + TraceId: $.TraceId, + ParentSpanId: $.ParentSpanId, + ServiceName: $.ServiceName, + Duration: $.Duration, + StatusCode: $.StatusCode, + TraceState: $.TraceState, + })) + .where(($) => [ + $.Timestamp.gte(rangeStart), + $.Timestamp.lt(rangeEnd), + $.OrgId.eq(param.string("orgId")), + envFilterMv($.DeploymentEnv), + ]) + + // Sample-weight expression: `1 / (1 - acceptanceProbability)` for spans + // carrying a `th:` TraceState threshold, else `1.0` (unsampled). The bit + // math is intentionally raw — these CH functions (reinterpret/unhex/ + // rightPad/pow/reverse/greatest) aren't worth promoting to first-class + // DSL helpers for a single call site. + const sampleWeightExpr = CH.multiIf( [ - _matchRegex(CH.rawExpr("c.TraceState"), "th:[0-9a-f]+"), - CH.rawExpr( - "1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(c.TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001)", - ), + [ + _matchRegex(CH.rawExpr("c.TraceState"), "th:[0-9a-f]+"), + CH.rawExpr( + "1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(c.TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001)", + ), + ], ], - ], - CH.rawExpr("1.0"), - ) + CH.rawExpr("1.0"), + ) + + // In a join, the main subquery's columns are auto-qualified with its alias + // (`p.ServiceName`), and joined columns are reached via `$..Column`. + return fromQuery(parentSpans, "p") + .innerJoinQuery(childSpans, "c", (p, c) => + p.SpanId.eq(c.ParentSpanId).and(p.TraceId.eq(c.TraceId)), + ) + .select(($) => ({ + sourceService: $.ServiceName, + targetService: $.c.ServiceName, + bucketCallCount: CH.count(), + bucketErrorCount: CH.countIf($.c.StatusCode.eq("Error")), + bucketDurationSumMs: CH.sum($.c.Duration.div(1000000)), + bucketMaxDurationMs: CH.max_($.c.Duration.div(1000000)), + bucketEstimatedSpanCount: CH.sum(sampleWeightExpr), + })) + .where(($) => [$.ServiceName.neq($.c.ServiceName)]) + .groupBy("sourceService", "targetService") + } - // In a join, the main subquery's columns are auto-qualified with its alias - // (`p.ServiceName`), and joined columns are reached via `$..Column`. - const liveJoinBranch = fromQuery(parentSpans, "p") - .innerJoinQuery(childSpans, "c", (p, c) => p.SpanId.eq(c.ParentSpanId).and(p.TraceId.eq(c.TraceId))) - .select(($) => ({ - sourceService: $.ServiceName, - targetService: $.c.ServiceName, - bucketCallCount: CH.count(), - bucketErrorCount: CH.countIf($.c.StatusCode.eq("Error")), - bucketDurationSumMs: CH.sum($.c.Duration.div(1000000)), - bucketMaxDurationMs: CH.max_($.c.Duration.div(1000000)), - bucketEstimatedSpanCount: CH.sum(sampleWeightExpr), - })) - .where(($) => [$.ServiceName.neq($.c.ServiceName)]) - .groupBy("sourceService", "targetService") + const startLiveBranch = liveJoinBranch(startDateTime, startEdgeEnd) + const endLiveBranch = liveJoinBranch(endEdgeStart, endDateTime) // Outer wrap: re-aggregate across both branches so avg duration uses // branch-summed numerators/denominators (avoids averaging averages). With // no further joins, columns from the union are accessed bare. - return fromUnion(unionAll(hourlyBranch, liveJoinBranch), "edges") + return fromUnion(unionAll(hourlyBranch, startLiveBranch, endLiveBranch), "edges") .select(($) => ({ sourceService: $.sourceService, targetService: $.targetService, @@ -445,10 +473,7 @@ export function serviceDbEdgesSQL( // talks to, mirroring the write-side `DB_SYSTEM_ATTR_SQL` / // `DB_NAMESPACE_ATTR_SQL` fragments so raw-branch keys merge with the MV's. const dbSystemExpr = ($: { SpanAttributes: CH.ColumnRef<"SpanAttributes", any> }) => - CH.coalesce( - CH.nullIf($.SpanAttributes.get("db.system.name"), ""), - $.SpanAttributes.get("db.system"), - ) + CH.coalesce(CH.nullIf($.SpanAttributes.get("db.system.name"), ""), $.SpanAttributes.get("db.system")) const dbNamespaceCoalesce = ($: { SpanAttributes: CH.ColumnRef<"SpanAttributes", any> }) => CH.coalesce( CH.nullIf($.SpanAttributes.get("db.namespace"), ""), @@ -732,9 +757,7 @@ const shapesHourlyFilters = ( // `undefined` = unscoped; `''` is a real value (the legacy/unknown node). // Collapse sealed hex → sentinel so a `dbNamespace: "hyperdrive"` filter also // matches pre-migration rows that still carry the raw Hyperdrive config ID. - params.dbNamespace !== undefined - ? collapseHyperdriveNs($.DbNamespace).eq(params.dbNamespace) - : undefined, + params.dbNamespace !== undefined ? collapseHyperdriveNs($.DbNamespace).eq(params.dbNamespace) : undefined, params.sourceService ? $.ServiceName.eq(params.sourceService) : undefined, params.deploymentEnv ? $.DeploymentEnv.eq(params.deploymentEnv) : undefined, ] diff --git a/packages/query-engine/src/ch/queries/services.test.ts b/packages/query-engine/src/ch/queries/services.test.ts index 2832e4e8c..ad24ed6a8 100644 --- a/packages/query-engine/src/ch/queries/services.test.ts +++ b/packages/query-engine/src/ch/queries/services.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest" import { compileCH, compileUnion } from "@maple-dev/clickhouse-builder" import { serviceOverviewQuery, + serviceCatalogQuery, serviceHealthBaselineQuery, serviceReleasesTimelineQuery, serviceEnvironmentsQuery, @@ -18,6 +19,28 @@ const baseParams = { bucketSeconds: 3600, } +describe("serviceCatalogQuery", () => { + it("aggregates by service name with pruning filters and limit-plus-one pagination", () => { + const { sql } = compileCH( + serviceCatalogQuery({ + deploymentEnvironment: "production", + serviceNamespace: "checkout", + limit: 21, + offset: 20, + }), + baseParams, + ) + expect(sql).toContain("FROM service_overview_spans") + expect(sql).toContain("OrgId = 'org_1'") + expect(sql).toContain("Timestamp >= '2024-01-01 00:00:00'") + expect(sql).toContain("DeploymentEnv = 'production'") + expect(sql).toContain("ServiceNamespace = 'checkout'") + expect(sql).toContain("GROUP BY serviceName") + expect(sql).toContain("LIMIT 21") + expect(sql).toContain("OFFSET 20") + }) +}) + // --------------------------------------------------------------------------- // serviceOverviewQuery // --------------------------------------------------------------------------- diff --git a/packages/query-engine/src/ch/queries/services.ts b/packages/query-engine/src/ch/queries/services.ts index 279508b1d..5e0fd760a 100644 --- a/packages/query-engine/src/ch/queries/services.ts +++ b/packages/query-engine/src/ch/queries/services.ts @@ -19,6 +19,8 @@ export interface ServiceOverviewOpts { environments?: readonly string[] namespaces?: readonly string[] commitShas?: readonly string[] + serviceName?: string + limit?: number } export interface ServiceOverviewOutput { @@ -36,6 +38,61 @@ export interface ServiceOverviewOutput { readonly estimatedSpanCount: number } +export interface ServiceCatalogOpts { + serviceName?: string + deploymentEnvironment?: string + serviceNamespace?: string + limit?: number + offset?: number +} + +export interface ServiceCatalogOutput { + readonly serviceName: string + readonly serviceNamespaces: readonly string[] + readonly deploymentEnvironments: readonly string[] + readonly spanCount: number + readonly errorCount: number + readonly estimatedErrorCount: number + readonly estimatedSpanCount: number + readonly p50LatencyMs: number + readonly p95LatencyMs: number + readonly p99LatencyMs: number +} + +/** Name-level public service catalog, intentionally aggregated across env/namespace. */ +export function serviceCatalogQuery(opts: ServiceCatalogOpts) { + return from(ServiceOverviewSpans) + .select(($) => ({ + serviceName: $.ServiceName, + serviceNamespaces: CH.rawExpr( + "arraySort(arrayFilter(x -> x != '', arrayDistinct(groupArray(ServiceNamespace))))", + ), + deploymentEnvironments: CH.rawExpr( + "arraySort(arrayFilter(x -> x != '', arrayDistinct(groupArray(DeploymentEnv))))", + ), + spanCount: CH.count(), + errorCount: CH.countIf($.StatusCode.eq("Error")), + estimatedErrorCount: CH.sumIf($.SampleRate, $.StatusCode.eq("Error")), + estimatedSpanCount: CH.sum($.SampleRate), + p50LatencyMs: CH.quantile(0.5)($.Duration).div(1000000), + p95LatencyMs: CH.quantile(0.95)($.Duration).div(1000000), + p99LatencyMs: CH.quantile(0.99)($.Duration).div(1000000), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + CH.when(opts.serviceName, (value: string) => $.ServiceName.eq(value)), + CH.when(opts.deploymentEnvironment, (value: string) => $.DeploymentEnv.eq(value)), + CH.when(opts.serviceNamespace, (value: string) => $.ServiceNamespace.eq(value)), + ]) + .groupBy("serviceName") + .orderBy(["estimatedSpanCount", "desc"], ["serviceName", "asc"]) + .limit(opts.limit ?? 20) + .offset(opts.offset ?? 0) + .format("JSON") +} + export function serviceOverviewQuery(opts: ServiceOverviewOpts) { return from(ServiceOverviewSpans) .select(($) => ({ @@ -59,13 +116,14 @@ export function serviceOverviewQuery(opts: ServiceOverviewOpts) { $.OrgId.eq(param.string("orgId")), $.Timestamp.gte(param.dateTime("startTime")), $.Timestamp.lte(param.dateTime("endTime")), + CH.when(opts.serviceName, (value: string) => $.ServiceName.eq(value)), opts.environments?.length ? CH.inList($.DeploymentEnv, opts.environments) : undefined, opts.namespaces?.length ? CH.inList($.ServiceNamespace, opts.namespaces) : undefined, opts.commitShas?.length ? CH.inList($.CommitSha, opts.commitShas) : undefined, ]) .groupBy("serviceName", "serviceNamespace", "environment", "commitSha") .orderBy(["throughput", "desc"]) - .limit(100) + .limit(opts.limit ?? 100) .format("JSON") } diff --git a/packages/query-engine/src/ch/queries/traces.test.ts b/packages/query-engine/src/ch/queries/traces.test.ts index 3f8e0516d..bcd671bf3 100644 --- a/packages/query-engine/src/ch/queries/traces.test.ts +++ b/packages/query-engine/src/ch/queries/traces.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from "vitest" import { compileCH } from "@maple-dev/clickhouse-builder" -import { slowTracesQuery, spanSearchQuery, tracesListQuery, tracesRootListQuery } from "./traces" +import { + slowTracesQuery, + spanSearchQuery, + traceSummariesQuery, + tracesListQuery, + tracesRootListQuery, +} from "./traces" const baseParams = { orgId: "org_1", @@ -9,6 +15,31 @@ const baseParams = { bucketSeconds: 3600, } +describe("traceSummariesQuery", () => { + it("uses the trace-list ordering key with org/time filters and deterministic pagination", () => { + const { sql } = compileCH( + traceSummariesQuery({ + serviceName: "api", + hasError: true, + limit: 21, + cursor: { timestamp: "2024-01-01 12:00:00", traceId: "trace123" }, + }), + baseParams, + ) + expect(sql).toContain("FROM trace_list_mv") + expect(sql).toContain("OrgId = 'org_1'") + expect(sql).toContain("Timestamp >= '2024-01-01 00:00:00'") + expect(sql).toContain("Timestamp <= '2024-01-02 00:00:00'") + expect(sql).toContain("ServiceName = 'api'") + expect(sql).toContain("HasError = 1") + expect(sql).toContain("GROUP BY traceId") + expect(sql).toContain("ORDER BY startTime DESC, traceId DESC") + expect(sql).toContain("LIMIT 21") + expect(sql).toContain("Timestamp < '2024-01-01 12:00:00'") + expect(sql).toContain("TraceId < 'trace123'") + }) +}) + // --------------------------------------------------------------------------- // tracesListQuery // --------------------------------------------------------------------------- diff --git a/packages/query-engine/src/ch/queries/traces.ts b/packages/query-engine/src/ch/queries/traces.ts index d1954d272..5c643af13 100644 --- a/packages/query-engine/src/ch/queries/traces.ts +++ b/packages/query-engine/src/ch/queries/traces.ts @@ -5,7 +5,7 @@ // --------------------------------------------------------------------------- import type { TracesMetric } from "../../query-engine" -import { compileCH } from "@maple-dev/clickhouse-builder" +import { compileCH, compileFnCall } from "@maple-dev/clickhouse-builder" import * as CH from "@maple-dev/clickhouse-builder/expr" import { param } from "@maple-dev/clickhouse-builder" import { from, type CHQuery, type ColumnAccessor } from "@maple-dev/clickhouse-builder" @@ -61,7 +61,11 @@ function metricSelectExprs( const apdex = needs.has("apdex") ? apdexExprs(durationMs, apdexThresholdMs, $.StatusCode.eq("Error")) - : { satisfiedCount: CH.lit(0), toleratingCount: CH.lit(0), apdexScore: CH.lit(0) } + : { + satisfiedCount: CH.lit(0), + toleratingCount: CH.lit(0), + apdexScore: CH.lit(0), + } return { count: CH.count(), @@ -763,6 +767,85 @@ export interface TracesRootListOutput { readonly hasError: number } +export interface TraceSummariesOpts { + serviceName?: string + spanName?: string + hasError?: boolean + minDurationMs?: number + maxDurationMs?: number + httpMethod?: string + httpStatusCode?: string + deploymentEnv?: string + namespace?: string + limit?: number + offset?: number + cursor?: { timestamp: string; traceId: string } +} + +export interface TraceSummaryOutput { + readonly traceId: string + readonly startTime: string + readonly durationMs: number + readonly rootSpanName: string + readonly rootSpanKind: string + readonly rootServiceName: string + readonly statusCode: string + readonly hasError: number + readonly deploymentEnvironment: string + readonly serviceNamespace: string + readonly httpMethod: string + readonly httpRoute: string + readonly httpStatusCode: string +} + +const argMin = (value: CH.Expr, ordering: CH.Expr): CH.Expr => + compileFnCall("argMin", value, ordering) + +/** Public trace catalog read over the root-span MV, ordered deterministically. */ +export function traceSummariesQuery(opts: TraceSummariesOpts) { + return from(TraceListMv) + .select(($) => ({ + traceId: $.TraceId, + startTime: CH.min_($.Timestamp), + durationMs: argMin($.Duration, $.Timestamp).div(1000000), + rootSpanName: argMin($.SpanName, $.Timestamp), + rootSpanKind: argMin($.SpanKind, $.Timestamp), + rootServiceName: argMin($.ServiceName, $.Timestamp), + statusCode: argMin($.StatusCode, $.Timestamp), + hasError: CH.max_($.HasError), + deploymentEnvironment: argMin($.DeploymentEnv, $.Timestamp), + serviceNamespace: argMin($.ServiceNamespace, $.Timestamp), + httpMethod: argMin($.HttpMethod, $.Timestamp), + httpRoute: argMin($.HttpRoute, $.Timestamp), + httpStatusCode: argMin($.HttpStatusCode, $.Timestamp), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + CH.when(opts.serviceName, (value: string) => $.ServiceName.eq(value)), + CH.when(opts.spanName, (value: string) => $.SpanName.eq(value)), + CH.when(opts.hasError, () => $.HasError.eq(1)), + CH.when(opts.hasError === false, () => $.HasError.eq(0)), + opts.minDurationMs !== undefined ? $.Duration.gte(opts.minDurationMs * 1000000) : undefined, + opts.maxDurationMs !== undefined ? $.Duration.lte(opts.maxDurationMs * 1000000) : undefined, + CH.when(opts.httpMethod, (value: string) => $.HttpMethod.eq(value)), + CH.when(opts.httpStatusCode, (value: string) => $.HttpStatusCode.eq(value)), + CH.when(opts.deploymentEnv, (value: string) => $.DeploymentEnv.eq(value)), + CH.when(opts.namespace, (value: string) => $.ServiceNamespace.eq(value)), + opts.cursor + ? $.Timestamp.lt(opts.cursor.timestamp).or( + $.Timestamp.eq(opts.cursor.timestamp).and($.TraceId.lt(opts.cursor.traceId)), + ) + : undefined, + ]) + .groupBy("traceId") + .orderBy(["startTime", "desc"], ["traceId", "desc"]) + .limit(opts.limit ?? 20) + .offset(opts.offset ?? 0) + .format("JSON") +} + /** * HTTP attribute keys projected into `rootSpanAttributes`. Mirrors the web app's * trace-list projection and the hierarchy query's `TREE_SPAN_ATTR_KEYS` so the diff --git a/packages/query-engine/src/execution/executor.test.ts b/packages/query-engine/src/execution/executor.test.ts index 3befd297d..03d698e71 100644 --- a/packages/query-engine/src/execution/executor.test.ts +++ b/packages/query-engine/src/execution/executor.test.ts @@ -2,8 +2,10 @@ import { assert, describe, it } from "@effect/vitest" import { Duration, Effect, Ref } from "effect" import { TestClock } from "effect/testing" import type { OrgId } from "@maple/domain" +import { RawSqlValidationError } from "@maple/domain/http" import { compile, listRuleChecksQuery } from "../ch" import { makeWarehouseExecutor } from "./executor" +import { WarehouseResponseLimitError } from "./response-limits" import type { ExecutionTenant, ResolvedWarehouseConfig, @@ -21,6 +23,7 @@ const tenant: ExecutionTenant = { // pipeline (the write path). alert_checks rows only ever land in the latter. const clickhouseConfig: ResolvedWarehouseConfig = { _tag: "clickhouse", + provider: "clickhouse", url: "https://byo.example.com", username: "default", password: "secret", @@ -28,9 +31,14 @@ const clickhouseConfig: ResolvedWarehouseConfig = { } const tinybirdConfig: ResolvedWarehouseConfig = { _tag: "tinybird", + provider: "tinybird", host: "https://api.tinybird.co", token: "tb_token", } +const tinybirdGatewayConfig: ResolvedWarehouseConfig = { + ...clickhouseConfig, + provider: "tinybird", +} const compiled = compile(listRuleChecksQuery({ limit: 1 }), { orgId: "org_test", @@ -48,8 +56,9 @@ const makeDeps = (createdTags: Array): Warehous } return client }, - resolveConfig: () => Effect.succeed({ config: clickhouseConfig, source: "org_override" as const }), - resolveIngestConfig: () => Effect.succeed({ config: tinybirdConfig, source: "managed" as const }), + resolveConfig: () => Effect.succeed({ config: clickhouseConfig, clientCacheKey: "read:org_test" }), + resolveRawSqlConfig: () => Effect.succeed({ config: clickhouseConfig, clientCacheKey: "raw:org_test" }), + resolveIngestConfig: () => Effect.succeed({ config: tinybirdConfig, clientCacheKey: "write:managed" }), }) describe("makeWarehouseExecutor pinToIngestConfig", () => { @@ -77,12 +86,11 @@ describe("makeWarehouseExecutor pinToIngestConfig", () => { // Capture the final SQL the executor hands to the client so a test can assert // whether a Tinybird-restricted setting (max_block_size) survived the strip for -// the resolved backend. The strip keys on the config `source`, not its `_tag`: -// the managed warehouse is Tinybird (SDK or its ClickHouse-compatible gateway, -// which surfaces as _tag "clickhouse" when CLICKHOUSE_URL is set) and enforces -// the restriction, so only a genuine per-org BYO ClickHouse keeps the setting. +// the resolved backend. Provider is independent from protocol and routing +// source: a Tinybird gateway uses the ClickHouse protocol but still enforces +// Tinybird's restricted-settings policy. const makeRecordingDeps = ( - resolved: { config: ResolvedWarehouseConfig; source: "managed" | "org_override" }, + resolved: { config: ResolvedWarehouseConfig; clientCacheKey: string }, sqls: Array, ): WarehouseExecutorDeps => ({ createClient: () => ({ @@ -93,15 +101,16 @@ const makeRecordingDeps = ( insert: async () => {}, }), resolveConfig: () => Effect.succeed(resolved), + resolveRawSqlConfig: () => Effect.succeed({ ...resolved, clientCacheKey: "raw:org_test" }), resolveIngestConfig: () => Effect.succeed(resolved), }) describe("makeWarehouseExecutor restricted-settings strip", () => { - it.effect("strips max_block_size for the managed Tinybird CH-gateway (_tag clickhouse, source managed)", () => + it.effect("strips max_block_size for the managed Tinybird CH-gateway", () => Effect.gen(function* () { const sqls: Array = [] const executor = makeWarehouseExecutor( - makeRecordingDeps({ config: clickhouseConfig, source: "managed" }, sqls), + makeRecordingDeps({ config: tinybirdGatewayConfig, clientCacheKey: "read:managed" }, sqls), ) yield* executor.compiledQuery(tenant, compiled, { context: "test", @@ -112,11 +121,25 @@ describe("makeWarehouseExecutor restricted-settings strip", () => { }), ) - it.effect("strips max_block_size for the managed Tinybird SDK backend (_tag tinybird, source managed)", () => + it.effect("keeps max_block_size for env-level vanilla ClickHouse", () => + Effect.gen(function* () { + const sqls: Array = [] + const executor = makeWarehouseExecutor( + makeRecordingDeps({ config: clickhouseConfig, clientCacheKey: "read:managed" }, sqls), + ) + yield* executor.compiledQuery(tenant, compiled, { + context: "test", + settings: { maxBlockSize: 512 }, + }) + assert.isTrue(sqls[0]?.includes("max_block_size=512")) + }), + ) + + it.effect("strips max_block_size for the managed Tinybird SDK backend", () => Effect.gen(function* () { const sqls: Array = [] const executor = makeWarehouseExecutor( - makeRecordingDeps({ config: tinybirdConfig, source: "managed" }, sqls), + makeRecordingDeps({ config: tinybirdConfig, clientCacheKey: "read:managed" }, sqls), ) yield* executor.compiledQuery(tenant, compiled, { context: "test", @@ -126,11 +149,11 @@ describe("makeWarehouseExecutor restricted-settings strip", () => { }), ) - it.effect("keeps max_block_size for a genuine BYO ClickHouse (_tag clickhouse, source org_override)", () => + it.effect("keeps max_block_size for a genuine BYO ClickHouse", () => Effect.gen(function* () { const sqls: Array = [] const executor = makeWarehouseExecutor( - makeRecordingDeps({ config: clickhouseConfig, source: "org_override" }, sqls), + makeRecordingDeps({ config: clickhouseConfig, clientCacheKey: "read:org_test" }, sqls), ) yield* executor.compiledQuery(tenant, compiled, { context: "test", @@ -141,6 +164,72 @@ describe("makeWarehouseExecutor restricted-settings strip", () => { ) }) +describe("makeWarehouseExecutor raw response limits", () => { + it.effect("maps a driver byte abort directly to RawSqlValidationError", () => + Effect.gen(function* () { + const executor = makeWarehouseExecutor({ + ...makeDeps([]), + createClient: () => ({ + sql: async () => { + throw new WarehouseResponseLimitError({ + kind: "bytes", + message: "raw response too large", + }) + }, + insert: async () => {}, + }), + }) + const error = yield* Effect.flip( + executor.rawSqlQuery(tenant, "SELECT 1 WHERE OrgId = 'org_test'"), + ) + assert.instanceOf(error, RawSqlValidationError) + assert.strictEqual(error.code, "ResourceLimit") + }), + ) +}) + +describe("makeWarehouseExecutor client cache partitions", () => { + it.effect("keeps standard reads, raw org reads, and managed writes in stable separate entries", () => + Effect.gen(function* () { + let nextClientId = 0 + const calls: Array<{ readonly clientId: number; readonly operation: "sql" | "insert" }> = [] + const executor = makeWarehouseExecutor({ + createClient: () => { + const clientId = ++nextClientId + return { + sql: async () => { + calls.push({ clientId, operation: "sql" }) + return { data: [] } + }, + insert: async () => { + calls.push({ clientId, operation: "insert" }) + }, + } + }, + resolveConfig: () => + Effect.succeed({ config: tinybirdConfig, clientCacheKey: "read:managed" }), + resolveRawSqlConfig: () => + Effect.succeed({ config: tinybirdConfig, clientCacheKey: "raw:org_test" }), + resolveIngestConfig: () => + Effect.succeed({ config: tinybirdConfig, clientCacheKey: "write:managed" }), + }) + + yield* executor.sqlQuery(tenant, "SELECT 1 WHERE OrgId = 'org_test'") + yield* executor.rawSqlQuery(tenant, "SELECT 1 WHERE OrgId = 'org_test'") + yield* executor.ingest(tenant, "traces", [{ TraceId: "trace" }]) + yield* executor.sqlQuery(tenant, "SELECT 2 WHERE OrgId = 'org_test'") + + assert.strictEqual(nextClientId, 3) + assert.deepStrictEqual(calls, [ + { clientId: 1, operation: "sql" }, + { clientId: 2, operation: "sql" }, + { clientId: 3, operation: "insert" }, + { clientId: 1, operation: "sql" }, + ]) + }), + ) +}) + // A client whose query never resolves — models a Tinybird request stuck in the // execution queue (the failure mode behind the 03:00–05:00 timeout storm, where // queries rode the ambient ~30s Worker fetch limit despite a server-side budget). @@ -149,8 +238,9 @@ const makeHangingDeps = (): WarehouseExecutorDeps => ({ sql: () => new Promise<{ data: never[] }>(() => {}), insert: async () => {}, }), - resolveConfig: () => Effect.succeed({ config: tinybirdConfig, source: "managed" as const }), - resolveIngestConfig: () => Effect.succeed({ config: tinybirdConfig, source: "managed" as const }), + resolveConfig: () => Effect.succeed({ config: tinybirdConfig, clientCacheKey: "read:managed" }), + resolveRawSqlConfig: () => Effect.succeed({ config: tinybirdConfig, clientCacheKey: "raw:org_test" }), + resolveIngestConfig: () => Effect.succeed({ config: tinybirdConfig, clientCacheKey: "write:managed" }), }) // Like makeHangingDeps, but counts how many times the client's `sql` is invoked @@ -164,8 +254,9 @@ const makeCountingHangingDeps = (counter: { count: number }): WarehouseExecutorD }, insert: async () => {}, }), - resolveConfig: () => Effect.succeed({ config: tinybirdConfig, source: "managed" as const }), - resolveIngestConfig: () => Effect.succeed({ config: tinybirdConfig, source: "managed" as const }), + resolveConfig: () => Effect.succeed({ config: tinybirdConfig, clientCacheKey: "read:managed" }), + resolveRawSqlConfig: () => Effect.succeed({ config: tinybirdConfig, clientCacheKey: "raw:org_test" }), + resolveIngestConfig: () => Effect.succeed({ config: tinybirdConfig, clientCacheKey: "write:managed" }), }) describe("makeWarehouseExecutor client timeout", () => { diff --git a/packages/query-engine/src/execution/executor.ts b/packages/query-engine/src/execution/executor.ts index 660a441dd..6c68137c8 100644 --- a/packages/query-engine/src/execution/executor.ts +++ b/packages/query-engine/src/execution/executor.ts @@ -1,5 +1,8 @@ import { Clock, Duration, Effect, Ref, Schedule } from "effect" import { + MAX_RAW_SQL_RESULT_BYTES, + MAX_RAW_SQL_RESULT_ROWS, + RawSqlValidationError, type WarehouseQueryRequest, WarehouseQueryResponse, WarehouseSchemaDriftError, @@ -16,6 +19,7 @@ import { stripTinybirdRestrictedSettings, } from "../profiles" import { mapWarehouseError, toWarehouseQueryError, type WarehouseSqlError } from "./errors" +import { WarehouseResponseLimitError } from "./response-limits" import { SQL_LOG_MAX, SQL_TRACE_MAX, @@ -42,7 +46,7 @@ interface CachedClient { const sqlClientCacheKey = (config: ResolvedWarehouseConfig): string => config._tag === "clickhouse" - ? `clickhouse:${config.url}:${config.username}:${config.password}:${config.database}` + ? `${config.provider}:clickhouse:${config.url}:${config.username}:${config.password}:${config.database}` : `tinybird:${config.host}:${config.token}` // Only retry transient upstream failures (5xx, 408, 429, network blips). Non-transient @@ -54,7 +58,7 @@ const TRANSIENT_RETRY_SCHEDULE = Schedule.exponential("100 millis", 2.0).pipe( Schedule.both(Schedule.recurs(2)), ) -const isTransientUpstreamError = (error: WarehouseSqlError): boolean => +const isTransientUpstreamError = (error: unknown): error is WarehouseUpstreamError => error instanceof WarehouseUpstreamError // Client-side ceiling for a single query attempt. Tinybird's server-side @@ -116,6 +120,7 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue sql: string, pipe: string, options?: SqlQueryOptions, + execution: "trusted" | "raw" = "trusted", ) { const startedAtMs = yield* Clock.currentTimeMillis yield* Effect.annotateCurrentSpan("orgId", tenant.orgId) @@ -139,9 +144,11 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue // config to stay symmetric with the write — otherwise a BYO-CH org reads an // empty table from its own ClickHouse. const resolveFn = - options?.pinToIngestConfig && deps.resolveIngestConfig - ? deps.resolveIngestConfig - : deps.resolveConfig + execution === "raw" + ? deps.resolveRawSqlConfig + : options?.pinToIngestConfig && deps.resolveIngestConfig + ? deps.resolveIngestConfig + : deps.resolveConfig const resolved = yield* resolveFn(tenant, pipe) if (options?.pinToIngestConfig) yield* Effect.annotateCurrentSpan("query.routing", "ingest") const peerService = resolved.config._tag === "clickhouse" ? "clickhouse" : "tinybird" @@ -150,14 +157,13 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue // Tinybird's managed warehouse restricts some settings (e.g. max_block_size: // "Usage of setting 'max_block_size' is restricted"). It is reached either via // the Tinybird SDK (_tag "tinybird") or its ClickHouse-compatible gateway - // (_tag "clickhouse", when CLICKHOUSE_URL is set); both enforce that policy. - // Only a genuine per-org BYO ClickHouse (source "org_override") supports those - // settings, so strip everywhere except there. Gating on `source` (not `_tag`) - // is what fixes the managed CH-gateway path. + // (_tag "clickhouse", when CLICKHOUSE_PROVIDER=tinybird); both enforce that + // policy. Vanilla ClickHouse supports those settings regardless of whether its + // credentials come from the environment or an org override. const settings = - resolved.source === "org_override" - ? resolveSettings(options) - : stripTinybirdRestrictedSettings(resolveSettings(options)) + resolved.config.provider === "tinybird" + ? stripTinybirdRestrictedSettings(resolveSettings(options)) + : resolveSettings(options) const sqlForClient = resolved.config._tag === "clickhouse" ? normalizeSqlForClickHouseClient(sql) : sql const finalSql = appendSettings(sqlForClient, settings) @@ -172,43 +178,55 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue if (options?.profile) yield* Effect.annotateCurrentSpan("query.profile", options.profile) if (settings) yield* Effect.annotateCurrentSpan("ch.settings", JSON.stringify(settings)) - const cacheKey = resolved.source === "managed" ? "__managed__" : tenant.orgId - const client = getCachedOrCreateClient(cacheKey, resolved.config, yield* Clock.currentTimeMillis) + const client = getCachedOrCreateClient( + resolved.clientCacheKey, + resolved.config, + yield* Clock.currentTimeMillis, + ) const attemptTimeoutMs = clientTimeoutMs(options?.profile, settings?.maxExecutionTime) const retryAttempts = yield* Ref.make(0) + const responseLimits = + execution === "raw" + ? { maxRows: MAX_RAW_SQL_RESULT_ROWS, maxBytes: MAX_RAW_SQL_RESULT_BYTES } + : undefined const queryAttempt = Effect.tryPromise({ - try: () => client.sql(finalSql), - catch: (error) => mapWarehouseError(pipe, error), + try: () => client.sql(finalSql, responseLimits === undefined ? undefined : { responseLimits }), + catch: (error) => + error instanceof WarehouseResponseLimitError + ? new RawSqlValidationError({ + code: "ResourceLimit", + message: error.message, + }) + : mapWarehouseError(pipe, error), }) // `db.duration_ms` measures warehouse execution only — captured here, after // config resolution + settings/client-cache preamble, immediately before the // query runs. `startedAtMs` (captured at span entry) feeds the separate // `db.total_duration_ms`, covering the whole executeSql span including preamble. const sqlStartedMs = yield* Clock.currentTimeMillis - const result = yield* ( - // Bound each attempt: don't let a queued query ride the ambient ~30s - // Worker fetch limit past its declared budget. Non-transient, so it - // fails fast rather than retrying into a struggling warehouse. Skipped - // entirely for the `unbounded` profile (explicit opt-out). + // Bound each attempt: don't let a queued query ride the ambient ~30s Worker + // fetch limit past its declared budget. The timeout is non-transient, so it + // fails fast instead of retrying. The `unbounded` profile explicitly opts out. + const boundedAttempt = attemptTimeoutMs === undefined ? queryAttempt : queryAttempt.pipe( Effect.timeoutOrElse({ duration: Duration.millis(attemptTimeoutMs), orElse: () => - // Constructed directly via `toWarehouseQueryError` (non-transient) — this - // timeout must never flow through `mapWarehouseError`, whose transient - // regex could reclassify a "timeout" message and feed it back into the - // retry loop, re-amplifying load on an already-struggling warehouse. + // Constructed directly via `toWarehouseQueryError` so a transient + // message matcher cannot feed this client timeout into the retry loop. Effect.fail( toWarehouseQueryError( pipe, - new Error(`Warehouse query exceeded ${attemptTimeoutMs}ms client timeout`), + new Error( + `Warehouse query exceeded ${attemptTimeoutMs}ms client timeout`, + ), ), ), }), ) - ).pipe( + const result = yield* boundedAttempt.pipe( Effect.tapError((error) => isTransientUpstreamError(error) ? Ref.update(retryAttempts, (n) => n + 1) : Effect.void, ), @@ -251,6 +269,18 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue return result.data }) + const executeTrustedSql = ( + tenant: ExecutionTenant, + sql: string, + pipe: string, + options?: SqlQueryOptions, + ) => + executeSql(tenant, sql, pipe, options, "trusted").pipe( + // A trusted driver call never receives response limits, so this branch is + // an impossible implementation defect rather than part of its error API. + Effect.catchTag("@maple/http/errors/RawSqlValidationError", Effect.die), + ) + const query = Effect.fn("WarehouseQueryService.query")(function* ( tenant: ExecutionTenant, payload: WarehouseQueryRequest, @@ -278,7 +308,7 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue }) } - const rows = yield* executeSql(tenant, compiled.sql, payload.pipeName, options) + const rows = yield* executeTrustedSql(tenant, compiled.sql, payload.pipeName, options) const decodedRows = yield* compiled.decodeRows(rows).pipe( Effect.mapError( (error) => @@ -312,7 +342,21 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue message: "SQL query must contain OrgId filter (sqlQuery)", }) } - return yield* executeSql(tenant, sql, "sqlQuery", options) + return yield* executeTrustedSql(tenant, sql, "sqlQuery", options) + }) + + const rawSqlQuery = Effect.fn("WarehouseQueryService.rawSqlQuery")(function* ( + tenant: ExecutionTenant, + sql: string, + options?: Pick, + ) { + if (!sql.includes("OrgId")) { + return yield* new RawSqlValidationError({ + code: "MissingOrgFilter", + message: "Raw SQL must contain the expanded OrgId filter", + }) + } + return yield* executeSql(tenant, sql, "rawSqlQuery", options, "raw") }) const compiledQuery = Effect.fn("WarehouseQueryService.compiledQuery")(function* ( @@ -368,15 +412,20 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue // ClickHouse READ override (routing writes through the override 500'd every // insert and broke demo-seed onboarding). Falls back to the read resolver. const resolveForIngest = deps.resolveIngestConfig ?? deps.resolveConfig - const resolved = yield* resolveForIngest(tenant, label) + const resolved = yield* resolveForIngest(tenant, label).pipe( + Effect.mapError((error) => toWarehouseQueryError(label, error)), + ) // Insert through the same client the read path uses (official // @clickhouse/client-web for ClickHouse, Tinybird Events API for // Tinybird) so the wire protocol is handled correctly — a hand-rolled // `?query=INSERT … FORMAT JSONEachRow` POST had its query param dropped // by managed ClickHouse, which then parsed the NDJSON body as SQL. - const cacheKey = resolved.source === "managed" ? "__managed__" : tenant.orgId - const client = getCachedOrCreateClient(cacheKey, resolved.config, yield* Clock.currentTimeMillis) + const client = getCachedOrCreateClient( + resolved.clientCacheKey, + resolved.config, + yield* Clock.currentTimeMillis, + ) yield* Effect.tryPromise({ try: () => client.insert(datasource, rows), @@ -451,6 +500,7 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue return { query, sqlQuery, + rawSqlQuery, compiledQuery, compiledQueryFirst, ingest, diff --git a/packages/query-engine/src/execution/index.ts b/packages/query-engine/src/execution/index.ts index d39e5a123..54008dea0 100644 --- a/packages/query-engine/src/execution/index.ts +++ b/packages/query-engine/src/execution/index.ts @@ -2,3 +2,4 @@ export * from "./ports" export * from "./errors" export * from "./fingerprint" export * from "./executor" +export * from "./response-limits" diff --git a/packages/query-engine/src/execution/ports.ts b/packages/query-engine/src/execution/ports.ts index f6d538e2a..65dd86051 100644 --- a/packages/query-engine/src/execution/ports.ts +++ b/packages/query-engine/src/execution/ports.ts @@ -1,6 +1,7 @@ import type { Effect, Option } from "effect" import type { OrgId } from "@maple/domain" import type { + RawSqlValidationError, WarehouseQueryRequest, WarehouseQueryResponse, WarehouseQueryError, @@ -38,10 +39,13 @@ export type SqlQueryOptions = { pinToIngestConfig?: boolean } +export type WarehouseProvider = "clickhouse" | "tinybird" + /** Resolved upstream connection config for a tenant's queries. */ export type ResolvedWarehouseConfig = | { readonly _tag: "clickhouse" + readonly provider: WarehouseProvider readonly url: string readonly username: string readonly password: string @@ -49,33 +53,50 @@ export type ResolvedWarehouseConfig = } | { readonly _tag: "tinybird" + readonly provider: "tinybird" readonly host: string readonly token: string } /** Minimal client interface — raw SQL execution plus row inserts. */ export interface WarehouseSqlClient { - readonly sql: (sql: string) => Promise<{ data: ReadonlyArray> }> + readonly sql: ( + sql: string, + options?: { + readonly responseLimits?: { + readonly maxRows: number + readonly maxBytes: number + } + }, + ) => Promise<{ data: ReadonlyArray> }> readonly insert: (datasource: string, rows: ReadonlyArray) => Promise } /** * The injected dependencies of the warehouse executor. The host app provides * the driver construction (`createClient`) and the per-org config resolution - * (`resolveConfig`, which reads the org-override DB row / env and emits the - * `clientSource` / `db.client` span annotations); the executor itself — error - * mapping, retry, client cache, OrgId scoping, span instrumentation — lives in - * this package. + * (`resolveConfig` / `resolveRawSqlConfig`, which read the org-override DB row + * or env and return stable logical cache partitions); the executor itself — + * error mapping, retry, client cache, OrgId scoping, span instrumentation — + * lives in this package. */ +export interface ResolvedWarehouseTarget { + readonly config: ResolvedWarehouseConfig + /** Stable logical cache partition; config changes are detected independently. */ + readonly clientCacheKey: string +} + export interface WarehouseExecutorDeps { readonly createClient: (config: ResolvedWarehouseConfig) => WarehouseSqlClient readonly resolveConfig: ( tenant: ExecutionTenant, label: string, - ) => Effect.Effect< - { readonly config: ResolvedWarehouseConfig; readonly source: "managed" | "org_override" }, - WarehouseQueryError - > + ) => Effect.Effect + /** Resolve the isolated credentials used exclusively for user-authored SQL. */ + readonly resolveRawSqlConfig: ( + tenant: ExecutionTenant, + label: string, + ) => Effect.Effect /** * Config resolver for the WRITE path (`ingest`). Inserts must land in the * managed pipeline (Tinybird in the cloud), NOT a per-org BYO ClickHouse @@ -85,10 +106,7 @@ export interface WarehouseExecutorDeps { readonly resolveIngestConfig?: ( tenant: ExecutionTenant, label: string, - ) => Effect.Effect< - { readonly config: ResolvedWarehouseConfig; readonly source: "managed" | "org_override" }, - WarehouseQueryError - > + ) => Effect.Effect } export interface WarehouseQueryServiceShape { @@ -102,6 +120,12 @@ export interface WarehouseQueryServiceShape { sql: string, options?: SqlQueryOptions, ) => Effect.Effect>, WarehouseSqlError | WarehouseValidationError> + /** Execute validated user-authored SQL with tenant-scoped credentials and hard response limits. */ + readonly rawSqlQuery: ( + tenant: ExecutionTenant, + sql: string, + options?: Pick, + ) => Effect.Effect>, WarehouseSqlError | RawSqlValidationError> readonly compiledQuery: ( tenant: ExecutionTenant, compiled: CompiledQuery, diff --git a/packages/query-engine/src/execution/response-limits.ts b/packages/query-engine/src/execution/response-limits.ts new file mode 100644 index 000000000..6e1699b69 --- /dev/null +++ b/packages/query-engine/src/execution/response-limits.ts @@ -0,0 +1,13 @@ +import { Schema } from "effect" + +export const WarehouseResponseLimitKind = Schema.Literals(["rows", "bytes"]) +export type WarehouseResponseLimitKind = Schema.Schema.Type + +/** Driver-level abort used before a raw response can be fully buffered. */ +export class WarehouseResponseLimitError extends Schema.TaggedErrorClass()( + "@maple/query-engine/execution/WarehouseResponseLimitError", + { + kind: WarehouseResponseLimitKind, + message: Schema.String, + }, +) {} diff --git a/packages/query-engine/src/profiles/query-profile.ts b/packages/query-engine/src/profiles/query-profile.ts index 4521471d2..28f8ca1c5 100644 --- a/packages/query-engine/src/profiles/query-profile.ts +++ b/packages/query-engine/src/profiles/query-profile.ts @@ -39,7 +39,14 @@ export type WarehouseQuerySettings = { */ export const LOGS_BODY_SEARCH_SETTINGS: WarehouseQuerySettings = { maxBlockSize: 512 } -export type QueryProfileName = "discovery" | "list" | "aggregation" | "explain" | "unbounded" +export type QueryProfileName = + | "discovery" + | "list" + | "aggregation" + | "rawInteractive" + | "rawAlert" + | "explain" + | "unbounded" /** * The shared profile/settings selector carried by every warehouse query path @@ -63,6 +70,8 @@ export const QueryProfile: Record = { discovery: { maxExecutionTime: 5, maxMemoryUsage: 512_000_000 }, list: { maxExecutionTime: 15, maxMemoryUsage: 1_500_000_000 }, aggregation: { maxExecutionTime: 30, maxMemoryUsage: 4_000_000_000 }, + rawInteractive: { maxExecutionTime: 10, maxMemoryUsage: 512_000_000, maxThreads: 2 }, + rawAlert: { maxExecutionTime: 5, maxMemoryUsage: 256_000_000, maxThreads: 2 }, explain: { maxExecutionTime: 2, maxMemoryUsage: 128_000_000 }, unbounded: {}, } diff --git a/packages/query-engine/src/runtime/query-engine.ts b/packages/query-engine/src/runtime/query-engine.ts index 7cdaa58ed..af9bda734 100644 --- a/packages/query-engine/src/runtime/query-engine.ts +++ b/packages/query-engine/src/runtime/query-engine.ts @@ -22,13 +22,16 @@ import { QueryEngineExecutionError, QueryEngineTimeoutError, QueryEngineValidationError, + MAX_RAW_SQL_ALERT_GROUPS, + MAX_RAW_SQL_GROUP_KEY_LENGTH, + type RawSqlValidationError, type WarehouseError, } from "@maple/domain/http" import type { OrgId } from "@maple/domain" import { Array as Arr, Duration, Effect, Match, Option, Result, Schema } from "effect" import { LOGS_BODY_SEARCH_SETTINGS, type QueryProfileName, type WarehouseQuerySettings } from "../profiles" import { computeBucketSeconds } from "../datetime" -import { makeExpandMacros } from "./raw-sql" +import { makeExecuteRawSql } from "./raw-sql" import { decodeEvalSeries, encodeEvalPoints, type BucketGroupObs } from "./evaluate-bucket-codec" // Re-exported so `@maple/query-engine/runtime` consumers (apps/api) keep importing @@ -57,6 +60,11 @@ export interface QueryEngineWarehouse { readonly settings?: WarehouseQuerySettings }, ) => Effect.Effect>, WarehouseError> + readonly rawSqlQuery: ( + tenant: T, + sql: string, + options: { readonly profile: QueryProfileName; readonly context: string }, + ) => Effect.Effect>, WarehouseError | RawSqlValidationError> readonly compiledQuery: ( tenant: T, compiled: CH.CompiledQuery, @@ -117,7 +125,7 @@ export type QueryEngineDirectError = QueryEngineExecutionError | QueryEngineTime export type QueryEngineRouteError = QueryEngineValidationError | QueryEngineDirectError -const MAX_RANGE_SECONDS = 60 * 60 * 24 * 31 +export const MAX_QUERY_RANGE_SECONDS = 60 * 60 * 24 * 31 const MAX_LIST_RANGE_SECONDS = 60 * 60 * 24 * 7 const MAX_TIMESERIES_POINTS = 1_500 const MAX_BREAKDOWN_RANGE_SECONDS = 60 * 60 * 24 * 30 @@ -314,10 +322,10 @@ const validateTimeRange = Effect.fn("QueryEngineService.validateTimeRange")(func } const rangeSeconds = (endMs - startMs) / 1000 - if (rangeSeconds > MAX_RANGE_SECONDS) { + if (rangeSeconds > MAX_QUERY_RANGE_SECONDS) { return yield* new QueryEngineValidationError({ message: "Time range too large", - details: [`Maximum supported range is ${MAX_RANGE_SECONDS} seconds`], + details: [`Maximum supported range is ${MAX_QUERY_RANGE_SECONDS} seconds`], }) } @@ -347,41 +355,43 @@ const validateTraceAttributeFilters = Effect.fn("QueryEngineService.validateTrac }, ) -const validateMetricsAttributeFilters = Effect.fn( - "QueryEngineService.validateMetricsAttributeFilters", -)(function* (query: QuerySpec): Effect.fn.Return { - if (query.source !== "metrics") return - if (query.kind !== "timeseries" && query.kind !== "breakdown") return - - // `groupBy` is an array for timeseries, a single literal for breakdown. - const groupBy = query.groupBy - const wantsAttribute = Array.isArray(groupBy) ? groupBy.includes("attribute") : groupBy === "attribute" - const wantsResourceAttribute = Array.isArray(groupBy) - ? groupBy.includes("resource_attribute") - : groupBy === "resource_attribute" - if (wantsAttribute && !query.filters.groupByAttributeKey) { - // Mirror the traces guard: never silently downgrade an attribute grouping - // to a service grouping — the agent asked for a label breakdown. - return yield* new QueryEngineValidationError({ - message: "Invalid metrics attribute grouping", - details: ["groupBy=attribute requires filters.groupByAttributeKey"], - }) - } - if (wantsResourceAttribute && !query.filters.groupByResourceAttributeKey) { - return yield* new QueryEngineValidationError({ - message: "Invalid metrics attribute grouping", - details: ["groupBy=resource_attribute requires filters.groupByResourceAttributeKey"], - }) - } - if (wantsAttribute && wantsResourceAttribute) { - // The metrics queries carry a single attributeValue group column — one - // attribute dimension per query. - return yield* new QueryEngineValidationError({ - message: "Invalid metrics attribute grouping", - details: ["groupBy cannot combine attribute and resource_attribute"], - }) - } -}) +const validateMetricsAttributeFilters = Effect.fn("QueryEngineService.validateMetricsAttributeFilters")( + function* (query: QuerySpec): Effect.fn.Return { + if (query.source !== "metrics") return + if (query.kind !== "timeseries" && query.kind !== "breakdown") return + + // `groupBy` is an array for timeseries, a single literal for breakdown. + const groupBy = query.groupBy + const wantsAttribute = Array.isArray(groupBy) + ? groupBy.includes("attribute") + : groupBy === "attribute" + const wantsResourceAttribute = Array.isArray(groupBy) + ? groupBy.includes("resource_attribute") + : groupBy === "resource_attribute" + if (wantsAttribute && !query.filters.groupByAttributeKey) { + // Mirror the traces guard: never silently downgrade an attribute grouping + // to a service grouping — the agent asked for a label breakdown. + return yield* new QueryEngineValidationError({ + message: "Invalid metrics attribute grouping", + details: ["groupBy=attribute requires filters.groupByAttributeKey"], + }) + } + if (wantsResourceAttribute && !query.filters.groupByResourceAttributeKey) { + return yield* new QueryEngineValidationError({ + message: "Invalid metrics attribute grouping", + details: ["groupBy=resource_attribute requires filters.groupByResourceAttributeKey"], + }) + } + if (wantsAttribute && wantsResourceAttribute) { + // The metrics queries carry a single attributeValue group column — one + // attribute dimension per query. + return yield* new QueryEngineValidationError({ + message: "Invalid metrics attribute grouping", + details: ["groupBy cannot combine attribute and resource_attribute"], + }) + } + }, +) const validatePointBudget = Effect.fn("QueryEngineService.validatePointBudget")(function* ( request: QueryEngineExecuteRequest, @@ -1093,6 +1103,8 @@ export const makeQueryEngineExecute = (warehouse: QueryEn attributeKey: attributeFilter?.key, attributeValue: attributeFilter?.value, resourceAttributeFilters, + groupBy: request.query.groupBy, + seriesLimit: request.query.seriesLimit, }), { orgId: tenant.orgId, @@ -1140,6 +1152,8 @@ export const makeQueryEngineExecute = (warehouse: QueryEn attributeKey: attributeFilter?.key, attributeValue: attributeFilter?.value, resourceAttributeFilters, + groupBy: request.query.groupBy, + seriesLimit: request.query.seriesLimit, }), { orgId: tenant.orgId, @@ -1164,7 +1178,7 @@ export const makeQueryEngineExecute = (warehouse: QueryEn request.query.groupBy?.includes("none") || !request.query.groupBy?.length ? groupTimeSeriesRows( collapseMetricTimeseriesRows( - result as Array, + result as ReadonlyArray, request.query.metric, ), (row) => row.value, @@ -2099,8 +2113,9 @@ const RawSqlAlertRowSchema = Schema.Struct({ * and an optional `samples` column carries the sample count (else each row * counts as 1). Per group, `value` rows are collapsed with the reducer. */ -export const makeQueryEngineEvaluateRawSql = (warehouse: QueryEngineWarehouse) => - Effect.fn("QueryEngineService.evaluateRawSql")(function* ( +export const makeQueryEngineEvaluateRawSql = (warehouse: QueryEngineWarehouse) => { + const executeRawSql = makeExecuteRawSql(warehouse) + return Effect.fn("QueryEngineService.evaluateRawSql")(function* ( tenant: T, request: QueryEngineRawSqlEvaluateRequest, ): Effect.fn.Return, QueryEngineValidationError | WarehouseError> { @@ -2108,26 +2123,24 @@ export const makeQueryEngineEvaluateRawSql = (warehouse: yield* Effect.annotateCurrentSpan("query.reducer", request.reducer) const granularitySeconds = Math.max(request.windowMinutes * 60, 60) - const expanded = yield* makeExpandMacros({ + const { rows: rawRows } = yield* executeRawSql(tenant, { sql: request.sql, orgId: tenant.orgId, startTime: request.startTime, endTime: request.endTime, granularitySeconds, + workload: "alert", + context: "alertRawQuery", }).pipe( - Effect.mapError( - (error) => + Effect.catchTag("@maple/http/errors/RawSqlValidationError", (error) => + Effect.fail( new QueryEngineValidationError({ message: "Invalid raw SQL alert query", details: [error.message], }), + ), ), ) - - const rawRows = yield* annotateWarehouseError( - warehouse.sqlQuery(tenant, expanded.sql, { profile: "list", context: "alertRawQuery" }), - "alertRawQuery", - ) const rows = yield* Schema.decodeUnknownEffect(Schema.Array(RawSqlAlertRowSchema))(rawRows).pipe( Effect.mapError( () => @@ -2145,14 +2158,36 @@ export const makeQueryEngineEvaluateRawSql = (warehouse: for (const row of rows) { const rawGroup = row.group const groupKey = typeof rawGroup === "string" && rawGroup.length > 0 ? rawGroup : "all" + if (groupKey.length > MAX_RAW_SQL_GROUP_KEY_LENGTH) { + return yield* new QueryEngineValidationError({ + message: "Invalid raw SQL alert query", + details: [ + `Raw SQL alert group keys may contain at most ${MAX_RAW_SQL_GROUP_KEY_LENGTH} characters.`, + ], + }) + } const numValue = row.value == null ? null : Number(row.value) const value = numValue != null && Number.isFinite(numValue) ? numValue : null const rawSamples = row.samples == null ? 1 : Number(row.samples) - const sampleCount = Number.isFinite(rawSamples) ? rawSamples : 1 + if (!Number.isFinite(rawSamples) || rawSamples < 0) { + return yield* new QueryEngineValidationError({ + message: "Invalid raw SQL alert query", + details: ["Raw SQL alert samples must be finite and nonnegative."], + }) + } + const sampleCount = rawSamples const list = byGroup.get(groupKey) const obs = { value, sampleCount, hasData: value != null } if (list) list.push(obs) - else byGroup.set(groupKey, [obs]) + else { + if (byGroup.size >= MAX_RAW_SQL_ALERT_GROUPS) { + return yield* new QueryEngineValidationError({ + message: "Invalid raw SQL alert query", + details: [`Raw SQL alerts may return at most ${MAX_RAW_SQL_ALERT_GROUPS} groups.`], + }) + } + byGroup.set(groupKey, [obs]) + } } // No rows → emit a single no-data observation so the alert engine can @@ -2165,3 +2200,4 @@ export const makeQueryEngineEvaluateRawSql = (warehouse: yield* Effect.annotateCurrentSpan("result.groupCount", result.length) return result }) +} diff --git a/packages/query-engine/src/runtime/raw-sql.test.ts b/packages/query-engine/src/runtime/raw-sql.test.ts index 40e4d9010..6b286279b 100644 --- a/packages/query-engine/src/runtime/raw-sql.test.ts +++ b/packages/query-engine/src/runtime/raw-sql.test.ts @@ -1,190 +1,208 @@ import { assert, describe, it } from "@effect/vitest" import { Cause, Effect, Exit, Option } from "effect" -import { RawSqlValidationError } from "@maple/domain/http" -import { RawSqlChartService } from "./raw-sql" +import { + MAX_RAW_SQL_CELL_LENGTH, + MAX_RAW_SQL_LENGTH, + MAX_RAW_SQL_RESULT_BYTES, + MAX_RAW_SQL_RESULT_ROWS, + RawSqlValidationError, +} from "@maple/domain/http" +import { makeExecuteRawSql, prepareRawSql } from "./raw-sql" const baseInput = { orgId: "org_abc", startTime: "2026-05-14 00:00:00", endTime: "2026-05-14 06:00:00", granularitySeconds: 60, + workload: "interactive" as const, } const getError = (exit: Exit.Exit): unknown => { if (!Exit.isFailure(exit)) return undefined - const failure = Option.getOrUndefined(Exit.findErrorOption(exit)) - if (failure !== undefined) return failure - return Cause.squash(exit.cause) + return Option.getOrElse(Exit.findErrorOption(exit), () => Cause.squash(exit.cause)) } -const expandOk = (sql: string) => - Effect.gen(function* () { - const svc = yield* RawSqlChartService - return yield* svc.expandMacros({ ...baseInput, sql }) - }) +const prepareOk = (sql: string) => prepareRawSql({ ...baseInput, sql }) -const expandFail = (sql: string) => +const prepareFail = (sql: string, workload: "interactive" | "alert" = "interactive") => Effect.gen(function* () { - const svc = yield* RawSqlChartService - const exit = yield* Effect.exit(svc.expandMacros({ ...baseInput, sql })) - if (Exit.isSuccess(exit)) { - throw new Error(`expected failure, got success: ${JSON.stringify(exit.value)}`) - } - const failure = getError(exit) - if (!(failure instanceof RawSqlValidationError)) { - throw new Error(`expected RawSqlValidationError, got: ${String(failure)}`) + const exit = yield* Effect.exit(prepareRawSql({ ...baseInput, sql, workload })) + const error = getError(exit) + if (!(error instanceof RawSqlValidationError)) { + throw new Error(`expected RawSqlValidationError, got: ${String(error)}`) } - return failure + return error }) -describe("RawSqlChartService.expandMacros", () => { - it.layer(RawSqlChartService.layer)((it) => { - it.effect("rejects SQL missing $__orgFilter", () => - Effect.gen(function* () { - const svc = yield* RawSqlChartService - const exit = yield* Effect.exit( - svc.expandMacros({ - ...baseInput, - sql: "SELECT 1 FROM Logs", - }), - ) - assert.isTrue(Exit.isFailure(exit)) - const failure = getError(exit) - assert.instanceOf(failure, RawSqlValidationError) - assert.strictEqual((failure as RawSqlValidationError).code, "MissingOrgFilter") - }), - ) +describe("prepareRawSql", () => { + it.effect("rejects SQL missing $__orgFilter", () => + Effect.gen(function* () { + const error = yield* prepareFail("SELECT 1 FROM Logs") + assert.strictEqual(error.code, "MissingOrgFilter") + }), + ) - it.effect("rejects SQL with multiple statements", () => - Effect.gen(function* () { - const svc = yield* RawSqlChartService - const exit = yield* Effect.exit( - svc.expandMacros({ - ...baseInput, - sql: "SELECT 1 FROM Logs WHERE $__orgFilter; SELECT 2", - }), - ) - assert.isTrue(Exit.isFailure(exit)) - const failure = getError(exit) - assert.instanceOf(failure, RawSqlValidationError) - assert.strictEqual((failure as RawSqlValidationError).code, "MultipleStatements") - }), - ) + it.effect("requires a time filter for alerts only", () => + Effect.gen(function* () { + yield* prepareOk("SELECT 1 FROM Logs WHERE $__orgFilter") + const error = yield* prepareFail("SELECT 1 FROM Logs WHERE $__orgFilter", "alert") + assert.strictEqual(error.code, "InvalidMacro") + }), + ) - it.effect("does not flag semicolons inside string literals", () => - Effect.gen(function* () { - const result = yield* expandOk( - "SELECT 'a;b' AS x FROM Logs WHERE $__orgFilter AND $__timeFilter(Timestamp)", - ) - assert.include(result.sql, "OrgId = 'org_abc'") - }), - ) - - for (const keyword of [ - "INSERT", - "UPDATE", - "DELETE", - "DROP", - "ALTER", - "TRUNCATE", - "RENAME", - "ATTACH", - "DETACH", - "CREATE", - "GRANT", - "REVOKE", - "OPTIMIZE", - "SYSTEM", - "KILL", - ]) { - it.effect(`rejects deny-listed keyword ${keyword}`, () => - Effect.gen(function* () { - const failure = yield* expandFail( - `SELECT 1 FROM Logs WHERE $__orgFilter; ${keyword} TABLE Logs`, - ) - // Either MultipleStatements (because of ';') or DisallowedStatement — - // both correctly block the dangerous query. Tighten by also testing without ';'. - assert.include(["MultipleStatements", "DisallowedStatement"], failure.code) + it.effect("validates length and granularity", () => + Effect.gen(function* () { + assert.strictEqual((yield* prepareFail("")).code, "ResourceLimit") + assert.strictEqual( + (yield* prepareFail(`SELECT '${"x".repeat(MAX_RAW_SQL_LENGTH)}' WHERE $__orgFilter`)).code, + "ResourceLimit", + ) + const exit = yield* Effect.exit( + prepareRawSql({ + ...baseInput, + granularitySeconds: Number.NaN, + sql: "SELECT 1 WHERE $__orgFilter", }), ) + assert.instanceOf(getError(exit), RawSqlValidationError) + }), + ) - it.effect(`rejects standalone ${keyword} statement`, () => - Effect.gen(function* () { - const failure = yield* expandFail(`${keyword} TABLE Logs WHERE $__orgFilter`) - assert.strictEqual(failure.code, "DisallowedStatement") - }), + it.effect("rejects multiple statements but ignores semicolons inside strings", () => + Effect.gen(function* () { + assert.strictEqual( + (yield* prepareFail("SELECT 1 WHERE $__orgFilter; SELECT 2")).code, + "MultipleStatements", ) - } + const result = yield* prepareOk("SELECT 'a;b' AS value WHERE $__orgFilter") + assert.include(result.sql, "'a;b'") + }), + ) - it.effect("rejects unknown macros", () => + for (const keyword of [ + "INSERT", + "UPDATE", + "DELETE", + "DROP", + "ALTER", + "TRUNCATE", + "RENAME", + "ATTACH", + "DETACH", + "CREATE", + "GRANT", + "REVOKE", + "OPTIMIZE", + "SYSTEM", + "KILL", + ]) { + it.effect(`rejects deny-listed keyword ${keyword}`, () => Effect.gen(function* () { - const failure = yield* expandFail( - "SELECT $__bogus FROM Logs WHERE $__orgFilter AND $__timeFilter(Timestamp)", - ) - assert.strictEqual(failure.code, "UnresolvedMacro") + const error = yield* prepareFail(`${keyword} TABLE Logs WHERE $__orgFilter`) + assert.strictEqual(error.code, "DisallowedStatement") }), ) + } - it.effect("rejects malformed $__timeFilter column identifier", () => - Effect.gen(function* () { - const failure = yield* expandFail( - "SELECT 1 FROM Logs WHERE $__orgFilter AND $__timeFilter(1 OR 1=1)", - ) - assert.strictEqual(failure.code, "InvalidMacro") - }), - ) + it.effect("accepts SELECT/WITH and rejects other query forms", () => + Effect.gen(function* () { + yield* prepareOk("SELECT 1 WHERE $__orgFilter") + yield* prepareOk("WITH 1 AS value SELECT value WHERE $__orgFilter") + assert.strictEqual( + (yield* prepareFail("EXPLAIN SELECT 1 WHERE $__orgFilter")).code, + "DisallowedStatement", + ) + }), + ) - it.effect("expands the documented happy-path query", () => - Effect.gen(function* () { - const result = yield* expandOk( - "SELECT toStartOfInterval(Timestamp, INTERVAL $__interval_s SECOND) AS bucket, count() FROM Logs WHERE $__orgFilter AND $__timeFilter(Timestamp) GROUP BY bucket ORDER BY bucket", - ) - assert.include(result.sql, "OrgId = 'org_abc'") - assert.include(result.sql, "toDateTime('2026-05-14 00:00:00')") - assert.include(result.sql, "toDateTime('2026-05-14 06:00:00')") - assert.include(result.sql, "INTERVAL 60 SECOND") - assert.include(result.sql, "Timestamp >= toDateTime('2026-05-14 00:00:00')") - assert.include(result.sql, "Timestamp <= toDateTime('2026-05-14 06:00:00')") - assert.strictEqual(result.granularitySeconds, 60) - }), - ) + it.effect("validates and expands macros", () => + Effect.gen(function* () { + assert.strictEqual( + (yield* prepareFail( + "SELECT $__bogus FROM Logs WHERE $__orgFilter AND $__timeFilter(Timestamp)", + )).code, + "UnresolvedMacro", + ) + assert.strictEqual( + (yield* prepareFail("SELECT 1 FROM Logs WHERE $__orgFilter AND $__timeFilter(1 OR 1=1)")) + .code, + "InvalidMacro", + ) - it.effect("appends a default LIMIT when the user did not specify one", () => - Effect.gen(function* () { - const result = yield* expandOk( - "SELECT 1 FROM Logs WHERE $__orgFilter AND $__timeFilter(Timestamp)", - ) - assert.match(result.sql, /LIMIT 10000\s*$/) - }), - ) + const result = yield* prepareOk( + "SELECT toStartOfInterval(Timestamp, INTERVAL $__interval_s SECOND) AS bucket FROM Logs WHERE $__orgFilter AND $__timeFilter(Timestamp)", + ) + assert.include(result.sql, "OrgId = 'org_abc'") + assert.include(result.sql, "toDateTime('2026-05-14 00:00:00')") + assert.include(result.sql, "toDateTime('2026-05-14 06:00:00')") + assert.include(result.sql, "INTERVAL 60 SECOND") + assert.match(result.sql, /^SELECT \* FROM \(/) + assert.match(result.sql, /LIMIT 1001\s*$/) + }), + ) - it.effect("preserves the user's LIMIT if already present", () => - Effect.gen(function* () { - const result = yield* expandOk( - "SELECT 1 FROM Logs WHERE $__orgFilter AND $__timeFilter(Timestamp) LIMIT 7", - ) - assert.notInclude(result.sql, "LIMIT 10000") - assert.match(result.sql, /LIMIT 7/) - }), - ) + it.effect("escapes the org literal before validation", () => + Effect.gen(function* () { + const result = yield* prepareRawSql({ + ...baseInput, + orgId: "org'); DROP TABLE Logs --", + sql: "SELECT 1 FROM Logs WHERE $__orgFilter", + }) + assert.include(result.sql, "OrgId = 'org\\'); DROP TABLE Logs --'") + }), + ) +}) - it.effect("escapes single quotes in the orgId", () => - Effect.gen(function* () { - const svc = yield* RawSqlChartService - const exit = yield* Effect.exit( - svc.expandMacros({ - ...baseInput, - orgId: "org'); DROP TABLE Logs --", - sql: "SELECT 1 FROM Logs WHERE $__orgFilter", - }), - ) - assert.isTrue(Exit.isSuccess(exit)) - if (Exit.isSuccess(exit)) { - assert.include(exit.value.sql, "OrgId = 'org\\'); DROP TABLE Logs --'") - // Crucially the masked deny-list scan now sees an empty literal, so the - // DROP inside the literal does NOT trip the check. - } - }), - ) - }) +const executeRows = (rows: ReadonlyArray>) => + makeExecuteRawSql({ + rawSqlQuery: () => Effect.succeed(rows), + })( + {}, + { + ...baseInput, + sql: "SELECT 1 AS value WHERE $__orgFilter", + context: "test", + }, + ) + +describe("makeExecuteRawSql", () => { + it.effect("accepts exactly 1,000 rows and returns metadata", () => + Effect.gen(function* () { + const rows = Array.from({ length: MAX_RAW_SQL_RESULT_ROWS }, (_, value) => ({ value })) + const result = yield* executeRows(rows) + assert.strictEqual(result.rowCount, MAX_RAW_SQL_RESULT_ROWS) + assert.deepStrictEqual(result.columns, ["value"]) + }), + ) + + it.effect("rejects the 1,001-row sentinel", () => + Effect.gen(function* () { + const rows = Array.from({ length: MAX_RAW_SQL_RESULT_ROWS + 1 }, (_, value) => ({ value })) + const error = yield* Effect.flip(executeRows(rows)) + assert.strictEqual(error.code, "ResourceLimit") + assert.include(error.message, "rows") + }), + ) + + it.effect("rejects oversized cells, encoded output, and unserializable values", () => + Effect.gen(function* () { + const cellError = yield* Effect.flip( + executeRows([{ value: "x".repeat(MAX_RAW_SQL_CELL_LENGTH + 1) }]), + ) + assert.include(cellError.message, "cells") + + const bytesError = yield* Effect.flip( + executeRows( + Array.from({ length: 100 }, (_, value) => ({ + value, + payload: "x".repeat(Math.ceil(MAX_RAW_SQL_RESULT_BYTES / 100)), + })), + ), + ) + assert.include(bytesError.message, "bytes") + + const jsonError = yield* Effect.flip(executeRows([{ value: 1n }])) + assert.include(jsonError.message, "JSON serializable") + }), + ) }) diff --git a/packages/query-engine/src/runtime/raw-sql.ts b/packages/query-engine/src/runtime/raw-sql.ts index fe8c3a671..7deff35f6 100644 --- a/packages/query-engine/src/runtime/raw-sql.ts +++ b/packages/query-engine/src/runtime/raw-sql.ts @@ -1,21 +1,22 @@ -import { Context, Effect, Layer } from "effect" -import { RawSqlValidationError } from "@maple/domain/http" +import { Effect } from "effect" +import { + MAX_RAW_SQL_CELL_LENGTH, + MAX_RAW_SQL_LENGTH, + MAX_RAW_SQL_RESULT_BYTES, + MAX_RAW_SQL_RESULT_ROWS, + RawSqlValidationError, +} from "@maple/domain/http" +import type { QueryProfileName } from "../profiles" import { escapeClickHouseString } from "../sql" // --------------------------------------------------------------------------- -// Raw SQL chart macro expansion + safety checks. +// User-authored ClickHouse SQL: validation, macro expansion, and execution. // -// Users author a ClickHouse SQL string that references a handful of server- -// evaluated macros. We expand them with escaped values, then run a deny-list -// pass before handing the string off to WarehouseQueryService.sqlQuery. -// -// SECURITY NOTE: org isolation rests on three layers: -// 1. `$__orgFilter` MUST appear in the user SQL — expanded to OrgId = ''. -// 2. WarehouseQueryService.sqlQuery() string-matches "OrgId" as belt+braces. -// 3. Deny-listed statements (INSERT/DROP/…) are rejected pre-execution. -// -// TODO(security): migrate to ClickHouse row policies + a read-only CH user so -// org isolation is enforced at the DB layer instead of by SQL substring checks. +// Tenant isolation is enforced by the rawSqlQuery warehouse capability: +// Tinybird uses a per-org datasource-scoped JWT, BYO ClickHouse uses per-org +// credentials, and shared vanilla ClickHouse is limited to single-org mode. +// `$__orgFilter` remains mandatory as defense in depth and because OrgId is the +// leading sorting-key filter on Maple telemetry tables. // --------------------------------------------------------------------------- const COLUMN_IDENT_RE = /^[A-Za-z_][A-Za-z0-9_.]*$/ @@ -40,30 +41,48 @@ const DENY_LIST = [ const DENY_LIST_RE = new RegExp(`\\b(${DENY_LIST.join("|")})\\b`, "i") -const DEFAULT_ROW_CAP = 10_000 +/** One extra row is the overflow sentinel for the public 1,000-row cap. */ +export const RAW_SQL_FETCH_ROW_LIMIT = MAX_RAW_SQL_RESULT_ROWS + 1 + +export type RawSqlWorkload = "interactive" | "alert" -export interface ExpandMacrosInput { +export interface PrepareRawSqlInput { readonly sql: string readonly orgId: string readonly startTime: string readonly endTime: string readonly granularitySeconds: number + readonly workload: RawSqlWorkload } -export interface ExpandMacrosResult { +export interface PreparedRawSql { readonly sql: string readonly granularitySeconds: number } -export interface RawSqlChartServiceShape { - readonly expandMacros: ( - input: ExpandMacrosInput, - ) => Effect.Effect +export interface ExecuteRawSqlInput extends PrepareRawSqlInput { + readonly context: string +} + +export interface ExecuteRawSqlResult { + readonly rows: ReadonlyArray> + readonly columns: ReadonlyArray + readonly rowCount: number + readonly expandedSql: string + readonly granularitySeconds: number +} + +export interface RawSqlWarehouse { + readonly rawSqlQuery: ( + tenant: TTenant, + sql: string, + options: { readonly profile: QueryProfileName; readonly context: string }, + ) => Effect.Effect>, E> } /** - * Strip ClickHouse-style comments and string literals so deny-list scans and - * semicolon detection don't false-positive on words inside them. + * Strip ClickHouse-style comments and string literals so keyword and semicolon + * checks do not false-positive on their contents. */ function maskLiteralsAndComments(sql: string): string { let out = "" @@ -111,23 +130,31 @@ function maskLiteralsAndComments(sql: string): string { const fail = (code: RawSqlValidationError["code"], message: string) => Effect.fail(new RawSqlValidationError({ code, message })) -/** - * Pure macro-expansion + safety pass. Exported so non-service callers (e.g. - * raw-SQL alert evaluation in QueryEngineService) can reuse it without wiring - * the `RawSqlChartService` layer. - */ -export const makeExpandMacros = Effect.fn("RawSqlChartService.expandMacros")(function* ( - input: ExpandMacrosInput, -) { - let sql = input.sql - - if (!sql.includes("$__orgFilter")) { +/** Validate and expand a raw query without accessing the warehouse. */ +export const prepareRawSql = Effect.fn("RawSql.prepare")(function* (input: PrepareRawSqlInput) { + if (input.sql.length === 0 || input.sql.length > MAX_RAW_SQL_LENGTH) { + return yield* fail( + "ResourceLimit", + `Raw SQL must contain between 1 and ${MAX_RAW_SQL_LENGTH} characters`, + ) + } + if (!Number.isFinite(input.granularitySeconds) || input.granularitySeconds <= 0) { + return yield* fail("ResourceLimit", "Raw SQL granularity must be a positive finite number") + } + if (!input.sql.includes("$__orgFilter")) { return yield* fail( "MissingOrgFilter", "SQL must reference $__orgFilter so the query is scoped to your org.", ) } + if (input.workload === "alert" && !input.sql.includes("$__timeFilter(")) { + return yield* fail( + "InvalidMacro", + "Raw SQL alerts must reference $__timeFilter(...) to bound alert reads.", + ) + } + let sql = input.sql const orgLiteral = `'${escapeClickHouseString(input.orgId)}'` const startLiteral = `toDateTime('${escapeClickHouseString(input.startTime)}')` const endLiteral = `toDateTime('${escapeClickHouseString(input.endTime)}')` @@ -138,12 +165,6 @@ export const makeExpandMacros = Effect.fn("RawSqlChartService.expandMacros")(fun sql = sql.replaceAll("$__endTime", endLiteral) sql = sql.replaceAll("$__interval_s", String(granularity)) - // $__timeFilter(Column) -> Column >= AND Column <= - // Match anything inside the parens (greedy up to next `)`), then strictly - // validate the captured argument is a single column identifier. Catching - // the whole inner string lets us return InvalidMacro for injection - // attempts like `$__timeFilter(1 OR 1=1)` instead of letting them slip - // through to the UnresolvedMacro fallback. const timeFilterMatches = [...sql.matchAll(/\$__timeFilter\(([^)]*)\)/g)] for (const match of timeFilterMatches) { const column = match[1].trim() @@ -165,7 +186,6 @@ export const makeExpandMacros = Effect.fn("RawSqlChartService.expandMacros")(fun } const masked = maskLiteralsAndComments(sql) - if (masked.includes(";")) { return yield* fail( "MultipleStatements", @@ -177,25 +197,68 @@ export const makeExpandMacros = Effect.fn("RawSqlChartService.expandMacros")(fun if (denyMatch) { return yield* fail( "DisallowedStatement", - `Statement keyword '${denyMatch[1].toUpperCase()}' is not allowed in raw SQL charts.`, + `Statement keyword '${denyMatch[1].toUpperCase()}' is not allowed in raw SQL.`, ) } - - if (!/\blimit\b/i.test(masked)) { - sql = `${sql.trimEnd()}\nLIMIT ${DEFAULT_ROW_CAP}` + if (!/^\s*(?:SELECT|WITH)\b/i.test(masked)) { + return yield* fail( + "DisallowedStatement", + "Raw SQL must be a SELECT query (WITH common table expressions are supported).", + ) } return { - sql, + sql: `SELECT * FROM (\n${sql.trim()}\n) AS maple_raw_sql_limited\nLIMIT ${RAW_SQL_FETCH_ROW_LIMIT}`, granularitySeconds: granularity, - } satisfies ExpandMacrosResult + } satisfies PreparedRawSql }) -export class RawSqlChartService extends Context.Service()( - "@maple/api/services/RawSqlChartService", - { - make: Effect.succeed({ expandMacros: makeExpandMacros } satisfies RawSqlChartServiceShape), - }, -) { - static readonly layer = Layer.effect(this, this.make) +const rawSqlResultLimitError = (rows: ReadonlyArray>): string | null => { + if (rows.length > MAX_RAW_SQL_RESULT_ROWS) { + return `Raw SQL results may contain at most ${MAX_RAW_SQL_RESULT_ROWS} rows` + } + + let totalBytes = 2 + for (const row of rows) { + for (const value of Object.values(row)) { + if (typeof value === "string" && value.length > MAX_RAW_SQL_CELL_LENGTH) { + return `Raw SQL result cells may contain at most ${MAX_RAW_SQL_CELL_LENGTH} characters` + } + } + + let encoded: string + try { + encoded = JSON.stringify(row) ?? "null" + } catch { + return "Raw SQL results must be JSON serializable" + } + totalBytes += new TextEncoder().encode(encoded).byteLength + 1 + if (totalBytes > MAX_RAW_SQL_RESULT_BYTES) { + return `Raw SQL results may contain at most ${MAX_RAW_SQL_RESULT_BYTES} encoded bytes` + } + } + return null } + +/** Build the single prepare/execute workflow shared by HTTP, MCP, and alerts. */ +export const makeExecuteRawSql = (warehouse: RawSqlWarehouse) => + Effect.fn("RawSql.execute")(function* (tenant: TTenant, input: ExecuteRawSqlInput) { + const prepared = yield* prepareRawSql(input) + const rows = yield* warehouse.rawSqlQuery(tenant, prepared.sql, { + profile: input.workload === "alert" ? "rawAlert" : "rawInteractive", + context: input.context, + }) + + const limitError = rawSqlResultLimitError(rows) + if (limitError !== null) { + return yield* new RawSqlValidationError({ code: "ResourceLimit", message: limitError }) + } + + return { + rows, + columns: rows.length > 0 ? Object.keys(rows[0]) : [], + rowCount: rows.length, + expandedSql: prepared.sql, + granularitySeconds: prepared.granularitySeconds, + } satisfies ExecuteRawSqlResult + })