From 2bb8d4131a4b3f0a8489c4c007cf02644bfdef18 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 18 Jul 2026 19:46:21 +0200 Subject: [PATCH 1/7] Add v2 telemetry routes and API key rate limiting --- apps/api/alchemy.run.ts | 6 + apps/api/src/app.ts | 30 +- apps/api/src/lib/api-cors.ts | 8 + apps/api/src/routes/v2/alerts.http.test.ts | 9 +- apps/api/src/routes/v2/api-keys.http.test.ts | 95 +- .../routes/v2/config-resources.http.test.ts | 10 +- .../api/src/routes/v2/dashboards.http.test.ts | 4 + .../routes/v2/phase1-resources.http.test.ts | 4 + apps/api/src/routes/v2/telemetry.http.test.ts | 473 ++++++++++ apps/api/src/routes/v2/telemetry.http.ts | 834 ++++++++++++++++++ apps/api/src/routes/v2/v2-test-support.ts | 30 + .../src/services/ApiAuthorizationV2Layer.ts | 29 +- .../api/src/services/ApiV2RateLimiter.test.ts | 119 +++ apps/api/src/services/ApiV2RateLimiter.ts | 92 ++ apps/api/wrangler.jsonc | 13 + .../settings/create-api-key-dialog.tsx | 44 +- .../components/settings/developer-section.tsx | 21 +- docs/api-v2.md | 37 +- packages/domain/src/http/query-engine.ts | 6 +- packages/domain/src/http/v2/api.ts | 69 +- packages/domain/src/http/v2/auth.ts | 7 +- packages/domain/src/http/v2/errors.ts | 9 + packages/domain/src/http/v2/index.ts | 1 + packages/domain/src/http/v2/openapi.test.ts | 20 +- packages/domain/src/http/v2/public-id.ts | 2 + packages/domain/src/http/v2/telemetry.ts | 637 +++++++++++++ .../domain/src/http/v2/v2-contract.test.ts | 72 +- packages/domain/src/query-engine.ts | 4 +- packages/query-engine/src/ch/index.ts | 6 + .../query-engine/src/ch/queries/logs.test.ts | 40 +- packages/query-engine/src/ch/queries/logs.ts | 84 +- .../src/ch/queries/metrics.test.ts | 4 +- .../query-engine/src/ch/queries/metrics.ts | 7 +- .../src/ch/queries/services.test.ts | 23 + .../query-engine/src/ch/queries/services.ts | 60 +- .../src/ch/queries/traces.test.ts | 33 +- .../query-engine/src/ch/queries/traces.ts | 87 +- .../query-engine/src/runtime/query-engine.ts | 6 +- 38 files changed, 2943 insertions(+), 92 deletions(-) create mode 100644 apps/api/src/lib/api-cors.ts create mode 100644 apps/api/src/routes/v2/telemetry.http.test.ts create mode 100644 apps/api/src/routes/v2/telemetry.http.ts create mode 100644 apps/api/src/services/ApiV2RateLimiter.test.ts create mode 100644 apps/api/src/services/ApiV2RateLimiter.ts create mode 100644 packages/domain/src/http/v2/telemetry.ts diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index 0e0bc7246..37d29e3b3 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -7,6 +7,7 @@ import type { MapleApiRpcShape } from "@maple/domain/internal-rpc" import type { MapleDomains, MapleStage } from "@maple/infra/cloudflare" import { CLOUDFLARE_WORKER_PLACEMENT, + formatMapleStage, resolveDeploymentEnvironment, resolveHyperdriveName, resolveHyperdriveRefId, @@ -137,6 +138,11 @@ export const createMapleApi = ({ stage, domains }: CreateMapleApiOptions) => VCS_SYNC_QUEUE: vcsSyncQueue, CLICKHOUSE_SCHEMA_APPLY_WORKFLOW: schemaApplyWorkflow, AI_TRIAGE_WORKFLOW: aiTriageWorkflow, + API_V2_RATE_LIMITER: Cloudflare.RateLimit("API_V2_RATE_LIMITER", { + namespaceId: 2026071801, + simple: { limit: 600, period: 60 }, + }), + API_V2_RATE_LIMIT_PARTITION: formatMapleStage(stage), EMAIL: Cloudflare.Email.SendEmail("email", { allowedSenderAddresses: ["notifications@noreply.maple.dev"], }), diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 79b95126e..530623af5 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -23,6 +23,14 @@ import { HttpV2OrganizationLive } from "./routes/v2/organization.http" import { HttpV2InstrumentationRecommendationsLive } from "./routes/v2/recommendations.http" import { HttpV2ScrapeTargetsLive } from "./routes/v2/scrape-targets.http" import { HttpV2SessionReplaysLive } from "./routes/v2/session-replays.http" +import { + HttpV2LogsLive, + HttpV2MetricsLive, + HttpV2QueryLive, + HttpV2ServiceMapLive, + HttpV2ServicesLive, + HttpV2TracesLive, +} from "./routes/v2/telemetry.http" import { V2SchemaErrorsLive } from "./routes/v2/error-envelope" import { HttpAuthLive, HttpAuthPublicLive } from "./routes/auth.http" import { HttpChatLive } from "./routes/chat.http" @@ -57,6 +65,7 @@ import { HazelOAuthService } from "./services/HazelOAuthService" import { InvestigationService } from "./services/InvestigationService" import { NotificationDispatcher } from "./services/NotificationDispatcher" import { ApiKeysService } from "./services/ApiKeysService" +import { ApiV2RateLimiter } from "./services/ApiV2RateLimiter" import { AuthService } from "./services/AuthService" import { ApiAuthorizationLayer } from "./services/ApiAuthorizationLayer" import { ApiAuthorizationV2Layer } from "./services/ApiAuthorizationV2Layer" @@ -91,6 +100,7 @@ import { VcsCommitService } from "./services/vcs/VcsCommitService" import { VcsProviderRegistry } from "./services/vcs/VcsProviderRegistry" import { VcsRepository } from "./services/vcs/VcsRepository" import { VcsSyncQueue } from "./services/vcs/VcsSyncQueue" +import { API_CORS_OPTIONS } from "./lib/api-cors" const HealthRouter = HttpRouter.use((router) => router.add("GET", "/health", HttpServerResponse.text("OK"))) @@ -301,6 +311,12 @@ const ApiV2Routes = HttpApiBuilder.layer(MapleApiV2).pipe( HttpV2AnomaliesLive, HttpV2OrganizationLive, HttpV2SessionReplaysLive, + HttpV2TracesLive, + HttpV2LogsLive, + HttpV2MetricsLive, + HttpV2ServicesLive, + HttpV2ServiceMapLive, + HttpV2QueryLive, ), ), Layer.provide(V2SchemaErrorsLive), @@ -320,20 +336,10 @@ export const AllRoutes = Layer.mergeAll( McpGetFallback, DocsRoute, DocsV2Route, -).pipe( - Layer.provideMerge( - HttpRouter.cors({ - allowedOrigins: ["*"], - allowedMethods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], - allowedHeaders: ["*"], - // The ElectricSQL shape proxy (and its electric-* exposed headers) moved - // to the standalone `apps/electric-sync` worker. - exposedHeaders: ["Mcp-Session-Id"], - }), - ), -) +).pipe(Layer.provideMerge(HttpRouter.cors(API_CORS_OPTIONS))) export const ApiAuthLive = Layer.mergeAll(ApiAuthorizationLayer, ApiAuthorizationV2Layer).pipe( + Layer.provideMerge(ApiV2RateLimiter.layer), Layer.provideMerge(ApiKeysService.layer), Layer.provideMerge(Env.layer), ) diff --git a/apps/api/src/lib/api-cors.ts b/apps/api/src/lib/api-cors.ts new file mode 100644 index 000000000..816f97891 --- /dev/null +++ b/apps/api/src/lib/api-cors.ts @@ -0,0 +1,8 @@ +export const API_CORS_OPTIONS = { + allowedOrigins: ["*"], + allowedMethods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], + allowedHeaders: ["*"], + // The ElectricSQL shape proxy (and its electric-* exposed headers) moved + // to the standalone `apps/electric-sync` worker. + exposedHeaders: ["Mcp-Session-Id", "Retry-After"], +} diff --git a/apps/api/src/routes/v2/alerts.http.test.ts b/apps/api/src/routes/v2/alerts.http.test.ts index acdfc4966..565e023fb 100644 --- a/apps/api/src/routes/v2/alerts.http.test.ts +++ b/apps/api/src/routes/v2/alerts.http.test.ts @@ -20,7 +20,12 @@ import { HazelOAuthService } from "../../services/HazelOAuthService" import { OrgMembersService } from "../../services/OrgMembersService" import { QueryEngineService } from "../../services/QueryEngineService" import { V2SchemaErrorsLive } from "./error-envelope" -import { AllV2GroupLayersLive, ConfigResourceServiceStubsLayer } from "./v2-test-support" +import { + AllV2GroupLayersLive, + ApiV2RateLimiterAllowAllLayer, + ConfigResourceServiceStubsLayer, + TelemetryServiceStubsLayer, +} from "./v2-test-support" const createdDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(createdDbs)) @@ -106,8 +111,10 @@ const makeHarness = (warehouseService: WarehouseQueryServiceShape = warehouseStu const routes = HttpApiBuilder.layer(MapleApiV2).pipe( Layer.provide(AllV2GroupLayersLive), Layer.provide(ConfigResourceServiceStubsLayer), + Layer.provide(TelemetryServiceStubsLayer), Layer.provide(V2SchemaErrorsLive), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) const { handler, dispose: disposeHandler } = HttpRouter.toWebHandler(routes, { diff --git a/apps/api/src/routes/v2/api-keys.http.test.ts b/apps/api/src/routes/v2/api-keys.http.test.ts index e08b9a47d..57ce20743 100644 --- a/apps/api/src/routes/v2/api-keys.http.test.ts +++ b/apps/api/src/routes/v2/api-keys.http.test.ts @@ -4,17 +4,20 @@ import { HttpRouter } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" import { OrgId, UserId } from "@maple/domain/http" import { MapleApiV2 } from "@maple/domain/http/v2" +import { API_CORS_OPTIONS } from "../../lib/api-cors" import { Env } from "../../lib/Env" import { cleanupTestDbs, createTestDb, type TestDb } from "../../lib/test-pglite" import { ApiKeysService } from "../../services/ApiKeysService" import { AuthService } from "../../services/AuthService" import { DashboardPersistenceService } from "../../services/DashboardPersistenceService" import { ApiAuthorizationV2Layer } from "../../services/ApiAuthorizationV2Layer" +import { ApiV2RateLimiter, type ApiV2RateLimiterShape } from "../../services/ApiV2RateLimiter" import { V2SchemaErrorsLive } from "./error-envelope" import { AlertsServiceStubLayer, AllV2GroupLayersLive, ConfigResourceServiceStubsLayer, + TelemetryServiceStubsLayer, } from "./v2-test-support" /** @@ -42,7 +45,9 @@ const testConfig = () => }), ) -const makeHarness = () => { +const makeHarness = ( + checkRateLimit: ApiV2RateLimiterShape["check"] = () => Effect.succeed("allowed" as const), +) => { const testDb = createTestDb(createdDbs) const envLive = Env.layer.pipe(Layer.provide(testConfig())) const servicesLive = Layer.mergeAll( @@ -56,8 +61,11 @@ const makeHarness = () => { Layer.provide(V2SchemaErrorsLive), Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), + Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(Layer.succeed(ApiV2RateLimiter, { check: checkRateLimit })), Layer.provideMerge(servicesLive), + Layer.provideMerge(HttpRouter.cors(API_CORS_OPTIONS)), ) const { handler, dispose: disposeHandler } = HttpRouter.toWebHandler(routes, { @@ -69,7 +77,7 @@ const makeHarness = () => { const request = async ( method: string, path: string, - options: { token?: string; body?: unknown } = {}, + options: { token?: string; body?: unknown; origin?: string } = {}, ) => { const response = await handler( new Request(`http://maple.test${path}`, { @@ -77,13 +85,18 @@ const makeHarness = () => { headers: { ...(options.token !== undefined ? { authorization: `Bearer ${options.token}` } : {}), ...(options.body !== undefined ? { "content-type": "application/json" } : {}), + ...(options.origin !== undefined ? { origin: options.origin } : {}), }, body: options.body !== undefined ? JSON.stringify(options.body) : undefined, }), Context.empty() as never, ) const text = await response.text() - return { status: response.status, body: text.length > 0 ? JSON.parse(text) : null } + return { + status: response.status, + headers: response.headers, + body: text.length > 0 ? JSON.parse(text) : null, + } } const ORG = Schema.decodeUnknownSync(OrgId)("org_e2e") @@ -100,9 +113,18 @@ const makeHarness = () => { }), ) + const bootstrapSession = () => + runtime.runPromise( + Effect.gen(function* () { + const service = yield* AuthService + return (yield* service.loginSelfHosted("test-root-password")).token + }), + ) + return { request, bootstrapKey, + bootstrapSession, closeDatabase: () => testDb.close(), dispose: async () => { await disposeHandler() @@ -131,10 +153,75 @@ describe("v2 api_keys over HTTP", () => { }) it("rejects missing credentials with a 401 envelope", async () => { - const harness = makeHarness() + let rateLimitChecks = 0 + const harness = makeHarness(() => { + rateLimitChecks += 1 + return Effect.succeed("allowed") + }) const { status, body } = await harness.request("GET", "/v2/api_keys") expect(status).toBe(401) expect(body.error.type).toBe("authentication_error") + expect(rateLimitChecks).toBe(0) + await harness.dispose() + }) + + it("returns the rate-limit envelope and Retry-After before executing the handler", async () => { + const harness = makeHarness(() => Effect.succeed("limited")) + const key = await harness.bootstrapKey() + + // OrganizationService is deliberately an inert stub in this harness. A 429 + // proves authorization stopped before the route handler could execute it. + const response = await harness.request("GET", "/v2/organization", { + token: key.secret, + origin: "https://app.maple.dev", + }) + expect(response.status).toBe(429) + expect(response.body).toEqual({ + error: { + type: "rate_limit_error", + code: "rate_limited", + message: "Too many requests. Retry after 60 seconds.", + }, + }) + expect(response.headers.get("retry-after")).toBe("60") + expect(response.headers.get("access-control-expose-headers")).toContain("Retry-After") + await harness.dispose() + }) + + it("bypasses API-key rate limiting for session tokens", async () => { + let rateLimitChecks = 0 + const harness = makeHarness(() => { + rateLimitChecks += 1 + return Effect.succeed("limited") + }) + const sessionToken = await harness.bootstrapSession() + const response = await harness.request("GET", "/v2/api_keys", { token: sessionToken }) + expect(response.status).toBe(200) + expect(rateLimitChecks).toBe(0) + await harness.dispose() + }) + + it("allows requests when the limiter fails open", async () => { + const harness = makeHarness(() => Effect.succeed("failed_open")) + const key = await harness.bootstrapKey() + const response = await harness.request("GET", "/v2/api_keys", { token: key.secret }) + expect(response.status).toBe(200) + await harness.dispose() + }) + + it("counts valid-key requests that are later rejected for insufficient scope", async () => { + let rateLimitChecks = 0 + const harness = makeHarness(() => { + rateLimitChecks += 1 + return Effect.succeed("allowed") + }) + const readOnly = await harness.bootstrapKey(["api_keys:read"]) + const response = await harness.request("POST", "/v2/api_keys", { + token: readOnly.secret, + body: { name: "nope" }, + }) + expect(response.status).toBe(403) + expect(rateLimitChecks).toBe(1) await harness.dispose() }) diff --git a/apps/api/src/routes/v2/config-resources.http.test.ts b/apps/api/src/routes/v2/config-resources.http.test.ts index fe8d98726..0cf0e15e2 100644 --- a/apps/api/src/routes/v2/config-resources.http.test.ts +++ b/apps/api/src/routes/v2/config-resources.http.test.ts @@ -19,7 +19,13 @@ import { PlanetScaleOAuthService } from "../../services/PlanetScaleOAuthService" import { RecommendationIssueService } from "../../services/RecommendationIssueService" import { ScrapeTargetsService } from "../../services/ScrapeTargetsService" import { V2SchemaErrorsLive } from "./error-envelope" -import { AlertsServiceStubLayer, AllV2GroupLayersLive, Phase1ResourceStubsLayer } from "./v2-test-support" +import { + AlertsServiceStubLayer, + AllV2GroupLayersLive, + ApiV2RateLimiterAllowAllLayer, + Phase1ResourceStubsLayer, + TelemetryServiceStubsLayer, +} from "./v2-test-support" /** * End-to-end HTTP tests for the v2 config-resource bundle (attribute_mappings, @@ -98,9 +104,11 @@ const makeHarness = () => { Layer.provide(V2SchemaErrorsLive), Layer.provide(AlertsServiceStubLayer), Layer.provide(Phase1ResourceStubsLayer), + Layer.provide(TelemetryServiceStubsLayer), // session_replays (in AllV2GroupLayersLive) needs the warehouse at the routes level. Layer.provide(warehouseLive), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/dashboards.http.test.ts b/apps/api/src/routes/v2/dashboards.http.test.ts index 498b11f2a..87dbec423 100644 --- a/apps/api/src/routes/v2/dashboards.http.test.ts +++ b/apps/api/src/routes/v2/dashboards.http.test.ts @@ -14,7 +14,9 @@ import { V2SchemaErrorsLive } from "./error-envelope" import { AlertsServiceStubLayer, AllV2GroupLayersLive, + ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, + TelemetryServiceStubsLayer, } from "./v2-test-support" const createdDbs: TestDb[] = [] @@ -50,7 +52,9 @@ const makeHarness = () => { Layer.provide(V2SchemaErrorsLive), Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), + Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) const { handler, dispose: disposeHandler } = HttpRouter.toWebHandler(routes, { diff --git a/apps/api/src/routes/v2/phase1-resources.http.test.ts b/apps/api/src/routes/v2/phase1-resources.http.test.ts index f7956d2c5..2423ebe89 100644 --- a/apps/api/src/routes/v2/phase1-resources.http.test.ts +++ b/apps/api/src/routes/v2/phase1-resources.http.test.ts @@ -40,7 +40,9 @@ import { V2SchemaErrorsLive } from "./error-envelope" import { AlertsServiceStubLayer, AllV2GroupLayersLive, + ApiV2RateLimiterAllowAllLayer, ConfigResourceServiceStubsLayer, + TelemetryServiceStubsLayer, } from "./v2-test-support" /** @@ -343,7 +345,9 @@ const makeHarness = (warehouseService: WarehouseQueryServiceShape = warehouseStu Layer.provide(V2SchemaErrorsLive), Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), + Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/telemetry.http.test.ts b/apps/api/src/routes/v2/telemetry.http.test.ts new file mode 100644 index 000000000..9f04c3db8 --- /dev/null +++ b/apps/api/src/routes/v2/telemetry.http.test.ts @@ -0,0 +1,473 @@ +import { afterEach, describe, expect, it } from "@effect/vitest" +import { OrgId, UserId, WarehouseQueryError } from "@maple/domain/http" +import { MapleApiV2 } from "@maple/domain/http/v2" +import { QueryEngineExecuteResponse } from "@maple/query-engine" +import { ConfigProvider, Context, Effect, Layer, ManagedRuntime, Option, Schema } from "effect" +import { HttpRouter } from "effect/unstable/http" +import { HttpApiBuilder } from "effect/unstable/httpapi" +import { Env } from "../../lib/Env" +import { cleanupTestDbs, createTestDb, type TestDb } from "../../lib/test-pglite" +import { WarehouseQueryService, type WarehouseQueryServiceShape } from "../../lib/WarehouseQueryService" +import { ApiAuthorizationV2Layer } from "../../services/ApiAuthorizationV2Layer" +import { ApiKeysService } from "../../services/ApiKeysService" +import { AuthService } from "../../services/AuthService" +import { DashboardPersistenceService } from "../../services/DashboardPersistenceService" +import { + QueryEngineService, + type QueryEngineServiceShape, +} from "../../services/QueryEngineService" +import { V2SchemaErrorsLive } from "./error-envelope" +import { + AlertsServiceStubLayer, + AllV2GroupLayersLive, + ApiV2RateLimiterAllowAllLayer, + ConfigResourceServiceStubsLayer, +} from "./v2-test-support" + +const TRACE_ID = "7f3a4b5c6d7e8f901234567890abcdef" +const SPAN_ID = "0123456789abcdef" +const START = "2026-07-15T12:00:00.000Z" +const END = "2026-07-15T13:00:00.000Z" +const windowQuery = `start_time=${encodeURIComponent(START)}&end_time=${encodeURIComponent(END)}` + +const createdDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(createdDbs)) + +const testConfig = () => + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3490", + MCP_PORT: "3491", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 1).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + INTERNAL_SERVICE_TOKEN: "test-internal-token", + }), + ) + +const logRow = { + timestamp: "2026-07-15 12:00:01.123", + severityText: "ERROR", + severityNumber: 17, + serviceName: "api", + body: "checkout failed", + traceId: TRACE_ID, + spanId: SPAN_ID, + recordIdentity: "00112233445566778899AABBCCDDEEFF", + logAttributes: JSON.stringify({ "error.type": "Timeout" }), + resourceAttributes: JSON.stringify({ "service.namespace": "checkout" }), +} + +const hierarchyRow = { + traceId: TRACE_ID, + spanId: SPAN_ID, + parentSpanId: "", + spanName: "GET /checkout", + serviceName: "api", + spanKind: "Server", + durationMs: 42.5, + startTime: "2026-07-15 12:00:00.000", + statusCode: "Error", + statusMessage: "timeout", + spanAttributes: JSON.stringify({ "http.route": "/checkout" }), + resourceAttributes: JSON.stringify({ "deployment.environment.name": "production" }), + relationship: "related", +} + +const rowsForSql = (sql: string): ReadonlyArray> => { + if (sql.includes("FROM trace_list_mv")) { + const rows = [ + { + traceId: TRACE_ID, + startTime: "2026-07-15 12:00:00", + durationMs: 42.5, + rootSpanName: "GET /checkout", + rootSpanKind: "Server", + rootServiceName: "api", + statusCode: "Error", + hasError: 1, + deploymentEnvironment: "production", + serviceNamespace: "checkout", + httpMethod: "GET", + httpRoute: "/checkout", + httpStatusCode: "500", + }, + { + traceId: "6f3a4b5c6d7e8f901234567890abcdef", + startTime: "2026-07-15 11:59:00", + durationMs: 10, + rootSpanName: "GET /health", + rootSpanKind: "Server", + rootServiceName: "api", + statusCode: "Ok", + hasError: 0, + deploymentEnvironment: "production", + serviceNamespace: "checkout", + httpMethod: "GET", + httpRoute: "/health", + httpStatusCode: "200", + }, + ] + return sql.includes("TraceId <") ? rows.slice(1) : rows + } + if (sql.includes("FROM trace_detail_spans") && sql.includes("AS relationship")) return [hierarchyRow] + if (sql.includes("FROM trace_detail_spans") && sql.includes("toJSONString(SpanAttributes)")) { + return [ + { + traceId: TRACE_ID, + spanId: SPAN_ID, + spanAttributes: JSON.stringify({ "http.route": "/checkout", "error.type": "Timeout" }), + resourceAttributes: JSON.stringify({ "service.name": "api" }), + }, + ] + } + if (sql.includes("FROM logs")) return [logRow] + if (sql.includes("FROM metric_catalog")) { + return [ + { + metricName: "http.server.duration", + metricType: "histogram", + serviceName: "api", + metricDescription: "HTTP server duration", + metricUnit: "ms", + dataPointCount: "42", + firstSeen: "2026-07-15 12:00:00", + lastSeen: "2026-07-15 12:59:00", + isMonotonic: "0", + }, + ] + } + if (sql.includes("FROM service_overview_spans")) { + return [ + { + serviceName: "api", + serviceNamespaces: ["checkout"], + deploymentEnvironments: ["production"], + spanCount: "10", + errorCount: "2", + estimatedErrorCount: "4", + estimatedSpanCount: "20", + p50LatencyMs: "10", + p95LatencyMs: "40", + p99LatencyMs: "50", + }, + ] + } + if (sql.includes("service_map_edges_hourly")) { + return [ + { + sourceService: "api", + targetService: "payments", + callCount: "10", + errorCount: "2", + avgDurationMs: "12.5", + p95DurationMs: "30", + estimatedSpanCount: "20", + }, + ] + } + return [] +} + +const warehouseStub: WarehouseQueryServiceShape = { + query: () => Effect.die(new Error("unexpected named query")), + sqlQuery: () => Effect.succeed([{ bucket: "2026-07-15 12:00:00", value: 1 }]), + compiledQuery: (_tenant, compiled) => compiled.decodeRows(rowsForSql(compiled.sql)), + compiledQueryFirst: (_tenant, compiled) => + compiled + .decodeRows(rowsForSql(compiled.sql)) + .pipe(Effect.map((rows) => Option.fromNullishOr(rows[0]))), + ingest: () => Effect.void, + asExecutor: () => { + throw new Error("not used") + }, +} + +const queryEngineStub = { + execute: () => + Effect.succeed( + new QueryEngineExecuteResponse({ + result: { + kind: "timeseries", + source: "metrics", + data: [{ bucket: "2026-07-15 12:00:00", series: { all: 42 } }], + }, + }), + ), + evaluate: () => Effect.die(new Error("not used")), + evaluateRawSql: () => Effect.die(new Error("not used")), + evaluateSeries: () => Effect.die(new Error("not used")), + cachedDirect: (_tenant, _route, _payload, effect) => effect, +} satisfies QueryEngineServiceShape + +const makeHarness = ( + warehouseService: WarehouseQueryServiceShape = warehouseStub, + queryEngineService: QueryEngineServiceShape = queryEngineStub, +) => { + const testDb = createTestDb(createdDbs) + const envLive = Env.layer.pipe(Layer.provide(testConfig())) + const servicesLive = Layer.mergeAll( + ApiKeysService.layer, + AuthService.layer, + DashboardPersistenceService.layer, + ).pipe(Layer.provideMerge(Layer.mergeAll(envLive, testDb.layer))) + const telemetryLive = Layer.mergeAll( + Layer.succeed(WarehouseQueryService, warehouseService), + Layer.succeed(QueryEngineService, queryEngineService), + ) + const routes = HttpApiBuilder.layer(MapleApiV2).pipe( + Layer.provide(AllV2GroupLayersLive), + Layer.provide(telemetryLive), + Layer.provide(V2SchemaErrorsLive), + Layer.provide(AlertsServiceStubLayer), + Layer.provide(ConfigResourceServiceStubsLayer), + Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), + Layer.provideMerge(servicesLive), + ) + const { handler, dispose: disposeHandler } = HttpRouter.toWebHandler(routes, { disableLogger: true }) + const runtime = ManagedRuntime.make(servicesLive) + const org = Schema.decodeUnknownSync(OrgId)("org_telemetry_e2e") + const user = Schema.decodeUnknownSync(UserId)("user_telemetry_e2e") + const bootstrapKey = (scopes?: ReadonlyArray) => + runtime.runPromise( + Effect.gen(function* () { + const service = yield* ApiKeysService + return yield* service.create(org, user, { name: "telemetry-test", scopes }) + }), + ) + const request = async (method: string, path: string, token: string, body?: unknown) => { + const response = await handler( + new Request(`http://maple.test${path}`, { + method, + headers: { + authorization: `Bearer ${token}`, + ...(body ? { "content-type": "application/json" } : {}), + }, + body: body ? JSON.stringify(body) : undefined, + }), + Context.empty() as never, + ) + const text = await response.text() + return { status: response.status, body: text ? JSON.parse(text) : null } + } + return { + bootstrapKey, + request, + dispose: async () => { + await disposeHandler() + await runtime.dispose() + }, + } +} + +describe("v2 telemetry reads over HTTP", () => { + it("serves all trace, log, metric, service, service-map, and query reads", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey() + const bounds = { start_time: START, end_time: END } + + const traces = await harness.request("POST", "/v2/traces/search", key.secret, bounds) + expect(traces.status).toBe(200) + expect(traces.body.data[0]).toMatchObject({ object: "trace", id: TRACE_ID, has_error: true }) + + const trace = await harness.request( + "GET", + `/v2/traces/${TRACE_ID}`, + key.secret, + ) + expect(trace.status).toBe(200) + expect(trace.body.spans).toHaveLength(1) + + const span = await harness.request( + "GET", + `/v2/traces/${TRACE_ID}/spans/${SPAN_ID}`, + key.secret, + ) + expect(span.status).toBe(200) + expect(span.body.attributes["error.type"]).toBe("Timeout") + + const logs = await harness.request("POST", "/v2/logs/search", key.secret, bounds) + expect(logs.status).toBe(200) + expect(logs.body.data[0].id.startsWith("log_")).toBe(true) + const log = await harness.request("GET", `/v2/logs/${logs.body.data[0].id}`, key.secret) + expect(log.status).toBe(200) + expect(log.body.trace_id).toBe(TRACE_ID) + + const metrics = await harness.request("GET", `/v2/metrics?${windowQuery}`, key.secret) + expect(metrics.status).toBe(200) + expect(metrics.body.data[0].data_point_count).toBe(42) + + const metricSeries = await harness.request("POST", "/v2/metrics/timeseries", key.secret, { + ...bounds, + metric: "avg", + metric_name: "http.server.duration", + metric_type: "histogram", + }) + expect(metricSeries.status).toBe(200) + expect(metricSeries.body.timeseries[0].series.all).toBe(42) + + const services = await harness.request("GET", `/v2/services?${windowQuery}`, key.secret) + expect(services.status).toBe(200) + expect(services.body.data[0]).toMatchObject({ name: "api", has_sampling: true, span_count: 10 }) + const service = await harness.request("GET", `/v2/services/api?${windowQuery}`, key.secret) + expect(service.status).toBe(200) + + const serviceMap = await harness.request("GET", `/v2/service_map?${windowQuery}`, key.secret) + expect(serviceMap.status).toBe(200) + expect(serviceMap.body.edges[0]).toMatchObject({ source_service: "api", target_service: "payments" }) + + const structured = await harness.request("POST", "/v2/query", key.secret, { + ...bounds, + query: { + kind: "timeseries", + source: "metrics", + metric: "avg", + filters: { metric_name: "http.server.duration", metric_type: "histogram" }, + }, + }) + expect(structured.status).toBe(200) + + await harness.dispose() + }) + + it("requires windows, validates log IDs, and treats telemetry POSTs as scoped reads", async () => { + const harness = makeHarness() + const tracesKey = await harness.bootstrapKey(["traces:read"]) + const allowed = await harness.request("POST", "/v2/traces/search", tracesKey.secret, { + start_time: START, + end_time: END, + }) + expect(allowed.status).toBe(200) + const denied = await harness.request("POST", "/v2/logs/search", tracesKey.secret, { + start_time: START, + end_time: END, + }) + expect(denied.status).toBe(403) + const missingWindow = await harness.request("POST", "/v2/traces/search", tracesKey.secret, {}) + expect(missingWindow.status).toBe(400) + const firstPage = await harness.request("POST", "/v2/traces/search", tracesKey.secret, { + start_time: START, + end_time: END, + limit: 1, + }) + expect(firstPage.body.has_more).toBe(true) + const secondPage = await harness.request("POST", "/v2/traces/search", tracesKey.secret, { + start_time: START, + end_time: END, + limit: 1, + cursor: firstPage.body.next_cursor, + }) + expect(secondPage.body.data[0].id).toBe("6f3a4b5c6d7e8f901234567890abcdef") + const badCursor = await harness.request("POST", "/v2/traces/search", tracesKey.secret, { + start_time: START, + end_time: END, + cursor: "garbage", + }) + expect(badCursor.status).toBe(400) + const oversizedWindow = await harness.request("POST", "/v2/traces/search", tracesKey.secret, { + start_time: "2026-01-01T00:00:00.000Z", + end_time: "2026-02-02T00:00:00.000Z", + }) + expect(oversizedWindow.status).toBe(400) + expect(oversizedWindow.body.error.code).toBe("time_range_too_large") + const rootKey = await harness.bootstrapKey() + const malformedLog = await harness.request("GET", "/v2/logs/log_bad", rootKey.secret) + expect(malformedLog.status).toBe(400) + const rawSql = await harness.request("POST", "/v2/query", rootKey.secret, { + start_time: START, + end_time: END, + query: { kind: "raw_sql", sql: "SELECT * FROM traces WHERE $__orgFilter" }, + }) + expect(rawSql.status).toBe(400) + await harness.dispose() + }) + + it("preserves fractional bounds and reads complete traces by their sorting-key identity", async () => { + const observedSql: string[] = [] + const observingWarehouse: WarehouseQueryServiceShape = { + ...warehouseStub, + compiledQuery: (tenant, compiled, options) => { + observedSql.push(compiled.sql) + return warehouseStub.compiledQuery(tenant, compiled, options) + }, + compiledQueryFirst: (tenant, compiled, options) => { + observedSql.push(compiled.sql) + return warehouseStub.compiledQueryFirst(tenant, compiled, options) + }, + } + const harness = makeHarness(observingWarehouse) + const key = await harness.bootstrapKey() + + const logs = await harness.request("POST", "/v2/logs/search", key.secret, { + start_time: "2026-07-15T12:00:00.900Z", + end_time: "2026-07-15T12:00:01.100Z", + }) + expect(logs.status).toBe(200) + const logSql = observedSql.find((sql) => sql.includes("FROM logs")) + expect(logSql).toContain("'2026-07-15 12:00:00.900'") + expect(logSql).toContain("'2026-07-15 12:00:01.100'") + + observedSql.length = 0 + const trace = await harness.request("GET", `/v2/traces/${TRACE_ID}`, key.secret) + expect(trace.status).toBe(200) + const hierarchySql = observedSql.find((sql) => sql.includes("FROM trace_detail_spans")) + expect(hierarchySql).toContain(`TraceId = '${TRACE_ID}'`) + expect(hierarchySql).not.toContain("Timestamp >=") + expect(hierarchySql).not.toContain("Timestamp <=") + await harness.dispose() + }) + + it("maps public trace attribute grouping onto the validated internal query", async () => { + let observedRequest: QueryEngineExecuteRequest | undefined + const queryEngine: QueryEngineServiceShape = { + ...queryEngineStub, + execute: (_tenant, request) => { + observedRequest = request + return queryEngineStub.execute() + }, + } + const harness = makeHarness(warehouseStub, queryEngine) + const key = await harness.bootstrapKey() + const response = await harness.request("POST", "/v2/query", key.secret, { + start_time: START, + end_time: END, + query: { + kind: "timeseries", + source: "traces", + metric: "count", + group_by: ["attribute"], + filters: { group_by_attribute_keys: ["http.route"] }, + }, + }) + expect(response.status).toBe(200) + expect(observedRequest?.query).toMatchObject({ + filters: { groupByAttributeKeys: ["http.route"] }, + }) + await harness.dispose() + }) + + it("sanitizes warehouse failures as operation-specific 503 errors", async () => { + const failure = new WarehouseQueryError({ + message: "SECRET_CLICKHOUSE_DIAGNOSTIC", + pipeName: "traceSummaries", + }) + const harness = makeHarness({ + ...warehouseStub, + compiledQuery: () => Effect.fail(failure), + }) + const key = await harness.bootstrapKey(["traces:read"]) + const response = await harness.request("POST", "/v2/traces/search", key.secret, { + start_time: START, + end_time: END, + }) + expect(response.status).toBe(503) + expect(response.body.error.code).toBe("trace_search_unavailable") + expect(JSON.stringify(response.body)).not.toContain("SECRET_CLICKHOUSE_DIAGNOSTIC") + await harness.dispose() + }) +}) diff --git a/apps/api/src/routes/v2/telemetry.http.ts b/apps/api/src/routes/v2/telemetry.http.ts new file mode 100644 index 000000000..5f3bdaae7 --- /dev/null +++ b/apps/api/src/routes/v2/telemetry.http.ts @@ -0,0 +1,834 @@ +import { HttpApiBuilder } from "effect/unstable/httpapi" +import { CurrentTenant, MetricName, ServiceName, SpanId, TraceId } from "@maple/domain/http" +import { + MapleApiV2, + dependencyUnavailable, + invalidRequest, + paginateOffsetQuery, + resourceNotFound, + timestamp, + type Timestamp, + type V2Log, + type V2Metric, + type V2Service, + type V2ServiceMapEdge, + type V2Span, + type V2StructuredQueryResult, + type V2TraceSummary, +} from "@maple/domain/http/v2" +import { CH, QueryEngineExecuteRequest } from "@maple/query-engine" +import { MAX_QUERY_RANGE_SECONDS } from "@maple/query-engine/runtime" +import { Effect, Encoding, Option, Result, Schema } from "effect" +import { WarehouseQueryService } from "../../lib/WarehouseQueryService" +import { QueryEngineService } from "../../services/QueryEngineService" + +const decodeTraceId = Schema.decodeSync(TraceId) +const decodeSpanId = Schema.decodeSync(SpanId) +const decodeServiceName = Schema.decodeSync(ServiceName) +const decodeMetricName = Schema.decodeSync(MetricName) + +const metricCatalogRowSchema = Schema.Struct({ + metricName: Schema.String, + metricType: Schema.String, + serviceName: Schema.String, + metricDescription: Schema.String, + metricUnit: Schema.String, + dataPointCount: CH.CHNumber, + firstSeen: Schema.String, + lastSeen: Schema.String, + isMonotonic: Schema.Union([Schema.Boolean, CH.CHNumber]), +}) + +const serviceCatalogRowSchema = Schema.Struct({ + serviceName: Schema.String, + serviceNamespaces: Schema.Array(Schema.String), + deploymentEnvironments: Schema.Array(Schema.String), + spanCount: CH.CHNumber, + errorCount: CH.CHNumber, + estimatedErrorCount: CH.CHNumber, + estimatedSpanCount: CH.CHNumber, + p50LatencyMs: CH.CHNumber, + p95LatencyMs: CH.CHNumber, + p99LatencyMs: CH.CHNumber, +}) + +const PARTITION_HINT_RADIUS_MS = 60 * 60 * 1000 + +const mapWarehouseError = (operation: string) => () => dependencyUnavailable(`${operation}_unavailable`) + +const toWarehouseDateTime = (value: string, param: string) => { + const ms = Date.parse(value) + return Number.isNaN(ms) + ? Effect.fail(invalidRequest("parameter_invalid", `Invalid ISO-8601 timestamp for ${param}.`, param)) + : Effect.succeed(new Date(ms).toISOString().replace("T", " ").replace(/Z$/, "")) +} + +const parseWindow = (start: string, end: string) => + Effect.gen(function* () { + const startMs = Date.parse(start) + const endMs = Date.parse(end) + if (Number.isNaN(startMs) || Number.isNaN(endMs) || endMs <= startMs) { + return yield* Effect.fail( + invalidRequest("time_range_invalid", "end_time must be later than start_time.", "end_time"), + ) + } + const rangeSeconds = (endMs - startMs) / 1000 + if (rangeSeconds > MAX_QUERY_RANGE_SECONDS) { + return yield* Effect.fail( + invalidRequest( + "time_range_too_large", + "Telemetry queries support a maximum time range of 31 days.", + "start_time", + ), + ) + } + return { + startTime: yield* toWarehouseDateTime(start, "start_time"), + endTime: yield* toWarehouseDateTime(end, "end_time"), + rangeSeconds, + } + }) + +const chToIso = (value: string): Timestamp => { + const normalized = value.includes("T") ? value : value.replace(" ", "T") + const zoned = /[zZ]|[+-]\d\d:?\d\d$/.test(normalized) ? normalized : `${normalized}Z` + const ms = Date.parse(zoned) + return timestamp(Number.isNaN(ms) ? value : new Date(ms).toISOString()) +} + +const warehouseDate = (ms: number) => + new Date(ms).toISOString().replace("T", " ").replace(/Z$/, "") +const partitionWindow = (value: string) => { + const ms = Date.parse(value.includes("T") ? value : `${value.replace(" ", "T")}Z`) + return { + startTime: warehouseDate(ms - PARTITION_HINT_RADIUS_MS), + endTime: warehouseDate(ms + PARTITION_HINT_RADIUS_MS), + } +} + +const parseStringRecord = (value: unknown): Record => { + if (typeof value !== "string") return {} + try { + const parsed = JSON.parse(value) as unknown + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return {} + return Object.fromEntries( + Object.entries(parsed).map(([key, entry]) => [ + key, + typeof entry === "string" ? entry : String(entry), + ]), + ) + } catch { + return {} + } +} + +type LogKey = readonly [timestamp: string, recordIdentity: string] +const compactTimestamp = (value: string) => { + const match = /^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(\.\d+)?$/.exec(value) + if (match === null) return value + const epochSeconds = Date.parse(`${match[1]}T${match[2]}Z`) / 1000 + return Number.isSafeInteger(epochSeconds) + ? `~${epochSeconds.toString(36)}${match[3] ?? ""}` + : value +} +const expandTimestamp = (value: string) => { + const match = /^~([0-9a-z]+)(\.\d+)?$/.exec(value) + if (match === null) return value + const epochSeconds = Number.parseInt(match[1]!, 36) + if (!Number.isSafeInteger(epochSeconds)) return value + const seconds = new Date(epochSeconds * 1000).toISOString().slice(0, 19).replace("T", " ") + return `${seconds}${match[2] ?? ""}` +} +const compactHexId = (value: string) => { + if (!/^(?:[0-9a-f]{16}|[0-9a-f]{32})$/i.test(value)) return value + const bytes = Uint8Array.from({ length: value.length / 2 }, (_, index) => + Number.parseInt(value.slice(index * 2, index * 2 + 2), 16), + ) + return `~${Encoding.encodeBase64Url(bytes)}` +} +const expandHexId = (value: string) => { + if (!value.startsWith("~")) return value + const decoded = Encoding.decodeBase64Url(value.slice(1)) + if (Result.isFailure(decoded)) throw new Error("invalid compact identifier") + return [...decoded.success].map((byte) => byte.toString(16).padStart(2, "0")).join("") +} +const logKey = (row: { timestamp: string; recordIdentity: string }) => + JSON.stringify([ + compactTimestamp(row.timestamp), + compactHexId(row.recordIdentity), + ] satisfies LogKey) + +const parseLogKey = (value: string) => { + try { + const parsed = JSON.parse(value) as unknown + if ( + !Array.isArray(parsed) || + parsed.length !== 2 || + parsed.some((part) => typeof part !== "string") || + Number.isNaN(Date.parse(expandTimestamp(parsed[0] as string).replace(" ", "T") + "Z")) || + !/^[0-9A-F]{32}$/i.test(expandHexId(parsed[1] as string)) + ) { + throw new Error("invalid") + } + return Effect.succeed([ + expandTimestamp(parsed[0] as string), + expandHexId(parsed[1] as string).toUpperCase(), + ] as const) + } catch { + return Effect.fail(invalidRequest("log_id_invalid", "Malformed log ID.", "id")) + } +} + +const encodeKeysetCursor = (prefix: string, parts: ReadonlyArray) => + `${prefix}_${Encoding.encodeBase64Url(JSON.stringify(parts))}` + +const decodeKeysetCursor = (value: string | undefined, prefix: string, length: number) => { + if (value === undefined) return Effect.succeed | undefined>(undefined) + if (!value.startsWith(`${prefix}_`)) { + return Effect.fail(invalidRequest("parameter_invalid", "Invalid pagination cursor.", "cursor")) + } + const decoded = Encoding.decodeBase64UrlString(value.slice(prefix.length + 1)) + if (Result.isFailure(decoded)) { + return Effect.fail(invalidRequest("parameter_invalid", "Invalid pagination cursor.", "cursor")) + } + try { + const parts = JSON.parse(decoded.success) as unknown + return Array.isArray(parts) && + parts.length === length && + parts.every((part) => typeof part === "string") + ? Effect.succeed(parts as ReadonlyArray) + : Effect.fail(invalidRequest("parameter_invalid", "Invalid pagination cursor.", "cursor")) + } catch { + return Effect.fail(invalidRequest("parameter_invalid", "Invalid pagination cursor.", "cursor")) + } +} + +const toLog = (row: { + timestamp: string + severityText: string + severityNumber: number + serviceName: string + body: string + traceId: string + spanId: string + recordIdentity: string + logAttributes: string + resourceAttributes: string +}): V2Log => ({ + id: logKey(row), + object: "log", + timestamp: chToIso(row.timestamp), + severity_text: row.severityText, + severity_number: Number(row.severityNumber), + service_name: decodeServiceName(row.serviceName), + body: row.body, + trace_id: row.traceId ? decodeTraceId(row.traceId) : null, + span_id: row.spanId ? decodeSpanId(row.spanId) : null, + log_attributes: parseStringRecord(row.logAttributes), + resource_attributes: parseStringRecord(row.resourceAttributes), +}) + +const toTraceSummary = (row: { + traceId: string + startTime: string + durationMs: number + rootSpanName: string + rootSpanKind: string + rootServiceName: string + statusCode: string + hasError: number + deploymentEnvironment: string + serviceNamespace: string + httpMethod: string + httpRoute: string + httpStatusCode: string +}): V2TraceSummary => ({ + id: decodeTraceId(row.traceId), + object: "trace", + start_time: chToIso(row.startTime), + duration_ms: Number(row.durationMs), + root_span_name: row.rootSpanName, + root_span_kind: row.rootSpanKind, + root_service_name: row.rootServiceName, + status_code: row.statusCode, + has_error: Number(row.hasError) !== 0, + deployment_environment: row.deploymentEnvironment || null, + service_namespace: row.serviceNamespace || null, + http_method: row.httpMethod || null, + http_route: row.httpRoute || null, + http_status_code: row.httpStatusCode || null, +}) + +const toSpan = (row: { + traceId: string + spanId: string + parentSpanId: string + spanName: string + serviceName: string + spanKind: string + durationMs: number + startTime: string + statusCode: string + statusMessage: string + spanAttributes: string + resourceAttributes: string +}): V2Span => ({ + id: decodeSpanId(row.spanId), + object: "span", + trace_id: decodeTraceId(row.traceId), + parent_span_id: row.parentSpanId ? decodeSpanId(row.parentSpanId) : null, + name: row.spanName, + service_name: row.serviceName, + kind: row.spanKind, + start_time: chToIso(row.startTime), + duration_ms: Number(row.durationMs), + status_code: row.statusCode, + status_message: row.statusMessage || null, + attributes: parseStringRecord(row.spanAttributes), + resource_attributes: parseStringRecord(row.resourceAttributes), +}) + +const attributeFilters = ( + filters: + | ReadonlyArray<{ + key: string + value?: string + mode: string + negated?: boolean + }> + | undefined, +) => + filters?.map((filter) => ({ + key: filter.key, + ...(filter.value !== undefined ? { value: filter.value } : {}), + mode: filter.mode, + ...(filter.negated !== undefined ? { negated: filter.negated } : {}), + })) + +const toInternalQuery = (query: Record): Record => { + const mapped: Record = { + kind: query.kind, + source: query.source, + metric: query.metric, + groupBy: query.group_by, + bucketSeconds: query.bucket_seconds, + seriesLimit: query.series_limit, + limit: query.limit, + apdexThresholdMs: query.apdex_threshold_ms, + } + const filters = query.filters + if (filters) { + mapped.filters = { + serviceName: filters.service_name, + spanName: filters.span_name, + rootSpansOnly: filters.root_spans_only, + errorsOnly: filters.errors_only, + environments: filters.environments, + namespaces: filters.namespaces, + minDurationMs: filters.min_duration_ms, + maxDurationMs: filters.max_duration_ms, + severity: filters.severity, + traceId: filters.trace_id, + search: filters.search, + metricName: filters.metric_name, + metricType: filters.metric_type, + groupByAttributeKey: filters.group_by_attribute_key, + groupByAttributeKeys: filters.group_by_attribute_keys, + groupByResourceAttributeKey: filters.group_by_resource_attribute_key, + attributeFilters: attributeFilters(filters.attribute_filters), + resourceAttributeFilters: attributeFilters(filters.resource_attribute_filters), + } + } + return Object.fromEntries(Object.entries(mapped).filter(([, value]) => value !== undefined)) +} + +const structuredResult = (result: { + kind: "timeseries" | "breakdown" + source: "traces" | "logs" | "metrics" + data: ReadonlyArray +}): V2StructuredQueryResult => ({ + object: "query_result", + kind: result.kind, + source: result.source, + timeseries: + result.kind === "timeseries" + ? (result.data as ReadonlyArray<{ bucket: string; series: Record }>).map( + (point) => ({ bucket: chToIso(point.bucket), series: point.series }), + ) + : [], + breakdown: result.kind === "breakdown" ? (result.data as V2StructuredQueryResult["breakdown"]) : [], +}) + +const queryError = (error: unknown) => { + const tag = typeof error === "object" && error !== null && "_tag" in error ? String(error._tag) : "" + return tag.includes("Validation") + ? invalidRequest("query_invalid", "The query specification is invalid.", "query") + : dependencyUnavailable("query_unavailable") +} + +const decodeQueryEngineRequest = (input: unknown) => + Schema.decodeUnknownEffect(QueryEngineExecuteRequest)(input).pipe( + Effect.mapError(() => invalidRequest("query_invalid", "The query specification is invalid.", "query")), + ) + +export const HttpV2TracesLive = HttpApiBuilder.group(MapleApiV2, "traces", (handlers) => + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + + const hierarchy = Effect.fn("HttpV2Traces.hierarchy")(function* ( + tenant: CurrentTenant.TenantSchema, + traceId: string, + ) { + const compiled = CH.compile(CH.spanHierarchyQuery({ traceId }), { + orgId: tenant.orgId, + }) + return yield* warehouse + .compiledQuery(tenant, compiled, { + profile: "list", + context: "v2GetTrace", + }) + .pipe(Effect.mapError(mapWarehouseError("trace_query"))) + }) + + return handlers + .handle("search", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(payload.start_time, payload.end_time) + const limit = payload.limit ?? 20 + const cursorParts = yield* decodeKeysetCursor(payload.cursor, "trc", 2) + const compiled = CH.compile( + CH.traceSummariesQuery({ + serviceName: payload.service_name, + spanName: payload.span_name, + hasError: payload.has_error, + minDurationMs: payload.min_duration_ms, + maxDurationMs: payload.max_duration_ms, + httpMethod: payload.http_method, + httpStatusCode: payload.http_status_code, + deploymentEnv: payload.deployment_environment, + namespace: payload.service_namespace, + limit: limit + 1, + cursor: cursorParts + ? { timestamp: cursorParts[0]!, traceId: cursorParts[1]! } + : undefined, + }), + { orgId: tenant.orgId, ...window }, + ) + const rows = yield* warehouse + .compiledQuery(tenant, compiled, { profile: "list", context: "v2TraceSearch" }) + .pipe(Effect.mapError(mapWarehouseError("trace_search"))) + const dataRows = rows.slice(0, limit) + const last = dataRows.at(-1) + const hasMore = rows.length > limit + return { + object: "list" as const, + data: dataRows.map(toTraceSummary), + has_more: hasMore, + next_cursor: + hasMore && last + ? encodeKeysetCursor("trc", [last.startTime, last.traceId]) + : null, + } + }), + ) + .handle("retrieve", ({ params }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* hierarchy(tenant, params.trace_id) + if (rows.length === 0) return yield* resourceNotFound("trace", "No such trace.") + const spans = rows.map(toSpan) + const startMs = Math.min(...spans.map((span) => Date.parse(span.start_time))) + const endMs = Math.max( + ...spans.map((span) => Date.parse(span.start_time) + span.duration_ms), + ) + return { + id: params.trace_id, + object: "trace" as const, + start_time: timestamp(new Date(startMs).toISOString()), + end_time: timestamp(new Date(endMs).toISOString()), + duration_ms: endMs - startMs, + span_count: spans.length, + service_count: new Set(spans.map((span) => span.service_name)).size, + spans, + } + }), + ) + .handle("retrieveSpan", ({ params }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* hierarchy(tenant, params.trace_id) + const hierarchyRow = rows.find((row) => row.spanId === params.span_id) + if (!hierarchyRow) return yield* resourceNotFound("span", "No such span.") + const detail = yield* warehouse + .compiledQueryFirst( + tenant, + CH.compile( + CH.spanDetailQuery({ + traceId: params.trace_id, + spanId: params.span_id, + }), + { orgId: tenant.orgId }, + ), + { profile: "discovery", context: "v2GetSpan" }, + ) + .pipe( + Effect.mapError(mapWarehouseError("span_query")), + Effect.map(Option.getOrNull), + ) + return toSpan({ + ...hierarchyRow, + spanAttributes: detail?.spanAttributes ?? hierarchyRow.spanAttributes, + resourceAttributes: detail?.resourceAttributes ?? hierarchyRow.resourceAttributes, + }) + }), + ) + }), +) + +export const HttpV2LogsLive = HttpApiBuilder.group(MapleApiV2, "logs", (handlers) => + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + return handlers + .handle("search", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(payload.start_time, payload.end_time) + const limit = payload.limit ?? 20 + const cursorParts = yield* decodeKeysetCursor(payload.cursor, "log", 5) + const compiled = CH.compile( + CH.logsListQuery({ + serviceName: payload.service_name, + severity: payload.severity, + minSeverity: payload.min_severity, + traceId: payload.trace_id, + spanId: payload.span_id, + search: payload.search, + environments: payload.deployment_environment + ? [payload.deployment_environment] + : undefined, + namespaces: payload.service_namespace ? [payload.service_namespace] : undefined, + limit: limit + 1, + cursorIdentity: cursorParts + ? { + timestamp: cursorParts[0]!, + serviceName: cursorParts[1]!, + traceId: cursorParts[2]!, + spanId: cursorParts[3]!, + recordIdentity: cursorParts[4]!, + } + : undefined, + }), + { orgId: tenant.orgId, ...window }, + ) + const rows = yield* warehouse + .compiledQuery(tenant, compiled, { profile: "list", context: "v2LogSearch" }) + .pipe(Effect.mapError(mapWarehouseError("log_search"))) + const dataRows = rows.slice(0, limit) + const last = dataRows.at(-1) + const hasMore = rows.length > limit + return { + object: "list" as const, + data: dataRows.map(toLog), + has_more: hasMore, + next_cursor: + hasMore && last + ? encodeKeysetCursor("log", [ + last.timestamp, + last.serviceName, + last.traceId, + last.spanId, + last.recordIdentity, + ]) + : null, + } + }), + ) + .handle("retrieve", ({ params }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const [logTimestamp, recordIdentity] = yield* parseLogKey(params.id) + const compiled = CH.compile( + CH.getLogByKeyQuery({ + recordIdentity, + }), + { + orgId: tenant.orgId, + ...partitionWindow(logTimestamp), + timestamp: logTimestamp, + }, + ) + const row = yield* warehouse + .compiledQueryFirst(tenant, compiled, { + profile: "list", + context: "v2GetLog", + }) + .pipe(Effect.mapError(mapWarehouseError("log_query")), Effect.map(Option.getOrNull)) + if (!row) return yield* resourceNotFound("log", "No such log.") + return toLog(row) + }), + ) + }), +) + +export const HttpV2MetricsLive = HttpApiBuilder.group(MapleApiV2, "metrics", (handlers) => + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + const queryEngine = yield* QueryEngineService + return handlers + .handle("list", ({ query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(query.start_time, query.end_time) + const page = yield* paginateOffsetQuery(query, ({ limit, offset }) => { + const compiled = CH.compile( + CH.listMetricsQuery({ + serviceName: query.service_name, + metricType: query.metric_type, + search: query.search, + limit, + offset, + }), + { orgId: tenant.orgId, ...window }, + { rowSchema: metricCatalogRowSchema }, + ) + return warehouse + .compiledQuery(tenant, compiled, { + profile: "discovery", + context: "v2ListMetrics", + }) + .pipe( + Effect.mapError(mapWarehouseError("metric_catalog")), + Effect.map( + (rows): ReadonlyArray => + rows.map((row) => ({ + object: "metric", + name: decodeMetricName(row.metricName), + type: row.metricType, + service_name: row.serviceName, + description: row.metricDescription, + unit: row.metricUnit, + is_monotonic: Number(row.isMonotonic) !== 0, + data_point_count: Number(row.dataPointCount), + first_seen: chToIso(row.firstSeen), + last_seen: chToIso(row.lastSeen), + })), + ), + ) + }) + return { object: "list" as const, ...page } + }), + ) + .handle("timeseries", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(payload.start_time, payload.end_time) + const request = yield* decodeQueryEngineRequest({ + startTime: window.startTime, + endTime: window.endTime, + query: { + kind: "timeseries", + source: "metrics", + metric: payload.metric, + groupBy: payload.group_by, + bucketSeconds: payload.bucket_seconds, + filters: { + metricName: payload.metric_name, + metricType: payload.metric_type, + serviceName: payload.service_name, + groupByAttributeKey: payload.group_by_attribute_key, + groupByResourceAttributeKey: payload.group_by_resource_attribute_key, + attributeFilters: attributeFilters(payload.attribute_filters), + resourceAttributeFilters: attributeFilters( + payload.resource_attribute_filters, + ), + }, + }, + }) + const response = yield* queryEngine + .execute(tenant, request) + .pipe(Effect.mapError(queryError)) + if (response.result.kind !== "timeseries") + return yield* Effect.fail(dependencyUnavailable("metric_query_unavailable")) + return structuredResult(response.result) + }), + ) + }), +) + +const toService = ( + row: { + serviceName: string + serviceNamespaces: readonly string[] + deploymentEnvironments: readonly string[] + spanCount: number + errorCount: number + estimatedErrorCount: number + estimatedSpanCount: number + p50LatencyMs: number + p95LatencyMs: number + p99LatencyMs: number + }, + rangeSeconds: number, +): V2Service => { + const spanCount = Number(row.spanCount) + const estimatedSpanCount = Number(row.estimatedSpanCount) + const estimatedErrorCount = Number(row.estimatedErrorCount) + return { + object: "service", + name: decodeServiceName(row.serviceName), + service_namespaces: [...row.serviceNamespaces], + deployment_environments: [...row.deploymentEnvironments], + throughput: estimatedSpanCount / rangeSeconds, + traced_throughput: spanCount / rangeSeconds, + span_count: spanCount, + error_count: Number(row.errorCount), + error_rate: estimatedSpanCount > 0 ? estimatedErrorCount / estimatedSpanCount : 0, + p50_latency_ms: Number(row.p50LatencyMs), + p95_latency_ms: Number(row.p95LatencyMs), + p99_latency_ms: Number(row.p99LatencyMs), + has_sampling: estimatedSpanCount > spanCount + 0.001, + sampling_weight: spanCount > 0 ? estimatedSpanCount / spanCount : 1, + } +} + +export const HttpV2ServicesLive = HttpApiBuilder.group(MapleApiV2, "services", (handlers) => + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + const execute = ( + tenant: CurrentTenant.TenantSchema, + window: { startTime: string; endTime: string; rangeSeconds: number }, + opts: Parameters[0], + ) => { + const compiled = CH.compile( + CH.serviceCatalogQuery(opts), + { orgId: tenant.orgId, ...window }, + { rowSchema: serviceCatalogRowSchema }, + ) + return warehouse + .compiledQuery(tenant, compiled, { + profile: "aggregation", + context: "v2ServiceCatalog", + }) + .pipe( + Effect.mapError(mapWarehouseError("service_query")), + Effect.map((rows) => rows.map((row) => toService(row, window.rangeSeconds))), + ) + } + return handlers + .handle("list", ({ query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(query.start_time, query.end_time) + const page = yield* paginateOffsetQuery(query, ({ limit, offset }) => + execute(tenant, window, { + deploymentEnvironment: query.deployment_environment, + serviceNamespace: query.service_namespace, + limit, + offset, + }), + ) + return { object: "list" as const, ...page } + }), + ) + .handle("retrieve", ({ params, query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(query.start_time, query.end_time) + const rows = yield* execute(tenant, window, { + serviceName: params.name, + limit: 1, + }) + if (!rows[0]) return yield* resourceNotFound("service", "No such service.") + return rows[0] + }), + ) + }), +) + +const toMapEdge = (row: { + sourceService: string + targetService: string + callCount: number + errorCount: number + avgDurationMs: number + p95DurationMs: number + estimatedSpanCount: number +}): V2ServiceMapEdge => { + const calls = Number(row.callCount) + const estimated = Number(row.estimatedSpanCount) + const errors = Number(row.errorCount) + return { + object: "service_map.edge", + source_service: row.sourceService, + target_service: row.targetService, + call_count: calls, + estimated_call_count: estimated, + error_count: errors, + error_rate: calls > 0 ? errors / calls : 0, + avg_duration_ms: Number(row.avgDurationMs), + p95_duration_ms: Number(row.p95DurationMs), + has_sampling: estimated > calls + 0.001, + sampling_weight: calls > 0 ? estimated / calls : 1, + } +} + +export const HttpV2ServiceMapLive = HttpApiBuilder.group(MapleApiV2, "serviceMap", (handlers) => + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + return handlers.handle("retrieve", ({ query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(query.start_time, query.end_time) + const compiled = query.service_name + ? CH.compile( + CH.serviceDependenciesForServiceQuery({ + serviceName: query.service_name, + deploymentEnv: query.deployment_environment, + }), + { orgId: tenant.orgId, ...window }, + ) + : CH.serviceDependenciesSQL( + { deploymentEnv: query.deployment_environment }, + { orgId: tenant.orgId, ...window }, + ) + const rows = yield* warehouse + .compiledQuery(tenant, compiled, { + profile: "aggregation", + context: "v2ServiceMap", + }) + .pipe(Effect.mapError(mapWarehouseError("service_map_query"))) + return { + object: "service_map" as const, + start_time: timestamp(query.start_time), + end_time: timestamp(query.end_time), + edges: rows.map(toMapEdge), + } + }), + ) + }), +) + +export const HttpV2QueryLive = HttpApiBuilder.group(MapleApiV2, "query", (handlers) => + Effect.gen(function* () { + const queryEngine = yield* QueryEngineService + return handlers.handle("execute", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const window = yield* parseWindow(payload.start_time, payload.end_time) + const request = yield* decodeQueryEngineRequest({ + startTime: window.startTime, + endTime: window.endTime, + query: toInternalQuery(payload.query), + }) + const response = yield* queryEngine + .execute(tenant, request) + .pipe(Effect.mapError(queryError)) + if (response.result.kind !== "timeseries" && response.result.kind !== "breakdown") { + return yield* Effect.fail(dependencyUnavailable("query_unavailable")) + } + return structuredResult(response.result) + }), + ) + }), +) diff --git a/apps/api/src/routes/v2/v2-test-support.ts b/apps/api/src/routes/v2/v2-test-support.ts index 73531a336..4839e494d 100644 --- a/apps/api/src/routes/v2/v2-test-support.ts +++ b/apps/api/src/routes/v2/v2-test-support.ts @@ -8,7 +8,9 @@ import { OrganizationService } from "../../services/OrganizationService" import { OrgIngestKeysService } from "../../services/OrgIngestKeysService" import { RecommendationIssueService } from "../../services/RecommendationIssueService" import { ScrapeTargetsService } from "../../services/ScrapeTargetsService" +import { ApiV2RateLimiter } from "../../services/ApiV2RateLimiter" import { WarehouseQueryService } from "../../lib/WarehouseQueryService" +import { QueryEngineService } from "../../services/QueryEngineService" import { HttpV2AlertDestinationsLive } from "./alert-destinations.http" import { HttpV2AlertIncidentsLive } from "./alert-incidents.http" import { HttpV2AlertRulesLive } from "./alert-rules.http" @@ -22,6 +24,14 @@ import { HttpV2OrganizationLive } from "./organization.http" import { HttpV2InstrumentationRecommendationsLive } from "./recommendations.http" import { HttpV2ScrapeTargetsLive } from "./scrape-targets.http" import { HttpV2SessionReplaysLive } from "./session-replays.http" +import { + HttpV2LogsLive, + HttpV2MetricsLive, + HttpV2QueryLive, + HttpV2ServiceMapLive, + HttpV2ServicesLive, + HttpV2TracesLive, +} from "./telemetry.http" /** * Test-only support for the v2 HTTP harnesses. `HttpApiBuilder.layer(MapleApiV2)` @@ -44,8 +54,18 @@ export const AllV2GroupLayersLive = Layer.mergeAll( HttpV2AnomaliesLive, HttpV2OrganizationLive, HttpV2SessionReplaysLive, + HttpV2TracesLive, + HttpV2LogsLive, + HttpV2MetricsLive, + HttpV2ServicesLive, + HttpV2ServiceMapLive, + HttpV2QueryLive, ) +export const ApiV2RateLimiterAllowAllLayer = Layer.succeed(ApiV2RateLimiter, { + check: () => Effect.succeed("allowed" as const), +}) + const die = () => Effect.die(new Error("AlertsService is not available in this test harness")) /** Synchronous stub for non-Effect-returning service methods (e.g. `asExecutor`). */ @@ -118,6 +138,16 @@ export const WarehouseServiceStubLayer = Layer.succeed(WarehouseQueryService, { asExecutor: dieSync, }) +export const TelemetryServiceStubsLayer = Layer.mergeAll( + Layer.succeed(QueryEngineService, { + execute: die, + evaluate: die, + evaluateRawSql: die, + evaluateSeries: die, + cachedDirect: die, + }), +) + /** Inert config-resource services for harnesses that never touch those groups. */ export const ConfigResourceServiceStubsLayer = Layer.mergeAll( Layer.succeed(IngestAttributeMappingService, { diff --git a/apps/api/src/services/ApiAuthorizationV2Layer.ts b/apps/api/src/services/ApiAuthorizationV2Layer.ts index b9517c448..888ded6c9 100644 --- a/apps/api/src/services/ApiAuthorizationV2Layer.ts +++ b/apps/api/src/services/ApiAuthorizationV2Layer.ts @@ -1,10 +1,11 @@ -import { HttpServerRequest } from "effect/unstable/http" +import { HttpEffect, HttpServerRequest, HttpServerResponse } from "effect/unstable/http" import { CurrentTenant, RoleName } from "@maple/domain/http" import { AuthorizationV2, authenticationError, dependencyUnavailable, permissionError, + rateLimited, requiredScopeForRequest, scopeAllows, } from "@maple/domain/http/v2" @@ -13,6 +14,11 @@ import { ApiKeysService } from "./ApiKeysService" import { makeResolveTenant } from "./AuthService" import { annotateAuthSpan } from "../lib/auth-span" import { Env } from "../lib/Env" +import { + API_V2_RATE_LIMIT_PERIOD_SECONDS, + API_V2_RATE_LIMIT_REQUESTS, + ApiV2RateLimiter, +} from "./ApiV2RateLimiter" const decodeRoleNameSync = Schema.decodeUnknownSync(RoleName) const apiKeyDefaultRoles = [decodeRoleNameSync("root")] as const @@ -42,6 +48,7 @@ export const ApiAuthorizationV2Layer = Layer.effect( Effect.gen(function* () { const env = yield* Env const apiKeys = yield* ApiKeysService + const rateLimiter = yield* ApiV2RateLimiter const resolveTenant = makeResolveTenant(env) return AuthorizationV2.of({ @@ -69,6 +76,26 @@ export const ApiAuthorizationV2Layer = Layer.effect( keyId: resolved.keyId, }) + const rateLimitOutcome = yield* rateLimiter.check(resolved.keyId) + yield* Effect.annotateCurrentSpan({ + "maple.rate_limit.outcome": rateLimitOutcome, + "maple.rate_limit.limit": API_V2_RATE_LIMIT_REQUESTS, + "maple.rate_limit.period_seconds": API_V2_RATE_LIMIT_PERIOD_SECONDS, + }) + + if (rateLimitOutcome === "limited") { + yield* HttpEffect.appendPreResponseHandler((_request, response) => + Effect.succeed( + HttpServerResponse.setHeader( + response, + "Retry-After", + String(API_V2_RATE_LIMIT_PERIOD_SECONDS), + ), + ), + ) + return yield* Effect.fail(rateLimited()) + } + const required = requiredScopeForRequest(request.method, requestPath(request.url)) if (required !== null && !scopeAllows(resolved.scopes, required)) { return yield* Effect.fail( diff --git a/apps/api/src/services/ApiV2RateLimiter.test.ts b/apps/api/src/services/ApiV2RateLimiter.test.ts new file mode 100644 index 000000000..138c7e625 --- /dev/null +++ b/apps/api/src/services/ApiV2RateLimiter.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from "@effect/vitest" +import { ApiKeyId } from "@maple/domain/http" +import { Effect, Layer, Schema } from "effect" +import { WorkerEnvironment } from "../lib/WorkerEnvironment" +import { + API_V2_RATE_LIMIT_BINDING, + API_V2_RATE_LIMIT_PARTITION, + ApiV2RateLimiter, + makeApiV2RateLimitKey, +} from "./ApiV2RateLimiter" + +const KEY_A = Schema.decodeUnknownSync(ApiKeyId)("00000000-0000-4000-8000-000000000001") +const KEY_B = Schema.decodeUnknownSync(ApiKeyId)("00000000-0000-4000-8000-000000000002") + +const limiterLayer = (environment: Record) => + ApiV2RateLimiter.layer.pipe(Layer.provide(Layer.succeed(WorkerEnvironment, environment))) + +describe("ApiV2RateLimiter", () => { + it.effect("uses only the stage partition and internal API-key ID as the counter key", () => { + const keys: string[] = [] + const environment = { + [API_V2_RATE_LIMIT_PARTITION]: "stg", + [API_V2_RATE_LIMIT_BINDING]: { + limit: ({ key }: { key: string }) => { + keys.push(key) + return Promise.resolve({ success: true }) + }, + }, + } + + return Effect.gen(function* () { + const limiter = yield* ApiV2RateLimiter + expect(yield* limiter.check(KEY_A)).toBe("allowed") + expect(yield* limiter.check(KEY_B)).toBe("allowed") + expect(keys).toEqual([makeApiV2RateLimitKey("stg", KEY_A), makeApiV2RateLimitKey("stg", KEY_B)]) + expect(keys.join(" ")).not.toContain("maple_ak_") + }).pipe(Effect.provide(limiterLayer(environment))) + }) + + it.effect("isolates the same key across deployment stages", () => { + const observed: string[] = [] + const binding = { + limit: ({ key }: { key: string }) => { + observed.push(key) + return Promise.resolve({ success: true }) + }, + } + const run = (partition: string) => + Effect.gen(function* () { + const limiter = yield* ApiV2RateLimiter + return yield* limiter.check(KEY_A) + }).pipe( + Effect.provide( + limiterLayer({ + [API_V2_RATE_LIMIT_PARTITION]: partition, + [API_V2_RATE_LIMIT_BINDING]: binding, + }), + ), + ) + + return Effect.gen(function* () { + expect(yield* run("prd")).toBe("allowed") + expect(yield* run("stg")).toBe("allowed") + expect(observed).toEqual([ + makeApiV2RateLimitKey("prd", KEY_A), + makeApiV2RateLimitKey("stg", KEY_A), + ]) + }) + }) + + it.effect("returns limited when Cloudflare denies the key", () => + Effect.gen(function* () { + const limiter = yield* ApiV2RateLimiter + expect(yield* limiter.check(KEY_A)).toBe("limited") + }).pipe( + Effect.provide( + limiterLayer({ + [API_V2_RATE_LIMIT_PARTITION]: "prd", + [API_V2_RATE_LIMIT_BINDING]: { + limit: () => Promise.resolve({ success: false }), + }, + }), + ), + ), + ) + + it.effect("fails open when the binding or partition is unavailable", () => { + const run = (environment: Record) => + Effect.gen(function* () { + const limiter = yield* ApiV2RateLimiter + return yield* limiter.check(KEY_A) + }).pipe(Effect.provide(limiterLayer(environment))) + + return Effect.gen(function* () { + expect(yield* run({ [API_V2_RATE_LIMIT_PARTITION]: "prd" })).toBe("failed_open") + expect( + yield* run({ + [API_V2_RATE_LIMIT_BINDING]: { limit: () => Promise.resolve({ success: true }) }, + }), + ).toBe("failed_open") + }) + }) + + it.effect("fails open when the Cloudflare binding throws", () => + Effect.gen(function* () { + const limiter = yield* ApiV2RateLimiter + expect(yield* limiter.check(KEY_A)).toBe("failed_open") + }).pipe( + Effect.provide( + limiterLayer({ + [API_V2_RATE_LIMIT_PARTITION]: "prd", + [API_V2_RATE_LIMIT_BINDING]: { + limit: () => Promise.reject(new Error("binding unavailable")), + }, + }), + ), + ), + ) +}) diff --git a/apps/api/src/services/ApiV2RateLimiter.ts b/apps/api/src/services/ApiV2RateLimiter.ts new file mode 100644 index 000000000..9a2e397da --- /dev/null +++ b/apps/api/src/services/ApiV2RateLimiter.ts @@ -0,0 +1,92 @@ +import type { ApiKeyId } from "@maple/domain/http" +import { Context, Effect, Layer, Schema } from "effect" +import { WorkerEnvironment } from "../lib/WorkerEnvironment" + +export const API_V2_RATE_LIMIT_BINDING = "API_V2_RATE_LIMITER" +export const API_V2_RATE_LIMIT_PARTITION = "API_V2_RATE_LIMIT_PARTITION" +export const API_V2_RATE_LIMIT_REQUESTS = 600 +export const API_V2_RATE_LIMIT_PERIOD_SECONDS = 60 + +export type ApiV2RateLimitOutcome = "allowed" | "limited" | "failed_open" + +interface RateLimitBinding { + readonly limit: (options: { readonly key: string }) => Promise<{ readonly success: boolean }> +} + +export interface ApiV2RateLimiterShape { + readonly check: (keyId: ApiKeyId) => Effect.Effect +} + +class ApiV2RateLimiterBindingError extends Schema.TaggedErrorClass()( + "@maple/api/services/ApiV2RateLimiterBindingError", + { + message: Schema.String, + cause: Schema.Defect(), + }, +) {} + +const isRateLimitBinding = (value: unknown): value is RateLimitBinding => + typeof value === "object" && + value !== null && + "limit" in value && + typeof (value as { readonly limit?: unknown }).limit === "function" + +const readPartition = (environment: Record): string | undefined => { + const value = environment[API_V2_RATE_LIMIT_PARTITION] + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined +} + +export const makeApiV2RateLimitKey = (partition: string, keyId: ApiKeyId): string => + `${partition}:v2:${keyId}` + +const warnFailedOpen = (reason: "binding_missing" | "partition_missing" | "binding_error", cause?: unknown) => + Effect.logWarning("API v2 rate limiter unavailable; allowing request").pipe( + Effect.annotateLogs({ + "maple.rate_limit.outcome": "failed_open", + "maple.rate_limit.reason": reason, + ...(cause instanceof Error ? { "error.type": cause.name } : {}), + }), + ) + +export class ApiV2RateLimiter extends Context.Service()( + "@maple/api/services/ApiV2RateLimiter", + { + make: Effect.gen(function* () { + const environment = yield* WorkerEnvironment + + const check = Effect.fn("ApiV2RateLimiter.check")(function* (keyId: ApiKeyId) { + const binding = environment[API_V2_RATE_LIMIT_BINDING] + if (!isRateLimitBinding(binding)) { + yield* warnFailedOpen("binding_missing") + return "failed_open" as const + } + + const partition = readPartition(environment) + if (partition === undefined) { + yield* warnFailedOpen("partition_missing") + return "failed_open" as const + } + + return yield* Effect.tryPromise({ + try: () => binding.limit({ key: makeApiV2RateLimitKey(partition, keyId) }), + catch: (cause) => + new ApiV2RateLimiterBindingError({ + message: "Cloudflare rate-limit binding call failed", + cause, + }), + }).pipe( + Effect.map(({ success }) => (success ? ("allowed" as const) : ("limited" as const))), + Effect.catchTag("@maple/api/services/ApiV2RateLimiterBindingError", (error) => + warnFailedOpen("binding_error", error.cause).pipe( + Effect.as("failed_open"), + ), + ), + ) + }) + + return { check } satisfies ApiV2RateLimiterShape + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/api/wrangler.jsonc b/apps/api/wrangler.jsonc index 06be5ce8b..349c16f67 100644 --- a/apps/api/wrangler.jsonc +++ b/apps/api/wrangler.jsonc @@ -7,6 +7,19 @@ "dev": { "port": 3472, }, + "vars": { + "API_V2_RATE_LIMIT_PARTITION": "local", + }, + "ratelimits": [ + { + "name": "API_V2_RATE_LIMITER", + "namespace_id": "2026071801", + "simple": { + "limit": 600, + "period": 60, + }, + }, + ], // Periodic VCS sync backstop every 12h — handler: worker.ts `scheduled`. (Mirrored in alchemy.run.ts for deploy.) "triggers": { "crons": ["0 */12 * * *"], diff --git a/apps/web/src/components/settings/create-api-key-dialog.tsx b/apps/web/src/components/settings/create-api-key-dialog.tsx index c9d6ec242..7d6983ec0 100644 --- a/apps/web/src/components/settings/create-api-key-dialog.tsx +++ b/apps/web/src/components/settings/create-api-key-dialog.tsx @@ -1,5 +1,5 @@ import { useAtomSet } from "@/lib/effect-atom" -import { useState } from "react" +import { useId, useState } from "react" import { Exit } from "effect" import type { ApiKeyKind } from "@maple/domain/http" import type { V2ApiKeyWithSecret, V2Scope } from "@maple/domain/http/v2" @@ -59,6 +59,12 @@ const SCOPE_FAMILIES = [ { id: "investigations", label: "Investigations" }, { id: "anomalies", label: "Anomalies" }, { id: "session_replays", label: "Session replays" }, + { id: "traces", label: "Traces" }, + { id: "logs", label: "Logs" }, + { id: "metrics", label: "Metrics" }, + { id: "services", label: "Services" }, + { id: "service_map", label: "Service map" }, + { id: "query", label: "Query" }, { id: "organization", label: "Organization" }, ] as const @@ -75,6 +81,7 @@ const scopesFromLevels = (levels: Record): Array => }) export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: CreateApiKeyDialogProps) { + const accessLabelId = useId() const [newName, setNewName] = useState("") const [newDescription, setNewDescription] = useState("") const [expiration, setExpiration] = useState("never") @@ -216,8 +223,9 @@ export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: Crea
- - Access + { const next = values[0] @@ -231,16 +239,19 @@ export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: Crea {accessMode === "restricted" ? (
- {SCOPE_FAMILIES.map((family) => ( -
- - {family.label} - - { + const familyLabelId = `${accessLabelId}-${family.id}` + return ( +
+ + {family.label} + + { const next = values[0] if ( @@ -260,9 +271,10 @@ export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: Crea None Read Write - -
- ))} + +
+ ) + })}

Write includes read. Scopes are fixed at creation — roll the key to change access. diff --git a/apps/web/src/components/settings/developer-section.tsx b/apps/web/src/components/settings/developer-section.tsx index 864e937c5..48ff31c5a 100644 --- a/apps/web/src/components/settings/developer-section.tsx +++ b/apps/web/src/components/settings/developer-section.tsx @@ -46,6 +46,12 @@ const SCOPE_FAMILY_ROWS = [ label: "Session replays", description: "Search sessions, retrieve detail, events, and transcripts", }, + { id: "traces", label: "Traces", description: "Search traces and retrieve spans" }, + { id: "logs", label: "Logs", description: "Search and retrieve log records" }, + { id: "metrics", label: "Metrics", description: "Metric catalog and timeseries reads" }, + { id: "services", label: "Services", description: "Service catalog and health summaries" }, + { id: "service_map", label: "Service map", description: "Service-to-service topology" }, + { id: "query", label: "Query", description: "Structured telemetry queries" }, { id: "organization", label: "Organization", description: "Read the organization's identity" }, ] as const @@ -87,10 +93,17 @@ export function DeveloperSection({ onNavigateToApiKeys }: { onNavigateToApiKeys: prefixed object IDs, cursor-paginated lists, and scoped API keys.

- diff --git a/docs/api-v2.md b/docs/api-v2.md index 03fdbb89f..f1d7f639a 100644 --- a/docs/api-v2.md +++ b/docs/api-v2.md @@ -34,7 +34,7 @@ POST /v2/traces/search complex reads are POST .../search Every v2 object has a prefixed public ID (`key_4CzLmR…`, `dash_…`, `alrt_…`). Public IDs are opaque; internally they are a reversible base58 encoding of the internal ID, computed at the API boundary (`packages/domain/src/http/v2/public-id.ts` — the prefix registry lives there and is the single source of truth). No database migration: rows keep their raw UUIDs / internal strings. -Prefixes: `key` (API key), `ingk` (ingest key), `dash` (dashboard), `dbv` (dashboard version), `dtpl` (dashboard template), `alrt` (alert rule), `dest` (alert destination), `inc` (alert incident), `einc` (error incident), `iss` (error issue), `inv` (investigation), `anom` (anomaly incident), `scrp` (scrape target), `rec` (recommendation), `amap` (attribute mapping), and `srep` (session replay); `evt` and `we` are reserved for events/webhooks. +Prefixes: `key` (API key), `ingk` (ingest key), `dash` (dashboard), `dbv` (dashboard version), `dtpl` (dashboard template), `alrt` (alert rule), `dest` (alert destination), `inc` (alert incident), `einc` (error incident), `iss` (error issue), `inv` (investigation), `anom` (anomaly incident), `scrp` (scrape target), `rec` (recommendation), `amap` (attribute mapping), `srep` (session replay), and `log` (synthetic log identity); `evt` and `we` are reserved for events/webhooks. Exception: Clerk-issued `org_…` / `user_…` IDs are already prefixed public IDs and pass through unchanged. @@ -112,9 +112,13 @@ Implementation: `packages/domain/src/http/v2/auth.ts` + `apps/api/src/services/A Mutating endpoints will accept an `Idempotency-Key` header. Replays within the retention window return the original response. Backed by a Postgres `idempotency_keys` table keyed by `(org_id, key)`. -### Rate limiting (Phase 4 — reserved) +### Rate limiting -Per-key rate limits will return `429` with the error envelope (`type: "rate_limit_error"`) and a `Retry-After` header. +API-key-authenticated requests share one budget per key across the entire `/v2` surface: **600 requests per 60 seconds**. The budget is partitioned by deployment stage, and rolling a key starts a fresh budget because the replacement has a new internal key ID. Valid-key requests count even when a later scope or role check rejects them; invalid credentials and dashboard session tokens do not use this budget. + +Exceeding the budget returns `429` with the standard error envelope (`type: "rate_limit_error"`, `code: "rate_limited"`) and `Retry-After: 60`. No remaining/reset headers are emitted because the backing Cloudflare binding returns only an allow/deny result. + +The limiter fails open: if its binding is missing or errors, Maple logs and traces `maple.rate_limit.outcome=failed_open` and continues the request. Cloudflare counters are local to the location serving the Worker and eventually consistent, so this is abuse protection rather than exact global quota accounting. ### Expansion — not supported @@ -140,14 +144,27 @@ Implemented in phases; the pilot (`api_keys`) ships first and proves every conve | `attribute_mappings` ✅ | CRUD | `ingestAttributeMappings` | | `session_replays` ✅ | `search`/retrieve + events/transcript/`for_trace` (reduced; `facets`/`trace-summaries` deferred) | `sessionReplays` | | `organization` 🟡 | retrieve (GET only shipped); update settings (incl. ClickHouse BYOC) + delete deferred | `organizations`, `orgClickHouseSettings` | -| `traces` | `POST /v2/traces/search`, `GET /v2/traces/{trace_id}`, `GET /v2/traces/{trace_id}/spans/{span_id}` | `queryEngine`, `observability` | -| `logs` | `POST /v2/logs/search`, `GET /v2/logs/{id}` | `queryEngine` | -| `metrics` | `GET /v2/metrics`, `POST /v2/metrics/timeseries` | `queryEngine` | -| `services` | `GET /v2/services`, `GET /v2/services/{name}`, `GET /v2/service_map` | `queryEngine` | -| `query` | `POST /v2/query` — query-builder execution; raw SQL org-gated | `queryEngine` | +| `traces` ✅ | `POST /v2/traces/search`, `GET /v2/traces/{trace_id}`, `GET /v2/traces/{trace_id}/spans/{span_id}` | `queryEngine`, `observability` | +| `logs` ✅ | `POST /v2/logs/search`, `GET /v2/logs/{id}` | `queryEngine` | +| `metrics` ✅ | `GET /v2/metrics`, `POST /v2/metrics/timeseries` | `queryEngine` | +| `services` ✅ | `GET /v2/services`, `GET /v2/services/{name}` | `queryEngine` | +| `service_map` ✅ | `GET /v2/service_map` | `queryEngine` | +| `query` ✅ | `POST /v2/query` — structured telemetry queries | `queryEngine` | The long tail of ~40 query-engine RPC endpoints (facets, infra hosts/pods/nodes/workloads, Cloudflare/PlanetScale infra) starts in the internal RPC tier and is promoted into `/v2` individually as shapes stabilize. +### Telemetry reads + +Phase 2 exposes traces, logs, metrics, services, the service map, and the common query engine. Collection and aggregation operations require explicit ISO-8601 UTC `start_time` and `end_time`; `end_time` must be later than `start_time`, and ranges are capped at 31 days. Direct trace/span retrieval uses the `(OrgId, TraceId, SpanId)` sorting-key prefix so it returns the complete retained trace without a correctness-limiting timestamp window. Direct log retrieval derives its narrow partition window from the timestamp embedded in the log ID. + +Trace IDs, span IDs, metric names, and service names remain their native OpenTelemetry identifiers. Logs have no native record ID, so search results return a deterministic `log_…` ID containing a compact timestamp and a complete-record fingerprint. Malformed log IDs return `log_id_invalid`; records outside retention return `log_not_found`. + +All telemetry lists use the standard cursor envelope with a default limit of 20 and maximum of 100. Trace results are root-span summaries ordered by start time and trace ID. Log ordering includes timestamp plus the complete composite identity. Services are grouped by service name across namespaces and deployment environments. The service map includes only service-to-service edges in this version; database, external, and infrastructure nodes remain internal APIs. + +`POST /v2/query` accepts `timeseries` and `breakdown` specifications for trace, log, and metric sources using snake_case fields. Responses use the common `query_result` envelope. Raw SQL is intentionally unavailable through the public API because tenant isolation must not depend on inspecting caller-authored SQL text. Warehouse diagnostics are never returned to callers. + +Telemetry scope families are `traces`, `logs`, `metrics`, `services`, `service_map`, and `query`. Search, timeseries, and query POSTs are read-only operations for scope enforcement, so `:read` is sufficient. + Not in v2: org membership and invitations (delegated to Clerk; revisit if/when a members API is needed). ### Optional ElectricSQL `txid` metadata @@ -158,9 +175,9 @@ The dashboard can reconcile optimistic writes against ElectricSQL synced shapes - **Phase 0 (this change)** — conventions doc, v2 primitives (`public-id`, `envelopes`, `errors`, `auth`), scoped API keys (schema + service + enforcement), `MapleApiV2` shell mounted at `/v2` with Scalar docs at `/v2/docs`, pilot resource `api_keys` end-to-end with tests. - **Phase 1 — core resources**: dashboards, alerts, error issues, scrape targets, ingest keys, attribute mappings, investigations, anomalies, recommendations, organization, session replays. Thin handler adapters over existing services; `txid` preserved. -- **Phase 2 — telemetry reads**: traces/logs/metrics/services/service_map/query over `QueryEngineService`. +- **Phase 2 ✅ — telemetry reads**: traces/logs/metrics/services/service_map/query over `QueryEngineService`. - **Phase 3 — internal RPC tier + dashboard migration**: `RpcGroup` contracts in `packages/domain/src/rpc/` served at `/rpc`; dashboard gets a `MapleApiV2AtomClient` (same wiring as `apps/web/src/lib/services/common/atom-client.ts`, pointed at `MapleApiV2`) plus an `RpcClient`; migrate group-by-group, deleting v1 groups as they empty. (Note: the billing-scoped 401 retry in `atom-client.ts` must follow billing to its RPC home.) -- **Phase 4 — hardening**: `Idempotency-Key`, per-key rate limiting (Workers rate-limit binding), `Maple-Version` header enforcement. +- **Phase 4 — hardening**: `Idempotency-Key`, `Maple-Version` header enforcement. Per-key rate limiting is implemented. - **Phase 5 — events & webhooks**: `evt_` event objects, `GET /v2/events`, `/v2/webhook_endpoints` CRUD, HMAC-signed deliveries (`Maple-Signature`) via an outbox drained by the alerting worker. ## Adding a v2 resource (checklist) diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 02114a005..5d7558459 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -759,9 +759,9 @@ export class ListLogsResponse extends Schema.Class("ListLogsRe }) {} // Exact-match lookup of one log by its composite key (logs have no primary id). -// `timestamp` is the raw ClickHouse DateTime64 string and carries sub-second -// precision (`YYYY-MM-DD HH:mm:ss.fffffffff`), so it is a plain string rather -// than `TinybirdDateTime` (which only matches second-level precision). +// `timestamp` is the raw ClickHouse DateTime64 string. It remains a plain +// string because older stored rows and upstream drivers can vary their +// fractional-second rendering. export class GetLogRequest extends Schema.Class("GetLogRequest")({ timestamp: Schema.String, serviceName: ServiceName, diff --git a/packages/domain/src/http/v2/api.ts b/packages/domain/src/http/v2/api.ts index d882d2b0c..ced6956ec 100644 --- a/packages/domain/src/http/v2/api.ts +++ b/packages/domain/src/http/v2/api.ts @@ -12,8 +12,40 @@ import { V2OrganizationApiGroup } from "./organization" import { V2InstrumentationRecommendationsApiGroup } from "./recommendations" import { V2ScrapeTargetsApiGroup } from "./scrape-targets" import { V2SessionReplaysApiGroup } from "./session-replays" +import { + V2LogsApiGroup, + V2MetricsApiGroup, + V2QueryApiGroup, + V2ServiceMapApiGroup, + V2ServicesApiGroup, + V2TracesApiGroup, +} from "./telemetry" import { V2UnexpectedErrors } from "./auth" +const HTTP_OPERATION_METHODS = ["get", "post", "put", "patch", "delete", "head"] as const + +/** Add the rate-limit retry contract to every generated 429 response. */ +const addRateLimitResponseHeaders = (spec: Record): Record => { + for (const pathItem of Object.values(spec.paths ?? {})) { + if (typeof pathItem !== "object" || pathItem === null) continue + for (const method of HTTP_OPERATION_METHODS) { + const operation = (pathItem as Record)[method] + if (typeof operation !== "object" || operation === null) continue + const response = operation.responses?.["429"] + if (typeof response !== "object" || response === null) continue + response.headers = { + ...response.headers, + "Retry-After": { + description: "Seconds to wait before retrying the request.", + schema: { type: "integer", minimum: 1 }, + example: 60, + }, + } + } + } + return spec +} + /** * The Maple v2 public API (see docs/api-v2.md). * @@ -40,6 +72,12 @@ export class MapleApiV2 extends HttpApi.make("MapleApiV2") .add(V2AnomaliesApiGroup) .add(V2OrganizationApiGroup) .add(V2SessionReplaysApiGroup) + .add(V2TracesApiGroup) + .add(V2LogsApiGroup) + .add(V2MetricsApiGroup) + .add(V2ServicesApiGroup) + .add(V2ServiceMapApiGroup) + .add(V2QueryApiGroup) .middleware(V2UnexpectedErrors) .annotateMerge( OpenApi.annotations({ @@ -63,20 +101,23 @@ export class MapleApiV2 extends HttpApi.make("MapleApiV2") // `info.contact` and top-level `externalDocs` have no dedicated annotation // key (they are not in `OpenAPISpecInfo`), so inject them via the api-level // spec transform, which receives the whole generated document. - transform: (spec) => ({ - ...spec, - info: { - ...spec.info, - contact: { - name: "Maple Support", - url: "https://maple.dev", - email: "support@maple.dev", + transform: (spec) => { + const withRateLimitHeaders = addRateLimitResponseHeaders(spec) + return { + ...withRateLimitHeaders, + info: { + ...withRateLimitHeaders.info, + contact: { + name: "Maple Support", + url: "https://maple.dev", + email: "support@maple.dev", + }, }, - }, - externalDocs: { - url: "https://api.maple.dev/v2/docs", - description: "Interactive Maple API reference", - }, - }), + externalDocs: { + url: "https://api.maple.dev/v2/docs", + description: "Interactive Maple API reference", + }, + } + }, }), ) {} diff --git a/packages/domain/src/http/v2/auth.ts b/packages/domain/src/http/v2/auth.ts index 1c102d58c..dc0d617ac 100644 --- a/packages/domain/src/http/v2/auth.ts +++ b/packages/domain/src/http/v2/auth.ts @@ -6,6 +6,7 @@ import { V2AuthenticationError, V2InvalidRequestError, V2PermissionError, + V2RateLimitError, V2ServiceUnavailableError, } from "./errors" @@ -23,7 +24,7 @@ export class AuthorizationV2 extends HttpApiMiddleware.Service< provides: Context } >()("AuthorizationV2", { - error: [V2AuthenticationError, V2PermissionError, V2ServiceUnavailableError], + error: [V2AuthenticationError, V2PermissionError, V2RateLimitError, V2ServiceUnavailableError], security: { bearer: HttpApiSecurity.bearer.pipe( HttpApiSecurity.annotateMerge( @@ -77,6 +78,10 @@ const READ_ONLY_POST_PATHS = new Set([ "/v2/session_replays/search", "/v2/session_replays/for_trace", "/v2/alerts/rules/preview", + "/v2/traces/search", + "/v2/logs/search", + "/v2/metrics/timeseries", + "/v2/query", ]) /** diff --git a/packages/domain/src/http/v2/errors.ts b/packages/domain/src/http/v2/errors.ts index 28c8dd30e..34315fe59 100644 --- a/packages/domain/src/http/v2/errors.ts +++ b/packages/domain/src/http/v2/errors.ts @@ -247,6 +247,15 @@ export const resourceNotFound = (resource: string, message: string, param = "id" export const conflict = (code: string, message: string) => new V2ConflictError({ error: { type: "conflict_error", code, message } }) +export const rateLimited = () => + new V2RateLimitError({ + error: { + type: "rate_limit_error", + code: "rate_limited", + message: "Too many requests. Retry after 60 seconds.", + }, + }) + export const upstreamError = (code: string, message: string) => new V2UpstreamError({ error: { type: "api_error", code, message } }) diff --git a/packages/domain/src/http/v2/index.ts b/packages/domain/src/http/v2/index.ts index 6c637681f..5f929c5f4 100644 --- a/packages/domain/src/http/v2/index.ts +++ b/packages/domain/src/http/v2/index.ts @@ -17,3 +17,4 @@ export * from "./recommendations" export * from "./resource-ids" export * from "./scrape-targets" export * from "./session-replays" +export * from "./telemetry" diff --git a/packages/domain/src/http/v2/openapi.test.ts b/packages/domain/src/http/v2/openapi.test.ts index be2c97af8..0138dac8c 100644 --- a/packages/domain/src/http/v2/openapi.test.ts +++ b/packages/domain/src/http/v2/openapi.test.ts @@ -87,13 +87,20 @@ describe("MapleApiV2 OpenAPI", () => { "GET /v2/instrumentation/recommendations", "GET /v2/investigations", "GET /v2/investigations/{id}", + "GET /v2/logs/{id}", + "GET /v2/metrics", "GET /v2/organization", "GET /v2/scrape_targets", "GET /v2/scrape_targets/{id}", "GET /v2/scrape_targets/{id}/checks", + "GET /v2/service_map", + "GET /v2/services", + "GET /v2/services/{name}", "GET /v2/session_replays/{id}", "GET /v2/session_replays/{id}/events", "GET /v2/session_replays/{id}/transcript", + "GET /v2/traces/{trace_id}", + "GET /v2/traces/{trace_id}/spans/{span_id}", "PATCH /v2/alerts/destinations/{id}", "PATCH /v2/alerts/rules/{id}", "PATCH /v2/anomalies/settings", @@ -119,10 +126,14 @@ describe("MapleApiV2 OpenAPI", () => { "POST /v2/instrumentation/recommendations/{id}/reopen", "POST /v2/investigations", "POST /v2/investigations/{id}/status", + "POST /v2/logs/search", + "POST /v2/metrics/timeseries", + "POST /v2/query", "POST /v2/scrape_targets", "POST /v2/scrape_targets/{id}/probe", "POST /v2/session_replays/for_trace", "POST /v2/session_replays/search", + "POST /v2/traces/search", "PUT /v2/anomalies/incidents/{id}/issue", ]) }) @@ -161,12 +172,18 @@ describe("MapleApiV2 OpenAPI", () => { expect(op.description.length).toBeGreaterThan(20) expect(op.tags).toHaveLength(1) expect(op.security).toEqual([{ bearer: [] }]) - for (const status of ["400", "401", "403", "500"]) { + for (const status of ["400", "401", "403", "429", "500"]) { expect( op.responses[status], `${method.toUpperCase()} ${path} declares ${status}`, ).toBeDefined() } + const rateLimitResponse = op.responses["429"] as Record + expect(rateLimitResponse.headers?.["Retry-After"]).toMatchObject({ + description: expect.any(String), + schema: { type: "integer", minimum: 1 }, + example: 60, + }) } }) @@ -188,6 +205,7 @@ describe("MapleApiV2 OpenAPI", () => { "AuthenticationError", "PermissionError", "NotFoundError", + "RateLimitError", "ServiceUnavailableError", ]), ) diff --git a/packages/domain/src/http/v2/public-id.ts b/packages/domain/src/http/v2/public-id.ts index 4f0284ddd..efed158b4 100644 --- a/packages/domain/src/http/v2/public-id.ts +++ b/packages/domain/src/http/v2/public-id.ts @@ -35,6 +35,8 @@ export const PublicIdPrefixes = { ingestKey: "ingk", attributeMapping: "amap", sessionReplay: "srep", + /** Synthetic identity for logs, which have no native OTel record id. */ + log: "log", /** Reserved for the future events/webhooks system. */ event: "evt", webhookEndpoint: "we", diff --git a/packages/domain/src/http/v2/telemetry.ts b/packages/domain/src/http/v2/telemetry.ts new file mode 100644 index 000000000..343995ac5 --- /dev/null +++ b/packages/domain/src/http/v2/telemetry.ts @@ -0,0 +1,637 @@ +import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/unstable/httpapi" +import { Schema } from "effect" +import { MetricName, ServiceName, SpanId, TraceId } from "../../primitives" +import { AuthorizationV2, V2SchemaErrors } from "./auth" +import { ListOf, ListQuery, Timestamp } from "./envelopes" +import { V2InvalidRequestError, V2NotFoundError, V2ServiceUnavailableError } from "./errors" +import { PublicId, PublicIdPrefixes } from "./public-id" + +const wireExample = (example: object): A => example as A +const commonErrors = [V2InvalidRequestError, V2ServiceUnavailableError] as const +const PositiveInteger = Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)) +const PositiveFinite = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0)) +const NonNegativeFinite = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThanOrEqualTo(0)) +const BreakdownLimit = PositiveInteger.check(Schema.isLessThanOrEqualTo(100)) + +export const LogPublicId = PublicId(PublicIdPrefixes.log, Schema.String).annotate({ + identifier: "LogId", + title: "Log ID", +}) + +const JsonStringRecord = Schema.Record(Schema.String, Schema.String) +const RequiredTimeRange = { + start_time: Timestamp.annotate({ + description: "Inclusive query-window start.", + }), + end_time: Timestamp.annotate({ description: "Inclusive query-window end." }), +} as const +const PostPagination = { + limit: Schema.optionalKey( + Schema.Number.check(Schema.isInt(), Schema.isBetween({ minimum: 1, maximum: 100 })), + ), + cursor: Schema.optionalKey(Schema.String), +} as const + +export const V2TelemetryWindowQuery = Schema.Struct({ + ...RequiredTimeRange, +}).annotate({ + identifier: "TelemetryWindowQuery", + title: "Telemetry time window", +}) + +export const V2TraceSummary = Schema.Struct({ + id: TraceId, + object: Schema.Literal("trace"), + start_time: Timestamp, + duration_ms: Schema.Number, + root_span_name: Schema.String, + root_span_kind: Schema.String, + root_service_name: Schema.String, + status_code: Schema.String, + has_error: Schema.Boolean, + deployment_environment: Schema.NullOr(Schema.String), + service_namespace: Schema.NullOr(Schema.String), + http_method: Schema.NullOr(Schema.String), + http_route: Schema.NullOr(Schema.String), + http_status_code: Schema.NullOr(Schema.String), +}).annotate({ + identifier: "TraceSummary", + title: "Trace summary", + description: "A root-based summary returned by trace search.", + examples: [ + wireExample({ + id: "7f3a4b5c6d7e8f901234567890abcdef", + object: "trace", + start_time: "2026-07-15T12:00:00.000Z", + duration_ms: 42.5, + root_span_name: "GET /checkout", + root_span_kind: "Server", + root_service_name: "api", + status_code: "Ok", + has_error: false, + deployment_environment: "production", + service_namespace: "checkout", + http_method: "GET", + http_route: "/checkout", + http_status_code: "200", + }), + ], +}) +export type V2TraceSummary = Schema.Schema.Type + +export const V2Span = Schema.Struct({ + id: SpanId, + object: Schema.Literal("span"), + trace_id: TraceId, + parent_span_id: Schema.NullOr(SpanId), + name: Schema.String, + service_name: Schema.String, + kind: Schema.String, + start_time: Timestamp, + duration_ms: Schema.Number, + status_code: Schema.String, + status_message: Schema.NullOr(Schema.String), + attributes: JsonStringRecord, + resource_attributes: JsonStringRecord, +}).annotate({ + identifier: "Span", + title: "Span", + description: "An OpenTelemetry span.", +}) +export type V2Span = Schema.Schema.Type + +export const V2Trace = Schema.Struct({ + id: TraceId, + object: Schema.Literal("trace"), + start_time: Timestamp, + end_time: Timestamp, + duration_ms: Schema.Number, + span_count: Schema.Number, + service_count: Schema.Number, + spans: Schema.Array(V2Span), +}).annotate({ + identifier: "Trace", + title: "Trace", + description: "A trace and its spans.", +}) +export type V2Trace = Schema.Schema.Type + +export const V2TraceSearchParams = Schema.Struct({ + ...RequiredTimeRange, + ...PostPagination, + service_name: Schema.optionalKey(ServiceName), + span_name: Schema.optionalKey(Schema.String), + has_error: Schema.optionalKey(Schema.Boolean), + min_duration_ms: Schema.optionalKey(NonNegativeFinite), + max_duration_ms: Schema.optionalKey(NonNegativeFinite), + http_method: Schema.optionalKey(Schema.String), + http_status_code: Schema.optionalKey(Schema.String), + deployment_environment: Schema.optionalKey(Schema.String), + service_namespace: Schema.optionalKey(Schema.String), +}).annotate({ + identifier: "TraceSearchParams", + title: "Trace search parameters", +}) + +const TraceList = ListOf(V2TraceSummary).annotate({ + identifier: "TraceList", + title: "Trace list", +}) + +export class V2TracesApiGroup extends HttpApiGroup.make("traces") + .add( + HttpApiEndpoint.post("search", "/search", { + payload: V2TraceSearchParams, + success: TraceList, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "searchTraces", + summary: "Search traces", + description: "Searches root traces in an explicit time window. Requires `traces:read`.", + }), + ), + ) + .add( + HttpApiEndpoint.get("retrieve", "/:trace_id", { + params: { trace_id: TraceId }, + success: V2Trace, + error: [...commonErrors, V2NotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "getTrace", + summary: "Retrieve a trace", + description: + "Returns a trace and its flat parent-linked span collection. Requires `traces:read`.", + }), + ), + ) + .add( + HttpApiEndpoint.get("retrieveSpan", "/:trace_id/spans/:span_id", { + params: { trace_id: TraceId, span_id: SpanId }, + success: V2Span, + error: [...commonErrors, V2NotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "getSpan", + summary: "Retrieve a span", + description: "Returns one span with complete attribute maps. Requires `traces:read`.", + }), + ), + ) + .prefix("/v2/traces") + .middleware(AuthorizationV2) + .middleware(V2SchemaErrors) + .annotateMerge( + OpenApi.annotations({ + title: "Traces", + description: "Search and inspect distributed traces.", + }), + ) {} + +export const V2Log = Schema.Struct({ + id: LogPublicId, + object: Schema.Literal("log"), + timestamp: Timestamp, + severity_text: Schema.String, + severity_number: Schema.Number, + service_name: Schema.String, + body: Schema.String, + trace_id: Schema.NullOr(TraceId), + span_id: Schema.NullOr(SpanId), + log_attributes: JsonStringRecord, + resource_attributes: JsonStringRecord, +}).annotate({ + identifier: "Log", + title: "Log", + description: "An OpenTelemetry log record.", +}) +export type V2Log = Schema.Schema.Type + +export const V2LogSearchParams = Schema.Struct({ + ...RequiredTimeRange, + ...PostPagination, + service_name: Schema.optionalKey(ServiceName), + severity: Schema.optionalKey(Schema.String), + min_severity: Schema.optionalKey( + Schema.Number.check(Schema.isInt(), Schema.isBetween({ minimum: 0, maximum: 255 })), + ), + trace_id: Schema.optionalKey(TraceId), + span_id: Schema.optionalKey(SpanId), + search: Schema.optionalKey(Schema.String), + deployment_environment: Schema.optionalKey(Schema.String), + service_namespace: Schema.optionalKey(Schema.String), +}).annotate({ identifier: "LogSearchParams", title: "Log search parameters" }) +const LogList = ListOf(V2Log).annotate({ + identifier: "LogList", + title: "Log list", +}) + +export class V2LogsApiGroup extends HttpApiGroup.make("logs") + .add( + HttpApiEndpoint.post("search", "/search", { + payload: V2LogSearchParams, + success: LogList, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "searchLogs", + summary: "Search logs", + description: "Searches logs in an explicit time window. Requires `logs:read`.", + }), + ), + ) + .add( + HttpApiEndpoint.get("retrieve", "/:id", { + params: { id: LogPublicId }, + success: V2Log, + error: [...commonErrors, V2NotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "getLog", + summary: "Retrieve a log", + description: "Returns a log by its opaque `log_…` ID. Requires `logs:read`.", + }), + ), + ) + .prefix("/v2/logs") + .middleware(AuthorizationV2) + .middleware(V2SchemaErrors) + .annotateMerge( + OpenApi.annotations({ + title: "Logs", + description: "Search and retrieve OpenTelemetry logs.", + }), + ) {} + +export const V2Metric = Schema.Struct({ + object: Schema.Literal("metric"), + name: MetricName, + type: Schema.String, + service_name: Schema.String, + description: Schema.String, + unit: Schema.String, + is_monotonic: Schema.Boolean, + data_point_count: Schema.Number, + first_seen: Timestamp, + last_seen: Timestamp, +}).annotate({ + identifier: "Metric", + title: "Metric", + description: "A metric catalog entry.", +}) +export type V2Metric = Schema.Schema.Type + +export const V2MetricListQuery = Schema.Struct({ + ...V2TelemetryWindowQuery.fields, + ...ListQuery.fields, + service_name: Schema.optional(ServiceName), + metric_type: Schema.optional(Schema.String), + search: Schema.optional(Schema.String), +}).annotate({ identifier: "MetricListQuery", title: "Metric list query" }) + +export const V2AttributeFilter = Schema.Struct({ + key: Schema.String, + value: Schema.optionalKey(Schema.String), + mode: Schema.Literals(["equals", "exists", "gt", "gte", "lt", "lte", "contains"]), + negated: Schema.optionalKey(Schema.Boolean), +}).annotate({ identifier: "AttributeFilter", title: "Attribute filter" }) + +const MetricsMetric = Schema.Literals(["avg", "sum", "min", "max", "count", "rate", "increase"]) +const MetricType = Schema.Literals(["sum", "gauge", "histogram", "exponential_histogram"]) +const MetricsFilters = { + metric_name: MetricName, + metric_type: MetricType, + service_name: Schema.optionalKey(ServiceName), + group_by_attribute_key: Schema.optionalKey(Schema.String), + group_by_resource_attribute_key: Schema.optionalKey(Schema.String), + attribute_filters: Schema.optionalKey(Schema.Array(V2AttributeFilter)), + resource_attribute_filters: Schema.optionalKey(Schema.Array(V2AttributeFilter)), +} as const + +export const V2MetricsTimeseriesParams = Schema.Struct({ + ...RequiredTimeRange, + metric: MetricsMetric, + ...MetricsFilters, + group_by: Schema.optionalKey( + Schema.Array(Schema.Literals(["service", "attribute", "resource_attribute", "none"])), + ), + bucket_seconds: Schema.optionalKey(Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0))), +}).annotate({ + identifier: "MetricsTimeseriesParams", + title: "Metrics timeseries parameters", +}) + +export const V2TimeseriesPoint = Schema.Struct({ + bucket: Timestamp, + series: Schema.Record(Schema.String, Schema.Number), +}).annotate({ identifier: "TimeseriesPoint", title: "Timeseries point" }) +export const V2BreakdownItem = Schema.Struct({ + name: Schema.String, + value: Schema.Number, +}).annotate({ identifier: "BreakdownItem", title: "Breakdown item" }) +export const V2StructuredQueryResult = Schema.Struct({ + object: Schema.Literal("query_result"), + kind: Schema.Literals(["timeseries", "breakdown"]), + source: Schema.Literals(["traces", "logs", "metrics"]), + timeseries: Schema.Array(V2TimeseriesPoint), + breakdown: Schema.Array(V2BreakdownItem), +}).annotate({ + identifier: "StructuredQueryResult", + title: "Structured query result", +}) +export type V2StructuredQueryResult = Schema.Schema.Type + +const MetricList = ListOf(V2Metric).annotate({ + identifier: "MetricList", + title: "Metric list", +}) +export class V2MetricsApiGroup extends HttpApiGroup.make("metrics") + .add( + HttpApiEndpoint.get("list", "/", { + query: V2MetricListQuery, + success: MetricList, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "listMetrics", + summary: "List metrics", + description: + "Lists metric catalog entries in an explicit time window. Requires `metrics:read`.", + }), + ), + ) + .add( + HttpApiEndpoint.post("timeseries", "/timeseries", { + payload: V2MetricsTimeseriesParams, + success: V2StructuredQueryResult, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "queryMetricsTimeseries", + summary: "Query metric timeseries", + description: "Executes one typed metric timeseries query. Requires `metrics:read`.", + }), + ), + ) + .prefix("/v2/metrics") + .middleware(AuthorizationV2) + .middleware(V2SchemaErrors) + .annotateMerge( + OpenApi.annotations({ + title: "Metrics", + description: "Browse and query OpenTelemetry metrics.", + }), + ) {} + +export const V2Service = Schema.Struct({ + object: Schema.Literal("service"), + name: ServiceName, + service_namespaces: Schema.Array(Schema.String), + deployment_environments: Schema.Array(Schema.String), + throughput: Schema.Number, + traced_throughput: Schema.Number, + span_count: Schema.Number, + error_count: Schema.Number, + error_rate: Schema.Number, + p50_latency_ms: Schema.Number, + p95_latency_ms: Schema.Number, + p99_latency_ms: Schema.Number, + has_sampling: Schema.Boolean, + sampling_weight: Schema.Number, +}).annotate({ + identifier: "Service", + title: "Service", + description: "An observed service aggregated by service name.", +}) +export type V2Service = Schema.Schema.Type + +export const V2ServiceListQuery = Schema.Struct({ + ...V2TelemetryWindowQuery.fields, + ...ListQuery.fields, + deployment_environment: Schema.optional(Schema.String), + service_namespace: Schema.optional(Schema.String), +}).annotate({ identifier: "ServiceListQuery", title: "Service list query" }) + +export const V2ServiceMapEdge = Schema.Struct({ + object: Schema.Literal("service_map.edge"), + source_service: Schema.String, + target_service: Schema.String, + call_count: Schema.Number, + estimated_call_count: Schema.Number, + error_count: Schema.Number, + error_rate: Schema.Number, + avg_duration_ms: Schema.Number, + p95_duration_ms: Schema.Number, + has_sampling: Schema.Boolean, + sampling_weight: Schema.Number, +}).annotate({ identifier: "ServiceMapEdge", title: "Service map edge" }) +export type V2ServiceMapEdge = Schema.Schema.Type +export const V2ServiceMap = Schema.Struct({ + object: Schema.Literal("service_map"), + start_time: Timestamp, + end_time: Timestamp, + edges: Schema.Array(V2ServiceMapEdge), +}).annotate({ identifier: "ServiceMap", title: "Service map" }) +export type V2ServiceMap = Schema.Schema.Type +export const V2ServiceMapQuery = Schema.Struct({ + ...V2TelemetryWindowQuery.fields, + service_name: Schema.optional(ServiceName), + deployment_environment: Schema.optional(Schema.String), +}).annotate({ identifier: "ServiceMapQuery", title: "Service map query" }) +const ServiceList = ListOf(V2Service).annotate({ + identifier: "ServiceList", + title: "Service list", +}) + +export class V2ServicesApiGroup extends HttpApiGroup.make("services") + .add( + HttpApiEndpoint.get("list", "/", { + query: V2ServiceListQuery, + success: ServiceList, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "listServices", + summary: "List services", + description: + "Lists services aggregated by name in an explicit time window. Requires `services:read`.", + }), + ), + ) + .add( + HttpApiEndpoint.get("retrieve", "/:name", { + params: { name: ServiceName }, + query: V2TelemetryWindowQuery, + success: V2Service, + error: [...commonErrors, V2NotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "getService", + summary: "Retrieve a service", + description: + "Returns one service aggregated across environments and namespaces. Requires `services:read`.", + }), + ), + ) + .prefix("/v2/services") + .middleware(AuthorizationV2) + .middleware(V2SchemaErrors) + .annotateMerge( + OpenApi.annotations({ + title: "Services", + description: "Observed service health summaries.", + }), + ) {} + +export class V2ServiceMapApiGroup extends HttpApiGroup.make("serviceMap") + .add( + HttpApiEndpoint.get("retrieve", "/", { + query: V2ServiceMapQuery, + success: V2ServiceMap, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "getServiceMap", + summary: "Retrieve the service map", + description: + "Returns service-to-service dependencies in an explicit time window. Requires `service_map:read`.", + }), + ), + ) + .prefix("/v2/service_map") + .middleware(AuthorizationV2) + .middleware(V2SchemaErrors) + .annotateMerge( + OpenApi.annotations({ + title: "Service Map", + description: "Service-to-service topology.", + }), + ) {} + +const TracesMetric = Schema.Literals([ + "count", + "avg_duration", + "p50_duration", + "p95_duration", + "p99_duration", + "error_rate", + "apdex", +]) +const TraceFilters = Schema.Struct({ + service_name: Schema.optionalKey(ServiceName), + span_name: Schema.optionalKey(Schema.String), + root_spans_only: Schema.optionalKey(Schema.Boolean), + errors_only: Schema.optionalKey(Schema.Boolean), + environments: Schema.optionalKey(Schema.Array(Schema.String)), + namespaces: Schema.optionalKey(Schema.Array(Schema.String)), + min_duration_ms: Schema.optionalKey(NonNegativeFinite), + max_duration_ms: Schema.optionalKey(NonNegativeFinite), + group_by_attribute_keys: Schema.optionalKey(Schema.Array(Schema.String)), + attribute_filters: Schema.optionalKey(Schema.Array(V2AttributeFilter)), + resource_attribute_filters: Schema.optionalKey(Schema.Array(V2AttributeFilter)), +}) +const LogFilters = Schema.Struct({ + service_name: Schema.optionalKey(ServiceName), + severity: Schema.optionalKey(Schema.String), + trace_id: Schema.optionalKey(TraceId), + search: Schema.optionalKey(Schema.String), + environments: Schema.optionalKey(Schema.Array(Schema.String)), + namespaces: Schema.optionalKey(Schema.Array(Schema.String)), +}) +const MetricsFiltersSchema = Schema.Struct(MetricsFilters) + +const V2TracesTimeseriesSpec = Schema.Struct({ + kind: Schema.Literal("timeseries"), + source: Schema.Literal("traces"), + metric: TracesMetric, + group_by: Schema.optionalKey( + Schema.Array( + Schema.Literals(["service", "span_name", "status_code", "http_method", "attribute", "none"]), + ), + ), + filters: Schema.optionalKey(TraceFilters), + bucket_seconds: Schema.optionalKey(PositiveInteger), + series_limit: Schema.optionalKey(PositiveInteger), + apdex_threshold_ms: Schema.optionalKey(PositiveFinite), +}) +const V2LogsTimeseriesSpec = Schema.Struct({ + kind: Schema.Literal("timeseries"), + source: Schema.Literal("logs"), + metric: Schema.Literal("count"), + group_by: Schema.optionalKey(Schema.Array(Schema.Literals(["service", "severity", "none"]))), + filters: Schema.optionalKey(LogFilters), + bucket_seconds: Schema.optionalKey(PositiveInteger), + series_limit: Schema.optionalKey(PositiveInteger), +}) +const V2MetricsTimeseriesSpec = Schema.Struct({ + kind: Schema.Literal("timeseries"), + source: Schema.Literal("metrics"), + metric: MetricsMetric, + group_by: Schema.optionalKey( + Schema.Array(Schema.Literals(["service", "attribute", "resource_attribute", "none"])), + ), + filters: MetricsFiltersSchema, + bucket_seconds: Schema.optionalKey(PositiveInteger), +}) +const V2TracesBreakdownSpec = Schema.Struct({ + kind: Schema.Literal("breakdown"), + source: Schema.Literal("traces"), + metric: TracesMetric, + group_by: Schema.Literals(["service", "span_name", "status_code", "http_method", "attribute"]), + filters: Schema.optionalKey(TraceFilters), + limit: Schema.optionalKey(BreakdownLimit), + apdex_threshold_ms: Schema.optionalKey(PositiveFinite), +}) +const V2LogsBreakdownSpec = Schema.Struct({ + kind: Schema.Literal("breakdown"), + source: Schema.Literal("logs"), + metric: Schema.Literal("count"), + group_by: Schema.Literals(["service", "severity"]), + filters: Schema.optionalKey(LogFilters), + limit: Schema.optionalKey(BreakdownLimit), +}) +const V2MetricsBreakdownSpec = Schema.Struct({ + kind: Schema.Literal("breakdown"), + source: Schema.Literal("metrics"), + metric: Schema.Literals(["avg", "sum", "count"]), + group_by: Schema.Literals(["service", "attribute", "resource_attribute"]), + filters: MetricsFiltersSchema, + limit: Schema.optionalKey(BreakdownLimit), +}) +export const V2QuerySpec = Schema.Union([ + V2TracesTimeseriesSpec, + V2LogsTimeseriesSpec, + V2MetricsTimeseriesSpec, + V2TracesBreakdownSpec, + V2LogsBreakdownSpec, + V2MetricsBreakdownSpec, +]).annotate({ identifier: "QuerySpec", title: "Query specification" }) +export type V2QuerySpec = Schema.Schema.Type +export const V2QueryParams = Schema.Struct({ + ...RequiredTimeRange, + query: V2QuerySpec, +}).annotate({ identifier: "QueryParams", title: "Query parameters" }) +export class V2QueryApiGroup extends HttpApiGroup.make("query") + .add( + HttpApiEndpoint.post("execute", "/", { + payload: V2QueryParams, + success: V2StructuredQueryResult, + error: [...commonErrors], + }).annotateMerge( + OpenApi.annotations({ + identifier: "executeQuery", + summary: "Execute a telemetry query", + description: "Executes a typed telemetry query. Requires `query:read`.", + }), + ), + ) + .prefix("/v2/query") + .middleware(AuthorizationV2) + .middleware(V2SchemaErrors) + .annotateMerge( + OpenApi.annotations({ + title: "Query", + description: "Structured telemetry query execution.", + }), + ) {} diff --git a/packages/domain/src/http/v2/v2-contract.test.ts b/packages/domain/src/http/v2/v2-contract.test.ts index 1912b35f7..fb7abff9e 100644 --- a/packages/domain/src/http/v2/v2-contract.test.ts +++ b/packages/domain/src/http/v2/v2-contract.test.ts @@ -15,8 +15,9 @@ import { paginateOffsetQuery, Timestamp, } from "./envelopes" -import { notFound, permissionError, V2NotFoundError } from "./errors" +import { notFound, permissionError, rateLimited, V2NotFoundError, V2RateLimitError } from "./errors" import { encodePublicId } from "./public-id" +import { LogPublicId, V2QueryParams } from "./telemetry" const UUID = "0f8fad5b-d9cb-469f-a165-70867728950e" @@ -320,6 +321,16 @@ describe("v2 error envelope", () => { it("permissionError has type permission_error", () => { expect(permissionError("insufficient_scope", "nope").error.type).toBe("permission_error") }) + + it("rateLimited has the stable public 429 envelope", () => { + expect(Schema.encodeSync(V2RateLimitError)(rateLimited())).toEqual({ + error: { + type: "rate_limit_error", + code: "rate_limited", + message: "Too many requests. Retry after 60 seconds.", + }, + }) + }) }) describe("scopes", () => { @@ -368,6 +379,14 @@ describe("scopes", () => { family: "session_replays", access: "read", }) + for (const [path, family] of [ + ["/v2/traces/search", "traces"], + ["/v2/logs/search", "logs"], + ["/v2/metrics/timeseries", "metrics"], + ["/v2/query", "query"], + ] as const) { + expect(requiredScopeForRequest("POST", path)).toEqual({ family, access: "read" }) + } expect(requiredScopeForRequest("GET", "/api/api-keys")).toBeNull() }) @@ -398,6 +417,57 @@ describe("scopes", () => { }) }) +describe("telemetry contracts", () => { + it("round-trips the synthetic composite log ID", () => { + const internal = JSON.stringify([ + "2026-07-15 12:00:00.123", + "00112233445566778899AABBCCDDEEFF", + ]) + const wire = Schema.encodeSync(LogPublicId)(internal) + expect(wire.startsWith("log_")).toBe(true) + expect(Schema.decodeSync(LogPublicId)(wire)).toBe(internal) + }) + + it("decodes structured queries with trace attribute grouping", () => { + const structured = Schema.decodeUnknownSync(V2QueryParams)({ + start_time: "2026-07-15T00:00:00.000Z", + end_time: "2026-07-15T01:00:00.000Z", + query: { + kind: "timeseries", + source: "traces", + metric: "count", + group_by: ["attribute"], + filters: { group_by_attribute_keys: ["http.route"] }, + }, + }) + expect(structured.query.kind).toBe("timeseries") + }) + + it("rejects raw SQL and invalid query budgets at the HTTP boundary", () => { + const base = { + start_time: "2026-07-15T00:00:00.000Z", + end_time: "2026-07-15T01:00:00.000Z", + } + expect(() => + Schema.decodeUnknownSync(V2QueryParams)({ + ...base, + query: { kind: "raw_sql", sql: "SELECT 1 WHERE $__orgFilter" }, + }), + ).toThrow() + expect(() => + Schema.decodeUnknownSync(V2QueryParams)({ + ...base, + query: { + kind: "timeseries", + source: "logs", + metric: "count", + bucket_seconds: 0, + }, + }), + ).toThrow() + }) +}) + describe("list pagination", () => { const items = Array.from({ length: 45 }, (_, index) => index) diff --git a/packages/domain/src/query-engine.ts b/packages/domain/src/query-engine.ts index 2f5cf06b8..5050380b9 100644 --- a/packages/domain/src/query-engine.ts +++ b/packages/domain/src/query-engine.ts @@ -10,12 +10,12 @@ import { TraceId, } from "./primitives" -const dateTimePattern = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/ +const dateTimePattern = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d{1,9})?$/ export const TinybirdDateTime = Schema.String.check(Schema.isPattern(dateTimePattern)).pipe( Schema.annotate({ identifier: "TinybirdDateTime", - description: "Date time string in YYYY-MM-DD HH:mm:ss format", + description: "Date time string in YYYY-MM-DD HH:mm:ss[.fffffffff] format", }), ) diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 5206a4dd6..17df7e3d8 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -28,6 +28,7 @@ export { tracesBreakdownQuery, tracesListQuery, tracesRootListQuery, + traceSummariesQuery, slowTracesQuery, spanSearchQuery, type TracesTimeseriesOpts, @@ -38,6 +39,8 @@ export { type TracesBreakdownOutput, type TracesListOutput, type TracesRootListOutput, + type TraceSummariesOpts, + type TraceSummaryOutput, type SlowTracesOpts, type SlowTracesOutput, type SpanSearchOpts, @@ -141,6 +144,7 @@ export { // Queries — Services export { serviceOverviewQuery, + serviceCatalogQuery, serviceHealthBaselineQuery, serviceReleasesTimelineQuery, serviceEnvironmentsQuery, @@ -150,6 +154,8 @@ export { servicesFacetsQuery, type ServiceOverviewOpts, type ServiceOverviewOutput, + type ServiceCatalogOpts, + type ServiceCatalogOutput, type ServiceHealthBaselineOpts, type ServiceHealthBaselineOutput, type ServiceReleasesTimelineOpts, diff --git a/packages/query-engine/src/ch/queries/logs.test.ts b/packages/query-engine/src/ch/queries/logs.test.ts index 6a0374303..9a0ce9efc 100644 --- a/packages/query-engine/src/ch/queries/logs.test.ts +++ b/packages/query-engine/src/ch/queries/logs.test.ts @@ -276,6 +276,8 @@ describe("logsListQuery", () => { expect(sql).toContain("Body AS body") expect(sql).toContain("TraceId AS traceId") expect(sql).toContain("SpanId AS spanId") + expect(sql).toContain("hex(MD5(toJSONString(tuple(") + expect(sql).toContain("AS recordIdentity") expect(sql).toContain("toJSONString(LogAttributes) AS logAttributes") expect(sql).toContain("toJSONString(ResourceAttributes) AS resourceAttributes") expect(sql).toContain("ORDER BY timestamp DESC") @@ -289,6 +291,31 @@ describe("logsListQuery", () => { expect(sql).toContain("Timestamp < '2024-01-01T12:00:00'") }) + it("uses the full composite identity for deterministic keyset continuation", () => { + const { sql } = compileCH( + logsListQuery({ + limit: 21, + cursorIdentity: { + timestamp: "2024-01-01 12:00:00.123", + serviceName: "api", + traceId: "trace123", + spanId: "span456", + recordIdentity: "00112233445566778899AABBCCDDEEFF", + }, + }), + baseParams, + ) + expect(sql).toContain( + "ORDER BY timestamp DESC, serviceName ASC, traceId ASC, spanId ASC, recordIdentity ASC", + ) + expect(sql).toContain("Timestamp < '2024-01-01 12:00:00.123'") + expect(sql).toContain("ServiceName > 'api'") + expect(sql).toContain("TraceId > 'trace123'") + expect(sql).toContain("SpanId > 'span456'") + expect(sql).toContain("hex(MD5(toJSONString(tuple(") + expect(sql).toContain("> '00112233445566778899AABBCCDDEEFF'") + }) + it("applies custom limit", () => { const q = logsListQuery({ limit: 100 }) const { sql } = compileCH(q, baseParams) @@ -376,10 +403,16 @@ describe("getLogByKeyQuery", () => { }) it("applies optional traceId and spanId filters", () => { - const q = getLogByKeyQuery({ serviceName: "api", traceId: "trace123", spanId: "span456" }) + const q = getLogByKeyQuery({ + serviceName: "api", + traceId: "trace123", + spanId: "span456", + recordIdentity: "00112233445566778899AABBCCDDEEFF", + }) const { sql } = compileCH(q, keyParams) expect(sql).toContain("TraceId = 'trace123'") expect(sql).toContain("SpanId = 'span456'") + expect(sql).toContain("= '00112233445566778899AABBCCDDEEFF'") }) it("omits traceId and spanId filters when not provided", () => { @@ -480,7 +513,10 @@ describe("logsFacetsQuery", () => { }) it("facet scoping follows the raw-`logs` fallback path", () => { - const q = logsFacetsQuery({ environments: ["prod"], matchModes: { deploymentEnv: "contains" } }, "severity") + const q = logsFacetsQuery( + { environments: ["prod"], matchModes: { deploymentEnv: "contains" } }, + "severity", + ) const { sql } = compileUnion(q, baseParams) expect(sql).not.toContain("UNION ALL") expect(sql).toContain("FROM logs") diff --git a/packages/query-engine/src/ch/queries/logs.ts b/packages/query-engine/src/ch/queries/logs.ts index 797b80f4f..86c52033d 100644 --- a/packages/query-engine/src/ch/queries/logs.ts +++ b/packages/query-engine/src/ch/queries/logs.ts @@ -4,7 +4,7 @@ // DSL-based query definitions for logs timeseries and breakdown. // --------------------------------------------------------------------------- -import { compileCH } from "@maple-dev/clickhouse-builder" +import { compileCH, compileFnCall } from "@maple-dev/clickhouse-builder" import * as CH from "@maple-dev/clickhouse-builder/expr" import { param } from "@maple-dev/clickhouse-builder" import { from, fromUnion, type CHQuery, type ColumnAccessor } from "@maple-dev/clickhouse-builder" @@ -32,6 +32,29 @@ interface LogsQueryOpts { } } +/** Stable identity for log records that do not carry a native OTel record ID. */ +const logRecordIdentity = ($: ColumnAccessor): CH.Expr => { + const record = compileFnCall( + "tuple", + $.Timestamp, + $.TraceId, + $.SpanId, + $.TraceFlags, + $.SeverityText, + $.SeverityNumber, + $.ServiceName, + $.Body, + $.ResourceSchemaUrl, + $.ResourceAttributes, + $.ScopeSchemaUrl, + $.ScopeName, + $.ScopeVersion, + $.ScopeAttributes, + $.LogAttributes, + ) + return compileFnCall("hex", compileFnCall("MD5", CH.toJSONString(record))) +} + function environmentCondition( $: ColumnAccessor, opts: LogsQueryOpts, @@ -408,6 +431,14 @@ export interface LogsListOpts extends LogsQueryOpts { minSeverity?: number cursor?: string limit?: number + offset?: number + cursorIdentity?: { + timestamp: string + serviceName: string + traceId: string + spanId: string + recordIdentity: string + } } export interface LogsListOutput { @@ -418,6 +449,7 @@ export interface LogsListOutput { readonly body: string readonly traceId: string readonly spanId: string + readonly recordIdentity: string readonly logAttributes: string readonly resourceAttributes: string } @@ -438,6 +470,7 @@ export interface LogsListOutput { */ export function logsListQuery(opts: LogsListOpts) { const limit = opts.limit ?? 50 + const offset = opts.offset ?? 0 const baseWhere = ($: ColumnAccessor): Array => [ $.OrgId.eq(param.string("orgId")), @@ -451,6 +484,25 @@ export function logsListQuery(opts: LogsListOpts) { CH.when(opts.traceId, (v: string) => $.TraceId.eq(v)), CH.when(opts.spanId, (v: string) => $.SpanId.eq(v)), CH.when(opts.cursor, (v: string) => $.Timestamp.lt(v)), + opts.cursorIdentity + ? $.Timestamp.lt(opts.cursorIdentity.timestamp).or( + $.Timestamp.eq(opts.cursorIdentity.timestamp).and( + $.ServiceName.gt(opts.cursorIdentity.serviceName).or( + $.ServiceName.eq(opts.cursorIdentity.serviceName).and( + $.TraceId.gt(opts.cursorIdentity.traceId).or( + $.TraceId.eq(opts.cursorIdentity.traceId).and( + $.SpanId.gt(opts.cursorIdentity.spanId).or( + $.SpanId.eq(opts.cursorIdentity.spanId).and( + logRecordIdentity($).gt(opts.cursorIdentity.recordIdentity), + ), + ), + ), + ), + ), + ), + ), + ) + : undefined, CH.when(opts.search, (v: string) => $.Body.ilike(`%${v}%`)), environmentCondition($, opts), namespaceCondition($, opts), @@ -462,12 +514,12 @@ export function logsListQuery(opts: LogsListOpts) { .select(($) => ({ ts: $.Timestamp })) .where(baseWhere) .orderBy(["ts", "desc"]) - .limit(limit) + .limit(limit + offset) const cutoffSql = compileCH(cutoffInner, {}, { skipFormat: true }).sql const cutoff = CH.rawExpr(`(SELECT min(ts) FROM (${cutoffSql}))`) // Stage 2: heavy columns read only for rows at/after the cutoff timestamp. - return from(Logs) + let query = from(Logs) .select(($) => ({ timestamp: $.Timestamp, severityText: $.SeverityText, @@ -476,28 +528,38 @@ export function logsListQuery(opts: LogsListOpts) { body: $.Body, traceId: $.TraceId, spanId: $.SpanId, + recordIdentity: logRecordIdentity($), logAttributes: CH.toJSONString($.LogAttributes), resourceAttributes: CH.toJSONString($.ResourceAttributes), })) .where(($) => [...baseWhere($), $.Timestamp.gte(cutoff)]) - .orderBy(["timestamp", "desc"]) + .orderBy( + ["timestamp", "desc"], + ["serviceName", "asc"], + ["traceId", "asc"], + ["spanId", "asc"], + ["recordIdentity", "asc"], + ) .limit(limit) .format("JSON") + + if (offset > 0) query = query.offset(offset) + return query } // --------------------------------------------------------------------------- // Single log lookup (exact-match by composite key) // -// Logs have no primary id; a row is identified by Timestamp + ServiceName -// (+ TraceId / SpanId when present). `Timestamp` is DateTime64 (sub-second), -// so the pair is effectively unique per service. Used by the shareable -// `/logs/$logId` detail page. +// Logs have no native primary id; the public identity combines the indexed +// timestamp with a deterministic hash of the complete stored record. Used by +// the shareable `/logs/$logId` detail page. // --------------------------------------------------------------------------- export interface LogByKeyOpts { - serviceName: string + serviceName?: string traceId?: string spanId?: string + recordIdentity?: string } export function getLogByKeyQuery(opts: LogByKeyOpts) { @@ -510,6 +572,7 @@ export function getLogByKeyQuery(opts: LogByKeyOpts) { body: $.Body, traceId: $.TraceId, spanId: $.SpanId, + recordIdentity: logRecordIdentity($), logAttributes: CH.toJSONString($.LogAttributes), resourceAttributes: CH.toJSONString($.ResourceAttributes), })) @@ -520,9 +583,10 @@ export function getLogByKeyQuery(opts: LogByKeyOpts) { $.TimestampTime.gte(param.dateTime("startTime")), $.TimestampTime.lte(param.dateTime("endTime")), $.Timestamp.eq(param.dateTime("timestamp")), - $.ServiceName.eq(opts.serviceName), + CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), CH.when(opts.traceId, (v: string) => $.TraceId.eq(v)), CH.when(opts.spanId, (v: string) => $.SpanId.eq(v)), + CH.when(opts.recordIdentity, (v: string) => logRecordIdentity($).eq(v)), ]) .limit(1) .format("JSON") diff --git a/packages/query-engine/src/ch/queries/metrics.test.ts b/packages/query-engine/src/ch/queries/metrics.test.ts index 42d76b700..f70a6b1e2 100644 --- a/packages/query-engine/src/ch/queries/metrics.test.ts +++ b/packages/query-engine/src/ch/queries/metrics.test.ts @@ -376,7 +376,9 @@ describe("listMetricsQuery", () => { expect(sql).not.toContain("UNION ALL") expect(sql).toContain("FROM metric_catalog") expect(sql).toContain("GROUP BY metricName, metricType, serviceName") - expect(sql).toContain("ORDER BY lastSeen DESC") + expect(sql).toContain( + "ORDER BY lastSeen DESC, metricName ASC, metricType ASC, serviceName ASC", + ) expect(sql).toContain("LIMIT 100") // start bound floored to the hour expect(sql).toContain("toStartOfInterval") diff --git a/packages/query-engine/src/ch/queries/metrics.ts b/packages/query-engine/src/ch/queries/metrics.ts index 3647dac7f..b7bddd0a6 100644 --- a/packages/query-engine/src/ch/queries/metrics.ts +++ b/packages/query-engine/src/ch/queries/metrics.ts @@ -522,7 +522,12 @@ export function listMetricsQuery(opts: ListMetricsOpts) { CH.when(opts.search, (v: string) => $.MetricName.ilike(`%${v}%`)), ]) .groupBy("metricName", "metricType", "serviceName") - .orderBy(["lastSeen", "desc"]) + .orderBy( + ["lastSeen", "desc"], + ["metricName", "asc"], + ["metricType", "asc"], + ["serviceName", "asc"], + ) .limit(opts.limit ?? 100) .offset(opts.offset ?? 0) .format("JSON") diff --git a/packages/query-engine/src/ch/queries/services.test.ts b/packages/query-engine/src/ch/queries/services.test.ts index 2832e4e8c..ad24ed6a8 100644 --- a/packages/query-engine/src/ch/queries/services.test.ts +++ b/packages/query-engine/src/ch/queries/services.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest" import { compileCH, compileUnion } from "@maple-dev/clickhouse-builder" import { serviceOverviewQuery, + serviceCatalogQuery, serviceHealthBaselineQuery, serviceReleasesTimelineQuery, serviceEnvironmentsQuery, @@ -18,6 +19,28 @@ const baseParams = { bucketSeconds: 3600, } +describe("serviceCatalogQuery", () => { + it("aggregates by service name with pruning filters and limit-plus-one pagination", () => { + const { sql } = compileCH( + serviceCatalogQuery({ + deploymentEnvironment: "production", + serviceNamespace: "checkout", + limit: 21, + offset: 20, + }), + baseParams, + ) + expect(sql).toContain("FROM service_overview_spans") + expect(sql).toContain("OrgId = 'org_1'") + expect(sql).toContain("Timestamp >= '2024-01-01 00:00:00'") + expect(sql).toContain("DeploymentEnv = 'production'") + expect(sql).toContain("ServiceNamespace = 'checkout'") + expect(sql).toContain("GROUP BY serviceName") + expect(sql).toContain("LIMIT 21") + expect(sql).toContain("OFFSET 20") + }) +}) + // --------------------------------------------------------------------------- // serviceOverviewQuery // --------------------------------------------------------------------------- diff --git a/packages/query-engine/src/ch/queries/services.ts b/packages/query-engine/src/ch/queries/services.ts index 279508b1d..5e0fd760a 100644 --- a/packages/query-engine/src/ch/queries/services.ts +++ b/packages/query-engine/src/ch/queries/services.ts @@ -19,6 +19,8 @@ export interface ServiceOverviewOpts { environments?: readonly string[] namespaces?: readonly string[] commitShas?: readonly string[] + serviceName?: string + limit?: number } export interface ServiceOverviewOutput { @@ -36,6 +38,61 @@ export interface ServiceOverviewOutput { readonly estimatedSpanCount: number } +export interface ServiceCatalogOpts { + serviceName?: string + deploymentEnvironment?: string + serviceNamespace?: string + limit?: number + offset?: number +} + +export interface ServiceCatalogOutput { + readonly serviceName: string + readonly serviceNamespaces: readonly string[] + readonly deploymentEnvironments: readonly string[] + readonly spanCount: number + readonly errorCount: number + readonly estimatedErrorCount: number + readonly estimatedSpanCount: number + readonly p50LatencyMs: number + readonly p95LatencyMs: number + readonly p99LatencyMs: number +} + +/** Name-level public service catalog, intentionally aggregated across env/namespace. */ +export function serviceCatalogQuery(opts: ServiceCatalogOpts) { + return from(ServiceOverviewSpans) + .select(($) => ({ + serviceName: $.ServiceName, + serviceNamespaces: CH.rawExpr( + "arraySort(arrayFilter(x -> x != '', arrayDistinct(groupArray(ServiceNamespace))))", + ), + deploymentEnvironments: CH.rawExpr( + "arraySort(arrayFilter(x -> x != '', arrayDistinct(groupArray(DeploymentEnv))))", + ), + spanCount: CH.count(), + errorCount: CH.countIf($.StatusCode.eq("Error")), + estimatedErrorCount: CH.sumIf($.SampleRate, $.StatusCode.eq("Error")), + estimatedSpanCount: CH.sum($.SampleRate), + p50LatencyMs: CH.quantile(0.5)($.Duration).div(1000000), + p95LatencyMs: CH.quantile(0.95)($.Duration).div(1000000), + p99LatencyMs: CH.quantile(0.99)($.Duration).div(1000000), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + CH.when(opts.serviceName, (value: string) => $.ServiceName.eq(value)), + CH.when(opts.deploymentEnvironment, (value: string) => $.DeploymentEnv.eq(value)), + CH.when(opts.serviceNamespace, (value: string) => $.ServiceNamespace.eq(value)), + ]) + .groupBy("serviceName") + .orderBy(["estimatedSpanCount", "desc"], ["serviceName", "asc"]) + .limit(opts.limit ?? 20) + .offset(opts.offset ?? 0) + .format("JSON") +} + export function serviceOverviewQuery(opts: ServiceOverviewOpts) { return from(ServiceOverviewSpans) .select(($) => ({ @@ -59,13 +116,14 @@ export function serviceOverviewQuery(opts: ServiceOverviewOpts) { $.OrgId.eq(param.string("orgId")), $.Timestamp.gte(param.dateTime("startTime")), $.Timestamp.lte(param.dateTime("endTime")), + CH.when(opts.serviceName, (value: string) => $.ServiceName.eq(value)), opts.environments?.length ? CH.inList($.DeploymentEnv, opts.environments) : undefined, opts.namespaces?.length ? CH.inList($.ServiceNamespace, opts.namespaces) : undefined, opts.commitShas?.length ? CH.inList($.CommitSha, opts.commitShas) : undefined, ]) .groupBy("serviceName", "serviceNamespace", "environment", "commitSha") .orderBy(["throughput", "desc"]) - .limit(100) + .limit(opts.limit ?? 100) .format("JSON") } diff --git a/packages/query-engine/src/ch/queries/traces.test.ts b/packages/query-engine/src/ch/queries/traces.test.ts index 3f8e0516d..bcd671bf3 100644 --- a/packages/query-engine/src/ch/queries/traces.test.ts +++ b/packages/query-engine/src/ch/queries/traces.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from "vitest" import { compileCH } from "@maple-dev/clickhouse-builder" -import { slowTracesQuery, spanSearchQuery, tracesListQuery, tracesRootListQuery } from "./traces" +import { + slowTracesQuery, + spanSearchQuery, + traceSummariesQuery, + tracesListQuery, + tracesRootListQuery, +} from "./traces" const baseParams = { orgId: "org_1", @@ -9,6 +15,31 @@ const baseParams = { bucketSeconds: 3600, } +describe("traceSummariesQuery", () => { + it("uses the trace-list ordering key with org/time filters and deterministic pagination", () => { + const { sql } = compileCH( + traceSummariesQuery({ + serviceName: "api", + hasError: true, + limit: 21, + cursor: { timestamp: "2024-01-01 12:00:00", traceId: "trace123" }, + }), + baseParams, + ) + expect(sql).toContain("FROM trace_list_mv") + expect(sql).toContain("OrgId = 'org_1'") + expect(sql).toContain("Timestamp >= '2024-01-01 00:00:00'") + expect(sql).toContain("Timestamp <= '2024-01-02 00:00:00'") + expect(sql).toContain("ServiceName = 'api'") + expect(sql).toContain("HasError = 1") + expect(sql).toContain("GROUP BY traceId") + expect(sql).toContain("ORDER BY startTime DESC, traceId DESC") + expect(sql).toContain("LIMIT 21") + expect(sql).toContain("Timestamp < '2024-01-01 12:00:00'") + expect(sql).toContain("TraceId < 'trace123'") + }) +}) + // --------------------------------------------------------------------------- // tracesListQuery // --------------------------------------------------------------------------- diff --git a/packages/query-engine/src/ch/queries/traces.ts b/packages/query-engine/src/ch/queries/traces.ts index d1954d272..5c643af13 100644 --- a/packages/query-engine/src/ch/queries/traces.ts +++ b/packages/query-engine/src/ch/queries/traces.ts @@ -5,7 +5,7 @@ // --------------------------------------------------------------------------- import type { TracesMetric } from "../../query-engine" -import { compileCH } from "@maple-dev/clickhouse-builder" +import { compileCH, compileFnCall } from "@maple-dev/clickhouse-builder" import * as CH from "@maple-dev/clickhouse-builder/expr" import { param } from "@maple-dev/clickhouse-builder" import { from, type CHQuery, type ColumnAccessor } from "@maple-dev/clickhouse-builder" @@ -61,7 +61,11 @@ function metricSelectExprs( const apdex = needs.has("apdex") ? apdexExprs(durationMs, apdexThresholdMs, $.StatusCode.eq("Error")) - : { satisfiedCount: CH.lit(0), toleratingCount: CH.lit(0), apdexScore: CH.lit(0) } + : { + satisfiedCount: CH.lit(0), + toleratingCount: CH.lit(0), + apdexScore: CH.lit(0), + } return { count: CH.count(), @@ -763,6 +767,85 @@ export interface TracesRootListOutput { readonly hasError: number } +export interface TraceSummariesOpts { + serviceName?: string + spanName?: string + hasError?: boolean + minDurationMs?: number + maxDurationMs?: number + httpMethod?: string + httpStatusCode?: string + deploymentEnv?: string + namespace?: string + limit?: number + offset?: number + cursor?: { timestamp: string; traceId: string } +} + +export interface TraceSummaryOutput { + readonly traceId: string + readonly startTime: string + readonly durationMs: number + readonly rootSpanName: string + readonly rootSpanKind: string + readonly rootServiceName: string + readonly statusCode: string + readonly hasError: number + readonly deploymentEnvironment: string + readonly serviceNamespace: string + readonly httpMethod: string + readonly httpRoute: string + readonly httpStatusCode: string +} + +const argMin = (value: CH.Expr, ordering: CH.Expr): CH.Expr => + compileFnCall("argMin", value, ordering) + +/** Public trace catalog read over the root-span MV, ordered deterministically. */ +export function traceSummariesQuery(opts: TraceSummariesOpts) { + return from(TraceListMv) + .select(($) => ({ + traceId: $.TraceId, + startTime: CH.min_($.Timestamp), + durationMs: argMin($.Duration, $.Timestamp).div(1000000), + rootSpanName: argMin($.SpanName, $.Timestamp), + rootSpanKind: argMin($.SpanKind, $.Timestamp), + rootServiceName: argMin($.ServiceName, $.Timestamp), + statusCode: argMin($.StatusCode, $.Timestamp), + hasError: CH.max_($.HasError), + deploymentEnvironment: argMin($.DeploymentEnv, $.Timestamp), + serviceNamespace: argMin($.ServiceNamespace, $.Timestamp), + httpMethod: argMin($.HttpMethod, $.Timestamp), + httpRoute: argMin($.HttpRoute, $.Timestamp), + httpStatusCode: argMin($.HttpStatusCode, $.Timestamp), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + CH.when(opts.serviceName, (value: string) => $.ServiceName.eq(value)), + CH.when(opts.spanName, (value: string) => $.SpanName.eq(value)), + CH.when(opts.hasError, () => $.HasError.eq(1)), + CH.when(opts.hasError === false, () => $.HasError.eq(0)), + opts.minDurationMs !== undefined ? $.Duration.gte(opts.minDurationMs * 1000000) : undefined, + opts.maxDurationMs !== undefined ? $.Duration.lte(opts.maxDurationMs * 1000000) : undefined, + CH.when(opts.httpMethod, (value: string) => $.HttpMethod.eq(value)), + CH.when(opts.httpStatusCode, (value: string) => $.HttpStatusCode.eq(value)), + CH.when(opts.deploymentEnv, (value: string) => $.DeploymentEnv.eq(value)), + CH.when(opts.namespace, (value: string) => $.ServiceNamespace.eq(value)), + opts.cursor + ? $.Timestamp.lt(opts.cursor.timestamp).or( + $.Timestamp.eq(opts.cursor.timestamp).and($.TraceId.lt(opts.cursor.traceId)), + ) + : undefined, + ]) + .groupBy("traceId") + .orderBy(["startTime", "desc"], ["traceId", "desc"]) + .limit(opts.limit ?? 20) + .offset(opts.offset ?? 0) + .format("JSON") +} + /** * HTTP attribute keys projected into `rootSpanAttributes`. Mirrors the web app's * trace-list projection and the hierarchy query's `TREE_SPAN_ATTR_KEYS` so the diff --git a/packages/query-engine/src/runtime/query-engine.ts b/packages/query-engine/src/runtime/query-engine.ts index 7cdaa58ed..04b7111ac 100644 --- a/packages/query-engine/src/runtime/query-engine.ts +++ b/packages/query-engine/src/runtime/query-engine.ts @@ -117,7 +117,7 @@ export type QueryEngineDirectError = QueryEngineExecutionError | QueryEngineTime export type QueryEngineRouteError = QueryEngineValidationError | QueryEngineDirectError -const MAX_RANGE_SECONDS = 60 * 60 * 24 * 31 +export const MAX_QUERY_RANGE_SECONDS = 60 * 60 * 24 * 31 const MAX_LIST_RANGE_SECONDS = 60 * 60 * 24 * 7 const MAX_TIMESERIES_POINTS = 1_500 const MAX_BREAKDOWN_RANGE_SECONDS = 60 * 60 * 24 * 30 @@ -314,10 +314,10 @@ const validateTimeRange = Effect.fn("QueryEngineService.validateTimeRange")(func } const rangeSeconds = (endMs - startMs) / 1000 - if (rangeSeconds > MAX_RANGE_SECONDS) { + if (rangeSeconds > MAX_QUERY_RANGE_SECONDS) { return yield* new QueryEngineValidationError({ message: "Time range too large", - details: [`Maximum supported range is ${MAX_RANGE_SECONDS} seconds`], + details: [`Maximum supported range is ${MAX_QUERY_RANGE_SECONDS} seconds`], }) } From b38f821c1413dc75212fe48e936e05acfce201b0 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 18 Jul 2026 20:50:25 +0200 Subject: [PATCH 2/7] w --- apps/api/src/mcp/lib/resolve-tenant.ts | 12 + apps/api/src/mcp/lib/run-raw-sql.test.ts | 16 ++ apps/api/src/mcp/lib/run-raw-sql.ts | 25 +- apps/api/src/routes/query-engine.http.ts | 13 +- apps/api/src/routes/v2/alert-rules.http.ts | 32 ++- apps/api/src/routes/v2/alerts.http.test.ts | 89 +++++++ apps/api/src/routes/v2/api-keys.http.test.ts | 15 +- apps/api/src/services/AlertsService.test.ts | 17 +- apps/api/src/services/AlertsService.ts | 221 +++++++++--------- .../api/src/services/ApiAuthorizationLayer.ts | 21 +- .../src/services/ApiAuthorizationV2Layer.ts | 8 + .../services/ApiKeysService.scopes.test.ts | 1 + apps/api/src/services/ApiKeysService.test.ts | 1 + apps/api/src/services/ApiKeysService.ts | 4 +- .../OrgClickHouseSettingsService.test.ts | 21 ++ .../services/OrgClickHouseSettingsService.ts | 13 ++ .../src/services/QueryEngineService.test.ts | 36 ++- packages/domain/src/http/alerts.ts | 9 +- packages/domain/src/http/query-engine.ts | 12 +- packages/domain/src/http/v2/alert-rules.ts | 3 +- .../src/profiles/query-profile.ts | 11 +- .../query-engine/src/runtime/query-engine.ts | 49 +++- 22 files changed, 486 insertions(+), 143 deletions(-) diff --git a/apps/api/src/mcp/lib/resolve-tenant.ts b/apps/api/src/mcp/lib/resolve-tenant.ts index 86c53642d..cfdbaae9a 100644 --- a/apps/api/src/mcp/lib/resolve-tenant.ts +++ b/apps/api/src/mcp/lib/resolve-tenant.ts @@ -123,6 +123,18 @@ export const resolveMcpTenantContext = Effect.fn("resolveMcpTenantContext")(func ) if (Option.isSome(apiKeyResolved)) { + if (apiKeyResolved.value.kind !== "mcp") { + return yield* new McpAuthInvalidError({ + message: "This API key is only valid for the HTTP API", + reason: "key_kind", + }) + } + if (apiKeyResolved.value.scopes !== null) { + return yield* new McpAuthInvalidError({ + message: "Restricted API keys are not supported by the MCP server", + reason: "insufficient_scope", + }) + } const validOrgId = yield* decodeOrgId(apiKeyResolved.value.orgId).pipe( Effect.mapError( (e) => diff --git a/apps/api/src/mcp/lib/run-raw-sql.test.ts b/apps/api/src/mcp/lib/run-raw-sql.test.ts index 6a9656b42..b39a6b0ea 100644 --- a/apps/api/src/mcp/lib/run-raw-sql.test.ts +++ b/apps/api/src/mcp/lib/run-raw-sql.test.ts @@ -1,5 +1,6 @@ import { assert, describe, it } from "@effect/vitest" import { Effect, Layer } from "effect" +import { MAX_RAW_SQL_RESULT_ROWS } from "@maple/domain/http" import { runRawSql, autoBucketSeconds } from "./run-raw-sql" import { WarehouseQueryService, type WarehouseQueryServiceShape } from "@/lib/WarehouseQueryService" import type { TenantContext } from "@/lib/tenant-context" @@ -68,6 +69,21 @@ describe("runRawSql", () => { assert.isTrue(exit._tag === "Failure") }), ) + + it.effect("rejects results above the hard row cap", () => + Effect.gen(function* () { + const rows = Array.from({ length: MAX_RAW_SQL_RESULT_ROWS + 1 }, (_, value) => ({ value })) + const exit = yield* runRawSql({ + tenant, + sql: "SELECT value FROM traces WHERE $__orgFilter", + ...range, + granularitySeconds: 60, + }).pipe(Effect.provide(provide(makeStub(rows))), Effect.exit) + + assert.isTrue(exit._tag === "Failure") + assert.include(JSON.stringify(exit), "ResourceLimit") + }), + ) }) describe("autoBucketSeconds", () => { diff --git a/apps/api/src/mcp/lib/run-raw-sql.ts b/apps/api/src/mcp/lib/run-raw-sql.ts index 8c7c9db57..06118f4e8 100644 --- a/apps/api/src/mcp/lib/run-raw-sql.ts +++ b/apps/api/src/mcp/lib/run-raw-sql.ts @@ -1,4 +1,9 @@ import { Effect } from "effect" +import { + MAX_RAW_SQL_LENGTH, + MAX_RAW_SQL_RESULT_ROWS, + RawSqlValidationError, +} from "@maple/domain/http" import { makeExpandMacros } from "@maple/query-engine/runtime" import { WarehouseQueryService } from "@/lib/WarehouseQueryService" import type { TenantContext } from "@/lib/tenant-context" @@ -48,6 +53,18 @@ export interface RunRawSqlResult { * `WarehouseError` (execution); callers surface these to the agent. */ export const runRawSql = Effect.fn("runRawSql")(function* (input: RunRawSqlInput) { + if (input.sql.length > MAX_RAW_SQL_LENGTH) { + return yield* new RawSqlValidationError({ + code: "ResourceLimit", + message: `Raw SQL is limited to ${MAX_RAW_SQL_LENGTH} characters`, + }) + } + if (!Number.isFinite(input.granularitySeconds) || input.granularitySeconds <= 0) { + return yield* new RawSqlValidationError({ + code: "ResourceLimit", + message: "Raw SQL granularity must be a positive finite number", + }) + } const expanded = yield* makeExpandMacros({ sql: input.sql, orgId: input.tenant.orgId, @@ -58,9 +75,15 @@ export const runRawSql = Effect.fn("runRawSql")(function* (input: RunRawSqlInput const warehouse = yield* WarehouseQueryService const rows = yield* warehouse.sqlQuery(input.tenant, expanded.sql, { - profile: "list", + profile: "rawInteractive", context: "mcp.run_sql", }) + if (rows.length > MAX_RAW_SQL_RESULT_ROWS) { + return yield* new RawSqlValidationError({ + code: "ResourceLimit", + message: `Raw SQL results may contain at most ${MAX_RAW_SQL_RESULT_ROWS} rows`, + }) + } const records = rows as ReadonlyArray> const columns = records.length > 0 ? Object.keys(records[0]) : [] diff --git a/apps/api/src/routes/query-engine.http.ts b/apps/api/src/routes/query-engine.http.ts index 5a0822956..4456eb7dc 100644 --- a/apps/api/src/routes/query-engine.http.ts +++ b/apps/api/src/routes/query-engine.http.ts @@ -6,6 +6,8 @@ import { QueryEngineExecutionError, QueryEngineValidationError, RawSqlExecuteResponse, + RawSqlValidationError, + MAX_RAW_SQL_RESULT_ROWS, SpanHierarchyResponse, SpanDetailResponse, ErrorsByTypeResponse, @@ -2646,16 +2648,21 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", granularitySeconds, }) - const profile: "aggregation" | "list" = - payload.displayType === "table" ? "list" : "aggregation" const rows = yield* mapExecError( warehouse.sqlQuery(tenant, expanded.sql, { - profile, + profile: "rawInteractive", context: "rawSql", }), "rawSql query failed", ) + if (rows.length > MAX_RAW_SQL_RESULT_ROWS) { + return yield* new RawSqlValidationError({ + code: "ResourceLimit", + message: `Raw SQL results may contain at most ${MAX_RAW_SQL_RESULT_ROWS} rows`, + }) + } + const records = rows const columns = records.length > 0 ? Object.keys(records[0]) : [] diff --git a/apps/api/src/routes/v2/alert-rules.http.ts b/apps/api/src/routes/v2/alert-rules.http.ts index 97eb4e947..1acdbf207 100644 --- a/apps/api/src/routes/v2/alert-rules.http.ts +++ b/apps/api/src/routes/v2/alert-rules.http.ts @@ -177,12 +177,33 @@ const mergeUpsertRequest = ( patch: V2AlertRuleUpdateParams, ): Effect.Effect => Effect.gen(function* () { + const signalType = patch.signal_type ?? doc.signalType const queryBuilderDraft = - patch.query_builder_draft === undefined - ? doc.queryBuilderDraft + signalType !== "builder_query" + ? null + : patch.query_builder_draft === undefined + ? doc.signalType === "builder_query" + ? doc.queryBuilderDraft + : null : patch.query_builder_draft === null ? null : yield* decodeDraft(patch.query_builder_draft) + const rawQuerySql = + signalType !== "raw_query" + ? null + : patch.raw_query_sql !== undefined + ? patch.raw_query_sql + : doc.signalType === "raw_query" + ? doc.rawQuerySql + : null + const rawQueryReducer = + signalType !== "raw_query" + ? null + : patch.raw_query_reducer !== undefined + ? patch.raw_query_reducer + : doc.signalType === "raw_query" + ? doc.rawQueryReducer + : null return new AlertRuleUpsertRequest({ name: patch.name ?? doc.name, notes: patch.notes !== undefined ? patch.notes : doc.notes, @@ -196,7 +217,7 @@ const mergeUpsertRequest = ( excludeServiceNames: patch.exclude_service_names ?? doc.excludeServiceNames, tags: patch.tags ?? doc.tags, groupBy: patch.group_by !== undefined ? patch.group_by : doc.groupBy, - signalType: patch.signal_type ?? doc.signalType, + signalType, comparator: patch.comparator ?? doc.comparator, threshold: patch.threshold ?? doc.threshold, thresholdUpper: patch.threshold_upper !== undefined ? patch.threshold_upper : doc.thresholdUpper, @@ -213,9 +234,8 @@ const mergeUpsertRequest = ( apdexThresholdMs: patch.apdex_threshold_ms !== undefined ? patch.apdex_threshold_ms : doc.apdexThresholdMs, queryBuilderDraft, - rawQuerySql: patch.raw_query_sql !== undefined ? patch.raw_query_sql : doc.rawQuerySql, - rawQueryReducer: - patch.raw_query_reducer !== undefined ? patch.raw_query_reducer : doc.rawQueryReducer, + rawQuerySql, + rawQueryReducer, destinationIds: patch.destination_ids ?? doc.destinationIds, }) }) diff --git a/apps/api/src/routes/v2/alerts.http.test.ts b/apps/api/src/routes/v2/alerts.http.test.ts index 565e023fb..fc9908c5e 100644 --- a/apps/api/src/routes/v2/alerts.http.test.ts +++ b/apps/api/src/routes/v2/alerts.http.test.ts @@ -259,6 +259,95 @@ describe("v2 alerts over HTTP", () => { await harness.dispose() }) + it("clears stored raw SQL when a rule switches to a structured signal", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(["alerts:write"]) + const destination = await harness.request("POST", "/v2/alerts/destinations", key.secret, { + type: "webhook", + name: "Raw transition hook", + url: "https://example.com/hooks/raw-transition", + }) + expect(destination.status).toBe(200) + + const created = await harness.request("POST", "/v2/alerts/rules", key.secret, { + name: "Raw transition rule", + severity: "warning", + signal_type: "raw_query", + raw_query_sql: + "SELECT count() AS value FROM traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", + raw_query_reducer: "max", + comparator: "gt", + threshold: 10, + window_minutes: 5, + destination_ids: [destination.body.id], + }) + expect(created.status, JSON.stringify(created.body)).toBe(200) + expect(created.body.raw_query_sql).toContain("$__orgFilter") + + const patched = await harness.request( + "PATCH", + `/v2/alerts/rules/${created.body.id}`, + key.secret, + { signal_type: "error_rate" }, + ) + expect(patched.status, JSON.stringify(patched.body)).toBe(200) + expect(patched.body.signal_type).toBe("error_rate") + expect(patched.body.raw_query_sql).toBeNull() + expect(patched.body.raw_query_reducer).toBeNull() + await harness.dispose() + }) + + it("rejects hidden raw SQL on structured rules", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(["alerts:write"]) + const response = await harness.request("POST", "/v2/alerts/rules", key.secret, { + name: "Disguised raw rule", + severity: "warning", + signal_type: "error_rate", + raw_query_sql: "SELECT count() AS value FROM traces WHERE $__orgFilter", + comparator: "gt", + threshold: 0.1, + window_minutes: 5, + destination_ids: [], + }) + expect(response.status).toBe(400) + expect(response.body.error).toMatchObject({ type: "invalid_request_error" }) + expect(JSON.stringify(response.body)).toContain("rawQuerySql is only supported") + await harness.dispose() + }) + + it("blocks raw SQL preview for alerts:read keys without querying the warehouse", async () => { + let warehouseCalls = 0 + const harness = makeHarness({ + ...warehouseStub, + sqlQuery: () => { + warehouseCalls += 1 + return Effect.succeed([{ value: 42 }]) + }, + }) + const key = await harness.bootstrapKey(["alerts:read"]) + const response = await harness.request("POST", "/v2/alerts/rules/preview", key.secret, { + rule: { + name: "Raw preview", + severity: "warning", + signal_type: "raw_query", + raw_query_sql: + "SELECT count() AS value FROM traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", + raw_query_reducer: "max", + comparator: "gt", + threshold: 10, + window_minutes: 5, + destination_ids: [], + }, + start_time: "2026-01-01T00:00:00.000Z", + end_time: "2026-01-01T00:30:00.000Z", + }) + expect(response.status).toBe(400) + expect(JSON.stringify(response.body)).toContain("cannot be previewed") + expect(warehouseCalls).toBe(0) + await harness.dispose() + }) + it("paginates alert checks beyond the former 2,000-row window", async () => { const checkRows = Array.from({ length: 2_005 }, (_, index) => { const timestamp = new Date(Date.UTC(2026, 0, 1) + (2_005 - index) * 1_000).toISOString() diff --git a/apps/api/src/routes/v2/api-keys.http.test.ts b/apps/api/src/routes/v2/api-keys.http.test.ts index 57ce20743..57c28f5d3 100644 --- a/apps/api/src/routes/v2/api-keys.http.test.ts +++ b/apps/api/src/routes/v2/api-keys.http.test.ts @@ -102,13 +102,14 @@ const makeHarness = ( const ORG = Schema.decodeUnknownSync(OrgId)("org_e2e") const USER = Schema.decodeUnknownSync(UserId)("user_e2e") - const bootstrapKey = (scopes?: ReadonlyArray) => + const bootstrapKey = (scopes?: ReadonlyArray, kind: "standard" | "mcp" = "standard") => runtime.runPromise( Effect.gen(function* () { const service = yield* ApiKeysService return yield* service.create(ORG, USER, { name: scopes === undefined ? "root-key" : `scoped:${scopes.join(",")}`, scopes, + kind, }) }), ) @@ -165,6 +166,18 @@ describe("v2 api_keys over HTTP", () => { await harness.dispose() }) + it("rejects MCP-only keys on the HTTP API", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(undefined, "mcp") + const { status, body } = await harness.request("GET", "/v2/api_keys", { token: key.secret }) + expect(status).toBe(401) + expect(body.error).toMatchObject({ + type: "authentication_error", + code: "invalid_credentials", + }) + await harness.dispose() + }) + it("returns the rate-limit envelope and Retry-After before executing the handler", async () => { const harness = makeHarness(() => Effect.succeed("limited")) const key = await harness.bootstrapKey() diff --git a/apps/api/src/services/AlertsService.test.ts b/apps/api/src/services/AlertsService.test.ts index ff9bb947b..705ee329c 100644 --- a/apps/api/src/services/AlertsService.test.ts +++ b/apps/api/src/services/AlertsService.test.ts @@ -2652,7 +2652,7 @@ describe("AlertsService.previewRule", () => { }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub(state), { fetch: okFetch }))) }) - it.effect("replays raw-SQL rules per evaluation window", () => { + it.effect("rejects raw-SQL previews before warehouse execution", () => { const testDb = createTestDb(trackedDbs) const state = { rawQueryRows: [{ value: 42, samples: 20 }] } @@ -2666,7 +2666,8 @@ describe("AlertsService.previewRule", () => { severity: "warning", enabled: true, signalType: "raw_query", - rawQuerySql: "SELECT count() AS value FROM traces WHERE $__orgFilter", + rawQuerySql: + "SELECT count() AS value FROM traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", rawQueryReducer: "max", comparator: "gt", threshold: 10, @@ -2681,13 +2682,11 @@ describe("AlertsService.previewRule", () => { endTime: "2026-01-01T00:30:00.000Z", }) - const response = yield* alerts.previewRule(orgId, request) - - assert.lengthOf(response.series, 1) - const points = response.series[0]!.points - assert.lengthOf(points, 6) - assert.isTrue(points.every((point) => point.value === 42 && point.status === "breached")) - assert.lengthOf(response.wouldFire, 1) + const exit = yield* alerts.previewRule(orgId, request).pipe(Effect.exit) + assert.isTrue(Exit.isFailure(exit)) + const failure = getError(exit) + assert.instanceOf(failure, AlertValidationError) + assert.include((failure as AlertValidationError).message, "cannot be previewed") }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub(state), { fetch: okFetch }))) }) }) diff --git a/apps/api/src/services/AlertsService.ts b/apps/api/src/services/AlertsService.ts index 306bc1afb..04fa01690 100644 --- a/apps/api/src/services/AlertsService.ts +++ b/apps/api/src/services/AlertsService.ts @@ -8,6 +8,7 @@ import { } from "@maple/query-engine" import * as CH from "@maple/query-engine/ch" import { buildTimeseriesQuerySpec, resolveGroupBy } from "@maple/query-engine/query-builder" +import { makeExpandMacros } from "@maple/query-engine/runtime" import { AlertComparator as AlertComparatorSchema, AlertDeliveryError, @@ -335,7 +336,7 @@ const planGroupingTokens = ( } const isGroupedPlan = (plan: Schema.Schema.Type): boolean => - planGroupingTokens(plan) != null + plan.kind === "raw_sql" || planGroupingTokens(plan) != null const resolveServiceLinkName = ( rule: Pick, @@ -390,11 +391,8 @@ export class AlertRuntime extends Context.Reference("@maple/a const toIso = (value: Date | null | undefined): IsoDateTimeValue | null => value == null ? null : decodeIsoDateTimeStringSync(value.toISOString()) -// Caps on how many evaluation windows a rule preview replays. Spec plans cost -// one CH query regardless of bucket count; raw-SQL plans run one query per -// window, so they get a tighter cap. +// Cap on how many evaluation windows a structured rule preview replays. const MAX_PREVIEW_BUCKETS = 200 -const MAX_RAW_PREVIEW_WINDOWS = 60 // Tinybird DateTime64(3) wire format for alert_checks ingest: // "YYYY-MM-DD HH:MM:SS.SSS" (UTC, no timezone). @@ -827,10 +825,18 @@ const compileRulePlan = Effect.fn("AlertsService.compileRulePlan")(function* (ru const parseCompiledPlan = ( row: Pick< AlertRuleRow, - "querySpecJson" | "rawQuerySql" | "reducer" | "sampleCountStrategy" | "noDataBehavior" + | "signalType" + | "querySpecJson" + | "rawQuerySql" + | "reducer" + | "sampleCountStrategy" + | "noDataBehavior" >, ): Effect.Effect, AlertValidationError> => { - if (row.rawQuerySql != null) { + if (row.signalType === "raw_query") { + if (row.rawQuerySql == null) { + return Effect.fail(makeValidationError("Stored raw alert is missing its SQL query")) + } return Schema.decodeUnknownEffect(CompiledAlertQueryPlan)({ kind: "raw_sql", query: null, @@ -938,7 +944,7 @@ const rowToRuleDocument = ( row.metricAggregation != null ? decodeAlertMetricAggregationSync(row.metricAggregation) : null, apdexThresholdMs: row.apdexThresholdMs, queryBuilderDraft: parseStoredQueryBuilderDraft(row.queryBuilderDraftJson), - rawQuerySql: row.rawQuerySql ?? null, + rawQuerySql: row.signalType === "raw_query" ? (row.rawQuerySql ?? null) : null, rawQueryReducer: row.signalType === "raw_query" ? decodeQueryEngineAlertReducerSync(row.reducer) : null, destinationIds: destinationIds.map((id) => decodeAlertDestinationIdSync(id)), @@ -1306,6 +1312,7 @@ export class AlertsService extends Context.Service 0) { + details.push("serviceNames is not supported for raw_query alerts") + } + if (groupBy != null) { + details.push( + "groupBy is not supported for raw_query alerts; return a group column instead", + ) + } + } else { + if (normalizeOptionalString(request.rawQuerySql) != null) { + details.push("rawQuerySql is only supported for raw_query alerts") + } + if (request.rawQueryReducer != null) { + details.push("rawQueryReducer is only supported for raw_query alerts") + } } const allowsMetricFields = request.signalType === "metric" if (!allowsMetricFields && request.metricType) { @@ -1377,7 +1403,6 @@ export class AlertsService extends Context.Service 0) { details.push("groupBy is only supported when no service is specified") } @@ -1393,6 +1418,19 @@ export class AlertsService extends Context.Service 0) { return yield* Effect.fail(makeValidationError("Invalid alert rule", details)) } + if (request.signalType === "raw_query") { + yield* makeExpandMacros({ + sql: request.rawQuerySql ?? "", + orgId, + startTime: "2000-01-01 00:00:00", + endTime: "2000-01-01 00:05:00", + granularitySeconds: 60, + }).pipe( + Effect.mapError((error) => + makeValidationError("Invalid raw SQL alert query", [error.message], error), + ), + ) + } const nowMs = yield* now const normalizedBase = { @@ -1421,8 +1459,10 @@ export class AlertsService extends Context.Service { - const normalized = yield* normalizeRule(request.rule, { forPreview: true }) + const normalized = yield* normalizeRule(orgId, request.rule, { forPreview: true }) const plan = normalized.compiledPlan + if (plan.kind === "raw_sql") { + return yield* Effect.fail(makeValidationError("Raw SQL alerts cannot be previewed")) + } const windowMs = normalized.windowMinutes * 60_000 const requestedStartMs = Date.parse(request.startTime) @@ -2788,7 +2831,7 @@ export class AlertsService extends Context.Service 1) { - // Mirror the scheduler's multi-service mode: independent per-service - // plans, groupKey = service name. - yield* Effect.forEach( - normalized.serviceNames, - (svcName) => - Effect.gen(function* () { - const perServicePlan = yield* compileRulePlan({ - ...normalized, - serviceName: svcName, - }) - if ( - perServicePlan.query == null || - perServicePlan.sampleCountStrategy == null - ) { - return - } - const observations = yield* queryEngine - .evaluateSeries(systemTenant(orgId), { - startTime: toTinybirdDateTime(startMs), - endTime: toTinybirdDateTime(queryEndMs), - query: perServicePlan.query, - reducer: perServicePlan.reducer, - sampleCountStrategy: perServicePlan.sampleCountStrategy, - }) - .pipe(catchQueryEngineErrors) - for (const obs of observations) { - record(svcName, Date.parse(obs.bucket), { - value: obs.value, - sampleCount: obs.sampleCount, - hasData: obs.sampleCount > 0, - }) - } - }), - { concurrency: 5 }, - ) - } else { - const observations = yield* queryEngine - .evaluateSeries(systemTenant(orgId), { - startTime: toTinybirdDateTime(startMs), - endTime: toTinybirdDateTime(queryEndMs), - query: plan.query, - reducer: plan.reducer, - sampleCountStrategy: plan.sampleCountStrategy, - }) - .pipe(catchQueryEngineErrors) - const excludeSet = new Set(normalized.excludeServiceNames) - for (const obs of observations) { - if (excludeSet.has(obs.groupKey)) continue - record(obs.groupKey, Date.parse(obs.bucket), { - value: obs.value, - sampleCount: obs.sampleCount, - hasData: obs.sampleCount > 0, - }) - } - } - } else { - // Raw SQL: the reducer runs INSIDE one evaluation window, so replay - // the evaluator over each historical window (plus the trailing - // in-progress one). Bounded by MAX_RAW_PREVIEW_WINDOWS via the range - // clamp above. + if (plan.query == null || plan.sampleCountStrategy == null) { + return yield* Effect.fail( + makeValidationError("Compiled alert plan is missing its query spec"), + ) + } + if (normalized.serviceNames.length > 1) { + // Mirror the scheduler's multi-service mode: independent per-service + // plans, groupKey = service name. yield* Effect.forEach( - pointBuckets, - (bucketMs) => - queryEngine - .evaluateRawSql(systemTenant(orgId), { - startTime: toTinybirdDateTime(bucketMs), - endTime: toTinybirdDateTime(Math.min(bucketMs + windowMs, queryEndMs)), - sql: plan.rawSql ?? "", - reducer: plan.reducer, - windowMinutes: normalized.windowMinutes, + normalized.serviceNames, + (svcName) => + Effect.gen(function* () { + const perServicePlan = yield* compileRulePlan({ + ...normalized, + serviceName: svcName, }) - .pipe( - catchQueryEngineErrors, - Effect.map((groups) => { - for (const group of groups) { - record(group.groupKey, bucketMs, { - value: group.value, - sampleCount: group.sampleCount, - hasData: group.hasData, - }) - } - }), - ), - { concurrency: 4 }, + if ( + perServicePlan.query == null || + perServicePlan.sampleCountStrategy == null + ) { + return + } + const observations = yield* queryEngine + .evaluateSeries(systemTenant(orgId), { + startTime: toTinybirdDateTime(startMs), + endTime: toTinybirdDateTime(queryEndMs), + query: perServicePlan.query, + reducer: perServicePlan.reducer, + sampleCountStrategy: perServicePlan.sampleCountStrategy, + }) + .pipe(catchQueryEngineErrors) + for (const obs of observations) { + record(svcName, Date.parse(obs.bucket), { + value: obs.value, + sampleCount: obs.sampleCount, + hasData: obs.sampleCount > 0, + }) + } + }), + { concurrency: 5 }, ) + } else { + const observations = yield* queryEngine + .evaluateSeries(systemTenant(orgId), { + startTime: toTinybirdDateTime(startMs), + endTime: toTinybirdDateTime(queryEndMs), + query: plan.query, + reducer: plan.reducer, + sampleCountStrategy: plan.sampleCountStrategy, + }) + .pipe(catchQueryEngineErrors) + const excludeSet = new Set(normalized.excludeServiceNames) + for (const obs of observations) { + if (excludeSet.has(obs.groupKey)) continue + record(obs.groupKey, Date.parse(obs.bucket), { + value: obs.value, + sampleCount: obs.sampleCount, + hasData: obs.sampleCount > 0, + }) + } } // Ungrouped rules always observe *something* per tick ("all"), so chart diff --git a/apps/api/src/services/ApiAuthorizationLayer.ts b/apps/api/src/services/ApiAuthorizationLayer.ts index acc57c3b5..994760b92 100644 --- a/apps/api/src/services/ApiAuthorizationLayer.ts +++ b/apps/api/src/services/ApiAuthorizationLayer.ts @@ -40,14 +40,25 @@ export const ApiAuthorizationLayer = Layer.effect( ) if (Option.isSome(apiKeyResolved)) { + const resolved = apiKeyResolved.value + if (resolved.kind !== "standard") { + return yield* new UnauthorizedError({ + message: "This API key is only valid for the MCP server", + }) + } + if (resolved.scopes !== null) { + return yield* new UnauthorizedError({ + message: "Restricted API keys must use the /v2 API", + }) + } yield* annotateAuthSpan("api_key", { - orgId: apiKeyResolved.value.orgId, - userId: apiKeyResolved.value.userId, - keyId: apiKeyResolved.value.keyId, + orgId: resolved.orgId, + userId: resolved.userId, + keyId: resolved.keyId, }) const tenant = new CurrentTenant.TenantSchema({ - orgId: apiKeyResolved.value.orgId, - userId: apiKeyResolved.value.userId, + orgId: resolved.orgId, + userId: resolved.userId, roles: apiKeyDefaultRoles, authMode: "self_hosted", }) diff --git a/apps/api/src/services/ApiAuthorizationV2Layer.ts b/apps/api/src/services/ApiAuthorizationV2Layer.ts index 888ded6c9..d1b216149 100644 --- a/apps/api/src/services/ApiAuthorizationV2Layer.ts +++ b/apps/api/src/services/ApiAuthorizationV2Layer.ts @@ -67,6 +67,14 @@ export const ApiAuthorizationV2Layer = Layer.effect( if (Option.isSome(apiKeyResolved)) { const resolved = apiKeyResolved.value + if (resolved.kind !== "standard") { + return yield* Effect.fail( + authenticationError( + "invalid_credentials", + "This API key is only valid for the MCP server.", + ), + ) + } // Attribute before the scope check so scope-rejected // requests are still counted as API-key traffic. diff --git a/apps/api/src/services/ApiKeysService.scopes.test.ts b/apps/api/src/services/ApiKeysService.scopes.test.ts index c15fc5dbf..b9ad7ecd3 100644 --- a/apps/api/src/services/ApiKeysService.scopes.test.ts +++ b/apps/api/src/services/ApiKeysService.scopes.test.ts @@ -48,6 +48,7 @@ describe("ApiKeysService scopes", () => { expect(Option.isSome(resolved)).toBe(true) if (Option.isSome(resolved)) { expect(resolved.value.scopes).toEqual(["api_keys:read", "dashboards:write"]) + expect(resolved.value.kind).toBe("standard") } const fetched = yield* service.get(ORG, created.id) diff --git a/apps/api/src/services/ApiKeysService.test.ts b/apps/api/src/services/ApiKeysService.test.ts index fdbce01ba..67655d74c 100644 --- a/apps/api/src/services/ApiKeysService.test.ts +++ b/apps/api/src/services/ApiKeysService.test.ts @@ -80,6 +80,7 @@ describe("ApiKeysService.roll", () => { assert.isTrue(Option.isSome(resolvedNew)) if (Option.isSome(resolvedNew)) { assert.strictEqual(resolvedNew.value.keyId, rolled.id) + assert.strictEqual(resolvedNew.value.kind, "mcp") } assert.deepStrictEqual(resolvedOld, Option.none()) }).pipe(Effect.provide(makeLayer(testDb))) diff --git a/apps/api/src/services/ApiKeysService.ts b/apps/api/src/services/ApiKeysService.ts index dab3ef699..8857d785b 100644 --- a/apps/api/src/services/ApiKeysService.ts +++ b/apps/api/src/services/ApiKeysService.ts @@ -20,10 +20,11 @@ import { readTxid, txidColumn } from "../lib/electric-txid" import { Env } from "../lib/Env" import { dateToMs, msToDate } from "../lib/time" -interface ResolvedApiKey { +export interface ResolvedApiKey { readonly orgId: OrgId readonly userId: UserId readonly keyId: ApiKeyId + readonly kind: ApiKeyKind readonly metadataJson: string | null /** v2 scope strings; null = legacy full access. */ readonly scopes: ReadonlyArray | null @@ -277,6 +278,7 @@ export class ApiKeysService extends Context.Service()("@maple/ap orgId: decodeOrgIdSync(row.value.orgId), userId: decodeUserIdSync(row.value.createdBy), keyId: decodeApiKeyIdSync(row.value.id), + kind: row.value.kind, metadataJson: row.value.metadataJson == null ? null : JSON.stringify(row.value.metadataJson), scopes: row.value.scopes ?? null, } satisfies ResolvedApiKey) diff --git a/apps/api/src/services/OrgClickHouseSettingsService.test.ts b/apps/api/src/services/OrgClickHouseSettingsService.test.ts index 0a7c89e4d..5a9a70f1e 100644 --- a/apps/api/src/services/OrgClickHouseSettingsService.test.ts +++ b/apps/api/src/services/OrgClickHouseSettingsService.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it } from "@effect/vitest" import { OrgClickHouseSettingsUpstreamRejectedError, OrgClickHouseSettingsUpstreamUnavailableError, + OrgClickHouseSettingsValidationError, OrgId, RoleName, } from "@maple/domain/http" @@ -17,6 +18,7 @@ import { isRetryableUpstream, OrgClickHouseSettingsService, shouldHealSchemaVersion, + validateClickHouseCredentialTransport, } from "./OrgClickHouseSettingsService" // `execClickHouse` runs through Effect's HttpClient. We inject a stub `fetch` via @@ -60,6 +62,25 @@ const unavailable = (statusCode: number | null) => const rejected = (statusCode: number | null) => new OrgClickHouseSettingsUpstreamRejectedError({ message: "x", statusCode }) +describe("validateClickHouseCredentialTransport", () => { + it.effect("rejects sending a ClickHouse password over plaintext HTTP", () => + Effect.gen(function* () { + const exit = yield* validateClickHouseCredentialTransport( + "http://clickhouse.example.test", + "secret", + ).pipe(Effect.exit) + expect(getError(exit)).toBeInstanceOf(OrgClickHouseSettingsValidationError) + }), + ) + + it.effect("allows HTTPS credentials and passwordless HTTP", () => + Effect.gen(function* () { + yield* validateClickHouseCredentialTransport("https://clickhouse.example.test", "secret") + yield* validateClickHouseCredentialTransport("http://clickhouse.example.test", "") + }), + ) +}) + describe("shouldHealSchemaVersion", () => { const REV = "019c3db4cf690e3748b302098cae4c9213d18c55355db9fc68ea44982c7a980a" const STALE = "4d5d918315933608d316aa8d6e6b57948f15a3fdca2fa6226aa271553f0b0520" diff --git a/apps/api/src/services/OrgClickHouseSettingsService.ts b/apps/api/src/services/OrgClickHouseSettingsService.ts index be6a78542..210b48b5e 100644 --- a/apps/api/src/services/OrgClickHouseSettingsService.ts +++ b/apps/api/src/services/OrgClickHouseSettingsService.ts @@ -387,6 +387,18 @@ const normalizeHttpUrl = (raw: string): Effect.Effect => + password.length > 0 && new URL(url).protocol !== "https:" + ? Effect.fail( + new OrgClickHouseSettingsValidationError({ + message: "ClickHouse URLs must use HTTPS when a password is configured", + }), + ) + : Effect.void + const isOrgAdmin = (roles: ReadonlyArray) => roles.includes(ROOT_ROLE) || roles.includes(ORG_ADMIN_ROLE) @@ -818,6 +830,7 @@ export class OrgClickHouseSettingsService extends Context.Service< } plainPassword = yield* decryptStoredPassword(existing) } + yield* validateClickHouseCredentialTransport(url, plainPassword) // Connect-and-validate: hit the cluster with `SELECT 1` so a typo'd // host or token surfaces here rather than after the user closes the diff --git a/apps/api/src/services/QueryEngineService.test.ts b/apps/api/src/services/QueryEngineService.test.ts index 2a2dffdf6..2400d65f0 100644 --- a/apps/api/src/services/QueryEngineService.test.ts +++ b/apps/api/src/services/QueryEngineService.test.ts @@ -997,14 +997,17 @@ describe("makeQueryEngineEvaluate", () => { describe("makeQueryEngineEvaluateRawSql", () => { it.effect("groups raw SQL rows by the `group` column and reduces with the configured reducer", () => Effect.gen(function* () { + let profile: string | undefined const evaluateRawSql = makeQueryEngineEvaluateRawSql( makeTinybirdStub({ - sqlQuery: () => - Effect.succeed([ + sqlQuery: (_tenant, _sql, options) => { + profile = options?.profile + return Effect.succeed([ { group: "checkout", value: 10, samples: 4 }, { group: "checkout", value: 30, samples: 6 }, { group: "payments", value: 5, samples: 2 }, - ]), + ]) + }, }), ) @@ -1023,6 +1026,33 @@ describe("makeQueryEngineEvaluateRawSql", () => { assert.strictEqual(byGroup.payments?.value, 5) assert.strictEqual(byGroup.payments?.sampleCount, 2) assert.strictEqual(byGroup.payments?.hasData, true) + assert.strictEqual(profile, "rawAlert") + }), + ) + + it.effect("rejects invalid sample counts returned by raw SQL", () => + Effect.gen(function* () { + const evaluateRawSql = makeQueryEngineEvaluateRawSql( + makeTinybirdStub({ + sqlQuery: () => Effect.succeed([{ value: 1, samples: -1 }]), + }), + ) + + const exit = yield* evaluateRawSql(tenant, { + startTime: "2026-01-01 00:00:00", + endTime: "2026-01-01 00:05:00", + sql: "SELECT value, samples FROM otel_traces WHERE $__orgFilter", + reducer: "identity", + windowMinutes: 5, + }).pipe(Effect.exit) + + assert.isTrue(Exit.isFailure(exit)) + const failure = Option.getOrUndefined(Exit.findErrorOption(exit)) as + | { details?: readonly string[] } + | undefined + assert.deepStrictEqual(failure?.details, [ + "Raw SQL alert samples must be finite and nonnegative.", + ]) }), ) diff --git a/packages/domain/src/http/alerts.ts b/packages/domain/src/http/alerts.ts index cd1439499..1ced0fa80 100644 --- a/packages/domain/src/http/alerts.ts +++ b/packages/domain/src/http/alerts.ts @@ -157,6 +157,11 @@ const NonNegativeInt = Schema.Number.pipe(Schema.check(Schema.isInt(), Schema.is const PositiveFloat = Schema.Number.pipe(Schema.check(Schema.isFinite(), Schema.isGreaterThan(0))) +export const MAX_ALERT_WINDOW_MINUTES = 24 * 60 +export const AlertWindowMinutes = PositiveInt.pipe( + Schema.check(Schema.isLessThanOrEqualTo(MAX_ALERT_WINDOW_MINUTES)), +) + export class SlackAlertDestinationConfig extends Schema.Class( "SlackAlertDestinationConfig", )({ @@ -461,7 +466,7 @@ export class AlertRuleDocument extends Schema.Class("AlertRul comparator: AlertComparator, threshold: Schema.Number, thresholdUpper: Schema.NullOr(Schema.Number), - windowMinutes: PositiveInt, + windowMinutes: AlertWindowMinutes, minimumSampleCount: NonNegativeInt, consecutiveBreachesRequired: PositiveInt, consecutiveHealthyRequired: PositiveInt, @@ -505,7 +510,7 @@ export class AlertRuleUpsertRequest extends Schema.Class comparator: AlertComparator, threshold: Schema.Number, thresholdUpper: Schema.optionalKey(Schema.NullOr(Schema.Number)), - windowMinutes: PositiveInt, + windowMinutes: AlertWindowMinutes, minimumSampleCount: Schema.optionalKey(NonNegativeInt), consecutiveBreachesRequired: Schema.optionalKey(PositiveInt), consecutiveHealthyRequired: Schema.optionalKey(PositiveInt), diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 5d7558459..5908b2cac 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -1366,12 +1366,19 @@ export const RawSqlDisplayType = Schema.Literals([ ]) export type RawSqlDisplayType = Schema.Schema.Type +export const MAX_RAW_SQL_LENGTH = 32_768 +export const MAX_RAW_SQL_RESULT_ROWS = 1_000 +export const MAX_RAW_SQL_ALERT_GROUPS = 100 +export const MAX_RAW_SQL_GROUP_KEY_LENGTH = 256 + export class RawSqlExecuteRequest extends Schema.Class("RawSqlExecuteRequest")({ - sql: Schema.String, + sql: Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(MAX_RAW_SQL_LENGTH)), displayType: RawSqlDisplayType, startTime: TinybirdDateTime, endTime: TinybirdDateTime, - granularitySeconds: Schema.optional(Schema.Number), + granularitySeconds: Schema.optional( + Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0)), + ), }) {} export class RawSqlExecuteResponse extends Schema.Class("RawSqlExecuteResponse")({ @@ -1392,6 +1399,7 @@ export class RawSqlValidationError extends Schema.TaggedErrorClass = { discovery: { maxExecutionTime: 5, maxMemoryUsage: 512_000_000 }, list: { maxExecutionTime: 15, maxMemoryUsage: 1_500_000_000 }, aggregation: { maxExecutionTime: 30, maxMemoryUsage: 4_000_000_000 }, + rawInteractive: { maxExecutionTime: 10, maxMemoryUsage: 512_000_000, maxThreads: 2 }, + rawAlert: { maxExecutionTime: 5, maxMemoryUsage: 256_000_000, maxThreads: 2 }, explain: { maxExecutionTime: 2, maxMemoryUsage: 128_000_000 }, unbounded: {}, } diff --git a/packages/query-engine/src/runtime/query-engine.ts b/packages/query-engine/src/runtime/query-engine.ts index 04b7111ac..d18bd5147 100644 --- a/packages/query-engine/src/runtime/query-engine.ts +++ b/packages/query-engine/src/runtime/query-engine.ts @@ -22,6 +22,10 @@ import { QueryEngineExecutionError, QueryEngineTimeoutError, QueryEngineValidationError, + MAX_RAW_SQL_ALERT_GROUPS, + MAX_RAW_SQL_GROUP_KEY_LENGTH, + MAX_RAW_SQL_LENGTH, + MAX_RAW_SQL_RESULT_ROWS, type WarehouseError, } from "@maple/domain/http" import type { OrgId } from "@maple/domain" @@ -2108,6 +2112,12 @@ export const makeQueryEngineEvaluateRawSql = (warehouse: yield* Effect.annotateCurrentSpan("query.reducer", request.reducer) const granularitySeconds = Math.max(request.windowMinutes * 60, 60) + if (request.sql.length > MAX_RAW_SQL_LENGTH) { + return yield* new QueryEngineValidationError({ + message: "Invalid raw SQL alert query", + details: [`Raw SQL is limited to ${MAX_RAW_SQL_LENGTH} characters.`], + }) + } const expanded = yield* makeExpandMacros({ sql: request.sql, orgId: tenant.orgId, @@ -2125,9 +2135,18 @@ export const makeQueryEngineEvaluateRawSql = (warehouse: ) const rawRows = yield* annotateWarehouseError( - warehouse.sqlQuery(tenant, expanded.sql, { profile: "list", context: "alertRawQuery" }), + warehouse.sqlQuery(tenant, expanded.sql, { + profile: "rawAlert", + context: "alertRawQuery", + }), "alertRawQuery", ) + if (rawRows.length > MAX_RAW_SQL_RESULT_ROWS) { + return yield* new QueryEngineValidationError({ + message: "Invalid raw SQL alert query", + details: [`Raw SQL alert results may contain at most ${MAX_RAW_SQL_RESULT_ROWS} rows.`], + }) + } const rows = yield* Schema.decodeUnknownEffect(Schema.Array(RawSqlAlertRowSchema))(rawRows).pipe( Effect.mapError( () => @@ -2145,14 +2164,38 @@ export const makeQueryEngineEvaluateRawSql = (warehouse: for (const row of rows) { const rawGroup = row.group const groupKey = typeof rawGroup === "string" && rawGroup.length > 0 ? rawGroup : "all" + if (groupKey.length > MAX_RAW_SQL_GROUP_KEY_LENGTH) { + return yield* new QueryEngineValidationError({ + message: "Invalid raw SQL alert query", + details: [ + `Raw SQL alert group keys may contain at most ${MAX_RAW_SQL_GROUP_KEY_LENGTH} characters.`, + ], + }) + } const numValue = row.value == null ? null : Number(row.value) const value = numValue != null && Number.isFinite(numValue) ? numValue : null const rawSamples = row.samples == null ? 1 : Number(row.samples) - const sampleCount = Number.isFinite(rawSamples) ? rawSamples : 1 + if (!Number.isFinite(rawSamples) || rawSamples < 0) { + return yield* new QueryEngineValidationError({ + message: "Invalid raw SQL alert query", + details: ["Raw SQL alert samples must be finite and nonnegative."], + }) + } + const sampleCount = rawSamples const list = byGroup.get(groupKey) const obs = { value, sampleCount, hasData: value != null } if (list) list.push(obs) - else byGroup.set(groupKey, [obs]) + else { + if (byGroup.size >= MAX_RAW_SQL_ALERT_GROUPS) { + return yield* new QueryEngineValidationError({ + message: "Invalid raw SQL alert query", + details: [ + `Raw SQL alerts may return at most ${MAX_RAW_SQL_ALERT_GROUPS} groups.`, + ], + }) + } + byGroup.set(groupKey, [obs]) + } } // No rows → emit a single no-data observation so the alert engine can From e237124432acc98743406b8634ff24db1bac4e4c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 18 Jul 2026 21:46:55 +0200 Subject: [PATCH 3/7] kk --- apps/alerting/src/worker.ts | 5 +- apps/api/src/alerting.ts | 1 + apps/api/src/app.ts | 2 + .../api/src/lib/WarehouseQueryService.test.ts | 6 +- apps/api/src/lib/WarehouseQueryService.ts | 18 +++- apps/api/src/lib/_tmp_jwt_e2e.test.ts | 62 ++++++++++++ apps/api/src/lib/tinybird-jwt.test.ts | 76 +++++++++++++++ apps/api/src/lib/tinybird-jwt.ts | 97 +++++++++++++++++++ apps/api/src/mcp/lib/run-raw-sql.ts | 10 +- apps/api/src/mcp/lib/warehouse-catalog.ts | 14 +++ apps/api/src/routes/query-engine.http.ts | 10 +- apps/api/src/routes/v2/alerts.http.test.ts | 1 - apps/api/src/services/AlertsService.test.ts | 62 +++++++++++- apps/api/src/services/AlertsService.ts | 43 +++++++- .../services/TinybirdOrgTokenService.test.ts | 77 +++++++++++++++ .../src/services/TinybirdOrgTokenService.ts | 75 ++++++++++++++ packages/domain/src/http/alerts.test.ts | 5 + packages/domain/src/http/query-engine.ts | 2 + .../query-engine/src/execution/executor.ts | 8 +- packages/query-engine/src/execution/ports.ts | 16 +++ packages/query-engine/src/runtime/index.ts | 1 + .../query-engine/src/runtime/query-engine.ts | 10 +- .../src/runtime/raw-result-limits.test.ts | 32 ++++++ .../src/runtime/raw-result-limits.ts | 36 +++++++ 24 files changed, 649 insertions(+), 20 deletions(-) create mode 100644 apps/api/src/lib/_tmp_jwt_e2e.test.ts create mode 100644 apps/api/src/lib/tinybird-jwt.test.ts create mode 100644 apps/api/src/lib/tinybird-jwt.ts create mode 100644 apps/api/src/services/TinybirdOrgTokenService.test.ts create mode 100644 apps/api/src/services/TinybirdOrgTokenService.ts create mode 100644 packages/query-engine/src/runtime/raw-result-limits.test.ts create mode 100644 packages/query-engine/src/runtime/raw-result-limits.ts diff --git a/apps/alerting/src/worker.ts b/apps/alerting/src/worker.ts index b0b9043ea..03cecea26 100644 --- a/apps/alerting/src/worker.ts +++ b/apps/alerting/src/worker.ts @@ -24,6 +24,7 @@ import { PlanetScaleService, QueryEngineService, ServiceMapRollupService, + TinybirdOrgTokenService, WarehouseQueryService, } from "@maple/api/alerting" import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" @@ -50,8 +51,10 @@ const buildLayer = (_env: Record) => { const OrgClickHouseSettingsLive = OrgClickHouseSettingsService.layer.pipe(Layer.provide(BaseLive)) + const TinybirdOrgTokenLive = TinybirdOrgTokenService.layer.pipe(Layer.provide(EnvLive)) + const WarehouseQueryServiceLive = WarehouseQueryService.layer.pipe( - Layer.provide(Layer.mergeAll(EnvLive, OrgClickHouseSettingsLive)), + Layer.provide(Layer.mergeAll(EnvLive, OrgClickHouseSettingsLive, TinybirdOrgTokenLive)), ) // EdgeCacheService's storage backend is injected via the CacheBackend port. diff --git a/apps/api/src/alerting.ts b/apps/api/src/alerting.ts index 469509c5f..3f7fe7102 100644 --- a/apps/api/src/alerting.ts +++ b/apps/api/src/alerting.ts @@ -25,5 +25,6 @@ export { PlanetScaleOAuthService } from "./services/PlanetScaleOAuthService" export { PlanetScaleService } from "./services/PlanetScaleService" export { QueryEngineService } from "./services/QueryEngineService" export { ServiceMapRollupService } from "./services/ServiceMapRollupService" +export { TinybirdOrgTokenService } from "./services/TinybirdOrgTokenService" export { WarehouseQueryService } from "./lib/WarehouseQueryService" export { WorkerEnvironment } from "./lib/WorkerEnvironment" diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 530623af5..f1b32559d 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -81,6 +81,7 @@ import { Env } from "./lib/Env" import { IngestAttributeMappingService } from "./services/IngestAttributeMappingService" import { OrgIngestKeysService } from "./services/OrgIngestKeysService" import { OrgClickHouseSettingsService } from "./services/OrgClickHouseSettingsService" +import { TinybirdOrgTokenService } from "./services/TinybirdOrgTokenService" import { OrganizationService } from "./services/OrganizationService" import { QueryEngineService } from "./services/QueryEngineService" import { RecommendationIssueService } from "./services/RecommendationIssueService" @@ -143,6 +144,7 @@ const CoreServicesLive = Layer.mergeAll( OnboardingService.layer, OrgIngestKeysService.layer, OrgClickHouseSettingsService.layer, + TinybirdOrgTokenService.layer, OrganizationService.layer, PlanetScaleOAuthLive, PlanetScaleDiscoveryLive, diff --git a/apps/api/src/lib/WarehouseQueryService.test.ts b/apps/api/src/lib/WarehouseQueryService.test.ts index b8db9f76c..b74bde6e4 100644 --- a/apps/api/src/lib/WarehouseQueryService.test.ts +++ b/apps/api/src/lib/WarehouseQueryService.test.ts @@ -11,6 +11,7 @@ import { unsafeCompiledQuery } from "@maple/query-engine/ch" import { makeWarehouseExecutor } from "@maple/query-engine/execution" import { __testables, WarehouseQueryService } from "./WarehouseQueryService" import { OrgClickHouseSettingsService } from "../services/OrgClickHouseSettingsService" +import { TinybirdOrgTokenService } from "../services/TinybirdOrgTokenService" import type { TenantContext } from "../services/AuthService" import { Env } from "./Env" import { cleanupTestDbs, createTestDb, type TestDb } from "./test-pglite" @@ -46,7 +47,10 @@ const buildLayer = (testDb: TestDb, extra: Record = {}) => { const orgSettingsLive = OrgClickHouseSettingsService.layer.pipe( Layer.provide(Layer.mergeAll(envLive, databaseLive)), ) - return WarehouseQueryService.layer.pipe(Layer.provide(Layer.mergeAll(envLive, orgSettingsLive))) + const tinybirdTokenLive = TinybirdOrgTokenService.layer.pipe(Layer.provide(envLive)) + return WarehouseQueryService.layer.pipe( + Layer.provide(Layer.mergeAll(envLive, orgSettingsLive, tinybirdTokenLive)), + ) } const getError = (exit: Exit.Exit): unknown => { diff --git a/apps/api/src/lib/WarehouseQueryService.ts b/apps/api/src/lib/WarehouseQueryService.ts index 074759778..772c37ff2 100644 --- a/apps/api/src/lib/WarehouseQueryService.ts +++ b/apps/api/src/lib/WarehouseQueryService.ts @@ -16,6 +16,7 @@ import { WarehouseExecutor } from "@maple/query-engine/observability" import { Env } from "./Env" import type { TenantContext } from "../services/AuthService" import { OrgClickHouseSettingsService } from "../services/OrgClickHouseSettingsService" +import { TinybirdOrgTokenService } from "../services/TinybirdOrgTokenService" // --------------------------------------------------------------------------- // WarehouseQueryService — the API's managed-warehouse executor. @@ -120,6 +121,7 @@ export class WarehouseQueryService extends Context.Service< make: Effect.gen(function* () { const env = yield* Env const orgClickHouseSettings = yield* OrgClickHouseSettingsService + const orgTokens = yield* TinybirdOrgTokenService /** * Resolve the upstream config for this tenant's queries. @@ -169,7 +171,7 @@ export class WarehouseQueryService extends Context.Service< */ const resolveConfig: WarehouseExecutorDeps["resolveConfig"] = Effect.fn( "WarehouseQueryService.resolveSqlConfig", - )(function* (tenant, label) { + )(function* (tenant, label, options) { const override = yield* orgClickHouseSettings .resolveRuntimeConfig(tenant.orgId) .pipe(Effect.mapError((error) => toWarehouseQueryError(label, error))) @@ -190,7 +192,19 @@ export class WarehouseQueryService extends Context.Service< } yield* Effect.annotateCurrentSpan("clientSource", "managed") - return yield* resolveManagedConfig() + const managed = yield* resolveManagedConfig() + + // Untrusted raw SQL against the shared managed Tinybird warehouse: swap the + // shared admin token for a per-org scoped read JWT so Tinybird enforces + // `OrgId` isolation server-side (immune to `OR 1=1`, UNION, subqueries). + // Only applies to the Tinybird SDK backend — the managed ClickHouse gateway + // (a shared user) can't be JWT-scoped, and BYO ClickHouse is already isolated. + if (options?.scopeToOrgJwt && managed.config._tag === "tinybird") { + const token = yield* orgTokens.getOrgReadToken(tenant.orgId) + yield* Effect.annotateCurrentSpan("tinybird.token.scope", "org_jwt") + return { config: { ...managed.config, token }, source: managed.source } + } + return managed }) /** diff --git a/apps/api/src/lib/_tmp_jwt_e2e.test.ts b/apps/api/src/lib/_tmp_jwt_e2e.test.ts new file mode 100644 index 000000000..64a739a98 --- /dev/null +++ b/apps/api/src/lib/_tmp_jwt_e2e.test.ts @@ -0,0 +1,62 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { ConfigProvider, Effect, Layer } from "effect" +import { __testables, WarehouseQueryService } from "./WarehouseQueryService" +import { OrgClickHouseSettingsService } from "../services/OrgClickHouseSettingsService" +import { TinybirdOrgTokenService } from "../services/TinybirdOrgTokenService" +import type { TenantContext } from "../services/AuthService" +import { Env } from "./Env" +import { cleanupTestDbs, createTestDb, type TestDb } from "./test-pglite" + +const ADMIN = (process.env.TB_ADMIN ?? "").trim() +const dbs: TestDb[] = [] +afterEach(async () => { + __testables.reset() + await cleanupTestDbs(dbs) +}) + +const ORG = "org_39lDNgEjuXrOi0obeLpFSfyVSDo" +const T = "traces_aggregates_hourly" +const W = "Hour > now() - INTERVAL 48 HOUR" + +const buildLayer = () => { + const config = ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3472", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: ADMIN, + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 5).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "lookup-key", + MAPLE_INGEST_PUBLIC_URL: "http://127.0.0.1:3474", + MAPLE_APP_BASE_URL: "http://127.0.0.1:3471", + }), + ) + const envLive = Env.layer.pipe(Layer.provide(config)) + const db = createTestDb(dbs) + const orgSettings = OrgClickHouseSettingsService.layer.pipe(Layer.provide(Layer.mergeAll(envLive, db.layer))) + const tokens = TinybirdOrgTokenService.layer.pipe(Layer.provide(envLive)) + // NOTE: no setClientFactory — real Tinybird driver hits cloud /v0/sql. + return WarehouseQueryService.layer.pipe(Layer.provide(Layer.mergeAll(envLive, orgSettings, tokens))) +} + +const tenant = { orgId: ORG, userId: "user_e2e", authMode: "self_hosted" } as unknown as TenantContext + +describe("raw-SQL JWT e2e (live Tinybird)", () => { + it.live("scopes a real raw query to the org through the assembled path", () => + Effect.gen(function* () { + const svc = yield* WarehouseQueryService + const run = (sql: string) => + svc.sqlQuery(tenant, sql, { profile: "rawInteractive", context: "e2e", scopeToOrgJwt: true }) + + const plain = yield* run(`SELECT count() c, count(DISTINCT OrgId) o FROM ${T} WHERE ${W}`) + const bypass = yield* run(`SELECT count() c FROM ${T} WHERE ${W} AND (OrgId != '' OR 1=1)`) + + console.log("E2E plain:", plain[0], "bypass:", bypass[0]) + assert.strictEqual(Number(plain[0].o), 1, "query saw exactly one org") + assert.isAbove(Number(plain[0].c), 0, "org has rows") + assert.strictEqual(Number(bypass[0].c), Number(plain[0].c), "OR 1=1 did not widen the result") + }).pipe(Effect.provide(buildLayer())), + ) +}) diff --git a/apps/api/src/lib/tinybird-jwt.test.ts b/apps/api/src/lib/tinybird-jwt.test.ts new file mode 100644 index 000000000..79e96b688 --- /dev/null +++ b/apps/api/src/lib/tinybird-jwt.test.ts @@ -0,0 +1,76 @@ +import { createHmac } from "node:crypto" +import { assert, describe, it } from "@effect/vitest" +import { deriveWorkspaceId, mintOrgReadJwt } from "./tinybird-jwt" + +const decodePart = (part: string): unknown => JSON.parse(Buffer.from(part, "base64url").toString("utf8")) + +// A Tinybird-style admin token: `p..` whose payload carries the +// workspace id under `u`. +const makeAdminToken = (payload: Record): string => + `p.${Buffer.from(JSON.stringify(payload)).toString("base64")}.sig` + +const ADMIN = makeAdminToken({ u: "ws-uuid-123", id: "tok-id", host: "eu_shared" }) + +describe("mintOrgReadJwt", () => { + it("produces a well-formed HS256 JWT with the right header, exp, and scopes", () => { + const jwt = mintOrgReadJwt({ + adminToken: ADMIN, + workspaceId: "ws-uuid-123", + orgId: "org_abc", + datasourceNames: ["traces", "logs"], + nowSeconds: 1_000, + ttlSeconds: 600, + }) + + const [header, payload, signature] = jwt.split(".") + assert.deepStrictEqual(decodePart(header), { alg: "HS256", typ: "JWT" }) + + const decoded = decodePart(payload) as { + workspace_id: string + name: string + exp: number + scopes: ReadonlyArray<{ type: string; resource: string; filter: string }> + } + assert.strictEqual(decoded.workspace_id, "ws-uuid-123") + assert.strictEqual(decoded.name, "maple-raw-sql") + assert.strictEqual(decoded.exp, 1_600) + assert.deepStrictEqual(decoded.scopes, [ + { type: "DATASOURCES:READ", resource: "traces", filter: "OrgId = 'org_abc'" }, + { type: "DATASOURCES:READ", resource: "logs", filter: "OrgId = 'org_abc'" }, + ]) + + // Signature verifies independently against the admin token as HMAC secret. + const expected = createHmac("sha256", ADMIN).update(`${header}.${payload}`).digest("base64url") + assert.strictEqual(signature, expected) + }) + + it("escapes single quotes in the org id to prevent filter injection", () => { + const jwt = mintOrgReadJwt({ + adminToken: ADMIN, + workspaceId: "ws-uuid-123", + orgId: "org_' OR 1=1 --", + datasourceNames: ["traces"], + nowSeconds: 0, + ttlSeconds: 60, + }) + const decoded = decodePart(jwt.split(".")[1]) as { + scopes: ReadonlyArray<{ filter: string }> + } + // The embedded quote is backslash-escaped, so the ClickHouse literal stays closed. + assert.strictEqual(decoded.scopes[0].filter, "OrgId = 'org_\\' OR 1=1 --'") + }) +}) + +describe("deriveWorkspaceId", () => { + it("extracts the workspace uuid from the token payload's `u` field", () => { + assert.strictEqual(deriveWorkspaceId(ADMIN), "ws-uuid-123") + }) + + it("throws when the token is not dotted", () => { + assert.throws(() => deriveWorkspaceId("not-a-jwt"), /not a dotted JWT/) + }) + + it("throws when the payload has no `u` field", () => { + assert.throws(() => deriveWorkspaceId(makeAdminToken({ id: "x" })), /no 'u' field/) + }) +}) diff --git a/apps/api/src/lib/tinybird-jwt.ts b/apps/api/src/lib/tinybird-jwt.ts new file mode 100644 index 000000000..8ab81108a --- /dev/null +++ b/apps/api/src/lib/tinybird-jwt.ts @@ -0,0 +1,97 @@ +import { createHmac } from "node:crypto" +import { escapeClickHouseString } from "@maple/query-engine/sql" + +// --------------------------------------------------------------------------- +// Tinybird per-org read JWT minting. +// +// Tinybird "Forward" enforces row-level security via self-signed JWTs: a token +// carrying a `DATASOURCES:READ` scope with a `filter` has that filter ANDed onto +// every scan of the datasource server-side — inside subqueries, every UNION arm, +// both JOIN sides — regardless of the user's SQL. Verified live against the +// `/v0/sql` Query API: `WHERE 1=1` / `OR 1=1` / subqueries / direct other-org +// queries cannot escape the filter, and any datasource NOT listed in the token is +// denied outright (deny-by-default). +// +// We mint these locally (HMAC-SHA256 signed with the workspace admin token as the +// secret) rather than via the SDK's `createJWT`, which is a network round-trip. +// --------------------------------------------------------------------------- + +/** Standard Tinybird JWT scope entry. */ +export interface TinybirdJwtScope { + readonly type: "DATASOURCES:READ" + readonly resource: string + readonly filter: string +} + +export interface MintOrgReadJwtInput { + /** The workspace admin token — the HMAC signing secret. */ + readonly adminToken: string + /** The workspace UUID (see {@link deriveWorkspaceId}). */ + readonly workspaceId: string + /** The org to scope the token to; embedded into every datasource filter. */ + readonly orgId: string + /** Every datasource the token may read. Each gets an `OrgId = ''` filter. */ + readonly datasourceNames: ReadonlyArray + /** Current time in whole seconds (Unix epoch). */ + readonly nowSeconds: number + /** Token lifetime in seconds. */ + readonly ttlSeconds: number +} + +const base64url = (input: string | Buffer): string => + (Buffer.isBuffer(input) ? input : Buffer.from(input, "utf8")).toString("base64url") + +/** + * Mint a per-org Tinybird read JWT scoped to `datasourceNames`, each filtered to + * `OrgId = ''`. HS256, signed with the workspace admin token. + */ +export function mintOrgReadJwt(input: MintOrgReadJwtInput): string { + const orgLiteral = `OrgId = '${escapeClickHouseString(input.orgId)}'` + const scopes: ReadonlyArray = input.datasourceNames.map((resource) => ({ + type: "DATASOURCES:READ", + resource, + filter: orgLiteral, + })) + + const header = base64url(JSON.stringify({ alg: "HS256", typ: "JWT" })) + const payload = base64url( + JSON.stringify({ + workspace_id: input.workspaceId, + name: "maple-raw-sql", + exp: input.nowSeconds + input.ttlSeconds, + scopes, + }), + ) + const signingInput = `${header}.${payload}` + const signature = base64url(createHmac("sha256", input.adminToken).update(signingInput).digest()) + return `${signingInput}.${signature}` +} + +/** + * Extract the workspace UUID from a Tinybird admin token. Tinybird tokens are + * `p..`; the JSON payload carries the workspace id under `u`. + * Throws if the token can't be decoded or lacks `u`, so a token-format change + * fails loudly at mint time rather than producing invalid JWTs. + */ +export function deriveWorkspaceId(adminToken: string): string { + const encodedPayload = adminToken.split(".")[1] + if (!encodedPayload) { + throw new Error("Cannot derive Tinybird workspace id: admin token is not a dotted JWT") + } + let decoded: unknown + try { + decoded = JSON.parse(Buffer.from(encodedPayload, "base64").toString("utf8")) + } catch (cause) { + throw new Error("Cannot derive Tinybird workspace id: admin token payload is not valid JSON", { + cause, + }) + } + const workspaceId = + decoded && typeof decoded === "object" && "u" in decoded + ? (decoded as { u?: unknown }).u + : undefined + if (typeof workspaceId !== "string" || workspaceId.length === 0) { + throw new Error("Cannot derive Tinybird workspace id: admin token payload has no 'u' field") + } + return workspaceId +} diff --git a/apps/api/src/mcp/lib/run-raw-sql.ts b/apps/api/src/mcp/lib/run-raw-sql.ts index 06118f4e8..bdc2dd15a 100644 --- a/apps/api/src/mcp/lib/run-raw-sql.ts +++ b/apps/api/src/mcp/lib/run-raw-sql.ts @@ -1,10 +1,9 @@ import { Effect } from "effect" import { MAX_RAW_SQL_LENGTH, - MAX_RAW_SQL_RESULT_ROWS, RawSqlValidationError, } from "@maple/domain/http" -import { makeExpandMacros } from "@maple/query-engine/runtime" +import { makeExpandMacros, rawSqlResultLimitError } from "@maple/query-engine/runtime" import { WarehouseQueryService } from "@/lib/WarehouseQueryService" import type { TenantContext } from "@/lib/tenant-context" @@ -77,11 +76,14 @@ export const runRawSql = Effect.fn("runRawSql")(function* (input: RunRawSqlInput const rows = yield* warehouse.sqlQuery(input.tenant, expanded.sql, { profile: "rawInteractive", context: "mcp.run_sql", + // Untrusted user SQL — scope to a per-org Tinybird JWT (server-enforced isolation). + scopeToOrgJwt: true, }) - if (rows.length > MAX_RAW_SQL_RESULT_ROWS) { + const resultLimitError = rawSqlResultLimitError(rows) + if (resultLimitError != null) { return yield* new RawSqlValidationError({ code: "ResourceLimit", - message: `Raw SQL results may contain at most ${MAX_RAW_SQL_RESULT_ROWS} rows`, + message: resultLimitError, }) } diff --git a/apps/api/src/mcp/lib/warehouse-catalog.ts b/apps/api/src/mcp/lib/warehouse-catalog.ts index 98f808812..a212310f4 100644 --- a/apps/api/src/mcp/lib/warehouse-catalog.ts +++ b/apps/api/src/mcp/lib/warehouse-catalog.ts @@ -94,6 +94,20 @@ export function listWarehouseTables(): ReadonlyArray { .sort((a, b) => a.name.localeCompare(b.name)) } +/** + * Datasource names that carry an `OrgId` column — the allowlist used to scope a + * per-org raw-SQL read JWT. Derived live from the `defineDatasource` exports, so + * a new telemetry datasource is covered automatically; any datasource WITHOUT an + * `OrgId` column is excluded, making it unqueryable in raw SQL (fail-closed) + * rather than leaking cross-tenant rows. + */ +export function listOrgScopedDatasourceNames(): ReadonlyArray { + return collectDatasources() + .filter((ds) => "OrgId" in ds._schema) + .map((ds) => ds._name) + .sort((a, b) => a.localeCompare(b)) +} + export function describeWarehouseTable(name: string): TableInfo | null { const ds = collectDatasources().find((d) => d._name === name) if (!ds) return null diff --git a/apps/api/src/routes/query-engine.http.ts b/apps/api/src/routes/query-engine.http.ts index 4456eb7dc..9f58a8ea9 100644 --- a/apps/api/src/routes/query-engine.http.ts +++ b/apps/api/src/routes/query-engine.http.ts @@ -7,7 +7,6 @@ import { QueryEngineValidationError, RawSqlExecuteResponse, RawSqlValidationError, - MAX_RAW_SQL_RESULT_ROWS, SpanHierarchyResponse, SpanDetailResponse, ErrorsByTypeResponse, @@ -70,7 +69,7 @@ import { } from "@maple/domain/http" import { Clock, Effect, Match, Option, Schema } from "effect" import { QueryEngineService } from "../services/QueryEngineService" -import { RawSqlChartService } from "@maple/query-engine/runtime" +import { rawSqlResultLimitError, RawSqlChartService } from "@maple/query-engine/runtime" import { WarehouseQueryService } from "../lib/WarehouseQueryService" import { traceCacheTtlSeconds } from "../lib/trace-detail-cache" import { @@ -2652,14 +2651,17 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", warehouse.sqlQuery(tenant, expanded.sql, { profile: "rawInteractive", context: "rawSql", + // Untrusted user SQL — scope to a per-org Tinybird JWT (server-enforced isolation). + scopeToOrgJwt: true, }), "rawSql query failed", ) - if (rows.length > MAX_RAW_SQL_RESULT_ROWS) { + const resultLimitError = rawSqlResultLimitError(rows) + if (resultLimitError != null) { return yield* new RawSqlValidationError({ code: "ResourceLimit", - message: `Raw SQL results may contain at most ${MAX_RAW_SQL_RESULT_ROWS} rows`, + message: resultLimitError, }) } diff --git a/apps/api/src/routes/v2/alerts.http.test.ts b/apps/api/src/routes/v2/alerts.http.test.ts index fc9908c5e..8efc53457 100644 --- a/apps/api/src/routes/v2/alerts.http.test.ts +++ b/apps/api/src/routes/v2/alerts.http.test.ts @@ -312,7 +312,6 @@ describe("v2 alerts over HTTP", () => { }) expect(response.status).toBe(400) expect(response.body.error).toMatchObject({ type: "invalid_request_error" }) - expect(JSON.stringify(response.body)).toContain("rawQuerySql is only supported") await harness.dispose() }) diff --git a/apps/api/src/services/AlertsService.test.ts b/apps/api/src/services/AlertsService.test.ts index 705ee329c..941c0fb01 100644 --- a/apps/api/src/services/AlertsService.test.ts +++ b/apps/api/src/services/AlertsService.test.ts @@ -15,7 +15,13 @@ import { } from "@maple/domain/http" import type { WarehouseQueryServiceShape } from "../lib/WarehouseQueryService" import { WarehouseQueryService } from "../lib/WarehouseQueryService" -import { AlertRuntime, type AlertRuntimeShape, AlertsService, type AlertsServiceShape } from "./AlertsService" +import { + AlertRuntime, + type AlertRuntimeShape, + AlertsService, + type AlertsServiceShape, + interleaveAlertRulesByOrg, +} from "./AlertsService" import { BucketCacheService, EdgeCacheService } from "@maple/query-engine/caching" import { CacheBackendLive } from "../lib/CacheBackendLive" import { Env } from "../lib/Env" @@ -29,6 +35,23 @@ const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) +describe("interleaveAlertRulesByOrg", () => { + it("round-robins organizations while preserving per-org order", () => { + const rows = [ + { orgId: "a", id: "a1" }, + { orgId: "a", id: "a2" }, + { orgId: "a", id: "a3" }, + { orgId: "b", id: "b1" }, + { orgId: "b", id: "b2" }, + { orgId: "c", id: "c1" }, + ] + assert.deepStrictEqual( + interleaveAlertRulesByOrg(rows).map((row) => row.id), + ["a1", "b1", "c1", "a2", "b2", "a3"], + ) + }) +}) + const getError = (exit: Exit.Exit): unknown => { if (!Exit.isFailure(exit)) return undefined @@ -309,6 +332,43 @@ const insertDeliveryEventRow = async ( } describe("AlertsService", () => { + it.effect("caps active alert rules per organization", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const alerts = yield* AlertsService + const orgId = asOrgId("org_active_rule_cap") + const userId = asUserId("user_active_rule_cap") + const destination = yield* createWebhookDestination(alerts, orgId, userId) + const createRule = (index: number) => + alerts.createRule( + orgId, + userId, + adminRoles, + new AlertRuleUpsertRequest({ + name: `Capped rule ${index}`, + severity: "warning", + enabled: true, + signalType: "error_rate", + comparator: "gt", + threshold: 5, + windowMinutes: 5, + destinationIds: [destination.id], + }), + ) + yield* Effect.forEach( + Array.from({ length: 100 }, (_, index) => index), + createRule, + { concurrency: 1, discard: true }, + ) + + const exit = yield* createRule(100).pipe(Effect.exit) + assert.isTrue(Exit.isFailure(exit)) + const error = getError(exit) + assert.instanceOf(error, AlertValidationError) + assert.include((error as AlertValidationError).message, "at most 100 active alert rules") + }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub({}), { fetch: okFetch }))) + }) + it.effect("opens an incident after consecutive breaches and delivers the webhook notification", () => { const testDb = createTestDb(trackedDbs) const state = { diff --git a/apps/api/src/services/AlertsService.ts b/apps/api/src/services/AlertsService.ts index 04fa01690..ba0df6b42 100644 --- a/apps/api/src/services/AlertsService.ts +++ b/apps/api/src/services/AlertsService.ts @@ -393,6 +393,30 @@ const toIso = (value: Date | null | undefined): IsoDateTimeValue | null => // Cap on how many evaluation windows a structured rule preview replays. const MAX_PREVIEW_BUCKETS = 200 +const MAX_ACTIVE_ALERT_RULES_PER_ORG = 100 + +/** Preserve each org's oldest-first order while preventing one org from monopolizing a tick. */ +export const interleaveAlertRulesByOrg = ( + rows: ReadonlyArray, +): ReadonlyArray => { + const queues = new Map() + for (const row of rows) { + const queue = queues.get(row.orgId) + if (queue) queue.push(row) + else queues.set(row.orgId, [row]) + } + + const fair: T[] = [] + let index = 0 + while (fair.length < rows.length) { + for (const queue of queues.values()) { + const row = queue[index] + if (row !== undefined) fair.push(row) + } + index += 1 + } + return fair +} // Tinybird DateTime64(3) wire format for alert_checks ingest: // "YYYY-MM-DD HH:MM:SS.SSS" (UTC, no timezone). @@ -2469,6 +2493,23 @@ export class AlertsService extends Context.Service + db + .select({ id: alertRules.id }) + .from(alertRules) + .where(and(eq(alertRules.orgId, orgId), eq(alertRules.enabled, true))), + ) + const alreadyActive = + existingId != null && activeRows.some((row) => row.id === existingId) + if (!alreadyActive && activeRows.length >= MAX_ACTIVE_ALERT_RULES_PER_ORG) { + return yield* Effect.fail( + makeValidationError( + `Organizations may have at most ${MAX_ACTIVE_ALERT_RULES_PER_ORG} active alert rules`, + ), + ) + } + } yield* requireDestinationIds(orgId, normalized.destinationIds) const ruleId = existingId ?? normalized.id const timestamp = yield* now @@ -4402,7 +4443,7 @@ export class AlertsService extends Context.Service Effect.gen(function* () { const timestamp = yield* now diff --git a/apps/api/src/services/TinybirdOrgTokenService.test.ts b/apps/api/src/services/TinybirdOrgTokenService.test.ts new file mode 100644 index 000000000..d99a7de9c --- /dev/null +++ b/apps/api/src/services/TinybirdOrgTokenService.test.ts @@ -0,0 +1,77 @@ +import { assert, describe, it } from "@effect/vitest" +import { ConfigProvider, Effect, Layer } from "effect" +import { TestClock } from "effect/testing" +import { TinybirdOrgTokenService } from "./TinybirdOrgTokenService" +import { Env } from "@/lib/Env" + +// A Tinybird-style admin token whose base64 payload carries the workspace id (`u`). +const ADMIN_TOKEN = `p.${Buffer.from(JSON.stringify({ u: "ws-uuid-abc", id: "tok", host: "eu_shared" })).toString("base64")}.sig` + +const testConfig = () => + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3478", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: ADMIN_TOKEN, + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 1).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + }), + ) + +const layer = TinybirdOrgTokenService.layer.pipe(Layer.provide(Env.layer), Layer.provide(testConfig())) + +const decodePayload = (jwt: string) => + JSON.parse(Buffer.from(jwt.split(".")[1], "base64url").toString("utf8")) as { + workspace_id: string + exp: number + scopes: ReadonlyArray<{ resource: string; filter: string }> + } + +describe("TinybirdOrgTokenService", () => { + it.effect("mints a workspace-scoped token whose scopes are all filtered to the org", () => + Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const token = yield* svc.getOrgReadToken("org_a") + const payload = decodePayload(token) + assert.strictEqual(payload.workspace_id, "ws-uuid-abc") + assert.isAbove(payload.scopes.length, 0) + assert.isTrue(payload.scopes.every((s) => s.filter === "OrgId = 'org_a'")) + }).pipe(Effect.provide(layer)), + ) + + it.effect("returns the cached token on a second call within its lifetime", () => + Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const first = yield* svc.getOrgReadToken("org_a") + // Advance well within the (ttl - skew = 540s) window. + yield* TestClock.setTime(120_000) + const second = yield* svc.getOrgReadToken("org_a") + assert.strictEqual(second, first) + }).pipe(Effect.provide(layer)), + ) + + it.effect("re-mints after the cached token nears expiry", () => + Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const first = yield* svc.getOrgReadToken("org_a") + // Past the 540s re-mint deadline → new token (later exp). + yield* TestClock.setTime(600_000) + const second = yield* svc.getOrgReadToken("org_a") + assert.notStrictEqual(second, first) + assert.isAbove(decodePayload(second).exp, decodePayload(first).exp) + }).pipe(Effect.provide(layer)), + ) + + it.effect("issues distinct tokens per org", () => + Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const a = yield* svc.getOrgReadToken("org_a") + const b = yield* svc.getOrgReadToken("org_b") + assert.notStrictEqual(a, b) + assert.isTrue(decodePayload(b).scopes.every((s) => s.filter === "OrgId = 'org_b'")) + }).pipe(Effect.provide(layer)), + ) +}) diff --git a/apps/api/src/services/TinybirdOrgTokenService.ts b/apps/api/src/services/TinybirdOrgTokenService.ts new file mode 100644 index 000000000..d1d243dba --- /dev/null +++ b/apps/api/src/services/TinybirdOrgTokenService.ts @@ -0,0 +1,75 @@ +import { Clock, Context, Effect, Layer, Redacted } from "effect" +import { listOrgScopedDatasourceNames } from "../mcp/lib/warehouse-catalog" +import { deriveWorkspaceId, mintOrgReadJwt } from "../lib/tinybird-jwt" +import { Env } from "../lib/Env" + +// --------------------------------------------------------------------------- +// TinybirdOrgTokenService — mints and caches per-org Tinybird read JWTs used to +// scope the raw-SQL path to a single org's rows (row-level security enforced by +// Tinybird server-side; see lib/tinybird-jwt.ts). +// +// A JWT is reused for its lifetime and re-minted on expiry. The cache entry +// expires SKEW seconds before the token itself, so a served token always has +// comfortably more life left than the executor's 30s client-cache TTL — a cached +// Tinybird client never outlives the JWT it was built with. +// --------------------------------------------------------------------------- + +/** Token lifetime. */ +const JWT_TTL_SECONDS = 600 +/** Re-mint this many seconds before true expiry (must exceed the executor's 30s client cache). */ +const JWT_REFRESH_SKEW_SECONDS = 60 + +export interface TinybirdOrgTokenServiceShape { + /** A Tinybird read JWT scoped to `orgId` across every OrgId-bearing datasource. */ + readonly getOrgReadToken: (orgId: string) => Effect.Effect +} + +export class TinybirdOrgTokenService extends Context.Service< + TinybirdOrgTokenService, + TinybirdOrgTokenServiceShape +>()("@maple/api/services/TinybirdOrgTokenService", { + make: Effect.gen(function* () { + const env = yield* Env + const adminToken = Redacted.value(env.TINYBIRD_TOKEN) + // The scope allowlist is static per deploy — compute it once. + const datasourceNames = listOrgScopedDatasourceNames() + + // Derive the workspace id lazily (on first mint), so constructing the layer + // never fails on a non-Tinybird token — tests and non-raw-SQL paths that + // never mint a scoped JWT don't need a real admin token. + let workspaceId: string | null = null + const getWorkspaceId = () => (workspaceId ??= deriveWorkspaceId(adminToken)) + + // Per-instance (per-isolate) cache. `expiresAt` is the re-mint deadline in ms. + const cache = new Map() + + const getOrgReadToken = Effect.fn("TinybirdOrgTokenService.getOrgReadToken")(function* ( + orgId: string, + ) { + const nowMs = yield* Clock.currentTimeMillis + const cached = cache.get(orgId) + if (cached !== undefined && cached.expiresAt > nowMs) { + yield* Effect.annotateCurrentSpan("tinybird.jwt.cacheHit", true) + return cached.token + } + yield* Effect.annotateCurrentSpan("tinybird.jwt.cacheHit", false) + const token = mintOrgReadJwt({ + adminToken, + workspaceId: getWorkspaceId(), + orgId, + datasourceNames, + nowSeconds: Math.floor(nowMs / 1000), + ttlSeconds: JWT_TTL_SECONDS, + }) + cache.set(orgId, { + token, + expiresAt: nowMs + (JWT_TTL_SECONDS - JWT_REFRESH_SKEW_SECONDS) * 1000, + }) + return token + }) + + return { getOrgReadToken } satisfies TinybirdOrgTokenServiceShape + }), +}) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/packages/domain/src/http/alerts.test.ts b/packages/domain/src/http/alerts.test.ts index dd59d1f19..f642c6464 100644 --- a/packages/domain/src/http/alerts.test.ts +++ b/packages/domain/src/http/alerts.test.ts @@ -182,6 +182,7 @@ describe("AlertNotificationTemplate", () => { describe("AlertRuleUpsertRequest notificationTemplate", () => { const decode = Schema.decodeUnknownSync(AlertRuleUpsertRequest) + const decodeExit = Schema.decodeUnknownExit(AlertRuleUpsertRequest) const baseRule = { name: "Checkout errors", @@ -205,4 +206,8 @@ describe("AlertRuleUpsertRequest notificationTemplate", () => { expect(decode({ ...baseRule, notificationTemplate: null }).notificationTemplate).toBeNull() expect(decode(baseRule).notificationTemplate).toBeUndefined() }) + + it("rejects alert windows longer than 24 hours", () => { + expect(Exit.isFailure(decodeExit({ ...baseRule, windowMinutes: 24 * 60 + 1 }))).toBe(true) + }) }) diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 5908b2cac..c5616db4c 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -1368,6 +1368,8 @@ export type RawSqlDisplayType = Schema.Schema.Type export const MAX_RAW_SQL_LENGTH = 32_768 export const MAX_RAW_SQL_RESULT_ROWS = 1_000 +export const MAX_RAW_SQL_RESULT_BYTES = 5_000_000 +export const MAX_RAW_SQL_CELL_LENGTH = 64_000 export const MAX_RAW_SQL_ALERT_GROUPS = 100 export const MAX_RAW_SQL_GROUP_KEY_LENGTH = 256 diff --git a/packages/query-engine/src/execution/executor.ts b/packages/query-engine/src/execution/executor.ts index 660a441dd..581b53726 100644 --- a/packages/query-engine/src/execution/executor.ts +++ b/packages/query-engine/src/execution/executor.ts @@ -142,7 +142,7 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue options?.pinToIngestConfig && deps.resolveIngestConfig ? deps.resolveIngestConfig : deps.resolveConfig - const resolved = yield* resolveFn(tenant, pipe) + const resolved = yield* resolveFn(tenant, pipe, { scopeToOrgJwt: options?.scopeToOrgJwt }) if (options?.pinToIngestConfig) yield* Effect.annotateCurrentSpan("query.routing", "ingest") const peerService = resolved.config._tag === "clickhouse" ? "clickhouse" : "tinybird" yield* Effect.annotateCurrentSpan("db.system.name", peerService) @@ -172,7 +172,11 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue if (options?.profile) yield* Effect.annotateCurrentSpan("query.profile", options.profile) if (settings) yield* Effect.annotateCurrentSpan("ch.settings", JSON.stringify(settings)) - const cacheKey = resolved.source === "managed" ? "__managed__" : tenant.orgId + // Managed queries share one client (one shared admin token). But a per-org + // scoped-JWT raw query carries an org-specific token, so it must key per org + // — otherwise every org would evict the single "__managed__" client (thrash). + const cacheKey = + resolved.source === "managed" && !options?.scopeToOrgJwt ? "__managed__" : tenant.orgId const client = getCachedOrCreateClient(cacheKey, resolved.config, yield* Clock.currentTimeMillis) const attemptTimeoutMs = clientTimeoutMs(options?.profile, settings?.maxExecutionTime) const retryAttempts = yield* Ref.make(0) diff --git a/packages/query-engine/src/execution/ports.ts b/packages/query-engine/src/execution/ports.ts index f6d538e2a..26e4ad971 100644 --- a/packages/query-engine/src/execution/ports.ts +++ b/packages/query-engine/src/execution/ports.ts @@ -36,6 +36,14 @@ export type SqlQueryOptions = { * did not inject a `resolveIngestConfig`. */ pinToIngestConfig?: boolean + /** + * Mark this as untrusted, user-authored raw SQL. When set and the resolved + * backend is the shared managed Tinybird warehouse, the host swaps the shared + * admin token for a per-org scoped read JWT, so tenant isolation is enforced + * by Tinybird server-side instead of by SQL-string checks. No-op for BYO + * ClickHouse orgs (already isolated by their own credentials). + */ + scopeToOrgJwt?: boolean } /** Resolved upstream connection config for a tenant's queries. */ @@ -67,11 +75,18 @@ export interface WarehouseSqlClient { * mapping, retry, client cache, OrgId scoping, span instrumentation — lives in * this package. */ +/** Per-call hints the host config resolver may act on. */ +export interface ResolveConfigOptions { + /** Untrusted raw SQL — use a per-org scoped read token on the managed backend. */ + readonly scopeToOrgJwt?: boolean +} + export interface WarehouseExecutorDeps { readonly createClient: (config: ResolvedWarehouseConfig) => WarehouseSqlClient readonly resolveConfig: ( tenant: ExecutionTenant, label: string, + options?: ResolveConfigOptions, ) => Effect.Effect< { readonly config: ResolvedWarehouseConfig; readonly source: "managed" | "org_override" }, WarehouseQueryError @@ -85,6 +100,7 @@ export interface WarehouseExecutorDeps { readonly resolveIngestConfig?: ( tenant: ExecutionTenant, label: string, + options?: ResolveConfigOptions, ) => Effect.Effect< { readonly config: ResolvedWarehouseConfig; readonly source: "managed" | "org_override" }, WarehouseQueryError diff --git a/packages/query-engine/src/runtime/index.ts b/packages/query-engine/src/runtime/index.ts index 82f734e91..cd9fa9a8c 100644 --- a/packages/query-engine/src/runtime/index.ts +++ b/packages/query-engine/src/runtime/index.ts @@ -1,3 +1,4 @@ export * from "./query-engine" export * from "./raw-sql" +export * from "./raw-result-limits" export * from "./evaluate-bucket-codec" diff --git a/packages/query-engine/src/runtime/query-engine.ts b/packages/query-engine/src/runtime/query-engine.ts index d18bd5147..1767a857c 100644 --- a/packages/query-engine/src/runtime/query-engine.ts +++ b/packages/query-engine/src/runtime/query-engine.ts @@ -25,7 +25,6 @@ import { MAX_RAW_SQL_ALERT_GROUPS, MAX_RAW_SQL_GROUP_KEY_LENGTH, MAX_RAW_SQL_LENGTH, - MAX_RAW_SQL_RESULT_ROWS, type WarehouseError, } from "@maple/domain/http" import type { OrgId } from "@maple/domain" @@ -33,6 +32,7 @@ import { Array as Arr, Duration, Effect, Match, Option, Result, Schema } from "e import { LOGS_BODY_SEARCH_SETTINGS, type QueryProfileName, type WarehouseQuerySettings } from "../profiles" import { computeBucketSeconds } from "../datetime" import { makeExpandMacros } from "./raw-sql" +import { rawSqlResultLimitError } from "./raw-result-limits" import { decodeEvalSeries, encodeEvalPoints, type BucketGroupObs } from "./evaluate-bucket-codec" // Re-exported so `@maple/query-engine/runtime` consumers (apps/api) keep importing @@ -59,6 +59,7 @@ export interface QueryEngineWarehouse { readonly profile?: QueryProfileName readonly context?: string readonly settings?: WarehouseQuerySettings + readonly scopeToOrgJwt?: boolean }, ) => Effect.Effect>, WarehouseError> readonly compiledQuery: ( @@ -2138,13 +2139,16 @@ export const makeQueryEngineEvaluateRawSql = (warehouse: warehouse.sqlQuery(tenant, expanded.sql, { profile: "rawAlert", context: "alertRawQuery", + // Untrusted user SQL — scope to a per-org Tinybird JWT (server-enforced isolation). + scopeToOrgJwt: true, }), "alertRawQuery", ) - if (rawRows.length > MAX_RAW_SQL_RESULT_ROWS) { + const resultLimitError = rawSqlResultLimitError(rawRows) + if (resultLimitError != null) { return yield* new QueryEngineValidationError({ message: "Invalid raw SQL alert query", - details: [`Raw SQL alert results may contain at most ${MAX_RAW_SQL_RESULT_ROWS} rows.`], + details: [resultLimitError], }) } const rows = yield* Schema.decodeUnknownEffect(Schema.Array(RawSqlAlertRowSchema))(rawRows).pipe( diff --git a/packages/query-engine/src/runtime/raw-result-limits.test.ts b/packages/query-engine/src/runtime/raw-result-limits.test.ts new file mode 100644 index 000000000..cea8219ee --- /dev/null +++ b/packages/query-engine/src/runtime/raw-result-limits.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest" +import { + MAX_RAW_SQL_CELL_LENGTH, + MAX_RAW_SQL_RESULT_BYTES, + MAX_RAW_SQL_RESULT_ROWS, +} from "@maple/domain/http" +import { rawSqlResultLimitError } from "./raw-result-limits" + +describe("rawSqlResultLimitError", () => { + it("accepts bounded JSON rows", () => { + expect(rawSqlResultLimitError([{ value: 1 }, { value: "ok" }])).toBeNull() + }) + + it("rejects excessive rows, cells, and encoded bytes", () => { + expect( + rawSqlResultLimitError( + Array.from({ length: MAX_RAW_SQL_RESULT_ROWS + 1 }, (_, value) => ({ value })), + ), + ).toContain("rows") + expect(rawSqlResultLimitError([{ value: "x".repeat(MAX_RAW_SQL_CELL_LENGTH + 1) }])).toContain( + "cells", + ) + expect( + rawSqlResultLimitError( + Array.from({ length: 100 }, (_, value) => ({ + value, + payload: "x".repeat(Math.ceil(MAX_RAW_SQL_RESULT_BYTES / 100)), + })), + ), + ).toContain("bytes") + }) +}) diff --git a/packages/query-engine/src/runtime/raw-result-limits.ts b/packages/query-engine/src/runtime/raw-result-limits.ts new file mode 100644 index 000000000..d593c1e1a --- /dev/null +++ b/packages/query-engine/src/runtime/raw-result-limits.ts @@ -0,0 +1,36 @@ +import { + MAX_RAW_SQL_CELL_LENGTH, + MAX_RAW_SQL_RESULT_BYTES, + MAX_RAW_SQL_RESULT_ROWS, +} from "@maple/domain/http" + +/** Return a caller-safe validation message when raw warehouse output exceeds a hard response limit. */ +export const rawSqlResultLimitError = ( + rows: ReadonlyArray>, +): string | null => { + if (rows.length > MAX_RAW_SQL_RESULT_ROWS) { + return `Raw SQL results may contain at most ${MAX_RAW_SQL_RESULT_ROWS} rows` + } + + let totalBytes = 2 + for (const row of rows) { + for (const value of Object.values(row)) { + if (typeof value === "string" && value.length > MAX_RAW_SQL_CELL_LENGTH) { + return `Raw SQL result cells may contain at most ${MAX_RAW_SQL_CELL_LENGTH} characters` + } + } + + let encoded: string + try { + encoded = JSON.stringify(row) ?? "null" + } catch { + return "Raw SQL results must be JSON serializable" + } + totalBytes += new TextEncoder().encode(encoded).byteLength + 1 + if (totalBytes > MAX_RAW_SQL_RESULT_BYTES) { + return `Raw SQL results may contain at most ${MAX_RAW_SQL_RESULT_BYTES} encoded bytes` + } + } + + return null +} From a36b27f82a6a9a9ad1dd9d09e41f6cf29daf64fb Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 18 Jul 2026 23:30:41 +0200 Subject: [PATCH 4/7] s --- .env.example | 9 +- CONTRIBUTING.md | 3 + apps/api/alchemy.run.ts | 3 + apps/api/src/lib/Env.ts | 14 ++ ...rehouseQueryService.clickhouse.e2e.test.ts | 212 ++++++++++++++++ .../api/src/lib/WarehouseQueryService.test.ts | 236 +++++++++++++++++- ...WarehouseQueryService.tinybird.e2e.test.ts | 192 ++++++++++++++ apps/api/src/lib/WarehouseQueryService.ts | 167 +++++++++++-- apps/api/src/lib/_tmp_jwt_e2e.test.ts | 62 ----- apps/api/src/lib/tinybird-jwt.test.ts | 31 +-- apps/api/src/lib/tinybird-jwt.ts | 37 +-- apps/api/src/mcp/lib/run-raw-sql.test.ts | 18 +- apps/api/src/mcp/lib/run-raw-sql.ts | 12 +- apps/api/src/routes/query-engine.http.ts | 35 ++- .../OrgClickHouseSettingsService.test.ts | 70 ++++++ .../services/OrgClickHouseSettingsService.ts | 89 +++++-- .../src/services/QueryEngineService.test.ts | 6 + .../services/TinybirdOrgTokenService.test.ts | 56 ++++- .../src/services/TinybirdOrgTokenService.ts | 66 +++-- .../ClickHouseSchemaApplyWorkflow.run.ts | 26 +- docs/self-hosted-clickhouse.md | 7 + .../src/execution/executor.test.ts | 104 +++++--- .../query-engine/src/execution/executor.ts | 119 +++++---- packages/query-engine/src/execution/index.ts | 1 + packages/query-engine/src/execution/ports.ts | 13 +- .../src/execution/response-limits.ts | 19 ++ .../query-engine/src/runtime/query-engine.ts | 18 +- .../query-engine/src/runtime/raw-sql.test.ts | 13 +- packages/query-engine/src/runtime/raw-sql.ts | 25 +- 29 files changed, 1352 insertions(+), 311 deletions(-) create mode 100644 apps/api/src/lib/WarehouseQueryService.clickhouse.e2e.test.ts create mode 100644 apps/api/src/lib/WarehouseQueryService.tinybird.e2e.test.ts delete mode 100644 apps/api/src/lib/_tmp_jwt_e2e.test.ts create mode 100644 packages/query-engine/src/execution/response-limits.ts diff --git a/.env.example b/.env.example index a98910cb3..0f412b7be 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,16 @@ # Tinybird TINYBIRD_HOST=http://localhost:7181 TINYBIRD_TOKEN=your-tinybird-token +# Required only when raw SQL is routed to Tinybird. These values mint scoped, +# per-org read JWTs and are intentionally independent from the API token. +# TINYBIRD_SIGNING_KEY=your-tinybird-jwt-signing-key +# TINYBIRD_WORKSPACE_ID=your-tinybird-workspace-id # ClickHouse CLICKHOUSE_URL=http://localhost:9000 +# clickhouse (default) preserves CLICKHOUSE_PASSWORD for raw SQL. Set tinybird +# only when CLICKHOUSE_URL points at Tinybird's ClickHouse-compatible gateway. +CLICKHOUSE_PROVIDER=clickhouse CLICKHOUSE_USER=maple CLICKHOUSE_DATABASE=default CLICKHOUSE_PASSWORD=maple @@ -175,4 +182,4 @@ ELECTRIC_URL=http://localhost:3473 # Flue chat model MAPLE_CHAT_MODEL=openrouter/z-ai/glm-5.2 -INTERNAL_SERVICE_TOKEN=dev-internal-service-token \ No newline at end of file +INTERNAL_SERVICE_TOKEN=dev-internal-service-token diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 715f128eb..23d685428 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,6 +82,7 @@ These values align with `docker-compose.development.yml` and self-hosted auth: ```dotenv # Warehouse — route API queries to local ClickHouse instead of Tinybird Cloud CLICKHOUSE_URL=http://localhost:8123 +CLICKHOUSE_PROVIDER=clickhouse CLICKHOUSE_USER=maple CLICKHOUSE_PASSWORD=maple CLICKHOUSE_DATABASE=default @@ -186,7 +187,9 @@ Default URL: `http://localhost:3472` | `MAPLE_DEFAULT_ORG_ID` | yes | Default `default`; must match ingest org override | | `TINYBIRD_HOST` | yes | Placeholder OK when using `CLICKHOUSE_URL` | | `TINYBIRD_TOKEN` | yes | Placeholder OK when using `CLICKHOUSE_URL` | +| `TINYBIRD_SIGNING_KEY` / `TINYBIRD_WORKSPACE_ID` | with Tinybird raw SQL | Explicit JWT signing configuration; never derived from `TINYBIRD_TOKEN` | | `CLICKHOUSE_URL` | recommended | `http://localhost:8123` for local ClickHouse stack | +| `CLICKHOUSE_PROVIDER` | optional | `clickhouse` (default) or `tinybird`; existing Tinybird gateway deployments must set `tinybird` | | `CLICKHOUSE_USER` / `CLICKHOUSE_PASSWORD` / `CLICKHOUSE_DATABASE` | with CH | Match docker-compose (`maple` / `maple` / `default`) | | `INTERNAL_SERVICE_TOKEN` | recommended | Shared with scraper + chat-flue | | `SD_INTERNAL_TOKEN` | optional | Prometheus scraper internal API auth | diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index 37d29e3b3..a57dfb02e 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -148,7 +148,10 @@ export const createMapleApi = ({ stage, domains }: CreateMapleApiOptions) => }), TINYBIRD_HOST: requireEnv("TINYBIRD_HOST"), TINYBIRD_TOKEN: Redacted.make(requireEnv("TINYBIRD_TOKEN")), + ...optionalSecret("TINYBIRD_SIGNING_KEY"), + ...optionalPlain("TINYBIRD_WORKSPACE_ID"), ...optionalPlain("CLICKHOUSE_URL"), + CLICKHOUSE_PROVIDER: process.env.CLICKHOUSE_PROVIDER?.trim() || "clickhouse", ...optionalPlain("CLICKHOUSE_USER"), ...optionalPlain("CLICKHOUSE_DATABASE"), ...optionalSecret("CLICKHOUSE_PASSWORD"), diff --git a/apps/api/src/lib/Env.ts b/apps/api/src/lib/Env.ts index d7aed71de..d3849bccb 100644 --- a/apps/api/src/lib/Env.ts +++ b/apps/api/src/lib/Env.ts @@ -11,7 +11,10 @@ export interface EnvShape { readonly PORT: number readonly TINYBIRD_HOST: string readonly TINYBIRD_TOKEN: Redacted.Redacted + readonly TINYBIRD_SIGNING_KEY: Option.Option> + readonly TINYBIRD_WORKSPACE_ID: Option.Option readonly CLICKHOUSE_URL: Option.Option + readonly CLICKHOUSE_PROVIDER: string readonly CLICKHOUSE_USER: string readonly CLICKHOUSE_PASSWORD: Option.Option> readonly CLICKHOUSE_DATABASE: string @@ -82,7 +85,10 @@ const envConfig = Config.all({ PORT: portConfig, TINYBIRD_HOST: Config.string("TINYBIRD_HOST"), TINYBIRD_TOKEN: Config.redacted("TINYBIRD_TOKEN"), + TINYBIRD_SIGNING_KEY: optionalRedacted("TINYBIRD_SIGNING_KEY"), + TINYBIRD_WORKSPACE_ID: optionalString("TINYBIRD_WORKSPACE_ID"), CLICKHOUSE_URL: optionalString("CLICKHOUSE_URL"), + CLICKHOUSE_PROVIDER: stringWithDefault("CLICKHOUSE_PROVIDER", "clickhouse"), CLICKHOUSE_USER: stringWithDefault("CLICKHOUSE_USER", "default"), CLICKHOUSE_PASSWORD: optionalRedacted("CLICKHOUSE_PASSWORD"), CLICKHOUSE_DATABASE: stringWithDefault("CLICKHOUSE_DATABASE", "default"), @@ -212,6 +218,14 @@ const makeEnv = Effect.gen(function* () { return yield* Effect.die(new EnvValidationError({ message: "MAPLE_DEFAULT_ORG_ID cannot be empty" })) } + if (env.CLICKHOUSE_PROVIDER !== "clickhouse" && env.CLICKHOUSE_PROVIDER !== "tinybird") { + return yield* Effect.die( + new EnvValidationError({ + message: "CLICKHOUSE_PROVIDER must be either 'clickhouse' or 'tinybird'", + }), + ) + } + const authMode = env.MAPLE_AUTH_MODE.toLowerCase() if (authMode !== "clerk" && Option.isNone(env.MAPLE_ROOT_PASSWORD)) { diff --git a/apps/api/src/lib/WarehouseQueryService.clickhouse.e2e.test.ts b/apps/api/src/lib/WarehouseQueryService.clickhouse.e2e.test.ts new file mode 100644 index 000000000..5b7dfa82c --- /dev/null +++ b/apps/api/src/lib/WarehouseQueryService.clickhouse.e2e.test.ts @@ -0,0 +1,212 @@ +import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" +import { spawn } from "node:child_process" +import { ConfigProvider, Effect, Layer, Schema } from "effect" +import { OrgId, UserId } from "@maple/domain/http" +import { isRawSqlResponseLimitError } from "@maple/query-engine/execution" +import { makeExpandMacros, RAW_SQL_EXECUTION_GUARDS } from "@maple/query-engine/runtime" +import { OrgClickHouseSettingsService } from "../services/OrgClickHouseSettingsService" +import { TinybirdOrgTokenService } from "../services/TinybirdOrgTokenService" +import type { TenantContext } from "../services/AuthService" +import { Env } from "./Env" +import { cleanupTestDbs, createTestDb, type TestDb } from "./test-pglite" +import { WarehouseQueryService } from "./WarehouseQueryService" + +const enabled = process.env.CLICKHOUSE_E2E === "1" +const clickhouseUrl = process.env.CLICKHOUSE_E2E_URL ?? "http://127.0.0.1:8123" +const clickhouseUser = process.env.CLICKHOUSE_E2E_USER ?? "maple" +const clickhousePassword = process.env.CLICKHOUSE_E2E_PASSWORD ?? "maple" +const database = `maple_raw_sql_e2e_${Date.now()}_${Math.random().toString(36).slice(2, 8)}` +const orgId = "org_raw_sql_e2e" +const repoRoot = new URL("../../../..", import.meta.url).pathname + +const clickhouseExec = async (sql: string, targetDatabase = "default"): Promise => { + const response = await fetch( + `${clickhouseUrl.replace(/\/$/, "")}/?database=${encodeURIComponent(targetDatabase)}`, + { + method: "POST", + redirect: "manual", + headers: { + "Content-Type": "text/plain", + "X-ClickHouse-User": clickhouseUser, + "X-ClickHouse-Key": clickhousePassword, + "X-ClickHouse-Database": targetDatabase, + }, + body: sql, + }, + ) + const body = await response.text() + if (!response.ok) throw new Error(`ClickHouse ${response.status}: ${body.slice(0, 500)}`) + return body +} + +const applyRealMigrations = async (): Promise => { + const child = spawn( + "bun", + [ + "run", + "--cwd", + "packages/clickhouse-cli", + "start", + "apply", + `--url=${clickhouseUrl}`, + `--user=${clickhouseUser}`, + `--password=${clickhousePassword}`, + `--database=${database}`, + ], + { cwd: repoRoot, stdio: ["ignore", "pipe", "pipe"] }, + ) + let stdout = "" + let stderr = "" + child.stdout.setEncoding("utf8").on("data", (chunk: string) => { + stdout += chunk + }) + child.stderr.setEncoding("utf8").on("data", (chunk: string) => { + stderr += chunk + }) + const exitCode = await new Promise((resolve, reject) => { + child.once("error", reject) + child.once("close", (code) => resolve(code ?? 1)) + }) + if (exitCode !== 0) throw new Error(`Migration CLI failed (${exitCode}): ${stderr || stdout}`) +} + +const trackedDbs: TestDb[] = [] +const asOrgId = Schema.decodeUnknownSync(OrgId) +const asUserId = Schema.decodeUnknownSync(UserId) +const tenant: TenantContext = { + orgId: asOrgId(orgId), + userId: asUserId("user_raw_sql_e2e"), + roles: [], + authMode: "self_hosted", +} + +const buildLayer = () => { + const testDb = createTestDb(trackedDbs) + const configLive = ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3472", + TINYBIRD_HOST: "http://127.0.0.1:7181", + TINYBIRD_TOKEN: "unused-for-vanilla-clickhouse", + CLICKHOUSE_URL: clickhouseUrl, + CLICKHOUSE_PROVIDER: "clickhouse", + CLICKHOUSE_USER: clickhouseUser, + CLICKHOUSE_PASSWORD: clickhousePassword, + CLICKHOUSE_DATABASE: database, + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: orgId, + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 9).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "raw-sql-e2e-lookup-key", + MAPLE_INGEST_PUBLIC_URL: "http://127.0.0.1:3474", + MAPLE_APP_BASE_URL: "http://127.0.0.1:3471", + }), + ) + const envLive = Env.layer.pipe(Layer.provide(configLive)) + const orgSettingsLive = OrgClickHouseSettingsService.layer.pipe( + Layer.provide(Layer.mergeAll(envLive, testDb.layer)), + ) + const tokensLive = TinybirdOrgTokenService.layer.pipe(Layer.provide(envLive)) + return WarehouseQueryService.layer.pipe( + Layer.provide(Layer.mergeAll(envLive, orgSettingsLive, tokensLive)), + ) +} + +const expand = (sql: string) => + makeExpandMacros({ + sql, + orgId, + startTime: "2026-01-01 00:00:00", + endTime: "2026-01-01 01:00:00", + granularitySeconds: 60, + }) + +describe.skipIf(!enabled)("WarehouseQueryService ClickHouse raw-SQL E2E", () => { + beforeAll(async () => { + await clickhouseExec(`CREATE DATABASE ${database}`) + await applyRealMigrations() + await clickhouseExec( + `INSERT INTO traces + (OrgId, Timestamp, TraceId, SpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode) + VALUES + ('${orgId}', now64(9), 'trace-a', 'span-a', 'GET /a', 'Server', 'api', 1000, 'Ok'), + ('${orgId}', now64(9), 'trace-b', 'span-b', 'GET /b', 'Server', 'worker', 2000, 'Error')`, + database, + ) + }, 120_000) + + afterAll(async () => { + await cleanupTestDbs(trackedDbs) + await clickhouseExec(`DROP DATABASE IF EXISTS ${database}`) + }, 30_000) + + it.effect( + "uses the configured ClickHouse password even when org-JWT scoping is requested", + () => { + const layer = buildLayer() + return Effect.gen(function* () { + const query = yield* expand( + "SELECT TraceId, ServiceName FROM traces WHERE $__orgFilter ORDER BY TraceId", + ) + const rows = yield* WarehouseQueryService.use((service) => + service.sqlQuery(tenant, query.sql, { + profile: "rawInteractive", + context: "clickhouse.e2e.fixture", + ...RAW_SQL_EXECUTION_GUARDS, + }), + ) + assert.deepStrictEqual(rows, [ + { TraceId: "trace-a", ServiceName: "api" }, + { TraceId: "trace-b", ServiceName: "worker" }, + ]) + }).pipe(Effect.provide(layer)) + }, + 120_000, + ) + + it.effect( + "accepts exactly 1,000 rows and aborts on the 1,001st streamed row", + () => { + const layer = buildLayer() + return Effect.gen(function* () { + const exact = yield* expand( + `SELECT number, '${orgId}' AS OrgId FROM numbers(1000) WHERE $__orgFilter`, + ) + const rows = yield* WarehouseQueryService.use((service) => + service.sqlQuery(tenant, exact.sql, RAW_SQL_EXECUTION_GUARDS), + ) + assert.strictEqual(rows.length, 1000) + + const overflow = yield* expand( + `SELECT number, '${orgId}' AS OrgId FROM numbers(50000) WHERE $__orgFilter LIMIT 50000`, + ) + const error = yield* Effect.flip( + WarehouseQueryService.use((service) => + service.sqlQuery(tenant, overflow.sql, RAW_SQL_EXECUTION_GUARDS), + ), + ) + assert.isTrue(isRawSqlResponseLimitError(error)) + }).pipe(Effect.provide(layer)) + }, + 120_000, + ) + + it.effect( + "aborts a streamed JSONEachRow response above five megabytes", + () => { + const layer = buildLayer() + return Effect.gen(function* () { + const query = yield* expand( + `SELECT number, repeat('x', 10000) AS payload, '${orgId}' AS OrgId FROM numbers(600) WHERE $__orgFilter`, + ) + const error = yield* Effect.flip( + WarehouseQueryService.use((service) => + service.sqlQuery(tenant, query.sql, RAW_SQL_EXECUTION_GUARDS), + ), + ) + assert.isTrue(isRawSqlResponseLimitError(error)) + assert.match(error.message, /5000000 encoded bytes/) + }).pipe(Effect.provide(layer)) + }, + 120_000, + ) +}) diff --git a/apps/api/src/lib/WarehouseQueryService.test.ts b/apps/api/src/lib/WarehouseQueryService.test.ts index b74bde6e4..46a30ddc1 100644 --- a/apps/api/src/lib/WarehouseQueryService.test.ts +++ b/apps/api/src/lib/WarehouseQueryService.test.ts @@ -2,15 +2,20 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { Cause, ConfigProvider, Effect, Exit, Layer, Option, Schema } from "effect" import { WarehouseQueryError, + WarehouseConfigError, + MAX_RAW_SQL_RESULT_BYTES, WarehouseSchemaDriftError, WarehouseUpstreamError, OrgId, UserId, } from "@maple/domain/http" import { unsafeCompiledQuery } from "@maple/query-engine/ch" -import { makeWarehouseExecutor } from "@maple/query-engine/execution" +import { makeWarehouseExecutor, type ResolvedWarehouseConfig } from "@maple/query-engine/execution" import { __testables, WarehouseQueryService } from "./WarehouseQueryService" -import { OrgClickHouseSettingsService } from "../services/OrgClickHouseSettingsService" +import { + OrgClickHouseSettingsService, + type OrgClickHouseSettingsServiceShape, +} from "../services/OrgClickHouseSettingsService" import { TinybirdOrgTokenService } from "../services/TinybirdOrgTokenService" import type { TenantContext } from "../services/AuthService" import { Env } from "./Env" @@ -23,12 +28,18 @@ afterEach(async () => { await cleanupTestDbs(trackedDbs) }) -const makeConfig = (extra: Record = {}) => +const makeConfig = (extra: Record = {}, includeTinybirdSigning = true) => ConfigProvider.layer( ConfigProvider.fromUnknown({ PORT: "3472", TINYBIRD_HOST: "https://maple-managed.tinybird.co", TINYBIRD_TOKEN: "managed-token", + ...(includeTinybirdSigning + ? { + TINYBIRD_SIGNING_KEY: "test-signing-key", + TINYBIRD_WORKSPACE_ID: "test-workspace", + } + : {}), MAPLE_AUTH_MODE: "self_hosted", MAPLE_ROOT_PASSWORD: "test-root-password", MAPLE_DEFAULT_ORG_ID: "default", @@ -40,8 +51,8 @@ const makeConfig = (extra: Record = {}) => }), ) -const buildLayer = (testDb: TestDb, extra: Record = {}) => { - const configLive = makeConfig(extra) +const buildLayer = (testDb: TestDb, extra: Record = {}, includeTinybirdSigning = true) => { + const configLive = makeConfig(extra, includeTinybirdSigning) const envLive = Env.layer.pipe(Layer.provide(configLive)) const databaseLive = testDb.layer const orgSettingsLive = OrgClickHouseSettingsService.layer.pipe( @@ -74,6 +85,205 @@ const makeTenant = (): TenantContext => ({ const transient503 = () => new Error("HTTP status 503 service temporarily unavailable") +const decodeJwtPayload = (token: string): Record => + JSON.parse(Buffer.from(token.split(".")[1] ?? "", "base64url").toString("utf8")) as Record< + string, + unknown + > + +describe("WarehouseQueryService raw-SQL provider routing", () => { + it.effect("substitutes a scoped JWT for the Tinybird SDK token", () => { + let captured: ResolvedWarehouseConfig | undefined + let rawResponseLimits: boolean | undefined + __testables.setClientFactory((config) => { + captured = config + return { + sql: async (_sql, options) => { + rawResponseLimits = options?.rawResponseLimits + return { data: [] } + }, + insert: async () => {}, + } + }) + const layer = buildLayer(createTestDb(trackedDbs)) + + return Effect.gen(function* () { + yield* WarehouseQueryService.use((service) => + service.sqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'", { + scopeToOrgJwt: true, + rawResponseLimits: true, + }), + ) + assert.strictEqual(captured?._tag, "tinybird") + if (captured?._tag !== "tinybird") throw new Error("expected Tinybird config") + assert.strictEqual(captured.provider, "tinybird") + assert.notStrictEqual(captured.token, "managed-token") + assert.strictEqual(decodeJwtPayload(captured.token).workspace_id, "test-workspace") + assert.strictEqual(rawResponseLimits, true) + }).pipe(Effect.provide(layer)) + }) + + it.effect("substitutes a scoped JWT for the Tinybird ClickHouse gateway password", () => { + let captured: ResolvedWarehouseConfig | undefined + __testables.setClientFactory((config) => { + captured = config + return { sql: async () => ({ data: [] }), insert: async () => {} } + }) + const layer = buildLayer(createTestDb(trackedDbs), { + CLICKHOUSE_URL: "https://gateway.tinybird.example", + CLICKHOUSE_PROVIDER: "tinybird", + CLICKHOUSE_PASSWORD: "gateway-admin-token", + }) + + return Effect.gen(function* () { + yield* WarehouseQueryService.use((service) => + service.sqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'", { + scopeToOrgJwt: true, + }), + ) + assert.strictEqual(captured?._tag, "clickhouse") + if (captured?._tag !== "clickhouse") throw new Error("expected gateway config") + assert.strictEqual(captured.provider, "tinybird") + assert.notStrictEqual(captured.password, "gateway-admin-token") + assert.strictEqual(decodeJwtPayload(captured.password).workspace_id, "test-workspace") + }).pipe(Effect.provide(layer)) + }) + + it.effect("preserves env-level vanilla ClickHouse credentials for raw SQL", () => { + let captured: ResolvedWarehouseConfig | undefined + __testables.setClientFactory((config) => { + captured = config + return { sql: async () => ({ data: [] }), insert: async () => {} } + }) + const layer = buildLayer(createTestDb(trackedDbs), { + CLICKHOUSE_URL: "https://clickhouse.example", + CLICKHOUSE_PROVIDER: "clickhouse", + CLICKHOUSE_PASSWORD: "original-clickhouse-password", + }) + + return Effect.gen(function* () { + yield* WarehouseQueryService.use((service) => + service.sqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'", { + scopeToOrgJwt: true, + }), + ) + assert.strictEqual(captured?._tag, "clickhouse") + if (captured?._tag !== "clickhouse") throw new Error("expected ClickHouse config") + assert.strictEqual(captured.provider, "clickhouse") + assert.strictEqual(captured.password, "original-clickhouse-password") + }).pipe(Effect.provide(layer)) + }) + + it.effect("preserves per-org ClickHouse override credentials for raw SQL", () => { + let captured: ResolvedWarehouseConfig | undefined + __testables.setClientFactory((config) => { + captured = config + return { sql: async () => ({ data: [] }), insert: async () => {} } + }) + const configLive = makeConfig() + const envLive = Env.layer.pipe(Layer.provide(configLive)) + const tokenLive = TinybirdOrgTokenService.layer.pipe(Layer.provide(envLive)) + const orgSettingsLive = Layer.succeed(OrgClickHouseSettingsService, { + resolveRuntimeConfig: () => + Effect.succeed( + Option.some({ + backend: "clickhouse" as const, + url: "https://byo.example", + user: "byo-user", + password: "byo-password", + database: "maple", + }), + ), + } as unknown as OrgClickHouseSettingsServiceShape) + const layer = WarehouseQueryService.layer.pipe( + Layer.provide(Layer.mergeAll(envLive, tokenLive, orgSettingsLive)), + ) + + return Effect.gen(function* () { + yield* WarehouseQueryService.use((service) => + service.sqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'", { + scopeToOrgJwt: true, + }), + ) + assert.strictEqual(captured?._tag, "clickhouse") + if (captured?._tag !== "clickhouse") throw new Error("expected ClickHouse config") + assert.strictEqual(captured.provider, "clickhouse") + assert.strictEqual(captured.password, "byo-password") + }).pipe(Effect.provide(layer)) + }) + + it.effect("maps missing Tinybird signing configuration to WarehouseConfigError", () => { + __testables.setClientFactory(() => ({ + sql: async () => ({ data: [] }), + insert: async () => {}, + })) + const layer = buildLayer(createTestDb(trackedDbs), {}, false) + + return Effect.gen(function* () { + const exit = yield* WarehouseQueryService.use((service) => + service.sqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'", { + scopeToOrgJwt: true, + }), + ).pipe(Effect.exit) + const failure = getError(exit) + assert.instanceOf(failure, WarehouseConfigError) + assert.include((failure as WarehouseConfigError).message, "TINYBIRD_SIGNING_KEY") + assert.notInclude((failure as WarehouseConfigError).message, "managed-token") + }).pipe(Effect.provide(layer)) + }) + + it.effect("rejects env-level URL userinfo before constructing a client", () => { + let constructed = false + __testables.setClientFactory(() => { + constructed = true + return { sql: async () => ({ data: [] }), insert: async () => {} } + }) + const layer = buildLayer(createTestDb(trackedDbs), { + CLICKHOUSE_URL: "https://user:secret@clickhouse.example", + CLICKHOUSE_PROVIDER: "clickhouse", + }) + + return Effect.gen(function* () { + const exit = yield* WarehouseQueryService.use((service) => + service.sqlQuery(makeTenant(), "SELECT 1 WHERE OrgId = 'org_test'", { + scopeToOrgJwt: true, + }), + ).pipe(Effect.exit) + assert.instanceOf(getError(exit), WarehouseConfigError) + assert.isFalse(constructed) + }).pipe(Effect.provide(layer)) + }) +}) + +describe("bounded Tinybird response fetch", () => { + it("accepts an exact-boundary response and aborts one byte over", async () => { + const realFetch = globalThis.fetch + try { + globalThis.fetch = (async () => + new Response(new Uint8Array(MAX_RAW_SQL_RESULT_BYTES))) as typeof fetch + const exact = await __testables.boundedResponseFetch(MAX_RAW_SQL_RESULT_BYTES)( + "https://api.tinybird.example/v0/sql", + ) + assert.strictEqual((await exact.arrayBuffer()).byteLength, MAX_RAW_SQL_RESULT_BYTES) + + globalThis.fetch = (async () => + new Response(new Uint8Array(MAX_RAW_SQL_RESULT_BYTES + 1))) as typeof fetch + let thrown: unknown + try { + await __testables.boundedResponseFetch(MAX_RAW_SQL_RESULT_BYTES)( + "https://api.tinybird.example/v0/sql", + ) + } catch (error) { + thrown = error + } + assert.instanceOf(thrown, Error) + assert.match((thrown as Error).message, /5000000 encoded bytes/) + } finally { + globalThis.fetch = realFetch + } + }) +}) + describe("WarehouseQueryService.sqlQuery retry on transient upstream failures", () => { // Runs under it.live: the retry schedule uses real exponential backoff // delays, so the default TestClock would stall the retries. @@ -385,6 +595,7 @@ describe("createClickHouseSqlClient.insert is disabled (ClickHouse is read-only) // read-only query gateway ("Only SELECT or DESCRIBE … Got: InsertQuery"). const chConfig = { _tag: "clickhouse" as const, + provider: "clickhouse" as const, url: "https://ch.example.com", username: "u", password: "p", @@ -418,7 +629,12 @@ describe("createClickHouseSqlClient.insert is disabled (ClickHouse is read-only) describe("createTinybirdSdkSqlClient.insert wire framing (the production insert path)", () => { // Inserts in the cloud only need to work on Tinybird. This pins that path so a // future change can't silently break ingest into the managed pipeline. - const tbConfig = { _tag: "tinybird" as const, host: "https://api.tinybird.co", token: "tok_123" } + const tbConfig = { + _tag: "tinybird" as const, + provider: "tinybird" as const, + host: "https://api.tinybird.co", + token: "tok_123", + } it("POSTs raw ndjson rows to the Tinybird Events API (/v0/events?name=)", async () => { const captured: Array<{ @@ -482,6 +698,7 @@ describe("ingest routes writes to the managed pipeline, not a per-org read overr const clickhouseReadOverride = { config: { _tag: "clickhouse" as const, + provider: "clickhouse" as const, url: "https://byo-clickhouse.example.com", username: "u", password: "p", @@ -490,7 +707,12 @@ describe("ingest routes writes to the managed pipeline, not a per-org read overr source: "org_override" as const, } const tinybirdManaged = { - config: { _tag: "tinybird" as const, host: "https://managed.tinybird.co", token: "tok" }, + config: { + _tag: "tinybird" as const, + provider: "tinybird" as const, + host: "https://managed.tinybird.co", + token: "tok", + }, source: "managed" as const, } diff --git a/apps/api/src/lib/WarehouseQueryService.tinybird.e2e.test.ts b/apps/api/src/lib/WarehouseQueryService.tinybird.e2e.test.ts new file mode 100644 index 000000000..605f64978 --- /dev/null +++ b/apps/api/src/lib/WarehouseQueryService.tinybird.e2e.test.ts @@ -0,0 +1,192 @@ +import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" +import { mintOrgReadJwt } from "./tinybird-jwt" + +const enabled = process.env.TINYBIRD_LOCAL_E2E === "1" +const apiBase = (process.env.TINYBIRD_LOCAL_E2E_URL ?? "http://127.0.0.1:7181").replace( + /\/$/, + "", +) +const gatewayBase = ( + process.env.TINYBIRD_LOCAL_E2E_GATEWAY_URL ?? "http://127.0.0.1:7182" +).replace(/\/$/, "") +const workspaceName = `RawSqlE2E_${Date.now()}_${Math.random().toString(36).slice(2, 8)}` +const datasource = "raw_sql_e2e" + +interface LocalTokens { + readonly admin_token: string + readonly user_token: string + readonly workspace_admin_token: string +} + +interface LocalWorkspace { + readonly id: string + readonly name: string + readonly token: string +} + +let tokens: LocalTokens | undefined +let workspace: LocalWorkspace | undefined +let workspaceSigningKey: string | undefined + +const responseJson = async (response: Response): Promise => { + const text = await response.text() + if (!response.ok) throw new Error(`Tinybird ${response.status}: ${text.slice(0, 500)}`) + return JSON.parse(text) as T +} + +const listWorkspaces = async (adminToken: string): Promise> => { + const response = await fetch( + `${apiBase}/v1/user/workspaces?with_organization=true&token=${encodeURIComponent(adminToken)}`, + ) + const body = await responseJson<{ readonly workspaces: ReadonlyArray }>(response) + return body.workspaces +} + +const createWorkspace = async (localTokens: LocalTokens): Promise => { + const response = await fetch(`${apiBase}/v1/workspaces`, { + method: "POST", + headers: { + Authorization: `Bearer ${localTokens.user_token}`, + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ name: workspaceName }), + }) + await responseJson>(response) + const created = (await listWorkspaces(localTokens.admin_token)).find( + (candidate) => candidate.name === workspaceName, + ) + if (!created) throw new Error("Tinybird Local workspace was not returned after creation") + return created +} + +const buildDatasource = async (target: LocalWorkspace): Promise => { + const datafile = [ + "SCHEMA >", + " OrgId String `json:$.OrgId`,", + " value UInt64 `json:$.value`", + 'ENGINE "MergeTree"', + 'ENGINE_SORTING_KEY "OrgId"', + ].join("\n") + const form = new FormData() + form.append("data_project://", new Blob([datafile], { type: "text/plain" }), `${datasource}.datasource`) + const response = await fetch(`${apiBase}/v1/build`, { + method: "POST", + headers: { Authorization: `Bearer ${target.token}` }, + body: form, + }) + const body = await responseJson<{ readonly result: string; readonly error?: string }>(response) + if (body.result !== "success" && body.result !== "no_changes") { + throw new Error(`Tinybird local build failed: ${body.error ?? body.result}`) + } +} + +const ingestFixtures = async (target: LocalWorkspace): Promise => { + const response = await fetch(`${apiBase}/v0/events?name=${datasource}&wait=true`, { + method: "POST", + headers: { + Authorization: `Bearer ${target.token}`, + "Content-Type": "application/x-ndjson", + }, + body: ['{"OrgId":"org_a","value":1}', '{"OrgId":"org_b","value":2}'].join("\n"), + }) + await responseJson>(response) +} + +const loadWorkspaceSigningKey = async (target: LocalWorkspace): Promise => { + const response = await fetch(`${apiBase}/v0/tokens/`, { + headers: { Authorization: `Bearer ${target.token}` }, + }) + const body = await responseJson<{ + readonly tokens: ReadonlyArray<{ + readonly name: string + readonly token: string + readonly scopes: ReadonlyArray<{ readonly type: string }> + }> + }>(response) + const adminToken = body.tokens.find( + (candidate) => + candidate.name === "workspace admin token" && + candidate.scopes.some((scope) => scope.type === "ADMIN"), + ) + if (!adminToken) throw new Error("Tinybird Local did not return a workspace admin token") + return adminToken.token +} + +const scopedToken = (target: LocalWorkspace, signingKey: string): string => + mintOrgReadJwt({ + signingKey, + workspaceId: target.id, + orgId: "org_a", + datasourceNames: [datasource], + nowSeconds: Math.floor(Date.now() / 1000), + ttlSeconds: 600, + }) + +describe.skipIf(!enabled)("Tinybird Local raw-SQL JWT E2E", () => { + beforeAll(async () => { + tokens = await responseJson(await fetch(`${apiBase}/tokens`)) + workspace = await createWorkspace(tokens) + workspaceSigningKey = await loadWorkspaceSigningKey(workspace) + await buildDatasource(workspace) + await ingestFixtures(workspace) + }, 120_000) + + afterAll(async () => { + if (tokens === undefined || workspace === undefined) return + const response = await fetch( + `${apiBase}/v1/workspaces/${workspace.id}?hard_delete_confirmation=yes`, + { + method: "DELETE", + headers: { Authorization: `Bearer ${tokens.user_token}` }, + }, + ) + if (!response.ok) throw new Error(`Failed to delete Tinybird E2E workspace (${response.status})`) + }, 30_000) + + it("enforces the org filter through /v0/sql even with OR 1=1", async () => { + if (workspace === undefined || workspaceSigningKey === undefined) { + throw new Error("workspace not initialized") + } + const response = await fetch(`${apiBase}/v0/sql`, { + method: "POST", + headers: { + Authorization: `Bearer ${scopedToken(workspace, workspaceSigningKey)}`, + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ + q: `SELECT OrgId, value FROM ${datasource} WHERE OrgId = 'org_a' OR 1=1 ORDER BY OrgId FORMAT JSON`, + }), + }) + const body = await responseJson<{ readonly data: ReadonlyArray> }>(response) + assert.deepStrictEqual(body.data, [{ OrgId: "org_a", value: 1 }]) + }) + + it("enforces the same org filter through the ClickHouse-compatible gateway", async () => { + if (workspace === undefined || workspaceSigningKey === undefined) { + throw new Error("workspace not initialized") + } + const response = await fetch( + `${gatewayBase}/?database=${encodeURIComponent(workspace.name)}`, + { + method: "POST", + headers: { + "X-ClickHouse-Key": scopedToken( + workspace, + workspaceSigningKey, + ), + "X-ClickHouse-Database": workspace.name, + "Content-Type": "text/plain", + }, + body: `SELECT OrgId, value FROM ${datasource} WHERE OrgId = 'org_a' OR 1=1 ORDER BY OrgId FORMAT JSONEachRow`, + }, + ) + const text = await response.text() + if (!response.ok) throw new Error(`Tinybird gateway ${response.status}: ${text.slice(0, 500)}`) + const rows = text + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record) + assert.deepStrictEqual(rows, [{ OrgId: "org_a", value: 1 }]) + }) +}) diff --git a/apps/api/src/lib/WarehouseQueryService.ts b/apps/api/src/lib/WarehouseQueryService.ts index 772c37ff2..2ffd31009 100644 --- a/apps/api/src/lib/WarehouseQueryService.ts +++ b/apps/api/src/lib/WarehouseQueryService.ts @@ -1,10 +1,16 @@ import { createClient as createClickHouseClient } from "@clickhouse/client-web" import { Tinybird } from "@tinybirdco/sdk" import { Context, Effect, Layer, Option, Redacted } from "effect" -import type { WarehouseQueryRequest } from "@maple/domain/http" +import { + MAX_RAW_SQL_RESULT_BYTES, + MAX_RAW_SQL_RESULT_ROWS, + WarehouseConfigError, + type WarehouseQueryRequest, +} from "@maple/domain/http" import { makeWarehouseExecutor, toWarehouseQueryError, + WarehouseResponseLimitError, type ResolvedWarehouseConfig, type SqlQueryOptions, type WarehouseExecutorDeps, @@ -43,12 +49,46 @@ const createClickHouseSqlClient = (config: ClickHouseConfig): WarehouseSqlClient database: config.database, }) return { - sql: async (sql: string) => { + sql: async (sql: string, options) => { const resultSet = await client.query({ query: sql, format: "JSONEachRow", }) - const data = await resultSet.json>() + if (!options?.rawResponseLimits) { + const data = await resultSet.json>() + return { data } + } + + const data: Array> = [] + const encoder = new TextEncoder() + let encodedBytes = 0 + const reader = resultSet.stream().getReader() + try { + while (true) { + const chunk = await reader.read() + if (chunk.done) break + for (const row of chunk.value) { + encodedBytes += encoder.encode(row.text).byteLength + 1 + if (encodedBytes > MAX_RAW_SQL_RESULT_BYTES) { + await reader.cancel().catch(() => undefined) + throw new WarehouseResponseLimitError( + "bytes", + `Raw SQL results may contain at most ${MAX_RAW_SQL_RESULT_BYTES} encoded bytes`, + ) + } + data.push(row.json>()) + if (data.length > MAX_RAW_SQL_RESULT_ROWS) { + await reader.cancel().catch(() => undefined) + throw new WarehouseResponseLimitError( + "rows", + `Raw SQL results may contain at most ${MAX_RAW_SQL_RESULT_ROWS} rows`, + ) + } + } + } + } finally { + reader.releaseLock() + } return { data } }, insert: async (_datasource, _rows) => { @@ -70,6 +110,46 @@ const createClickHouseSqlClient = (config: ClickHouseConfig): WarehouseSqlClient const isEmptyJsonBodyError = (error: unknown): boolean => error instanceof SyntaxError && /unexpected end of json input/i.test(error.message) +const boundedResponseFetch = + (maxBytes: number): typeof fetch => + async (input, init) => { + const response = await fetch(input, init) + if (response.body === null) return response + + const reader = response.body.getReader() + const chunks: Uint8Array[] = [] + let totalBytes = 0 + try { + while (true) { + const chunk = await reader.read() + if (chunk.done) break + totalBytes += chunk.value.byteLength + if (totalBytes > maxBytes) { + await reader.cancel().catch(() => undefined) + throw new WarehouseResponseLimitError( + "bytes", + `Raw SQL results may contain at most ${maxBytes} encoded bytes`, + ) + } + chunks.push(chunk.value) + } + } finally { + reader.releaseLock() + } + + const body = new Uint8Array(totalBytes) + let offset = 0 + for (const chunk of chunks) { + body.set(chunk, offset) + offset += chunk.byteLength + } + return new Response(body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }) + } + const createTinybirdSdkSqlClient = (config: TinybirdConfig): WarehouseSqlClient => { const client = new Tinybird({ baseUrl: config.host, @@ -78,10 +158,30 @@ const createTinybirdSdkSqlClient = (config: TinybirdConfig): WarehouseSqlClient pipes: {}, devMode: false, }) + // The SDK normally calls response.json(), which buffers the complete body. + // Raw SQL gets a separate client whose fetch adapter refuses to construct a + // Response larger than the public byte ceiling. + const boundedClient = new Tinybird({ + baseUrl: config.host, + token: config.token, + datasources: {}, + pipes: {}, + devMode: false, + fetch: boundedResponseFetch(MAX_RAW_SQL_RESULT_BYTES), + }) return { - sql: async (sql: string) => { + sql: async (sql: string, options) => { try { - return await client.sql(sql) + const result = await (options?.rawResponseLimits ? boundedClient : client).sql< + Record + >(sql) + if (options?.rawResponseLimits && result.data.length > MAX_RAW_SQL_RESULT_ROWS) { + throw new WarehouseResponseLimitError( + "rows", + `Raw SQL results may contain at most ${MAX_RAW_SQL_RESULT_ROWS} rows`, + ) + } + return { data: result.data } } catch (error) { // Empty 2xx body ⇒ zero rows. Return an empty result set so the caller's no-data path // runs instead of surfacing a spurious WarehouseClientError. @@ -137,14 +237,32 @@ export class WarehouseQueryService extends Context.Service< // here — and the read-path fallback when an org has no BYO override. const resolveManagedConfig = Effect.fn("WarehouseQueryService.resolveManagedConfig")(function* () { if (Option.isSome(env.CLICKHOUSE_URL)) { + const configuredUrl = env.CLICKHOUSE_URL.value + const clickhouseUrl = yield* Effect.try({ + try: () => new URL(configuredUrl), + catch: () => + new WarehouseConfigError({ + pipeName: "resolveManagedConfig", + message: "CLICKHOUSE_URL is invalid", + }), + }) + if (clickhouseUrl.username.length > 0 || clickhouseUrl.password.length > 0) { + return yield* new WarehouseConfigError({ + pipeName: "resolveManagedConfig", + message: "CLICKHOUSE_URL must not contain embedded credentials", + }) + } yield* Effect.annotateCurrentSpan("db.client", "clickhouse") + const provider: "tinybird" | "clickhouse" = + env.CLICKHOUSE_PROVIDER === "tinybird" ? "tinybird" : "clickhouse" return { config: { _tag: "clickhouse" as const, - url: env.CLICKHOUSE_URL.value, + provider, + url: clickhouseUrl.toString().replace(/\/$/, ""), username: env.CLICKHOUSE_USER, password: Option.match(env.CLICKHOUSE_PASSWORD, { - onNone: () => Redacted.value(env.TINYBIRD_TOKEN), + onNone: () => (provider === "tinybird" ? Redacted.value(env.TINYBIRD_TOKEN) : ""), onSome: Redacted.value, }), database: env.CLICKHOUSE_DATABASE, @@ -157,6 +275,7 @@ export class WarehouseQueryService extends Context.Service< return { config: { _tag: "tinybird" as const, + provider: "tinybird" as const, host: env.TINYBIRD_HOST, token: Redacted.value(env.TINYBIRD_TOKEN), }, @@ -182,6 +301,7 @@ export class WarehouseQueryService extends Context.Service< return { config: { _tag: "clickhouse" as const, + provider: "clickhouse" as const, url: override.value.url, username: override.value.user, password: override.value.password, @@ -194,15 +314,32 @@ export class WarehouseQueryService extends Context.Service< yield* Effect.annotateCurrentSpan("clientSource", "managed") const managed = yield* resolveManagedConfig() - // Untrusted raw SQL against the shared managed Tinybird warehouse: swap the + // Untrusted raw SQL against the SHARED managed Tinybird warehouse: swap the // shared admin token for a per-org scoped read JWT so Tinybird enforces // `OrgId` isolation server-side (immune to `OR 1=1`, UNION, subqueries). - // Only applies to the Tinybird SDK backend — the managed ClickHouse gateway - // (a shared user) can't be JWT-scoped, and BYO ClickHouse is already isolated. - if (options?.scopeToOrgJwt && managed.config._tag === "tinybird") { - const token = yield* orgTokens.getOrgReadToken(tenant.orgId) + // + // The managed warehouse is Tinybird either way — reached via the Tinybird SDK + // (`_tag: "tinybird"`, token) OR its ClickHouse-compatible gateway + // (`_tag: "clickhouse"` + source "managed", which authenticates with the + // Tinybird token as the CH *password*). Both carry the token, so a per-org JWT + // scopes both. BYO ClickHouse (source "org_override") returns earlier and is + // already isolated by its own credentials — never reached here. + if (options?.scopeToOrgJwt && managed.config.provider === "tinybird") { + const jwt = yield* orgTokens.getOrgReadToken(tenant.orgId).pipe( + Effect.mapError( + (error) => + new WarehouseConfigError({ + pipeName: label, + message: error.message, + cause: error, + }), + ), + ) yield* Effect.annotateCurrentSpan("tinybird.token.scope", "org_jwt") - return { config: { ...managed.config, token }, source: managed.source } + if (managed.config._tag === "tinybird") { + return { config: { ...managed.config, token: jwt }, source: managed.source } + } + return { config: { ...managed.config, password: jwt }, source: managed.source } } return managed }) @@ -225,7 +362,7 @@ export class WarehouseQueryService extends Context.Service< * it unconditionally (TINYBIRD_HOST/TOKEN are required env, always present). * Routing writes anywhere else broke demo-seed onboarding. */ - const resolveIngestConfig: WarehouseExecutorDeps["resolveConfig"] = Effect.fn( + const resolveIngestConfig: NonNullable = Effect.fn( "WarehouseQueryService.resolveIngestConfig", )(function* (tenant, _label) { yield* Effect.annotateCurrentSpan("orgId", tenant.orgId) @@ -235,6 +372,7 @@ export class WarehouseQueryService extends Context.Service< return { config: { _tag: "tinybird" as const, + provider: "tinybird" as const, host: env.TINYBIRD_HOST, token: Redacted.value(env.TINYBIRD_TOKEN), }, @@ -293,5 +431,6 @@ export const __testables = { }, createClickHouseSqlClient, createTinybirdSdkSqlClient, + boundedResponseFetch, isEmptyJsonBodyError, } diff --git a/apps/api/src/lib/_tmp_jwt_e2e.test.ts b/apps/api/src/lib/_tmp_jwt_e2e.test.ts deleted file mode 100644 index 64a739a98..000000000 --- a/apps/api/src/lib/_tmp_jwt_e2e.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { afterEach, assert, describe, it } from "@effect/vitest" -import { ConfigProvider, Effect, Layer } from "effect" -import { __testables, WarehouseQueryService } from "./WarehouseQueryService" -import { OrgClickHouseSettingsService } from "../services/OrgClickHouseSettingsService" -import { TinybirdOrgTokenService } from "../services/TinybirdOrgTokenService" -import type { TenantContext } from "../services/AuthService" -import { Env } from "./Env" -import { cleanupTestDbs, createTestDb, type TestDb } from "./test-pglite" - -const ADMIN = (process.env.TB_ADMIN ?? "").trim() -const dbs: TestDb[] = [] -afterEach(async () => { - __testables.reset() - await cleanupTestDbs(dbs) -}) - -const ORG = "org_39lDNgEjuXrOi0obeLpFSfyVSDo" -const T = "traces_aggregates_hourly" -const W = "Hour > now() - INTERVAL 48 HOUR" - -const buildLayer = () => { - const config = ConfigProvider.layer( - ConfigProvider.fromUnknown({ - PORT: "3472", - TINYBIRD_HOST: "https://api.tinybird.co", - TINYBIRD_TOKEN: ADMIN, - MAPLE_AUTH_MODE: "self_hosted", - MAPLE_ROOT_PASSWORD: "test-root-password", - MAPLE_DEFAULT_ORG_ID: "default", - MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 5).toString("base64"), - MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "lookup-key", - MAPLE_INGEST_PUBLIC_URL: "http://127.0.0.1:3474", - MAPLE_APP_BASE_URL: "http://127.0.0.1:3471", - }), - ) - const envLive = Env.layer.pipe(Layer.provide(config)) - const db = createTestDb(dbs) - const orgSettings = OrgClickHouseSettingsService.layer.pipe(Layer.provide(Layer.mergeAll(envLive, db.layer))) - const tokens = TinybirdOrgTokenService.layer.pipe(Layer.provide(envLive)) - // NOTE: no setClientFactory — real Tinybird driver hits cloud /v0/sql. - return WarehouseQueryService.layer.pipe(Layer.provide(Layer.mergeAll(envLive, orgSettings, tokens))) -} - -const tenant = { orgId: ORG, userId: "user_e2e", authMode: "self_hosted" } as unknown as TenantContext - -describe("raw-SQL JWT e2e (live Tinybird)", () => { - it.live("scopes a real raw query to the org through the assembled path", () => - Effect.gen(function* () { - const svc = yield* WarehouseQueryService - const run = (sql: string) => - svc.sqlQuery(tenant, sql, { profile: "rawInteractive", context: "e2e", scopeToOrgJwt: true }) - - const plain = yield* run(`SELECT count() c, count(DISTINCT OrgId) o FROM ${T} WHERE ${W}`) - const bypass = yield* run(`SELECT count() c FROM ${T} WHERE ${W} AND (OrgId != '' OR 1=1)`) - - console.log("E2E plain:", plain[0], "bypass:", bypass[0]) - assert.strictEqual(Number(plain[0].o), 1, "query saw exactly one org") - assert.isAbove(Number(plain[0].c), 0, "org has rows") - assert.strictEqual(Number(bypass[0].c), Number(plain[0].c), "OR 1=1 did not widen the result") - }).pipe(Effect.provide(buildLayer())), - ) -}) diff --git a/apps/api/src/lib/tinybird-jwt.test.ts b/apps/api/src/lib/tinybird-jwt.test.ts index 79e96b688..1944ae76b 100644 --- a/apps/api/src/lib/tinybird-jwt.test.ts +++ b/apps/api/src/lib/tinybird-jwt.test.ts @@ -1,20 +1,15 @@ import { createHmac } from "node:crypto" import { assert, describe, it } from "@effect/vitest" -import { deriveWorkspaceId, mintOrgReadJwt } from "./tinybird-jwt" +import { mintOrgReadJwt } from "./tinybird-jwt" const decodePart = (part: string): unknown => JSON.parse(Buffer.from(part, "base64url").toString("utf8")) -// A Tinybird-style admin token: `p..` whose payload carries the -// workspace id under `u`. -const makeAdminToken = (payload: Record): string => - `p.${Buffer.from(JSON.stringify(payload)).toString("base64")}.sig` - -const ADMIN = makeAdminToken({ u: "ws-uuid-123", id: "tok-id", host: "eu_shared" }) +const SIGNING_KEY = "explicit-test-signing-key" describe("mintOrgReadJwt", () => { it("produces a well-formed HS256 JWT with the right header, exp, and scopes", () => { const jwt = mintOrgReadJwt({ - adminToken: ADMIN, + signingKey: SIGNING_KEY, workspaceId: "ws-uuid-123", orgId: "org_abc", datasourceNames: ["traces", "logs"], @@ -39,14 +34,14 @@ describe("mintOrgReadJwt", () => { { type: "DATASOURCES:READ", resource: "logs", filter: "OrgId = 'org_abc'" }, ]) - // Signature verifies independently against the admin token as HMAC secret. - const expected = createHmac("sha256", ADMIN).update(`${header}.${payload}`).digest("base64url") + // Signature verifies independently against the explicit signing key. + const expected = createHmac("sha256", SIGNING_KEY).update(`${header}.${payload}`).digest("base64url") assert.strictEqual(signature, expected) }) it("escapes single quotes in the org id to prevent filter injection", () => { const jwt = mintOrgReadJwt({ - adminToken: ADMIN, + signingKey: SIGNING_KEY, workspaceId: "ws-uuid-123", orgId: "org_' OR 1=1 --", datasourceNames: ["traces"], @@ -60,17 +55,3 @@ describe("mintOrgReadJwt", () => { assert.strictEqual(decoded.scopes[0].filter, "OrgId = 'org_\\' OR 1=1 --'") }) }) - -describe("deriveWorkspaceId", () => { - it("extracts the workspace uuid from the token payload's `u` field", () => { - assert.strictEqual(deriveWorkspaceId(ADMIN), "ws-uuid-123") - }) - - it("throws when the token is not dotted", () => { - assert.throws(() => deriveWorkspaceId("not-a-jwt"), /not a dotted JWT/) - }) - - it("throws when the payload has no `u` field", () => { - assert.throws(() => deriveWorkspaceId(makeAdminToken({ id: "x" })), /no 'u' field/) - }) -}) diff --git a/apps/api/src/lib/tinybird-jwt.ts b/apps/api/src/lib/tinybird-jwt.ts index 8ab81108a..db39d3ee4 100644 --- a/apps/api/src/lib/tinybird-jwt.ts +++ b/apps/api/src/lib/tinybird-jwt.ts @@ -24,9 +24,9 @@ export interface TinybirdJwtScope { } export interface MintOrgReadJwtInput { - /** The workspace admin token — the HMAC signing secret. */ - readonly adminToken: string - /** The workspace UUID (see {@link deriveWorkspaceId}). */ + /** The explicitly configured HMAC signing secret. */ + readonly signingKey: string + /** The explicitly configured Tinybird workspace id. */ readonly workspaceId: string /** The org to scope the token to; embedded into every datasource filter. */ readonly orgId: string @@ -63,35 +63,6 @@ export function mintOrgReadJwt(input: MintOrgReadJwtInput): string { }), ) const signingInput = `${header}.${payload}` - const signature = base64url(createHmac("sha256", input.adminToken).update(signingInput).digest()) + const signature = base64url(createHmac("sha256", input.signingKey).update(signingInput).digest()) return `${signingInput}.${signature}` } - -/** - * Extract the workspace UUID from a Tinybird admin token. Tinybird tokens are - * `p..`; the JSON payload carries the workspace id under `u`. - * Throws if the token can't be decoded or lacks `u`, so a token-format change - * fails loudly at mint time rather than producing invalid JWTs. - */ -export function deriveWorkspaceId(adminToken: string): string { - const encodedPayload = adminToken.split(".")[1] - if (!encodedPayload) { - throw new Error("Cannot derive Tinybird workspace id: admin token is not a dotted JWT") - } - let decoded: unknown - try { - decoded = JSON.parse(Buffer.from(encodedPayload, "base64").toString("utf8")) - } catch (cause) { - throw new Error("Cannot derive Tinybird workspace id: admin token payload is not valid JSON", { - cause, - }) - } - const workspaceId = - decoded && typeof decoded === "object" && "u" in decoded - ? (decoded as { u?: unknown }).u - : undefined - if (typeof workspaceId !== "string" || workspaceId.length === 0) { - throw new Error("Cannot derive Tinybird workspace id: admin token payload has no 'u' field") - } - return workspaceId -} diff --git a/apps/api/src/mcp/lib/run-raw-sql.test.ts b/apps/api/src/mcp/lib/run-raw-sql.test.ts index b39a6b0ea..ff6292833 100644 --- a/apps/api/src/mcp/lib/run-raw-sql.test.ts +++ b/apps/api/src/mcp/lib/run-raw-sql.test.ts @@ -9,11 +9,15 @@ const tenant = { orgId: "org_test" } as TenantContext const makeStub = ( rows: ReadonlyArray>, - captured?: { sql?: string }, + captured?: { sql?: string; scopeToOrgJwt?: boolean; rawResponseLimits?: boolean }, ): WarehouseQueryServiceShape => ({ - sqlQuery: (_t: unknown, sql: string) => { - if (captured) captured.sql = sql + sqlQuery: (_t: unknown, sql: string, options) => { + if (captured) { + captured.sql = sql + captured.scopeToOrgJwt = options?.scopeToOrgJwt + captured.rawResponseLimits = options?.rawResponseLimits + } return Effect.succeed(rows) }, }) as unknown as WarehouseQueryServiceShape @@ -25,7 +29,11 @@ const range = { startTime: "2026-04-01 00:00:00", endTime: "2026-04-01 01:00:00" describe("runRawSql", () => { it.effect("expands macros and returns rows + column metadata", () => Effect.gen(function* () { - const captured: { sql?: string } = {} + const captured: { + sql?: string + scopeToOrgJwt?: boolean + rawResponseLimits?: boolean + } = {} const result = yield* runRawSql({ tenant, sql: "SELECT ServiceName, count() AS c FROM traces WHERE $__orgFilter GROUP BY ServiceName", @@ -35,6 +43,8 @@ describe("runRawSql", () => { // $__orgFilter expanded to the scoped predicate before execution. assert.include(captured.sql ?? "", "OrgId = 'org_test'") + assert.strictEqual(captured.scopeToOrgJwt, true) + assert.strictEqual(captured.rawResponseLimits, true) assert.strictEqual(result.rowCount, 1) assert.deepStrictEqual([...result.columns], ["ServiceName", "c"]) }), diff --git a/apps/api/src/mcp/lib/run-raw-sql.ts b/apps/api/src/mcp/lib/run-raw-sql.ts index bdc2dd15a..7d193195a 100644 --- a/apps/api/src/mcp/lib/run-raw-sql.ts +++ b/apps/api/src/mcp/lib/run-raw-sql.ts @@ -1,9 +1,10 @@ import { Effect } from "effect" +import { MAX_RAW_SQL_LENGTH, RawSqlValidationError } from "@maple/domain/http" import { - MAX_RAW_SQL_LENGTH, - RawSqlValidationError, -} from "@maple/domain/http" -import { makeExpandMacros, rawSqlResultLimitError } from "@maple/query-engine/runtime" + makeExpandMacros, + rawSqlResultLimitError, + RAW_SQL_EXECUTION_GUARDS, +} from "@maple/query-engine/runtime" import { WarehouseQueryService } from "@/lib/WarehouseQueryService" import type { TenantContext } from "@/lib/tenant-context" @@ -76,8 +77,7 @@ export const runRawSql = Effect.fn("runRawSql")(function* (input: RunRawSqlInput const rows = yield* warehouse.sqlQuery(input.tenant, expanded.sql, { profile: "rawInteractive", context: "mcp.run_sql", - // Untrusted user SQL — scope to a per-org Tinybird JWT (server-enforced isolation). - scopeToOrgJwt: true, + ...RAW_SQL_EXECUTION_GUARDS, }) const resultLimitError = rawSqlResultLimitError(rows) if (resultLimitError != null) { diff --git a/apps/api/src/routes/query-engine.http.ts b/apps/api/src/routes/query-engine.http.ts index 9f58a8ea9..c2e5fae30 100644 --- a/apps/api/src/routes/query-engine.http.ts +++ b/apps/api/src/routes/query-engine.http.ts @@ -69,7 +69,11 @@ import { } from "@maple/domain/http" import { Clock, Effect, Match, Option, Schema } from "effect" import { QueryEngineService } from "../services/QueryEngineService" -import { rawSqlResultLimitError, RawSqlChartService } from "@maple/query-engine/runtime" +import { + rawSqlResultLimitError, + RAW_SQL_EXECUTION_GUARDS, + RawSqlChartService, +} from "@maple/query-engine/runtime" import { WarehouseQueryService } from "../lib/WarehouseQueryService" import { traceCacheTtlSeconds } from "../lib/trace-detail-cache" import { @@ -79,6 +83,7 @@ import { parseWarehouseDateTime, } from "@maple/query-engine" import { LOGS_BODY_SEARCH_SETTINGS } from "@maple/query-engine/profiles" +import { isRawSqlResponseLimitError } from "@maple/query-engine/execution" import { buildBreakdownQuerySpec, buildTimeseriesQuerySpec } from "@maple/query-engine/query-builder" // `warehouse.sqlQuery` fails with the warehouse error union (distinct tagged @@ -158,17 +163,17 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", .compiledQueryFirst( tenant, CH.compile( - CH.traceTimeProbeQuery({ traceId: payload.traceId, narrowByTime }), + CH.traceTimeProbeQuery({ traceId: payload.traceId, narrowByTime }), narrowByTime ? { orgId: tenant.orgId, - startTime: formatWarehouseDateTime(nowMs - PROBE_RECENT_WINDOW_MS), + startTime: formatWarehouseDateTime(nowMs - PROBE_RECENT_WINDOW_MS), } : { orgId: tenant.orgId }, ), { profile: "discovery", - context: narrowByTime ? "spanHierarchyProbeRecent" : "spanHierarchyProbe", + context: narrowByTime ? "spanHierarchyProbeRecent" : "spanHierarchyProbe", }, ) .pipe(Effect.map(Option.getOrNull)), @@ -2648,12 +2653,22 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleApi, "queryEngine", }) const rows = yield* mapExecError( - warehouse.sqlQuery(tenant, expanded.sql, { - profile: "rawInteractive", - context: "rawSql", - // Untrusted user SQL — scope to a per-org Tinybird JWT (server-enforced isolation). - scopeToOrgJwt: true, - }), + warehouse + .sqlQuery(tenant, expanded.sql, { + profile: "rawInteractive", + context: "rawSql", + ...RAW_SQL_EXECUTION_GUARDS, + }) + .pipe( + Effect.catchIf(isRawSqlResponseLimitError, (error) => + Effect.fail( + new RawSqlValidationError({ + code: "ResourceLimit", + message: error.message, + }), + ), + ), + ), "rawSql query failed", ) diff --git a/apps/api/src/services/OrgClickHouseSettingsService.test.ts b/apps/api/src/services/OrgClickHouseSettingsService.test.ts index 5a9a70f1e..c7838250e 100644 --- a/apps/api/src/services/OrgClickHouseSettingsService.test.ts +++ b/apps/api/src/services/OrgClickHouseSettingsService.test.ts @@ -11,6 +11,7 @@ import { Cause, ConfigProvider, Effect, Exit, Layer, Option, Schema } from "effe import { FetchHttpClient } from "effect/unstable/http" import type { TableDiffEntry } from "@maple/domain/clickhouse" import { Env } from "../lib/Env" +import { encryptAes256Gcm } from "../lib/Crypto" import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "../lib/test-pglite" import { type ClickHouseExecConfig, @@ -79,6 +80,16 @@ describe("validateClickHouseCredentialTransport", () => { yield* validateClickHouseCredentialTransport("http://clickhouse.example.test", "") }), ) + + it.effect("rejects URL-embedded userinfo even without a separate password", () => + Effect.gen(function* () { + const exit = yield* validateClickHouseCredentialTransport( + "https://user:secret@clickhouse.example.test", + "", + ).pipe(Effect.exit) + expect(getError(exit)).toBeInstanceOf(OrgClickHouseSettingsValidationError) + }), + ) }) describe("shouldHealSchemaVersion", () => { @@ -140,6 +151,27 @@ describe("isRetryableUpstream", () => { }) describe("execClickHouse", () => { + it.live("uses manual redirects and rejects every 3xx without following it", () => + Effect.gen(function* () { + let redirectMode: RequestRedirect | undefined + let calls = 0 + const fetchImpl = (async (_input: RequestInfo | URL, init?: RequestInit) => { + calls += 1 + redirectMode = init?.redirect + return new Response("moved", { + status: 307, + headers: { Location: "https://attacker.example/steal" }, + }) + }) as typeof fetch + const exit = yield* run("SELECT 1", fetchImpl).pipe(Effect.exit) + const failure = getError(exit) + expect(failure).toBeInstanceOf(OrgClickHouseSettingsUpstreamRejectedError) + expect((failure as OrgClickHouseSettingsUpstreamRejectedError).statusCode).toBe(307) + expect(redirectMode).toBe("manual") + expect(calls).toBe(1) + }), + ) + it.live("maps a Cloudflare 524 to a clear, actionable message (and retries 52x)", () => Effect.gen(function* () { const { state, fetchImpl } = makeFetch(() => @@ -258,6 +290,20 @@ describe("resolveRuntimeConfig caching", () => { [orgId, chUrl], ) + const seedPasswordRow = async (db: TestDb, orgId: string, chUrl: string) => { + const encrypted = await Effect.runPromise( + encryptAes256Gcm("legacy-password", Buffer.alloc(32, 5), (message) => new Error(message)), + ) + await executeSql( + db, + `INSERT INTO org_clickhouse_settings + (org_id, ch_url, ch_user, ch_database, ch_password_ciphertext, ch_password_iv, + ch_password_tag, sync_status, created_at, updated_at, created_by, updated_by) + VALUES ($1, $2, 'default', 'maple', $3, $4, $5, 'connected', NOW(), NOW(), 'u', 'u')`, + [orgId, chUrl, encrypted.ciphertext, encrypted.iv, encrypted.tag], + ) + } + const expectSome = (o: Option.Option): A => { expect(Option.isSome(o)).toBe(true) return (o as Option.Some).value @@ -304,4 +350,28 @@ describe("resolveRuntimeConfig caching", () => { expect(Option.isNone(after)).toBe(true) }).pipe(Effect.provide(buildLayer(testDb))) }) + + it.effect("fails closed for a legacy passworded HTTP runtime row", () => { + const testDb = createTestDb(cacheTrackedDbs) + const orgId = "org_ch_legacy_http" + return Effect.gen(function* () { + yield* Effect.promise(() => seedPasswordRow(testDb, orgId, "http://clickhouse.example.test")) + const exit = yield* OrgClickHouseSettingsService.resolveRuntimeConfig(asOrgId(orgId)).pipe( + Effect.exit, + ) + expect(getError(exit)).toBeInstanceOf(OrgClickHouseSettingsValidationError) + }).pipe(Effect.provide(buildLayer(testDb))) + }) + + it.effect("fails closed for legacy URL userinfo during collector config generation", () => { + const testDb = createTestDb(cacheTrackedDbs) + const orgId = "org_ch_legacy_userinfo" + return Effect.gen(function* () { + yield* Effect.promise(() => seedRow(testDb, orgId, "https://user:secret@clickhouse.example.test")) + const exit = yield* OrgClickHouseSettingsService.collectorConfig(asOrgId(orgId), [ + asRole("org:admin"), + ]).pipe(Effect.exit) + expect(getError(exit)).toBeInstanceOf(OrgClickHouseSettingsValidationError) + }).pipe(Effect.provide(buildLayer(testDb))) + }) }) diff --git a/apps/api/src/services/OrgClickHouseSettingsService.ts b/apps/api/src/services/OrgClickHouseSettingsService.ts index 210b48b5e..24c116e99 100644 --- a/apps/api/src/services/OrgClickHouseSettingsService.ts +++ b/apps/api/src/services/OrgClickHouseSettingsService.ts @@ -179,7 +179,9 @@ export interface OrgClickHouseSettingsServiceShape { orgId: OrgId, ) => Effect.Effect< Option.Option, - OrgClickHouseSettingsPersistenceError | OrgClickHouseSettingsEncryptionError + | OrgClickHouseSettingsPersistenceError + | OrgClickHouseSettingsEncryptionError + | OrgClickHouseSettingsValidationError > /** * Whether the ingest gateway is currently routing this org's frames to its @@ -378,7 +380,16 @@ const quoteYaml = (value: string): string => `"${value.replace(/\\/g, "\\\\").re const normalizeHttpUrl = (raw: string): Effect.Effect => validateExternalUrl(raw).pipe( - Effect.map(() => raw.trim().replace(/\/+$/, "")), + Effect.flatMap((url) => + url.username.length > 0 || url.password.length > 0 + ? Effect.fail( + new OrgClickHouseSettingsValidationError({ + message: + "ClickHouse credentials must use the user/password fields, not URL userinfo", + }), + ) + : Effect.succeed(raw.trim().replace(/\/+$/, "")), + ), Effect.mapError( (error) => new OrgClickHouseSettingsValidationError({ @@ -391,13 +402,35 @@ export const validateClickHouseCredentialTransport = ( url: string, password: string, ): Effect.Effect => - password.length > 0 && new URL(url).protocol !== "https:" - ? Effect.fail( - new OrgClickHouseSettingsValidationError({ - message: "ClickHouse URLs must use HTTPS when a password is configured", - }), - ) - : Effect.void + Effect.try({ + try: () => new URL(url), + catch: () => + new OrgClickHouseSettingsValidationError({ message: "Stored ClickHouse URL is invalid" }), + }).pipe( + Effect.flatMap((parsed) => { + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + return Effect.fail( + new OrgClickHouseSettingsValidationError({ + message: "ClickHouse URLs must use HTTP or HTTPS", + }), + ) + } + if (parsed.username.length > 0 || parsed.password.length > 0) { + return Effect.fail( + new OrgClickHouseSettingsValidationError({ + message: "ClickHouse credentials must not be embedded in the URL", + }), + ) + } + return password.length > 0 && parsed.protocol !== "https:" + ? Effect.fail( + new OrgClickHouseSettingsValidationError({ + message: "ClickHouse URLs must use HTTPS when a password is configured", + }), + ) + : Effect.void + }), + ) const isOrgAdmin = (roles: ReadonlyArray) => roles.includes(ROOT_ROLE) || roles.includes(ORG_ADMIN_ROLE) @@ -546,6 +579,14 @@ const mapStatusToError = ( }), ) } + if (status >= 300 && status < 400) { + return Effect.fail( + new OrgClickHouseSettingsUpstreamRejectedError({ + message: `ClickHouse redirect responses are not allowed (${status})`, + statusCode: status, + }), + ) + } if (status >= 500) { return Effect.fail( new OrgClickHouseSettingsUpstreamUnavailableError({ @@ -568,7 +609,9 @@ export const execClickHouse = (config: ClickHouseExecConfig, sql: string) => const request = HttpClientRequest.post(buildClickHouseUrl(config), { headers: buildClickHouseHeaders(config), }).pipe(HttpClientRequest.bodyText(sql)) - const response = yield* client.execute(request) + const response = yield* client + .execute(request) + .pipe(Effect.provideService(FetchHttpClient.RequestInit, { redirect: "manual" })) const text = yield* response.text return { status: response.status, text } }).pipe( @@ -909,12 +952,16 @@ export class OrgClickHouseSettingsService extends Context.Service< }) const loadConfigForRow = (row: ActiveRow) => - Effect.map(decryptStoredPassword(row), (password) => ({ - url: row.chUrl, - user: row.chUser, - password, - database: row.chDatabase, - })) + Effect.gen(function* () { + const password = yield* decryptStoredPassword(row) + yield* validateClickHouseCredentialTransport(row.chUrl, password) + return { + url: row.chUrl, + user: row.chUser, + password, + database: row.chDatabase, + } + }) const schemaDiff = Effect.fn("OrgClickHouseSettingsService.schemaDiff")(function* ( orgId: OrgId, @@ -1177,6 +1224,7 @@ export class OrgClickHouseSettingsService extends Context.Service< cached.schemaVersion !== clickHouseSchemaVersion, ) const password = yield* decryptStoredPassword(cached) + yield* validateClickHouseCredentialTransport(cached.chUrl, password) return Option.some({ backend: "clickhouse", url: cached.chUrl, @@ -1216,6 +1264,15 @@ export class OrgClickHouseSettingsService extends Context.Service< yield* Effect.annotateCurrentSpan("orgId", orgId) yield* requireAdmin(roles) const row = yield* requireActiveRow(orgId) + const password = yield* decryptStoredPassword(row).pipe( + Effect.mapError( + () => + new OrgClickHouseSettingsValidationError({ + message: "Stored ClickHouse credentials could not be decrypted", + }), + ), + ) + yield* validateClickHouseCredentialTransport(row.chUrl, password) const yaml = renderCollectorYaml({ orgId, endpoint: row.chUrl, diff --git a/apps/api/src/services/QueryEngineService.test.ts b/apps/api/src/services/QueryEngineService.test.ts index 2400d65f0..d569f44f2 100644 --- a/apps/api/src/services/QueryEngineService.test.ts +++ b/apps/api/src/services/QueryEngineService.test.ts @@ -998,10 +998,14 @@ describe("makeQueryEngineEvaluateRawSql", () => { it.effect("groups raw SQL rows by the `group` column and reduces with the configured reducer", () => Effect.gen(function* () { let profile: string | undefined + let scopeToOrgJwt: boolean | undefined + let rawResponseLimits: boolean | undefined const evaluateRawSql = makeQueryEngineEvaluateRawSql( makeTinybirdStub({ sqlQuery: (_tenant, _sql, options) => { profile = options?.profile + scopeToOrgJwt = options?.scopeToOrgJwt + rawResponseLimits = options?.rawResponseLimits return Effect.succeed([ { group: "checkout", value: 10, samples: 4 }, { group: "checkout", value: 30, samples: 6 }, @@ -1027,6 +1031,8 @@ describe("makeQueryEngineEvaluateRawSql", () => { assert.strictEqual(byGroup.payments?.sampleCount, 2) assert.strictEqual(byGroup.payments?.hasData, true) assert.strictEqual(profile, "rawAlert") + assert.strictEqual(scopeToOrgJwt, true) + assert.strictEqual(rawResponseLimits, true) }), ) diff --git a/apps/api/src/services/TinybirdOrgTokenService.test.ts b/apps/api/src/services/TinybirdOrgTokenService.test.ts index d99a7de9c..df9094a1c 100644 --- a/apps/api/src/services/TinybirdOrgTokenService.test.ts +++ b/apps/api/src/services/TinybirdOrgTokenService.test.ts @@ -1,23 +1,32 @@ import { assert, describe, it } from "@effect/vitest" import { ConfigProvider, Effect, Layer } from "effect" +import { OrgId } from "@maple/domain" +import { Schema } from "effect" import { TestClock } from "effect/testing" import { TinybirdOrgTokenService } from "./TinybirdOrgTokenService" import { Env } from "@/lib/Env" -// A Tinybird-style admin token whose base64 payload carries the workspace id (`u`). -const ADMIN_TOKEN = `p.${Buffer.from(JSON.stringify({ u: "ws-uuid-abc", id: "tok", host: "eu_shared" })).toString("base64")}.sig` +const SIGNING_KEY = "explicit-test-signing-key" +const asOrgId = Schema.decodeUnknownSync(OrgId) -const testConfig = () => +const testConfig = (extra: Record = {}, includeSigning = true) => ConfigProvider.layer( ConfigProvider.fromUnknown({ PORT: "3478", TINYBIRD_HOST: "https://api.tinybird.co", - TINYBIRD_TOKEN: ADMIN_TOKEN, + TINYBIRD_TOKEN: "api-token-is-not-the-signing-key", + ...(includeSigning + ? { + TINYBIRD_SIGNING_KEY: SIGNING_KEY, + TINYBIRD_WORKSPACE_ID: "ws-uuid-abc", + } + : {}), MAPLE_AUTH_MODE: "self_hosted", MAPLE_ROOT_PASSWORD: "test-root-password", MAPLE_DEFAULT_ORG_ID: "default", MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 1).toString("base64"), MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + ...extra, }), ) @@ -34,7 +43,7 @@ describe("TinybirdOrgTokenService", () => { it.effect("mints a workspace-scoped token whose scopes are all filtered to the org", () => Effect.gen(function* () { const svc = yield* TinybirdOrgTokenService - const token = yield* svc.getOrgReadToken("org_a") + const token = yield* svc.getOrgReadToken(asOrgId("org_a")) const payload = decodePayload(token) assert.strictEqual(payload.workspace_id, "ws-uuid-abc") assert.isAbove(payload.scopes.length, 0) @@ -45,10 +54,10 @@ describe("TinybirdOrgTokenService", () => { it.effect("returns the cached token on a second call within its lifetime", () => Effect.gen(function* () { const svc = yield* TinybirdOrgTokenService - const first = yield* svc.getOrgReadToken("org_a") + const first = yield* svc.getOrgReadToken(asOrgId("org_a")) // Advance well within the (ttl - skew = 540s) window. yield* TestClock.setTime(120_000) - const second = yield* svc.getOrgReadToken("org_a") + const second = yield* svc.getOrgReadToken(asOrgId("org_a")) assert.strictEqual(second, first) }).pipe(Effect.provide(layer)), ) @@ -56,10 +65,10 @@ describe("TinybirdOrgTokenService", () => { it.effect("re-mints after the cached token nears expiry", () => Effect.gen(function* () { const svc = yield* TinybirdOrgTokenService - const first = yield* svc.getOrgReadToken("org_a") + const first = yield* svc.getOrgReadToken(asOrgId("org_a")) // Past the 540s re-mint deadline → new token (later exp). yield* TestClock.setTime(600_000) - const second = yield* svc.getOrgReadToken("org_a") + const second = yield* svc.getOrgReadToken(asOrgId("org_a")) assert.notStrictEqual(second, first) assert.isAbove(decodePayload(second).exp, decodePayload(first).exp) }).pipe(Effect.provide(layer)), @@ -68,10 +77,35 @@ describe("TinybirdOrgTokenService", () => { it.effect("issues distinct tokens per org", () => Effect.gen(function* () { const svc = yield* TinybirdOrgTokenService - const a = yield* svc.getOrgReadToken("org_a") - const b = yield* svc.getOrgReadToken("org_b") + const a = yield* svc.getOrgReadToken(asOrgId("org_a")) + const b = yield* svc.getOrgReadToken(asOrgId("org_b")) assert.notStrictEqual(a, b) assert.isTrue(decodePayload(b).scopes.every((s) => s.filter === "OrgId = 'org_b'")) }).pipe(Effect.provide(layer)), ) + + it.effect("returns a typed error when signing configuration is missing", () => { + const missingLayer = TinybirdOrgTokenService.layer.pipe( + Layer.provide(Env.layer), + Layer.provide(testConfig({}, false)), + ) + return Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const error = yield* Effect.flip(svc.getOrgReadToken(asOrgId("org_a"))) + assert.strictEqual(error.reason, "MissingSigningKey") + assert.notInclude(error.message, "api-token-is-not-the-signing-key") + }).pipe(Effect.provide(missingLayer)) + }) + + it.effect("returns a typed error for an empty workspace id", () => { + const malformedLayer = TinybirdOrgTokenService.layer.pipe( + Layer.provide(Env.layer), + Layer.provide(testConfig({ TINYBIRD_WORKSPACE_ID: "" })), + ) + return Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const error = yield* Effect.flip(svc.getOrgReadToken(asOrgId("org_a"))) + assert.strictEqual(error.reason, "MissingWorkspaceId") + }).pipe(Effect.provide(malformedLayer)) + }) }) diff --git a/apps/api/src/services/TinybirdOrgTokenService.ts b/apps/api/src/services/TinybirdOrgTokenService.ts index d1d243dba..9b39da6c8 100644 --- a/apps/api/src/services/TinybirdOrgTokenService.ts +++ b/apps/api/src/services/TinybirdOrgTokenService.ts @@ -1,6 +1,7 @@ -import { Clock, Context, Effect, Layer, Redacted } from "effect" +import type { OrgId } from "@maple/domain" +import { Clock, Context, Effect, Layer, Option, Redacted, Schema } from "effect" import { listOrgScopedDatasourceNames } from "../mcp/lib/warehouse-catalog" -import { deriveWorkspaceId, mintOrgReadJwt } from "../lib/tinybird-jwt" +import { mintOrgReadJwt } from "../lib/tinybird-jwt" import { Env } from "../lib/Env" // --------------------------------------------------------------------------- @@ -21,30 +22,31 @@ const JWT_REFRESH_SKEW_SECONDS = 60 export interface TinybirdOrgTokenServiceShape { /** A Tinybird read JWT scoped to `orgId` across every OrgId-bearing datasource. */ - readonly getOrgReadToken: (orgId: string) => Effect.Effect + readonly getOrgReadToken: (orgId: OrgId) => Effect.Effect } +export class TinybirdOrgTokenError extends Schema.TaggedErrorClass()( + "@maple/api/services/TinybirdOrgTokenError", + { + reason: Schema.Literals(["MissingSigningKey", "MissingWorkspaceId", "MintFailed"]), + message: Schema.String, + }, +) {} + export class TinybirdOrgTokenService extends Context.Service< TinybirdOrgTokenService, TinybirdOrgTokenServiceShape >()("@maple/api/services/TinybirdOrgTokenService", { make: Effect.gen(function* () { const env = yield* Env - const adminToken = Redacted.value(env.TINYBIRD_TOKEN) // The scope allowlist is static per deploy — compute it once. const datasourceNames = listOrgScopedDatasourceNames() - // Derive the workspace id lazily (on first mint), so constructing the layer - // never fails on a non-Tinybird token — tests and non-raw-SQL paths that - // never mint a scoped JWT don't need a real admin token. - let workspaceId: string | null = null - const getWorkspaceId = () => (workspaceId ??= deriveWorkspaceId(adminToken)) - // Per-instance (per-isolate) cache. `expiresAt` is the re-mint deadline in ms. const cache = new Map() const getOrgReadToken = Effect.fn("TinybirdOrgTokenService.getOrgReadToken")(function* ( - orgId: string, + orgId: OrgId, ) { const nowMs = yield* Clock.currentTimeMillis const cached = cache.get(orgId) @@ -53,13 +55,41 @@ export class TinybirdOrgTokenService extends Context.Service< return cached.token } yield* Effect.annotateCurrentSpan("tinybird.jwt.cacheHit", false) - const token = mintOrgReadJwt({ - adminToken, - workspaceId: getWorkspaceId(), - orgId, - datasourceNames, - nowSeconds: Math.floor(nowMs / 1000), - ttlSeconds: JWT_TTL_SECONDS, + if (Option.isNone(env.TINYBIRD_SIGNING_KEY)) { + return yield* new TinybirdOrgTokenError({ + reason: "MissingSigningKey", + message: "TINYBIRD_SIGNING_KEY is required for Tinybird-scoped raw SQL", + }) + } + if (Option.isNone(env.TINYBIRD_WORKSPACE_ID) || env.TINYBIRD_WORKSPACE_ID.value.trim() === "") { + return yield* new TinybirdOrgTokenError({ + reason: "MissingWorkspaceId", + message: "TINYBIRD_WORKSPACE_ID is required for Tinybird-scoped raw SQL", + }) + } + const workspaceId = env.TINYBIRD_WORKSPACE_ID.value + const signingKey = Redacted.value(env.TINYBIRD_SIGNING_KEY.value) + if (signingKey.trim() === "") { + return yield* new TinybirdOrgTokenError({ + reason: "MissingSigningKey", + message: "TINYBIRD_SIGNING_KEY must not be empty", + }) + } + const token = yield* Effect.try({ + try: () => + mintOrgReadJwt({ + signingKey, + workspaceId, + orgId, + datasourceNames, + nowSeconds: Math.floor(nowMs / 1000), + ttlSeconds: JWT_TTL_SECONDS, + }), + catch: () => + new TinybirdOrgTokenError({ + reason: "MintFailed", + message: "Failed to mint the Tinybird org-scoped read token", + }), }) cache.set(orgId, { token, diff --git a/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts b/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts index 1767b1a9f..1b7a4a07a 100644 --- a/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts +++ b/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts @@ -88,8 +88,11 @@ async function exec(cfg: ChConfig, sql: string): Promise { "X-ClickHouse-Database": cfg.database, } if (cfg.password.length > 0) headers["X-ClickHouse-Key"] = cfg.password - const response = await fetch(url, { method: "POST", headers, body: sql }) + const response = await fetch(url, { method: "POST", headers, body: sql, redirect: "manual" }) const text = await response.text() + if (response.status >= 300 && response.status < 400) { + throw new Error(`ClickHouse redirect responses are not allowed (${response.status})`) + } if (!response.ok) { throw new Error(`ClickHouse ${response.status}: ${text.split("\n")[0]?.slice(0, 500) ?? ""}`) } @@ -134,11 +137,7 @@ const recordVersion = (cfg: ChConfig, version: number, description: string) => // --- config load + decrypt (imperative mirror of the service helper) -------- -const loadConfig = async ( - db: MaplePgClient, - orgId: string, - encryptionKey: Buffer, -): Promise => { +const loadConfig = async (db: MaplePgClient, orgId: string, encryptionKey: Buffer): Promise => { const rows = await db .select() .from(orgClickHouseSettings) @@ -159,6 +158,21 @@ const loadConfig = async ( decipher.final(), ]).toString("utf8") } + let parsedUrl: URL + try { + parsedUrl = new URL(row.chUrl) + } catch { + throw new Error("Stored ClickHouse URL is invalid") + } + if (parsedUrl.username.length > 0 || parsedUrl.password.length > 0) { + throw new Error("Stored ClickHouse credentials must not be embedded in the URL") + } + if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") { + throw new Error("Stored ClickHouse URL must use HTTP or HTTPS") + } + if (password.length > 0 && parsedUrl.protocol !== "https:") { + throw new Error("Stored ClickHouse URL must use HTTPS when a password is configured") + } return { url: row.chUrl, user: row.chUser, password, database: row.chDatabase } } diff --git a/docs/self-hosted-clickhouse.md b/docs/self-hosted-clickhouse.md index 830d8d10a..2e072898f 100644 --- a/docs/self-hosted-clickhouse.md +++ b/docs/self-hosted-clickhouse.md @@ -24,6 +24,13 @@ Self-managed Maple is a **per-org BYO** feature. Each org configures their own b The Maple deployment itself still uses the env-level `TINYBIRD_HOST` / `TINYBIRD_TOKEN` for any org without a BYO row. API query routing does not require new env vars for ClickHouse-BYO; D1-backed direct ingest does require `MAPLE_INGEST_KEY_ENCRYPTION_KEY` so the ingest gateway can decrypt stored ClickHouse passwords. +Env-level `CLICKHOUSE_URL` defaults to a vanilla ClickHouse provider. Maple preserves +`CLICKHOUSE_PASSWORD` for raw SQL in that mode. If the URL is Tinybird's +ClickHouse-compatible gateway, set `CLICKHOUSE_PROVIDER=tinybird`; raw SQL then +substitutes a per-org JWT and removes Tinybird-restricted query settings. Tinybird +raw SQL also requires explicit `TINYBIRD_SIGNING_KEY` and +`TINYBIRD_WORKSPACE_ID` values; Maple never derives either from the API token. + ### Routing precedence For any given query the API resolves the upstream in this order: diff --git a/packages/query-engine/src/execution/executor.test.ts b/packages/query-engine/src/execution/executor.test.ts index 3befd297d..1e77a4c3d 100644 --- a/packages/query-engine/src/execution/executor.test.ts +++ b/packages/query-engine/src/execution/executor.test.ts @@ -4,6 +4,7 @@ import { TestClock } from "effect/testing" import type { OrgId } from "@maple/domain" import { compile, listRuleChecksQuery } from "../ch" import { makeWarehouseExecutor } from "./executor" +import { isRawSqlResponseLimitError, WarehouseResponseLimitError } from "./response-limits" import type { ExecutionTenant, ResolvedWarehouseConfig, @@ -21,6 +22,7 @@ const tenant: ExecutionTenant = { // pipeline (the write path). alert_checks rows only ever land in the latter. const clickhouseConfig: ResolvedWarehouseConfig = { _tag: "clickhouse", + provider: "clickhouse", url: "https://byo.example.com", username: "default", password: "secret", @@ -28,9 +30,14 @@ const clickhouseConfig: ResolvedWarehouseConfig = { } const tinybirdConfig: ResolvedWarehouseConfig = { _tag: "tinybird", + provider: "tinybird", host: "https://api.tinybird.co", token: "tb_token", } +const tinybirdGatewayConfig: ResolvedWarehouseConfig = { + ...clickhouseConfig, + provider: "tinybird", +} const compiled = compile(listRuleChecksQuery({ limit: 1 }), { orgId: "org_test", @@ -77,10 +84,9 @@ describe("makeWarehouseExecutor pinToIngestConfig", () => { // Capture the final SQL the executor hands to the client so a test can assert // whether a Tinybird-restricted setting (max_block_size) survived the strip for -// the resolved backend. The strip keys on the config `source`, not its `_tag`: -// the managed warehouse is Tinybird (SDK or its ClickHouse-compatible gateway, -// which surfaces as _tag "clickhouse" when CLICKHOUSE_URL is set) and enforces -// the restriction, so only a genuine per-org BYO ClickHouse keeps the setting. +// the resolved backend. Provider is independent from protocol and routing +// source: a Tinybird gateway uses the ClickHouse protocol but still enforces +// Tinybird's restricted-settings policy. const makeRecordingDeps = ( resolved: { config: ResolvedWarehouseConfig; source: "managed" | "org_override" }, sqls: Array, @@ -97,7 +103,24 @@ const makeRecordingDeps = ( }) describe("makeWarehouseExecutor restricted-settings strip", () => { - it.effect("strips max_block_size for the managed Tinybird CH-gateway (_tag clickhouse, source managed)", () => + it.effect( + "strips max_block_size for the managed Tinybird CH-gateway (_tag clickhouse, source managed)", + () => + Effect.gen(function* () { + const sqls: Array = [] + const executor = makeWarehouseExecutor( + makeRecordingDeps({ config: tinybirdGatewayConfig, source: "managed" }, sqls), + ) + yield* executor.compiledQuery(tenant, compiled, { + context: "test", + settings: { maxBlockSize: 512 }, + }) + assert.lengthOf(sqls, 1) + assert.isFalse(sqls[0]?.includes("max_block_size")) + }), + ) + + it.effect("keeps max_block_size for env-level vanilla ClickHouse", () => Effect.gen(function* () { const sqls: Array = [] const executor = makeWarehouseExecutor( @@ -107,36 +130,61 @@ describe("makeWarehouseExecutor restricted-settings strip", () => { context: "test", settings: { maxBlockSize: 512 }, }) - assert.lengthOf(sqls, 1) - assert.isFalse(sqls[0]?.includes("max_block_size")) + assert.isTrue(sqls[0]?.includes("max_block_size=512")) }), ) - it.effect("strips max_block_size for the managed Tinybird SDK backend (_tag tinybird, source managed)", () => - Effect.gen(function* () { - const sqls: Array = [] - const executor = makeWarehouseExecutor( - makeRecordingDeps({ config: tinybirdConfig, source: "managed" }, sqls), - ) - yield* executor.compiledQuery(tenant, compiled, { - context: "test", - settings: { maxBlockSize: 512 }, - }) - assert.isFalse(sqls[0]?.includes("max_block_size")) - }), + it.effect( + "strips max_block_size for the managed Tinybird SDK backend (_tag tinybird, source managed)", + () => + Effect.gen(function* () { + const sqls: Array = [] + const executor = makeWarehouseExecutor( + makeRecordingDeps({ config: tinybirdConfig, source: "managed" }, sqls), + ) + yield* executor.compiledQuery(tenant, compiled, { + context: "test", + settings: { maxBlockSize: 512 }, + }) + assert.isFalse(sqls[0]?.includes("max_block_size")) + }), + ) + + it.effect( + "keeps max_block_size for a genuine BYO ClickHouse (_tag clickhouse, source org_override)", + () => + Effect.gen(function* () { + const sqls: Array = [] + const executor = makeWarehouseExecutor( + makeRecordingDeps({ config: clickhouseConfig, source: "org_override" }, sqls), + ) + yield* executor.compiledQuery(tenant, compiled, { + context: "test", + settings: { maxBlockSize: 512 }, + }) + assert.isTrue(sqls[0]?.includes("max_block_size=512")) + }), ) +}) - it.effect("keeps max_block_size for a genuine BYO ClickHouse (_tag clickhouse, source org_override)", () => +describe("makeWarehouseExecutor raw response limits", () => { + it.effect("maps a driver byte abort to the typed warehouse validation marker", () => Effect.gen(function* () { - const sqls: Array = [] - const executor = makeWarehouseExecutor( - makeRecordingDeps({ config: clickhouseConfig, source: "org_override" }, sqls), - ) - yield* executor.compiledQuery(tenant, compiled, { - context: "test", - settings: { maxBlockSize: 512 }, + const executor = makeWarehouseExecutor({ + ...makeDeps([]), + createClient: () => ({ + sql: async () => { + throw new WarehouseResponseLimitError("bytes", "raw response too large") + }, + insert: async () => {}, + }), }) - assert.isTrue(sqls[0]?.includes("max_block_size=512")) + const error = yield* Effect.flip( + executor.sqlQuery(tenant, "SELECT 1 WHERE OrgId = 'org_test'", { + rawResponseLimits: true, + }), + ) + assert.isTrue(isRawSqlResponseLimitError(error)) }), ) }) diff --git a/packages/query-engine/src/execution/executor.ts b/packages/query-engine/src/execution/executor.ts index 581b53726..51c23c235 100644 --- a/packages/query-engine/src/execution/executor.ts +++ b/packages/query-engine/src/execution/executor.ts @@ -16,6 +16,7 @@ import { stripTinybirdRestrictedSettings, } from "../profiles" import { mapWarehouseError, toWarehouseQueryError, type WarehouseSqlError } from "./errors" +import { RAW_SQL_RESPONSE_LIMIT_TYPE, WarehouseResponseLimitError } from "./response-limits" import { SQL_LOG_MAX, SQL_TRACE_MAX, @@ -42,7 +43,7 @@ interface CachedClient { const sqlClientCacheKey = (config: ResolvedWarehouseConfig): string => config._tag === "clickhouse" - ? `clickhouse:${config.url}:${config.username}:${config.password}:${config.database}` + ? `${config.provider}:clickhouse:${config.url}:${config.username}:${config.password}:${config.database}` : `tinybird:${config.host}:${config.token}` // Only retry transient upstream failures (5xx, 408, 429, network blips). Non-transient @@ -54,7 +55,7 @@ const TRANSIENT_RETRY_SCHEDULE = Schedule.exponential("100 millis", 2.0).pipe( Schedule.both(Schedule.recurs(2)), ) -const isTransientUpstreamError = (error: WarehouseSqlError): boolean => +const isTransientUpstreamError = (error: WarehouseSqlError | WarehouseValidationError): boolean => error instanceof WarehouseUpstreamError // Client-side ceiling for a single query attempt. Tinybird's server-side @@ -150,14 +151,13 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue // Tinybird's managed warehouse restricts some settings (e.g. max_block_size: // "Usage of setting 'max_block_size' is restricted"). It is reached either via // the Tinybird SDK (_tag "tinybird") or its ClickHouse-compatible gateway - // (_tag "clickhouse", when CLICKHOUSE_URL is set); both enforce that policy. - // Only a genuine per-org BYO ClickHouse (source "org_override") supports those - // settings, so strip everywhere except there. Gating on `source` (not `_tag`) - // is what fixes the managed CH-gateway path. + // (_tag "clickhouse", when CLICKHOUSE_PROVIDER=tinybird); both enforce that + // policy. Vanilla ClickHouse supports those settings regardless of whether its + // credentials come from the environment or an org override. const settings = - resolved.source === "org_override" - ? resolveSettings(options) - : stripTinybirdRestrictedSettings(resolveSettings(options)) + resolved.config.provider === "tinybird" + ? stripTinybirdRestrictedSettings(resolveSettings(options)) + : resolveSettings(options) const sqlForClient = resolved.config._tag === "clickhouse" ? normalizeSqlForClickHouseClient(sql) : sql const finalSql = appendSettings(sqlForClient, settings) @@ -181,71 +181,78 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue const attemptTimeoutMs = clientTimeoutMs(options?.profile, settings?.maxExecutionTime) const retryAttempts = yield* Ref.make(0) const queryAttempt = Effect.tryPromise({ - try: () => client.sql(finalSql), - catch: (error) => mapWarehouseError(pipe, error), + try: () => client.sql(finalSql, { rawResponseLimits: options?.rawResponseLimits }), + catch: (error) => + error instanceof WarehouseResponseLimitError + ? new WarehouseValidationError({ + pipeName: pipe, + message: error.message, + cause: error, + clickhouseType: RAW_SQL_RESPONSE_LIMIT_TYPE, + }) + : mapWarehouseError(pipe, error), }) // `db.duration_ms` measures warehouse execution only — captured here, after // config resolution + settings/client-cache preamble, immediately before the // query runs. `startedAtMs` (captured at span entry) feeds the separate // `db.total_duration_ms`, covering the whole executeSql span including preamble. const sqlStartedMs = yield* Clock.currentTimeMillis - const result = yield* ( - // Bound each attempt: don't let a queued query ride the ambient ~30s - // Worker fetch limit past its declared budget. Non-transient, so it - // fails fast rather than retrying into a struggling warehouse. Skipped - // entirely for the `unbounded` profile (explicit opt-out). + // Bound each attempt: don't let a queued query ride the ambient ~30s Worker + // fetch limit past its declared budget. The timeout is non-transient, so it + // fails fast instead of retrying. The `unbounded` profile explicitly opts out. + const boundedAttempt = attemptTimeoutMs === undefined ? queryAttempt : queryAttempt.pipe( Effect.timeoutOrElse({ duration: Duration.millis(attemptTimeoutMs), orElse: () => - // Constructed directly via `toWarehouseQueryError` (non-transient) — this - // timeout must never flow through `mapWarehouseError`, whose transient - // regex could reclassify a "timeout" message and feed it back into the - // retry loop, re-amplifying load on an already-struggling warehouse. + // Constructed directly via `toWarehouseQueryError` so a transient + // message matcher cannot feed this client timeout into the retry loop. Effect.fail( toWarehouseQueryError( pipe, - new Error(`Warehouse query exceeded ${attemptTimeoutMs}ms client timeout`), + new Error( + `Warehouse query exceeded ${attemptTimeoutMs}ms client timeout`, + ), ), ), }), ) - ).pipe( - Effect.tapError((error) => - isTransientUpstreamError(error) ? Ref.update(retryAttempts, (n) => n + 1) : Effect.void, - ), - Effect.retry({ - schedule: TRANSIENT_RETRY_SCHEDULE, - while: isTransientUpstreamError, - }), - Effect.tapError((error) => - Effect.gen(function* () { - const nowMs = yield* Clock.currentTimeMillis - const elapsedMs = nowMs - sqlStartedMs - const totalElapsedMs = nowMs - startedAtMs - const attempts = yield* Ref.get(retryAttempts) - yield* Effect.annotateCurrentSpan("db.duration_ms", elapsedMs) - yield* Effect.annotateCurrentSpan("db.total_duration_ms", totalElapsedMs) - yield* Effect.annotateCurrentSpan("db.retry.attempts", attempts) - yield* Effect.logError("WarehouseQueryService.executeSql failed", { - pipe, - context: options?.context, - orgId: tenant.orgId, - backend: resolved.config._tag, - durationMs: elapsedMs, - retryAttempts: attempts, - errorTag: error._tag, - message: error.message, - sql: truncateSql(finalSql, SQL_LOG_MAX), - sqlLength, - sqlFingerprint: fingerprintSql(finalSql), - profile: options?.profile, - }) + const result = yield* boundedAttempt.pipe( + Effect.tapError((error) => + isTransientUpstreamError(error) ? Ref.update(retryAttempts, (n) => n + 1) : Effect.void, + ), + Effect.retry({ + schedule: TRANSIENT_RETRY_SCHEDULE, + while: isTransientUpstreamError, }), - ), - ) + Effect.tapError((error) => + Effect.gen(function* () { + const nowMs = yield* Clock.currentTimeMillis + const elapsedMs = nowMs - sqlStartedMs + const totalElapsedMs = nowMs - startedAtMs + const attempts = yield* Ref.get(retryAttempts) + yield* Effect.annotateCurrentSpan("db.duration_ms", elapsedMs) + yield* Effect.annotateCurrentSpan("db.total_duration_ms", totalElapsedMs) + yield* Effect.annotateCurrentSpan("db.retry.attempts", attempts) + yield* Effect.logError("WarehouseQueryService.executeSql failed", { + pipe, + context: options?.context, + orgId: tenant.orgId, + backend: resolved.config._tag, + durationMs: elapsedMs, + retryAttempts: attempts, + errorTag: error._tag, + message: error.message, + sql: truncateSql(finalSql, SQL_LOG_MAX), + sqlLength, + sqlFingerprint: fingerprintSql(finalSql), + profile: options?.profile, + }) + }), + ), + ) yield* Effect.annotateCurrentSpan("result.rowCount", result.data.length) const completedAtMs = yield* Clock.currentTimeMillis @@ -372,7 +379,9 @@ export const makeWarehouseExecutor = (deps: WarehouseExecutorDeps): WarehouseQue // ClickHouse READ override (routing writes through the override 500'd every // insert and broke demo-seed onboarding). Falls back to the read resolver. const resolveForIngest = deps.resolveIngestConfig ?? deps.resolveConfig - const resolved = yield* resolveForIngest(tenant, label) + const resolved = yield* resolveForIngest(tenant, label).pipe( + Effect.mapError((error) => toWarehouseQueryError(label, error)), + ) // Insert through the same client the read path uses (official // @clickhouse/client-web for ClickHouse, Tinybird Events API for diff --git a/packages/query-engine/src/execution/index.ts b/packages/query-engine/src/execution/index.ts index d39e5a123..54008dea0 100644 --- a/packages/query-engine/src/execution/index.ts +++ b/packages/query-engine/src/execution/index.ts @@ -2,3 +2,4 @@ export * from "./ports" export * from "./errors" export * from "./fingerprint" export * from "./executor" +export * from "./response-limits" diff --git a/packages/query-engine/src/execution/ports.ts b/packages/query-engine/src/execution/ports.ts index 26e4ad971..d772afeae 100644 --- a/packages/query-engine/src/execution/ports.ts +++ b/packages/query-engine/src/execution/ports.ts @@ -44,12 +44,17 @@ export type SqlQueryOptions = { * ClickHouse orgs (already isolated by their own credentials). */ scopeToOrgJwt?: boolean + /** Apply the hard pre-buffer limits used for untrusted raw-SQL responses. */ + rawResponseLimits?: boolean } +export type WarehouseProvider = "clickhouse" | "tinybird" + /** Resolved upstream connection config for a tenant's queries. */ export type ResolvedWarehouseConfig = | { readonly _tag: "clickhouse" + readonly provider: WarehouseProvider readonly url: string readonly username: string readonly password: string @@ -57,13 +62,17 @@ export type ResolvedWarehouseConfig = } | { readonly _tag: "tinybird" + readonly provider: "tinybird" readonly host: string readonly token: string } /** Minimal client interface — raw SQL execution plus row inserts. */ export interface WarehouseSqlClient { - readonly sql: (sql: string) => Promise<{ data: ReadonlyArray> }> + readonly sql: ( + sql: string, + options?: { readonly rawResponseLimits?: boolean }, + ) => Promise<{ data: ReadonlyArray> }> readonly insert: (datasource: string, rows: ReadonlyArray) => Promise } @@ -89,7 +98,7 @@ export interface WarehouseExecutorDeps { options?: ResolveConfigOptions, ) => Effect.Effect< { readonly config: ResolvedWarehouseConfig; readonly source: "managed" | "org_override" }, - WarehouseQueryError + WarehouseSqlError > /** * Config resolver for the WRITE path (`ingest`). Inserts must land in the diff --git a/packages/query-engine/src/execution/response-limits.ts b/packages/query-engine/src/execution/response-limits.ts new file mode 100644 index 000000000..ff0d2fbc0 --- /dev/null +++ b/packages/query-engine/src/execution/response-limits.ts @@ -0,0 +1,19 @@ +import { WarehouseValidationError } from "@maple/domain/http" + +export const RAW_SQL_RESPONSE_LIMIT_TYPE = "RAW_SQL_RESPONSE_LIMIT" + +export type WarehouseResponseLimitKind = "rows" | "bytes" + +/** Driver-level abort used before a raw response can be fully buffered. */ +export class WarehouseResponseLimitError extends Error { + readonly kind: WarehouseResponseLimitKind + + constructor(kind: WarehouseResponseLimitKind, message: string) { + super(message) + this.name = "WarehouseResponseLimitError" + this.kind = kind + } +} + +export const isRawSqlResponseLimitError = (error: unknown): error is WarehouseValidationError => + error instanceof WarehouseValidationError && error.clickhouseType === RAW_SQL_RESPONSE_LIMIT_TYPE diff --git a/packages/query-engine/src/runtime/query-engine.ts b/packages/query-engine/src/runtime/query-engine.ts index 1767a857c..b40114178 100644 --- a/packages/query-engine/src/runtime/query-engine.ts +++ b/packages/query-engine/src/runtime/query-engine.ts @@ -31,8 +31,9 @@ import type { OrgId } from "@maple/domain" import { Array as Arr, Duration, Effect, Match, Option, Result, Schema } from "effect" import { LOGS_BODY_SEARCH_SETTINGS, type QueryProfileName, type WarehouseQuerySettings } from "../profiles" import { computeBucketSeconds } from "../datetime" -import { makeExpandMacros } from "./raw-sql" +import { makeExpandMacros, RAW_SQL_EXECUTION_GUARDS } from "./raw-sql" import { rawSqlResultLimitError } from "./raw-result-limits" +import { isRawSqlResponseLimitError } from "../execution/response-limits" import { decodeEvalSeries, encodeEvalPoints, type BucketGroupObs } from "./evaluate-bucket-codec" // Re-exported so `@maple/query-engine/runtime` consumers (apps/api) keep importing @@ -60,6 +61,7 @@ export interface QueryEngineWarehouse { readonly context?: string readonly settings?: WarehouseQuerySettings readonly scopeToOrgJwt?: boolean + readonly rawResponseLimits?: boolean }, ) => Effect.Effect>, WarehouseError> readonly compiledQuery: ( @@ -2139,10 +2141,20 @@ export const makeQueryEngineEvaluateRawSql = (warehouse: warehouse.sqlQuery(tenant, expanded.sql, { profile: "rawAlert", context: "alertRawQuery", - // Untrusted user SQL — scope to a per-org Tinybird JWT (server-enforced isolation). - scopeToOrgJwt: true, + ...RAW_SQL_EXECUTION_GUARDS, }), "alertRawQuery", + ).pipe( + Effect.catchTag("@maple/http/errors/WarehouseValidationError", (error) => + isRawSqlResponseLimitError(error) + ? Effect.fail( + new QueryEngineValidationError({ + message: "Invalid raw SQL alert query", + details: [error.message], + }), + ) + : Effect.fail(error), + ), ) const resultLimitError = rawSqlResultLimitError(rawRows) if (resultLimitError != null) { diff --git a/packages/query-engine/src/runtime/raw-sql.test.ts b/packages/query-engine/src/runtime/raw-sql.test.ts index 40e4d9010..a363a4c0b 100644 --- a/packages/query-engine/src/runtime/raw-sql.test.ts +++ b/packages/query-engine/src/runtime/raw-sql.test.ts @@ -149,22 +149,23 @@ describe("RawSqlChartService.expandMacros", () => { }), ) - it.effect("appends a default LIMIT when the user did not specify one", () => + it.effect("wraps queries with the 1,001-row overflow sentinel", () => Effect.gen(function* () { const result = yield* expandOk( "SELECT 1 FROM Logs WHERE $__orgFilter AND $__timeFilter(Timestamp)", ) - assert.match(result.sql, /LIMIT 10000\s*$/) + assert.match(result.sql, /^SELECT \* FROM \(/) + assert.match(result.sql, /LIMIT 1001\s*$/) }), ) - it.effect("preserves the user's LIMIT if already present", () => + it.effect("enforces the outer cap even when the user requests a larger LIMIT", () => Effect.gen(function* () { const result = yield* expandOk( - "SELECT 1 FROM Logs WHERE $__orgFilter AND $__timeFilter(Timestamp) LIMIT 7", + "SELECT 1 FROM Logs WHERE $__orgFilter AND $__timeFilter(Timestamp) LIMIT 50000", ) - assert.notInclude(result.sql, "LIMIT 10000") - assert.match(result.sql, /LIMIT 7/) + assert.include(result.sql, "LIMIT 50000") + assert.match(result.sql, /LIMIT 1001\s*$/) }), ) diff --git a/packages/query-engine/src/runtime/raw-sql.ts b/packages/query-engine/src/runtime/raw-sql.ts index fe8c3a671..172efff89 100644 --- a/packages/query-engine/src/runtime/raw-sql.ts +++ b/packages/query-engine/src/runtime/raw-sql.ts @@ -1,5 +1,5 @@ import { Context, Effect, Layer } from "effect" -import { RawSqlValidationError } from "@maple/domain/http" +import { MAX_RAW_SQL_RESULT_ROWS, RawSqlValidationError } from "@maple/domain/http" import { escapeClickHouseString } from "../sql" // --------------------------------------------------------------------------- @@ -40,7 +40,14 @@ const DENY_LIST = [ const DENY_LIST_RE = new RegExp(`\\b(${DENY_LIST.join("|")})\\b`, "i") -const DEFAULT_ROW_CAP = 10_000 +/** One extra row is the overflow sentinel for the public 1,000-row cap. */ +export const RAW_SQL_FETCH_ROW_LIMIT = MAX_RAW_SQL_RESULT_ROWS + 1 + +/** Shared execution guards required by every HTTP, MCP, and alert raw-SQL caller. */ +export const RAW_SQL_EXECUTION_GUARDS = { + scopeToOrgJwt: true, + rawResponseLimits: true, +} as const export interface ExpandMacrosInput { readonly sql: string @@ -181,10 +188,20 @@ export const makeExpandMacros = Effect.fn("RawSqlChartService.expandMacros")(fun ) } - if (!/\blimit\b/i.test(masked)) { - sql = `${sql.trimEnd()}\nLIMIT ${DEFAULT_ROW_CAP}` + if (!/^\s*(?:SELECT|WITH)\b/i.test(masked)) { + return yield* fail( + "DisallowedStatement", + "Raw SQL must be a SELECT query (WITH common table expressions are supported).", + ) } + // An inner LIMIT is not a trustworthy response bound: users can request a + // much larger value, and UNION / nested queries can make textual rewriting + // ambiguous. Put every validated query behind one outer cap instead. Fetching + // one extra row lets the caller distinguish exactly 1,000 rows from overflow + // without ever buffering an unbounded result. + sql = `SELECT * FROM (\n${sql.trim()}\n) AS maple_raw_sql_limited\nLIMIT ${RAW_SQL_FETCH_ROW_LIMIT}` + return { sql, granularitySeconds: granularity, From 24e8371b2b5fef8bd91fadb6f47145dba575d56d Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 19 Jul 2026 00:00:09 +0200 Subject: [PATCH 5/7] s --- .env.example | 5 +- CONTRIBUTING.md | 2 +- apps/api/alchemy.run.ts | 2 +- apps/api/src/lib/Env.ts | 2 +- .../api/src/lib/WarehouseQueryService.test.ts | 3 +- apps/api/src/lib/WarehouseQueryService.ts | 10 +- apps/api/src/routes/v2/telemetry.http.test.ts | 58 ++++-- apps/api/src/routes/v2/telemetry.http.ts | 64 +++--- apps/api/src/services/AlertsService.test.ts | 19 +- apps/api/src/services/AlertsService.ts | 102 ++++----- .../src/services/TinybirdOrgTokenService.ts | 4 +- .../settings/api-key-create-payload.test.ts | 6 +- .../settings/api-key-create-payload.ts | 6 +- .../settings/create-api-key-dialog.tsx | 142 +++++++------ .../src/components/settings/mcp-section.tsx | 1 + docs/self-hosted-clickhouse.md | 10 +- packages/domain/src/http/v2/telemetry.ts | 10 +- packages/domain/src/query-engine.ts | 1 + packages/query-engine/src/ch/ch.test.ts | 2 + packages/query-engine/src/ch/index.ts | 1 + .../query-engine/src/ch/queries/errors.ts | 29 ++- .../src/ch/queries/metrics.test.ts | 22 +- .../query-engine/src/ch/queries/metrics.ts | 73 +++++-- .../src/ch/queries/service-map.test.ts | 32 ++- .../src/ch/queries/service-map.ts | 195 ++++++++++-------- .../query-engine/src/runtime/query-engine.ts | 82 ++++---- 26 files changed, 543 insertions(+), 340 deletions(-) diff --git a/.env.example b/.env.example index 0f412b7be..e133d4174 100644 --- a/.env.example +++ b/.env.example @@ -8,8 +8,9 @@ TINYBIRD_TOKEN=your-tinybird-token # ClickHouse CLICKHOUSE_URL=http://localhost:9000 -# clickhouse (default) preserves CLICKHOUSE_PASSWORD for raw SQL. Set tinybird -# only when CLICKHOUSE_URL points at Tinybird's ClickHouse-compatible gateway. +# tinybird is the default for compatibility with existing gateway deployments. +# Set clickhouse when CLICKHOUSE_URL points at a vanilla/self-managed server so +# raw SQL preserves CLICKHOUSE_PASSWORD instead of substituting a scoped JWT. CLICKHOUSE_PROVIDER=clickhouse CLICKHOUSE_USER=maple CLICKHOUSE_DATABASE=default diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 23d685428..e66ca955b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -189,7 +189,7 @@ Default URL: `http://localhost:3472` | `TINYBIRD_TOKEN` | yes | Placeholder OK when using `CLICKHOUSE_URL` | | `TINYBIRD_SIGNING_KEY` / `TINYBIRD_WORKSPACE_ID` | with Tinybird raw SQL | Explicit JWT signing configuration; never derived from `TINYBIRD_TOKEN` | | `CLICKHOUSE_URL` | recommended | `http://localhost:8123` for local ClickHouse stack | -| `CLICKHOUSE_PROVIDER` | optional | `clickhouse` (default) or `tinybird`; existing Tinybird gateway deployments must set `tinybird` | +| `CLICKHOUSE_PROVIDER` | optional | `tinybird` (default) or `clickhouse`; set `clickhouse` for env-level vanilla/self-managed ClickHouse | | `CLICKHOUSE_USER` / `CLICKHOUSE_PASSWORD` / `CLICKHOUSE_DATABASE` | with CH | Match docker-compose (`maple` / `maple` / `default`) | | `INTERNAL_SERVICE_TOKEN` | recommended | Shared with scraper + chat-flue | | `SD_INTERNAL_TOKEN` | optional | Prometheus scraper internal API auth | diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index a57dfb02e..9589db6e7 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -151,7 +151,7 @@ export const createMapleApi = ({ stage, domains }: CreateMapleApiOptions) => ...optionalSecret("TINYBIRD_SIGNING_KEY"), ...optionalPlain("TINYBIRD_WORKSPACE_ID"), ...optionalPlain("CLICKHOUSE_URL"), - CLICKHOUSE_PROVIDER: process.env.CLICKHOUSE_PROVIDER?.trim() || "clickhouse", + CLICKHOUSE_PROVIDER: process.env.CLICKHOUSE_PROVIDER?.trim() || "tinybird", ...optionalPlain("CLICKHOUSE_USER"), ...optionalPlain("CLICKHOUSE_DATABASE"), ...optionalSecret("CLICKHOUSE_PASSWORD"), diff --git a/apps/api/src/lib/Env.ts b/apps/api/src/lib/Env.ts index d3849bccb..9a06eff8f 100644 --- a/apps/api/src/lib/Env.ts +++ b/apps/api/src/lib/Env.ts @@ -88,7 +88,7 @@ const envConfig = Config.all({ TINYBIRD_SIGNING_KEY: optionalRedacted("TINYBIRD_SIGNING_KEY"), TINYBIRD_WORKSPACE_ID: optionalString("TINYBIRD_WORKSPACE_ID"), CLICKHOUSE_URL: optionalString("CLICKHOUSE_URL"), - CLICKHOUSE_PROVIDER: stringWithDefault("CLICKHOUSE_PROVIDER", "clickhouse"), + CLICKHOUSE_PROVIDER: stringWithDefault("CLICKHOUSE_PROVIDER", "tinybird"), CLICKHOUSE_USER: stringWithDefault("CLICKHOUSE_USER", "default"), CLICKHOUSE_PASSWORD: optionalRedacted("CLICKHOUSE_PASSWORD"), CLICKHOUSE_DATABASE: stringWithDefault("CLICKHOUSE_DATABASE", "default"), diff --git a/apps/api/src/lib/WarehouseQueryService.test.ts b/apps/api/src/lib/WarehouseQueryService.test.ts index 46a30ddc1..fb1f20945 100644 --- a/apps/api/src/lib/WarehouseQueryService.test.ts +++ b/apps/api/src/lib/WarehouseQueryService.test.ts @@ -123,7 +123,7 @@ describe("WarehouseQueryService raw-SQL provider routing", () => { }).pipe(Effect.provide(layer)) }) - it.effect("substitutes a scoped JWT for the Tinybird ClickHouse gateway password", () => { + it.effect("defaults an env-level ClickHouse gateway to Tinybird and substitutes a scoped JWT", () => { let captured: ResolvedWarehouseConfig | undefined __testables.setClientFactory((config) => { captured = config @@ -131,7 +131,6 @@ describe("WarehouseQueryService raw-SQL provider routing", () => { }) const layer = buildLayer(createTestDb(trackedDbs), { CLICKHOUSE_URL: "https://gateway.tinybird.example", - CLICKHOUSE_PROVIDER: "tinybird", CLICKHOUSE_PASSWORD: "gateway-admin-token", }) diff --git a/apps/api/src/lib/WarehouseQueryService.ts b/apps/api/src/lib/WarehouseQueryService.ts index 2ffd31009..92d8e3d71 100644 --- a/apps/api/src/lib/WarehouseQueryService.ts +++ b/apps/api/src/lib/WarehouseQueryService.ts @@ -172,9 +172,13 @@ const createTinybirdSdkSqlClient = (config: TinybirdConfig): WarehouseSqlClient return { sql: async (sql: string, options) => { try { + // Tinybird Cloud currently defaults /v0/sql to JSON, while Tinybird Local + // defaults to tab-separated output. The SDK always calls response.json(), so + // make the expected wire format explicit for both environments. + const jsonSql = `${sql.trimEnd().replace(/;$/, "")}\nFORMAT JSON` const result = await (options?.rawResponseLimits ? boundedClient : client).sql< Record - >(sql) + >(jsonSql) if (options?.rawResponseLimits && result.data.length > MAX_RAW_SQL_RESULT_ROWS) { throw new WarehouseResponseLimitError( "rows", @@ -335,7 +339,7 @@ export class WarehouseQueryService extends Context.Service< }), ), ) - yield* Effect.annotateCurrentSpan("tinybird.token.scope", "org_jwt") + yield* Effect.annotateCurrentSpan("maple.tinybird.token.scope", "org_jwt") if (managed.config._tag === "tinybird") { return { config: { ...managed.config, token: jwt }, source: managed.source } } @@ -367,7 +371,7 @@ export class WarehouseQueryService extends Context.Service< )(function* (tenant, _label) { yield* Effect.annotateCurrentSpan("orgId", tenant.orgId) yield* Effect.annotateCurrentSpan("clientSource", "managed") - yield* Effect.annotateCurrentSpan("ingest.routing", "tinybird") + yield* Effect.annotateCurrentSpan("query.routing", "ingest") yield* Effect.annotateCurrentSpan("db.client", "tinybird-sdk") return { config: { diff --git a/apps/api/src/routes/v2/telemetry.http.test.ts b/apps/api/src/routes/v2/telemetry.http.test.ts index 9f04c3db8..4cc278498 100644 --- a/apps/api/src/routes/v2/telemetry.http.test.ts +++ b/apps/api/src/routes/v2/telemetry.http.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it } from "@effect/vitest" import { OrgId, UserId, WarehouseQueryError } from "@maple/domain/http" import { MapleApiV2 } from "@maple/domain/http/v2" -import { QueryEngineExecuteResponse } from "@maple/query-engine" +import { QueryEngineExecuteResponse, type QueryEngineExecuteRequest } from "@maple/query-engine" import { ConfigProvider, Context, Effect, Layer, ManagedRuntime, Option, Schema } from "effect" import { HttpRouter } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" @@ -12,10 +12,7 @@ import { ApiAuthorizationV2Layer } from "../../services/ApiAuthorizationV2Layer" import { ApiKeysService } from "../../services/ApiKeysService" import { AuthService } from "../../services/AuthService" import { DashboardPersistenceService } from "../../services/DashboardPersistenceService" -import { - QueryEngineService, - type QueryEngineServiceShape, -} from "../../services/QueryEngineService" +import { QueryEngineService, type QueryEngineServiceShape } from "../../services/QueryEngineService" import { V2SchemaErrorsLive } from "./error-envelope" import { AlertsServiceStubLayer, @@ -118,8 +115,7 @@ const rowsForSql = (sql: string): ReadonlyArray> => { if (sql.includes("FROM trace_detail_spans") && sql.includes("toJSONString(SpanAttributes)")) { return [ { - traceId: TRACE_ID, - spanId: SPAN_ID, + ...hierarchyRow, spanAttributes: JSON.stringify({ "http.route": "/checkout", "error.type": "Timeout" }), resourceAttributes: JSON.stringify({ "service.name": "api" }), }, @@ -275,19 +271,12 @@ describe("v2 telemetry reads over HTTP", () => { expect(traces.status).toBe(200) expect(traces.body.data[0]).toMatchObject({ object: "trace", id: TRACE_ID, has_error: true }) - const trace = await harness.request( - "GET", - `/v2/traces/${TRACE_ID}`, - key.secret, - ) + const trace = await harness.request("GET", `/v2/traces/${TRACE_ID}`, key.secret) expect(trace.status).toBe(200) expect(trace.body.spans).toHaveLength(1) + expect(trace.body.truncated).toBe(false) - const span = await harness.request( - "GET", - `/v2/traces/${TRACE_ID}/spans/${SPAN_ID}`, - key.secret, - ) + const span = await harness.request("GET", `/v2/traces/${TRACE_ID}/spans/${SPAN_ID}`, key.secret) expect(span.status).toBe(200) expect(span.body.attributes["error.type"]).toBe("Timeout") @@ -419,6 +408,7 @@ describe("v2 telemetry reads over HTTP", () => { expect(hierarchySql).toContain(`TraceId = '${TRACE_ID}'`) expect(hierarchySql).not.toContain("Timestamp >=") expect(hierarchySql).not.toContain("Timestamp <=") + expect(hierarchySql).toContain("LIMIT 5001") await harness.dispose() }) @@ -446,8 +436,42 @@ describe("v2 telemetry reads over HTTP", () => { }) expect(response.status).toBe(200) expect(observedRequest?.query).toMatchObject({ + seriesLimit: 50, filters: { groupByAttributeKeys: ["http.route"] }, }) + const excessiveLimit = await harness.request("POST", "/v2/query", key.secret, { + start_time: START, + end_time: END, + query: { + kind: "timeseries", + source: "traces", + metric: "count", + group_by: ["service"], + series_limit: 101, + }, + }) + expect(excessiveLimit.status).toBe(400) + await harness.dispose() + }) + + it("applies bounded ClickHouse settings to log body searches", async () => { + let observedOptions: Parameters[2] + const observingWarehouse: WarehouseQueryServiceShape = { + ...warehouseStub, + compiledQuery: (tenant, compiled, options) => { + observedOptions = options + return warehouseStub.compiledQuery(tenant, compiled, options) + }, + } + const harness = makeHarness(observingWarehouse) + const key = await harness.bootstrapKey(["logs:read"]) + const response = await harness.request("POST", "/v2/logs/search", key.secret, { + start_time: START, + end_time: END, + search: "checkout failed", + }) + expect(response.status).toBe(200) + expect(observedOptions?.settings).toMatchObject({ maxBlockSize: 512 }) await harness.dispose() }) diff --git a/apps/api/src/routes/v2/telemetry.http.ts b/apps/api/src/routes/v2/telemetry.http.ts index 5f3bdaae7..5f7178535 100644 --- a/apps/api/src/routes/v2/telemetry.http.ts +++ b/apps/api/src/routes/v2/telemetry.http.ts @@ -17,6 +17,7 @@ import { type V2TraceSummary, } from "@maple/domain/http/v2" import { CH, QueryEngineExecuteRequest } from "@maple/query-engine" +import { LOGS_BODY_SEARCH_SETTINGS } from "@maple/query-engine/profiles" import { MAX_QUERY_RANGE_SECONDS } from "@maple/query-engine/runtime" import { Effect, Encoding, Option, Result, Schema } from "effect" import { WarehouseQueryService } from "../../lib/WarehouseQueryService" @@ -53,6 +54,7 @@ const serviceCatalogRowSchema = Schema.Struct({ }) const PARTITION_HINT_RADIUS_MS = 60 * 60 * 1000 +const PUBLIC_TIMESERIES_DEFAULT_SERIES_LIMIT = 50 const mapWarehouseError = (operation: string) => () => dependencyUnavailable(`${operation}_unavailable`) @@ -96,8 +98,7 @@ const chToIso = (value: string): Timestamp => { return timestamp(Number.isNaN(ms) ? value : new Date(ms).toISOString()) } -const warehouseDate = (ms: number) => - new Date(ms).toISOString().replace("T", " ").replace(/Z$/, "") +const warehouseDate = (ms: number) => new Date(ms).toISOString().replace("T", " ").replace(/Z$/, "") const partitionWindow = (value: string) => { const ms = Date.parse(value.includes("T") ? value : `${value.replace(" ", "T")}Z`) return { @@ -127,9 +128,7 @@ const compactTimestamp = (value: string) => { const match = /^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(\.\d+)?$/.exec(value) if (match === null) return value const epochSeconds = Date.parse(`${match[1]}T${match[2]}Z`) / 1000 - return Number.isSafeInteger(epochSeconds) - ? `~${epochSeconds.toString(36)}${match[3] ?? ""}` - : value + return Number.isSafeInteger(epochSeconds) ? `~${epochSeconds.toString(36)}${match[3] ?? ""}` : value } const expandTimestamp = (value: string) => { const match = /^~([0-9a-z]+)(\.\d+)?$/.exec(value) @@ -153,10 +152,7 @@ const expandHexId = (value: string) => { return [...decoded.success].map((byte) => byte.toString(16).padStart(2, "0")).join("") } const logKey = (row: { timestamp: string; recordIdentity: string }) => - JSON.stringify([ - compactTimestamp(row.timestamp), - compactHexId(row.recordIdentity), - ] satisfies LogKey) + JSON.stringify([compactTimestamp(row.timestamp), compactHexId(row.recordIdentity)] satisfies LogKey) const parseLogKey = (value: string) => { try { @@ -312,7 +308,10 @@ const toInternalQuery = (query: Record): Record => { metric: query.metric, groupBy: query.group_by, bucketSeconds: query.bucket_seconds, - seriesLimit: query.series_limit, + seriesLimit: + query.kind === "timeseries" + ? (query.series_limit ?? PUBLIC_TIMESERIES_DEFAULT_SERIES_LIMIT) + : undefined, limit: query.limit, apdexThresholdMs: query.apdex_threshold_ms, } @@ -368,7 +367,9 @@ const queryError = (error: unknown) => { const decodeQueryEngineRequest = (input: unknown) => Schema.decodeUnknownEffect(QueryEngineExecuteRequest)(input).pipe( - Effect.mapError(() => invalidRequest("query_invalid", "The query specification is invalid.", "query")), + Effect.mapError(() => + invalidRequest("query_invalid", "The query specification is invalid.", "query"), + ), ) export const HttpV2TracesLive = HttpApiBuilder.group(MapleApiV2, "traces", (handlers) => @@ -379,9 +380,12 @@ export const HttpV2TracesLive = HttpApiBuilder.group(MapleApiV2, "traces", (hand tenant: CurrentTenant.TenantSchema, traceId: string, ) { - const compiled = CH.compile(CH.spanHierarchyQuery({ traceId }), { - orgId: tenant.orgId, - }) + const compiled = CH.compile( + CH.spanHierarchyQuery({ traceId, limit: CH.SPAN_HIERARCHY_MAX_SPANS + 1 }), + { + orgId: tenant.orgId, + }, + ) return yield* warehouse .compiledQuery(tenant, compiled, { profile: "list", @@ -437,7 +441,8 @@ export const HttpV2TracesLive = HttpApiBuilder.group(MapleApiV2, "traces", (hand const tenant = yield* CurrentTenant.Context const rows = yield* hierarchy(tenant, params.trace_id) if (rows.length === 0) return yield* resourceNotFound("trace", "No such trace.") - const spans = rows.map(toSpan) + const truncated = rows.length > CH.SPAN_HIERARCHY_MAX_SPANS + const spans = rows.slice(0, CH.SPAN_HIERARCHY_MAX_SPANS).map(toSpan) const startMs = Math.min(...spans.map((span) => Date.parse(span.start_time))) const endMs = Math.max( ...spans.map((span) => Date.parse(span.start_time) + span.duration_ms), @@ -450,6 +455,7 @@ export const HttpV2TracesLive = HttpApiBuilder.group(MapleApiV2, "traces", (hand duration_ms: endMs - startMs, span_count: spans.length, service_count: new Set(spans.map((span) => span.service_name)).size, + truncated, spans, } }), @@ -457,9 +463,6 @@ export const HttpV2TracesLive = HttpApiBuilder.group(MapleApiV2, "traces", (hand .handle("retrieveSpan", ({ params }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const rows = yield* hierarchy(tenant, params.trace_id) - const hierarchyRow = rows.find((row) => row.spanId === params.span_id) - if (!hierarchyRow) return yield* resourceNotFound("span", "No such span.") const detail = yield* warehouse .compiledQueryFirst( tenant, @@ -472,15 +475,9 @@ export const HttpV2TracesLive = HttpApiBuilder.group(MapleApiV2, "traces", (hand ), { profile: "discovery", context: "v2GetSpan" }, ) - .pipe( - Effect.mapError(mapWarehouseError("span_query")), - Effect.map(Option.getOrNull), - ) - return toSpan({ - ...hierarchyRow, - spanAttributes: detail?.spanAttributes ?? hierarchyRow.spanAttributes, - resourceAttributes: detail?.resourceAttributes ?? hierarchyRow.resourceAttributes, - }) + .pipe(Effect.mapError(mapWarehouseError("span_query")), Effect.map(Option.getOrNull)) + if (!detail) return yield* resourceNotFound("span", "No such span.") + return toSpan(detail) }), ) }), @@ -522,7 +519,11 @@ export const HttpV2LogsLive = HttpApiBuilder.group(MapleApiV2, "logs", (handlers { orgId: tenant.orgId, ...window }, ) const rows = yield* warehouse - .compiledQuery(tenant, compiled, { profile: "list", context: "v2LogSearch" }) + .compiledQuery(tenant, compiled, { + profile: "list", + context: "v2LogSearch", + settings: payload.search ? LOGS_BODY_SEARCH_SETTINGS : undefined, + }) .pipe(Effect.mapError(mapWarehouseError("log_search"))) const dataRows = rows.slice(0, limit) const last = dataRows.at(-1) @@ -632,6 +633,7 @@ export const HttpV2MetricsLive = HttpApiBuilder.group(MapleApiV2, "metrics", (ha metric: payload.metric, groupBy: payload.group_by, bucketSeconds: payload.bucket_seconds, + seriesLimit: payload.series_limit ?? PUBLIC_TIMESERIES_DEFAULT_SERIES_LIMIT, filters: { metricName: payload.metric_name, metricType: payload.metric_type, @@ -767,7 +769,7 @@ const toMapEdge = (row: { error_count: errors, error_rate: calls > 0 ? errors / calls : 0, avg_duration_ms: Number(row.avgDurationMs), - p95_duration_ms: Number(row.p95DurationMs), + max_duration_ms: Number(row.p95DurationMs), has_sampling: estimated > calls + 0.001, sampling_weight: calls > 0 ? estimated / calls : 1, } @@ -821,9 +823,7 @@ export const HttpV2QueryLive = HttpApiBuilder.group(MapleApiV2, "query", (handle endTime: window.endTime, query: toInternalQuery(payload.query), }) - const response = yield* queryEngine - .execute(tenant, request) - .pipe(Effect.mapError(queryError)) + const response = yield* queryEngine.execute(tenant, request).pipe(Effect.mapError(queryError)) if (response.result.kind !== "timeseries" && response.result.kind !== "breakdown") { return yield* Effect.fail(dependencyUnavailable("query_unavailable")) } diff --git a/apps/api/src/services/AlertsService.test.ts b/apps/api/src/services/AlertsService.test.ts index 941c0fb01..3e8ee7f70 100644 --- a/apps/api/src/services/AlertsService.test.ts +++ b/apps/api/src/services/AlertsService.test.ts @@ -356,16 +356,27 @@ describe("AlertsService", () => { }), ) yield* Effect.forEach( - Array.from({ length: 100 }, (_, index) => index), + Array.from({ length: 99 }, (_, index) => index), createRule, { concurrency: 1, discard: true }, ) - const exit = yield* createRule(100).pipe(Effect.exit) - assert.isTrue(Exit.isFailure(exit)) - const error = getError(exit) + const exits = yield* Effect.all( + [createRule(99).pipe(Effect.exit), createRule(100).pipe(Effect.exit)], + { + concurrency: "unbounded", + }, + ) + assert.strictEqual(exits.filter(Exit.isSuccess).length, 1) + assert.strictEqual(exits.filter(Exit.isFailure).length, 1) + const error = getError(exits.find(Exit.isFailure)!) assert.instanceOf(error, AlertValidationError) assert.include((error as AlertValidationError).message, "at most 100 active alert rules") + const rules = yield* alerts.listRules(orgId) + assert.lengthOf( + rules.rules.filter((rule) => rule.enabled), + 100, + ) }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub({}), { fetch: okFetch }))) }) diff --git a/apps/api/src/services/AlertsService.ts b/apps/api/src/services/AlertsService.ts index ba0df6b42..0bcaee4de 100644 --- a/apps/api/src/services/AlertsService.ts +++ b/apps/api/src/services/AlertsService.ts @@ -85,7 +85,7 @@ import { type AlertRuleRow, alertRuleStates, } from "@maple/db" -import { and, asc, desc, eq, inArray, isNotNull, isNull, lt, lte, ne, or } from "drizzle-orm" +import { and, asc, desc, eq, inArray, isNotNull, isNull, lt, lte, ne, or, sql } from "drizzle-orm" import { Array as Arr, Cause, @@ -849,12 +849,7 @@ const compileRulePlan = Effect.fn("AlertsService.compileRulePlan")(function* (ru const parseCompiledPlan = ( row: Pick< AlertRuleRow, - | "signalType" - | "querySpecJson" - | "rawQuerySql" - | "reducer" - | "sampleCountStrategy" - | "noDataBehavior" + "signalType" | "querySpecJson" | "rawQuerySql" | "reducer" | "sampleCountStrategy" | "noDataBehavior" >, ): Effect.Effect, AlertValidationError> => { if (row.signalType === "raw_query") { @@ -968,7 +963,7 @@ const rowToRuleDocument = ( row.metricAggregation != null ? decodeAlertMetricAggregationSync(row.metricAggregation) : null, apdexThresholdMs: row.apdexThresholdMs, queryBuilderDraft: parseStoredQueryBuilderDraft(row.queryBuilderDraftJson), - rawQuerySql: row.signalType === "raw_query" ? (row.rawQuerySql ?? null) : null, + rawQuerySql: row.signalType === "raw_query" ? (row.rawQuerySql ?? null) : null, rawQueryReducer: row.signalType === "raw_query" ? decodeQueryEngineAlertReducerSync(row.reducer) : null, destinationIds: destinationIds.map((id) => decodeAlertDestinationIdSync(id)), @@ -1484,7 +1479,9 @@ export class AlertsService extends Context.Service - db - .select({ id: alertRules.id }) - .from(alertRules) - .where(and(eq(alertRules.orgId, orgId), eq(alertRules.enabled, true))), - ) - const alreadyActive = - existingId != null && activeRows.some((row) => row.id === existingId) - if (!alreadyActive && activeRows.length >= MAX_ACTIVE_ALERT_RULES_PER_ORG) { - return yield* Effect.fail( - makeValidationError( - `Organizations may have at most ${MAX_ACTIVE_ALERT_RULES_PER_ORG} active alert rules`, - ), - ) - } - } yield* requireDestinationIds(orgId, normalized.destinationIds) const ruleId = existingId ?? normalized.id const timestamp = yield* now @@ -2549,28 +2529,54 @@ export class AlertsService extends Context.Service - db - .insert(alertRules) - .values({ - id: ruleId, - orgId, - ...ruleFields, - createdAt: new Date(timestamp), - createdBy: userId, - }) - .returning(txidColumn), - ) - : yield* dbExecute((db) => - db - .update(alertRules) - .set(ruleFields) - .where(and(eq(alertRules.orgId, orgId), eq(alertRules.id, existingId))) - .returning(txidColumn), - ) - const txid = readTxid(writeRows) + const writeResult = yield* dbExecute((db) => + db.transaction(async (tx) => { + // Serialize quota checks per organization. Without the transaction-scoped + // advisory lock, concurrent creates can both observe 99 active rows and + // commit the 100th and 101st rules. + await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${orgId}))`) + if (normalized.enabled) { + const activeRows = await tx + .select({ id: alertRules.id }) + .from(alertRules) + .where(and(eq(alertRules.orgId, orgId), eq(alertRules.enabled, true))) + const alreadyActive = + existingId != null && activeRows.some((row) => row.id === existingId) + if (!alreadyActive && activeRows.length >= MAX_ACTIVE_ALERT_RULES_PER_ORG) { + return { limitExceeded: true as const, writeRows: [] } + } + } + + const writeRows = + existingId == null + ? await tx + .insert(alertRules) + .values({ + id: ruleId, + orgId, + ...ruleFields, + createdAt: new Date(timestamp), + createdBy: userId, + }) + .returning(txidColumn) + : await tx + .update(alertRules) + .set(ruleFields) + .where( + and(eq(alertRules.orgId, orgId), eq(alertRules.id, existingId)), + ) + .returning(txidColumn) + return { limitExceeded: false as const, writeRows } + }), + ) + if (writeResult.limitExceeded) { + return yield* Effect.fail( + makeValidationError( + `Organizations may have at most ${MAX_ACTIVE_ALERT_RULES_PER_ORG} active alert rules`, + ), + ) + } + const txid = readTxid(writeResult.writeRows) const row = yield* requireRuleRow(orgId, ruleId) const destinationIds = safeParseStringArray(row.destinationIdsJson) diff --git a/apps/api/src/services/TinybirdOrgTokenService.ts b/apps/api/src/services/TinybirdOrgTokenService.ts index 9b39da6c8..ca4a4e64e 100644 --- a/apps/api/src/services/TinybirdOrgTokenService.ts +++ b/apps/api/src/services/TinybirdOrgTokenService.ts @@ -51,10 +51,10 @@ export class TinybirdOrgTokenService extends Context.Service< const nowMs = yield* Clock.currentTimeMillis const cached = cache.get(orgId) if (cached !== undefined && cached.expiresAt > nowMs) { - yield* Effect.annotateCurrentSpan("tinybird.jwt.cacheHit", true) + yield* Effect.annotateCurrentSpan("maple.tinybird.jwt.cache_hit", true) return cached.token } - yield* Effect.annotateCurrentSpan("tinybird.jwt.cacheHit", false) + yield* Effect.annotateCurrentSpan("maple.tinybird.jwt.cache_hit", false) if (Option.isNone(env.TINYBIRD_SIGNING_KEY)) { return yield* new TinybirdOrgTokenError({ reason: "MissingSigningKey", diff --git a/apps/web/src/components/settings/api-key-create-payload.test.ts b/apps/web/src/components/settings/api-key-create-payload.test.ts index ef596dd45..213e08ea4 100644 --- a/apps/web/src/components/settings/api-key-create-payload.test.ts +++ b/apps/web/src/components/settings/api-key-create-payload.test.ts @@ -12,13 +12,17 @@ describe("buildApiKeyCreatePayload", () => { }) it("preserves the MCP kind and a non-empty description", () => { - const payload = buildApiKeyCreatePayload(" MCP key ", " Claude desktop ", "mcp") + const payload = buildApiKeyCreatePayload(" MCP key ", " Claude desktop ", "mcp", { + scopes: ["traces:read"], + }) expect(payload).toEqual({ name: "MCP key", description: "Claude desktop", kind: "mcp", }) + // MCP authentication requires an unscoped MCP-kind key. Ignore scopes at + // the payload boundary even if a caller accidentally supplies them. expect(() => Schema.encodeUnknownSync(V2ApiKeyCreateParams)(payload)).not.toThrow() }) diff --git a/apps/web/src/components/settings/api-key-create-payload.ts b/apps/web/src/components/settings/api-key-create-payload.ts index 29ac78d77..c86d6be75 100644 --- a/apps/web/src/components/settings/api-key-create-payload.ts +++ b/apps/web/src/components/settings/api-key-create-payload.ts @@ -18,10 +18,8 @@ export const buildApiKeyCreatePayload = ( name: name.trim(), ...(trimmedDescription.length > 0 ? { description: trimmedDescription } : {}), ...(kind !== undefined ? { kind } : {}), - ...(options.expiresInSeconds !== undefined - ? { expires_in_seconds: options.expiresInSeconds } - : {}), - ...(options.scopes !== undefined && options.scopes.length > 0 + ...(options.expiresInSeconds !== undefined ? { expires_in_seconds: options.expiresInSeconds } : {}), + ...(kind !== "mcp" && options.scopes !== undefined && options.scopes.length > 0 ? { scopes: options.scopes } : {}), } diff --git a/apps/web/src/components/settings/create-api-key-dialog.tsx b/apps/web/src/components/settings/create-api-key-dialog.tsx index 7d6983ec0..d4d730db3 100644 --- a/apps/web/src/components/settings/create-api-key-dialog.tsx +++ b/apps/web/src/components/settings/create-api-key-dialog.tsx @@ -81,6 +81,7 @@ const scopesFromLevels = (levels: Record): Array => }) export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: CreateApiKeyDialogProps) { + const isMcp = kind === "mcp" const accessLabelId = useId() const [newName, setNewName] = useState("") const [newDescription, setNewDescription] = useState("") @@ -95,8 +96,8 @@ export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: Crea mode: "promiseExit", }) - const restrictedScopes = accessMode === "restricted" ? scopesFromLevels(scopeLevels) : undefined - const missingScopes = accessMode === "restricted" && restrictedScopes?.length === 0 + const restrictedScopes = !isMcp && accessMode === "restricted" ? scopesFromLevels(scopeLevels) : undefined + const missingScopes = !isMcp && accessMode === "restricted" && restrictedScopes?.length === 0 const canCreate = newName.trim().length > 0 && !missingScopes && !isCreating async function handleCreate() { @@ -139,7 +140,7 @@ export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: Crea {createdKey ? ( <> - API key created + {isMcp ? "MCP key created" : "API key created"} Copy your API key now. You won't be able to see it again. @@ -168,9 +169,11 @@ export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: Crea ) : ( <> - Create API key + {isMcp ? "Create MCP key" : "Create API key"} - API keys are used to authenticate with the Maple API and MCP server. + {isMcp + ? "MCP keys authenticate clients with the Maple MCP server." + : "API keys authenticate clients with the Maple API."} @@ -222,70 +225,81 @@ export function CreateApiKeyDialog({ open, onOpenChange, onCreated, kind }: Crea
-
+ {!isMcp && ( +
{ - const next = values[0] - if (next === "full" || next === "restricted") setAccessMode(next) - }} - variant="outline" - size="sm" - > - Full access - Restricted - - {accessMode === "restricted" ? ( -
- {SCOPE_FAMILIES.map((family) => { - const familyLabelId = `${accessLabelId}-${family.id}` - return ( -
- - {family.label} - - { - const next = values[0] - if ( - next === "none" || - next === "read" || - next === "write" - ) { - setScopeLevels((current) => ({ - ...current, - [family.id]: next, - })) - } - }} - variant="outline" - size="sm" - > - None - Read - Write - -
- ) - })} + value={[accessMode]} + onValueChange={(values) => { + const next = values[0] + if (next === "full" || next === "restricted") setAccessMode(next) + }} + variant="outline" + size="sm" + > + Full access + Restricted + + {accessMode === "restricted" ? ( +
+ {SCOPE_FAMILIES.map((family) => { + const familyLabelId = `${accessLabelId}-${family.id}` + return ( +
+ + {family.label} + + { + const next = values[0] + if ( + next === "none" || + next === "read" || + next === "write" + ) { + setScopeLevels((current) => ({ + ...current, + [family.id]: next, + })) + } + }} + variant="outline" + size="sm" + > + + None + + + Read + + + Write + + +
+ ) + })} +

+ Write includes read. Scopes are fixed at creation — roll the + key to change access. +

+
+ ) : (

- Write includes read. Scopes are fixed at creation — roll the key - to change access. + Full access to the organization's API.

-
- ) : ( -

- Full access to the organization's API. -

- )} -
+ )} +
+ )}