diff --git a/apps/api/src/services/PlanetScaleConnectionService.ts b/apps/api/src/services/PlanetScaleConnectionService.ts index f8457080..cb7c3fe9 100644 --- a/apps/api/src/services/PlanetScaleConnectionService.ts +++ b/apps/api/src/services/PlanetScaleConnectionService.ts @@ -179,14 +179,14 @@ export class PlanetScaleConnectionService extends Context.Service< const decodeHttpSd = Schema.decodeUnknownEffect(Schema.fromJsonString(HttpSdResponse)) /** - * Whether the bearer can perform an ACTUAL branch-metrics scrape. The SD - * endpoint on api.planetscale.com accepting the bearer proves nothing - * about the data plane: the discovered hosts (metrics.psdb.cloud) only - * document service-token auth and reject OAuth bearers with 403. Probe one - * discovered endpoint with the bearer so `readMetricsEndpoints` reflects - * scraping, not listing. Inconclusive outcomes (no branches discovered - * yet, transport blip, undecodable payload) keep the control-plane answer - * instead of pausing a possibly-working target. + * Whether an ACTUAL branch-metrics scrape works. The SD endpoint on + * api.planetscale.com accepting the credential proves nothing about the + * data plane: metrics.psdb.cloud authenticates with the signed `?sig=&exp=` + * URL params minted in the SD response, so we probe a discovered endpoint + * via its `signedUrl` and let `readMetricsEndpoints` reflect scraping, not + * listing. Inconclusive outcomes (no branches discovered yet, transport + * blip, undecodable payload) keep the control-plane answer instead of + * pausing a possibly-working target. */ const probeDataPlaneScrape = Effect.fn("PlanetScaleConnectionService.probeDataPlaneScrape")( function* (organization: string, bearer: string) { @@ -205,7 +205,8 @@ export class PlanetScaleConnectionService extends Context.Service< const first = groups.flatMap((group) => subTargetsFromGroup(group).ok)[0] if (first === undefined) return "inconclusive" as const - const scrape = yield* probeUrl(first.url, bearer) + // signedUrl carries the `?sig=&exp=` params the data plane requires. + const scrape = yield* probeUrl(first.signedUrl, bearer) return scrape.status >= 200 && scrape.status < 300 ? ("ok" as const) : ("rejected" as const) diff --git a/apps/api/src/services/PlanetScaleDiscoveryService.test.ts b/apps/api/src/services/PlanetScaleDiscoveryService.test.ts index c0a41e15..f04d3c94 100644 --- a/apps/api/src/services/PlanetScaleDiscoveryService.test.ts +++ b/apps/api/src/services/PlanetScaleDiscoveryService.test.ts @@ -225,6 +225,59 @@ describe("PlanetScaleDiscoveryService", () => { }).pipe(Effect.provide(makeLayer(testDb))) }) + it.effect("promotes __param_* meta labels to signed scrape-url query params", () => { + const testDb = createTestDb(trackedDbs) + const recorded: Array = [] + return Effect.gen(function* () { + const discovery = yield* PlanetScaleDiscoveryService + const row = yield* createPlanetScaleTargetRow("my-org") + + // PlanetScale authenticates the metrics data plane with a signed, expiring + // URL: the http_sd group carries `__param_sig`/`__param_exp`, which + // Prometheus promotes to `?sig=&exp=` on the scrape URL. Dropping them + // yields `403 invalid signature` on every scrape (the real prod outage). + const SIGNED_PAYLOAD = [ + { + targets: ["metrics.psdb.cloud"], + labels: { + __metrics_path__: "/metrics/branch/3a1nf2gvu9rf", + __scheme__: "https", + __param_sig: "abc-_signature123", + __param_exp: "1784238737", + planetscale_branch_name: "main", + planetscale_database_name: "mydb", + }, + }, + ] + + const entries = yield* discovery.discover(row).pipe( + Effect.provideService( + FetchHttpClient.Fetch, + stubFetch(recorded, () => Response.json(SIGNED_PAYLOAD)), + ), + ) + + assert.strictEqual(entries.length, 1) + // Base url + fiber-identity key stay param-free so identity is stable as + // PlanetScale rotates the signature each refresh. + assert.strictEqual(entries[0]?.url, "https://metrics.psdb.cloud/metrics/branch/3a1nf2gvu9rf") + assert.strictEqual( + entries[0]?.subTargetKey, + "metrics.psdb.cloud/metrics/branch/3a1nf2gvu9rf", + ) + // signedUrl carries the auth params the data plane actually verifies. + assert.strictEqual( + entries[0]?.signedUrl, + "https://metrics.psdb.cloud/metrics/branch/3a1nf2gvu9rf?sig=abc-_signature123&exp=1784238737", + ) + // The signed params must never leak into the metric labels. + assert.deepStrictEqual(entries[0]?.labels, { + planetscale_branch_name: "main", + planetscale_database_name: "mydb", + }) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + it.effect("caches discovery for the TTL and refreshes after it elapses", () => { const testDb = createTestDb(trackedDbs) const recorded: Array = [] diff --git a/apps/api/src/services/PlanetScaleDiscoveryService.ts b/apps/api/src/services/PlanetScaleDiscoveryService.ts index 112a3c31..85fd26cf 100644 --- a/apps/api/src/services/PlanetScaleDiscoveryService.ts +++ b/apps/api/src/services/PlanetScaleDiscoveryService.ts @@ -20,10 +20,14 @@ type ScrapeTargetRow = typeof scrapeTargets.$inferSelect /** * Resolves PlanetScale `planetscale`-type scrape targets into their concrete * per-database-branch scrape endpoints via PlanetScale's Prometheus http_sd - * discovery API (`GET /v1/organizations/{org}/metrics`). Managed targets - * (`authType "planetscale_oauth"`) authenticate with the org's OAuth grant - * (`Authorization: Bearer …`); manual escape-hatch targets keep the service - * token scheme (`Authorization: token {ID}:{SECRET}`). + * discovery API (`GET /v1/organizations/{org}/metrics`). The Authorization + * header authenticates the DISCOVERY call only: managed targets + * (`authType "planetscale_oauth"`) use the org's OAuth grant + * (`Authorization: Bearer …`), manual escape-hatch targets the service-token + * scheme (`Authorization: token {ID}:{SECRET}`). The metrics DATA PLANE + * (`metrics.psdb.cloud`) does not use that header at all — it authenticates + * with a signed, expiring URL (`?sig=…&exp=…`) that the SD response mints per + * branch (see {@link subTargetsFromGroup} / {@link PlanetScaleSubTarget.signedUrl}). * * Discovery results are cached in-memory per target with a 10-minute TTL * (PlanetScale's documented refresh cadence). On refresh failure stale entries @@ -32,8 +36,20 @@ type ScrapeTargetRow = typeof scrapeTargets.$inferSelect */ export interface PlanetScaleSubTarget { - /** Concrete per-branch scrape URL (`https://{host}{__metrics_path__}`). */ + /** + * Per-branch scrape endpoint WITHOUT auth params (`https://{host}{__metrics_path__}`). + * Stable across discovery refreshes, so it's the scraper-facing target url + * (fiber identity + `instance` label). NOT fetched directly — see `signedUrl`. + */ readonly url: string + /** + * `url` plus PlanetScale's signed, expiring auth query params (`?sig=…&exp=…`), + * promoted from the http_sd `__param_*` meta labels. This is the URL that + * actually authenticates against the metrics data plane; fetching `url` + * without them returns `403 invalid signature`. Equals `url` when the SD group + * carried no `__param_*` labels. + */ + readonly signedUrl: string /** Stable discriminator: `planetscale_database_branch_id` SD label, falling back to `host:port` + metrics path. */ readonly subTargetKey: string /** SD labels minus `__`-prefixed Prometheus meta labels. */ @@ -81,10 +97,19 @@ export const subTargetsFromGroup = (group: { const sdLabels = group.labels ?? {} const scheme = sdLabels.__scheme__ ?? "https" const path = sdLabels.__metrics_path__ ?? "/metrics" + // PlanetScale authenticates the metrics data plane with a signed, expiring URL + // (`?sig=…&exp=…`) minted per branch in the http_sd response — NOT the discovery + // credential (the service token / OAuth bearer only auths the SD listing). + // Prometheus convention promotes `__param_` meta labels to `?=` + // query params on the scrape URL; forward them or every scrape returns + // `403 invalid signature`. const labels: Record = {} + const authParams = new URLSearchParams() for (const [key, value] of Object.entries(sdLabels)) { - if (!key.startsWith("__")) labels[key] = value + if (key.startsWith("__param_")) authParams.set(key.slice("__param_".length), value) + else if (!key.startsWith("__")) labels[key] = value } + const query = authParams.toString() const branchId = sdLabels.planetscale_database_branch_id const ok: Array = [] @@ -99,14 +124,16 @@ export const subTargetsFromGroup = (group: { } // The no-branch-id fallback keys on host + path (not bare host): groups // that differ only by `__metrics_path__` are distinct endpoints, and a - // host-only key would silently collapse them away in dedupe. + // host-only key would silently collapse them away in dedupe. The signed + // `?sig=&exp=` params are deliberately excluded from the key so per-branch + // fiber identity stays stable as PlanetScale rotates them each refresh. const subTargetKey = branchId && group.targets.length === 1 ? branchId : branchId ? `${branchId}:${hostPort}` : `${hostPort}${path}` - ok.push({ url, subTargetKey, labels }) + ok.push({ url, signedUrl: query ? `${url}?${query}` : url, subTargetKey, labels }) } return { ok, dropped } } diff --git a/apps/api/src/services/ScrapeTargetsService.ts b/apps/api/src/services/ScrapeTargetsService.ts index 1853d463..bb4bb360 100644 --- a/apps/api/src/services/ScrapeTargetsService.ts +++ b/apps/api/src/services/ScrapeTargetsService.ts @@ -844,11 +844,12 @@ export class ScrapeTargetsService extends Context.Service entry.subTargetKey === subTargetKey) if (!match) { @@ -859,7 +860,7 @@ export class ScrapeTargetsService extends Context.Service