fix(planetscale): forward signed data-plane auth params to branch scrapes

apps/api/src/services/PlanetScaleConnectionService.ts

@@ -179,14 +179,14 @@ export class PlanetScaleConnectionService extends Context.Service<
 		const decodeHttpSd = Schema.decodeUnknownEffect(Schema.fromJsonString(HttpSdResponse))
 
 		/**
-		 * Whether the bearer can perform an ACTUAL branch-metrics scrape. The SD
-		 * endpoint on api.planetscale.com accepting the bearer proves nothing
-		 * about the data plane: the discovered hosts (metrics.psdb.cloud) only
-		 * document service-token auth and reject OAuth bearers with 403. Probe one
-		 * discovered endpoint with the bearer so `readMetricsEndpoints` reflects
-		 * scraping, not listing. Inconclusive outcomes (no branches discovered
-		 * yet, transport blip, undecodable payload) keep the control-plane answer
-		 * instead of pausing a possibly-working target.
+		 * Whether an ACTUAL branch-metrics scrape works. The SD endpoint on
+		 * api.planetscale.com accepting the credential proves nothing about the
+		 * data plane: metrics.psdb.cloud authenticates with the signed `?sig=&exp=`
+		 * URL params minted in the SD response, so we probe a discovered endpoint
+		 * via its `signedUrl` and let `readMetricsEndpoints` reflect scraping, not
+		 * listing. Inconclusive outcomes (no branches discovered yet, transport
+		 * blip, undecodable payload) keep the control-plane answer instead of
+		 * pausing a possibly-working target.
 		 */
 		const probeDataPlaneScrape = Effect.fn("PlanetScaleConnectionService.probeDataPlaneScrape")(
 			function* (organization: string, bearer: string) {
@@ -205,7 +205,8 @@ export class PlanetScaleConnectionService extends Context.Service<
 					const first = groups.flatMap((group) => subTargetsFromGroup(group).ok)[0]
 					if (first === undefined) return "inconclusive" as const
 
-					const scrape = yield* probeUrl(first.url, bearer)
+					// signedUrl carries the `?sig=&exp=` params the data plane requires.
+					const scrape = yield* probeUrl(first.signedUrl, bearer)
 					return scrape.status >= 200 && scrape.status < 300
 						? ("ok" as const)
 						: ("rejected" as const)

apps/api/src/services/PlanetScaleDiscoveryService.test.ts

@@ -225,6 +225,59 @@ describe("PlanetScaleDiscoveryService", () => {
 		}).pipe(Effect.provide(makeLayer(testDb)))
 	})
 
+	it.effect("promotes __param_* meta labels to signed scrape-url query params", () => {
+		const testDb = createTestDb(trackedDbs)
+		const recorded: Array<RecordedRequest> = []
+		return Effect.gen(function* () {
+			const discovery = yield* PlanetScaleDiscoveryService
+			const row = yield* createPlanetScaleTargetRow("my-org")
+
+			// PlanetScale authenticates the metrics data plane with a signed, expiring
+			// URL: the http_sd group carries `__param_sig`/`__param_exp`, which
+			// Prometheus promotes to `?sig=&exp=` on the scrape URL. Dropping them
+			// yields `403 invalid signature` on every scrape (the real prod outage).
+			const SIGNED_PAYLOAD = [
+				{
+					targets: ["metrics.psdb.cloud"],
+					labels: {
+						__metrics_path__: "/metrics/branch/3a1nf2gvu9rf",
+						__scheme__: "https",
+						__param_sig: "abc-_signature123",
+						__param_exp: "1784238737",
+						planetscale_branch_name: "main",
+						planetscale_database_name: "mydb",
+					},
+				},
+			]
+
+			const entries = yield* discovery.discover(row).pipe(
+				Effect.provideService(
+					FetchHttpClient.Fetch,
+					stubFetch(recorded, () => Response.json(SIGNED_PAYLOAD)),
+				),
+			)
+
+			assert.strictEqual(entries.length, 1)
+			// Base url + fiber-identity key stay param-free so identity is stable as
+			// PlanetScale rotates the signature each refresh.
+			assert.strictEqual(entries[0]?.url, "https://metrics.psdb.cloud/metrics/branch/3a1nf2gvu9rf")
+			assert.strictEqual(
+				entries[0]?.subTargetKey,
+				"metrics.psdb.cloud/metrics/branch/3a1nf2gvu9rf",
+			)
+			// signedUrl carries the auth params the data plane actually verifies.
+			assert.strictEqual(
+				entries[0]?.signedUrl,
+				"https://metrics.psdb.cloud/metrics/branch/3a1nf2gvu9rf?sig=abc-_signature123&exp=1784238737",
+			)
+			// The signed params must never leak into the metric labels.
+			assert.deepStrictEqual(entries[0]?.labels, {
+				planetscale_branch_name: "main",
+				planetscale_database_name: "mydb",
+			})
+		}).pipe(Effect.provide(makeLayer(testDb)))
+	})
+
 	it.effect("caches discovery for the TTL and refreshes after it elapses", () => {
 		const testDb = createTestDb(trackedDbs)
 		const recorded: Array<RecordedRequest> = []

apps/api/src/services/PlanetScaleDiscoveryService.ts

@@ -20,10 +20,14 @@ type ScrapeTargetRow = typeof scrapeTargets.$inferSelect
 /**
  * Resolves PlanetScale `planetscale`-type scrape targets into their concrete
  * per-database-branch scrape endpoints via PlanetScale's Prometheus http_sd
- * discovery API (`GET /v1/organizations/{org}/metrics`). Managed targets
- * (`authType "planetscale_oauth"`) authenticate with the org's OAuth grant
- * (`Authorization: Bearer …`); manual escape-hatch targets keep the service
- * token scheme (`Authorization: token {ID}:{SECRET}`).
+ * discovery API (`GET /v1/organizations/{org}/metrics`). The Authorization
+ * header authenticates the DISCOVERY call only: managed targets
+ * (`authType "planetscale_oauth"`) use the org's OAuth grant
+ * (`Authorization: Bearer …`), manual escape-hatch targets the service-token
+ * scheme (`Authorization: token {ID}:{SECRET}`). The metrics DATA PLANE
+ * (`metrics.psdb.cloud`) does not use that header at all — it authenticates
+ * with a signed, expiring URL (`?sig=…&exp=…`) that the SD response mints per
+ * branch (see {@link subTargetsFromGroup} / {@link PlanetScaleSubTarget.signedUrl}).
  *
  * Discovery results are cached in-memory per target with a 10-minute TTL
  * (PlanetScale's documented refresh cadence). On refresh failure stale entries
@@ -32,8 +36,20 @@ type ScrapeTargetRow = typeof scrapeTargets.$inferSelect
  */
 
 export interface PlanetScaleSubTarget {
-	/** Concrete per-branch scrape URL (`https://{host}{__metrics_path__}`). */
+	/**
+	 * Per-branch scrape endpoint WITHOUT auth params (`https://{host}{__metrics_path__}`).
+	 * Stable across discovery refreshes, so it's the scraper-facing target url
+	 * (fiber identity + `instance` label). NOT fetched directly — see `signedUrl`.
+	 */
 	readonly url: string
+	/**
+	 * `url` plus PlanetScale's signed, expiring auth query params (`?sig=…&exp=…`),
+	 * promoted from the http_sd `__param_*` meta labels. This is the URL that
+	 * actually authenticates against the metrics data plane; fetching `url`
+	 * without them returns `403 invalid signature`. Equals `url` when the SD group
+	 * carried no `__param_*` labels.
+	 */
+	readonly signedUrl: string
 	/** Stable discriminator: `planetscale_database_branch_id` SD label, falling back to `host:port` + metrics path. */
 	readonly subTargetKey: string
 	/** SD labels minus `__`-prefixed Prometheus meta labels. */
@@ -81,10 +97,19 @@ export const subTargetsFromGroup = (group: {
 	const sdLabels = group.labels ?? {}
 	const scheme = sdLabels.__scheme__ ?? "https"
 	const path = sdLabels.__metrics_path__ ?? "/metrics"
+	// PlanetScale authenticates the metrics data plane with a signed, expiring URL
+	// (`?sig=…&exp=…`) minted per branch in the http_sd response — NOT the discovery
+	// credential (the service token / OAuth bearer only auths the SD listing).
+	// Prometheus convention promotes `__param_<name>` meta labels to `?<name>=`
+	// query params on the scrape URL; forward them or every scrape returns
+	// `403 invalid signature`.
 	const labels: Record<string, string> = {}
+	const authParams = new URLSearchParams()
 	for (const [key, value] of Object.entries(sdLabels)) {
-		if (!key.startsWith("__")) labels[key] = value
+		if (key.startsWith("__param_")) authParams.set(key.slice("__param_".length), value)
+		else if (!key.startsWith("__")) labels[key] = value
 	}
+	const query = authParams.toString()
 	const branchId = sdLabels.planetscale_database_branch_id
 
 	const ok: Array<PlanetScaleSubTarget> = []
@@ -99,14 +124,16 @@ export const subTargetsFromGroup = (group: {
 		}
 		// The no-branch-id fallback keys on host + path (not bare host): groups
 		// that differ only by `__metrics_path__` are distinct endpoints, and a
-		// host-only key would silently collapse them away in dedupe.
+		// host-only key would silently collapse them away in dedupe. The signed
+		// `?sig=&exp=` params are deliberately excluded from the key so per-branch
+		// fiber identity stays stable as PlanetScale rotates them each refresh.
 		const subTargetKey =
 			branchId && group.targets.length === 1
 				? branchId
 				: branchId
 					? `${branchId}:${hostPort}`
 					: `${hostPort}${path}`
-		ok.push({ url, subTargetKey, labels })
+		ok.push({ url, signedUrl: query ? `${url}?${query}` : url, subTargetKey, labels })
 	}
 	return { ok, dropped }
 }

apps/api/src/services/ScrapeTargetsService.ts

@@ -844,11 +844,12 @@ export class ScrapeTargetsService extends Context.Service<ScrapeTargetsService,
 
 				let scrapeUrl = row.value.url
 				if (row.value.targetType === "planetscale") {
-					// Resolve the per-branch endpoint from the discovery cache. The
-					// scrape carries the same auth header as the SD call — required
-					// there too: metrics.psdb.cloud accepts service tokens but rejects
-					// OAuth bearers with 403, which is why finalize only enables
-					// bearer-auth targets that passed a data-plane scrape probe.
+					// Resolve the per-branch endpoint from the discovery cache and use
+					// its SIGNED url: PlanetScale authenticates the metrics data plane
+					// with the short-lived `?sig=&exp=` params minted in the SD response,
+					// not the Authorization header (that only auths the discovery
+					// listing). The header built below is still sent but the data plane
+					// ignores it.
 					const subTargets = yield* discovery.discover(row.value)
 					const match = subTargets.find((entry) => entry.subTargetKey === subTargetKey)
 					if (!match) {
@@ -859,7 +860,7 @@ export class ScrapeTargetsService extends Context.Service<ScrapeTargetsService,
 							}),
 						)
 					}
-					scrapeUrl = match.url
+					scrapeUrl = match.signedUrl
 				}
 
 				const headers = yield* authHeadersForRow(row.value)