diff --git a/.gitignore b/.gitignore index f2471a36d3..74875645f9 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,9 @@ postgres/pgsql-test/output/ .env.local graphql/server/logs/ graphql/server/*.heapsnapshot +scripts/scale-validate/out/ +scripts/scale-validate/fleet*.json +scripts/scale-validate/drifted.json +scripts/scale-spike/ +packages/perf-harness/perf-out/ +/metrics.jsonl diff --git a/graphile/graphile-cache/src/__tests__/counters.test.ts b/graphile/graphile-cache/src/__tests__/counters.test.ts new file mode 100644 index 0000000000..5b3ec01be3 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/counters.test.ts @@ -0,0 +1,120 @@ +import { + cacheCounters, + clearMatchingEntries, + getCacheConfig, + getCacheCounters, + graphileCache, + type GraphileCacheEntry +} from '../graphile-cache'; + +/** + * Counter instrumentation regression tests. The counters are process-global and + * mutable, so every assertion is a DELTA against a snapshot captured immediately + * before the action — robust to test ordering and to fire-and-forget disposals from + * other suites. The eviction and disposal increments happen synchronously inside + * graphileCache.delete() (the dispose callback and the pre-await head of disposeEntry), + * so no tick is needed except for the async drain-timeout path. + */ + +const tick = (ms = 60) => new Promise((resolve) => setTimeout(resolve, ms)); + +const makeEntry = (releaseDelayMs = 0): GraphileCacheEntry => { + const release = jest.fn( + () => new Promise((resolve) => setTimeout(resolve, releaseDelayMs)) + ); + return { + pgl: { release } as unknown as GraphileCacheEntry['pgl'], + serv: {} as GraphileCacheEntry['serv'], + handler: {} as GraphileCacheEntry['handler'], + // Never .listen()ed in production; close() invokes its callback synchronously here. + httpServer: { close: (cb: () => void) => cb() } as unknown as GraphileCacheEntry['httpServer'], + cacheKey: 'mock', + createdAt: Date.now() + }; +}; + +describe('cacheCounters increment on eviction/disposal', () => { + afterEach(async () => { + graphileCache.clear(); + await tick(); + }); + + it('counts an LRU eviction (fresh entry, plain delete) and a disposal', () => { + const before = getCacheCounters(); + const key = 'counter-lru-1'; + graphileCache.set(key, makeEntry(0)); + graphileCache.delete(key); // fresh + not manual → reason 'lru'; disposeEntry runs + + const after = getCacheCounters(); + expect(after.evictions.lru).toBe(before.evictions.lru + 1); + expect(after.disposals).toBe(before.disposals + 1); + }); + + it('counts a TTL eviction when the entry is older than the configured ttl', () => { + const before = getCacheCounters(); + const key = 'counter-ttl-1'; + const entry = makeEntry(0); + // Backdate our own createdAt beyond the idle-ttl so getEvictionReason returns 'ttl'. + entry.createdAt = Date.now() - getCacheConfig().ttl - 1_000; + graphileCache.set(key, entry); + graphileCache.delete(key); + + const after = getCacheCounters(); + expect(after.evictions.ttl).toBe(before.evictions.ttl + 1); + }); + + it('counts a manual eviction via clearMatchingEntries', () => { + const before = getCacheCounters(); + graphileCache.set('counter-manual-1', makeEntry(0)); + const cleared = clearMatchingEntries(/^counter-manual-/); + + expect(cleared).toBe(1); + const after = getCacheCounters(); + expect(after.evictions.manual).toBe(before.evictions.manual + 1); + }); +}); + +describe('cacheCounters.drainTimeouts', () => { + const prev = process.env.GRAPHILE_CACHE_DRAIN_TIMEOUT_MS; + afterEach(async () => { + if (prev === undefined) delete process.env.GRAPHILE_CACHE_DRAIN_TIMEOUT_MS; + else process.env.GRAPHILE_CACHE_DRAIN_TIMEOUT_MS = prev; + graphileCache.clear(); + await tick(); + }); + + it('increments when disposal drains past the timeout with a request still in flight', async () => { + process.env.GRAPHILE_CACHE_DRAIN_TIMEOUT_MS = '1'; // one 100ms poll cycle, then give up + const before = getCacheCounters(); + const key = 'counter-drain-1'; + const entry = makeEntry(0); + entry.inflight = 1; // never drops → drain loop hits the timeout branch + + graphileCache.set(key, entry); + graphileCache.delete(key); // fire-and-forget disposeEntry begins draining + + await tick(250); // allow the poll cycle + timeout to elapse + + const after = getCacheCounters(); + expect(after.drainTimeouts).toBe(before.drainTimeouts + 1); + // Disposal still proceeds after the drain timeout. + expect((entry.pgl as unknown as { release: jest.Mock }).release).toHaveBeenCalledTimes(1); + }); +}); + +describe('getCacheCounters snapshot isolation', () => { + it('returns a deep copy that cannot mutate the live counters', () => { + const before = getCacheCounters(); + const snapshot = getCacheCounters(); + snapshot.disposals += 1_000; + snapshot.evictions.lru += 1_000; + snapshot.drainTimeouts += 1_000; + + const after = getCacheCounters(); + expect(after.disposals).toBe(before.disposals); + expect(after.evictions.lru).toBe(before.evictions.lru); + expect(after.drainTimeouts).toBe(before.drainTimeouts); + // Sanity: the live object is the same instance across reads. + expect(cacheCounters.disposals).toBe(after.disposals); + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/disposal-and-cap.test.ts b/graphile/graphile-cache/src/__tests__/disposal-and-cap.test.ts new file mode 100644 index 0000000000..f272704899 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/disposal-and-cap.test.ts @@ -0,0 +1,201 @@ +import { EventEmitter } from 'events'; +import { + ensureCacheHeadroom, + getCacheConfig, + graphileCache, + type GraphileCacheEntry, + invokeEntryHandler, +} from '../graphile-cache'; + +/** + * Regression tests for the schema-builder OOM fix. + * + * 1. Disposal guard is entry-scoped, not key-scoped. The previous string-keyed guard + * skipped pgl.release() for a rebuilt entry that shared a key with an entry still + * mid-release (the "same-key disposal race"). These tests pin the corrected behaviour: + * every distinct entry is released exactly once, while the SAME entry is never + * double-disposed. + * 2. getCacheConfig honours GRAPHILE_CACHE_MAX and otherwise returns a heap-aware default + * bounded to a sane range (a count-only default of 50 × ~0.5GB/instance was the OOM). + */ + +const tick = (ms = 60) => new Promise((resolve) => setTimeout(resolve, ms)); + +const makeEntry = (releaseDelayMs = 0): GraphileCacheEntry => { + const release = jest.fn( + () => new Promise((resolve) => setTimeout(resolve, releaseDelayMs)), + ); + return { + pgl: { release } as unknown as GraphileCacheEntry['pgl'], + serv: {} as GraphileCacheEntry['serv'], + handler: {} as GraphileCacheEntry['handler'], + // Never .listen()ed in production; close() invokes its callback synchronously here. + httpServer: { close: (cb: () => void) => cb() } as unknown as GraphileCacheEntry['httpServer'], + cacheKey: 'mock', + createdAt: Date.now(), + }; +}; + +describe('graphile-cache disposal guard (same-key race)', () => { + it('disposes a rebuilt entry on the same key while a prior entry is mid-release', async () => { + const key = 'race-key-1'; + const a = makeEntry(40); // a.release() stays pending while b is disposed + const b = makeEntry(0); + + graphileCache.set(key, a); + graphileCache.delete(key); // -> disposeEntry(a) begins, a.release() in flight + graphileCache.set(key, b); // key was free; inserts b + graphileCache.delete(key); // -> disposeEntry(b) MUST NOT be skipped by the guard + + await tick(120); + + // The bug: the old string-keyed guard left `key` parked while a.release() was pending, + // so b.release() was never called (would be 0 here). + expect((a.pgl as unknown as { release: jest.Mock }).release).toHaveBeenCalledTimes(1); + expect((b.pgl as unknown as { release: jest.Mock }).release).toHaveBeenCalledTimes(1); + }); + + it('never disposes the same entry twice', async () => { + const key = 'race-key-2'; + const a = makeEntry(0); + + graphileCache.set(key, a); + graphileCache.delete(key); // dispose a + graphileCache.set(key, a); // re-insert the SAME entry object + graphileCache.delete(key); // dispose again -> guard must skip (already disposed) + + await tick(); + + expect((a.pgl as unknown as { release: jest.Mock }).release).toHaveBeenCalledTimes(1); + }); +}); + +describe('graphile-cache heap-aware capacity', () => { + const prev = process.env.GRAPHILE_CACHE_MAX; + afterEach(() => { + if (prev === undefined) delete process.env.GRAPHILE_CACHE_MAX; + else process.env.GRAPHILE_CACHE_MAX = prev; + }); + + it('honours an explicit GRAPHILE_CACHE_MAX', () => { + process.env.GRAPHILE_CACHE_MAX = '7'; + expect(getCacheConfig().max).toBe(7); + }); + + it('falls back to a bounded heap-aware default when unset', () => { + delete process.env.GRAPHILE_CACHE_MAX; + const { max } = getCacheConfig(); + // Bounded within [1, 50]: at least one instance must be admitted, but the + // floor must never exceed what the heap budget says fits (a floor of 3 + // over-admitted 1.3GB instances on a 2GB heap and aborted the process). + expect(max).toBeGreaterThanOrEqual(1); + expect(max).toBeLessThanOrEqual(50); + }); + + it('ignores a non-numeric GRAPHILE_CACHE_MAX (falls back to heap-aware default)', () => { + process.env.GRAPHILE_CACHE_MAX = 'not-a-number'; + const { max } = getCacheConfig(); + expect(max).toBeGreaterThanOrEqual(1); + expect(max).toBeLessThanOrEqual(50); + }); + + it('admits only one instance when the per-instance estimate exceeds half the heap', () => { + delete process.env.GRAPHILE_CACHE_MAX; + const prevEstimate = process.env.GRAPHILE_CACHE_INSTANCE_HEAP_BYTES; + // Larger than any plausible test-runner heap budget => budgeted count is 0 + // and the floor of 1 must win (never the old floor of 3). + process.env.GRAPHILE_CACHE_INSTANCE_HEAP_BYTES = String(64 * 1024 * 1024 * 1024); + try { + expect(getCacheConfig().max).toBe(1); + } finally { + if (prevEstimate === undefined) delete process.env.GRAPHILE_CACHE_INSTANCE_HEAP_BYTES; + else process.env.GRAPHILE_CACHE_INSTANCE_HEAP_BYTES = prevEstimate; + } + }); +}); + +// A minimal express-like response: an EventEmitter so invokeEntryHandler can hook +// 'finish'/'close', cast to the express type it expects. +const makeRes = () => new EventEmitter() as unknown as Parameters[2]; +const fakeReq = {} as Parameters[1]; +const fakeNext = (() => undefined) as Parameters[3]; + +describe('invokeEntryHandler in-flight refcounting', () => { + it('increments while serving and releases exactly once across finish+close', () => { + const entry = makeEntry(); + const handler = jest.fn(); + entry.handler = handler as unknown as GraphileCacheEntry['handler']; + + const res = makeRes(); + expect(invokeEntryHandler(entry, fakeReq, res, fakeNext)).toBe(true); + expect(handler).toHaveBeenCalledTimes(1); + expect(entry.inflight).toBe(1); + + (res as unknown as EventEmitter).emit('finish'); + expect(entry.inflight).toBe(0); + // 'close' always follows 'finish' in Node; the release must be idempotent. + (res as unknown as EventEmitter).emit('close'); + expect(entry.inflight).toBe(0); + }); + + it('refuses a disposing entry without invoking the handler', () => { + const entry = makeEntry(); + const handler = jest.fn(); + entry.handler = handler as unknown as GraphileCacheEntry['handler']; + entry.disposing = true; + + expect(invokeEntryHandler(entry, fakeReq, makeRes(), fakeNext)).toBe(false); + expect(handler).not.toHaveBeenCalled(); + }); +}); + +describe('disposeEntry drains in-flight requests before release', () => { + it('waits for inflight to reach 0, then releases exactly once', async () => { + const key = 'drain-key-1'; + const entry = makeEntry(0); + entry.inflight = 1; // simulate a request mid-flight + + graphileCache.set(key, entry); + graphileCache.delete(key); // disposal starts; must poll instead of releasing + + await tick(120); // one poll cycle elapsed, request still in flight + expect((entry.pgl as unknown as { release: jest.Mock }).release).not.toHaveBeenCalled(); + + entry.inflight = 0; // request completes + await tick(250); // allow the next poll cycle to observe it + + expect((entry.pgl as unknown as { release: jest.Mock }).release).toHaveBeenCalledTimes(1); + }); +}); + +describe('ensureCacheHeadroom', () => { + const prevMax = process.env.GRAPHILE_CACHE_MAX; + afterEach(() => { + if (prevMax === undefined) delete process.env.GRAPHILE_CACHE_MAX; + else process.env.GRAPHILE_CACHE_MAX = prevMax; + graphileCache.clear(); + }); + + it('evicts least-recently-used entries until a build slot is free', async () => { + process.env.GRAPHILE_CACHE_MAX = '2'; + const a = makeEntry(0); + const b = makeEntry(0); + graphileCache.set('headroom-a', a); + graphileCache.set('headroom-b', b); + graphileCache.get('headroom-a'); // touch a → b becomes LRU-oldest + + const evicted = ensureCacheHeadroom(1); + + expect(evicted).toBe(1); + expect(graphileCache.has('headroom-a')).toBe(true); + expect(graphileCache.has('headroom-b')).toBe(false); + await tick(); // let the fire-and-forget disposal settle before clear() + }); + + it('no-ops when under capacity', () => { + process.env.GRAPHILE_CACHE_MAX = '5'; + graphileCache.set('headroom-c', makeEntry(0)); + expect(ensureCacheHeadroom(1)).toBe(0); + expect(graphileCache.has('headroom-c')).toBe(true); + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/governor.test.ts b/graphile/graphile-cache/src/__tests__/governor.test.ts new file mode 100644 index 0000000000..19ae8df6e6 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/governor.test.ts @@ -0,0 +1,83 @@ +import { + computeCapacityFromBudget, + getCacheCounters, + getMemoryPressure, + shouldRefuseBuild +} from '../graphile-cache'; + +const MB = 1024 * 1024; + +describe('computeCapacityFromBudget (two-constraint capacity model)', () => { + it('matches the live capacity-2 proof: 3584MB heap with 1.45GB instances', () => { + // r2-div-k1: two ~1.35GB instances resident (heapMax 2750MB), rebuild safe + // because evict-before-build frees a slot ahead of the transient peak. + expect(computeCapacityFromBudget(3584 * MB, 1450 * MB, 256 * MB, 768 * MB)).toBe(2); + }); + + it('gives capacity 1 on a 2GB heap with measured 1.45GB instances', () => { + expect(computeCapacityFromBudget(2048 * MB, 1450 * MB, 256 * MB, 768 * MB)).toBe(1); + }); + + it('never returns less than 1 even when nothing fits', () => { + expect(computeCapacityFromBudget(896 * MB, 1450 * MB, 256 * MB, 768 * MB)).toBe(1); + }); + + it('caps at 50 for tiny instances on big heaps', () => { + expect(computeCapacityFromBudget(64 * 1024 * MB, 4 * MB, 256 * MB, 768 * MB)).toBe(50); + }); + + it('applies the rebuild constraint when it is the binding one', () => { + // Residency alone would allow 3 (3*1000+256 <= 3500), but a rebuild with + // 2 residents (2*1000 + 256 + 900 = 3156 <= 3500) is the binding bound: + // byResidency = floor((3500-256)/1000) = 3 + // byRebuild = floor((3500-256-900)/1000) + 1 = 2 + 1 = 3 + expect(computeCapacityFromBudget(3500 * MB, 1000 * MB, 256 * MB, 900 * MB)).toBe(3); + // Shrink the heap so the rebuild constraint bites: + // byResidency = floor((3300-256)/1000) = 3 + // byRebuild = floor((3300-256-900)/1000) + 1 = 2 + 1 = 3 -> still 3; + // push harder: + // byResidency(3256) = 3, byRebuild(3256) = floor(2100/1000)+1 = 3 + // byResidency(3200) = 2 — residency binds again. Use a bigger reserve: + expect(computeCapacityFromBudget(3300 * MB, 1000 * MB, 256 * MB, 1500 * MB)).toBe(2); + }); +}); + +describe('memory governor pressure gate', () => { + const cleanup: Array<() => void> = []; + afterEach(() => { + while (cleanup.length) cleanup.pop()!(); + }); + + const setEnv = (key: string, value: string) => { + const prev = process.env[key]; + process.env[key] = value; + cleanup.push(() => { + if (prev === undefined) delete process.env[key]; + else process.env[key] = prev; + }); + }; + + it('reports ok at normal test-process heap levels', () => { + expect(getMemoryPressure().level).toBe('ok'); + }); + + it('refuses builds and counts them at critical pressure', () => { + // Force the critical watermark below any real process usage. + setEnv('GRAPHILE_MEMORY_GOVERNOR_CRITICAL', '0.0001'); + setEnv('GRAPHILE_MEMORY_GOVERNOR_ELEVATED', '0.00005'); + const before = getCacheCounters().buildRefusals; + const verdict = shouldRefuseBuild(); + expect(verdict.level).toBe('critical'); + expect(verdict.refuseBuild).toBe(true); + expect(getCacheCounters().buildRefusals).toBe(before + 1); + }); + + it('does not refuse builds at elevated pressure', () => { + setEnv('GRAPHILE_MEMORY_GOVERNOR_ELEVATED', '0.0001'); + const before = getCacheCounters().buildRefusals; + const verdict = shouldRefuseBuild(); + expect(verdict.level).toBe('elevated'); + expect(verdict.refuseBuild).toBe(false); + expect(getCacheCounters().buildRefusals).toBe(before); + }); +}); diff --git a/graphile/graphile-cache/src/create-instance.ts b/graphile/graphile-cache/src/create-instance.ts index fc4c625ae2..5787020a5a 100644 --- a/graphile/graphile-cache/src/create-instance.ts +++ b/graphile/graphile-cache/src/create-instance.ts @@ -10,6 +10,11 @@ const log = new Logger('graphile-cache:create'); interface GraphileInstanceOptions { preset: any; cacheKey: string; + /** + * Database name backing this instance's pg pool. Stored on the entry so the + * graphile-cache pgCache cleanup callback can evict it when its pool is disposed. + */ + dbname?: string; /** * When true, a RealtimeManager is created and started alongside the * PostGraphile instance. The pool is extracted from the preset's @@ -37,7 +42,7 @@ interface GraphileInstanceOptions { export const createGraphileInstance = async ( opts: GraphileInstanceOptions ): Promise => { - const { preset, cacheKey, enableRealtime = false } = opts; + const { preset, cacheKey, dbname, enableRealtime = false } = opts; const pgl = postgraphile(preset); const serv = pgl.createServ(grafserv); @@ -53,7 +58,9 @@ export const createGraphileInstance = async ( handler, httpServer, cacheKey, + dbname, createdAt: Date.now(), + inflight: 0, }; if (enableRealtime) { diff --git a/graphile/graphile-cache/src/graphile-cache.ts b/graphile/graphile-cache/src/graphile-cache.ts index 26bbb24c0a..ef257a6c62 100644 --- a/graphile/graphile-cache/src/graphile-cache.ts +++ b/graphile/graphile-cache/src/graphile-cache.ts @@ -1,8 +1,14 @@ import { EventEmitter } from 'events'; +import { getHeapStatistics } from 'node:v8'; import { Logger } from '@pgpmjs/logger'; import { LRUCache } from 'lru-cache'; import { pgCache } from 'pg-cache'; -import type { Express } from 'express'; +import type { + Express, + NextFunction as ExpressNextFunction, + Request as ExpressRequest, + Response as ExpressResponse, +} from 'express'; import type { Server as HttpServer } from 'http'; import type { PostGraphileInstance } from 'postgraphile'; import type { GrafservBase } from 'grafserv'; @@ -12,11 +18,45 @@ const log = new Logger('graphile-cache'); // --- Time Constants --- export const ONE_HOUR_MS = 1000 * 60 * 60; export const FIVE_MINUTES_MS = 1000 * 60 * 5; -const ONE_DAY = ONE_HOUR_MS * 24; -const ONE_YEAR = ONE_DAY * 366; +const SIX_HOURS_MS = ONE_HOUR_MS * 6; // --- Eviction Types --- -export type EvictionReason = 'lru' | 'ttl' | 'manual'; +export type EvictionReason = 'lru' | 'ttl' | 'manual' | 'governor'; + +// --- Cache Counters (in-process metrics) --- +export interface CacheCounters { + evictions: { lru: number; ttl: number; manual: number; governor: number }; + disposals: number; + drainTimeouts: number; + buildRefusals: number; +} + +/** + * Cumulative in-process cache counters for the metrics sampler. Mutated in the LRU + * dispose callback (by eviction reason), in disposeEntry (each distinct disposal), and + * in the drain-timeout warning branch. Zero overhead when the sampler is disabled — the + * increments are a handful of integer bumps on paths that already do real teardown work. + * Read a stable snapshot via getCacheCounters(). + */ +export const cacheCounters: CacheCounters = { + evictions: { lru: 0, ttl: 0, manual: 0, governor: 0 }, + disposals: 0, + drainTimeouts: 0, + buildRefusals: 0 +}; + +/** + * Snapshot the cache counters. Returns a deep copy so callers cannot mutate the live + * counters through the returned object. + */ +export function getCacheCounters(): CacheCounters { + return { + evictions: { ...cacheCounters.evictions }, + disposals: cacheCounters.disposals, + drainTimeouts: cacheCounters.drainTimeouts, + buildRefusals: cacheCounters.buildRefusals + }; +} // --- Cache Event Emitter --- export interface CacheEvictionEvent { @@ -43,30 +83,128 @@ export interface CacheConfig { ttl: number; } +/** + * Empirically, a PostGraphile v5 instance that has served at least one GraphQL + * request retains ~0.5 GB of V8 heap (the fully-materialised GraphQL schema plus + * grafast's per-schema plan machinery; a build-only instance is far smaller, but + * every *cached* instance is, by definition, one that serves requests). The cache + * therefore CANNOT be bounded by entry count alone: `max` heavy instances need + * `max * ~0.5GB` of resident heap, and once that exceeds the V8 old-space limit the + * process OOMs as distinct hosts fill the cache. A fixed default of 50 implies + * ~24 GB of resident schemas — far beyond any normal heap — which is the root cause + * of the schema-builder OOM. We derive a heap-aware default that budgets a fraction + * of the heap for cached instances. Override explicitly with GRAPHILE_CACHE_MAX, and + * tune the per-instance estimate with GRAPHILE_CACHE_INSTANCE_HEAP_BYTES. + */ +// ~0.5 GB retained per query-serving instance (measured against real provisioned apps +// on a small catalog). NOTE: instance heap scales with the TOTAL pg catalog +// (~21 KB per pg_class row measured) — on large multi-tenant catalogs set +// GRAPHILE_CACHE_INSTANCE_HEAP_BYTES to a measured value (e.g. ~1.45 GB at 61k rows). +const DEFAULT_INSTANCE_HEAP_BYTES = 512 * 1024 * 1024; +// Heap reserved for the server itself: express, pools, request working set +// (~+250 MB was measured at 200 rps on top of the resident instance). +const DEFAULT_BASE_RESERVE_BYTES = 256 * 1024 * 1024; +// Heap reserved for ONE in-flight schema build's transient allocations +// (introspection payload + gather + schema construction; >700 MB measured at a +// 61k-pg_class catalog). Builds are serialized by the server's BuildSemaphore, +// and ensureCacheHeadroom evicts down to (max - 1) residents before each build, +// so exactly one transient of this size coexists with (max - 1) residents. +const DEFAULT_BUILD_RESERVE_BYTES = 768 * 1024 * 1024; +// At least one instance must be admitted or nothing can ever build — but never +// more than the heap budget says fit. A floor of 3 admitted two extra builds +// the heap could not hold and aborted the process (V8 heap-limit SIGABRT) +// before eviction ever ran. +const MIN_CACHE_MAX = 1; +const MAX_CACHE_MAX = 50; + +const parseEnvInt = (value: string | undefined, fallback: number): number => { + if (!value) return fallback; + const n = parseInt(value, 10); + return Number.isFinite(n) && n > 0 ? n : fallback; +}; + +/** + * Pure capacity math (exported for tests and diagnostics). + * + * Two constraints bound how many instances fit: + * residency: max * perInstance + baseReserve <= heapLimit + * rebuild: (max - 1) * perInstance + baseReserve + * + buildReserve <= heapLimit + * (the rebuild constraint has (max - 1) residents because evict-before-build + * frees one slot before the transient allocation peaks — validated live: a + * 3584 MB heap holds TWO ~1.35 GB instances and still rebuilds safely). + */ +export function computeCapacityFromBudget( + heapLimit: number, + perInstance: number, + baseReserve: number = DEFAULT_BASE_RESERVE_BYTES, + buildReserve: number = DEFAULT_BUILD_RESERVE_BYTES, +): number { + const byResidency = Math.floor((heapLimit - baseReserve) / perInstance); + const byRebuild = Math.floor((heapLimit - baseReserve - buildReserve) / perInstance) + 1; + const budgeted = Math.min(byResidency, byRebuild); + return Math.min(MAX_CACHE_MAX, Math.max(MIN_CACHE_MAX, budgeted)); +} + +/** + * Heap-aware default for the maximum number of cached PostGraphile instances. + * See computeCapacityFromBudget for the model. Tunables: + * GRAPHILE_CACHE_INSTANCE_HEAP_BYTES — measured per-instance retained heap + * GRAPHILE_CACHE_BASE_RESERVE_BYTES — server base + request working set + * GRAPHILE_CACHE_BUILD_RESERVE_BYTES — one build's transient peak + */ +function computeHeapAwareMax(): number { + try { + const heapLimit = getHeapStatistics().heap_size_limit; // reflects --max-old-space-size + const perInstance = parseEnvInt( + process.env.GRAPHILE_CACHE_INSTANCE_HEAP_BYTES, + DEFAULT_INSTANCE_HEAP_BYTES, + ); + const baseReserve = parseEnvInt( + process.env.GRAPHILE_CACHE_BASE_RESERVE_BYTES, + DEFAULT_BASE_RESERVE_BYTES, + ); + const buildReserve = parseEnvInt( + process.env.GRAPHILE_CACHE_BUILD_RESERVE_BYTES, + DEFAULT_BUILD_RESERVE_BYTES, + ); + return computeCapacityFromBudget(heapLimit, perInstance, baseReserve, buildReserve); + } catch { + return MIN_CACHE_MAX; + } +} + /** * Get cache configuration from environment variables * * Supports: - * - GRAPHILE_CACHE_MAX: Maximum number of entries (default: 50) - * - GRAPHILE_CACHE_TTL_MS: TTL in milliseconds - * - Production default: ONE_YEAR + * - GRAPHILE_CACHE_MAX: Maximum number of entries. Default is heap-aware + * (see computeHeapAwareMax) rather than a fixed 50, because each cached + * instance retains ~0.5 GB and a count-only cap OOMs the process. + * - GRAPHILE_CACHE_INSTANCE_HEAP_BYTES: per-instance heap estimate for the + * heap-aware default (default ~512 MB). + * - GRAPHILE_CACHE_TTL_MS: idle TTL in milliseconds (updateAgeOnGet refreshes it + * on every hit, so this is an idle-expiry, not an absolute lifetime) + * - Production default: SIX_HOURS_MS — an idle tenant's ~0.5 GB instance is + * reclaimed within hours instead of pinned for a year; a later request + * simply rebuilds it * - Development default: FIVE_MINUTES_MS * - * NOTE: This value should be <= PG_CACHE_MAX (also default: 50) so that - * every cached PostGraphile instance has a live pool backing it. + * NOTE: This value should be <= PG_CACHE_MAX so that every cached PostGraphile + * instance has a live pool backing it. */ export function getCacheConfig(): CacheConfig { const isDevelopment = process.env.NODE_ENV === 'development'; const max = process.env.GRAPHILE_CACHE_MAX - ? parseInt(process.env.GRAPHILE_CACHE_MAX, 10) - : 50; + ? parseEnvInt(process.env.GRAPHILE_CACHE_MAX, computeHeapAwareMax()) + : computeHeapAwareMax(); const ttl = process.env.GRAPHILE_CACHE_TTL_MS ? parseInt(process.env.GRAPHILE_CACHE_TTL_MS, 10) : isDevelopment ? FIVE_MINUTES_MS - : ONE_YEAR; + : SIX_HOURS_MS; return { max, ttl }; } @@ -89,16 +227,61 @@ export interface GraphileCacheEntry { httpServer: HttpServer; cacheKey: string; createdAt: number; + /** + * Database name backing this instance's pg pool. Used to evict the entry when its + * pool is disposed (see the pgCache cleanup callback). Distinct from cacheKey, which + * on the public server is the request HOST, not the database. + */ + dbname?: string; /** Optional RealtimeManager for cursor-tracked subscription delivery */ realtimeManager?: { stop(): Promise } | null; + /** + * Number of requests currently executing against this entry's handler. + * Maintained by invokeEntryHandler; disposeEntry drains to 0 (bounded by + * GRAPHILE_CACHE_DRAIN_TIMEOUT_MS) before releasing the instance so eviction + * cannot tear down a schema mid-request. + */ + inflight?: number; + /** Set at the start of disposal; routing must treat the entry as a cache miss. */ + disposing?: boolean; } -// Track disposed entries to prevent double-disposal -const disposedKeys = new Set(); +// Track disposed entries to prevent double-disposal. Keyed by ENTRY IDENTITY (a WeakSet), +// NOT by the cache key. A key-scoped guard caused a same-key disposal race: while entry A +// for key K was mid `pgl.release()` (K parked in the guard), a rebuilt entry B for the SAME +// key K could be evicted and its disposeEntry would short-circuit, silently skipping +// B.pgl.release(). Guarding by entry identity disposes every distinct entry exactly once +// while still allowing a rebuilt entry on the same key to be released. +const disposedEntries = new WeakSet(); // Track keys that are being manually evicted for accurate eviction reason const manualEvictionKeys = new Set(); +// Track keys evicted by the memory governor for accurate eviction reason +const governorEvictionKeys = new Set(); + +// Number of entries currently DRAINING (evicted from the cache but their ~GB of +// heap still live until in-flight requests finish and pgl.release() completes). +// Invisible to ensureCacheHeadroom (they are no longer cache entries), so build +// admission must consult it: a build transient stacked on undrained instances is +// what OOMed the soak (flush -> rebuild race). +let drainingCount = 0; + +export const getDrainingCount = (): number => drainingCount; + +/** + * Hold a build until evicted instances have actually released their memory — + * or heap pressure is comfortable anyway, or the bounded wait expires (drains + * are themselves bounded by GRAPHILE_CACHE_DRAIN_TIMEOUT_MS). + */ +export async function waitForDrainSettle(timeoutMs = 20_000): Promise { + const start = Date.now(); + while (drainingCount > 0 && Date.now() - start < timeoutMs) { + if (getMemoryPressure().level === 'ok') return; + await new Promise((resolve) => setTimeout(resolve, 200)); + } +} + /** * Dispose a PostGraphile v5 cache entry * @@ -106,20 +289,53 @@ const manualEvictionKeys = new Set(); * 1. Closing the HTTP server if listening * 2. Releasing the PostGraphile instance (which internally releases grafserv) * - * Uses disposedKeys set to prevent double-disposal when closeAllCaches() - * explicitly disposes entries and then clear() triggers the dispose callback. + * Uses the disposedEntries WeakSet to prevent double-disposal of the same entry when + * closeAllCaches() explicitly disposes entries and then clear() triggers the dispose + * callback for the same entry. */ const disposeEntry = async (entry: GraphileCacheEntry, key: string): Promise => { - // Prevent double-disposal - if (disposedKeys.has(key)) { + // Prevent double-disposal of the SAME entry (guard by identity, not by key — see the + // disposedEntries declaration for why). + if (disposedEntries.has(entry)) { return; } - disposedKeys.add(key); + disposedEntries.add(entry); + cacheCounters.disposals += 1; + entry.disposing = true; + drainingCount += 1; + try { + await disposeEntryInner(entry, key); + } finally { + drainingCount -= 1; + } +}; + +const disposeEntryInner = async (entry: GraphileCacheEntry, key: string): Promise => { + + // Drain in-flight requests before tearing the instance down. The entry is already + // out of the cache (dispose fires post-removal), so no NEW requests can route here — + // invokeEntryHandler also refuses entries with `disposing` set. Bounded wait: a wedged + // request must not pin ~0.5 GB forever. + const drainTimeoutMs = parseEnvInt(process.env.GRAPHILE_CACHE_DRAIN_TIMEOUT_MS, 30_000); + const drainStart = Date.now(); + while ((entry.inflight ?? 0) > 0 && Date.now() - drainStart < drainTimeoutMs) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + if ((entry.inflight ?? 0) > 0) { + cacheCounters.drainTimeouts += 1; + log.warn( + `Disposing PostGraphile[${key}] with ${entry.inflight} request(s) still in flight after ${drainTimeoutMs}ms drain timeout`, + ); + } log.debug(`Disposing PostGraphile[${key}]`); try { - // Close HTTP server if it's listening - if (entry.httpServer?.listening) { + // Close the HTTP server. create-instance builds it via createServer() but never + // .listen()s it, so `.listening` is always false — the old guard meant close() never + // ran. Closing unconditionally detaches grafserv's 'upgrade' listener; when the server + // was never listening, close() simply invokes the callback with ERR_SERVER_NOT_RUNNING + // (a harmless no-op here), so the Promise always resolves. + if (entry.httpServer) { await new Promise((resolve) => { entry.httpServer.close(() => resolve()); }); @@ -138,15 +354,18 @@ const disposeEntry = async (entry: GraphileCacheEntry, key: string): Promise { + if (governorEvictionKeys.has(key)) { + governorEvictionKeys.delete(key); + return 'governor'; + } if (manualEvictionKeys.has(key)) { manualEvictionKeys.delete(key); return 'manual'; @@ -165,6 +384,20 @@ const getEvictionReason = (key: string, entry: GraphileCacheEntry): EvictionReas // Get initial cache configuration const initialConfig = getCacheConfig(); +// Surface the resolved instance cap once at startup. Because each cached PostGraphile +// instance retains ~0.5 GB, an oversized cap (relative to the heap) is the primary cause +// of schema-builder OOM — make the chosen value visible to operators. +try { + const heapLimitMb = Math.round(getHeapStatistics().heap_size_limit / (1024 * 1024)); + const source = process.env.GRAPHILE_CACHE_MAX ? 'GRAPHILE_CACHE_MAX' : 'heap-aware default'; + log.info( + `graphileCache max=${initialConfig.max} instances (${source}); heap limit ~${heapLimitMb}MB. ` + + `Each query-serving instance retains ~0.5GB; raise --max-old-space-size or lower GRAPHILE_CACHE_MAX if memory-constrained.`, + ); +} catch { + // ignore — logging is best-effort +} + // --- Graphile Cache --- export const graphileCache = new LRUCache({ max: initialConfig.max, @@ -173,6 +406,7 @@ export const graphileCache = new LRUCache({ dispose: (entry, key) => { // Determine eviction reason before disposal const reason = getEvictionReason(key, entry); + cacheCounters.evictions[reason] += 1; // Emit eviction event cacheEvents.emitEviction({ key, reason, entry }); @@ -187,6 +421,169 @@ export const graphileCache = new LRUCache({ } }); +/** + * Invoke an entry's Express handler with in-flight refcounting. + * + * Returns false WITHOUT invoking when the entry is already being disposed — + * callers must treat that as a cache miss (rebuild) or retry. On invocation, + * the refcount is incremented and released exactly once when the response + * finishes or the connection closes, which is what disposeEntry drains on. + */ +export function invokeEntryHandler( + entry: GraphileCacheEntry, + req: ExpressRequest, + res: ExpressResponse, + next: ExpressNextFunction, +): boolean { + if (entry.disposing) { + return false; + } + entry.inflight = (entry.inflight ?? 0) + 1; + let released = false; + const release = () => { + if (!released) { + released = true; + entry.inflight = Math.max(0, (entry.inflight ?? 1) - 1); + } + }; + // 'close' fires after 'finish' and also on aborted connections; release is idempotent. + res.once('finish', release); + res.once('close', release); + entry.handler(req, res, next); + return true; +} + +/** + * Evict least-recently-used entries until there is room for `slots` new entries + * under the configured max. Called BEFORE building a new instance so the build's + * transient allocation (hundreds of MB) lands on a cache that has already shed + * an instance, instead of stacking a full cache + a build peak (the OOM shape). + */ +export function ensureCacheHeadroom(slots = 1): number { + const { max } = getCacheConfig(); + let evicted = 0; + while (graphileCache.size > Math.max(0, max - slots)) { + // rkeys() iterates least-recently-used first + const oldestKey = graphileCache.rkeys().next().value as string | undefined; + if (oldestKey === undefined) break; + graphileCache.delete(oldestKey); + evicted++; + } + if (evicted > 0) { + log.info(`Evicted ${evicted} instance(s) before build to keep heap headroom (max=${max})`); + } + return evicted; +} + +// --- Memory Governor --- +export type MemoryPressureLevel = 'ok' | 'elevated' | 'critical'; + +export interface MemoryPressure { + level: MemoryPressureLevel; + heapUsed: number; + heapLimit: number; + ratio: number; +} + +const parseEnvFloat = (value: string | undefined, fallback: number): number => { + if (!value) return fallback; + const n = parseFloat(value); + return Number.isFinite(n) && n > 0 && n < 1 ? n : fallback; +}; + +/** + * Current heap pressure against V8's actually-exhaustible headroom + * (used / (used + total_available_size)). + * + * elevated (default >=85%): proactively evict idle instances ahead of the LRU + * schedule. critical (default >=92%): additionally REFUSE to start new schema + * builds (the caller responds 503; resident instances keep serving). A build's + * transient allocations are the largest single spike the process makes — at + * critical pressure admitting one converts degraded service into a V8 + * heap-limit abort for every tenant on the box. + * Tunables: GRAPHILE_MEMORY_GOVERNOR_ELEVATED / GRAPHILE_MEMORY_GOVERNOR_CRITICAL. + */ +export function getMemoryPressure(): MemoryPressure { + const stats = getHeapStatistics(); + const heapLimit = stats.heap_size_limit; + const heapUsed = process.memoryUsage().heapUsed; + // Ratio denominator is used + V8's own remaining-allocatable estimate, NOT + // heap_size_limit: the limit includes young-generation and reserved space the + // old space can never use, so with --max-old-space-size=512 the limit reads + // ~738MB while the process aborts near ~508MB used — a 0.69 "ratio" at the + // moment of death, leaving 0.85/0.92 watermarks unreachable exactly when they + // matter. used/(used+available) approaches 1.0 as the abort nears at every + // heap size. + const available = stats.total_available_size ?? Math.max(0, heapLimit - heapUsed); + const exhaustible = heapUsed + available; + const ratio = exhaustible > 0 ? heapUsed / exhaustible : 0; + const elevated = parseEnvFloat(process.env.GRAPHILE_MEMORY_GOVERNOR_ELEVATED, 0.85); + const critical = parseEnvFloat(process.env.GRAPHILE_MEMORY_GOVERNOR_CRITICAL, 0.92); + const level: MemoryPressureLevel = ratio >= critical ? 'critical' : ratio >= elevated ? 'elevated' : 'ok'; + return { level, heapUsed, heapLimit, ratio }; +} + +/** + * Gate for new schema builds. Returns the pressure snapshot; when + * `refuseBuild` is true the caller must not start a build (respond 503 and + * count it). Cheap enough for the request path: one memoryUsage() call. + */ +export function shouldRefuseBuild(): MemoryPressure & { refuseBuild: boolean } { + const pressure = getMemoryPressure(); + const refuseBuild = pressure.level === 'critical'; + if (refuseBuild) { + cacheCounters.buildRefusals += 1; + log.warn( + `Refusing new schema build at critical heap pressure ` + + `(${Math.round(pressure.ratio * 100)}% of exhaustible headroom used; ` + + `heap limit ${Math.round(pressure.heapLimit / 1048576)}MB)`, + ); + } + return { ...pressure, refuseBuild }; +} + +let governorTimer: ReturnType | null = null; + +/** + * Periodic proactive eviction: at elevated/critical pressure, evict the + * least-recently-used instance (skipping in-flight ones) each tick until + * pressure clears. Disabled with GRAPHILE_MEMORY_GOVERNOR=0. + */ +export function startMemoryGovernor(intervalMs = 10_000): () => void { + if (process.env.GRAPHILE_MEMORY_GOVERNOR === '0') return () => {}; + if (governorTimer) return stopMemoryGovernor; + governorTimer = setInterval(() => { + const pressure = getMemoryPressure(); + if (pressure.level === 'ok') return; + // Evict ONE entry per tick, LRU first, preferring idle entries. + let victim: string | undefined; + for (const key of graphileCache.rkeys()) { + const entry = graphileCache.peek(key); + if (entry && !entry.disposing && (entry.inflight ?? 0) === 0) { + victim = key; + break; + } + if (victim === undefined) victim = key; // fall back to LRU even if busy + } + if (victim === undefined) return; + governorEvictionKeys.add(victim); + log.warn( + `Memory governor evicting PostGraphile[${victim}] at ${pressure.level} pressure ` + + `(heap ${Math.round(pressure.ratio * 100)}%)`, + ); + graphileCache.delete(victim); + }, intervalMs); + if (typeof governorTimer.unref === 'function') governorTimer.unref(); + return stopMemoryGovernor; +} + +export function stopMemoryGovernor(): void { + if (governorTimer) { + clearInterval(governorTimer); + governorTimer = null; + } +} + // --- Cache Stats --- export interface CacheStats { size: number; @@ -235,9 +632,12 @@ export function clearMatchingEntries(pattern: RegExp): number { const unregister = pgCache.registerCleanupCallback((pgPoolKey: string) => { log.debug(`pgPool[${pgPoolKey}] disposed - checking graphile entries`); - // Remove graphile entries that reference this pool key + // Remove graphile entries backed by this pool. Match on the entry's dbname (the pool is + // keyed by database name), NOT on cacheKey: on the public server cacheKey is the request + // HOST, which never contains the database name, so the old `cacheKey.includes(pgPoolKey)` + // test never matched and this safety valve was dead. graphileCache.forEach((entry, k) => { - if (entry.cacheKey.includes(pgPoolKey)) { + if (entry.dbname === pgPoolKey) { log.debug(`Removing graphileCache[${k}] due to pgPool[${pgPoolKey}] disposal`); manualEvictionKeys.add(k); graphileCache.delete(k); @@ -281,11 +681,12 @@ export const closeAllCaches = async (verbose = false): Promise => { // Wait for all disposals to complete await Promise.allSettled(disposePromises); - // Clear the cache after disposal (dispose callback will no-op due to disposedKeys) + // Clear the cache after disposal (dispose callback no-ops: each entry is already + // in the disposedEntries WeakSet from the explicit disposeEntry above). graphileCache.clear(); - // Clear disposed keys tracking after full cleanup - disposedKeys.clear(); + // The disposedEntries WeakSet needs no explicit clearing — entries are reclaimed by + // GC once the cache no longer references them. manualEvictionKeys.clear(); // Close pg pools diff --git a/graphile/graphile-cache/src/index.ts b/graphile/graphile-cache/src/index.ts index e64be532b9..3dec3e4442 100644 --- a/graphile/graphile-cache/src/index.ts +++ b/graphile/graphile-cache/src/index.ts @@ -25,6 +25,28 @@ export { CacheStats, getCacheStats, + // In-process metrics counters + CacheCounters, + cacheCounters, + getCacheCounters, + + // In-flight refcounting + pre-build headroom + invokeEntryHandler, + ensureCacheHeadroom, + + // Capacity model + memory governor + computeCapacityFromBudget, + MemoryPressure, + MemoryPressureLevel, + getMemoryPressure, + shouldRefuseBuild, + startMemoryGovernor, + stopMemoryGovernor, + + // Drain-aware build admission + getDrainingCount, + waitForDrainSettle, + // Clear matching entries clearMatchingEntries } from './graphile-cache'; diff --git a/graphile/graphile-i18n/src/plugin.ts b/graphile/graphile-i18n/src/plugin.ts index ebe3ab14a4..81f1a2acad 100644 --- a/graphile/graphile-i18n/src/plugin.ts +++ b/graphile/graphile-i18n/src/plugin.ts @@ -74,7 +74,7 @@ export function createI18nPlugin(options: I18nPluginOptions = {}): GraphileConfi // Closure-scoped state shared between init and field hooks let i18nRegistry: Record = {}; - const localeTypeCache: Record = {}; + let localeTypeCache: Record = {}; return { name: 'I18nPlugin', @@ -85,6 +85,7 @@ export function createI18nPlugin(options: I18nPluginOptions = {}): GraphileConfi init: { callback(_, build) { i18nRegistry = {}; + localeTypeCache = {}; for (const [, codec] of Object.entries(build.input.pgRegistry.pgCodecs)) { const c = codec as PgCodecWithAttributes; @@ -238,10 +239,13 @@ export function createI18nPlugin(options: I18nPluginOptions = {}): GraphileConfi .map(f => `coalesce(v."${f.column}", b."${f.column}") as "${f.column}"`) .join(', '); + const baseTableRef = `"${schemaName}"."${baseTable}"`; + const translationTableRef = `"${schemaName}"."${translationTable}"`; + // Build the SQL query template const sqlQuery = `SELECT v."${langCodeColumn}" AS "lang_code", ${coalescedCols} - FROM "${schemaName}"."${baseTable}" b - LEFT JOIN "${schemaName}"."${translationTable}" v + FROM ${baseTableRef} b + LEFT JOIN ${translationTableRef} v ON v."${fkColumn}" = b."${pkColumn}" AND array_position($2::text[], v."${langCodeColumn}") IS NOT NULL WHERE b."${pkColumn}" = $1::${pkType} @@ -265,18 +269,20 @@ export function createI18nPlugin(options: I18nPluginOptions = {}): GraphileConfi $baseCols[column] = $parent.get(column); } const $withPgClient = (grafastContext() as any).get('withPgClient'); + const $pgSettings = (grafastContext() as any).get('pgSettings'); const $langCodes = (grafastContext() as any).get('langCodes'); // Combine all inputs into a single step const $input = object({ id: $id, withPgClient: $withPgClient, + pgSettings: $pgSettings, langCodes: $langCodes, ...$baseCols, }); return lambda($input, async (input: any) => { - const { id, withPgClient, langCodes: ctxLangCodes, ...baseCols } = input; + const { id, withPgClient, pgSettings, langCodes: ctxLangCodes, ...baseCols } = input; const langs: string[] = ctxLangCodes ?? defaultLanguages; if (!withPgClient || !id) { @@ -287,7 +293,7 @@ export function createI18nPlugin(options: I18nPluginOptions = {}): GraphileConfi return result; } - const row = await withPgClient(null, async (client: any) => { + const row = await withPgClient(pgSettings, async (client: any) => { const { rows } = await client.query(sqlQuery, [id, langs]); return rows[0] ?? null; }); diff --git a/graphile/graphile-llm/src/__tests__/agent-discovery.test.ts b/graphile/graphile-llm/src/__tests__/agent-discovery.test.ts new file mode 100644 index 0000000000..c0b9e14c7f --- /dev/null +++ b/graphile/graphile-llm/src/__tests__/agent-discovery.test.ts @@ -0,0 +1,148 @@ +/** + * Agent discovery tenant-isolation tests (pure, no DB required). + * + * Verifies the cross-tenant bleed fix: discovery is filtered by the requesting + * tenant's database id (resolved from jwt.claims.database_id in pgSettings and + * passed as $1), and the per-database cache is keyed by that id — never by + * dbname alone — so a shared/pooled instance cannot return another tenant's + * agent config. + */ + +import { clearAgentDiscoveryCache, getAgentDiscovery } from '../plugins/agent-discovery-plugin'; + +// ─── Fake pool ─────────────────────────────────────────────────────────────── + +interface QueryCall { + sql: string; + values: unknown[] | undefined; +} + +/** + * A fake pg Pool whose `query` scopes rows by the $1 database id — mirroring how + * the real (shared) control-plane metaschema tables behave once the WHERE + * clause is applied. Records every call for assertions. + */ +function makeFakePool(rowsByDatabaseId: Record) { + const calls: QueryCall[] = []; + const pool = { + calls, + query: async (sql: string, values?: unknown[]) => { + calls.push({ sql, values }); + const id = values?.[0] as string | null; + const row = id != null ? rowsByDatabaseId[id] : undefined; + return { rows: row ? [row] : [] }; + } + }; + return pool; +} + +const TENANT_1 = '11111111-1111-1111-1111-111111111111'; +const TENANT_2 = '22222222-2222-2222-2222-222222222222'; + +const ROWS = { + [TENANT_1]: { + schema_name: 'tenant1_agent', + thread_table_name: 'agent_thread', + message_table_name: 'agent_message', + task_table_name: 'agent_task' + }, + [TENANT_2]: { + schema_name: 'tenant2_agent', + thread_table_name: 't2_thread', + message_table_name: 't2_message', + task_table_name: 't2_task' + } +}; + +// ─── Tests ─────────────────────────────────────────────────────────────────── + +describe('getAgentDiscovery — tenant isolation', () => { + beforeEach(() => { + clearAgentDiscoveryCache(); + }); + + it('filters by the tenant database id from pgSettings, passed as $1', async () => { + const pool = makeFakePool(ROWS); + + const result = await getAgentDiscovery(pool as any, 'shared-db', { + 'jwt.claims.database_id': TENANT_1 + }); + + expect(pool.calls).toHaveLength(1); + expect(pool.calls[0].sql).toContain('WHERE s.database_id = $1'); + expect(pool.calls[0].values).toEqual([TENANT_1]); + + expect(result).not.toBeNull(); + expect(result!.thread).toEqual({ schemaName: 'tenant1_agent', tableName: 'agent_thread' }); + expect(result!.message).toEqual({ schemaName: 'tenant1_agent', tableName: 'agent_message' }); + expect(result!.task).toEqual({ schemaName: 'tenant1_agent', tableName: 'agent_task' }); + }); + + it('does NOT bleed across tenants sharing one instance/dbname', async () => { + const pool = makeFakePool(ROWS); + + const t1 = await getAgentDiscovery(pool as any, 'shared-db', { + 'jwt.claims.database_id': TENANT_1 + }); + const t2 = await getAgentDiscovery(pool as any, 'shared-db', { + 'jwt.claims.database_id': TENANT_2 + }); + + // Each tenant gets its own config despite the same dbname. + expect(t1!.thread!.schemaName).toBe('tenant1_agent'); + expect(t2!.thread!.schemaName).toBe('tenant2_agent'); + + // Two distinct database ids ⇒ two separate cache entries ⇒ two queries. + expect(pool.calls.map((c) => c.values)).toEqual([[TENANT_1], [TENANT_2]]); + }); + + it('caches by database id (same id ⇒ a single query)', async () => { + const pool = makeFakePool(ROWS); + + const a = await getAgentDiscovery(pool as any, 'shared-db', { + 'jwt.claims.database_id': TENANT_1 + }); + const b = await getAgentDiscovery(pool as any, 'shared-db', { + 'jwt.claims.database_id': TENANT_1 + }); + + expect(pool.calls).toHaveLength(1); + expect(a).toEqual(b); + }); + + it('fails closed when database id is absent — passes null as $1, keys under :nodb', async () => { + const pool = makeFakePool(ROWS); + + // No pgSettings at all. + const r1 = await getAgentDiscovery(pool as any, 'shared-db'); + // Empty pgSettings (no jwt.claims.database_id). + const r2 = await getAgentDiscovery(pool as any, 'shared-db', {}); + + expect(r1).toBeNull(); + expect(r2).toBeNull(); + + // First call queries with null $1; second is served from the ':nodb' cache. + expect(pool.calls).toHaveLength(1); + expect(pool.calls[0].values).toEqual([null]); + + // A different dbname without an id uses a different ':nodb' key ⇒ new query. + await getAgentDiscovery(pool as any, 'other-db'); + expect(pool.calls).toHaveLength(2); + }); + + it('caches null (unprovisioned tenant) without re-querying', async () => { + // Pool returns no rows for this (valid) tenant id ⇒ discovery is null. + const pool = makeFakePool({}); + + const first = await getAgentDiscovery(pool as any, 'shared-db', { + 'jwt.claims.database_id': TENANT_1 + }); + const second = await getAgentDiscovery(pool as any, 'shared-db', { + 'jwt.claims.database_id': TENANT_1 + }); + + expect(first).toBeNull(); + expect(second).toBeNull(); + expect(pool.calls).toHaveLength(1); + }); +}); diff --git a/graphile/graphile-llm/src/__tests__/rag-unqualified.test.ts b/graphile/graphile-llm/src/__tests__/rag-unqualified.test.ts new file mode 100644 index 0000000000..d89fa1f17e --- /dev/null +++ b/graphile/graphile-llm/src/__tests__/rag-unqualified.test.ts @@ -0,0 +1,96 @@ +/** + * RAG chunk-table qualification tests (pure, no DB / no Ollama required). + * + * Verifies the generated chunk search SQL references the tenant-data table + * schema-qualified when a chunks schema is known, and falls back to a bare + * table name only when no schema is available. + */ + +import { buildChunkSearchSql, discoverChunkTables } from '../plugins/rag-plugin'; +import type { ChunkTableInfo } from '../types'; + +// ─── Fixtures ──────────────────────────────────────────────────────────────── + +function baseChunkTable(overrides: Partial = {}): ChunkTableInfo { + return { + parentCodecName: 'articles', + chunksSchema: 'llm_test', + chunksTableName: 'articles_chunks', + parentFkField: 'parent_id', + parentPkField: 'id', + embeddingField: 'embedding', + contentField: 'content', + ...overrides + }; +} + +function makeBuild() { + return { + options: {}, + input: { + pgRegistry: { + pgCodecs: { + articles: { + name: 'articles', + attributes: { id: {}, title: {} }, + extensions: { + pg: { schemaName: 'llm_test' }, + tags: { + hasChunks: { + chunksTable: 'articles_chunks', + parentFk: 'parent_id', + parentPk: 'id', + embeddingField: 'embedding', + contentField: 'content' + } + } + } + } + } + } + } + }; +} + +// ─── buildChunkSearchSql ───────────────────────────────────────────────────── + +describe('buildChunkSearchSql — chunk table qualification', () => { + it('emits a SCHEMA-QUALIFIED reference by default', () => { + const { text } = buildChunkSearchSql(baseChunkTable(), '[1,0,0]', 5, null); + expect(text).toContain('FROM "llm_test"."articles_chunks"'); + }); + + it('emits an UNQUALIFIED reference when no schema is known (existing fallback)', () => { + const { text } = buildChunkSearchSql( + baseChunkTable({ chunksSchema: null }), + '[1,0,0]', + 5, + null + ); + expect(text).toContain('FROM "articles_chunks"'); + }); + + it('preserves parameter order/values (no maxDistance)', () => { + const { values } = buildChunkSearchSql(baseChunkTable(), '[1,0,0]', 7, null); + expect(values).toEqual(['[1,0,0]', 7]); + }); + + it('preserves parameter order/values (with maxDistance)', () => { + const { text, values } = buildChunkSearchSql(baseChunkTable(), '[1,0,0]', 7, 0.4); + expect(values).toEqual(['[1,0,0]', 0.4, 7]); + expect(text).toContain('LIMIT $3'); + }); +}); + +// ─── discoverChunkTables ───────────────────────────────────────────────────── + +describe('discoverChunkTables', () => { + it('discovers chunk tables and emits schema-qualified SQL', () => { + const tables = discoverChunkTables(makeBuild()); + expect(tables).toHaveLength(1); + expect(tables[0].chunksSchema).toBe('llm_test'); + + const { text } = buildChunkSearchSql(tables[0], '[1,0,0]', 5, null); + expect(text).toContain('FROM "llm_test"."articles_chunks"'); + }); +}); diff --git a/graphile/graphile-llm/src/plugins/agent-discovery-plugin.ts b/graphile/graphile-llm/src/plugins/agent-discovery-plugin.ts index ad020c4dc3..735ca173dc 100644 --- a/graphile/graphile-llm/src/plugins/agent-discovery-plugin.ts +++ b/graphile/graphile-llm/src/plugins/agent-discovery-plugin.ts @@ -41,6 +41,9 @@ export function clearAgentDiscoveryCache(): void { // ─── Discovery Query ──────────────────────────────────────────────────────── +// Control-plane metaschema_* tables stay fully qualified. The schema row is +// filtered by the requesting tenant's database id ($1) so a shared/pooled +// instance never bleeds another tenant's agent config across the join. const DISCOVERY_SQL = ` SELECT s.schema_name, @@ -49,18 +52,31 @@ const DISCOVERY_SQL = ` acm.task_table_name FROM metaschema_modules_public.agent_chat_module acm JOIN metaschema_public.schema s ON s.id = acm.schema_id + WHERE s.database_id = $1 LIMIT 1 `; /** * Look up agent table info for a database, querying the module config table. - * Results are cached per-database with a 60s TTL. + * + * The requesting tenant is identified by its database id, resolved from + * `jwt.claims.database_id` in the per-request pgSettings — the same way the + * billing config cache obtains it (see config-cache.ts / metering-plugin.ts). + * The query filters on that id so a shared instance cannot return another + * tenant's config. When the id is absent we fail closed (no discovery) and + * cache under a `:nodb` key to avoid poisoning a real tenant's entry. + * + * Results are cached per database id with a 60s TTL. */ export async function getAgentDiscovery( pool: Pool, - dbname: string + dbname: string, + pgSettings?: Record | null ): Promise { - const cached = agentDiscoveryCache.get(dbname); + const databaseId = pgSettings?.['jwt.claims.database_id'] ?? null; + const cacheKey = databaseId ?? `${dbname}:nodb`; + + const cached = agentDiscoveryCache.get(cacheKey); if (cached !== undefined) { return cached; } @@ -68,7 +84,7 @@ export async function getAgentDiscovery( let discovery: AgentDiscovery | null = null; try { - const { rows } = await pool.query(DISCOVERY_SQL); + const { rows } = await pool.query(DISCOVERY_SQL, [databaseId]); if (rows.length > 0) { const row = rows[0]; @@ -90,6 +106,6 @@ export async function getAgentDiscovery( // Module table doesn't exist in this database — not provisioned } - agentDiscoveryCache.set(dbname, discovery); + agentDiscoveryCache.set(cacheKey, discovery); return discovery; } diff --git a/graphile/graphile-llm/src/plugins/rag-plugin.ts b/graphile/graphile-llm/src/plugins/rag-plugin.ts index 3c1a3e15cf..d60c4f3715 100644 --- a/graphile/graphile-llm/src/plugins/rag-plugin.ts +++ b/graphile/graphile-llm/src/plugins/rag-plugin.ts @@ -86,8 +86,10 @@ function parseHasChunksTag(raw: any, codec: any): ChunkTableInfo | null { /** * Discover all chunk-aware tables from the pgRegistry. + * + * Exported for unit testing. */ -function discoverChunkTables(build: any): ChunkTableInfo[] { +export function discoverChunkTables(build: any): ChunkTableInfo[] { const chunkTables: ChunkTableInfo[] = []; const pgRegistry = build.input?.pgRegistry ?? build.pgRegistry; if (!pgRegistry) return chunkTables; @@ -111,17 +113,19 @@ function discoverChunkTables(build: any): ChunkTableInfo[] { /** * Build a SQL query string to search a chunks table for similar embeddings. + * + * Exported for unit testing. */ -function buildChunkSearchSql( +export function buildChunkSearchSql( table: ChunkTableInfo, vectorString: string, limit: number, maxDistance: number | null ): { text: string; values: any[] } { - const schema = table.chunksSchema; - const qualifiedTable = schema - ? `"${schema}"."${table.chunksTableName}"` - : `"${table.chunksTableName}"`; + // Keep the schema-qualified reference when a schema is known. + const qualifiedTable = !table.chunksSchema + ? `"${table.chunksTableName}"` + : `"${table.chunksSchema}"."${table.chunksTableName}"`; const embeddingCol = `"${table.embeddingField}"`; const contentCol = `"${table.contentField}"`; diff --git a/graphile/graphile-presigned-url-plugin/__tests__/storage-module-cache.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/storage-module-cache.test.ts new file mode 100644 index 0000000000..54ed68236d --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/__tests__/storage-module-cache.test.ts @@ -0,0 +1,209 @@ +/** + * Unit tests for resolveStorageConfigFromCodec. + * + * These are pure (no DB) tests covering both the single-tenant path (exact + * physical schema match) and the blueprint-pooling path, where the codec's + * build-time schema belongs to the REPRESENTATIVE tenant but the request's + * configs carry a DIFFERENT tenant's actual (differently-hashed) schema. In + * that case the resolver must fall back to matching on the LOGICAL schema + * name (the tenant hash prefix stripped from both sides). + */ + +import { resolveStorageConfigFromCodec } from '../src/storage-module-cache'; +import type { StorageModuleConfig } from '../src/types'; + +// --- Fixtures ------------------------------------------------------------- + +// Two real-shaped hashed tenant schemas that share the SAME logical suffix +// (`storage-public`). Under pooling, the shared instance is built against the +// representative tenant's schema, while requests route to other tenants. +const REPRESENTATIVE_SCHEMA = 'marketplace-db-tenant1-5e6b13b2-storage-public'; +const TENANT_SCHEMA = 'marketplace-db-tenant2-35a03232-storage-public'; + +function makeConfig( + overrides: Partial & + Pick, +): StorageModuleConfig { + return { + bucketsQualifiedName: `"${overrides.schemaName}"."${overrides.bucketsTableName}"`, + filesQualifiedName: `"${overrides.schemaName}"."${overrides.filesTableName}"`, + scope: 'app', + entityTableId: null, + entityQualifiedName: null, + endpoint: null, + publicUrlPrefix: null, + provider: null, + allowedOrigins: null, + uploadUrlExpirySeconds: 900, + downloadUrlExpirySeconds: 3600, + defaultMaxFileSize: 200 * 1024 * 1024, + maxFilenameLength: 1024, + cacheTtlSeconds: 3600, + hasPathShares: false, + maxBulkFiles: 100, + maxBulkTotalSize: 1073741824, + ...overrides, + }; +} + +function makeCodec(schemaName: string | undefined, name: string | undefined) { + return { name: name as string, extensions: { pg: { schemaName, name } } }; +} + +// --- Tests ---------------------------------------------------------------- + +describe('resolveStorageConfigFromCodec', () => { + describe('single-tenant (exact physical schema match)', () => { + it('matches a files codec by exact schema + files table name', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(TENANT_SCHEMA, 'app_files'); + + expect(resolveStorageConfigFromCodec(codec, [config])).toBe(config); + }); + + it('matches a buckets codec by exact schema + buckets table name', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(TENANT_SCHEMA, 'app_buckets'); + + expect(resolveStorageConfigFromCodec(codec, [config])).toBe(config); + }); + + it('matches plain (non-hashed) schemas exactly, unaffected by hash stripping', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: 'app_public', + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec('app_public', 'app_files'); + + expect(resolveStorageConfigFromCodec(codec, [config])).toBe(config); + }); + }); + + describe('blueprint pooling (logical schema match across tenants)', () => { + it('matches a files codec built for the representative tenant against another tenant config', () => { + // Build-time codec: representative tenant. Request config: a DIFFERENT tenant. + const tenantConfig = makeConfig({ + id: 'sm-tenant2', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(REPRESENTATIVE_SCHEMA, 'app_files'); + + // Physical schemas differ; logical suffix (`storage-public`) is shared. + expect(REPRESENTATIVE_SCHEMA).not.toEqual(TENANT_SCHEMA); + expect(resolveStorageConfigFromCodec(codec, [tenantConfig])).toBe(tenantConfig); + }); + + it('matches a buckets codec built for the representative tenant against another tenant config', () => { + const tenantConfig = makeConfig({ + id: 'sm-tenant2', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(REPRESENTATIVE_SCHEMA, 'app_buckets'); + + expect(resolveStorageConfigFromCodec(codec, [tenantConfig])).toBe(tenantConfig); + }); + + it('does NOT match when the logical schema suffix differs', () => { + // A codec whose logical schema is `analytics-public` must not resolve to a + // `storage-public` config even though both are hashed tenant schemas. + const tenantConfig = makeConfig({ + id: 'sm-tenant2', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec('marketplace-db-tenant1-5e6b13b2-analytics-public', 'app_files'); + + expect(resolveStorageConfigFromCodec(codec, [tenantConfig])).toBeNull(); + }); + }); + + describe('exact-physical priority', () => { + it('prefers an exact physical match over a logical-only match', () => { + const exactConfig = makeConfig({ + id: 'sm-exact', + schemaName: REPRESENTATIVE_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const logicalOnlyConfig = makeConfig({ + id: 'sm-logical', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(REPRESENTATIVE_SCHEMA, 'app_files'); + + // exactConfig listed AFTER the logical-only one to prove priority is by + // match quality, not array order. + const result = resolveStorageConfigFromCodec(codec, [logicalOnlyConfig, exactConfig]); + expect(result).toBe(exactConfig); + }); + }); + + describe('non-matches and guards', () => { + it('returns null when the table name matches nothing', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(REPRESENTATIVE_SCHEMA, 'some_unrelated_table'); + + expect(resolveStorageConfigFromCodec(codec, [config])).toBeNull(); + }); + + it('returns null when the codec has no schemaName', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = makeCodec(undefined, 'app_files'); + + expect(resolveStorageConfigFromCodec(codec, [config])).toBeNull(); + }); + + it('returns null when the codec has no resolvable table name', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = { name: undefined as unknown as string, extensions: { pg: { schemaName: REPRESENTATIVE_SCHEMA } } }; + + expect(resolveStorageConfigFromCodec(codec, [config])).toBeNull(); + }); + + it('falls back to codec.name for the table name when extensions.pg.name is absent', () => { + const config = makeConfig({ + id: 'sm-1', + schemaName: TENANT_SCHEMA, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + }); + const codec = { name: 'app_files', extensions: { pg: { schemaName: REPRESENTATIVE_SCHEMA } } }; + + expect(resolveStorageConfigFromCodec(codec, [config])).toBe(config); + }); + }); +}); diff --git a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts index 6ea403fbb5..5cf993558e 100644 --- a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts +++ b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts @@ -354,12 +354,46 @@ export async function loadAllStorageModules( return configs; } +/** + * Strip a tenant hash prefix from a physical schema name, returning the + * logical schema name. + * + * Hashed multi-tenant schemas are named like + * `marketplace-db-tenant1-5e6b13b2-app-public`, where `-5e6b13b2-` is an + * 8-hex-char tenant hash separating the tenant prefix from the logical schema + * suffix (`app-public`). This removes everything up to and including the first + * such hash segment. If no hash segment is present (e.g. a plain schema like + * `app_public` or a control-plane schema like `metaschema_public`), the name + * is returned unchanged. + * + * (Duplicated locally rather than imported to avoid a cross-package dependency + * for a one-line helper.) + */ +function stripSchemaHashPrefix(name: string): string { + const match = /-[0-9a-f]{8}-/.exec(name); + return match ? name.slice(match.index + match[0].length) : name; +} + /** * Resolve the storage module config from a PostGraphile pgCodec. * * Matches the codec's schema + table name against cached storage modules. * Works for both files codecs (@storageFiles) and buckets codecs (@storageBuckets). * + * Matching is two-tier: + * 1. Exact physical schema match — used for single-tenant / non-pooled + * instances where the codec's build-time schema equals the request + * tenant's actual schema. Keeps existing behavior unchanged. + * 2. Logical schema match — strip the tenant hash prefix from BOTH the + * codec's schema and each config's schema before comparing. Under + * blueprint pooling a single shared instance serves every tenant of a + * given schema-shape, so the codec's build-time `schemaName` belongs to + * the representative tenant while `allConfigs` carry the request tenant's + * actual (differently-hashed) schema. Both share the same logical suffix + * (e.g. `app-public`). Safe because `allConfigs` is already filtered to a + * single databaseId, so (tableName, logicalSchema) uniquely identifies a + * module. + * * @param pgCodec - The PostGraphile codec (has extensions.pg.schemaName, name) * @param allConfigs - All storage module configs for this database * @returns The matching StorageModuleConfig or null @@ -373,10 +407,24 @@ export function resolveStorageConfigFromCodec( if (!schemaName || !tableName) return null; - return allConfigs.find((c) => + // Priority 1: exact physical schema match (non-pooled / single-tenant). + const exact = allConfigs.find((c) => (c.filesTableName === tableName && c.schemaName === schemaName) || (c.bucketsTableName === tableName && c.schemaName === schemaName), - ) || null; + ); + if (exact) return exact; + + // Priority 2: logical schema match (blueprint pooling — the codec's + // build-time schema belongs to the representative tenant; strip the tenant + // hash from both sides so the shared logical suffix lines up). + const logicalSchema = stripSchemaHashPrefix(schemaName); + return allConfigs.find((c) => { + const configLogicalSchema = stripSchemaHashPrefix(c.schemaName); + return ( + (c.filesTableName === tableName && configLogicalSchema === logicalSchema) || + (c.bucketsTableName === tableName && configLogicalSchema === logicalSchema) + ); + }) || null; } // --- Bucket metadata cache --- diff --git a/graphile/graphile-search/src/__tests__/search-config.test.ts b/graphile/graphile-search/src/__tests__/search-config.test.ts index 1ead0f6b6b..12c142b4d8 100644 --- a/graphile/graphile-search/src/__tests__/search-config.test.ts +++ b/graphile/graphile-search/src/__tests__/search-config.test.ts @@ -423,3 +423,104 @@ describe('VectorNearbyInput includeChunks field (Phase E)', () => { expect(fields.includeChunks.description).toContain('chunks'); }); }); + +// ─── Adapter SQL qualification ──────────────────────────────────────────────── +// +// Tenant-data SQL (bm25 index names, chunk table references) is always emitted +// fully schema-qualified. Tenant routing happens elsewhere (the rewriting pool +// maps canonical→tenant schemas at query time), so the adapters do not +// special-case pooled instances. + +describe('adapter SQL qualification', () => { + // Mock sql object that mimics pg-sql2 behavior (same shape as above). + const mockSql = { + identifier: (name: string) => `"${name}"`, + value: (val: any) => `'${val}'`, + literal: (val: any) => `'${val}'`, + raw: (s: string) => s, + fragment: (strings: TemplateStringsArray, ...values: any[]) => { + let result = ''; + strings.forEach((str, i) => { + result += str; + if (i < values.length) result += String(values[i]); + }); + return result; + }, + join: (parts: any[], sep: string) => parts.join(sep), + parens: (expr: any) => `(${expr})`, + }; + const sql = Object.assign( + (strings: TemplateStringsArray, ...values: any[]) => { + let result = ''; + strings.forEach((str: string, i: number) => { + result += str; + if (i < values.length) result += String(values[i]); + }); + return result; + }, + mockSql, + ); + + // ── bm25: index name passed to to_bm25query ── + describe('bm25 adapter — index name', () => { + const adapter = createBm25Adapter(); + const bm25Column = { + attributeName: 'body', + adapterData: { + bm25Index: { + schemaName: 'app_private', + tableName: 'documents', + columnName: 'body', + indexName: 'documents_body_bm25_idx', + }, + }, + }; + + it('emits the schema-qualified index name', () => { + const result = adapter.buildFilterApply( + sql, + 'tbl' as any, + bm25Column, + { query: 'hello' }, + {}, + ); + + expect(result).not.toBeNull(); + const scoreStr = String(result!.scoreExpression); + // Default behavior: fully qualified "schema"."index" + expect(scoreStr).toContain('"app_private"."documents_body_bm25_idx"'); + }); + }); + + // ── pgvector: chunk table reference ── + describe('pgvector adapter — chunk table reference', () => { + const adapter = createPgvectorAdapter(); + const chunkColumn = { + attributeName: 'embedding', + adapterData: { + chunksInfo: { + chunksSchema: 'app_private', + chunksTableName: 'doc_chunks', + parentFkField: 'document_id', + parentPkField: 'row_id', + embeddingField: 'vec', + }, + }, + }; + + it('emits the schema-qualified chunk table', () => { + const result = adapter.buildFilterApply( + sql, + 'tbl' as any, + chunkColumn, + { vector: [1, 0, 0], metric: 'COSINE' }, + {}, + ); + + expect(result).not.toBeNull(); + const scoreStr = String(result!.scoreExpression); + expect(scoreStr).toContain('"app_private"."doc_chunks"'); + expect(scoreStr).toContain('LEAST'); + }); + }); +}); diff --git a/graphile/graphile-search/src/adapters/bm25.ts b/graphile/graphile-search/src/adapters/bm25.ts index 26ea207f38..d2bb76f7a4 100644 --- a/graphile/graphile-search/src/adapters/bm25.ts +++ b/graphile/graphile-search/src/adapters/bm25.ts @@ -180,9 +180,11 @@ export function createBm25Adapter( const bm25Index = columnData.bm25Index; const columnExpr = sql`${alias}.${sql.identifier(column.attributeName)}`; - // Use quoteQualifiedIdentifier to produce the qualified index name - const qualifiedIndexName = `"${bm25Index.schemaName}"."${bm25Index.indexName}"`; - const bm25queryExpr = sql`to_bm25query(${sql.value(query)}, ${sql.value(qualifiedIndexName)})`; + // The store lookup (getBm25IndexForAttribute) keys off the build-time + // codec schema — correct, because the store was populated by the same + // representative gather. + const indexNameArg = `"${bm25Index.schemaName}"."${bm25Index.indexName}"`; + const bm25queryExpr = sql`to_bm25query(${sql.value(query)}, ${sql.value(indexNameArg)})`; const scoreExpr = sql`(${columnExpr} <@> ${bm25queryExpr})`; // Check for chunk-aware querying diff --git a/graphile/graphile-search/src/codecs/bm25-codec.ts b/graphile/graphile-search/src/codecs/bm25-codec.ts index 117ca88340..5fd82bce32 100644 --- a/graphile/graphile-search/src/codecs/bm25-codec.ts +++ b/graphile/graphile-search/src/codecs/bm25-codec.ts @@ -36,6 +36,15 @@ export interface Bm25IndexInfo { * * Key: "schemaName.tableName.columnName" * Value: Bm25IndexInfo + * + * NOTE (blueprint pooling): this store is process-global and is cleared + + * repopulated per introspection run (see pgIntrospection_introspection). + * Concurrent gathers are serialized by the server's build semaphore, so a + * single global store is tolerated for now. Under blueprint pooling the store + * is keyed by the representative tenant's schema; the BM25 adapter reads the + * index NAME from these entries and always passes the schema-qualified index + * name to to_bm25query. Under pooling the rewriting pool maps canonical→tenant + * schemas at query time, resolving the correct tenant index. */ export const bm25IndexStore = new Map(); diff --git a/graphile/graphile-settings/src/plugins/meta-schema/cache.ts b/graphile/graphile-settings/src/plugins/meta-schema/cache.ts index 3e9f909b7a..916d6712b0 100644 --- a/graphile/graphile-settings/src/plugins/meta-schema/cache.ts +++ b/graphile/graphile-settings/src/plugins/meta-schema/cache.ts @@ -1,5 +1,41 @@ import type { TableMeta } from './types'; +/** + * Per-build table metadata, keyed by the PostGraphile `build` object. + * + * The server constructs multiple PostGraphile schemas concurrently in a single + * process, so the `init` and `GraphQLObjectType_fields` hooks of different + * builds interleave. Keying the collected metadata on the build object (instead + * of a single shared array) guarantees each schema's `_meta` field serves its + * OWN tables and never bleeds another concurrent build's data. + * + * A WeakMap is used (rather than a property on the build) because graphile-build + * freezes the build object before the `init` hook runs, so it cannot be mutated + * with an own property. The WeakMap also lets entries be garbage-collected once + * a build is discarded. + */ +const tablesMetaByBuild = new WeakMap(); + +export function setTablesMetaForBuild(build: object, tablesMeta: TableMeta[]): void { + tablesMetaByBuild.set(build, tablesMeta); +} + +export function getTablesMetaForBuild(build: object): TableMeta[] { + return tablesMetaByBuild.get(build) ?? []; +} + +/** + * Flat "last build wins" mirror of the most recently collected table metadata. + * + * Retained ONLY for out-of-process codegen consumers — graphile-schema's + * `buildIntrospectionJSON` and graphql/codegen's `DatabaseSchemaSource` — which + * read this (re-exported as `_cachedTablesMeta`) as a side-effect AFTER a single + * `buildSchemaSDL()` call and have no reference to the `build` object. Those + * paths build one schema at a time, so last-write-wins is safe for them. + * + * Do NOT read this from the concurrent schema-serving path (the `_meta` field): + * use getTablesMetaForBuild(build) instead, or concurrent builds will bleed. + */ export let cachedTablesMeta: TableMeta[] = []; export function getCachedTablesMeta(): TableMeta[] { diff --git a/graphile/graphile-settings/src/plugins/meta-schema/plugin.ts b/graphile/graphile-settings/src/plugins/meta-schema/plugin.ts index 2144c323fc..11d66ed0e7 100644 --- a/graphile/graphile-settings/src/plugins/meta-schema/plugin.ts +++ b/graphile/graphile-settings/src/plugins/meta-schema/plugin.ts @@ -1,5 +1,9 @@ import type { GraphileConfig } from 'graphile-config'; -import { getCachedTablesMeta, setCachedTablesMeta } from './cache'; +import { + getTablesMetaForBuild, + setCachedTablesMeta, + setTablesMetaForBuild, +} from './cache'; import { extendQueryWithMetaField } from './graphql-meta-field'; import { collectTablesMeta } from './table-meta-builder'; import type { MetaBuild } from './types'; @@ -18,16 +22,21 @@ export const MetaSchemaPlugin: GraphileConfig.Plugin = { hooks: { init(input, rawBuild) { const build = rawBuild as unknown as MetaBuild; - setCachedTablesMeta(collectTablesMeta(build)); + const tablesMeta = collectTablesMeta(build); + // Keyed by this build so the _meta field resolver reads its own tables, + // even when other builds run concurrently in the same process. + setTablesMetaForBuild(rawBuild, tablesMeta); + // Flat mirror for out-of-process codegen consumers (see cache.ts). + setCachedTablesMeta(tablesMeta); return input; }, - GraphQLObjectType_fields(rawFields, _rawBuild, rawContext) { + GraphQLObjectType_fields(rawFields, rawBuild, rawContext) { const context = rawContext as unknown as QueryTypeContext; if (context.Self?.name !== 'Query') return rawFields; return extendQueryWithMetaField( rawFields as unknown as Record, - getCachedTablesMeta(), + getTablesMetaForBuild(rawBuild), ) as typeof rawFields; }, }, diff --git a/graphile/graphile-settings/src/plugins/meta-schema/table-meta-builder.ts b/graphile/graphile-settings/src/plugins/meta-schema/table-meta-builder.ts index 0b98fccb26..e71ffb7375 100644 --- a/graphile/graphile-settings/src/plugins/meta-schema/table-meta-builder.ts +++ b/graphile/graphile-settings/src/plugins/meta-schema/table-meta-builder.ts @@ -115,11 +115,28 @@ function buildTableMeta( }; } +/** + * Strip the tenant hash prefix (`-<8hex>-`) from a physical + * schema name, returning the logical name; names without a hash segment are + * returned unchanged. Local duplicate of the server-side helper to avoid a + * cross-package dependency. + */ +function stripSchemaHashPrefix(name: string): string { + const match = /-[0-9a-f]{8}-/.exec(name); + if (!match) return name; + return name.slice(match.index + match[0].length); +} + export function collectTablesMeta(build: MetaBuild): TableMeta[] { const configuredSchemas = getConfiguredSchemas(build); const context = createBuildContext(build); const seenCodecs = new Set(); const tablesMeta: TableMeta[] = []; + // Shared (pooled) instances serve many tenants: reporting the build-time + // PHYSICAL schema name would leak the representative tenant's hashed schema + // identifier to every other tenant via _meta. When schema.constructivePooled + // is set, report the logical (hash-stripped) name instead. + const pooled = !!(build.options as any)?.constructivePooled; for (const resource of Object.values(build.input.pgRegistry.pgResources || {})) { if (!isTableResource(resource)) continue; @@ -137,7 +154,9 @@ export function collectTablesMeta(build: MetaBuild): TableMeta[] { continue; } - tablesMeta.push(buildTableMeta(resource, schemaName, context)); + tablesMeta.push( + buildTableMeta(resource, pooled ? stripSchemaHashPrefix(schemaName) : schemaName, context) + ); } return tablesMeta; diff --git a/graphql/server/package.json b/graphql/server/package.json index dbf1da23d4..96d45f7bf6 100644 --- a/graphql/server/package.json +++ b/graphql/server/package.json @@ -78,6 +78,7 @@ }, "devDependencies": { "@aws-sdk/client-s3": "^3.1052.0", + "@dataplan/pg": "1.0.3", "@types/cookie-parser": "^1.4.10", "@types/cors": "^2.8.17", "@types/express": "^5.0.6", @@ -88,6 +89,7 @@ "graphile-test": "workspace:*", "makage": "^0.3.0", "nodemon": "^3.1.14", + "pg-introspection": "1.0.1", "ts-node": "^10.9.2" } } diff --git a/graphql/server/src/diagnostics/__tests__/metrics-sampler.test.ts b/graphql/server/src/diagnostics/__tests__/metrics-sampler.test.ts new file mode 100644 index 0000000000..79c8cfadc2 --- /dev/null +++ b/graphql/server/src/diagnostics/__tests__/metrics-sampler.test.ts @@ -0,0 +1,127 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { collectMetricsSample, startMetricsSampler } from '../metrics-sampler'; + +const originalEnv = { ...process.env }; + +const tmpFile = (): string => + path.join(os.tmpdir(), `metrics-sampler-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}.jsonl`); + +const waitForFileContent = async (file: string, timeoutMs = 2_000): Promise => { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const content = await fs.promises.readFile(file, 'utf8'); + if (content.trim().length > 0) { + return content; + } + } catch { + // file not created yet + } + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error(`metrics file ${file} never received content`); +}; + +afterEach(() => { + process.env = { ...originalEnv }; +}); + +describe('collectMetricsSample', () => { + it('produces a well-formed, JSON-serializable sample', () => { + const sample = collectMetricsSample(); + + expect(typeof sample.ts).toBe('string'); + expect(Number.isNaN(Date.parse(sample.ts))).toBe(false); + expect(typeof sample.rss).toBe('number'); + expect(typeof sample.heapUsed).toBe('number'); + expect(typeof sample.heapTotal).toBe('number'); + expect(typeof sample.external).toBe('number'); + expect(typeof sample.heap_size_limit).toBe('number'); + expect(typeof sample.cache.size).toBe('number'); + expect(typeof sample.cache.keys).toBe('number'); + // counters merge cache counters + graphile counters + build queue depth + expect(typeof sample.counters.disposals).toBe('number'); + expect(typeof sample.counters.evictions.lru).toBe('number'); + expect(typeof sample.counters.builds).toBe('number'); + expect(typeof sample.counters.poolingAttaches).toBe('number'); + expect(typeof sample.counters.buildQueueDepth).toBe('number'); + expect(typeof sample.counters.rewritePool.rewrittenQueries).toBe('number'); + expect(typeof sample.counters.introspectionFilter.swaps).toBe('number'); + expect(typeof sample.counters.introspectionFilter.discoveries).toBe('number'); + expect(typeof sample.gc).toBe('object'); + + // Round-trips through JSON without loss. + expect(() => JSON.parse(JSON.stringify(sample))).not.toThrow(); + }); +}); + +describe('startMetricsSampler (disabled)', () => { + it('returns null and writes nothing when GRAPHILE_DEBUG_METRICS is unset', async () => { + delete process.env.GRAPHILE_DEBUG_METRICS; + const file = tmpFile(); + process.env.GRAPHILE_DEBUG_METRICS_FILE = file; + + const handle = startMetricsSampler(); + expect(handle).toBeNull(); + + await new Promise((resolve) => setTimeout(resolve, 60)); + expect(fs.existsSync(file)).toBe(false); + }); + + it('returns null when GRAPHILE_DEBUG_METRICS is a falsy string', () => { + process.env.GRAPHILE_DEBUG_METRICS = '0'; + expect(startMetricsSampler()).toBeNull(); + process.env.GRAPHILE_DEBUG_METRICS = 'false'; + expect(startMetricsSampler()).toBeNull(); + }); +}); + +describe('startMetricsSampler (enabled)', () => { + it("writes a parseable JSON line to the configured file when enabled ('1')", async () => { + const file = tmpFile(); + process.env.GRAPHILE_DEBUG_METRICS = '1'; + process.env.GRAPHILE_DEBUG_METRICS_FILE = file; + process.env.GRAPHILE_DEBUG_METRICS_INTERVAL_MS = '50'; + + const handle = startMetricsSampler(); + expect(handle).not.toBeNull(); + + try { + const content = await waitForFileContent(file); + const firstLine = content.trim().split('\n')[0]; + const parsed = JSON.parse(firstLine); + + expect(typeof parsed.ts).toBe('string'); + expect(typeof parsed.rss).toBe('number'); + expect(typeof parsed.heap_size_limit).toBe('number'); + expect(parsed.cache).toBeDefined(); + expect(typeof parsed.counters.disposals).toBe('number'); + expect(typeof parsed.counters.buildQueueDepth).toBe('number'); + expect(parsed.gc).toBeDefined(); + } finally { + handle?.stop(); + await fs.promises.rm(file, { force: true }); + } + }); + + it("also treats 'true' as enabled", async () => { + const file = tmpFile(); + process.env.GRAPHILE_DEBUG_METRICS = 'true'; + process.env.GRAPHILE_DEBUG_METRICS_FILE = file; + process.env.GRAPHILE_DEBUG_METRICS_INTERVAL_MS = '50'; + + const handle = startMetricsSampler(); + expect(handle).not.toBeNull(); + + try { + const content = await waitForFileContent(file); + expect(() => JSON.parse(content.trim().split('\n')[0])).not.toThrow(); + } finally { + handle?.stop(); + await fs.promises.rm(file, { force: true }); + } + }); +}); diff --git a/graphql/server/src/diagnostics/connection-error-guard.ts b/graphql/server/src/diagnostics/connection-error-guard.ts new file mode 100644 index 0000000000..2a846730ce --- /dev/null +++ b/graphql/server/src/diagnostics/connection-error-guard.ts @@ -0,0 +1,97 @@ +import { Logger } from '@pgpmjs/logger'; + +const log = new Logger('connection-guard'); + +/** + * Process-level guard for connection-class failures. + * + * A multi-tenant server holds many long-lived PostgreSQL connections (service + * pools, LISTEN/NOTIFY, introspection checkouts). When PostgreSQL restarts or + * crash-recovers (failover, OOM-killed backend, admin restart), every one of + * those sockets dies at once. Most owners handle their own errors (pg-cache + * pools, the schema listener), but a checkout that is mid-query when the + * server goes away can surface as an unhandled 'error' event on a raw pg + * Client — which kills the WHOLE process, turning a ~15s PostgreSQL recovery + * into a full outage for every tenant (observed live during scale validation: + * an introspection backend was OOM-killed, PG crash-recovered, and the node + * process exited on the orphaned client error). + * + * This guard absorbs ONLY connection-class errors (logged, counted); anything + * else preserves fatal semantics via process.exit(1). Pools reconnect on the + * next checkout, so the correct behavior after PG recovery is a brief burst + * of 5xx followed by self-healing — not process death. + * + * Disable with GRAPHILE_CONNECTION_GUARD=0. + */ +const CONN_ERROR_CODES = new Set([ + '57P01', // admin_shutdown (pg_terminate_backend / restart) + '57P02', // crash_shutdown + '57P03', // cannot_connect_now (recovery in progress) + '08006', // connection_failure + '08003', // connection_does_not_exist + '53300', // too_many_connections + 'ECONNRESET', + 'ECONNREFUSED', + 'EPIPE' +]); + +const CONN_ERROR_RE = + /connection terminated|terminating connection|server closed the connection|connection ended|client has encountered a connection error/i; + +export const isConnectionClassError = (err: unknown): boolean => { + if (!(err instanceof Error)) return false; + const code = (err as Error & { code?: string }).code; + if (code && CONN_ERROR_CODES.has(code)) return true; + return CONN_ERROR_RE.test(err.message || ''); +}; + +export interface ConnectionErrorGuardCounters { + absorbedExceptions: number; + absorbedRejections: number; +} + +const counters: ConnectionErrorGuardCounters = { + absorbedExceptions: 0, + absorbedRejections: 0 +}; + +export const getConnectionErrorGuardCounters = (): ConnectionErrorGuardCounters => ({ ...counters }); + +let installed = false; + +export function installConnectionErrorGuard(): void { + if (installed) return; + if (process.env.GRAPHILE_CONNECTION_GUARD === '0') return; + installed = true; + + process.on('uncaughtException', (err) => { + if (isConnectionClassError(err)) { + counters.absorbedExceptions++; + log.error( + `Absorbed connection-class exception (PG restart/failover?): ${(err as Error).message} ` + + `[absorbed=${counters.absorbedExceptions}] — pools reconnect on next checkout` + ); + return; + } + // Preserve fatal semantics for everything else. + // eslint-disable-next-line no-console + console.error('Uncaught exception (fatal):', err); + process.exit(1); + }); + + process.on('unhandledRejection', (reason) => { + if (isConnectionClassError(reason)) { + counters.absorbedRejections++; + log.error( + `Absorbed connection-class rejection (PG restart/failover?): ${(reason as Error).message} ` + + `[absorbed=${counters.absorbedRejections}]` + ); + return; + } + // eslint-disable-next-line no-console + console.error('Unhandled rejection (fatal):', reason); + process.exit(1); + }); + + log.info('connection-error guard installed (disable with GRAPHILE_CONNECTION_GUARD=0)'); +} diff --git a/graphql/server/src/diagnostics/metrics-sampler.ts b/graphql/server/src/diagnostics/metrics-sampler.ts new file mode 100644 index 0000000000..dceab46552 --- /dev/null +++ b/graphql/server/src/diagnostics/metrics-sampler.ts @@ -0,0 +1,219 @@ +import fs from 'node:fs'; +import { constants as perfConstants, PerformanceObserver } from 'node:perf_hooks'; +import v8 from 'node:v8'; +import { Logger } from '@pgpmjs/logger'; +import { getCacheCounters, getCacheStats } from 'graphile-cache'; + +import { getBuildQueueDepth, getGraphileCounters } from '../middleware/graphile'; +import { getIntrospectionFilterCounters } from '../middleware/introspection-filter'; +import { getRewritePoolCounters } from '../middleware/rewrite-pool'; +import { getConnectionErrorGuardCounters } from './connection-error-guard'; + +const log = new Logger('metrics-sampler'); + +const DEFAULT_INTERVAL_MS = 10_000; +const DEFAULT_METRICS_FILE = './metrics.jsonl'; + +// ============================================================================= +// GC tracking (perf_hooks) +// +// A single process-global PerformanceObserver accumulates GC pause time and counts +// by kind. Registered lazily the first time the sampler starts (so there is zero +// overhead when GRAPHILE_DEBUG_METRICS is off) and torn down when the sampler stops. +// ============================================================================= + +interface GcKindStat { + count: number; + totalPauseMs: number; +} + +type GcStats = Record; + +const gcStats: GcStats = {}; +let gcObserver: PerformanceObserver | null = null; + +// Map perf_hooks GC kind flags to stable, human-readable names for the metrics line. +const GC_KIND_NAMES: Record = { + [perfConstants.NODE_PERFORMANCE_GC_MINOR]: 'minor', + [perfConstants.NODE_PERFORMANCE_GC_MAJOR]: 'major', + [perfConstants.NODE_PERFORMANCE_GC_INCREMENTAL]: 'incremental', + [perfConstants.NODE_PERFORMANCE_GC_WEAKCB]: 'weakcb' +}; + +const startGcObserver = (): void => { + if (gcObserver) { + return; + } + try { + const observer = new PerformanceObserver((list) => { + for (const entry of list.getEntries()) { + // `detail.kind` is the modern accessor; fall back to the deprecated top-level + // `kind` for older Node. Guard: unknown kinds are bucketed by their numeric flag. + const kind = + (entry as { detail?: { kind?: number } }).detail?.kind ?? + (entry as unknown as { kind?: number }).kind; + const name = + kind != null && GC_KIND_NAMES[kind] ? GC_KIND_NAMES[kind] : `kind_${kind ?? 'unknown'}`; + const stat = gcStats[name] ?? (gcStats[name] = { count: 0, totalPauseMs: 0 }); + stat.count += 1; + stat.totalPauseMs += entry.duration; + } + }); + observer.observe({ entryTypes: ['gc'] }); + gcObserver = observer; + } catch (err) { + // GC observation is best-effort; a platform without 'gc' entries just omits GC data. + gcObserver = null; + log.warn('GC PerformanceObserver unavailable; metrics will omit GC data', err); + } +}; + +const stopGcObserver = (): void => { + if (gcObserver) { + try { + gcObserver.disconnect(); + } catch { + // ignore — teardown is best-effort + } + gcObserver = null; + } +}; + +const snapshotGcStats = (): GcStats => { + const out: GcStats = {}; + for (const [name, stat] of Object.entries(gcStats)) { + out[name] = { count: stat.count, totalPauseMs: stat.totalPauseMs }; + } + return out; +}; + +// ============================================================================= +// Env parsing +// ============================================================================= + +const isEnabled = (): boolean => { + const raw = process.env.GRAPHILE_DEBUG_METRICS; + if (!raw) { + return false; + } + const normalized = raw.trim().toLowerCase(); + return normalized === '1' || normalized === 'true'; +}; + +const getIntervalMs = (): number => { + const raw = process.env.GRAPHILE_DEBUG_METRICS_INTERVAL_MS; + const parsed = raw ? Number.parseInt(raw, 10) : DEFAULT_INTERVAL_MS; + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_INTERVAL_MS; +}; + +const getMetricsFile = (): string => + process.env.GRAPHILE_DEBUG_METRICS_FILE || DEFAULT_METRICS_FILE; + +// ============================================================================= +// Sample +// ============================================================================= + +export interface MetricsSample { + ts: string; + rss: number; + heapUsed: number; + heapTotal: number; + external: number; + heap_size_limit: number; + cache: { + size: number; + keys: number; + }; + counters: ReturnType & + ReturnType & { + buildQueueDepth: number; + connGuard: ReturnType; + rewritePool: ReturnType; + introspectionFilter: ReturnType; + }; + gc: GcStats; +} + +/** + * Collect a single metrics sample. Cheap: a memoryUsage() call, a v8 heap-stats read, + * cache size/key-count, and integer counter snapshots. Exposed for tests and future + * promotion to a /metrics endpoint. + */ +export const collectMetricsSample = (): MetricsSample => { + const mem = process.memoryUsage(); + const cacheStats = getCacheStats(); + return { + ts: new Date().toISOString(), + rss: mem.rss, + heapUsed: mem.heapUsed, + heapTotal: mem.heapTotal, + external: mem.external, + heap_size_limit: v8.getHeapStatistics().heap_size_limit, + cache: { + size: cacheStats.size, + keys: cacheStats.keys.length + }, + counters: { + ...getCacheCounters(), + ...getGraphileCounters(), + buildQueueDepth: getBuildQueueDepth(), + connGuard: getConnectionErrorGuardCounters(), + rewritePool: getRewritePoolCounters(), + introspectionFilter: getIntrospectionFilterCounters() + }, + gc: snapshotGcStats() + }; +}; + +// ============================================================================= +// Sampler +// ============================================================================= + +export interface MetricsSamplerHandle { + stop(): void; +} + +/** + * Start the in-process metrics sampler. + * + * When GRAPHILE_DEBUG_METRICS is '1' or 'true', appends one JSON line per + * GRAPHILE_DEBUG_METRICS_INTERVAL_MS (default 10000ms) to GRAPHILE_DEBUG_METRICS_FILE + * (default ./metrics.jsonl). The interval timer is unref'd so it never keeps the + * process alive, and file writes swallow errors so metrics can never affect the server. + * + * Returns null (a true no-op, zero overhead) when disabled. + */ +export const startMetricsSampler = (): MetricsSamplerHandle | null => { + if (!isEnabled()) { + return null; + } + + startGcObserver(); + + const intervalMs = getIntervalMs(); + const file = getMetricsFile(); + + const writeSample = (): void => { + const line = `${JSON.stringify(collectMetricsSample())}\n`; + // Fire-and-forget; a failed metrics write must never impact request handling. + fs.appendFile(file, line, () => { + // swallow errors + }); + }; + + // Emit an immediate baseline sample so the file has a line without waiting a full + // interval, then sample periodically. + writeSample(); + + const timer = setInterval(writeSample, intervalMs); + timer.unref(); + + log.info(`Metrics sampler writing every ${intervalMs}ms to ${file}`); + + return { + stop(): void { + clearInterval(timer); + stopGcObserver(); + } + }; +}; diff --git a/graphql/server/src/middleware/__tests__/blueprint.test.ts b/graphql/server/src/middleware/__tests__/blueprint.test.ts new file mode 100644 index 0000000000..03b46e496a --- /dev/null +++ b/graphql/server/src/middleware/__tests__/blueprint.test.ts @@ -0,0 +1,131 @@ +import { + computeBlueprintKey, + stripSchemaHashPrefix +} from '../blueprint'; + +describe('stripSchemaHashPrefix', () => { + it('strips a hashed tenant prefix down to the logical schema', () => { + expect(stripSchemaHashPrefix('marketplace-db-tenant1-5e6b13b2-app-public')).toBe( + 'app-public' + ); + }); + + it('returns control-plane / non-hashed schema names unchanged', () => { + expect(stripSchemaHashPrefix('services_public')).toBe('services_public'); + expect(stripSchemaHashPrefix('metaschema_public')).toBe('metaschema_public'); + expect(stripSchemaHashPrefix('app-public')).toBe('app-public'); + }); + + it('handles multi-dash database names in the prefix', () => { + expect(stripSchemaHashPrefix('my-cool-db-name-deadbeef-auth-private')).toBe( + 'auth-private' + ); + expect(stripSchemaHashPrefix('app-5e6b13b2-public')).toBe('public'); + }); + + it('only strips through the FIRST 8-hex segment, keeping later ones intact', () => { + expect( + stripSchemaHashPrefix('marketplace-db-tenant1-5e6b13b2-app-deadbeef-zone') + ).toBe('app-deadbeef-zone'); + }); + + it('does not treat non-hex or wrong-length segments as a hash', () => { + // 'tenant11' is 8 chars but not all hex; 'abcdef1' is 7 hex chars. + expect(stripSchemaHashPrefix('marketplace-tenant11-app-public')).toBe( + 'marketplace-tenant11-app-public' + ); + expect(stripSchemaHashPrefix('marketplace-abcdef1-app-public')).toBe( + 'marketplace-abcdef1-app-public' + ); + }); +}); + +describe('computeBlueprintKey', () => { + const base = { + logicalSchemas: ['app-public', 'auth-public'], + shapeFingerprint: 'fingerprint-a', + flags: { a: 1, b: 2 } as Record, + apiName: 'customer-api', + mode: 'domain-lookup' + }; + + it('produces a stable "bp:"-prefixed sha256 hex key', () => { + const key = computeBlueprintKey(base); + expect(key).toMatch(/^bp:[0-9a-f]{64}$/); + expect(computeBlueprintKey(base)).toBe(key); + }); + + it('is stable across the key order of flags', () => { + const key1 = computeBlueprintKey({ ...base, flags: { a: 1, b: 2 } }); + const key2 = computeBlueprintKey({ ...base, flags: { b: 2, a: 1 } }); + expect(key1).toBe(key2); + }); + + it('is stable across the order of logical schemas', () => { + const key1 = computeBlueprintKey({ + ...base, + logicalSchemas: ['app-public', 'auth-public'] + }); + const key2 = computeBlueprintKey({ + ...base, + logicalSchemas: ['auth-public', 'app-public'] + }); + expect(key1).toBe(key2); + }); + + it('differs when the shape fingerprint differs', () => { + const key1 = computeBlueprintKey({ ...base, shapeFingerprint: 'fingerprint-a' }); + const key2 = computeBlueprintKey({ ...base, shapeFingerprint: 'fingerprint-b' }); + expect(key1).not.toBe(key2); + }); + + it('differs when flag values differ', () => { + const key1 = computeBlueprintKey({ ...base, flags: { a: 1, b: 2 } }); + const key2 = computeBlueprintKey({ ...base, flags: { a: 1, b: 3 } }); + expect(key1).not.toBe(key2); + }); + + it('treats undefined flags like an empty flag set', () => { + const key1 = computeBlueprintKey({ ...base, flags: undefined }); + const key2 = computeBlueprintKey({ ...base, flags: {} }); + expect(key1).toBe(key2); + }); + + it('differs when the mode or api name differs', () => { + expect(computeBlueprintKey({ ...base, mode: 'domain-lookup' })).not.toBe( + computeBlueprintKey({ ...base, mode: 'api-name-header' }) + ); + expect(computeBlueprintKey({ ...base, apiName: 'customer-api' })).not.toBe( + computeBlueprintKey({ ...base, apiName: 'other-api' }) + ); + }); + + it('treats null and empty api name identically', () => { + expect(computeBlueprintKey({ ...base, apiName: null })).toBe( + computeBlueprintKey({ ...base, apiName: '' }) + ); + }); +}); + +describe('computeBlueprintKey dbname isolation (W3 fix)', () => { + const { computeBlueprintKey } = require('../blueprint'); + const base = { + logicalSchemas: ['app-public'], + shapeFingerprint: 'f'.repeat(64), + flags: undefined as Record | undefined, + apiName: 'api', + mode: 'public' + }; + + it('same-shape tenants in DIFFERENT physical databases get DIFFERENT keys', () => { + const a = computeBlueprintKey({ ...base, dbname: 'db_a' }); + const b = computeBlueprintKey({ ...base, dbname: 'db_b' }); + expect(a).not.toBe(b); + }); + + it('same dbname yields a stable key', () => { + expect(computeBlueprintKey({ ...base, dbname: 'db_a' })).toBe( + computeBlueprintKey({ ...base, dbname: 'db_a' }) + ); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/build-timeout.test.ts b/graphql/server/src/middleware/__tests__/build-timeout.test.ts new file mode 100644 index 0000000000..742c2df013 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/build-timeout.test.ts @@ -0,0 +1,57 @@ +import { BuildRefusedError, raceBuildAgainstTimeout } from '../graphile'; + +describe('BuildRefusedError', () => { + it('carries the 503 mapping contract (code + name) and the pressure ratio', () => { + const err = new BuildRefusedError(0.934); + expect(err.code).toBe('SERVICE_OVERLOADED'); + expect(err.name).toBe('BuildRefusedError'); + expect(err.message).toContain('0.93'); + expect(err).toBeInstanceOf(Error); + }); +}); + +describe('raceBuildAgainstTimeout', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('returns the built value and disarms the timer when the build wins', async () => { + const instance = { handler: 'built' }; + const result = await raceBuildAgainstTimeout(Promise.resolve(instance), 180_000); + expect(result).toBe(instance); + // The load-bearing assertion: an armed timer here retains the built instance + // through the race reaction chain for the full timeout (the evict-churn OOM). + expect(jest.getTimerCount()).toBe(0); + }); + + it('returns null when the wait expires, leaving no armed timer', async () => { + const never = new Promise(() => {}); + const race = raceBuildAgainstTimeout(never, 180_000); + jest.advanceTimersByTime(180_000); + await expect(race).resolves.toBeNull(); + expect(jest.getTimerCount()).toBe(0); + }); + + it('disarms the timer when the build rejects', async () => { + const boom = new Error('build failed'); + await expect(raceBuildAgainstTimeout(Promise.reject(boom), 180_000)).rejects.toBe(boom); + expect(jest.getTimerCount()).toBe(0); + }); + + it('propagates a late build failure to no one after a timeout (no unhandled timer state)', async () => { + let rejectBuild: (err: Error) => void; + const build = new Promise((_resolve, reject) => { + rejectBuild = reject; + }); + build.catch(() => { /* the dispatcher attaches its own handler; mirror that */ }); + const race = raceBuildAgainstTimeout(build, 1_000); + jest.advanceTimersByTime(1_000); + await expect(race).resolves.toBeNull(); + expect(jest.getTimerCount()).toBe(0); + rejectBuild!(new Error('late failure')); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-pooling.test.ts b/graphql/server/src/middleware/__tests__/graphile-pooling.test.ts new file mode 100644 index 0000000000..669586f65f --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-pooling.test.ts @@ -0,0 +1,224 @@ +import type { Pool } from 'pg'; + +import type { ApiStructure } from '../../types'; +import { isBlueprintPoolingEnabled } from '../blueprint'; +import { clearPoolDecisions, computePoolDecision, resolvePoolDecision } from '../pooling-decision'; + +type QueryResult = { rows: Array> }; + +const makeApi = (overrides: Partial = {}): ApiStructure => + ({ + dbname: 'constructive', + anonRole: 'anon', + roleName: 'authenticated', + schema: ['marketplace-db-tenant1-5e6b13b2-app-public'], + apiModules: [], + ...overrides + } as ApiStructure); + +// A mock Pool whose query() resolves queued results in call order. computePoolDecision +// issues a single shape-fingerprint query. +const makePool = (results: QueryResult[]): { pool: Pool; query: jest.Mock } => { + const query = jest.fn(); + results.forEach((r) => query.mockResolvedValueOnce(r)); + return { pool: { query } as unknown as Pool, query }; +}; + +describe('computePoolDecision', () => { + afterEach(() => { + clearPoolDecisions(); + jest.clearAllMocks(); + }); + + it('falls back (pooling=false, key=svc_key) when realtime is enabled, without touching the DB', async () => { + const { pool, query } = makePool([]); + const api = makeApi({ databaseSettings: { enableRealtime: true } as ApiStructure['databaseSettings'] }); + + const decision = await computePoolDecision('svc-1', api, pool); + + expect(decision).toEqual({ key: 'svc-1', pooling: false }); + expect(query).not.toHaveBeenCalled(); + }); + + it('falls back when the API exposes no schemas, without touching the DB', async () => { + const { pool, query } = makePool([]); + const api = makeApi({ schema: [] }); + + const decision = await computePoolDecision('svc-2', api, pool); + + expect(decision).toEqual({ key: 'svc-2', pooling: false }); + expect(query).not.toHaveBeenCalled(); + }); + + it('pools a clean single-tenant shape and returns a stable bp: key', async () => { + const rows = [ + { nspname: 'marketplace-db-tenant1-5e6b13b2-app-public', relname: 'products' }, + { nspname: 'marketplace-db-tenant1-5e6b13b2-app-public', relname: 'orders' } + ]; + const { pool, query } = makePool([{ rows }, { rows: [] }]); + const api = makeApi(); + + const decision = await computePoolDecision('svc-3', api, pool); + + expect(decision.pooling).toBe(true); + expect(decision.key).toMatch(/^bp:[0-9a-f]{64}$/); + expect(query).toHaveBeenCalledTimes(1); // single catalog scan feeds the fingerprint + }); + + it('produces the SAME bp: key for two different physical tenants of the same shape', async () => { + const shapeRows = (hash: string) => [ + { nspname: `marketplace-db-tenant-${hash}-app-public`, relname: 'orders' }, + { nspname: `marketplace-db-tenant-${hash}-app-public`, relname: 'products' } + ]; + + const t1 = makePool([{ rows: shapeRows('5e6b13b2') }, { rows: [] }]); + const d1 = await computePoolDecision( + 'svc-t1', + makeApi({ schema: ['marketplace-db-tenant-5e6b13b2-app-public'] }), + t1.pool + ); + + const t2 = makePool([{ rows: shapeRows('deadbeef') }, { rows: [] }]); + const d2 = await computePoolDecision( + 'svc-t2', + makeApi({ schema: ['marketplace-db-tenant-deadbeef-app-public'] }), + t2.pool + ); + + expect(d1.pooling).toBe(true); + expect(d2.pooling).toBe(true); + expect(d1.key).toBe(d2.key); // same logical shape → shared blueprint instance + }); + + it('threads api.logicalSchemas into the key, deriving it from physical schema only when absent', async () => { + // Identical fingerprint rows across all three calls: only logicalSchemas varies, + // so any key difference is attributable solely to the logicalSchemas input. + const fpRows = [{ nspname: 'shop-5e6b13b2-app-public', relname: 'orders' }]; + + // Explicit logicalSchemas that DIFFER from the stripped physical name. + const a = makePool([{ rows: fpRows }, { rows: [] }]); + const dExplicitDiff = await computePoolDecision( + 'svc-a', + makeApi({ schema: ['shop-5e6b13b2-app-public'], logicalSchemas: ['custom-logical'] }), + a.pool + ); + + // No logicalSchemas → derived from schema.map(stripSchemaHashPrefix) = ['app-public']. + const b = makePool([{ rows: fpRows }, { rows: [] }]); + const dDerived = await computePoolDecision( + 'svc-b', + makeApi({ schema: ['shop-5e6b13b2-app-public'], logicalSchemas: undefined }), + b.pool + ); + + // Explicit logicalSchemas EQUAL to the derived value. + const c = makePool([{ rows: fpRows }, { rows: [] }]); + const dExplicitSame = await computePoolDecision( + 'svc-c', + makeApi({ schema: ['shop-5e6b13b2-app-public'], logicalSchemas: ['app-public'] }), + c.pool + ); + + expect(dExplicitDiff.key).not.toBe(dDerived.key); // provided logicalSchemas is honored + expect(dExplicitSame.key).toBe(dDerived.key); // ?? fallback derivation matches + }); + + it('relation-name collisions across schemas are poolable under qualified SQL', async () => { + const rows = [ + { nspname: 'shop-5e6b13b2-auth-public', relname: 'identity_providers' }, + { nspname: 'shop-5e6b13b2-auth-private', relname: 'identity_providers' } + ]; + const { pool } = makePool([{ rows }]); + const api = makeApi({ schema: ['shop-5e6b13b2-auth-public', 'shop-5e6b13b2-auth-private'] }); + + const decision = await computePoolDecision('svc-4', api, pool); + + expect(decision.pooling).toBe(true); + expect(decision.key).toMatch(/^bp:[0-9a-f]{64}$/); + }); + + it('falls back (never throws) when a catalog probe rejects', async () => { + const query = jest.fn().mockRejectedValue(new Error('connection reset')); + const api = makeApi(); + + const decision = await computePoolDecision('svc-5', api, { query } as unknown as Pool); + + expect(decision).toEqual({ key: 'svc-5', pooling: false, transient: true }); + }); +}); + +describe('resolvePoolDecision caching', () => { + afterEach(() => { + clearPoolDecisions(); + jest.clearAllMocks(); + }); + + it('memoizes the decision per svc_key (one probe pair) until cleared', async () => { + const { pool, query } = makePool([ + { rows: [{ nspname: 'shop-5e6b13b2-app-public', relname: 'orders' }] }, + { rows: [] } + ]); + const api = makeApi({ schema: ['shop-5e6b13b2-app-public'] }); + + const first = await resolvePoolDecision('svc-cache', api, pool); + const second = await resolvePoolDecision('svc-cache', api, pool); + + expect(first).toBe(second); // same cached object identity + expect(first.pooling).toBe(true); + expect(query).toHaveBeenCalledTimes(1); // NOT 2 — second call served from cache + + // clearPoolDecisions() forces the next request to re-probe. + clearPoolDecisions(); + query.mockResolvedValueOnce({ rows: [{ nspname: 'shop-5e6b13b2-app-public', relname: 'orders' }] }); + await resolvePoolDecision('svc-cache', api, pool); + expect(query).toHaveBeenCalledTimes(2); + }); +}); + +describe('blueprint pooling flag gate (flag-off ⇒ key stays svc_key in the dispatcher)', () => { + const original = process.env.GRAPHILE_BLUEPRINT_POOLING; + + afterEach(() => { + if (original === undefined) delete process.env.GRAPHILE_BLUEPRINT_POOLING; + else process.env.GRAPHILE_BLUEPRINT_POOLING = original; + }); + + it('is disabled by default — the dispatcher never runs a decision and uses req.svc_key', () => { + delete process.env.GRAPHILE_BLUEPRINT_POOLING; + expect(isBlueprintPoolingEnabled()).toBe(false); + }); + + it("enables only on '1' or 'true'", () => { + process.env.GRAPHILE_BLUEPRINT_POOLING = '1'; + expect(isBlueprintPoolingEnabled()).toBe(true); + process.env.GRAPHILE_BLUEPRINT_POOLING = 'true'; + expect(isBlueprintPoolingEnabled()).toBe(true); + process.env.GRAPHILE_BLUEPRINT_POOLING = 'yes'; + expect(isBlueprintPoolingEnabled()).toBe(false); + }); +}); + +describe('transient probe failures are not memoized (W3 fix)', () => { + afterEach(() => { + clearPoolDecisions(); + jest.clearAllMocks(); + }); + + it('re-probes on the next request after a thrown catalog probe', async () => { + const query = jest.fn(); + query.mockRejectedValueOnce(new Error('connection reset')); + query.mockResolvedValueOnce({ + rows: [{ nspname: 'shop-5e6b13b2-app-public', relname: 'orders' }] + }); + const pool = { query } as unknown as Pool; + const api = makeApi({ schema: ['shop-5e6b13b2-app-public'] }); + + const first = await resolvePoolDecision('svc-transient', api, pool); + expect(first.pooling).toBe(false); + expect(first.transient).toBe(true); + + const second = await resolvePoolDecision('svc-transient', api, pool); + expect(second.pooling).toBe(true); // re-probed and now poolable + expect(query).toHaveBeenCalledTimes(2); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-recoalesce.test.ts b/graphql/server/src/middleware/__tests__/graphile-recoalesce.test.ts new file mode 100644 index 0000000000..7ea244f2f1 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-recoalesce.test.ts @@ -0,0 +1,79 @@ +import type { NextFunction, Request, Response } from 'express'; +import type { GraphileCacheEntry } from 'graphile-cache'; + +import { BuildRefusedError, clearInFlightMap, graphile, setInFlightForTest } from '../graphile'; + +/** + * Regression test for the Phase C re-coalesce load-shed hole. + * + * Contract (commit 46636d4b4): BuildRefusedError maps to 503 SERVICE_OVERLOADED + + * Retry-After "for the requester and every coalesced waiter." A re-coalescing + * waiter awaits a fresh in-flight build at the Phase C re-coalesce await. Pre-fix + * that await was unguarded, so a BuildRefusedError surfacing there escaped to the + * outer catch and became a generic 500 INTERNAL_ERROR with no Retry-After — + * contradicting the contract on the exact memory-pressure path the error exists for. + * + * The re-coalesce await is reached before the request-time pressure gate, so the + * real (empty) cache naturally misses and no build machinery runs; seeding the + * in-flight map with a rejecting promise drives the path deterministically. + */ +describe('graphile dispatcher — Phase C re-coalesce refusal mapping', () => { + const key = 'svc-recoalesce-test'; + + const makeRes = () => { + const headers: Record = {}; + const res: any = { headersSent: false }; + res.setHeader = jest.fn((name: string, value: string) => { + headers[name] = value; + }); + res.status = jest.fn((code: number) => { + res.statusCode = code; + return res; + }); + res.json = jest.fn((payload: unknown) => { + res.body = payload; + return res; + }); + return { res: res as Response, headers }; + }; + + const makeReq = (): Request => + ({ + requestId: 'test-req', + svc_key: key, + api: { + dbname: 'constructive', + anonRole: 'anon', + roleName: 'authenticated', + schema: ['app-public'] + } + } as unknown as Request); + + afterEach(() => { + clearInFlightMap(); + }); + + it('maps a BuildRefusedError on the re-coalesce await to 503 SERVICE_OVERLOADED + Retry-After', async () => { + // A rejecting in-flight promise that is NOT self-deleting, so it survives from + // Phase B (which falls through on the rejection) into the Phase C re-coalesce + // read — mirroring a fresh owner's build that refuses under late heap pressure. + const refused: Promise = Promise.reject(new BuildRefusedError(0.98)); + refused.catch(() => { /* guard: both awaits inside the dispatcher handle it */ }); + setInFlightForTest(key, refused); + + const handler = graphile({} as never); + const { res, headers } = makeRes(); + const next = jest.fn() as unknown as NextFunction; + + await handler(makeReq(), res, next); + + expect((res.status as jest.Mock)).toHaveBeenCalledWith(503); + expect(headers['Retry-After']).toBe('15'); + expect((res.json as jest.Mock).mock.calls[0][0]).toEqual({ + error: { code: 'SERVICE_OVERLOADED', message: 'Server is at critical memory pressure; retry shortly' } + }); + // The pre-fix defect: a generic 500 with no Retry-After from the outer catch. + expect((res.status as jest.Mock)).not.toHaveBeenCalledWith(500); + expect(next).not.toHaveBeenCalled(); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/introspection-filter.test.ts b/graphql/server/src/middleware/__tests__/introspection-filter.test.ts new file mode 100644 index 0000000000..0c13f4a96d --- /dev/null +++ b/graphql/server/src/middleware/__tests__/introspection-filter.test.ts @@ -0,0 +1,349 @@ +import { makeIntrospectionQuery } from 'pg-introspection'; + +import { + buildFilteredIntrospectionQuery, + createIntrospectionFilterCounters, + createIntrospectionFilterPool, + createIntrospectionInterceptor, + getIntrospectionFilterCounters, + isIntrospectionFilterEnabled, + isIntrospectionQuery +} from '../introspection-filter'; +import { + createRewriteCounters, + createRewritingPool, + POOL_SCHEMAS_GUC +} from '../rewrite-pool'; + +// The exact runtime gate substring (single backslash before the underscore). +// Derived here so tests fail loudly if the installed package ever drifts. +const GATE_SUBSTRING = + "in (select namespaces._id from namespaces where nspname <> 'information_schema' and nspname not like 'pg\\_%')"; + +const STOCK = makeIntrospectionQuery(); + +const countOccurrences = (haystack: string, needle: string): number => + haystack.split(needle).length - 1; + +const originalEnv = { ...process.env }; +afterEach(() => { + process.env = { ...originalEnv }; +}); + +// ============================================================================= +// Env flag +// ============================================================================= + +describe('isIntrospectionFilterEnabled', () => { + it("is true only for '1' or 'true'", () => { + process.env.GRAPHILE_INTROSPECTION_FILTER = '1'; + expect(isIntrospectionFilterEnabled()).toBe(true); + process.env.GRAPHILE_INTROSPECTION_FILTER = 'true'; + expect(isIntrospectionFilterEnabled()).toBe(true); + process.env.GRAPHILE_INTROSPECTION_FILTER = '0'; + expect(isIntrospectionFilterEnabled()).toBe(false); + delete process.env.GRAPHILE_INTROSPECTION_FILTER; + expect(isIntrospectionFilterEnabled()).toBe(false); + }); +}); + +// ============================================================================= +// 1. THE PINNING TEST +// ============================================================================= + +describe('buildFilteredIntrospectionQuery — pinned against installed pg-introspection', () => { + it('recognizes the stock query', () => { + expect(isIntrospectionQuery(STOCK)).toBe(true); + // The stock text carries the gate exactly four times. + expect(countOccurrences(STOCK, GATE_SUBSTRING)).toBe(4); + }); + + it('rewrites all four gates and preserves the pg_catalog type branch', () => { + const filtered = buildFilteredIntrospectionQuery(STOCK, ['t1-aaaa-app_public', 'public']); + expect(filtered).not.toBeNull(); + // The injected `= any (array[...])` gate appears exactly four times... + expect(countOccurrences(filtered as string, 'nspname = any (array[')).toBe(4); + // ...and zero stock gates remain. + expect(countOccurrences(filtered as string, GATE_SUBSTRING)).toBe(0); + // pg_catalog types survive via the intact OR branch. + expect((filtered as string).includes("'pg_catalog'::regnamespace")).toBe(true); + // Sorted, escaped literals present. + expect((filtered as string).includes("array['public', 't1-aaaa-app_public']::text[]")).toBe(true); + }); +}); + +// ============================================================================= +// 2. Gate mismatch +// ============================================================================= + +describe('buildFilteredIntrospectionQuery — gate mismatch', () => { + it('returns null when the gate does not appear exactly four times', () => { + // Remove one of the four gate occurrences -> three remain. + const doctored = STOCK.replace(GATE_SUBSTRING, 'in (select namespaces._id from namespaces where true)'); + expect(countOccurrences(doctored, GATE_SUBSTRING)).toBe(3); + expect(buildFilteredIntrospectionQuery(doctored, ['public'])).toBeNull(); + }); +}); + +// ============================================================================= +// 3. Literal escaping / defensive drops +// ============================================================================= + +describe('buildFilteredIntrospectionQuery — literal escaping', () => { + it("doubles single quotes and drops pg_ / information_schema names", () => { + const filtered = buildFilteredIntrospectionQuery(STOCK, [ + "o'brien", + 'pg_temp_1', + 'information_schema', + 'public' + ]); + expect(filtered).not.toBeNull(); + const text = filtered as string; + expect(text.includes("'o''brien'")).toBe(true); + expect(text.includes("'public'")).toBe(true); + expect(text.includes("'pg_temp_1'")).toBe(false); + // 'information_schema' appears in the stock text elsewhere, but never as an + // injected array literal. + expect(text.includes("array['information_schema'")).toBe(false); + expect(text.includes(", 'information_schema'")).toBe(false); + }); +}); + +// ============================================================================= +// 4. isIntrospectionQuery negatives +// ============================================================================= + +describe('isIntrospectionQuery — negatives', () => { + it('rejects the adaptor settings query, DEALLOCATE, and a plain SELECT', () => { + const settings = + 'select set_config(el->>0, el->>1, true) from json_array_elements($1::json) el'; + expect(isIntrospectionQuery(settings)).toBe(false); + expect(isIntrospectionQuery('deallocate all')).toBe(false); + expect(isIntrospectionQuery('select introspection_version from t')).toBe(false); + }); + + it('rejects a long query that merely mentions introspection_version', () => { + const long = 'select ' + 'a,'.repeat(3000) + " 'introspection_version' from t"; + expect(long.length).toBeGreaterThan(5000); + expect(isIntrospectionQuery(long)).toBe(false); + }); +}); + +// ============================================================================= +// 5. Closure loop + interceptor +// ============================================================================= + +const scriptedClient = (rounds: any[][]) => { + let i = 0; + return { + query: jest.fn((_sql: string, _values?: any[]) => { + const rows = rounds[i] ?? []; + i += 1; + return Promise.resolve({ rows }); + }) + }; +}; + +describe('createIntrospectionInterceptor — closure loop', () => { + beforeEach(() => { + process.env.GRAPHILE_INTROSPECTION_FILTER = '1'; + }); + + it('accumulates served ∪ public ∪ discovered, then filters', async () => { + const counters = createIntrospectionFilterCounters(); + const interceptor = createIntrospectionInterceptor({ servedSchemas: ['t1'], counters }); + // Round 1 discovers services_public; round 2 discovers nothing. + const client = scriptedClient([[{ nspname: 'services_public' }], []]); + + const swapped = await (interceptor(STOCK, client) as Promise); + + // keep set = public, t1, services_public + expect(swapped.includes("'public'")).toBe(true); + expect(swapped.includes("'t1'")).toBe(true); + expect(swapped.includes("'services_public'")).toBe(true); + expect(countOccurrences(swapped, GATE_SUBSTRING)).toBe(0); + + expect(counters.discoveries).toBe(1); + expect(counters.swaps).toBe(1); + expect(counters.closureTruncations).toBe(0); + expect(counters.keepNamespaceCount).toBe(3); + // Two closure rounds ran (the second returned nothing, ending the loop). + expect(client.query).toHaveBeenCalledTimes(2); + }); + + it('memoizes: a second call reuses the first discovery', async () => { + const counters = createIntrospectionFilterCounters(); + const interceptor = createIntrospectionInterceptor({ servedSchemas: ['t1'], counters }); + const client = scriptedClient([[], []]); + + const a = await (interceptor(STOCK, client) as Promise); + const b = await (interceptor(STOCK, client) as Promise); + expect(a).toBe(b); + expect(counters.discoveries).toBe(1); + expect(client.query).toHaveBeenCalledTimes(1); // one round (empty) only + }); + + it('rejects when the discovery query throws', async () => { + const counters = createIntrospectionFilterCounters(); + const interceptor = createIntrospectionInterceptor({ servedSchemas: ['t1'], counters }); + const client = { + query: jest.fn(() => Promise.reject(new Error('boom'))) + }; + await expect(interceptor(STOCK, client) as Promise).rejects.toThrow('boom'); + expect(counters.discoveries).toBe(1); + expect(counters.swaps).toBe(0); + }); + + it('returns null when disabled, served-empty, or non-introspection', () => { + const client = scriptedClient([[]]); + // disabled + process.env.GRAPHILE_INTROSPECTION_FILTER = '0'; + expect(createIntrospectionInterceptor({ servedSchemas: ['t1'] })(STOCK, client)).toBeNull(); + // enabled again + process.env.GRAPHILE_INTROSPECTION_FILTER = '1'; + // no served schemas + expect(createIntrospectionInterceptor({ servedSchemas: [] })(STOCK, client)).toBeNull(); + // non-introspection text + expect(createIntrospectionInterceptor({ servedSchemas: ['t1'] })('select 1', client)).toBeNull(); + }); +}); + +// ============================================================================= +// 6. createIntrospectionFilterPool (dedicated) +// ============================================================================= + +const introspectingClient = (rounds: any[][]) => { + let round = 0; + const query = jest.fn((arg: any, _values?: any[]) => { + if (typeof arg === 'string' && arg.includes('kept as')) { + const rows = rounds[round] ?? []; + round += 1; + return Promise.resolve({ rows }); + } + return Promise.resolve({ rows: [] }); + }); + return { query, release: jest.fn(), on: jest.fn() }; +}; + +const sentConfigTexts = (client: { query: jest.Mock }): string[] => + client.query.mock.calls.map((c: any[]) => (typeof c[0] === 'string' ? c[0] : c[0]?.text)); + +describe('createIntrospectionFilterPool — dedicated wrapper', () => { + beforeEach(() => { + process.env.GRAPHILE_INTROSPECTION_FILTER = '1'; + }); + + it('swaps the introspection query and leaves other queries untouched', async () => { + const counters = createIntrospectionFilterCounters(); + const client = introspectingClient([[], []]); + const pool = { connect: jest.fn().mockResolvedValue(client) }; + const wrapped = createIntrospectionFilterPool(pool, { servedSchemas: ['t1'], counters }); + + const c = await wrapped.connect(); + + // A normal query passes straight through. + await c.query('select 1 from t'); + expect(client.query).toHaveBeenCalledWith('select 1 from t'); + + // Introspection is discovered + swapped. + await c.query({ text: STOCK }); + const introspectionCall = client.query.mock.calls.find( + (call: any[]) => call[0] && typeof call[0] === 'object' && typeof call[0].text === 'string' && + call[0].text.includes('as introspection') + ); + expect(introspectionCall).toBeDefined(); + const swapped = introspectionCall[0].text as string; + expect(countOccurrences(swapped, 'nspname = any (array[')).toBe(4); + expect(swapped.includes("'t1'")).toBe(true); + expect(counters.swaps).toBe(1); + }); + + it('wraps clients handed back via callback-style connect', (done) => { + const client = introspectingClient([[]]); + const pool = { + connect: (cb: any) => cb(null, client, jest.fn()) + }; + const wrapped = createIntrospectionFilterPool(pool, { servedSchemas: ['t1'] }); + + wrapped.connect((err: any, c: any) => { + expect(err).toBeNull(); + c.query('select 1') + .then(() => { + expect(client.query).toHaveBeenCalledWith('select 1'); + done(); + }) + .catch(done); + }); + }); +}); + +// ============================================================================= +// 7. rewrite-pool integration +// ============================================================================= + +const logicalName = (name: string): string => name.split('-').slice(2).join('-'); +const CANON_APP = 'app-11111111-app-public'; +const TENANT_APP = 'app-22222222-app-public'; + +const settingsQuery = (schemas: string[]) => ({ + text: 'select set_config(el->>0, el->>1, true) from json_array_elements($1::json) el', + values: [JSON.stringify([['role', 'app_user'], [POOL_SCHEMAS_GUC, JSON.stringify(schemas)]])] +}); + +describe('createRewritingPool — introspection filter integration', () => { + beforeEach(() => { + process.env.GRAPHILE_INTROSPECTION_FILTER = '1'; + }); + + it('applies settings, swaps introspection (not identifier-rewritten), still rewrites tenant SQL', async () => { + const client = introspectingClient([[], []]); + const pool = { connect: jest.fn().mockResolvedValue(client), query: jest.fn(), on: jest.fn() }; + const counters = createRewriteCounters(); + const wrapped = createRewritingPool(pool, { + canonicalSchemas: [CANON_APP], + logicalName, + counters, + introspectionFilter: { servedSchemas: [CANON_APP] } + }); + + const c = await wrapped.connect(); + + // Settings establish the tenant map. + await c.query(settingsQuery([TENANT_APP, 'public'])); + expect(counters.settingsParses).toBe(1); + + // Introspection: swapped, NOT identifier-rewritten, no fail-closed. + await c.query({ text: STOCK }); + const introspectionCall = client.query.mock.calls.find( + (call: any[]) => call[0] && typeof call[0] === 'object' && + typeof call[0].text === 'string' && call[0].text.includes('as introspection') + ); + expect(introspectionCall).toBeDefined(); + const swapped = introspectionCall[0].text as string; + expect(countOccurrences(swapped, 'nspname = any (array[')).toBe(4); + expect(countOccurrences(swapped, GATE_SUBSTRING)).toBe(0); + // Introspection was not treated as a rewrite target. + expect(counters.failClosed).toBe(0); + + // A normal tenant query is still identifier-rewritten as before. + await c.query({ text: `select from "${CANON_APP}"."t"` }); + const rewritten = sentConfigTexts(client).find((t) => t && t.includes(TENANT_APP)); + expect(rewritten).toBe(`select from "${TENANT_APP}"."t"`); + expect(counters.rewrittenQueries).toBeGreaterThanOrEqual(1); + }); + + it('feeds the process-wide introspection-filter counters', async () => { + const before = getIntrospectionFilterCounters().swaps; + const client = introspectingClient([[]]); + const pool = { connect: jest.fn().mockResolvedValue(client), query: jest.fn(), on: jest.fn() }; + const wrapped = createRewritingPool(pool, { + canonicalSchemas: [CANON_APP], + logicalName, + introspectionFilter: { servedSchemas: [CANON_APP] } + }); + const c = await wrapped.connect(); + await c.query(settingsQuery([TENANT_APP, 'public'])); + await c.query({ text: STOCK }); + expect(getIntrospectionFilterCounters().swaps).toBe(before + 1); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/rewrite-pool.test.ts b/graphql/server/src/middleware/__tests__/rewrite-pool.test.ts new file mode 100644 index 0000000000..0113e8ad8e --- /dev/null +++ b/graphql/server/src/middleware/__tests__/rewrite-pool.test.ts @@ -0,0 +1,637 @@ +import { + containsQuotedIdentifier, + createRewriteCounters, + createRewritingPool, + getRewritePoolCounters, + POOL_SCHEMAS_GUC, + RewriteCounters, + rewriteQuotedIdentifiers +} from '../rewrite-pool'; + +// ----------------------------------------------------------------------------- +// Fixtures: physical schemas shaped `--`; the logical name is +// everything past the first two dash segments, so a1111111/a2222222 tenants of +// the same shape collapse to the same logical name. +// ----------------------------------------------------------------------------- + +const logicalName = (name: string): string => name.split('-').slice(2).join('-'); + +const CANON_APP = 'app-11111111-app-public'; +const CANON_AUTH = 'app-11111111-auth-public'; +const TENANT_APP = 'app-22222222-app-public'; +const TENANT_AUTH = 'app-22222222-auth-public'; + +const canonicalSchemas = [CANON_APP, CANON_AUTH]; +const tenantSchemas = [TENANT_APP, TENANT_AUTH, 'public']; + +const settingsQuery = (schemas: string[]) => ({ + text: 'select set_config(el->>0, el->>1, true) from json_array_elements($1::json) el', + values: [JSON.stringify([['role', 'app_user'], [POOL_SCHEMAS_GUC, JSON.stringify(schemas)]])] +}); + +const makeClient = () => ({ + query: jest.fn().mockResolvedValue({ rows: [] }), + release: jest.fn(), + on: jest.fn(), + escapeIdentifier: jest.fn((s: string) => `"${s}"`) +}); + +const makePool = (client: ReturnType) => ({ + query: jest.fn().mockResolvedValue({ rows: [] }), + connect: jest.fn().mockResolvedValue(client), + on: jest.fn(), + end: jest.fn() +}); + +const setup = (overrides: Partial[1]> = {}) => { + const counters = createRewriteCounters(); + const client = makeClient(); + const pool = makePool(client); + const wrapped = createRewritingPool(pool, { + canonicalSchemas, + logicalName, + counters, + ...overrides + }); + return { counters, client, pool, wrapped }; +}; + +// Text of the Nth (0-based) call to the raw client query mock. +const sentText = (client: ReturnType, n: number): string => { + const arg = client.query.mock.calls[n][0]; + return typeof arg === 'string' ? arg : arg.text; +}; + +// ============================================================================= +// Pure lexer: rewriteQuotedIdentifiers +// ============================================================================= + +describe('rewriteQuotedIdentifiers', () => { + const map = new Map([[CANON_APP, TENANT_APP]]); + + it('replaces every quoted occurrence, including ::cast enum refs', () => { + const input = `select "${CANON_APP}"."t".*, x::"${CANON_APP}"."my_enum" from "${CANON_APP}"."t"`; + const expected = `select "${TENANT_APP}"."t".*, x::"${TENANT_APP}"."my_enum" from "${TENANT_APP}"."t"`; + expect(rewriteQuotedIdentifiers(input, map)).toBe(expected); + }); + + it('returns the input unchanged (byte-identical) when nothing matches', () => { + const input = 'select "users"."id" from "users"'; + expect(rewriteQuotedIdentifiers(input, map)).toBe(input); + }); + + describe('never touches non-identifier regions', () => { + const m = new Map([['canon', 'TENANT']]); + + it('single-quoted literal incl. doubled-quote escape', () => { + const input = `select '"canon" it''s "canon"', "canon"`; + expect(rewriteQuotedIdentifiers(input, m)).toBe(`select '"canon" it''s "canon"', "TENANT"`); + }); + + it("E-string with backslash escapes and embedded quotes", () => { + // Actual text: E'a\'b "canon" c' || "canon" + const input = `E'a\\'b "canon" c' || "canon"`; + expect(rewriteQuotedIdentifiers(input, m)).toBe(`E'a\\'b "canon" c' || "TENANT"`); + }); + + it('$$ and $tag$ dollar-quoted bodies', () => { + const input = `$$ "canon" $$ || $tag$ x "canon" y $tag$ || "canon"`; + expect(rewriteQuotedIdentifiers(input, m)).toBe(`$$ "canon" $$ || $tag$ x "canon" y $tag$ || "TENANT"`); + }); + + it('line comments and nested block comments', () => { + const line = `-- "canon"\n"canon"`; + expect(rewriteQuotedIdentifiers(line, m)).toBe(`-- "canon"\n"TENANT"`); + + const block = `/* "canon" /* "canon" */ still "canon" */ "canon"`; + expect(rewriteQuotedIdentifiers(block, m)).toBe(`/* "canon" /* "canon" */ still "canon" */ "TENANT"`); + }); + }); + + it('does not replace an identifier that merely prefixes a longer one', () => { + const m = new Map([['canon', 'TENANT']]); + expect(rewriteQuotedIdentifiers('"canon_extra" "canon"', m)).toBe('"canon_extra" "TENANT"'); + }); + + it('matches on the unescaped value of an identifier with embedded ""', () => { + const m = new Map([['ca"non', 'TENANT']]); + expect(rewriteQuotedIdentifiers('"ca""non"', m)).toBe('"TENANT"'); + }); + + it('escapes a " inside the replacement value', () => { + const m = new Map([['canon', 'ten"ant']]); + expect(rewriteQuotedIdentifiers('"canon"', m)).toBe('"ten""ant"'); + }); +}); + +// ============================================================================= +// Pure lexer: containsQuotedIdentifier +// ============================================================================= + +describe('containsQuotedIdentifier', () => { + const names = new Set([CANON_APP]); + + it('true only for a complete quoted token match', () => { + expect(containsQuotedIdentifier(`from "${CANON_APP}"."t"`, names)).toBe(true); + expect(containsQuotedIdentifier(`from "${CANON_APP}_extra"."t"`, names)).toBe(false); + }); + + it('ignores matches inside strings and comments', () => { + expect(containsQuotedIdentifier(`select '"${CANON_APP}"'`, names)).toBe(false); + expect(containsQuotedIdentifier(`-- "${CANON_APP}"`, names)).toBe(false); + expect(containsQuotedIdentifier(`$$ "${CANON_APP}" $$`, names)).toBe(false); + }); +}); + +// ============================================================================= +// Wrapper: end-to-end +// ============================================================================= + +describe('createRewritingPool — settings + rewriting', () => { + it('passes the settings query through byte-identical, then rewrites later queries', async () => { + const { client, wrapped } = setup(); + const c = await wrapped.connect(); + + const sq = settingsQuery(tenantSchemas); + await c.query(sq); + // Forwarded untouched (same object, identical text/values). + expect(client.query).toHaveBeenNthCalledWith(1, sq); + + await c.query({ text: `select * from "${CANON_APP}"."users" join "${CANON_AUTH}"."roles" using (id)` }); + expect(sentText(client, 1)).toBe( + `select * from "${TENANT_APP}"."users" join "${TENANT_AUTH}"."roles" using (id)` + ); + }); + + it('supports string and (text, values) call forms', async () => { + const { client, wrapped } = setup(); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + + await c.query(`select from "${CANON_APP}"."t"`); + expect(sentText(client, 1)).toBe(`select from "${TENANT_APP}"."t"`); + + await c.query(`select from "${CANON_APP}"."t" where x = $1`, [1]); + expect(sentText(client, 2)).toBe(`select from "${TENANT_APP}"."t" where x = $1`); + expect(client.query.mock.calls[2][1]).toEqual([1]); + }); + + it('does not mutate the caller-supplied query config', async () => { + const { client, wrapped } = setup(); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + + const config = { text: `select from "${CANON_APP}"."t"`, name: 'p1' }; + await c.query(config); + expect(config.text).toBe(`select from "${CANON_APP}"."t"`); + expect(config.name).toBe('p1'); + // The forwarded object is a clone with rewritten fields. + expect(client.query.mock.calls[1][0]).not.toBe(config); + expect(sentText(client, 1)).toBe(`select from "${TENANT_APP}"."t"`); + }); + + it('wraps clients handed back via callback-style connect', (done) => { + const client = makeClient(); + const pool = { + connect: (cb: any) => cb(null, client, jest.fn()), + query: jest.fn() + }; + const wrapped = createRewritingPool(pool, { canonicalSchemas, logicalName }); + + wrapped.connect((err: any, c: any) => { + expect(err).toBeNull(); + c.query(settingsQuery(tenantSchemas)); + c.query({ text: `select from "${CANON_APP}"."t"` }); + expect(sentText(client, 1)).toBe(`select from "${TENANT_APP}"."t"`); + done(); + }); + }); + + it('forwards non-query members bound to the raw client', async () => { + const { client, wrapped } = setup(); + const c = await wrapped.connect(); + expect(c.escapeIdentifier('x')).toBe('"x"'); + expect(client.escapeIdentifier).toHaveBeenCalledWith('x'); + }); +}); + +// ============================================================================= +// Wrapper: identity mode (canonical tenant querying itself) +// ============================================================================= + +describe('createRewritingPool — identity mode', () => { + it('leaves text and prepared names untouched when the tenant is canonical', async () => { + const { client, counters, wrapped } = setup(); + const c = await wrapped.connect(); + await c.query(settingsQuery([CANON_APP, CANON_AUTH, 'public'])); + + await c.query({ text: `select from "${CANON_APP}"."t"`, name: 'p1' }); + expect(sentText(client, 1)).toBe(`select from "${CANON_APP}"."t"`); + expect(client.query.mock.calls[1][0].name).toBe('p1'); + expect(counters.rewrittenQueries).toBe(0); + expect(counters.nameRewrites).toBe(0); + }); +}); + +// ============================================================================= +// Wrapper: prepared-statement name namespacing +// ============================================================================= + +describe('createRewritingPool — prepared name namespacing', () => { + it('is deterministic per tenant and differs across tenants', async () => { + const a = setup(); + const ca = await a.wrapped.connect(); + await ca.query(settingsQuery(tenantSchemas)); + await ca.query({ text: 'select 1', name: 'stmt' }); + await ca.query({ text: 'select 2', name: 'stmt' }); + const nameA1 = a.client.query.mock.calls[1][0].name; + const nameA2 = a.client.query.mock.calls[2][0].name; + expect(nameA1).toMatch(/^bp_[0-9a-f]{24}$/); + expect(nameA2).toBe(nameA1); // same tenant + original name => same + + const b = setup(); + const cb = await b.wrapped.connect(); + await cb.query(settingsQuery(['app-33333333-app-public', 'app-33333333-auth-public', 'public'])); + await cb.query({ text: 'select 1', name: 'stmt' }); + const nameB1 = b.client.query.mock.calls[1][0].name; + expect(nameB1).not.toBe(nameA1); // different tenant => different name + }); +}); + +// ============================================================================= +// Wrapper: prepared-statement lifecycle (the wrapper owns eviction + DEALLOCATE) +// ============================================================================= + +describe('createRewritingPool — prepared-statement lifecycle', () => { + // The `deallocate "bp_…"` statements the raw client received, in order. + const deallocs = (client: ReturnType): string[] => + client.query.mock.calls + .map((call: any[]) => call[0]) + .filter((arg: any): arg is string => typeof arg === 'string' && arg.startsWith('deallocate')); + + it('passes DEALLOCATE text through untouched (adaptor concern, not ours)', async () => { + const { client, wrapped } = setup(); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + + await c.query('deallocate stmt'); + await c.query('deallocate all'); + expect(sentText(client, 1)).toBe('deallocate stmt'); + expect(sentText(client, 2)).toBe('deallocate all'); + }); + + it('evicts the oldest hashed name and deallocates it once the cap is exceeded', async () => { + const { client, wrapped } = setup({ maxPreparedNames: 2 }); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + + await c.query({ text: 'select 1', name: 'a' }); + await c.query({ text: 'select 2', name: 'b' }); + const hashedA = client.query.mock.calls[1][0].name; + expect(hashedA).toMatch(/^bp_[0-9a-f]{24}$/); + + await c.query({ text: 'select 3', name: 'c' }); // 3rd distinct name -> evict 'a' + + expect(deallocs(client)).toEqual([`deallocate "${hashedA}"`]); + }); + + it('clears node-postgres parsedStatements bookkeeping for the evicted name', async () => { + const { client, wrapped } = setup({ maxPreparedNames: 2 }); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + + await c.query({ text: 'select 1', name: 'a' }); + const hashedA = client.query.mock.calls[1][0].name; + // Simulate node-postgres having parsed/prepared under the hashed name. + (client as any).connection = { parsedStatements: { [hashedA]: 'select 1' } }; + + await c.query({ text: 'select 2', name: 'b' }); + await c.query({ text: 'select 3', name: 'c' }); // evict 'a' + await new Promise((resolve) => setImmediate(resolve)); // flush the fire-and-forget then() + + // Without this cleanup, re-preparing 'a' would skip Parse and fail on PG + // with "prepared statement does not exist". + expect((client as any).connection.parsedStatements[hashedA]).toBeUndefined(); + }); + + it('refreshes recency: re-touching name 1 evicts name 2 instead', async () => { + const { client, wrapped } = setup({ maxPreparedNames: 2 }); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + + await c.query({ text: 'select 1', name: 'a' }); // [a] + await c.query({ text: 'select 2', name: 'b' }); // [a, b] + const hashedA = client.query.mock.calls[1][0].name; + const hashedB = client.query.mock.calls[2][0].name; + + await c.query({ text: 'select 1 again', name: 'a' }); // refresh a -> [b, a] + await c.query({ text: 'select 3', name: 'c' }); // insert c -> evict b + + expect(deallocs(client)).toEqual([`deallocate "${hashedB}"`]); + expect(deallocs(client)).not.toContain(`deallocate "${hashedA}"`); + }); + + it('re-using the same name neither grows the LRU nor deallocates', async () => { + const { client, wrapped } = setup({ maxPreparedNames: 2 }); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + + await c.query({ text: 'select 1', name: 'a' }); + await c.query({ text: 'select 1 v2', name: 'a' }); + await c.query({ text: 'select 1 v3', name: 'a' }); + + expect(deallocs(client)).toEqual([]); + }); + + it('keeps the prepared-name LRU across release (it lives on the physical connection)', async () => { + const { client, wrapped } = setup({ maxPreparedNames: 2 }); + + const c1 = await wrapped.connect(); + await c1.query(settingsQuery(tenantSchemas)); + await c1.query({ text: 'select 1', name: 'a' }); // [a] + const hashedA = client.query.mock.calls[1][0].name; + c1.release(); + + // Same physical client returns; tenant state reset, prepared-name LRU intact. + const c2 = await wrapped.connect(); + await c2.query(settingsQuery(tenantSchemas)); + await c2.query({ text: 'select 2', name: 'b' }); // [a, b] + await c2.query({ text: 'select 3', name: 'c' }); // evict a (survived release) + + expect(deallocs(client)).toEqual([`deallocate "${hashedA}"`]); + }); + + it('honors DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE for the default cap', async () => { + const original = process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE; + process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE = '1'; + try { + const { client, wrapped } = setup(); // no explicit maxPreparedNames + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + + await c.query({ text: 'select 1', name: 'a' }); + const hashedA = client.query.mock.calls[1][0].name; + await c.query({ text: 'select 2', name: 'b' }); // cap 1 -> evict a + + expect(deallocs(client)).toEqual([`deallocate "${hashedA}"`]); + } finally { + if (original === undefined) delete process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE; + else process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE = original; + } + }); +}); + +// ============================================================================= +// Wrapper: fail-closed +// ============================================================================= + +describe('createRewritingPool — fail closed', () => { + it('rejects a rewrite-needing query issued before any settings query', async () => { + const { client, counters, wrapped } = setup(); + const c = await wrapped.connect(); + + await expect(c.query({ text: `select from "${CANON_APP}"."t"` })).rejects.toThrow( + /requires a tenant mapping but none is established/ + ); + expect(counters.failClosed).toBe(1); + expect(client.query).not.toHaveBeenCalled(); + }); + + it('passes benign non-rewrite statements through untouched', async () => { + const { client, counters, wrapped } = setup(); + const c = await wrapped.connect(); + + await c.query('begin'); + await c.query({ text: 'listen "x"' }); + await c.query({ text: 'select 1 from pg_catalog.pg_class' }); + await c.query({ text: 'select set_config($1,$2,true)', values: ['role', 'app_user'] }); + + expect(counters.failClosed).toBe(0); + expect(client.query.mock.calls.map((call: any[]) => call[0])).toEqual([ + 'begin', + { text: 'listen "x"' }, + { text: 'select 1 from pg_catalog.pg_class' }, + { text: 'select set_config($1,$2,true)', values: ['role', 'app_user'] } + ]); + }); + + it('fails closed with a callback when called callback-style', async () => { + const { wrapped } = setup(); + const c = await wrapped.connect(); + const err: Error = await new Promise((resolve) => { + c.query({ text: `select from "${CANON_APP}"."t"` }, (e: Error) => resolve(e)); + }); + expect(err.message).toMatch(/requires a tenant mapping/); + }); +}); + +// ============================================================================= +// Wrapper: settings detection keys on the adaptor statement TEXT, not on +// user-influenceable parameter contents (A1 — misclassification hardening). +// ============================================================================= + +describe('createRewritingPool — settings detection is text-gated', () => { + it('does not misclassify a data query as settings when a bind value contains the GUC substring', async () => { + const { client, counters, wrapped } = setup(); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + expect(counters.settingsParses).toBe(1); + + // A genuine data query whose user-controlled bind value contains the GUC + // string. Under parameter-only detection this was treated as the settings + // query and forwarded UNREWRITTEN (canonical SQL on the tenant connection). + await c.query({ + text: `select from "${CANON_APP}"."t" where x = $1`, + values: [`${POOL_SCHEMAS_GUC} = whatever`] + }); + + // Rewritten to the tenant schema — not forwarded with canonical identifiers. + expect(sentText(client, 1)).toBe(`select from "${TENANT_APP}"."t" where x = $1`); + // The malicious value was NOT parsed as settings; checkout map is intact. + expect(counters.settingsParses).toBe(1); + }); + + it('a GUC-substring bind value cannot null the established tenant map (no self-DoS)', async () => { + const { client, counters, wrapped } = setup(); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + + // Malicious non-JSON value that, under parameter-only detection, would have + // re-run applySettings and nulled tenantMap (forcing later fail-closed). + await c.query({ + text: `select from "${CANON_APP}"."t"`, + values: [`${POOL_SCHEMAS_GUC}: not json`] + }); + expect(sentText(client, 1)).toBe(`select from "${TENANT_APP}"."t"`); + + // Sibling query still rewrites (map not clobbered) rather than failing closed. + await c.query({ text: `select from "${CANON_AUTH}"."r"` }); + expect(sentText(client, 2)).toBe(`select from "${TENANT_AUTH}"."r"`); + expect(counters.failClosed).toBe(0); + expect(counters.settingsParses).toBe(1); + }); + + it('a crafted GUC-bearing JSON bind value cannot remap the checkout to another tenant', async () => { + const { client, wrapped } = setup(); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); // maps canonical -> app-22222222 + + const victim = ['app-99999999-app-public', 'app-99999999-auth-public', 'public']; + // Same value shape the real settings query would carry, but on a DATA query. + await c.query({ + text: `select from "${CANON_APP}"."t"`, + values: [JSON.stringify([[POOL_SCHEMAS_GUC, JSON.stringify(victim)]])] + }); + + // Still the original tenant (22222222), never the injected victim (99999999). + expect(sentText(client, 1)).toBe(`select from "${TENANT_APP}"."t"`); + }); + + it('fails closed (does not swallow as settings) for a pre-settings query carrying a GUC value', async () => { + const { client, counters, wrapped } = setup(); + const c = await wrapped.connect(); + + await expect( + c.query({ text: `select from "${CANON_APP}"."t"`, values: [`${POOL_SCHEMAS_GUC} x`] }) + ).rejects.toThrow(/requires a tenant mapping/); + expect(counters.failClosed).toBe(1); + expect(counters.settingsParses).toBe(0); + expect(client.query).not.toHaveBeenCalled(); + }); +}); + +// ============================================================================= +// Wrapper: release clears state +// ============================================================================= + +describe('createRewritingPool — release clears state', () => { + it('re-checkout of the same client requires a fresh settings query', async () => { + const { client, wrapped } = setup(); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + await c.query({ text: `select from "${CANON_APP}"."t"` }); // ok + + c.release(); + expect(client.release).toHaveBeenCalled(); + + await expect(c.query({ text: `select from "${CANON_APP}"."t"` })).rejects.toThrow( + /requires a tenant mapping/ + ); + }); +}); + +// ============================================================================= +// Wrapper: logical mismatch => invalid checkout +// ============================================================================= + +describe('createRewritingPool — logical mismatch', () => { + it('marks the checkout invalid and fails closed with the reason', async () => { + const { counters, wrapped } = setup(); + const c = await wrapped.connect(); + // Tenant list is missing an auth-public counterpart. + await c.query(settingsQuery([TENANT_APP, 'public'])); + + await expect(c.query({ text: `select from "${CANON_AUTH}"."t"` })).rejects.toThrow( + /auth-public/ + ); + expect(counters.failClosed).toBe(1); + }); +}); + +// ============================================================================= +// Wrapper: LRU memo cap +// ============================================================================= + +describe('createRewritingPool — LRU memo cap', () => { + it('respects the cap while keeping every rewrite correct', async () => { + const { client, counters, wrapped } = setup({ maxMemoEntries: 2 }); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + + const q1 = { text: `select from "${CANON_APP}"."a"` }; + const q2 = { text: `select from "${CANON_APP}"."b"` }; + const q3 = { text: `select from "${CANON_APP}"."c"` }; + + await c.query(q1); + await c.query(q2); + await c.query(q3); // fills then evicts q1 (cap 2) + expect(counters.memoHits).toBe(0); + + await c.query(q1); // q1 was evicted => miss (proves the cap held) + expect(counters.memoHits).toBe(0); + + await c.query(q3); // q3 still cached => hit + expect(counters.memoHits).toBe(1); + + // Every rewrite stayed correct regardless of eviction. + expect(sentText(client, 1)).toBe(`select from "${TENANT_APP}"."a"`); + expect(sentText(client, 3)).toBe(`select from "${TENANT_APP}"."c"`); + expect(sentText(client, 4)).toBe(`select from "${TENANT_APP}"."a"`); + expect(sentText(client, 5)).toBe(`select from "${TENANT_APP}"."c"`); + }); +}); + +// ============================================================================= +// Wrapper: counters +// ============================================================================= + +describe('createRewritingPool — counters', () => { + it('increments each counter exactly as specified', async () => { + const { wrapped, counters } = setup(); + const c = await wrapped.connect(); + + await c.query(settingsQuery(tenantSchemas)); // settingsParses + await c.query({ text: `select from "${CANON_APP}"."t"` }); // rewrittenQueries (miss) + await c.query({ text: `select from "${CANON_APP}"."t"` }); // rewrittenQueries + memoHits + await c.query({ text: 'select 1', name: 'p1' }); // nameRewrites + + const expected: RewriteCounters = { + settingsParses: 1, + rewrittenQueries: 2, + memoHits: 1, + failClosed: 0, + nameRewrites: 1 + }; + expect(counters).toEqual(expected); + }); + + it('shares a caller-provided counters object', async () => { + const counters = createRewriteCounters(); + const { wrapped } = setup({ counters }); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + expect(counters.settingsParses).toBe(1); + }); + + it('feeds the process-wide getRewritePoolCounters() snapshot when no counters are passed', async () => { + const before = getRewritePoolCounters().settingsParses; + // No explicit counters -> activity lands in the module-level sink. + const client = makeClient(); + const wrapped = createRewritingPool(makePool(client), { canonicalSchemas, logicalName }); + const c = await wrapped.connect(); + await c.query(settingsQuery(tenantSchemas)); + + expect(getRewritePoolCounters().settingsParses).toBe(before + 1); + // Returns a snapshot copy, not the live object. + const snap = getRewritePoolCounters(); + snap.settingsParses = -999; + expect(getRewritePoolCounters().settingsParses).toBe(before + 1); + }); +}); + +// ============================================================================= +// Pool-level one-shot query +// ============================================================================= + +describe('createRewritingPool — pool.query one-shot', () => { + it('passes benign queries through and fails closed on rewrite-needing ones', async () => { + const { pool, wrapped } = setup(); + + await wrapped.query('select 1'); + expect(pool.query).toHaveBeenCalledWith('select 1'); + + await expect(wrapped.query({ text: `select from "${CANON_APP}"."t"` })).rejects.toThrow( + /requires a tenant mapping/ + ); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/settings-text-pin.test.ts b/graphql/server/src/middleware/__tests__/settings-text-pin.test.ts new file mode 100644 index 0000000000..ebc706a689 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/settings-text-pin.test.ts @@ -0,0 +1,24 @@ +import { readFileSync } from 'fs'; + +import { SETTINGS_QUERY_TEXT } from '../rewrite-pool'; + +/** + * Pins SETTINGS_QUERY_TEXT against the INSTALLED @dataplan/pg adaptor source. + * + * The rewriting pool classifies the adaptor's pgSettings statement by exact + * text equality (defense against user bind values that merely contain the + * pool_schemas GUC substring). If a dependency bump changes the emitted + * statement, classification silently stops matching: applySettings never runs, + * no checkout ever gets a tenant mapping, and every pooled data query fails + * closed. That is the SAFE direction, but it disables pooling entirely and + * only surfaces at runtime — this test makes the drift fail loudly here + * instead. Mirrors the pg-introspection pinning approach in + * introspection-filter.test.ts. + */ +describe('SETTINGS_QUERY_TEXT pins the installed @dataplan/pg adaptor', () => { + it('the adaptor source still emits the exact statement we byte-match', () => { + const adaptorPath = require.resolve('@dataplan/pg/adaptors/pg'); + const source = readFileSync(adaptorPath, 'utf8'); + expect(source).toContain(SETTINGS_QUERY_TEXT); + }); +}); diff --git a/graphql/server/src/middleware/api.ts b/graphql/server/src/middleware/api.ts index 005be581fd..cd20402413 100644 --- a/graphql/server/src/middleware/api.ts +++ b/graphql/server/src/middleware/api.ts @@ -14,6 +14,7 @@ import { getPgPool } from 'pg-cache'; import errorPage50x from '../errors/50x'; import errorPage404Message from '../errors/404-message'; import { ApiConfigResult, ApiError, ApiOptions, ApiStructure, AuthSettings, DatabaseSettings, PubkeyChallengeSettings, RlsModule, WebauthnSettings } from '../types'; +import { stripSchemaHashPrefix } from './blueprint'; import './types'; const log = new Logger('api'); @@ -266,6 +267,7 @@ const toApiStructure = (row: ApiRow, opts: ApiOptions, settings: ResolvedModuleS anonRole: row.anon_role || 'anon', roleName: row.role_name || 'authenticated', schema: row.schemas || [], + logicalSchemas: (row.schemas || []).map(stripSchemaHashPrefix), apiModules: [], rlsModule: settings.rlsModule, domains: [], diff --git a/graphql/server/src/middleware/blueprint.ts b/graphql/server/src/middleware/blueprint.ts new file mode 100644 index 0000000000..71984ba30d --- /dev/null +++ b/graphql/server/src/middleware/blueprint.ts @@ -0,0 +1,129 @@ +import { createHash } from 'node:crypto'; +import type { Pool } from 'pg'; + +// ============================================================================= +// Blueprint pooling identity helpers +// +// OPT-IN "blueprint pooling" shares one PostGraphile instance per schema-shape. +// Tenants are routed per request by the rewriting pool (canonical→tenant schema- +// identifier rewrite; see ./rewrite-pool). These helpers derive the identity of a +// shared instance (shape fingerprint + blueprint key). +// ============================================================================= + +/** + * Whether blueprint pooling is enabled via the GRAPHILE_BLUEPRINT_POOLING env + * flag. Accepts '1' or 'true'; anything else (incl. unset) means disabled. + */ +export const isBlueprintPoolingEnabled = (): boolean => { + const value = process.env.GRAPHILE_BLUEPRINT_POOLING; + return value === '1' || value === 'true'; +}; + +/** + * Strip the tenant hash prefix from a physical schema name. + * + * Tenant physical schemas look like `-<8hex>-`, e.g. + * `marketplace-db-tenant1-5e6b13b2-app-public` -> `app-public`. We strip through + * the FIRST `-<8 lowercase hex>-` occurrence and return the remainder. Control + * plane schemas (e.g. `services_public`) contain no such segment and are + * returned unchanged. + */ +export const stripSchemaHashPrefix = (name: string): string => { + const match = /-[0-9a-f]{8}-/.exec(name); + if (!match) return name; + return name.slice(match.index + match[0].length); +}; + +export interface SchemaRelation { + nspname: string; + relname: string; +} + +/** + * Single catalog scan backing the shape fingerprint. + */ +export const fetchSchemaRelations = async ( + pool: Pool, + physicalSchemas: string[] +): Promise => { + const result = await pool.query( + `SELECT n.nspname, c.relname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = ANY($1) AND c.relkind IN ('r','v','m','p') + ORDER BY 1, 2`, + [physicalSchemas] + ); + return result.rows; +}; + +/** + * Pure fingerprint over pre-fetched relations: sha256 of the sorted + * `[logicalSchema, relname]` pairs (physical names mapped to logical form so + * same-shape tenants collapse to the same fingerprint). + * + * NOTE (v1 granularity): the fingerprint covers relation NAMES only — not + * columns, functions or enum labels. Same-relname column drift between tenants + * is not detected here; it is bounded by flushService's flush-all-pooled-on- + * schema:update semantics. Extending the fingerprint to attributes/procs is a + * documented follow-up. + */ +export const fingerprintFromRelations = (rows: SchemaRelation[]): string => { + const pairs: [string, string][] = rows.map((row) => [ + stripSchemaHashPrefix(row.nspname), + row.relname + ]); + pairs.sort((a, b) => { + if (a[0] !== b[0]) return a[0] < b[0] ? -1 : 1; + if (a[1] !== b[1]) return a[1] < b[1] ? -1 : 1; + return 0; + }); + return createHash('sha256').update(JSON.stringify(pairs)).digest('hex'); +}; + +/** + * Compute a fingerprint of the relational shape shared by a set of physical + * schemas. Physical schema names are mapped to their logical form (hash prefix + * stripped) so that different tenants of the same shape collapse to the same + * fingerprint. The fingerprint is a sha256 hex digest over the sorted + * `[logicalSchema, relname]` pairs. + */ +export const computeShapeFingerprint = async ( + pool: Pool, + physicalSchemas: string[] +): Promise => fingerprintFromRelations(await fetchSchemaRelations(pool, physicalSchemas)); + +/** + * Compute the stable blueprint key that identifies a poolable PostGraphile + * instance. Two requests share an instance iff they agree on logical schemas, + * shape fingerprint, gather/schema flags, api name and mode. Inputs are + * normalized (schemas sorted, flag entries sorted by key) so the key is + * independent of ordering. + */ +export const computeBlueprintKey = (input: { + logicalSchemas: string[]; + shapeFingerprint: string; + flags: Record | undefined; + apiName?: string | null; + mode: string; + /** + * Physical database backing the instance's pool. REQUIRED for correctness in + * multi-database fleets: without it, same-shape tenants living in DIFFERENT + * physical databases would share one instance whose pool points at only one + * of them. + */ + dbname?: string | null; +}): string => { + const payload = { + s: [...input.logicalSchemas].sort(), + f: input.shapeFingerprint, + g: Object.entries(input.flags || {}).sort((a, b) => + a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0 + ), + a: input.apiName || '', + m: input.mode, + d: input.dbname || '' + }; + + return 'bp:' + createHash('sha256').update(JSON.stringify(payload)).digest('hex'); +}; diff --git a/graphql/server/src/middleware/flush.ts b/graphql/server/src/middleware/flush.ts index 653d74631c..70b7bc8a72 100644 --- a/graphql/server/src/middleware/flush.ts +++ b/graphql/server/src/middleware/flush.ts @@ -2,9 +2,11 @@ import { ConstructiveOptions } from '@constructive-io/graphql-types'; import { Logger } from '@pgpmjs/logger'; import { svcCache } from '@pgpmjs/server-utils'; import { NextFunction, Request, Response } from 'express'; -import { graphileCache } from 'graphile-cache'; +import { clearMatchingEntries, graphileCache } from 'graphile-cache'; import { getPgPool } from 'pg-cache'; import './types'; // for Request type +import { isBlueprintPoolingEnabled } from './blueprint'; +import { clearPoolDecisions, clearPoolDecisionsForDatabase } from './pooling-decision'; const log = new Logger('flush'); @@ -17,6 +19,12 @@ export const flush = async ( // TODO: check bearer for a flush / special key graphileCache.delete((req as any).svc_key); svcCache.delete((req as any).svc_key); + // Under pooling the serving instance is stored under a `bp:` key, which the + // svc_key delete above cannot reach — mirror flushService's v1 semantics. + if (isBlueprintPoolingEnabled()) { + clearMatchingEntries(/^bp:/); + clearPoolDecisions(); + } res.status(200).send('OK'); return; } @@ -30,6 +38,23 @@ export const flushService = async ( const pgPool = getPgPool(opts.pg); log.info('flushing db ' + databaseId); + // Blueprint pooling: invalidate ONLY the changed database's decisions and the + // blueprint instance it was attached to. Fleet-wide `bp:` flushes on every + // schema:update turned each tenant PROVISION into a cold restart of every + // pooled instance — and the immediate rebuild raced the evicted instances' + // drain (~GB still live until release), which OOMed the 24h soak. New tenants + // have no memoized decisions, so provisioning is a no-op here (instances are + // shape-generic; a same-shape tenant attaches without any rebuild). + if (isBlueprintPoolingEnabled()) { + const bpKeys = clearPoolDecisionsForDatabase(databaseId); + for (const key of bpKeys) { + graphileCache.delete(key); + } + if (bpKeys.length > 0) { + log.info(`[pooling] flushed ${bpKeys.length} blueprint(s) for db ${databaseId}: ${bpKeys.join(', ')}`); + } + } + const api = new RegExp(`^api:${databaseId}:.*`); const schemata = new RegExp(`^schemata:${databaseId}:.*`); const meta = new RegExp(`^metaschema:api:${databaseId}`); diff --git a/graphql/server/src/middleware/graphile.ts b/graphql/server/src/middleware/graphile.ts index 477e61288b..fe7ed33b9d 100644 --- a/graphql/server/src/middleware/graphile.ts +++ b/graphql/server/src/middleware/graphile.ts @@ -3,19 +3,44 @@ import { getNodeEnv } from '@pgpmjs/env'; import type { ConstructiveOptions } from '@constructive-io/graphql-types'; import { Logger } from '@pgpmjs/logger'; import type { NextFunction, Request, RequestHandler, Response } from 'express'; -import type { GraphQLError, GraphQLFormattedError } from 'grafast/graphql'; -import { createGraphileInstance, type GraphileCacheEntry, graphileCache } from 'graphile-cache'; +// Type-only import of the canonical GraphQL error types. Imported from 'graphql' +// (a direct, version-pinned dependency) rather than the 'grafast/graphql' subpath so +// the types resolve under every moduleResolution mode — including ts-jest, which type- +// checks this file transitively via the diagnostics metrics sampler. Erased at runtime. +import type { GraphQLError, GraphQLFormattedError } from 'graphql'; +import type { Pool } from 'pg'; +import { + createGraphileInstance, + ensureCacheHeadroom, + getMemoryPressure, + type GraphileCacheEntry, + graphileCache, + invokeEntryHandler, + shouldRefuseBuild, + waitForDrainSettle, +} from 'graphile-cache'; import type { GraphileConfig } from 'graphile-config'; import { createConstructivePreset, makePgService } from 'graphile-settings'; import { getPgPool } from 'pg-cache'; import { getPgEnvOptions } from 'pg-env'; import './types'; // for Request type +import { isBlueprintPoolingEnabled, stripSchemaHashPrefix } from './blueprint'; +import { resolvePoolDecision } from './pooling-decision'; +import { + createIntrospectionFilterPool, + isIntrospectionFilterEnabled +} from './introspection-filter'; +import { createRewritingPool, POOL_SCHEMAS_GUC } from './rewrite-pool'; import { isGraphqlObservabilityEnabled } from '../diagnostics/observability'; import { HandlerCreationError } from '../errors/api-errors'; import { observeGraphileBuild } from './observability/graphile-build-stats'; import type { DatabaseSettings } from '../types'; import { AuthCookiePlugin } from '../plugins/auth-cookie-plugin'; +// Re-exported so flush.ts (and other callers) can invalidate pooling decisions +// through the graphile module surface. +export { clearPoolDecisions } from './pooling-decision'; + const maskErrorLog = new Logger('graphile:maskError'); const SAFE_ERROR_CODES = new Set([ @@ -191,6 +216,101 @@ export function clearInFlightMap(): void { creating.clear(); } +/** + * Seeds an in-flight creation promise for a key. Used for testing purposes to + * exercise the coalescing paths (Phase B / Phase C re-coalesce) deterministically. + */ +export function setInFlightForTest(key: string, promise: Promise): void { + creating.set(key, promise); +} + +// ============================================================================= +// Build Admission Control +// ============================================================================= + +/** + * Bounds how many PostGraphile schema builds run concurrently across ALL cache + * keys (the single-flight map only dedups builds for the SAME key). Each build + * transiently allocates hundreds of MB, so concurrent builds of different + * tenants stack those peaks on top of the resident cache — the OOM shape. + * Default 1: builds queue and run serially. Override with GRAPHILE_BUILD_CONCURRENCY. + */ +class BuildSemaphore { + private active = 0; + private waiters: Array<() => void> = []; + constructor(private readonly capacity: number) {} + + async acquire(): Promise { + if (this.active < this.capacity) { + this.active++; + return; + } + await new Promise((resolve) => this.waiters.push(resolve)); + this.active++; + } + + release(): void { + this.active = Math.max(0, this.active - 1); + const wake = this.waiters.shift(); + if (wake) wake(); + } + + /** Number of builds currently queued (blocked on a free slot). */ + get queueDepth(): number { + return this.waiters.length; + } +} + +const parseBuildConcurrency = (): number => { + const raw = process.env.GRAPHILE_BUILD_CONCURRENCY; + const n = raw ? parseInt(raw, 10) : 1; + return Number.isFinite(n) && n > 0 ? n : 1; +}; + +const buildSemaphore = new BuildSemaphore(parseBuildConcurrency()); + +/** + * Returns the number of PostGraphile schema builds currently queued behind the + * build-concurrency semaphore. A persistently non-zero depth means builds are + * arriving faster than they can be serialized — a leading indicator of build + * backpressure under load. Exposed for the metrics sampler. + */ +export function getBuildQueueDepth(): number { + return buildSemaphore.queueDepth; +} + +// ============================================================================= +// In-Process Build Counters (metrics) +// ============================================================================= + +export interface GraphileCounters { + /** Total PostGraphile schema builds started (past coalescing + the semaphore). */ + builds: number; + /** Requests routed to a shared blueprint (pooling) instance. */ + poolingAttaches: number; + /** Builds that were pooled (shared-blueprint) builds. */ + poolingBuilds: number; + /** Requests that gave up waiting on a build (GRAPHILE_BUILD_TIMEOUT_MS) and got 503. */ + buildWaitTimeouts: number; +} + +/** + * Cumulative in-process build counters for the metrics sampler. Mutated where a build + * starts and where a [pooling] attach/build happens. Zero overhead when the sampler is + * off — these are integer bumps on paths that already do real build work. + */ +const graphileCounters: GraphileCounters = { + builds: 0, + poolingAttaches: 0, + poolingBuilds: 0, + buildWaitTimeouts: 0 +}; + +/** Snapshot the build counters. Returns a copy so callers cannot mutate live state. */ +export function getGraphileCounters(): GraphileCounters { + return { ...graphileCounters }; +} + const log = new Logger('graphile'); const reqLabel = (req: Request): string => (req.requestId ? `[${req.requestId}]` : '[req]'); @@ -203,25 +323,52 @@ const reqLabel = (req: Request): string => (req.requestId ? `[${req.requestId}]` * (everything on except aggregates). */ const buildPreset = ( - pool: import('pg').Pool, + pool: Pool, schemas: string[], anonRole: string, roleName: string, databaseSettings?: DatabaseSettings, + options?: { pooling?: boolean }, ): GraphileConfig.Preset => { - return { + // When pooling, the instance is shared across tenants of the same schema-shape + // and routed per request via the rewriting pool (canonical→tenant schema- + // identifier rewrite; see ./rewrite-pool). + const pooling = options?.pooling === true; + // Pooled instances are built fully qualified against the canonical tenant's + // physical schemas; the rewriting pool swaps canonical→tenant schema + // identifiers per request (see ./rewrite-pool). Dedicated instances use the + // raw pool untouched. + // Introspection filter (opt-in via GRAPHILE_INTROSPECTION_FILTER): scope the + // instance's catalog introspection to the schemas it serves. Only active when + // the flag is on AND we have a concrete served-schema list; otherwise the pool + // selection below is byte-identical to today. Pooled instances receive the + // filter through the rewriting pool; dedicated instances get a thin filter-only + // wrapper on the raw pool. + const introspectionFilterActive = isIntrospectionFilterEnabled() && schemas.length > 0; + const servicePool = pooling + ? createRewritingPool(pool, { + canonicalSchemas: schemas, + logicalName: stripSchemaHashPrefix, + introspectionFilter: introspectionFilterActive ? { servedSchemas: schemas } : false + }) + : introspectionFilterActive + ? createIntrospectionFilterPool(pool, { servedSchemas: schemas }) + : pool; + const preset: GraphileConfig.Preset = { extends: [createConstructivePreset(databaseSettings)], plugins: [AuthCookiePlugin], pgServices: [ makePgService({ - pool, + pool: servicePool, schemas, }), ], grafserv: { graphqlPath: '/graphql', graphiqlPath: '/graphiql', - graphiql: true, + // GraphiQL (ruru) assets/handlers are per-instance overhead and an unnecessary + // prod surface — enable only in development, or explicitly via GRAPHILE_GRAPHIQL=true. + graphiql: getNodeEnv() === 'development' || process.env.GRAPHILE_GRAPHIQL === 'true', graphiqlOnGraphQLGET: false, maskError, }, @@ -232,6 +379,13 @@ const buildPreset = ( const req = (requestContext as { expressv4?: { req?: Request } })?.expressv4?.req; const context: Record = {}; + // De-closure the role names: read them from the resolved API on the request + // so a shared (pooled) instance uses the REQUESTING tenant's roles rather than + // whichever tenant happened to build it. Falls back to the build-time params. + const api = req?.api; + const role = api?.roleName ?? roleName; + const anon = api?.anonRole ?? anonRole; + if (req) { if (req.databaseId) { context['jwt.claims.database_id'] = req.databaseId; @@ -251,7 +405,7 @@ const buildPreset = ( if (req.token?.user_id) { const pgSettings: Record = { - role: roleName, + role, 'jwt.claims.token_id': req.token.id, 'jwt.claims.user_id': req.token.user_id, ...context, @@ -282,29 +436,110 @@ const buildPreset = ( pgSettings['request.id'] = req.requestId; } + // Pooled instances: hand the REQUESTING tenant's physical schema list to + // the rewriting pool via a transaction-local GUC; the wrapper maps the + // canonical schemas to these by logical name. Never set when not pooling. + if (pooling && api?.schema?.length) { + pgSettings[POOL_SCHEMAS_GUC] = JSON.stringify(api.schema); + } + return { pgSettings }; } } const anonSettings: Record = { - role: anonRole, + role: anon, ...context, }; if (req?.requestId) { anonSettings['request.id'] = req.requestId; } + // Pooled instances: hand the REQUESTING tenant's physical schema list to + // the rewriting pool via a transaction-local GUC; the wrapper maps the + // canonical schemas to these by logical name. Never set when not pooling. + if (pooling && api?.schema?.length) { + anonSettings[POOL_SCHEMAS_GUC] = JSON.stringify(api.schema); + } return { pgSettings: anonSettings, }; }, }, + }; + + if (pooling) { + // Shared (pooled) instances must not leak the canonical tenant's hashed + // schema names through tenant-facing metadata (e.g. _meta) — plugins read + // this flag and report logical schema names instead. SQL emission stays + // fully qualified (PostGraphile default). Cast: custom schema option, + // absent from base types. + (preset as any).schema = { constructivePooled: true }; + } + + return preset; }; + +/** + * Bound how long a REQUEST waits on a build, ALWAYS clearing the timeout timer. + * + * The build itself is not cancelled — makeSchema cannot be aborted — it completes + * in the background and fills the cache for retries. Resolves to the built value + * when the build wins, or `null` when the wait expires. + * + * Why the finally/clearTimeout is load-bearing: `Promise.race` subscribes reactions + * to BOTH arms. If the timeout timer is left armed after the build wins, Node's + * internal timers list keeps the timeout arm's reaction — and, transitively through + * the race result promise, the built ~15MB instance — reachable for the full + * `timeoutMs` (default 180s). Under evict-churn (GRAPHILE_CACHE_MAX=1) every built + * instance stays pinned past its cache eviction, stacking ~2 builds/s × ~15MB × 180s + * far beyond the heap ceiling and OOMing the process. `.unref()` only removes + * event-loop keepalive, NOT GC retention — the timer must be cleared. clearTimeout() + * removes the Timeout from the timers list so the whole chain is collectable the + * instant the cache evicts the entry. + */ +/** + * Thrown from inside the build critical section when heap pressure reaches + * critical between a request's admission check and the build's actual start. + * The dispatcher maps it (for the requester and every coalesced waiter) to + * 503 SERVICE_OVERLOADED instead of a generic 500. + */ +export class BuildRefusedError extends Error { + readonly code = 'SERVICE_OVERLOADED'; + constructor(ratio: number) { + super(`Schema build refused at critical heap pressure (ratio ${ratio.toFixed(2)})`); + this.name = 'BuildRefusedError'; + } +} + +export const raceBuildAgainstTimeout = async ( + buildPromise: Promise, + timeoutMs: number, +): Promise => { + let timer: ReturnType | undefined; + const timeoutPromise = new Promise((resolve) => { + timer = setTimeout(() => resolve(null), timeoutMs); + timer.unref?.(); + }); + try { + return await Promise.race([buildPromise, timeoutPromise]); + } finally { + // Disarm the timer whether the build won, timed out, or rejected — otherwise + // the armed Timeout pins the built instance via the race reaction chain. + clearTimeout(timer); + } }; export const graphile = (opts: ConstructiveOptions): RequestHandler => { const observabilityEnabled = isGraphqlObservabilityEnabled(opts.server?.host); + // Pause between build STARTS while heap pressure is elevated (ms; 0 disables). + // Bounds the allocation rate of an eviction-churn storm so GC keeps pace. + const buildPressureSpacingMs = (() => { + const n = parseInt(process.env.GRAPHILE_BUILD_PRESSURE_SPACING_MS || '', 10); + return Number.isFinite(n) && n >= 0 ? n : 250; + })(); + return async (req: Request, res: Response, next: NextFunction) => { const label = reqLabel(req); try { @@ -313,21 +548,51 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { log.error(`${label} Missing API info`); return res.status(500).send('Missing API info'); } - const key = req.svc_key; - if (!key) { + const svcKey = req.svc_key; + if (!svcKey) { log.error(`${label} Missing service cache key`); return res.status(500).send('Missing service cache key'); } const { dbname, anonRole, roleName, schema } = api; const schemaLabel = schema?.join(',') || 'unknown'; + // ========================================================================= + // Blueprint Pooling Decision (opt-in via GRAPHILE_BLUEPRINT_POOLING) + // + // When enabled, resolve a shared blueprint key so tenants of the same + // schema-shape attach to ONE instance, routed per request by the rewriting + // pool (the requesting tenant's schema list is set in grafast.context). + // Decisions are memoized per svc_key and cleared on flush. When the flag is + // OFF this block is skipped entirely: `key` stays exactly req.svc_key and no + // extra pool or queries are touched. + // ========================================================================= + const pgConfig = getPgEnvOptions({ ...opts.pg, database: dbname }); + let key = svcKey; + let pooling = false; + let pool: Pool | undefined; + if (isBlueprintPoolingEnabled() && api) { + // Blueprint keying needs the tenant pool to fingerprint the schema shape. + pool = getPgPool(pgConfig); + const decision = await resolvePoolDecision(svcKey, api, pool); + key = decision.key; + pooling = decision.pooling; + if (pooling) { + graphileCounters.poolingAttaches += 1; + log.info(`[pooling] svc=${svcKey} → ${key}`); + } + } + // ========================================================================= // Phase A: Cache Check (fast path) // ========================================================================= const cached = graphileCache.get(key); if (cached) { - log.debug(`${label} PostGraphile cache hit key=${key} db=${dbname} schemas=${schemaLabel}`); - return cached.handler(req, res, next); + if (invokeEntryHandler(cached, req, res, next)) { + log.debug(`${label} PostGraphile cache hit key=${key} db=${dbname} schemas=${schemaLabel}`); + return; + } + // Entry is mid-disposal — fall through and rebuild. + log.debug(`${label} PostGraphile cache hit on disposing entry key=${key}; rebuilding`); } log.debug(`${label} PostGraphile cache miss key=${key} db=${dbname} schemas=${schemaLabel}`); @@ -340,7 +605,11 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { log.debug(`${label} Coalescing request for PostGraphile[${key}] - waiting for in-flight creation`); try { const instance = await inFlight; - return instance.handler(req, res, next); + if (invokeEntryHandler(instance, req, res, next)) { + return; + } + log.debug(`${label} Coalesced instance already disposing for PostGraphile[${key}], retrying`); + // Fall through to Phase C to retry creation } catch (error) { log.warn(`${label} Coalesced request failed for PostGraphile[${key}], retrying`); // Fall through to Phase C to retry creation @@ -353,55 +622,180 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { // Re-check cache after coalesced request failure (another retry may have succeeded) const recheckedCache = graphileCache.get(key); - if (recheckedCache) { + if (recheckedCache && invokeEntryHandler(recheckedCache, req, res, next)) { log.debug(`${label} PostGraphile cache hit on re-check key=${key}`); - return recheckedCache.handler(req, res, next); + return; } // Re-check in-flight map (another retry may have started creation) const retryInFlight = creating.get(key); if (retryInFlight) { log.debug(`${label} Re-coalescing request for PostGraphile[${key}]`); - const retryInstance = await retryInFlight; - return retryInstance.handler(req, res, next); + try { + const retryInstance = await retryInFlight; + if (invokeEntryHandler(retryInstance, req, res, next)) { + return; + } + log.warn(`${label} Re-coalesced instance already disposing for PostGraphile[${key}]`); + return res.status(503).json({ + error: { code: 'SERVICE_ROTATING', message: 'Service is restarting, please retry' } + }); + } catch (error) { + // A refusal surfacing on the re-coalesce await is bound by the same + // contract as the owner path (763) and the request-time gate (640): + // 503 SERVICE_OVERLOADED + Retry-After for every coalesced waiter, never + // a generic 500. Other rejections fall through to retry the build here, + // exactly as Phase B does (605). + if (error instanceof BuildRefusedError) { + log.warn(`${label} ${error.message} key=${key}`); + res.setHeader('Retry-After', '15'); + return res.status(503).json({ + error: { code: 'SERVICE_OVERLOADED', message: 'Server is at critical memory pressure; retry shortly' } + }); + } + log.warn(`${label} Re-coalesced request failed for PostGraphile[${key}], retrying`); + // Fall through to build below. + } + } + + // Memory governor gate: at critical heap pressure a build's transient + // allocation (hundreds of MB to >700MB on large catalogs) would abort the + // whole process. Resident instances keep serving; only NEW builds refuse. + const pressure = shouldRefuseBuild(); + if (pressure.refuseBuild) { + res.setHeader('Retry-After', '15'); + return res.status(503).json({ + error: { + code: 'SERVICE_OVERLOADED', + message: 'Server is at critical memory pressure; retry shortly' + } + }); } log.info( `${label} Building PostGraphile v5 handler key=${key} db=${dbname} schemas=${schemaLabel} role=${roleName} anon=${anonRole}`, ); - const pgConfig = getPgEnvOptions({ - ...opts.pg, - database: dbname, - }); - // Route through pg-cache so the pool is tracked and can be cleaned up - // properly, preventing leaked connections during database teardown. - const pool = getPgPool(pgConfig); + // properly, preventing leaked connections during database teardown. When the + // pooling decision above already resolved the pool, reuse it (pg-cache memoizes + // regardless); otherwise resolve it here exactly as before (flag-off path). + const activePool = pool ?? getPgPool(pgConfig); // Create promise and store in in-flight map BEFORE try block - const preset = buildPreset(pool, schema || [], anonRole, roleName, api.databaseSettings); - const creationPromise = observeGraphileBuild( - { - cacheKey: key, - serviceKey: key, - databaseId: api.databaseId ?? null, - }, - () => createGraphileInstance({ - preset, - cacheKey: key, - enableRealtime: api.databaseSettings?.enableRealtime, - }), - { enabled: observabilityEnabled }, - ); + const preset = buildPreset(activePool, schema || [], anonRole, roleName, api.databaseSettings, { pooling }); + const creationPromise = (async (): Promise => { + // Serialize builds process-wide: each build transiently allocates hundreds of MB, + // and only same-key builds are deduped by the single-flight map above. + await buildSemaphore.acquire(); + try { + // While queued for the slot another request may have finished this key. + const builtMeanwhile = graphileCache.get(key); + if (builtMeanwhile) { + return builtMeanwhile; + } + // Evict the LRU instance BEFORE building so the build peak lands on freed + // headroom instead of stacking on a full cache. + ensureCacheHeadroom(1); + // Evicted != freed: a draining instance's ~GB stays live until its + // in-flight requests finish and release completes. Stacking the build + // transient on undrained instances OOMed the soak — wait them out + // (bounded; returns immediately when heap pressure is already ok). + await waitForDrainSettle(); + // The request-time pressure gate can be seconds stale by the time a + // queued build reaches this point: under an eviction-churn storm the + // dispatcher sustains tens of builds/s, each retaining ~15MB until GC + // catches up, so heap can cross from ok to fatal between two request + // admissions. Re-check at the point of allocation — and under elevated + // pressure pause once (semaphore held, so this spaces GLOBAL build + // starts) to give mark-compact a window before committing the next + // transient. + let latePressure = getMemoryPressure(); + if (latePressure.level === 'elevated' && buildPressureSpacingMs > 0) { + await new Promise((resolve) => setTimeout(resolve, buildPressureSpacingMs)); + latePressure = getMemoryPressure(); + } + if (latePressure.level === 'critical') { + const refusal = shouldRefuseBuild(); + if (refusal.refuseBuild) { + throw new BuildRefusedError(refusal.ratio); + } + } + // A build genuinely starts here: past single-flight coalescing, past the + // build semaphore, and past the built-meanwhile re-check. + graphileCounters.builds += 1; + if (pooling) { + graphileCounters.poolingBuilds += 1; + } + return await observeGraphileBuild( + { + cacheKey: key, + serviceKey: key, + databaseId: api.databaseId ?? null, + }, + () => createGraphileInstance({ + preset, + cacheKey: key, + dbname, + enableRealtime: api.databaseSettings?.enableRealtime, + }), + { enabled: observabilityEnabled }, + ); + } finally { + buildSemaphore.release(); + } + })(); creating.set(key, creationPromise); + // Populate the cache and clear the in-flight slot when the BUILD settles + // (not when this request finishes): a request that stops waiting + // (build-timeout below) must leave coalescing intact for followers, and + // the finished build must land in the cache for their retries. + creationPromise + .then((instance) => { + graphileCache.set(key, instance); + log.info(`${label} Cached PostGraphile v5 handler key=${key} db=${dbname}`); + }) + .catch(() => { + /* logged by the awaiting branch (or a coalesced waiter) below */ + }) + .finally(() => { + creating.delete(key); + }); + try { - const instance = await creationPromise; - graphileCache.set(key, instance); - log.info(`${label} Cached PostGraphile v5 handler key=${key} db=${dbname}`); - return instance.handler(req, res, next); + // Bound how long a REQUEST waits on a build (queue time + build time). + const timeoutMs = (() => { + const n = parseInt(process.env.GRAPHILE_BUILD_TIMEOUT_MS || '', 10); + return Number.isFinite(n) && n > 0 ? n : 180_000; + })(); + const instance = await raceBuildAgainstTimeout(creationPromise, timeoutMs); + if (instance === null) { + graphileCounters.buildWaitTimeouts += 1; + log.warn(`${label} Request timed out after ${timeoutMs}ms waiting for PostGraphile[${key}] build`); + res.setHeader('Retry-After', '15'); + return res.status(503).json({ + error: { code: 'BUILD_TIMEOUT', message: 'Schema build in progress; retry shortly' } + }); + } + if (invokeEntryHandler(instance, req, res, next)) { + return; + } + log.warn(`${label} Freshly built instance already disposing for PostGraphile[${key}]`); + return res.status(503).json({ + error: { code: 'SERVICE_ROTATING', message: 'Service is restarting, please retry' } + }); } catch (error) { + // Pressure refusal from inside the critical section: same contract as the + // request-time gate — 503 + Retry-After for the requester and every + // coalesced waiter, never a generic 500. + if (error instanceof BuildRefusedError) { + log.warn(`${label} ${error.message} key=${key}`); + res.setHeader('Retry-After', '15'); + return res.status(503).json({ + error: { code: 'SERVICE_OVERLOADED', message: 'Server is at critical memory pressure; retry shortly' } + }); + } log.error(`${label} Failed to create PostGraphile[${key}]:`, error); throw new HandlerCreationError( `Failed to create handler for ${key}: ${error instanceof Error ? error.message : String(error)}`, @@ -410,9 +804,6 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { cause: error instanceof Error ? error.message : String(error), }, ); - } finally { - // Always clean up in-flight tracker - creating.delete(key); } } catch (e: any) { log.error(`${label} PostGraphile middleware error`, e); diff --git a/graphql/server/src/middleware/introspection-filter.ts b/graphql/server/src/middleware/introspection-filter.ts new file mode 100644 index 0000000000..9f68fea734 --- /dev/null +++ b/graphql/server/src/middleware/introspection-filter.ts @@ -0,0 +1,456 @@ +import { Logger } from '@pgpmjs/logger'; + +// ============================================================================= +// Introspection filter: scope PostGraphile v5 catalog introspection to the +// namespaces an instance actually serves. +// +// PostGraphile v5's introspection SQL (pg-introspection makeIntrospectionQuery) +// is UNSCOPED: it ingests the entire pg_class catalog (tens of thousands of rows +// on a shared multi-tenant database) into every built instance, and the parsed +// result's internal cross-references pin ~1.4GB of heap per instance regardless +// of how few schemas that instance serves. This module intercepts the single, +// static, parameter-less introspection statement as it flows through the pg pool +// wrapper we already own and rewrites its four namespace gates so only the +// served schemas (plus their transitive cross-schema references, plus public and +// pg_catalog) are ingested. +// +// It is intentionally FAIL-OPEN: any shape it does not recognize (gate count != +// 4, a non-introspection query, an empty served-schema set, the feature flag +// off) falls straight back to today's stock behavior. The one non-open path is a +// throwing discovery query — that fails the instance build, which the caller +// already retries on later requests, and is preferable to serving an instance +// built on an under-scoped (data-losing) catalog. +// +// The rewrite touches SQL STRING LITERALS only (single-quoted schema names in an +// `= any(array[...])`); it never emits double-quoted identifiers, so the sibling +// rewrite-pool's identifier lexer neither sees nor rewrites anything here. +// ============================================================================= + +const log = new Logger('graphile:introspection-filter'); + +// The exact namespace-gate substring emitted by pg-introspection@1.0.1's +// makeIntrospectionQuery(). It appears EXACTLY 4 times (classes.relnamespace, +// constraints.connamespace, procs.pronamespace, types.typnamespace). The types +// gate is followed by `or (typnamespace = 'pg_catalog'::regnamespace)` which we +// leave intact so pg_catalog types survive. NOTE the single backslash before the +// underscore: that is the runtime string (the .js source shows a doubled +// backslash only because it sits in a template literal). Verified by test +// against the installed package rather than hand-transcription. +const GATE_SUBSTRING = + "in (select namespaces._id from namespaces where nspname <> 'information_schema' and nspname not like 'pg\\_%')"; + +// Cheap structural signature of the introspection statement. Ordinary +// GraphQL-generated SQL can never satisfy all three of these simultaneously. +const INTROSPECTION_PREFIX = 'with\n database as ('; +const INTROSPECTION_MARKER = ')::text as introspection'; +const INTROSPECTION_VERSION_TOKEN = 'introspection_version'; + +// ----------------------------------------------------------------------------- +// Env flag +// ----------------------------------------------------------------------------- + +/** + * Whether the introspection filter is enabled via GRAPHILE_INTROSPECTION_FILTER. + * Accepts '1' or 'true'; anything else (incl. unset) means disabled. Mirrors the + * GRAPHILE_BLUEPRINT_POOLING read in blueprint.ts. + */ +export const isIntrospectionFilterEnabled = (): boolean => { + const value = process.env.GRAPHILE_INTROSPECTION_FILTER; + return value === '1' || value === 'true'; +}; + +// ----------------------------------------------------------------------------- +// Counters +// ----------------------------------------------------------------------------- + +export interface IntrospectionFilterCounters { + /** Closure-discovery passes actually run (one per intercepted introspection). */ + discoveries: number; + /** Introspection queries whose text was swapped for a filtered one. */ + swaps: number; + /** Callback-form or otherwise unfilterable introspection queries passed through. */ + failures: number; + /** Times buildFilteredIntrospectionQuery returned null (gate count != 4). */ + gateMismatches: number; + /** Closure loops that hit the 5-round cap before converging. */ + closureTruncations: number; + /** Total namespaces retained by the most recent successful filter build. */ + keepNamespaceCount: number; +} + +/** Build a zeroed counters object (companion to the interceptor opts). */ +export const createIntrospectionFilterCounters = (): IntrospectionFilterCounters => ({ + discoveries: 0, + swaps: 0, + failures: 0, + gateMismatches: 0, + closureTruncations: 0, + keepNamespaceCount: 0 +}); + +// Process-wide sink: the default counters when an interceptor/pool is created +// without its own, so getIntrospectionFilterCounters() (read by the metrics +// sampler) reflects real activity aggregated across every filtered instance. +const moduleCounters = createIntrospectionFilterCounters(); + +/** Snapshot the process-wide introspection-filter counters (metrics sampler entry point). Returns a copy so callers cannot mutate live state. */ +export const getIntrospectionFilterCounters = (): IntrospectionFilterCounters => ({ + ...moduleCounters +}); + +/** + * Record a callback-form introspection query that had to be passed through + * unfiltered (callback semantics are never rewritten). Bumps the module sink so + * the rewrite-pool integration can report it without owning its own counters. + */ +export const noteIntrospectionCallbackPassthrough = (): void => { + moduleCounters.failures += 1; +}; + +// ----------------------------------------------------------------------------- +// Query detection +// ----------------------------------------------------------------------------- + +/** + * Cheap signature check for the pg-introspection statement. Robust against + * user/GraphQL-generated SQL: requires the length floor, the real CTE prefix, + * the trailing `)::text as introspection` marker AND the introspection_version + * token together. + */ +export const isIntrospectionQuery = (text: string): boolean => { + if (typeof text !== 'string' || text.length <= 5000) return false; + return ( + text.startsWith(INTROSPECTION_PREFIX) && + text.includes(INTROSPECTION_MARKER) && + text.includes(INTROSPECTION_VERSION_TOKEN) + ); +}; + +// ----------------------------------------------------------------------------- +// Filtered-query builder +// ----------------------------------------------------------------------------- + +/** Escape a value as a SQL string literal body (single quotes doubled). */ +const sqlLiteral = (value: string): string => "'" + value.replace(/'/g, "''") + "'"; + +const isRejectedNamespace = (name: string): boolean => + name.length === 0 || + name.includes('\u0000') || + name === 'information_schema' || + name.startsWith('pg_'); + +/** + * Rewrite the four namespace gates of the stock introspection text so only + * `keepNamespaces` (sorted, deduped, escaped) are ingested. Returns null when + * the gate substring does not appear exactly 4 times (the caller then falls back + * to the stock text). Names starting 'pg_' / equal to 'information_schema' / + * containing NUL are defensively dropped from the injected array; the intact + * `or (typnamespace = 'pg_catalog'::regnamespace)` branch keeps pg_catalog + * types. + */ +export const buildFilteredIntrospectionQuery = ( + stockText: string, + keepNamespaces: string[] +): string | null => { + const occurrences = stockText.split(GATE_SUBSTRING).length - 1; + if (occurrences !== 4) return null; + + const cleaned = Array.from(new Set(keepNamespaces.filter((n) => !isRejectedNamespace(n)))).sort(); + const arrayLiteral = + cleaned.length === 0 + ? "in (select namespaces._id from namespaces where false)" + : 'in (select namespaces._id from namespaces where nspname = any (array[' + + cleaned.map(sqlLiteral).join(', ') + + ']::text[]))'; + + return stockText.split(GATE_SUBSTRING).join(arrayLiteral); +}; + +// ----------------------------------------------------------------------------- +// Namespace-closure discovery +// ----------------------------------------------------------------------------- + +// One round of closure discovery. Given the already-kept namespaces ($1), return +// NEW namespaces referenced by objects living in kept namespaces, via three arms: +// 1. FK targets: pg_constraint(contype='f') in kept classes -> confrelid ns +// 2. attribute types: live attributes of kept classes -> atttypid ns, plus that +// type's typbasetype and typelem namespaces when nonzero +// 3. proc types: pg_proc in kept namespaces -> prorettype ns + arg-type ns +// Every arm schema-qualifies pg_catalog.* (no search_path) and excludes +// information_schema, pg_* namespaces, and names already in the kept set. +const CLOSURE_ROUND_SQL = ` +with kept as ( + select n.oid + from pg_catalog.pg_namespace n + where n.nspname = any ($1::text[]) +), +kept_classes as ( + select c.oid, c.relnamespace + from pg_catalog.pg_class c + where c.relnamespace in (select oid from kept) +), +referenced as ( + -- arm 1: foreign-key target namespaces + select tc.relnamespace as nsoid + from pg_catalog.pg_constraint con + join kept_classes kc on kc.oid = con.conrelid + join pg_catalog.pg_class tc on tc.oid = con.confrelid + where con.contype = 'f' + + union + + -- arm 2: attribute type namespaces (+ base/element type namespaces) + select t.typnamespace as nsoid + from pg_catalog.pg_attribute a + join kept_classes kc on kc.oid = a.attrelid + join pg_catalog.pg_type t on t.oid = a.atttypid + where a.attnum > 0 and not a.attisdropped + + union + select bt.typnamespace as nsoid + from pg_catalog.pg_attribute a + join kept_classes kc on kc.oid = a.attrelid + join pg_catalog.pg_type t on t.oid = a.atttypid + join pg_catalog.pg_type bt on bt.oid = t.typbasetype + where a.attnum > 0 and not a.attisdropped and t.typbasetype <> 0 + + union + select et.typnamespace as nsoid + from pg_catalog.pg_attribute a + join kept_classes kc on kc.oid = a.attrelid + join pg_catalog.pg_type t on t.oid = a.atttypid + join pg_catalog.pg_type et on et.oid = t.typelem + where a.attnum > 0 and not a.attisdropped and t.typelem <> 0 + + union + + -- arm 3: proc return-type and argument-type namespaces + select rt.typnamespace as nsoid + from pg_catalog.pg_proc p + join pg_catalog.pg_type rt on rt.oid = p.prorettype + where p.pronamespace in (select oid from kept) + + union + select at.typnamespace as nsoid + from pg_catalog.pg_proc p + cross join lateral unnest(p.proargtypes) as pat(argoid) + join pg_catalog.pg_type at on at.oid = pat.argoid + where p.pronamespace in (select oid from kept) +) +select distinct n.nspname +from referenced r +join pg_catalog.pg_namespace n on n.oid = r.nsoid +where n.nspname <> 'information_schema' + and n.nspname not like 'pg\\_%' + and n.nspname <> all ($1::text[]) +`; + +const MAX_CLOSURE_ROUNDS = 5; + +/** Minimal shape of the checked-out client the discovery needs. */ +interface RawClientLike { + query(sql: string, values?: any[]): Promise<{ rows: any[] }>; +} + +/** + * Iterate CLOSURE_ROUND_SQL in JS until a round adds nothing or the round cap is + * hit. Starting set = servedSchemas ∪ {'public'}. Runs sequential queries on the + * provided (exclusively checked-out) client. Returns the accumulated keep set and + * whether the cap truncated the walk. + */ +const discoverKeepNamespaces = async ( + client: RawClientLike, + servedSchemas: string[] +): Promise<{ keep: string[]; truncated: boolean }> => { + const keep = new Set(['public']); + for (const s of servedSchemas) { + if (typeof s === 'string' && s.length > 0) keep.add(s); + } + + let truncated = true; + for (let round = 0; round < MAX_CLOSURE_ROUNDS; round++) { + const result = await client.query(CLOSURE_ROUND_SQL, [Array.from(keep)]); + const added: string[] = []; + for (const row of result.rows) { + const name = row.nspname; + if (typeof name === 'string' && !keep.has(name)) { + keep.add(name); + added.push(name); + } + } + if (added.length === 0) { + truncated = false; + break; + } + } + + return { keep: Array.from(keep), truncated }; +}; + +// ----------------------------------------------------------------------------- +// Interceptor +// ----------------------------------------------------------------------------- + +export interface IntrospectionInterceptorOptions { + /** Physical schema names this instance serves (the discovery seed, sans public). */ + servedSchemas: string[]; + /** Optional shared counters; the module sink is used when omitted. */ + counters?: IntrospectionFilterCounters; +} + +/** + * The interceptor callback: given the stock introspection text and the raw + * checked-out client, resolves to the text to actually execute. Returns null + * (synchronously) when the filter must not apply — filter disabled, no served + * schemas, or the text is not the introspection query — so the caller runs the + * original query unchanged. + * + * On the filtering path it memoizes: the closure discovery + filtered text are + * computed once per interceptor (i.e. once per instance build) and reused for any + * subsequent introspection on the same interceptor. A gate mismatch records the + * reason, warns once, and returns the STOCK text (graceful fallback). A throwing + * discovery query rejects (the instance build fails and is retried later). + */ +export type IntrospectionInterceptor = ( + text: string, + rawClient: RawClientLike +) => Promise | null; + +export const createIntrospectionInterceptor = ( + opts: IntrospectionInterceptorOptions +): IntrospectionInterceptor => { + const counters = opts.counters ?? moduleCounters; + const servedSchemas = Array.isArray(opts.servedSchemas) ? opts.servedSchemas : []; + + let memo: Promise | null = null; + let warnedGateMismatch = false; + + return (text: string, rawClient: RawClientLike): Promise | null => { + if (!isIntrospectionFilterEnabled()) return null; + if (servedSchemas.length === 0) return null; + if (!isIntrospectionQuery(text)) return null; + + if (memo) return memo; + + memo = (async (): Promise => { + counters.discoveries += 1; + const { keep, truncated } = await discoverKeepNamespaces(rawClient, servedSchemas); + if (truncated) counters.closureTruncations += 1; + + const filtered = buildFilteredIntrospectionQuery(text, keep); + if (filtered === null) { + counters.gateMismatches += 1; + if (!warnedGateMismatch) { + warnedGateMismatch = true; + log.warn( + 'introspection gate substring not found exactly 4 times; falling back to unfiltered introspection' + ); + } + return text; + } + + counters.swaps += 1; + counters.keepNamespaceCount = keep.length; + return filtered; + })(); + + // A rejected discovery must not poison later attempts (the build is retried). + memo.catch(() => { + memo = null; + }); + + return memo; + }; +}; + +// ----------------------------------------------------------------------------- +// Dedicated-instance pool wrapper +// ----------------------------------------------------------------------------- + +export interface IntrospectionFilterPoolOptions { + /** Physical schema names this dedicated instance serves. */ + servedSchemas: string[]; + /** Optional shared counters; the module sink is used when omitted. */ + counters?: IntrospectionFilterCounters; +} + +/** + * Thin Proxy over a raw pg Pool for DEDICATED (non-pooled) instances: intercepts + * a checked-out client's introspection query and swaps it for the filtered text. + * Everything else passes through untouched. No settings parsing, identifier + * rewriting, or prepared-name logic (unlike rewrite-pool). The per-checkout + * interceptor closure is a per-connection memo; release restores nothing. + */ +export const createIntrospectionFilterPool = (pool: any, opts: IntrospectionFilterPoolOptions): any => { + const counters = opts.counters ?? moduleCounters; + const servedSchemas = opts.servedSchemas; + + const wrapClient = (client: any): any => { + const interceptor = createIntrospectionInterceptor({ servedSchemas, counters }); + + const interceptQuery = (rawQuery: any, thisArg: any, args: any[]): any => { + const first = args[0]; + + // Promise-form config object with a text field is the only introspection + // shape (the @dataplan/pg adaptor runs `client.query({ text })`). A callback + // last-arg means callback semantics: never filter, count and pass through. + let text: string | undefined; + if (typeof first === 'string') text = first; + else if (first && typeof first === 'object' && typeof first.text === 'string') text = first.text; + + const hasCallback = typeof args[args.length - 1] === 'function'; + + if (text !== undefined && isIntrospectionQuery(text)) { + if (hasCallback) { + counters.failures += 1; + return rawQuery.apply(thisArg, args); + } + const swapPromise = interceptor(text, { query: (sql, values) => thisArg.query(sql, values) }); + if (swapPromise === null) { + return rawQuery.apply(thisArg, args); + } + return Promise.resolve(swapPromise).then((swapped) => { + if (swapped === text) return rawQuery.apply(thisArg, args); + if (typeof first === 'string') { + return rawQuery.apply(thisArg, [swapped, ...args.slice(1)]); + } + return rawQuery.apply(thisArg, [{ ...first, text: swapped }, ...args.slice(1)]); + }); + } + + return rawQuery.apply(thisArg, args); + }; + + return new Proxy(client, { + get(target, prop, receiver) { + if (prop === 'query') { + return (...args: any[]) => interceptQuery(target.query, target, args); + } + const value = Reflect.get(target, prop, receiver); + return typeof value === 'function' ? value.bind(target) : value; + } + }); + }; + + return new Proxy(pool, { + get(target, prop, receiver) { + if (prop === 'connect') { + return (...args: any[]) => { + if (typeof args[0] === 'function') { + const cb = args[0]; + return target.connect((err: any, client: any, done: any) => { + if (err) { + cb(err, client, done); + return; + } + cb(err, wrapClient(client), done); + }); + } + return Promise.resolve(target.connect(...args)).then((client: any) => wrapClient(client)); + }; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === 'function' ? value.bind(target) : value; + } + }); +}; diff --git a/graphql/server/src/middleware/pooling-decision.ts b/graphql/server/src/middleware/pooling-decision.ts new file mode 100644 index 0000000000..a4656addef --- /dev/null +++ b/graphql/server/src/middleware/pooling-decision.ts @@ -0,0 +1,148 @@ +import { Logger } from '@pgpmjs/logger'; +import type { Pool } from 'pg'; + +import type { ApiStructure } from '../types'; +import { + computeBlueprintKey, + fetchSchemaRelations, + fingerprintFromRelations, + stripSchemaHashPrefix +} from './blueprint'; + +// ============================================================================= +// Blueprint Pooling: per-svc decision + decision cache +// +// Decides whether a service (svc_key) may attach to a SHARED PostGraphile +// instance (blueprint pooling, tenant-routed by the rewriting pool — see +// ./rewrite-pool) or must keep a per-tenant instance. +// Kept in its own module — free of the heavy graphile.ts import chain — so the +// decision logic is unit-testable without loading PostGraphile. +// ============================================================================= + +const log = new Logger('graphile:pooling'); + +export interface PoolDecision { + /** Effective graphileCache key: a `bp:` blueprint key when pooling, else the svc_key. */ + key: string; + /** Whether this svc attaches to a shared (blueprint-pooled) instance. */ + pooling: boolean; + /** + * True when the fallback came from a thrown catalog probe (a possibly + * transient DB error) rather than a structural opt-out. Transient decisions + * are NOT memoized, so the next request re-probes instead of permanently + * pinning the svc to a per-tenant instance until the next flush. + */ + transient?: boolean; + /** + * The tenant database this decision belongs to — lets flushService invalidate + * ONLY the affected database's decisions/blueprint instead of a fleet-wide + * flush on every schema:update. + */ + databaseId?: string; +} + +/** + * Memoized pooling decision per svc_key. Computing a decision runs one catalog + * probe (the shape fingerprint), so the result is cached and invalidated on + * flush (see clearPoolDecisions / flushService). + */ +const poolDecisions = new Map(); + +/** + * Clear all cached pooling decisions (explicit /flush admin route). + */ +export function clearPoolDecisions(): void { + poolDecisions.clear(); +} + +/** + * Invalidate ONLY the decisions belonging to one tenant database and return the + * `bp:` keys it was attached to (so flushService can rebuild exactly the + * affected blueprint). A schema change in tenant X: + * - X's decisions must recompute (its fingerprint may have changed), and + * - the blueprint instance X was pooled into must rebuild (its shared schema + * was derived from a catalog that included X's old shape), + * while every OTHER blueprint keeps serving untouched. A brand-new tenant has + * no memoized decisions => provisioning events are a no-op here (new tenants + * probe on first request; instances are shape-generic and need no rebuild to + * accept another same-shape tenant). + */ +export function clearPoolDecisionsForDatabase(databaseId: string): string[] { + const bpKeys = new Set(); + for (const [svcKey, decision] of poolDecisions) { + if (decision.databaseId === databaseId) { + if (decision.pooling && decision.key.startsWith('bp:')) { + bpKeys.add(decision.key); + } + poolDecisions.delete(svcKey); + } + } + return [...bpKeys]; +} + +/** + * Compute (WITHOUT caching) the blueprint-pooling decision for a service. + * + * Returns `{ pooling: false, key: svcKey }` — a per-tenant instance keyed by the + * normal svc_key — when the API opts out of pooling: realtime is enabled, there + * are no schemas, or the catalog probe throws. Otherwise returns + * `{ pooling: true, key: 'bp:...' }` so same-shape tenants share one instance. + * + * Exported for unit testing; the dispatcher uses the cached resolvePoolDecision. + */ +export const computePoolDecision = async ( + svcKey: string, + api: ApiStructure, + pool: Pool +): Promise => { + // Realtime instances hold a LISTEN/NOTIFY connection + RealtimeManager per + // instance and can't be safely shared across tenants — never pool them. + if (api.databaseSettings?.enableRealtime === true) { + return { key: svcKey, pooling: false }; + } + if (!api.schema || api.schema.length === 0) { + return { key: svcKey, pooling: false }; + } + + try { + // One catalog scan feeds the fingerprint. + const relations = await fetchSchemaRelations(pool, api.schema); + + const key = computeBlueprintKey({ + logicalSchemas: api.logicalSchemas ?? api.schema.map(stripSchemaHashPrefix), + shapeFingerprint: fingerprintFromRelations(relations), + flags: api.databaseSettings as Record | undefined, + apiName: (api as { apiName?: string | null }).apiName ?? null, + mode: 'public', + dbname: api.dbname + }); + return { key, pooling: true }; + } catch (err) { + log.warn( + `svc=${svcKey} not poolable — shape probe failed: ${err instanceof Error ? err.message : String(err)}; using per-tenant instance (not memoized)` + ); + return { key: svcKey, pooling: false, transient: true }; + } +}; + +/** + * Resolve the pooling decision for a service, memoized per svc_key. Exported for + * unit testing of the memoization/invalidation behaviour. + */ +export const resolvePoolDecision = async ( + svcKey: string, + api: ApiStructure, + pool: Pool +): Promise => { + const cached = poolDecisions.get(svcKey); + if (cached) return cached; + const decision = await computePoolDecision(svcKey, api, pool); + if (!decision.transient) { + // Stamp the owning database once, at the single memoization point, so + // flushService can invalidate per-database instead of fleet-wide. + const memoized: PoolDecision = { ...decision, databaseId: (api as any).databaseId ?? undefined }; + poolDecisions.set(svcKey, memoized); + return memoized; + } + return decision; +}; diff --git a/graphql/server/src/middleware/rewrite-pool.ts b/graphql/server/src/middleware/rewrite-pool.ts new file mode 100644 index 0000000000..528afd9cbe --- /dev/null +++ b/graphql/server/src/middleware/rewrite-pool.ts @@ -0,0 +1,696 @@ +import { createHash } from 'node:crypto'; + +import { + createIntrospectionInterceptor, + type IntrospectionInterceptor, + isIntrospectionQuery, + noteIntrospectionCallbackPassthrough +} from './introspection-filter'; + +// ============================================================================= +// Rewriting pool: per-request tenant schema rewriting for a pooled instance +// +// One PostGraphile instance is pooled across many tenants whose PostgreSQL +// schemas are shape-identical but differently prefixed (e.g. canonical +// `myapp-a1b2c3d4-app-public` vs tenant `other-9f8e7d6c-app-public`). The +// pooled instance emits fully-qualified SQL naming the CANONICAL tenant's +// schemas. This module wraps the `pg` Pool handed to the PostGraphile adaptor +// and, per request, rewrites the canonical schema identifiers to the requesting +// tenant's — plus namespaces prepared-statement names per tenant so one tenant's +// prepared plan never serves another. +// +// Tenant identity arrives inside the adaptor's pgSettings query via a custom GUC +// (POOL_SCHEMAS_GUC). Until a checkout has seen that GUC it has NO mapping; any +// query that would otherwise ship canonical SQL FAILS CLOSED rather than read +// the wrong tenant's data. +// +// The SQL rewriting is a single-pass lexer that rewrites ONLY double-quoted +// identifier tokens — never string literals, dollar-quoted bodies, comments or +// E-strings — so output is byte-identical to input except for replaced idents. +// ============================================================================= + +export const POOL_SCHEMAS_GUC = 'constructive.pool_schemas'; + +export interface RewriteCounters { + settingsParses: number; + rewrittenQueries: number; + memoHits: number; + failClosed: number; + nameRewrites: number; +} + +export interface RewritingPoolOptions { + /** Physical schema names the pooled instance was built against. */ + canonicalSchemas: string[]; + /** Maps a physical schema name to its logical (prefix-stripped) name. Injected. */ + logicalName: (physical: string) => string; + /** LRU cap for both the needs-rewrite and rewrite-result memos. Default 2000. */ + maxMemoEntries?: number; + /** + * Per-connection cap on the prepared-statement names we issue. Defaults to the + * adaptor's own DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE parse (unset -> 100). + */ + maxPreparedNames?: number; + /** Optional shared counters; a fresh set is created when omitted. */ + counters?: RewriteCounters; + /** + * When set, the pooled instance's catalog introspection query is scoped to the + * given served schemas (plus their transitive cross-schema references, public + * and pg_catalog) before it runs — see ./introspection-filter. `false`/omitted + * leaves introspection unfiltered (today's behavior). + */ + introspectionFilter?: { servedSchemas: string[] } | false; +} + +/** Build a zeroed counters object (companion to RewritingPoolOptions.counters). */ +export function createRewriteCounters(): RewriteCounters { + return { + settingsParses: 0, + rewrittenQueries: 0, + memoHits: 0, + failClosed: 0, + nameRewrites: 0 + }; +} + +// Process-wide counters: the default sink when a pool is created without its own +// counters, so getRewritePoolCounters() (read by the metrics sampler) reflects +// real pooled activity aggregated across every pooled instance. +const moduleCounters = createRewriteCounters(); + +/** Snapshot the process-wide rewrite-pool counters (metrics sampler entry point). Returns a copy so callers cannot mutate live state. */ +export function getRewritePoolCounters(): RewriteCounters { + return { ...moduleCounters }; +} + +// ----------------------------------------------------------------------------- +// SQL lexer +// ----------------------------------------------------------------------------- + +const isIdentChar = (ch: string | undefined): boolean => { + if (ch === undefined) return false; + return ( + (ch >= 'a' && ch <= 'z') || + (ch >= 'A' && ch <= 'Z') || + (ch >= '0' && ch <= '9') || + ch === '_' || + ch === '$' + ); +}; + +const isTagStartChar = (ch: string | undefined): boolean => { + if (ch === undefined) return false; + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '_'; +}; + +const isTagChar = (ch: string | undefined): boolean => + isTagStartChar(ch) || (ch !== undefined && ch >= '0' && ch <= '9'); + +/** + * If `text[i]` opens a dollar-quote tag (`$$` or `$tag$` where the tag follows + * unquoted-identifier rules), return the full opening tag (e.g. `$$`, `$body$`); + * otherwise null. Positional params like `$1` return null (digit-led tag). + */ +const matchDollarTag = (text: string, i: number): string | null => { + // text[i] === '$' + const next = text[i + 1]; + if (next === '$') return '$$'; + if (!isTagStartChar(next)) return null; + let j = i + 2; + while (j < text.length && isTagChar(text[j])) j++; + if (text[j] === '$') return text.slice(i, j + 1); + return null; +}; + +interface IdentifierToken { + /** Index of the opening `"`. */ + start: number; + /** Index just past the closing `"`. */ + end: number; + /** Unescaped identifier value (`""` collapsed to `"`). */ + value: string; +} + +/** + * Single left-to-right scan collecting every double-quoted identifier token, + * skipping over string literals, E-strings, dollar-quoted bodies, and line / + * (nested) block comments. This is the ONLY place identifier boundaries are + * decided; both public helpers below reuse it. + */ +const scanIdentifierTokens = (text: string): IdentifierToken[] => { + const tokens: IdentifierToken[] = []; + const n = text.length; + let i = 0; + + while (i < n) { + const ch = text[i]; + + // Line comment -- ... to end of line + if (ch === '-' && text[i + 1] === '-') { + i += 2; + while (i < n && text[i] !== '\n') i++; + continue; + } + + // Block comment /* ... */ (nested) + if (ch === '/' && text[i + 1] === '*') { + i += 2; + let depth = 1; + while (i < n && depth > 0) { + if (text[i] === '/' && text[i + 1] === '*') { + depth++; + i += 2; + } else if (text[i] === '*' && text[i + 1] === '/') { + depth--; + i += 2; + } else { + i++; + } + } + continue; + } + + // Double-quoted identifier — the only token we ever rewrite + if (ch === '"') { + const start = i; + i++; + let value = ''; + while (i < n) { + if (text[i] === '"') { + if (text[i + 1] === '"') { + value += '"'; + i += 2; + continue; + } + i++; + break; + } + value += text[i]; + i++; + } + tokens.push({ start, end: i, value }); + continue; + } + + // Dollar-quoted string $tag$ ... $tag$ + if (ch === '$') { + const tag = matchDollarTag(text, i); + if (tag !== null) { + i += tag.length; + const close = text.indexOf(tag, i); + i = close === -1 ? n : close + tag.length; + continue; + } + } + + // E-string: a standalone e/E immediately followed by a quote. Both \\-style + // backslash escapes AND '' count; a backslash escapes the next char. + if ((ch === 'e' || ch === 'E') && text[i + 1] === "'" && !isIdentChar(text[i - 1])) { + i += 2; + while (i < n) { + const c = text[i]; + if (c === '\\') { + i += 2; + continue; + } + if (c === "'") { + if (text[i + 1] === "'") { + i += 2; + continue; + } + i++; + break; + } + i++; + } + continue; + } + + // Regular single-quoted string literal ('' is an escaped quote) + if (ch === "'") { + i++; + while (i < n) { + if (text[i] === "'") { + if (text[i + 1] === "'") { + i += 2; + continue; + } + i++; + break; + } + i++; + } + continue; + } + + i++; + } + + return tokens; +}; + +/** + * Rewrite ONLY double-quoted identifier tokens whose unescaped value is a key of + * `map`. String literals, dollar-quoted bodies, comments, E-strings and + * unquoted words are never touched. Output is byte-identical to input except for + * replaced tokens (a replacement is re-quoted with embedded `"` doubled). + */ +export function rewriteQuotedIdentifiers(text: string, map: Map): string { + const tokens = scanIdentifierTokens(text); + if (tokens.length === 0) return text; + + let out = ''; + let last = 0; + for (const tok of tokens) { + const replacement = map.get(tok.value); + // Non-matches keep their original bytes: they stay inside the next verbatim + // slice because `last` only advances past tokens we actually replace. + if (replacement === undefined) continue; + out += text.slice(last, tok.start); + out += '"' + replacement.replace(/"/g, '""') + '"'; + last = tok.end; + } + if (last === 0) return text; + out += text.slice(last); + return out; +} + +/** + * True iff `text` contains any of `names` as a complete double-quoted identifier + * token (same lexer as rewriteQuotedIdentifiers — matches on the unescaped + * value, so an identifier merely prefixing a longer one does not count). + */ +export function containsQuotedIdentifier(text: string, names: Set): boolean { + const tokens = scanIdentifierTokens(text); + for (const tok of tokens) { + if (names.has(tok.value)) return true; + } + return false; +} + +// ----------------------------------------------------------------------------- +// Small insertion-order LRU (bounded memo). A stored `false`/'' is a real value, +// so callers distinguish a miss via `has()` rather than an undefined return. +// ----------------------------------------------------------------------------- + +class LruCache { + private readonly map = new Map(); + constructor(private readonly max: number) {} + + has(key: string): boolean { + return this.map.has(key); + } + + get(key: string): V | undefined { + if (!this.map.has(key)) return undefined; + const value = this.map.get(key); + // Touch: move to most-recently-used position. + this.map.delete(key); + this.map.set(key, value); + return value; + } + + set(key: string, value: V): void { + if (this.map.has(key)) this.map.delete(key); + this.map.set(key, value); + if (this.map.size > this.max) { + const oldest = this.map.keys().next().value; + this.map.delete(oldest); + } + } +} + +// ----------------------------------------------------------------------------- +// Pool / client wrapper +// ----------------------------------------------------------------------------- + +interface CheckoutState { + /** Canonical -> tenant physical schema map; null until a settings query lands. */ + tenantMap: Map | null; + /** Per-tenant namespace tag; null in identity mode (canonical querying itself). */ + tenantTag: string | null; + /** Set when the tenant schema set can't satisfy the canonical shape (fail closed). */ + invalidReason: string | null; + /** Per-checkout introspection interceptor (memoizes discovery); null when the filter is off. */ + introspectionInterceptor: IntrospectionInterceptor | null; +} + +const freshState = (): CheckoutState => ({ + tenantMap: null, + tenantTag: null, + invalidReason: null, + introspectionInterceptor: null +}); + +const sha256hex = (input: string): string => createHash('sha256').update(input).digest('hex'); + +/** Deterministic per-tenant prepared-statement name: same tenant+name -> same result. */ +const hashName = (tenantTag: string, name: string): string => + 'bp_' + sha256hex(tenantTag + ':' + name).slice(0, 24); + +// The adaptor's fixed pgSettings statement (@dataplan/pg adaptors/pg.js: the +// `select set_config(el->>0, el->>1, true) from json_array_elements($1::json) el` +// it issues with the JSON-serialized pgSettings as $1). Settings detection MUST +// key on THIS TEXT — never on parameter contents alone: a data query whose +// user-controlled bind value merely contains the GUC substring must never be +// classified as the settings query, or it would be forwarded UNREWRITTEN (canonical +// SQL on the tenant connection) and its value could remap or null the checkout's +// tenant state for sibling queries. This literal byte-matches SQL emitted upstream. +// Exported ONLY for the pinning test that asserts it against the installed +// @dataplan/pg source — a dependency bump that changes the statement must fail +// loudly at test time, not silently fail-closed at runtime. +export const SETTINGS_QUERY_TEXT = + 'select set_config(el->>0, el->>1, true) from json_array_elements($1::json) el'; + +const isSettingsText = (text: string): boolean => text === SETTINGS_QUERY_TEXT; + +const isSettingsValues = (values: any): boolean => + Array.isArray(values) && + values.length > 0 && + typeof values[0] === 'string' && + values[0].includes(POOL_SCHEMAS_GUC); + +// Per-connection prepared-statement state lives on the RAW client under this +// module-private Symbol: prepared statements are physical-connection scoped and +// must survive checkout/release (unlike per-checkout tenant state). +const BP_PREPARED = Symbol('constructive.rewrite-pool.prepared'); + +/** + * Replicate @dataplan/pg's DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE parse: + * unset / empty / non-numeric -> 100; an explicit 0 -> 0 (upstream then disables + * prepared statements entirely, so our LRU is never exercised). + */ +const parsePreparedStatementCacheSize = (raw: string | undefined): number => { + const fromEnv = raw ? parseInt(raw, 10) : null; + return !!fromEnv || fromEnv === 0 ? fromEnv : 100; +}; + +export function createRewritingPool(pool: any, opts: RewritingPoolOptions): any { + const counters = opts.counters ?? moduleCounters; + const maxMemoEntries = opts.maxMemoEntries ?? 2000; + const maxPreparedNames = + opts.maxPreparedNames ?? + parsePreparedStatementCacheSize(process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE); + const canonicalSchemas = opts.canonicalSchemas; + const canonicalNames = new Set(canonicalSchemas); + const logicalName = opts.logicalName; + + // Caches live at pool scope (shared across checkouts). `needs` depends only on + // the text; the rewrite result is disambiguated per tenant via tenantTag. + const needsCache = new LruCache(maxMemoEntries); + const rewriteCache = new LruCache(maxMemoEntries); + + const needsRewrite = (text: string): boolean => { + if (needsCache.has(text)) return needsCache.get(text); + const result = containsQuotedIdentifier(text, canonicalNames); + needsCache.set(text, result); + return result; + }; + + const rewriteText = (tenantTag: string, text: string, map: Map): string => { + const key = tenantTag + ' ' + text; + if (rewriteCache.has(key)) { + counters.rewrittenQueries += 1; + counters.memoHits += 1; + return rewriteCache.get(key); + } + const out = rewriteQuotedIdentifiers(text, map); + rewriteCache.set(key, out); + counters.rewrittenQueries += 1; + return out; + }; + + // We rename prepared statements to bp_, which defeats the adaptor's own + // dispose (it keys its LRU by the ORIGINAL name, but pg's parsedStatements is + // now keyed by our hashed name, so its `deallocate ` never fires). + // So the wrapper OWNS the lifecycle: an insertion-ordered LRU of hashed names, + // kept on the RAW client so it survives release, evicts the oldest and fires a + // best-effort DEALLOCATE for it once the per-connection cap is exceeded. + const trackPreparedName = (rawClient: any, hashedName: string): void => { + if (maxPreparedNames <= 0) return; + let store: Map = rawClient[BP_PREPARED]; + if (!store) { + store = new Map(); + rawClient[BP_PREPARED] = store; + } + if (store.has(hashedName)) { + // Refresh recency. + store.delete(hashedName); + store.set(hashedName, true); + return; + } + store.set(hashedName, true); + if (store.size > maxPreparedNames) { + const evicted = store.keys().next().value; + store.delete(evicted); + // Fire-and-forget on the raw connection (hashed names need no escaping). + Promise.resolve(rawClient.query('deallocate "' + evicted + '"')) + .then(() => { + // Mirror the adaptor's dispose: node-postgres tracks prepared names in + // connection.parsedStatements. Without clearing the entry, re-preparing + // the evicted name later would skip the Parse step and fail on PG with + // "prepared statement does not exist". + const parsed = rawClient.connection?.parsedStatements; + if (parsed) delete parsed[evicted]; + }) + .catch((err: any) => { + // eslint-disable-next-line no-console + console.error('[rewrite-pool] failed to deallocate prepared statement ' + evicted, err); + }); + } + }; + + // Parse the adaptor's pgSettings query and (re)build the checkout's tenant map. + const applySettings = (text: string, values: any[], state: CheckoutState): void => { + // Defense in depth: only the adaptor's fixed settings statement may mutate + // checkout tenant state. The call site already gates on this text; guard again + // so no stray query whose parameter merely contains the GUC substring can + // suppress rewriting or remap the checkout via parameter contents alone. + if (!isSettingsText(text)) return; + counters.settingsParses += 1; + + let tenantSchemas: string[] | null = null; + try { + const parsed = JSON.parse(values[0]); + let raw: any; + if (Array.isArray(parsed)) { + for (const entry of parsed) { + if (Array.isArray(entry) && entry[0] === POOL_SCHEMAS_GUC) { + raw = entry[1]; + break; + } + } + } else if (parsed && typeof parsed === 'object') { + raw = parsed[POOL_SCHEMAS_GUC]; + } + if (typeof raw === 'string') tenantSchemas = JSON.parse(raw); + else if (Array.isArray(raw)) tenantSchemas = raw; + } catch { + tenantSchemas = null; + } + + if (!Array.isArray(tenantSchemas)) { + state.tenantMap = null; + state.tenantTag = null; + state.invalidReason = `malformed ${POOL_SCHEMAS_GUC} value`; + return; + } + + const tenantByLogical = new Map(); + for (const t of tenantSchemas) tenantByLogical.set(logicalName(t), t); + + const map = new Map(); + for (const c of canonicalSchemas) { + const logical = logicalName(c); + const t = tenantByLogical.get(logical); + if (t === undefined) { + state.tenantMap = null; + state.tenantTag = null; + state.invalidReason = `canonical schema "${c}" (logical "${logical}") has no tenant schema in [${tenantSchemas.join(', ')}]`; + return; + } + if (t !== c) map.set(c, t); + } + + state.tenantMap = map; + state.invalidReason = null; + if (map.size === 0) { + // Canonical tenant querying itself: identity mode, no rewriting/namespacing. + state.tenantTag = null; + } else { + state.tenantTag = sha256hex([...tenantSchemas].sort().join(',')).slice(0, 16); + } + }; + + const failClosed = (state: CheckoutState, args: any[]): any => { + counters.failClosed += 1; + const reason = state.invalidReason ?? 'no tenant schema mapping has been applied to this connection'; + const err = new Error( + `[rewrite-pool] pooled query requires a tenant mapping but none is established on this connection: ${reason}` + ); + const last = args[args.length - 1]; + if (typeof last === 'function') { + last(err); + return undefined; + } + return Promise.reject(err); + }; + + const interceptQuery = (rawQuery: any, thisArg: any, args: any[], state: CheckoutState): any => { + const first = args[0]; + + // Introspection filter (runs BEFORE settings parsing and any needs-rewrite / + // fail-closed logic). The introspection statement carries schema names only + // as string literals, so the identifier rewrite pass never applies to it — + // we forward the swapped text directly, bypassing rewriting entirely. + if (state.introspectionInterceptor) { + let iText: string | undefined; + if (typeof first === 'string') iText = first; + else if (first && typeof first === 'object' && typeof first.text === 'string') iText = first.text; + if (iText !== undefined && isIntrospectionQuery(iText)) { + // A callback-form introspection query would break callback semantics if + // rewritten async — pass it through unfiltered and count it. + if (typeof args[args.length - 1] === 'function') { + noteIntrospectionCallbackPassthrough(); + return rawQuery.apply(thisArg, args); + } + const swap = state.introspectionInterceptor(iText, { + query: (sql: string, values?: any[]) => thisArg.query(sql, values) + }); + if (swap !== null) { + return Promise.resolve(swap).then((swapped: string) => { + if (swapped === iText) return rawQuery.apply(thisArg, args); + if (typeof first === 'string') { + return rawQuery.apply(thisArg, [swapped, ...args.slice(1)]); + } + return rawQuery.apply(thisArg, [{ ...first, text: swapped }, ...args.slice(1)]); + }); + } + // swap === null: filter disabled / no served schemas — fall through. + } + } + + let text: string; + let values: any; + let name: any; + let isConfig = false; + + if (typeof first === 'string') { + text = first; + if (Array.isArray(args[1])) values = args[1]; + } else if (first && typeof first === 'object' && typeof first.text === 'string') { + isConfig = true; + text = first.text; + values = first.values; + name = first.name; + } else { + // Unknown shape (e.g. a Submittable) — forward untouched. + return rawQuery.apply(thisArg, args); + } + + // Settings query: parse tenant identity, forward the query itself untouched. + // Classification keys on the adaptor's fixed statement TEXT (plus the GUC-bearing + // value shape), never on parameter contents alone — a data query carrying a + // user-controlled value that contains the GUC substring has different text and + // therefore falls through to the needs-rewrite / fail-closed path below. + if (isSettingsText(text) && isSettingsValues(values)) { + applySettings(text, values, state); + return rawQuery.apply(thisArg, args); + } + + let newText = text; + if (needsRewrite(text)) { + if (state.invalidReason !== null || state.tenantMap === null) { + return failClosed(state, args); + } + if (state.tenantMap.size > 0) { + newText = rewriteText(state.tenantTag, text, state.tenantMap); + } + // else identity mode: leave text unchanged. + } + + // Prepared-statement name namespacing (config form only). thisArg is the raw + // client on the checkout path (the only path where tenantTag is ever set), so + // the per-connection prepared-name LRU hangs off it. + let newName = name; + if (isConfig && name != null && state.tenantTag !== null) { + newName = hashName(state.tenantTag, String(name)); + counters.nameRewrites += 1; + trackPreparedName(thisArg, newName); + } + + if (newText === text && newName === name) { + return rawQuery.apply(thisArg, args); + } + + // Clone rather than mutate the caller's object. + if (isConfig) { + const cloned = { ...first, text: newText }; + if (newName !== name) cloned.name = newName; + return rawQuery.apply(thisArg, [cloned, ...args.slice(1)]); + } + return rawQuery.apply(thisArg, [newText, ...args.slice(1)]); + }; + + const introspectionFilter = opts.introspectionFilter; + + const wrapClient = (client: any): any => { + const state = freshState(); + // Per-checkout introspection interceptor: memoizes the closure discovery so a + // single build's introspection is filtered once and reused. A fresh interceptor + // per checkout keeps the memo connection-scoped. + if (introspectionFilter && Array.isArray(introspectionFilter.servedSchemas)) { + state.introspectionInterceptor = createIntrospectionInterceptor({ + servedSchemas: introspectionFilter.servedSchemas + }); + } + return new Proxy(client, { + get(target, prop, receiver) { + if (prop === 'query') { + return (...args: any[]) => interceptQuery(target.query, target, args, state); + } + if (prop === 'release') { + return (...args: any[]) => { + // Per-checkout state must not leak into the next borrower. + state.tenantMap = null; + state.tenantTag = null; + state.invalidReason = null; + state.introspectionInterceptor = null; + return target.release.apply(target, args); + }; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === 'function' ? value.bind(target) : value; + } + }); + }; + + return new Proxy(pool, { + get(target, prop, receiver) { + if (prop === 'connect') { + return (...args: any[]) => { + // Callback style: connect((err, client, release) => ...) + if (typeof args[0] === 'function') { + const cb = args[0]; + return target.connect((err: any, client: any, done: any) => { + if (err) { + cb(err, client, done); + return; + } + cb(err, wrapClient(client), done); + }); + } + // Promise style (used by the PostGraphile adaptor). + return Promise.resolve(target.connect(...args)).then((client: any) => wrapClient(client)); + }; + } + if (prop === 'query') { + // Pool-level one-shot: no established checkout => tenantMap stays null, + // so anything needing a rewrite fails closed; everything else passes. + return (...args: any[]) => interceptQuery(target.query, target, args, freshState()); + } + const value = Reflect.get(target, prop, receiver); + return typeof value === 'function' ? value.bind(target) : value; + } + }); +} diff --git a/graphql/server/src/server.ts b/graphql/server/src/server.ts index a88b48d9f4..3aad1050c1 100644 --- a/graphql/server/src/server.ts +++ b/graphql/server/src/server.ts @@ -10,11 +10,12 @@ import express, { Express, NextFunction, Request, RequestHandler, Response } fro import type { Server as HttpServer } from 'http'; import graphqlUpload from 'graphql-upload'; import { Pool, PoolClient } from 'pg'; -import { graphileCache, closeAllCaches } from 'graphile-cache'; +import { closeAllCaches, getMemoryPressure, graphileCache, startMemoryGovernor } from 'graphile-cache'; import { getPgPool } from 'pg-cache'; import requestIp from 'request-ip'; import type { DebugSamplerHandle } from './diagnostics/debug-sampler'; +import { installConnectionErrorGuard } from './diagnostics/connection-error-guard'; import { closeDebugDatabasePools } from './diagnostics/debug-db-snapshot'; import { isDevelopmentObservabilityMode, @@ -40,6 +41,7 @@ import { parseCookieValue, SESSION_COOKIE_NAME } from './middleware/cookie'; import { createAgenticRouter } from 'agentic-server'; import { createContextMiddleware, requestIdMiddleware } from '@constructive-io/express-context'; import { startDebugSampler } from './diagnostics/debug-sampler'; +import { collectMetricsSample, type MetricsSamplerHandle, startMetricsSampler } from './diagnostics/metrics-sampler'; const log = new Logger('server'); @@ -80,8 +82,11 @@ class Server { private closed = false; private httpServer: HttpServer | null = null; private debugSampler: DebugSamplerHandle | null = null; + private metricsSampler: MetricsSamplerHandle | null = null; + private stopGovernor: (() => void) | null = null; constructor(opts: ConstructiveOptions) { + installConnectionErrorGuard(); this.opts = getEnvOptions(opts); const effectiveOpts = this.opts; const observabilityRequested = isGraphqlObservabilityRequested(); @@ -126,6 +131,15 @@ class Server { } healthz(app); + // Operational metrics endpoint (heap/rss, cache size, build/eviction/guard + // counters, memory pressure) — same sample shape as the JSON-line sampler. + // Env-gated and loopback-only by default; front it with internal routing in + // production if remote scrape access is needed. + if (process.env.GRAPHILE_METRICS_ENDPOINT === '1') { + app.get('/metrics', localObservabilityOnly, (_req, res) => { + res.json({ ...collectMetricsSample(), memoryPressure: getMemoryPressure() }); + }); + } if (observabilityEnabled) { app.get('/debug/memory', localObservabilityOnly, debugMemory); app.get('/debug/db', localObservabilityOnly, createDebugDatabaseMiddleware(effectiveOpts)); @@ -209,6 +223,14 @@ class Server { this.app = app; this.debugSampler = observabilityEnabled ? startDebugSampler(effectiveOpts) : null; + // Gated purely on GRAPHILE_DEBUG_METRICS (not dev-only observability): this JSON-line + // sampler is designed for long production soak analysis and is a true no-op when the + // env flag is off. + this.metricsSampler = startMetricsSampler(); + // Proactive heap-pressure eviction (85%) + critical build refusal (92%) — + // the request-path gate lives in the graphile dispatcher; this timer only + // does the background evictions. Disable with GRAPHILE_MEMORY_GOVERNOR=0. + this.stopGovernor = startMemoryGovernor(); } listen(): HttpServer { @@ -320,6 +342,14 @@ class Server { await this.debugSampler.stop(); this.debugSampler = null; } + if (this.metricsSampler) { + this.metricsSampler.stop(); + this.metricsSampler = null; + } + if (this.stopGovernor) { + this.stopGovernor(); + this.stopGovernor = null; + } if (this.httpServer?.listening) { await new Promise((resolve) => this.httpServer!.close(() => resolve())); } diff --git a/packages/express-context/src/types.ts b/packages/express-context/src/types.ts index 39e0a79c56..f6a860a0e8 100644 --- a/packages/express-context/src/types.ts +++ b/packages/express-context/src/types.ts @@ -106,6 +106,12 @@ export interface ApiStructure { anonRole: string; roleName: string; schema: string[]; + /** + * Logical schema names (tenant hash prefix stripped) derived from `schema`. + * Used by blueprint pooling to key a shared PostGraphile instance by + * schema-shape rather than by physical tenant schema. + */ + logicalSchemas?: string[]; apiModules: ApiModule[]; rlsModule?: RlsModule; domains?: string[]; diff --git a/packages/perf-harness/.gitignore b/packages/perf-harness/.gitignore new file mode 100644 index 0000000000..c71e46e3f3 --- /dev/null +++ b/packages/perf-harness/.gitignore @@ -0,0 +1,3 @@ +dist/ +.smoke-out/ +perf-out/ diff --git a/packages/perf-harness/DESIGN.md b/packages/perf-harness/DESIGN.md new file mode 100644 index 0000000000..faf0846ec4 --- /dev/null +++ b/packages/perf-harness/DESIGN.md @@ -0,0 +1,168 @@ +# @constructive-io/perf-harness — Design + +Reusable performance / regression harness for the Constructive GraphQL server. +Port of the ad-hoc `scripts/scale-validate/*.mjs` tooling (built during the +2026-07 blueprint-pooling validation program) into a proper monorepo package, +so the next time a memory/scale regression appears it is re-run with one +command instead of hand-driven scripts. + +**The originals in `scripts/scale-validate/` are the behavioral source of +truth. Port faithfully: every CLI flag, default, output field, and exit code is +preserved unless this document says otherwise. Do not redesign algorithms.** + +## Package identity + +- Directory: `packages/perf-harness` (auto-included by `pnpm-workspace.yaml` glob `packages/*`; lerna derives from pnpm — no registration anywhere) +- Name: `@constructive-io/perf-harness`, version `0.1.0`, license MIT, author `Constructive ` +- Bin: `{ "perf-harness": "index.js", "cperf": "index.js" }` (dist-relative like `packages/cli`; shebang `#!/usr/bin/env node` as FIRST line of `src/index.ts` — makage preserves it; no makage config file exists or is needed) +- `main: index.js`, `module: esm/index.js`, `types: index.d.ts`, `publishConfig: { access: public, directory: dist }`, NO `files` field (no sibling has one) +- Scripts (verbatim sibling standard): + `clean: makage clean` / `prepack: npm run build` / `build: makage build` / `build:dev: makage build --dev` / `dev: ts-node ./src/index.ts` / `lint: eslint . --fix` / `test: jest --passWithNoTests` / `test:watch: jest --watch` +- dependencies: `pg ^8.21.0` (imported directly), `inquirerer ^4.9.1`, `@inquirerer/utils ^3.3.9`, `graphile-cache workspace:^` (lazy-required ONLY inside `report predict` — see below) +- devDependencies: `@types/pg ^8.20.0`, `@types/node ^22.19.11`, `makage ^0.3.0`, `ts-node ^10.9.2`, `@inquirerer/test ^1.4.3` +- Do NOT declare typescript / jest / ts-jest / eslint / graphql — root-hoisted or irrelevant. +- `tsconfig.json`, `tsconfig.esm.json`, `jest.config.js`: byte-identical to `packages/server-utils` versions (extends `../../tsconfig.json`; ESM variant outDir `dist/esm`, module es2022, declaration false; jest = ts-jest preset, testRegex `(/__tests__/.*|(\.|/)(test|spec))\.(jsx?|tsx?)$`, modulePathIgnorePatterns `['dist/*']`, NO trailing commas). +- NO eslint config file in the package (root `.eslintrc.json` governs). Code style: 2-space, single quotes, semicolons, no trailing commas, `_`-prefixed unused args. + +## Hard rules (safety + fidelity) + +1. **Hub guardrail**: `HUB_PORTS = [5432, 3000, 3001, 3002, 9000]`. EVERY command that dials Postgres or a server port calls `assertPortAllowed(port, allowHub)` and fails with a clear error naming the flag `--allow-hub` to override. PG port default is **5433, never 5432** (preserve the `_lib.mjs` comment's intent). +2. **Credentials**: never hardcoded. `--email` defaults to `seeder@gmail.com`; `--password` has NO default — resolution order: flag > `PERF_PASSWORD` env. Commands needing auth error out with guidance when it is missing. Never interpolate creds into shell strings (we spawn with arg arrays only). +3. **No absolute paths** in code. Repo root discovered by walking up from cwd to the first `pnpm-workspace.yaml` (`findRepoRoot()`); server command defaults to `['node', '/packages/cli/dist/index.js']`, overridable via `--server-cmd`. +4. All outputs under `--out-dir` (default `./perf-out`), created with mkdir -p. File naming: `metrics-