diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index afd5ac5b3..b6ef114f7 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -28,6 +28,8 @@ import { PrometheusScrapeProxyRouter } from "./routes/prometheus-scrape-proxy.ht import { ScraperInternalRouter } from "./routes/scraper-internal.http" import { VcsWebhookRouter } from "./routes/vcs-webhook.http" import { HttpQueryEngineLive } from "./routes/query-engine.http" +import { HttpRailwayLive } from "./routes/railway.http" +import { RailwayInternalRouter } from "./routes/railway-internal.http" import { HttpRecommendationIssuesLive } from "./routes/recommendation-issues.http" import { HttpScrapeTargetsLive } from "./routes/scrape-targets.http" import { HttpSessionReplaysLive } from "./routes/session-replay.http" @@ -58,6 +60,7 @@ import { OrgIngestKeysService } from "./services/OrgIngestKeysService" import { OrgClickHouseSettingsService } from "./services/OrgClickHouseSettingsService" import { OrganizationService } from "./services/OrganizationService" import { QueryEngineService } from "./services/QueryEngineService" +import { RailwayIntegrationService } from "./services/RailwayIntegrationService" import { RecommendationIssueService } from "./services/RecommendationIssueService" import { RawSqlChartService } from "@maple/query-engine/runtime" import { PlanetScaleDiscoveryService } from "./services/PlanetScaleDiscoveryService" @@ -103,6 +106,7 @@ const CoreServicesLive = Layer.mergeAll( // Shared with ScrapeTargetsService via layer memoization so the proxy and // the internal target list resolve sub-targets from one discovery cache. PlanetScaleDiscoveryService.layer, + RailwayIntegrationService.layer, ScrapeTargetsService.layer.pipe(Layer.provide(PlanetScaleDiscoveryService.layer)), IngestAttributeMappingService.layer, ).pipe(Layer.provideMerge(InfraLive)) @@ -233,6 +237,7 @@ const ApiRoutes = HttpApiBuilder.layer(MapleApi).pipe( Layer.provide(HttpOnboardingLive), Layer.provide(HttpOrgClickHouseSettingsLive), Layer.provide(HttpOrganizationsLive), + Layer.provide(HttpRailwayLive), Layer.provide(HttpScrapeTargetsLive), Layer.provide( Layer.mergeAll( @@ -249,6 +254,7 @@ export const AllRoutes = Layer.mergeAll( IntegrationsCallbackRouter, OAuthDiscoveryRouter, PrometheusScrapeProxyRouter, + RailwayInternalRouter, ScraperInternalRouter, VcsWebhookRouter, McpLive, diff --git a/apps/api/src/lib/RailwayApi.ts b/apps/api/src/lib/RailwayApi.ts new file mode 100644 index 000000000..ff4ba1f0f --- /dev/null +++ b/apps/api/src/lib/RailwayApi.ts @@ -0,0 +1,330 @@ +/** + * Minimal fetch-based client for Railway's public GraphQL API + * (https://backboard.railway.com/graphql/v2). Used by the api for token + * validation and project/environment/service discovery; the streaming/polling + * paths live in apps/railway-connector. + * + * Railway has no public OAuth — orgs paste a workspace or account API token. + * Both authenticate with `Authorization: Bearer`. The endpoint is a fixed + * trusted host, so plain fetch (no SSRF validation) is fine here; tests stub + * `globalThis.fetch`. + */ +import { Effect, Option, Schema } from "effect" + +export const RAILWAY_GRAPHQL_URL = "https://backboard.railway.com/graphql/v2" + +export class RailwayApiRequestError extends Error { + readonly unauthorized: boolean + readonly status: number | null + + constructor(message: string, options?: { unauthorized?: boolean; status?: number | null }) { + super(message) + this.name = "RailwayApiRequestError" + this.unauthorized = options?.unauthorized ?? false + this.status = options?.status ?? null + } +} + +const GraphqlErrorsSchema = Schema.Array( + Schema.Struct({ + message: Schema.optionalKey(Schema.String), + }), +) + +const decodeGraphqlErrors = Schema.decodeUnknownSync(GraphqlErrorsSchema) + +export interface RailwayGraphqlResult { + readonly data: unknown + readonly errors: ReadonlyArray<{ readonly message?: string }> +} + +/** Railway signals auth failures both via HTTP 401 and via GraphQL errors like "Not Authorized". */ +const isUnauthorizedMessage = (message: string): boolean => /not authorized|unauthorized/i.test(message) + +export const railwayGraphqlQuery = Effect.fn("RailwayApi.graphqlQuery")(function* ( + token: string, + request: { readonly query: string; readonly variables?: Record }, + options?: { readonly url?: string }, +) { + const url = options?.url ?? RAILWAY_GRAPHQL_URL + + const response = yield* Effect.tryPromise({ + try: async (signal) => { + const res = await fetch(url, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ + query: request.query, + ...(request.variables === undefined ? {} : { variables: request.variables }), + }), + signal, + }) + return { status: res.status, body: await res.text() } + }, + catch: (error) => + new RailwayApiRequestError( + error instanceof Error ? error.message : "Railway API request failed", + ), + }).pipe( + Effect.timeout(15_000), + Effect.catchTag("TimeoutError", () => + Effect.fail(new RailwayApiRequestError("Railway API request timed out")), + ), + ) + + if (response.status === 401 || response.status === 403) { + return yield* Effect.fail( + new RailwayApiRequestError("Railway rejected the API token", { + unauthorized: true, + status: response.status, + }), + ) + } + if (response.status >= 400) { + return yield* Effect.fail( + new RailwayApiRequestError(`Railway API returned HTTP ${response.status}`, { + status: response.status, + }), + ) + } + + const parsed = yield* Effect.try({ + try: () => JSON.parse(response.body) as { data?: unknown; errors?: unknown }, + catch: () => new RailwayApiRequestError("Railway API returned invalid JSON"), + }) + + let errors: ReadonlyArray<{ readonly message?: string }> = [] + if (Array.isArray(parsed.errors)) { + try { + errors = decodeGraphqlErrors(parsed.errors) + } catch { + errors = [{ message: "Railway API returned unrecognized GraphQL errors" }] + } + } + + return { + data: parsed.data ?? null, + errors, + } satisfies RailwayGraphqlResult +}) + +const graphqlErrorMessage = (errors: ReadonlyArray<{ readonly message?: string }>): string => + errors + .map((error) => error.message) + .filter((message): message is string => typeof message === "string" && message.length > 0) + .join("; ") || "Railway API returned GraphQL errors" + +/** Fail when a GraphQL-level error is present, flagging auth-shaped messages as unauthorized. */ +const requireData = ( + result: RailwayGraphqlResult, +): Effect.Effect => { + if (result.errors.length > 0) { + const message = graphqlErrorMessage(result.errors) + return Effect.fail( + new RailwayApiRequestError(message, { unauthorized: isUnauthorizedMessage(message) }), + ) + } + if (result.data === null) { + return Effect.fail(new RailwayApiRequestError("Railway API response carried no data")) + } + return Effect.succeed(result.data) +} + +// ── Token validation ────────────────────────────────────────────────────────── + +const MeSchema = Schema.Struct({ + me: Schema.Struct({ + name: Schema.optionalKey(Schema.NullOr(Schema.String)), + email: Schema.optionalKey(Schema.NullOr(Schema.String)), + }), +}) + +const decodeMe = Schema.decodeUnknownEffect(MeSchema) + +export interface RailwayTokenIdentity { + readonly tokenType: "account" | "workspace" + readonly accountName: string | null +} + +// ── Discovery ───────────────────────────────────────────────────────────────── + +/** + * Relay-style connection shapes from Railway's public schema. Per-environment + * services come from `serviceInstances` (a service only runs in environments + * where it has an instance); the project-level `services` list is the fallback + * when an environment carries no instances. + */ +const connection = (node: S) => + Schema.Struct({ + edges: Schema.Array(Schema.Struct({ node })), + }) + +const ProjectsSchema = Schema.Struct({ + projects: connection( + Schema.Struct({ + id: Schema.String, + name: Schema.String, + environments: connection( + Schema.Struct({ + id: Schema.String, + name: Schema.String, + serviceInstances: Schema.optionalKey( + connection( + Schema.Struct({ + serviceId: Schema.String, + serviceName: Schema.optionalKey(Schema.NullOr(Schema.String)), + }), + ), + ), + }), + ), + services: connection( + Schema.Struct({ + id: Schema.String, + name: Schema.String, + }), + ), + }), + ), +}) + +const decodeProjects = Schema.decodeUnknownEffect(ProjectsSchema) + +const PROJECTS_QUERY = ` +query mapleDiscovery { + projects { + edges { + node { + id + name + environments { + edges { + node { + id + name + serviceInstances { + edges { + node { + serviceId + serviceName + } + } + } + } + } + } + services { + edges { + node { + id + name + } + } + } + } + } + } +} +` + +const ME_QUERY = ` +query mapleValidateToken { + me { + name + email + } +} +` + +export interface RailwayDiscoveredServiceData { + readonly id: string + readonly name: string +} + +export interface RailwayDiscoveredEnvironmentData { + readonly id: string + readonly name: string + readonly services: ReadonlyArray +} + +export interface RailwayDiscoveredProjectData { + readonly id: string + readonly name: string + readonly environments: ReadonlyArray +} + +export const fetchRailwayProjects = Effect.fn("RailwayApi.fetchProjects")(function* ( + token: string, + options?: { readonly url?: string }, +) { + const result = yield* railwayGraphqlQuery(token, { query: PROJECTS_QUERY }, options) + const data = yield* requireData(result) + const decoded = yield* decodeProjects(data).pipe( + Effect.mapError(() => new RailwayApiRequestError("Railway projects response had an unexpected shape")), + ) + + return decoded.projects.edges.map(({ node: project }): RailwayDiscoveredProjectData => { + const projectServices = project.services.edges.map(({ node }) => ({ + id: node.id, + name: node.name, + })) + return { + id: project.id, + name: project.name, + environments: project.environments.edges.map(({ node: environment }) => { + const instances = environment.serviceInstances?.edges ?? [] + const services = + instances.length > 0 + ? instances.map(({ node }) => ({ + id: node.serviceId, + name: node.serviceName ?? node.serviceId, + })) + : projectServices + return { + id: environment.id, + name: environment.name, + services, + } + }), + } + }) +}) + +/** + * Validate a pasted token and classify it. Account (personal) tokens resolve + * `me`; workspace/team tokens cannot (no user principal) but can list + * projects, so a failed `me` falls through to the projects query before the + * token is rejected. + */ +export const validateRailwayToken = Effect.fn("RailwayApi.validateToken")(function* ( + token: string, + options?: { readonly url?: string }, +) { + const me = yield* railwayGraphqlQuery(token, { query: ME_QUERY }, options).pipe( + Effect.flatMap(requireData), + Effect.flatMap((data) => + decodeMe(data).pipe( + Effect.mapError(() => new RailwayApiRequestError("Railway me response had an unexpected shape")), + ), + ), + Effect.option, + ) + + if (Option.isSome(me)) { + return { + tokenType: "account", + accountName: me.value.me.name ?? me.value.me.email ?? null, + } satisfies RailwayTokenIdentity + } + + // `me` failed — a workspace token still lists projects. Let a hard failure + // here propagate: it means the token is invalid for both principals. + yield* fetchRailwayProjects(token, options) + return { + tokenType: "workspace", + accountName: null, + } satisfies RailwayTokenIdentity +}) diff --git a/apps/api/src/routes/railway-internal.http.ts b/apps/api/src/routes/railway-internal.http.ts new file mode 100644 index 000000000..ed72f2619 --- /dev/null +++ b/apps/api/src/routes/railway-internal.http.ts @@ -0,0 +1,178 @@ +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http" +import { Effect, Option, Redacted, Schema } from "effect" +import { + InternalRailwayTarget, + OrgId, + RailwayConnectionId, + RailwayMetricsIntervalSeconds, + RailwayResultReportList, + RailwayTargetId, + RailwayTokenType, + UserId, +} from "@maple/domain/http" +import { Env } from "../lib/Env" +import { isValidInternalBearer } from "../lib/internal-auth" +import { OrgIngestKeysService } from "../services/OrgIngestKeysService" +import { RailwayIntegrationService, type EnabledRailwayTarget } from "../services/RailwayIntegrationService" + +const decodeTargetIdSync = Schema.decodeUnknownSync(RailwayTargetId) +const decodeConnectionIdSync = Schema.decodeUnknownSync(RailwayConnectionId) +const decodeOrgIdSync = Schema.decodeUnknownSync(OrgId) +const decodeTokenTypeSync = Schema.decodeUnknownSync(RailwayTokenType) +const decodeMetricsIntervalSync = Schema.decodeUnknownSync(RailwayMetricsIntervalSeconds) + +/** Audit identity for lazily-created ingest keys (org_ingest_keys.created_by). */ +const RAILWAY_SYSTEM_USER = Schema.decodeUnknownSync(UserId)("system-railway-connector") +const decodeResultsEffect = Schema.decodeUnknownEffect(RailwayResultReportList) + +const errorText = (message: string, status: number) => + HttpServerResponse.text(message, { + status, + headers: { "content-type": "text/plain; charset=utf-8" }, + }) + +/** + * Marshal a joined connection×target row into the internal wire shape. A row + * whose brands fail to decode (bad id, interval out of range) yields `none` + * so one corrupt row cannot break the whole list. + */ +export const toInternalRailwayTarget = ( + entry: EnabledRailwayTarget, + ingestKey: string, +): Option.Option => { + try { + return Option.some( + new InternalRailwayTarget({ + id: decodeTargetIdSync(entry.target.id), + orgId: entry.target.orgId, + connectionId: decodeConnectionIdSync(entry.connection.id), + tokenType: decodeTokenTypeSync(entry.connection.tokenType), + token: entry.token, + projectId: entry.target.projectId, + projectName: entry.target.projectName, + environmentId: entry.target.environmentId, + environmentName: entry.target.environmentName, + services: entry.services.map((service) => ({ id: service.id, name: service.name })), + logsEnabled: entry.target.logsEnabled, + metricsEnabled: entry.target.metricsEnabled, + metricsIntervalSeconds: decodeMetricsIntervalSync(entry.target.metricsIntervalSeconds), + logWatermarkMs: entry.target.logWatermarkAt?.getTime() ?? null, + metricsWatermarkMs: entry.target.metricsWatermarkAt?.getTime() ?? null, + ingestKey, + }), + ) + } catch { + return Option.none() + } +} + +/** + * Internal endpoints backing the standalone railway-connector + * (apps/railway-connector). The connector polls + * `/api/internal/railway-targets` for the enabled target list (decrypted + * Railway tokens + org ingest keys ride this internal wire only) and reports + * outcomes to `/api/internal/railway-results`. + */ +export const RailwayInternalRouter = HttpRouter.use((router) => + Effect.gen(function* () { + const env = yield* Env + const service = yield* RailwayIntegrationService + const ingestKeys = yield* OrgIngestKeysService + const internalToken = Option.match(env.SD_INTERNAL_TOKEN, { + onNone: () => undefined, + onSome: Redacted.value, + }) + + const unauthorized = (req: HttpServerRequest.HttpServerRequest) => { + if (!internalToken) return errorText("Railway internal endpoints are not configured", 401) + if (!isValidInternalBearer(req.headers.authorization, internalToken)) { + return errorText("Unauthorized", 401) + } + return undefined + } + + const listTargets = (req: HttpServerRequest.HttpServerRequest) => + Effect.gen(function* () { + const denied = unauthorized(req) + if (denied) return denied + + const entries = yield* service + .listAllEnabledInternal() + .pipe(Effect.catch(() => Effect.succeed([] as ReadonlyArray))) + + // One public ingest key per org (lazily created on first use). The + // connector ingests with this key so pulled logs/metrics are billed + // and warehouse-routed identically to the org's own OTLP traffic. + const keyByOrg = new Map() + const ingestKeyForOrg = (orgId: string) => + Effect.gen(function* () { + const cached = keyByOrg.get(orgId) + if (cached !== undefined) return cached + const key: string | null = yield* ingestKeys + .getOrCreate(decodeOrgIdSync(orgId), RAILWAY_SYSTEM_USER) + .pipe( + Effect.map((keys): string | null => keys.publicKey), + Effect.catch(() => Effect.succeed(null)), + ) + keyByOrg.set(orgId, key) + return key + }) + + const targets: Array = [] + yield* Effect.forEach(entries, (entry) => + Effect.gen(function* () { + const ingestKey = yield* ingestKeyForOrg(entry.target.orgId) + if (ingestKey === null) { + yield* Effect.logWarning("Skipping Railway target (no ingest key)").pipe( + Effect.annotateLogs({ + railwayTargetId: entry.target.id, + orgId: entry.target.orgId, + }), + ) + return + } + const target = toInternalRailwayTarget(entry, ingestKey) + if (Option.isSome(target)) { + targets.push(target.value) + } else { + yield* Effect.logWarning("Skipping Railway target (invalid row)").pipe( + Effect.annotateLogs({ + railwayTargetId: entry.target.id, + orgId: entry.target.orgId, + }), + ) + } + }), + ) + + return yield* HttpServerResponse.json(targets) + }).pipe(Effect.withSpan("RailwayInternal.listTargets")) + + const recordResults = (req: HttpServerRequest.HttpServerRequest) => + Effect.gen(function* () { + const denied = unauthorized(req) + if (denied) return denied + + const body = yield* req.json.pipe(Effect.option) + if (Option.isNone(body)) return errorText("Invalid JSON body", 400) + + const results = yield* decodeResultsEffect(body.value).pipe(Effect.option) + if (Option.isNone(results)) return errorText("Invalid railway results payload", 400) + + yield* service + .recordResults(results.value) + .pipe( + Effect.catch((error) => + Effect.logWarning("Failed to persist railway results").pipe( + Effect.annotateLogs({ error: error.message }), + ), + ), + ) + + return yield* HttpServerResponse.json({ recorded: results.value.length }) + }).pipe(Effect.withSpan("RailwayInternal.recordResults")) + + yield* router.add("GET", "/api/internal/railway-targets", listTargets) + yield* router.add("POST", "/api/internal/railway-results", recordResults) + }), +) diff --git a/apps/api/src/routes/railway-internal.test.ts b/apps/api/src/routes/railway-internal.test.ts new file mode 100644 index 000000000..9c8a8ba4d --- /dev/null +++ b/apps/api/src/routes/railway-internal.test.ts @@ -0,0 +1,131 @@ +import { assert, describe, it } from "@effect/vitest" +import { Option, Schema } from "effect" +import { RailwayResultReportList } from "@maple/domain/http" +import type { RailwayConnectionRow, RailwayTargetRow } from "@maple/db" +import { toInternalRailwayTarget } from "./railway-internal.http" + +const TARGET_ID = "11111111-1111-4111-8111-111111111111" +const CONNECTION_ID = "22222222-2222-4222-8222-222222222222" +const INGEST_KEY = "maple_pk_test_key" + +const connection: RailwayConnectionRow = { + id: CONNECTION_ID, + orgId: "org_1", + tokenType: "account", + tokenCiphertext: "cipher", + tokenIv: "iv", + tokenTag: "tag", + externalAccountName: "David", + connectedByUserId: "user_1", + enabled: true, + lastValidatedAt: null, + lastValidationError: null, + lastValidationErrorAt: null, + createdAt: new Date(0), + updatedAt: new Date(0), +} + +const target: RailwayTargetRow = { + id: TARGET_ID, + orgId: "org_1", + connectionId: CONNECTION_ID, + projectId: "proj-1", + projectName: "maple-demo", + environmentId: "env-1", + environmentName: "production", + enabled: true, + logsEnabled: true, + metricsEnabled: true, + metricsIntervalSeconds: 300, + servicesJson: JSON.stringify([{ id: "svc-1", name: "api" }]), + servicesDiscoveredAt: new Date(0), + logWatermarkAt: new Date(5_000), + metricsWatermarkAt: null, + lastLogAt: null, + lastMetricsAt: null, + lastSuccessAt: null, + lastError: null, + lastErrorAt: null, + createdAt: new Date(0), + updatedAt: new Date(0), +} + +describe("toInternalRailwayTarget", () => { + it("marshals a joined row with the decrypted token and ingest key", () => { + const result = toInternalRailwayTarget( + { target, connection, token: "railway-token", services: [{ id: "svc-1", name: "api" }] }, + INGEST_KEY, + ) + assert.isTrue(Option.isSome(result)) + if (Option.isNone(result)) return + assert.strictEqual(result.value.id, TARGET_ID) + assert.strictEqual(result.value.connectionId, CONNECTION_ID) + assert.strictEqual(result.value.token, "railway-token") + assert.strictEqual(result.value.tokenType, "account") + assert.strictEqual(result.value.environmentId, "env-1") + assert.deepStrictEqual( + result.value.services.map((service) => ({ id: service.id, name: service.name })), + [{ id: "svc-1", name: "api" }], + ) + assert.strictEqual(result.value.logWatermarkMs, 5_000) + assert.isNull(result.value.metricsWatermarkMs) + assert.strictEqual(result.value.ingestKey, INGEST_KEY) + }) + + it("drops rows that violate the schema brands instead of failing the list", () => { + const outOfRange = toInternalRailwayTarget( + { + target: { ...target, metricsIntervalSeconds: 5 }, + connection, + token: "railway-token", + services: [], + }, + INGEST_KEY, + ) + assert.isTrue(Option.isNone(outOfRange)) + + const badTokenType = toInternalRailwayTarget( + { + target, + connection: { ...connection, tokenType: "project" }, + token: "railway-token", + services: [], + }, + INGEST_KEY, + ) + assert.isTrue(Option.isNone(badTokenType)) + }) +}) + +describe("RailwayResultReportList decoding", () => { + const decode = Schema.decodeUnknownSync(RailwayResultReportList) + + it("accepts log and metrics reports with optional fields", () => { + const reports = decode([ + { + targetId: TARGET_ID, + kind: "logs", + at: 1_750_000_000_000, + error: null, + recordsIngested: 250, + newLogWatermarkMs: 1_750_000_000_500, + }, + { + targetId: TARGET_ID, + kind: "metrics", + at: 1_750_000_000_000, + error: "Railway API returned HTTP 429", + rateLimited: true, + }, + ]) + assert.strictEqual(reports.length, 2) + assert.strictEqual(reports[0]!.kind, "logs") + assert.strictEqual(reports[1]!.rateLimited, true) + }) + + it("rejects reports with an unknown kind", () => { + assert.throws(() => + decode([{ targetId: TARGET_ID, kind: "traces", at: 0, error: null }]), + ) + }) +}) diff --git a/apps/api/src/routes/railway.http.ts b/apps/api/src/routes/railway.http.ts new file mode 100644 index 000000000..0f96759ca --- /dev/null +++ b/apps/api/src/routes/railway.http.ts @@ -0,0 +1,76 @@ +import { HttpApiBuilder } from "effect/unstable/httpapi" +import { CurrentTenant, MapleApi, RailwayForbiddenError } from "@maple/domain/http" +import { Effect } from "effect" +import type { RoleName } from "@maple/domain/http" +import { requireAdmin as requireAdminRole } from "../lib/auth" +import { RailwayIntegrationService } from "../services/RailwayIntegrationService" + +const requireAdmin = (roles: ReadonlyArray) => + requireAdminRole( + roles, + () => + new RailwayForbiddenError({ + message: "Only organization admins can manage the Railway integration", + }), + ) + +export const HttpRailwayLive = HttpApiBuilder.group(MapleApi, "railway", (handlers) => + Effect.gen(function* () { + const service = yield* RailwayIntegrationService + + return handlers + .handle("status", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + return yield* service.status(tenant.orgId) + }), + ) + .handle("connect", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles) + return yield* service.connect(tenant.orgId, tenant.userId, payload.token) + }), + ) + .handle("disconnect", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles) + return yield* service.disconnect(tenant.orgId) + }), + ) + .handle("discovery", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + return yield* service.discover(tenant.orgId) + }), + ) + .handle("listTargets", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + return yield* service.listTargets(tenant.orgId) + }), + ) + .handle("createTarget", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles) + return yield* service.createTarget(tenant.orgId, payload) + }), + ) + .handle("updateTarget", ({ params, payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles) + return yield* service.updateTarget(tenant.orgId, params.targetId, payload) + }), + ) + .handle("deleteTarget", ({ params }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireAdmin(tenant.roles) + return yield* service.deleteTarget(tenant.orgId, params.targetId) + }), + ) + }), +) diff --git a/apps/api/src/services/RailwayIntegrationService.test.ts b/apps/api/src/services/RailwayIntegrationService.test.ts new file mode 100644 index 000000000..a1b711f83 --- /dev/null +++ b/apps/api/src/services/RailwayIntegrationService.test.ts @@ -0,0 +1,425 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { ConfigProvider, Effect, Exit, Layer, Schema } from "effect" +import { + CreateRailwayTargetRequest, + OrgId, + RailwayMetricsIntervalSeconds, + RailwayResultReport, + UpdateRailwayTargetRequest, +} from "@maple/domain/http" +import { Env } from "../lib/Env" +import { cleanupTestDbs, createTestDb, type TestDb } from "../lib/test-pglite" +import { RailwayIntegrationService } from "./RailwayIntegrationService" + +const trackedDbs: TestDb[] = [] +const originalFetch = globalThis.fetch + +afterEach(async () => { + globalThis.fetch = originalFetch + await cleanupTestDbs(trackedDbs) +}) + +const makeConfig = () => + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3472", + MCP_PORT: "3473", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 9).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + }), + ) + +const makeLayer = (testDb: TestDb) => + RailwayIntegrationService.layer.pipe( + Layer.provide(testDb.layer), + Layer.provide(Env.layer), + Layer.provide(makeConfig()), + ) + +const asOrgId = Schema.decodeUnknownSync(OrgId) +const asInterval = Schema.decodeUnknownSync(RailwayMetricsIntervalSeconds) + +const ACCOUNT_TOKEN = "railway-account-token" +const WORKSPACE_TOKEN = "railway-workspace-token" + +const projectsFixture = { + projects: { + edges: [ + { + node: { + id: "proj-1", + name: "maple-demo", + environments: { + edges: [ + { + node: { + id: "env-1", + name: "production", + serviceInstances: { + edges: [ + { node: { serviceId: "svc-1", serviceName: "api" } }, + { node: { serviceId: "svc-2", serviceName: "worker" } }, + ], + }, + }, + }, + { + node: { + id: "env-2", + name: "staging", + serviceInstances: { edges: [] }, + }, + }, + ], + }, + services: { + edges: [ + { node: { id: "svc-1", name: "api" } }, + { node: { id: "svc-2", name: "worker" } }, + ], + }, + }, + }, + ], + }, +} + +/** + * Emulates Railway's GraphQL endpoint: account tokens resolve `me`, workspace + * tokens get a GraphQL "Not Authorized" for `me` but can list projects, and + * unknown tokens are rejected with HTTP 401. + */ +const stubRailwayFetch = () => { + const calls: Array<{ query: string; authorization: string | null }> = [] + globalThis.fetch = (async (input, init) => { + const headers = new Headers(init?.headers) + const authorization = headers.get("authorization") + const body = JSON.parse(String(init?.body ?? "{}")) as { query?: string } + const query = body.query ?? "" + calls.push({ query, authorization }) + + const token = authorization?.replace(/^Bearer /, "") + if (token !== ACCOUNT_TOKEN && token !== WORKSPACE_TOKEN) { + return new Response("Unauthorized", { status: 401 }) + } + + if (query.includes("mapleValidateToken")) { + if (token === WORKSPACE_TOKEN) { + return Response.json({ data: null, errors: [{ message: "Not Authorized" }] }) + } + return Response.json({ data: { me: { name: "David", email: "david@example.com" } } }) + } + + return Response.json({ data: projectsFixture }) + }) as typeof fetch + return calls +} + +describe("RailwayIntegrationService", () => { + it.effect("connect validates an account token and reports a connected status", () => { + const testDb = createTestDb(trackedDbs) + stubRailwayFetch() + return Effect.gen(function* () { + const service = yield* RailwayIntegrationService + const status = yield* service.connect(asOrgId("org_1"), "user_1", ACCOUNT_TOKEN) + + assert.isTrue(status.connected) + assert.strictEqual(status.tokenType, "account") + assert.strictEqual(status.accountName, "David") + assert.isNull(status.lastValidationError) + assert.strictEqual(status.targetCount, 0) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("connect classifies a workspace token via the projects fallback", () => { + const testDb = createTestDb(trackedDbs) + stubRailwayFetch() + return Effect.gen(function* () { + const service = yield* RailwayIntegrationService + const status = yield* service.connect(asOrgId("org_1"), "user_1", WORKSPACE_TOKEN) + + assert.isTrue(status.connected) + assert.strictEqual(status.tokenType, "workspace") + assert.isNull(status.accountName) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("connect rejects a token Railway does not accept", () => { + const testDb = createTestDb(trackedDbs) + stubRailwayFetch() + return Effect.gen(function* () { + const service = yield* RailwayIntegrationService + const exit = yield* service.connect(asOrgId("org_1"), "user_1", "bogus-token").pipe(Effect.exit) + + assert.isTrue(Exit.isFailure(exit)) + const status = yield* service.status(asOrgId("org_1")) + assert.isFalse(status.connected) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("discover annotates environments with existing target ids", () => { + const testDb = createTestDb(trackedDbs) + stubRailwayFetch() + return Effect.gen(function* () { + const service = yield* RailwayIntegrationService + const orgId = asOrgId("org_1") + yield* service.connect(orgId, "user_1", ACCOUNT_TOKEN) + const target = yield* service.createTarget( + orgId, + new CreateRailwayTargetRequest({ projectId: "proj-1", environmentId: "env-1" }), + ) + + const discovery = yield* service.discover(orgId) + assert.strictEqual(discovery.projects.length, 1) + const project = discovery.projects[0]! + assert.strictEqual(project.name, "maple-demo") + const production = project.environments.find((environment) => environment.id === "env-1")! + assert.strictEqual(production.targetId, target.id) + assert.deepStrictEqual( + production.services.map((service_) => service_.name), + ["api", "worker"], + ) + const staging = project.environments.find((environment) => environment.id === "env-2")! + assert.isNull(staging.targetId) + // No service instances in staging → falls back to the project-level services. + assert.strictEqual(staging.services.length, 2) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("createTarget stores names + services and rejects duplicates", () => { + const testDb = createTestDb(trackedDbs) + stubRailwayFetch() + return Effect.gen(function* () { + const service = yield* RailwayIntegrationService + const orgId = asOrgId("org_1") + yield* service.connect(orgId, "user_1", ACCOUNT_TOKEN) + + const target = yield* service.createTarget( + orgId, + new CreateRailwayTargetRequest({ projectId: "proj-1", environmentId: "env-1" }), + ) + assert.strictEqual(target.projectName, "maple-demo") + assert.strictEqual(target.environmentName, "production") + assert.strictEqual(target.serviceCount, 2) + assert.strictEqual(target.metricsIntervalSeconds, 300) + assert.isTrue(target.logsEnabled) + assert.isTrue(target.metricsEnabled) + + const duplicate = yield* service + .createTarget( + orgId, + new CreateRailwayTargetRequest({ projectId: "proj-1", environmentId: "env-1" }), + ) + .pipe(Effect.exit) + assert.isTrue(Exit.isFailure(duplicate)) + + const unknownEnv = yield* service + .createTarget( + orgId, + new CreateRailwayTargetRequest({ projectId: "proj-1", environmentId: "env-404" }), + ) + .pipe(Effect.exit) + assert.isTrue(Exit.isFailure(unknownEnv)) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("updateTarget toggles flags and deleteTarget removes the row", () => { + const testDb = createTestDb(trackedDbs) + stubRailwayFetch() + return Effect.gen(function* () { + const service = yield* RailwayIntegrationService + const orgId = asOrgId("org_1") + yield* service.connect(orgId, "user_1", ACCOUNT_TOKEN) + const target = yield* service.createTarget( + orgId, + new CreateRailwayTargetRequest({ projectId: "proj-1", environmentId: "env-1" }), + ) + + const updated = yield* service.updateTarget( + orgId, + target.id, + new UpdateRailwayTargetRequest({ + logsEnabled: false, + metricsIntervalSeconds: asInterval(120), + }), + ) + assert.isFalse(updated.logsEnabled) + assert.strictEqual(updated.metricsIntervalSeconds, 120) + assert.isTrue(updated.metricsEnabled) + + yield* service.deleteTarget(orgId, target.id) + const listed = yield* service.listTargets(orgId) + assert.strictEqual(listed.targets.length, 0) + + const missing = yield* service.deleteTarget(orgId, target.id).pipe(Effect.exit) + assert.isTrue(Exit.isFailure(missing)) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("listAllEnabledInternal returns the decrypted token and cached services", () => { + const testDb = createTestDb(trackedDbs) + stubRailwayFetch() + return Effect.gen(function* () { + const service = yield* RailwayIntegrationService + const orgId = asOrgId("org_1") + yield* service.connect(orgId, "user_1", ACCOUNT_TOKEN) + yield* service.createTarget( + orgId, + new CreateRailwayTargetRequest({ projectId: "proj-1", environmentId: "env-1" }), + ) + + const entries = yield* service.listAllEnabledInternal() + assert.strictEqual(entries.length, 1) + const entry = entries[0]! + // Round-trips through AES-256-GCM — proves the token was stored encrypted + // with the configured key, not in plaintext. + assert.strictEqual(entry.token, ACCOUNT_TOKEN) + assert.strictEqual(entry.target.environmentId, "env-1") + assert.deepStrictEqual( + entry.services.map((service_) => service_.id), + ["svc-1", "svc-2"], + ) + + // Disabled targets drop out of the work list. + const target = (yield* service.listTargets(orgId)).targets[0]! + yield* service.updateTarget(orgId, target.id, new UpdateRailwayTargetRequest({ enabled: false })) + const afterDisable = yield* service.listAllEnabledInternal() + assert.strictEqual(afterDisable.length, 0) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("recordResults advances watermarks monotonically and tracks health", () => { + const testDb = createTestDb(trackedDbs) + stubRailwayFetch() + return Effect.gen(function* () { + const service = yield* RailwayIntegrationService + const orgId = asOrgId("org_1") + yield* service.connect(orgId, "user_1", ACCOUNT_TOKEN) + const target = yield* service.createTarget( + orgId, + new CreateRailwayTargetRequest({ projectId: "proj-1", environmentId: "env-1" }), + ) + + yield* service.recordResults([ + new RailwayResultReport({ + targetId: target.id, + kind: "logs", + at: 2_000, + error: null, + recordsIngested: 10, + newLogWatermarkMs: 5_000, + }), + ]) + // An older/duplicate report must not rewind the watermark. + yield* service.recordResults([ + new RailwayResultReport({ + targetId: target.id, + kind: "logs", + at: 3_000, + error: null, + newLogWatermarkMs: 4_000, + }), + ]) + + const entries = yield* service.listAllEnabledInternal() + assert.strictEqual(entries[0]!.target.logWatermarkAt?.getTime(), 5_000) + + // A failing metrics report surfaces on target health without clearing lastLogAt. + yield* service.recordResults([ + new RailwayResultReport({ + targetId: target.id, + kind: "metrics", + at: 6_000, + error: "Railway API returned HTTP 500", + }), + ]) + const listed = yield* service.listTargets(orgId) + assert.strictEqual(listed.targets[0]!.lastError, "[metrics] Railway API returned HTTP 500") + assert.isNotNull(listed.targets[0]!.lastLogAt) + + const status = yield* service.status(orgId) + assert.strictEqual(status.erroredTargetCount, 1) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("discover flags the connection only on auth rejections, not transient outages", () => { + const testDb = createTestDb(trackedDbs) + stubRailwayFetch() + return Effect.gen(function* () { + const service = yield* RailwayIntegrationService + const orgId = asOrgId("org_1") + yield* service.connect(orgId, "user_1", ACCOUNT_TOKEN) + + // Transient Railway outage: discovery fails, but the card must NOT + // flip to "Reconnect needed". + globalThis.fetch = (async () => new Response("oops", { status: 500 })) as typeof fetch + const outage = yield* service.discover(orgId).pipe(Effect.exit) + assert.isTrue(Exit.isFailure(outage)) + const afterOutage = yield* service.status(orgId) + assert.isNull(afterOutage.lastValidationError) + + // A genuine token rejection does flag it. + globalThis.fetch = (async () => new Response("Unauthorized", { status: 401 })) as typeof fetch + const revoked = yield* service.discover(orgId).pipe(Effect.exit) + assert.isTrue(Exit.isFailure(revoked)) + const afterRevocation = yield* service.status(orgId) + assert.isNotNull(afterRevocation.lastValidationError) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("an unauthorized report flags the connection for reconnection", () => { + const testDb = createTestDb(trackedDbs) + stubRailwayFetch() + return Effect.gen(function* () { + const service = yield* RailwayIntegrationService + const orgId = asOrgId("org_1") + yield* service.connect(orgId, "user_1", ACCOUNT_TOKEN) + const target = yield* service.createTarget( + orgId, + new CreateRailwayTargetRequest({ projectId: "proj-1", environmentId: "env-1" }), + ) + + yield* service.recordResults([ + new RailwayResultReport({ + targetId: target.id, + kind: "logs", + at: 1_000, + error: "Railway rejected the API token", + unauthorized: true, + }), + ]) + + const status = yield* service.status(orgId) + assert.isNotNull(status.lastValidationError) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("disconnect removes the connection and its targets", () => { + const testDb = createTestDb(trackedDbs) + stubRailwayFetch() + return Effect.gen(function* () { + const service = yield* RailwayIntegrationService + const orgId = asOrgId("org_1") + yield* service.connect(orgId, "user_1", ACCOUNT_TOKEN) + yield* service.createTarget( + orgId, + new CreateRailwayTargetRequest({ projectId: "proj-1", environmentId: "env-1" }), + ) + + const result = yield* service.disconnect(orgId) + assert.isTrue(result.disconnected) + + const status = yield* service.status(orgId) + assert.isFalse(status.connected) + assert.strictEqual(status.targetCount, 0) + + const entries = yield* service.listAllEnabledInternal() + assert.strictEqual(entries.length, 0) + }).pipe(Effect.provide(makeLayer(testDb))) + }) +}) diff --git a/apps/api/src/services/RailwayIntegrationService.ts b/apps/api/src/services/RailwayIntegrationService.ts new file mode 100644 index 000000000..54b697a95 --- /dev/null +++ b/apps/api/src/services/RailwayIntegrationService.ts @@ -0,0 +1,759 @@ +import { randomUUID } from "node:crypto" +import { + CreateRailwayTargetRequest, + IsoDateTimeString, + OrgId, + RailwayConnectionId, + RailwayDiscoveredEnvironment, + RailwayDiscoveredProject, + RailwayDiscoveredService, + RailwayDiscoveryResponse, + RailwayDisconnectResponse, + RailwayEncryptionError, + RailwayIntegrationStatus, + RailwayMetricsIntervalSeconds, + RailwayNotFoundError, + RailwayPersistenceError, + RailwayTargetDeleteResponse, + RailwayTargetId, + RailwayTargetResponse, + RailwayTargetsListResponse, + RailwayTokenType, + RailwayUpstreamError, + RailwayValidationError, + type RailwayResultReport, + type UpdateRailwayTargetRequest, +} from "@maple/domain/http" +import { railwayConnections, railwayTargets, type RailwayConnectionRow, type RailwayTargetRow } from "@maple/db" +import { and, eq } from "drizzle-orm" +import { Clock, Context, Effect, Layer, Option, Redacted, Schema } from "effect" +import { decryptAes256Gcm, encryptAes256Gcm, parseBase64Aes256GcmKey } from "../lib/Crypto" +import { Database } from "../lib/DatabaseLive" +import { Env } from "../lib/Env" +import { + fetchRailwayProjects, + RailwayApiRequestError, + validateRailwayToken, + type RailwayDiscoveredProjectData, +} from "../lib/RailwayApi" + +const DEFAULT_METRICS_INTERVAL_SECONDS = 300 +/** Cached environment→services lists refresh at most this often (one discovery call per connection). */ +const SERVICES_REFRESH_TTL_MS = 60 * 60 * 1000 + +const decodeConnectionIdSync = Schema.decodeUnknownSync(RailwayConnectionId) +const decodeTargetIdSync = Schema.decodeUnknownSync(RailwayTargetId) +const decodeTokenTypeSync = Schema.decodeUnknownSync(RailwayTokenType) +const decodeMetricsIntervalSync = Schema.decodeUnknownSync(RailwayMetricsIntervalSeconds) +const decodeIsoDateTimeStringSync = Schema.decodeUnknownSync(IsoDateTimeString) + +const ServicesJsonSchema = Schema.Array( + Schema.Struct({ + id: Schema.String, + name: Schema.String, + }), +) +const decodeServicesJson = Schema.decodeUnknownSync(Schema.fromJsonString(ServicesJsonSchema)) + +/** Parse a target's cached services list; corrupt JSON degrades to an empty list. */ +export const parseServicesJson = ( + servicesJson: string | null, +): ReadonlyArray<{ readonly id: string; readonly name: string }> => { + if (servicesJson === null) return [] + try { + return decodeServicesJson(servicesJson) + } catch { + return [] + } +} + +/** The internal wire row handed to the railway-connector (before ingest keys are attached). */ +export interface EnabledRailwayTarget { + readonly target: RailwayTargetRow + readonly connection: RailwayConnectionRow + /** Decrypted Railway API token. */ + readonly token: string + readonly services: ReadonlyArray<{ readonly id: string; readonly name: string }> +} + +export interface RailwayIntegrationServiceShape { + readonly status: (orgId: OrgId) => Effect.Effect + readonly connect: ( + orgId: OrgId, + userId: string, + token: string, + ) => Effect.Effect< + RailwayIntegrationStatus, + RailwayValidationError | RailwayUpstreamError | RailwayPersistenceError | RailwayEncryptionError + > + readonly disconnect: (orgId: OrgId) => Effect.Effect + readonly discover: ( + orgId: OrgId, + ) => Effect.Effect< + RailwayDiscoveryResponse, + RailwayNotFoundError | RailwayUpstreamError | RailwayPersistenceError | RailwayEncryptionError + > + readonly listTargets: ( + orgId: OrgId, + ) => Effect.Effect + readonly createTarget: ( + orgId: OrgId, + request: CreateRailwayTargetRequest, + ) => Effect.Effect< + RailwayTargetResponse, + | RailwayNotFoundError + | RailwayValidationError + | RailwayUpstreamError + | RailwayPersistenceError + | RailwayEncryptionError + > + readonly updateTarget: ( + orgId: OrgId, + targetId: RailwayTargetId, + request: UpdateRailwayTargetRequest, + ) => Effect.Effect< + RailwayTargetResponse, + RailwayNotFoundError | RailwayValidationError | RailwayPersistenceError + > + readonly deleteTarget: ( + orgId: OrgId, + targetId: RailwayTargetId, + ) => Effect.Effect + /** + * Work list for the railway-connector: every enabled target of every enabled + * connection, with the connection token decrypted and the cached services + * list refreshed on a TTL. Rows that fail decryption are skipped (logged) + * so one corrupt connection cannot break the whole list. + */ + readonly listAllEnabledInternal: () => Effect.Effect< + ReadonlyArray, + RailwayPersistenceError + > + readonly recordResults: ( + reports: ReadonlyArray, + ) => Effect.Effect +} + +const toPersistenceError = (error: unknown) => + new RailwayPersistenceError({ + message: error instanceof Error ? error.message : "Railway integration persistence failed", + }) + +const toEncryptionError = (message: string) => new RailwayEncryptionError({ message }) + +const mapRailwayApiError = (error: RailwayApiRequestError): RailwayUpstreamError => + new RailwayUpstreamError({ message: error.message }) + +const parseEncryptionKey = (raw: string): Effect.Effect => + parseBase64Aes256GcmKey(raw, (message) => + toEncryptionError( + message === "Expected a non-empty base64 encryption key" + ? "MAPLE_INGEST_KEY_ENCRYPTION_KEY is required" + : message === "Expected base64 for exactly 32 bytes" + ? "MAPLE_INGEST_KEY_ENCRYPTION_KEY must be base64 for exactly 32 bytes" + : message, + ), + ) + +const iso = (date: Date | null): IsoDateTimeString | null => + date === null ? null : decodeIsoDateTimeStringSync(date.toISOString()) + +const rowToTargetResponse = (row: RailwayTargetRow): RailwayTargetResponse => + new RailwayTargetResponse({ + id: decodeTargetIdSync(row.id), + projectId: row.projectId, + projectName: row.projectName, + environmentId: row.environmentId, + environmentName: row.environmentName, + enabled: row.enabled, + logsEnabled: row.logsEnabled, + metricsEnabled: row.metricsEnabled, + metricsIntervalSeconds: decodeMetricsIntervalSync(row.metricsIntervalSeconds), + serviceCount: parseServicesJson(row.servicesJson).length, + lastLogAt: iso(row.lastLogAt), + lastMetricsAt: iso(row.lastMetricsAt), + lastSuccessAt: iso(row.lastSuccessAt), + lastError: row.lastError, + lastErrorAt: iso(row.lastErrorAt), + createdAt: decodeIsoDateTimeStringSync(row.createdAt.toISOString()), + updatedAt: decodeIsoDateTimeStringSync(row.updatedAt.toISOString()), + }) + +export class RailwayIntegrationService extends Context.Service< + RailwayIntegrationService, + RailwayIntegrationServiceShape +>()("@maple/api/services/RailwayIntegrationService", { + make: Effect.gen(function* () { + const database = yield* Database + const env = yield* Env + const encryptionKey = yield* parseEncryptionKey(Redacted.value(env.MAPLE_INGEST_KEY_ENCRYPTION_KEY)) + + const selectConnection = Effect.fn("RailwayIntegrationService.selectConnection")(function* ( + orgId: OrgId, + ) { + const rows = yield* database + .execute((db) => + db.select().from(railwayConnections).where(eq(railwayConnections.orgId, orgId)).limit(1), + ) + .pipe(Effect.mapError(toPersistenceError)) + return Option.fromNullishOr(rows[0]) + }) + + const requireConnection = Effect.fn("RailwayIntegrationService.requireConnection")(function* ( + orgId: OrgId, + ) { + const connection = yield* selectConnection(orgId) + if (Option.isSome(connection)) return connection.value + return yield* Effect.fail( + new RailwayNotFoundError({ message: "Railway is not connected for this organization" }), + ) + }) + + const decryptToken = ( + row: RailwayConnectionRow, + ): Effect.Effect => + decryptAes256Gcm( + { ciphertext: row.tokenCiphertext, iv: row.tokenIv, tag: row.tokenTag }, + encryptionKey, + () => toEncryptionError("Failed to decrypt the stored Railway token"), + ) + + const selectTargets = Effect.fn("RailwayIntegrationService.selectTargets")(function* ( + orgId: OrgId, + ) { + return yield* database + .execute((db) => db.select().from(railwayTargets).where(eq(railwayTargets.orgId, orgId))) + .pipe(Effect.mapError(toPersistenceError)) + }) + + const requireTarget = Effect.fn("RailwayIntegrationService.requireTarget")(function* ( + orgId: OrgId, + targetId: RailwayTargetId, + ) { + const rows = yield* database + .execute((db) => + db + .select() + .from(railwayTargets) + .where(and(eq(railwayTargets.orgId, orgId), eq(railwayTargets.id, targetId))) + .limit(1), + ) + .pipe(Effect.mapError(toPersistenceError)) + const row = Option.fromNullishOr(rows[0]) + if (Option.isSome(row)) return row.value + return yield* Effect.fail(new RailwayNotFoundError({ message: "Railway target not found" })) + }) + + const statusFor = Effect.fn("RailwayIntegrationService.statusFor")(function* (orgId: OrgId) { + const connection = yield* selectConnection(orgId) + const targets = yield* selectTargets(orgId) + const erroredTargetCount = targets.filter((target) => target.lastError !== null).length + return Option.match(connection, { + onNone: () => + new RailwayIntegrationStatus({ + connected: false, + connectionId: null, + tokenType: null, + accountName: null, + enabled: false, + lastValidatedAt: null, + lastValidationError: null, + targetCount: targets.length, + erroredTargetCount, + }), + onSome: (row) => + new RailwayIntegrationStatus({ + connected: true, + connectionId: decodeConnectionIdSync(row.id), + tokenType: decodeTokenTypeSync(row.tokenType), + accountName: row.externalAccountName, + enabled: row.enabled, + lastValidatedAt: iso(row.lastValidatedAt), + lastValidationError: row.lastValidationError, + targetCount: targets.length, + erroredTargetCount, + }), + }) + }) + + const connect = Effect.fn("RailwayIntegrationService.connect")(function* ( + orgId: OrgId, + userId: string, + token: string, + ) { + const trimmed = token.trim() + if (trimmed.length === 0) { + return yield* Effect.fail(new RailwayValidationError({ message: "A Railway API token is required" })) + } + + const identity = yield* validateRailwayToken(trimmed).pipe( + Effect.mapError((error) => + error.unauthorized + ? new RailwayValidationError({ + message: + "Railway rejected the token. Paste a workspace or account API token from Railway → Account Settings → Tokens.", + }) + : mapRailwayApiError(error), + ), + ) + + const encrypted = yield* encryptAes256Gcm(trimmed, encryptionKey, () => + toEncryptionError("Failed to encrypt the Railway token"), + ) + + const now = yield* Clock.currentTimeMillis + const values = { + tokenType: identity.tokenType, + tokenCiphertext: encrypted.ciphertext, + tokenIv: encrypted.iv, + tokenTag: encrypted.tag, + externalAccountName: identity.accountName, + connectedByUserId: userId, + enabled: true, + lastValidatedAt: new Date(now), + lastValidationError: null, + lastValidationErrorAt: null, + } + yield* database + .execute((db) => + db + .insert(railwayConnections) + .values({ + id: randomUUID(), + orgId, + createdAt: new Date(now), + updatedAt: new Date(now), + ...values, + }) + .onConflictDoUpdate({ + target: [railwayConnections.orgId], + set: { ...values, updatedAt: new Date(now) }, + }), + ) + .pipe(Effect.mapError(toPersistenceError)) + + return yield* statusFor(orgId) + }) + + const disconnect = Effect.fn("RailwayIntegrationService.disconnect")(function* (orgId: OrgId) { + yield* database + .execute((db) => db.delete(railwayTargets).where(eq(railwayTargets.orgId, orgId))) + .pipe(Effect.mapError(toPersistenceError)) + const deleted = yield* database + .execute((db) => + db + .delete(railwayConnections) + .where(eq(railwayConnections.orgId, orgId)) + .returning({ id: railwayConnections.id }), + ) + .pipe(Effect.mapError(toPersistenceError)) + return new RailwayDisconnectResponse({ disconnected: deleted.length > 0 }) + }) + + const discoverProjects = Effect.fn("RailwayIntegrationService.discoverProjects")(function* ( + connection: RailwayConnectionRow, + ) { + const token = yield* decryptToken(connection) + return yield* fetchRailwayProjects(token).pipe( + Effect.tapError((error) => + Effect.gen(function* () { + // Best-effort revocation surfacing; the caller still sees the + // upstream error. Only a genuine auth rejection flags the + // connection — a transient Railway outage (timeout, 5xx, DNS) + // must not flip the card to "Reconnect needed". + if (!error.unauthorized) return + const now = yield* Clock.currentTimeMillis + yield* database + .execute((db) => + db + .update(railwayConnections) + .set({ + lastValidationError: "Railway rejected the stored token", + lastValidationErrorAt: new Date(now), + updatedAt: new Date(now), + }) + .where(eq(railwayConnections.id, connection.id)), + ) + .pipe(Effect.ignore) + }), + ), + Effect.mapError(mapRailwayApiError), + ) + }) + + const discover = Effect.fn("RailwayIntegrationService.discover")(function* (orgId: OrgId) { + const connection = yield* requireConnection(orgId) + const projects = yield* discoverProjects(connection) + const targets = yield* selectTargets(orgId) + const targetByEnv = new Map( + targets.map((target) => [`${target.projectId}:${target.environmentId}`, target.id]), + ) + + // A successful discovery proves the token is still valid — clear any + // stale revocation flag and bump lastValidatedAt. + const now = yield* Clock.currentTimeMillis + yield* database + .execute((db) => + db + .update(railwayConnections) + .set({ + lastValidatedAt: new Date(now), + lastValidationError: null, + lastValidationErrorAt: null, + updatedAt: new Date(now), + }) + .where(eq(railwayConnections.id, connection.id)), + ) + .pipe(Effect.ignore) + + return new RailwayDiscoveryResponse({ + projects: projects.map( + (project) => + new RailwayDiscoveredProject({ + id: project.id, + name: project.name, + environments: project.environments.map( + (environment) => + new RailwayDiscoveredEnvironment({ + id: environment.id, + name: environment.name, + services: environment.services.map( + (service) => + new RailwayDiscoveredService({ id: service.id, name: service.name }), + ), + targetId: (() => { + const id = targetByEnv.get(`${project.id}:${environment.id}`) + return id === undefined ? null : decodeTargetIdSync(id) + })(), + }), + ), + }), + ), + }) + }) + + const listTargets = Effect.fn("RailwayIntegrationService.listTargets")(function* (orgId: OrgId) { + const rows = yield* selectTargets(orgId) + return new RailwayTargetsListResponse({ targets: rows.map(rowToTargetResponse) }) + }) + + const createTarget = Effect.fn("RailwayIntegrationService.createTarget")(function* ( + orgId: OrgId, + request: CreateRailwayTargetRequest, + ) { + const connection = yield* requireConnection(orgId) + const projects = yield* discoverProjects(connection) + const project = projects.find((candidate) => candidate.id === request.projectId) + if (project === undefined) { + return yield* Effect.fail( + new RailwayValidationError({ + message: "The Railway project was not found with the connected token", + }), + ) + } + const environment = project.environments.find( + (candidate) => candidate.id === request.environmentId, + ) + if (environment === undefined) { + return yield* Effect.fail( + new RailwayValidationError({ + message: "The Railway environment was not found in that project", + }), + ) + } + + const now = yield* Clock.currentTimeMillis + const id = randomUUID() + const inserted = yield* database + .execute((db) => + db + .insert(railwayTargets) + .values({ + id, + orgId, + connectionId: connection.id, + projectId: project.id, + projectName: project.name, + environmentId: environment.id, + environmentName: environment.name, + enabled: true, + logsEnabled: request.logsEnabled ?? true, + metricsEnabled: request.metricsEnabled ?? true, + metricsIntervalSeconds: + request.metricsIntervalSeconds ?? DEFAULT_METRICS_INTERVAL_SECONDS, + servicesJson: JSON.stringify(environment.services), + servicesDiscoveredAt: new Date(now), + createdAt: new Date(now), + updatedAt: new Date(now), + }) + .onConflictDoNothing() + .returning({ id: railwayTargets.id }), + ) + .pipe(Effect.mapError(toPersistenceError)) + + if (inserted.length === 0) { + return yield* Effect.fail( + new RailwayValidationError({ + message: "This Railway environment is already being ingested", + }), + ) + } + + const row = yield* requireTarget(orgId, decodeTargetIdSync(id)).pipe( + Effect.mapError(() => + new RailwayPersistenceError({ message: "Failed to load the created Railway target" }), + ), + ) + return rowToTargetResponse(row) + }) + + const updateTarget = Effect.fn("RailwayIntegrationService.updateTarget")(function* ( + orgId: OrgId, + targetId: RailwayTargetId, + request: UpdateRailwayTargetRequest, + ) { + yield* requireTarget(orgId, targetId) + + const now = yield* Clock.currentTimeMillis + const updates: Partial = { updatedAt: new Date(now) } + if (request.enabled !== undefined) updates.enabled = request.enabled + if (request.logsEnabled !== undefined) updates.logsEnabled = request.logsEnabled + if (request.metricsEnabled !== undefined) updates.metricsEnabled = request.metricsEnabled + if (request.metricsIntervalSeconds !== undefined) { + updates.metricsIntervalSeconds = request.metricsIntervalSeconds + } + + yield* database + .execute((db) => + db + .update(railwayTargets) + .set(updates) + .where(and(eq(railwayTargets.orgId, orgId), eq(railwayTargets.id, targetId))), + ) + .pipe(Effect.mapError(toPersistenceError)) + + const row = yield* requireTarget(orgId, targetId) + return rowToTargetResponse(row) + }) + + const deleteTarget = Effect.fn("RailwayIntegrationService.deleteTarget")(function* ( + orgId: OrgId, + targetId: RailwayTargetId, + ) { + const deleted = yield* database + .execute((db) => + db + .delete(railwayTargets) + .where(and(eq(railwayTargets.orgId, orgId), eq(railwayTargets.id, targetId))) + .returning({ id: railwayTargets.id }), + ) + .pipe(Effect.mapError(toPersistenceError)) + const row = Option.fromNullishOr(deleted[0]) + if (Option.isNone(row)) { + return yield* Effect.fail(new RailwayNotFoundError({ message: "Railway target not found" })) + } + return new RailwayTargetDeleteResponse({ id: decodeTargetIdSync(row.value.id) }) + }) + + /** Refresh a connection's cached env→services lists when stale; best-effort. */ + const refreshServices = Effect.fn("RailwayIntegrationService.refreshServices")(function* ( + connection: RailwayConnectionRow, + targets: ReadonlyArray, + now: number, + ) { + const stale = targets.some( + (target) => + target.servicesDiscoveredAt === null || + now - target.servicesDiscoveredAt.getTime() > SERVICES_REFRESH_TTL_MS, + ) + if (!stale) return targets + + const projects = yield* discoverProjects(connection).pipe(Effect.option) + if (Option.isNone(projects)) return targets + + const envById = new Map() + for (const project of projects.value) { + for (const environment of project.environments) { + envById.set(`${project.id}:${environment.id}`, environment) + } + } + + const refreshed: Array = [] + for (const target of targets) { + const environment = envById.get(`${target.projectId}:${target.environmentId}`) + if (environment === undefined) { + // Environment no longer visible with this token — keep the cached + // list; the connector will surface errors if it truly disappeared. + refreshed.push(target) + continue + } + const servicesJson = JSON.stringify(environment.services) + yield* database + .execute((db) => + db + .update(railwayTargets) + .set({ + servicesJson, + servicesDiscoveredAt: new Date(now), + environmentName: environment.name, + updatedAt: new Date(now), + }) + .where(eq(railwayTargets.id, target.id)), + ) + .pipe(Effect.ignore) + refreshed.push({ + ...target, + servicesJson, + servicesDiscoveredAt: new Date(now), + environmentName: environment.name, + }) + } + return refreshed + }) + + const listAllEnabledInternal = Effect.fn("RailwayIntegrationService.listAllEnabledInternal")( + function* () { + const connections = yield* database + .execute((db) => + db.select().from(railwayConnections).where(eq(railwayConnections.enabled, true)), + ) + .pipe(Effect.mapError(toPersistenceError)) + if (connections.length === 0) return [] + + const now = yield* Clock.currentTimeMillis + const results: Array = [] + + yield* Effect.forEach(connections, (connection) => + Effect.gen(function* () { + const token = yield* decryptToken(connection).pipe(Effect.option) + if (Option.isNone(token)) { + yield* Effect.logWarning("Skipping Railway connection (decryption failed)").pipe( + Effect.annotateLogs({ railwayConnectionId: connection.id, orgId: connection.orgId }), + ) + return + } + + const targets = yield* database + .execute((db) => + db + .select() + .from(railwayTargets) + .where( + and( + eq(railwayTargets.connectionId, connection.id), + eq(railwayTargets.enabled, true), + ), + ), + ) + .pipe(Effect.orElseSucceed(() => [] as Array)) + if (targets.length === 0) return + + const refreshed = yield* refreshServices(connection, targets, now) + for (const target of refreshed) { + results.push({ + target, + connection, + token: token.value, + services: parseServicesJson(target.servicesJson), + }) + } + }), + ) + + return results + }, + ) + + const recordResults = Effect.fn("RailwayIntegrationService.recordResults")(function* ( + reports: ReadonlyArray, + ) { + yield* Effect.forEach( + reports, + (report) => + Effect.gen(function* () { + const rows = yield* database + .execute((db) => + db + .select() + .from(railwayTargets) + .where(eq(railwayTargets.id, report.targetId)) + .limit(1), + ) + .pipe(Effect.mapError(toPersistenceError)) + const row = Option.fromNullishOr(rows[0]) + // Deleted targets can still be in-flight in the connector; drop their reports. + if (Option.isNone(row)) return + const target = row.value + + const at = new Date(report.at) + const updates: Partial = { updatedAt: at } + + if (report.error === null) { + updates.lastSuccessAt = at + updates.lastError = null + updates.lastErrorAt = null + if (report.kind === "logs") updates.lastLogAt = at + if (report.kind === "metrics") updates.lastMetricsAt = at + } else { + // Failure keeps last*At at the last good run so gaps stay visible. + updates.lastError = `[${report.kind}] ${report.error}` + updates.lastErrorAt = at + } + + // Watermarks only ever move forward — late/out-of-order reports must + // not rewind the backfill anchor. + if (report.newLogWatermarkMs !== undefined) { + const current = target.logWatermarkAt?.getTime() ?? 0 + if (report.newLogWatermarkMs > current) { + updates.logWatermarkAt = new Date(report.newLogWatermarkMs) + } + } + if (report.newMetricsWatermarkMs !== undefined) { + const current = target.metricsWatermarkAt?.getTime() ?? 0 + if (report.newMetricsWatermarkMs > current) { + updates.metricsWatermarkAt = new Date(report.newMetricsWatermarkMs) + } + } + + yield* database + .execute((db) => + db.update(railwayTargets).set(updates).where(eq(railwayTargets.id, target.id)), + ) + .pipe(Effect.mapError(toPersistenceError)) + + if (report.unauthorized === true) { + yield* database + .execute((db) => + db + .update(railwayConnections) + .set({ + lastValidationError: + "Railway rejected the stored token — reconnect the integration", + lastValidationErrorAt: at, + updatedAt: at, + }) + .where(eq(railwayConnections.id, target.connectionId)), + ) + .pipe(Effect.ignore) + } + }), + { discard: true }, + ) + }) + + return { + status: statusFor, + connect, + disconnect, + discover, + listTargets, + createTarget, + updateTarget, + deleteTarget, + listAllEnabledInternal, + recordResults, + } satisfies RailwayIntegrationServiceShape + }), +}) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/landing/src/content/docs/integrations/railway.md b/apps/landing/src/content/docs/integrations/railway.md new file mode 100644 index 000000000..487404b52 --- /dev/null +++ b/apps/landing/src/content/docs/integrations/railway.md @@ -0,0 +1,43 @@ +--- +title: "Railway" +description: "Connect Railway to Maple with an API token — Maple streams your services' runtime logs and collects CPU, memory, network, and disk metrics for every service." +group: "Integrations" +order: 4 +--- + +Railway has no log-drain feature — there is no setting to forward a service's stdout to an external intake URL. Its [public GraphQL API](https://docs.railway.com/integrations/api) is the only programmatic way at your logs and resource metrics. Maple supports this natively: you paste one API token, pick the environments to ingest, and Maple's connector streams runtime logs over Railway's GraphQL subscription API and polls per-service resource metrics — no code changes, sidecars, or forwarder services in your Railway project. + +## 1. Create an API token + +In Railway, open **Account Settings → Tokens** and create a token: + +- A **workspace token** (recommended) is scoped to one workspace and can read all of its projects. +- An **account token** works too and covers every workspace you belong to. +- **Project tokens are not supported** in this version — they authenticate differently and cannot enumerate projects. + +## 2. Connect it in Maple + +Open **Integrations → Railway** in the Maple dashboard and paste the token. Maple validates it against Railway's API, stores it encrypted (AES-256-GCM — it is never sent back to the browser), and discovers your projects, environments, and services. + +## 3. Pick environments + +Toggle **Ingest** on each project environment you want in Maple. Logs and metrics can be toggled independently per environment. Every Railway service in an ingested environment appears in Maple as its own service (`service.name` = the Railway service name, `service.namespace` = the project name), tagged with `cloud.provider=railway` and `railway.project/environment/service` resource attributes. + +## What you get + +- **Runtime logs**, live: each service's stdout/stderr as Maple log records, with Railway's severity mapped onto OTLP severity and the deployment id attached (`railway.deployment.id`). Build and deploy logs and HTTP edge logs are not ingested in this version. +- **Resource metrics**, polled every 5 minutes by default (configurable 1–60 min per environment): + - `railway.cpu.usage` / `railway.cpu.limit` (vCPU) + - `railway.memory.usage` / `railway.memory.limit` (bytes) + - `railway.network.rx` / `railway.network.tx` (bytes) + - `railway.disk.usage` (bytes) + +## Rate limits + +Railway rate-limits API calls by **your Railway plan** — 100 requests/hour on Free, 1,000 on Hobby, 10,000 on Pro. Metrics polling costs roughly one request per service per interval; the log stream is a single long-lived connection per environment. Maple reads Railway's rate-limit headers and automatically stretches poll intervals when the remaining budget runs low, but on the Free plan with many services expect metrics to arrive less often than the configured interval. + +## Notes + +- Ingested logs and metrics count toward your Maple ingestion volume like any other OTLP data. +- If you revoke or rotate the token in Railway, the integration card shows a **Reconnect needed** banner — paste the new token to resume. Ingestion pauses (it never falls back to another credential) until you do. +- Short gaps in the log stream can occur around reconnects (for example a Railway API deploy); metrics polling backfills up to an hour automatically. diff --git a/apps/railway-connector/Dockerfile b/apps/railway-connector/Dockerfile new file mode 100644 index 000000000..7fa65316d --- /dev/null +++ b/apps/railway-connector/Dockerfile @@ -0,0 +1,14 @@ +# Build context: repo root (workspace deps @maple/domain + @maple-dev/effect-sdk). +# Railway: set the service's config file path to apps/railway-connector/railway.json +# and keep the root directory at the repo root. Required env: MAPLE_API_URL, +# SD_INTERNAL_TOKEN, MAPLE_INGEST_URL (see src/Env.ts). Run exactly ONE replica — +# two would double-ingest every log line (no lease coordination in v1). +FROM oven/bun:1.2 AS runner +WORKDIR /app + +COPY . . +RUN bun install --frozen-lockfile && bun run --cwd lib/effect-sdk build + +EXPOSE 3476 + +CMD ["bun", "run", "apps/railway-connector/src/main.ts"] diff --git a/apps/railway-connector/package.json b/apps/railway-connector/package.json new file mode 100644 index 000000000..4af8c0def --- /dev/null +++ b/apps/railway-connector/package.json @@ -0,0 +1,25 @@ +{ + "name": "@maple/railway-connector", + "private": true, + "type": "module", + "scripts": { + "start": "bun run src/main.ts", + "dev": "bun --env-file=../../.env.local --watch src/main.ts", + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@effect/platform-bun": "catalog:effect", + "@maple-dev/effect-sdk": "workspace:*", + "@maple/domain": "workspace:*", + "effect": "catalog:effect" + }, + "devDependencies": { + "@effect/language-service": "catalog:effect", + "@effect/vitest": "4.0.0-beta.93", + "@types/bun": "^1.3.11", + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:" + } +} diff --git a/apps/railway-connector/railway.json b/apps/railway-connector/railway.json new file mode 100644 index 000000000..cbd493dee --- /dev/null +++ b/apps/railway-connector/railway.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://railway.com/railway.schema.json", + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "apps/railway-connector/Dockerfile", + "watchPatterns": ["apps/railway-connector/**", "packages/domain/**", "lib/effect-sdk/**"] + }, + "deploy": { + "healthcheckPath": "/health", + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 5 + } +} diff --git a/apps/railway-connector/src/ApiClient.ts b/apps/railway-connector/src/ApiClient.ts new file mode 100644 index 000000000..8f89db4f6 --- /dev/null +++ b/apps/railway-connector/src/ApiClient.ts @@ -0,0 +1,107 @@ +import { Context, Duration, Effect, Layer, Redacted, Schema } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" +import { + InternalRailwayTargetList, + type InternalRailwayTarget, + type RailwayResultReport, +} from "@maple/domain/http" +import { RailwayConnectorEnv } from "./Env" + +export class ApiRequestError extends Schema.TaggedErrorClass()( + "@maple/railway-connector/ApiRequestError", + { + message: Schema.String, + status: Schema.NullOr(Schema.Number), + }, +) {} + +export interface ApiClientShape { + /** Enabled Railway targets from `/api/internal/railway-targets`. */ + readonly listTargets: () => Effect.Effect, ApiRequestError> + /** Report log/metrics outcomes to `/api/internal/railway-results`. */ + readonly reportResults: ( + results: ReadonlyArray, + ) => Effect.Effect +} + +const decodeTargets = Schema.decodeUnknownEffect(InternalRailwayTargetList) + +export class ApiClient extends Context.Service()( + "@maple/railway-connector/ApiClient", + { + make: Effect.gen(function* () { + const env = yield* RailwayConnectorEnv + const client = yield* HttpClient.HttpClient + + const authHeaders = { + authorization: `Bearer ${Redacted.value(env.SD_INTERNAL_TOKEN)}`, + } + + // Bound every API round-trip so a stalled Worker fails fast and the + // caller re-buffers instead of hanging. + const REQUEST_TIMEOUT = Duration.seconds(30) + + const transportError = (error: { readonly message: string }) => + new ApiRequestError({ message: `Maple API unreachable: ${error.message}`, status: null }) + + const listTargets = Effect.fn("ApiClient.listTargets")(function* () { + const request = HttpClientRequest.get(`${env.MAPLE_API_URL}/api/internal/railway-targets`, { + headers: authHeaders, + }) + const response = yield* client + .execute(request) + .pipe(Effect.timeout(REQUEST_TIMEOUT), Effect.mapError(transportError)) + const text = yield* response.text.pipe(Effect.mapError(transportError)) + if (response.status < 200 || response.status >= 300) { + return yield* Effect.fail( + new ApiRequestError({ + message: `railway-targets returned HTTP ${response.status}: ${text.slice(0, 200)}`, + status: response.status, + }), + ) + } + return yield* Effect.try({ + try: () => JSON.parse(text) as unknown, + catch: () => + new ApiRequestError({ message: "railway-targets returned invalid JSON", status: null }), + }).pipe( + Effect.flatMap((json) => + decodeTargets(json).pipe( + Effect.mapError( + (error) => + new ApiRequestError({ + message: `railway-targets payload mismatch: ${error.message}`, + status: null, + }), + ), + ), + ), + ) + }) + + const reportResults = Effect.fn("ApiClient.reportResults")(function* ( + results: ReadonlyArray, + ) { + if (results.length === 0) return + const request = HttpClientRequest.post(`${env.MAPLE_API_URL}/api/internal/railway-results`, { + headers: authHeaders, + }).pipe(HttpClientRequest.bodyText(JSON.stringify(results), "application/json")) + const response = yield* client + .execute(request) + .pipe(Effect.timeout(REQUEST_TIMEOUT), Effect.mapError(transportError)) + if (response.status < 200 || response.status >= 300) { + return yield* Effect.fail( + new ApiRequestError({ + message: `railway-results returned HTTP ${response.status}`, + status: response.status, + }), + ) + } + }) + + return { listTargets, reportResults } satisfies ApiClientShape + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/railway-connector/src/Env.ts b/apps/railway-connector/src/Env.ts new file mode 100644 index 000000000..9e754765d --- /dev/null +++ b/apps/railway-connector/src/Env.ts @@ -0,0 +1,66 @@ +import { Config, Context, Effect, Layer, Redacted } from "effect" + +export interface RailwayConnectorEnvShape { + /** Base URL of the Maple API, e.g. `https://api.maple.dev`. */ + readonly MAPLE_API_URL: string + /** Shared internal bearer for the `/api/internal/railway-*` endpoints. */ + readonly SD_INTERNAL_TOKEN: Redacted.Redacted + /** + * Base URL of the Maple ingest gateway, e.g. `https://ingest.maple.dev`. + * Pulled logs/metrics are sent here as OTLP/JSON with each org's public + * ingest key so they get billed and warehouse-routed per org. + */ + readonly MAPLE_INGEST_URL: string + /** Railway public GraphQL API over HTTP (metrics query, discovery). */ + readonly RAILWAY_GRAPHQL_HTTP_URL: string + /** Railway public GraphQL API over WebSocket (environmentLogs subscription). */ + readonly RAILWAY_GRAPHQL_WS_URL: string + /** Max concurrent Railway metrics polls across all targets. */ + readonly RAILWAY_CONNECTOR_CONCURRENCY: number + /** How often the target list is refreshed, in seconds. */ + readonly RAILWAY_CONNECTOR_RECONCILE_INTERVAL_SECONDS: number + /** Port for the `/health` endpoint. */ + readonly PORT: number +} + +// Defaults target the local dev stack (`bun dev`: api on 3472, ingest on +// 3474, the docker-compose dev token) so the connector boots without extra +// configuration instead of crashing the turbo dev TUI. Production overrides +// the first three (see apps/railway-connector/railway.json deploy notes); a +// missing override degrades to visible per-reconcile warnings, never a crash +// loop. +const envConfig = Config.all({ + MAPLE_API_URL: Config.string("MAPLE_API_URL").pipe(Config.withDefault("http://127.0.0.1:3472")), + SD_INTERNAL_TOKEN: Config.redacted("SD_INTERNAL_TOKEN").pipe( + Config.withDefault(Redacted.make("maple-sd-dev-token")), + ), + MAPLE_INGEST_URL: Config.string("MAPLE_INGEST_URL").pipe(Config.withDefault("http://127.0.0.1:3474")), + RAILWAY_GRAPHQL_HTTP_URL: Config.string("RAILWAY_GRAPHQL_HTTP_URL").pipe( + Config.withDefault("https://backboard.railway.com/graphql/v2"), + ), + RAILWAY_GRAPHQL_WS_URL: Config.string("RAILWAY_GRAPHQL_WS_URL").pipe( + Config.withDefault("wss://backboard.railway.com/graphql/v2"), + ), + RAILWAY_CONNECTOR_CONCURRENCY: Config.number("RAILWAY_CONNECTOR_CONCURRENCY").pipe( + Config.withDefault(10), + ), + RAILWAY_CONNECTOR_RECONCILE_INTERVAL_SECONDS: Config.number( + "RAILWAY_CONNECTOR_RECONCILE_INTERVAL_SECONDS", + ).pipe(Config.withDefault(60)), + PORT: Config.number("PORT").pipe(Config.withDefault(3476)), +}) + +export class RailwayConnectorEnv extends Context.Service()( + "@maple/railway-connector/Env", + { + make: Effect.map(envConfig, (env) => ({ + ...env, + MAPLE_API_URL: env.MAPLE_API_URL.replace(/\/$/, ""), + MAPLE_INGEST_URL: env.MAPLE_INGEST_URL.replace(/\/$/, ""), + RAILWAY_GRAPHQL_HTTP_URL: env.RAILWAY_GRAPHQL_HTTP_URL.replace(/\/$/, ""), + RAILWAY_GRAPHQL_WS_URL: env.RAILWAY_GRAPHQL_WS_URL.replace(/\/$/, ""), + })), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/railway-connector/src/LogStream.test.ts b/apps/railway-connector/src/LogStream.test.ts new file mode 100644 index 000000000..740c515d2 --- /dev/null +++ b/apps/railway-connector/src/LogStream.test.ts @@ -0,0 +1,56 @@ +import { assert, describe, it } from "@effect/vitest" +import { + enqueueLogEntry, + MAX_BUFFERED_ENTRIES, + reconnectDelayMs, + type LogStreamState, +} from "./LogStream" +import type { RailwayLogEntry } from "./otlp/logs" + +const entry = (timestampMs: number, message = "line"): RailwayLogEntry => ({ + timestampMs, + message, + severity: null, + serviceId: null, + deploymentId: null, + attributes: [], +}) + +describe("enqueueLogEntry", () => { + it("drops entries at or below the watermark (reconnect replay overlap)", () => { + const state: LogStreamState = { buffer: [], droppedSinceLastReport: 0, watermarkMs: 5_000 } + enqueueLogEntry(state, entry(4_000)) + enqueueLogEntry(state, entry(5_000)) + enqueueLogEntry(state, entry(6_000)) + assert.strictEqual(state.buffer.length, 1) + assert.strictEqual(state.buffer[0]!.timestampMs, 6_000) + }) + + it("accepts everything when no watermark is set", () => { + const state: LogStreamState = { buffer: [], droppedSinceLastReport: 0, watermarkMs: null } + enqueueLogEntry(state, entry(1)) + enqueueLogEntry(state, entry(2)) + assert.strictEqual(state.buffer.length, 2) + }) + + it("evicts the oldest entry on overflow and counts the drop", () => { + const state: LogStreamState = { + buffer: Array.from({ length: MAX_BUFFERED_ENTRIES }, (_, index) => entry(index)), + droppedSinceLastReport: 0, + watermarkMs: null, + } + enqueueLogEntry(state, entry(MAX_BUFFERED_ENTRIES + 1, "newest")) + assert.strictEqual(state.buffer.length, MAX_BUFFERED_ENTRIES) + assert.strictEqual(state.droppedSinceLastReport, 1) + assert.strictEqual(state.buffer.at(-1)!.message, "newest") + assert.strictEqual(state.buffer[0]!.timestampMs, 1) + }) +}) + +describe("reconnectDelayMs", () => { + it("escalates exponentially and caps at five minutes", () => { + assert.strictEqual(reconnectDelayMs(1), 2_000) + assert.strictEqual(reconnectDelayMs(3), 8_000) + assert.strictEqual(reconnectDelayMs(20), 300_000) + }) +}) diff --git a/apps/railway-connector/src/LogStream.ts b/apps/railway-connector/src/LogStream.ts new file mode 100644 index 000000000..3458e21ba --- /dev/null +++ b/apps/railway-connector/src/LogStream.ts @@ -0,0 +1,206 @@ +/** + * Per-target log streaming: one `environmentLogs` GraphQL subscription per + * enabled (project, environment) target covers every service in the + * environment. Entries buffer in memory and flush to the ingest gateway as + * OTLP logs every few seconds; the connection reconnects with exponential + * backoff and the delivered watermark rides result reports back to the api. + */ +import { Clock, Duration, Effect, Result } from "effect" +import { RailwayResultReport, type InternalRailwayTarget } from "@maple/domain/http" +import { buildLogsExportRequest, decodeRailwayLogEntry, type RailwayLogEntry } from "./otlp/logs" +import { + ENVIRONMENT_LOGS_SUBSCRIPTION, + runGraphqlWsSubscription, + type WebSocketFactory, +} from "./RailwaySubscriptionClient" +import type { OtlpIngestShape } from "./OtlpIngest" + +const FLUSH_INTERVAL = Duration.seconds(3) +/** Flush immediately once this many entries are buffered. */ +export const FLUSH_BATCH_SIZE = 500 +/** Bound the in-memory buffer during ingest outages; oldest entries drop first. */ +export const MAX_BUFFERED_ENTRIES = 10_000 +/** Reconnect backoff bounds. */ +const RECONNECT_BASE_MS = 1_000 +const RECONNECT_MAX_MS = Duration.toMillis(Duration.minutes(5)) +/** A connection that survived this long resets the backoff counter. */ +const STABLE_CONNECTION_MS = 60_000 + +export interface LogStreamState { + buffer: RailwayLogEntry[] + droppedSinceLastReport: number + /** Epoch ms of the newest entry delivered to ingest. */ + watermarkMs: number | null +} + +/** + * Enqueue a decoded entry, dropping entries at/below the watermark (replay + * overlap after a reconnect) and evicting the oldest on overflow. + */ +export const enqueueLogEntry = (state: LogStreamState, entry: RailwayLogEntry): void => { + if (state.watermarkMs !== null && entry.timestampMs <= state.watermarkMs) return + if (state.buffer.length >= MAX_BUFFERED_ENTRIES) { + state.buffer.shift() + state.droppedSinceLastReport += 1 + } + state.buffer.push(entry) +} + +export const reconnectDelayMs = (consecutiveFailures: number): number => + Math.min(RECONNECT_MAX_MS, RECONNECT_BASE_MS * 2 ** consecutiveFailures) + +export interface LogStreamDeps { + readonly wsUrl: string + readonly otlp: Pick + readonly enqueueReport: (report: RailwayResultReport) => Effect.Effect + readonly webSocketFactory?: WebSocketFactory +} + +/** Run one target's log stream forever (flush loop + reconnecting subscription). */ +export const runLogStream = ( + target: InternalRailwayTarget, + deps: LogStreamDeps, +): Effect.Effect => + Effect.gen(function* () { + const state: LogStreamState = { + buffer: [], + droppedSinceLastReport: 0, + watermarkMs: target.logWatermarkMs, + } + + const flushOnce = Effect.gen(function* () { + if (state.buffer.length === 0) return + const now = yield* Clock.currentTimeMillis + const entries = state.buffer + state.buffer = [] + const dropped = state.droppedSinceLastReport + state.droppedSinceLastReport = 0 + + const request = buildLogsExportRequest(entries, { + projectId: target.projectId, + projectName: target.projectName, + environmentId: target.environmentId, + environmentName: target.environmentName, + services: target.services, + observedAtMs: now, + }) + if (request === null) return + + const outcome = yield* Effect.result(deps.otlp.sendLogs(target.ingestKey, request)) + if (Result.isSuccess(outcome)) { + const maxTs = entries.reduce((max, entry) => Math.max(max, entry.timestampMs), 0) + if (state.watermarkMs === null || maxTs > state.watermarkMs) state.watermarkMs = maxTs + yield* deps.enqueueReport( + new RailwayResultReport({ + targetId: target.id, + kind: "logs", + at: now, + error: null, + recordsIngested: entries.length, + ...(dropped > 0 ? { recordsDropped: dropped } : {}), + ...(state.watermarkMs !== null ? { newLogWatermarkMs: state.watermarkMs } : {}), + }), + ) + return + } + + // Ingest failure: a billing-limit 402 drops the batch (retrying cannot + // succeed until the limit resets); anything else re-buffers in front, + // bounded by MAX_BUFFERED_ENTRIES with oldest-first eviction. + const error = outcome.failure + if (error.status === 402) { + // `dropped` (pre-flush overflow evictions) was reset above — carry it + // into the count alongside the batch this branch is discarding. + state.droppedSinceLastReport += dropped + entries.length + } else { + const room = MAX_BUFFERED_ENTRIES - state.buffer.length + const requeued = entries.slice(Math.max(0, entries.length - room)) + state.droppedSinceLastReport += dropped + (entries.length - requeued.length) + state.buffer = [...requeued, ...state.buffer] + } + yield* deps.enqueueReport( + new RailwayResultReport({ + targetId: target.id, + kind: "logs", + at: now, + error: error.message, + ...(state.droppedSinceLastReport > 0 + ? { recordsDropped: state.droppedSinceLastReport } + : {}), + }), + ) + yield* Effect.logWarning("Railway log flush failed").pipe( + Effect.annotateLogs({ targetId: target.id, orgId: target.orgId, error: error.message }), + ) + }) + + // Flush on a cadence and immediately once a batch fills up. + yield* Effect.forkChild( + Effect.gen(function* () { + while (true) { + yield* Effect.sleep( + state.buffer.length >= FLUSH_BATCH_SIZE ? Duration.millis(50) : FLUSH_INTERVAL, + ) + yield* flushOnce + } + }), + ) + + let consecutiveFailures = 0 + while (true) { + const connectedAt = yield* Clock.currentTimeMillis + const outcome = yield* Effect.result( + runGraphqlWsSubscription({ + wsUrl: deps.wsUrl, + token: target.token, + query: ENVIRONMENT_LOGS_SUBSCRIPTION, + variables: { environmentId: target.environmentId }, + onNext: (data) => { + // `environmentLogs` streams either one entry or an array per tick. + const payload = + typeof data === "object" && data !== null + ? (data as { environmentLogs?: unknown }).environmentLogs + : undefined + const items = Array.isArray(payload) ? payload : payload !== undefined ? [payload] : [] + for (const item of items) { + const entry = decodeRailwayLogEntry(item) + if (entry !== null) enqueueLogEntry(state, entry) + } + }, + ...(deps.webSocketFactory !== undefined + ? { webSocketFactory: deps.webSocketFactory } + : {}), + }), + ) + + const now = yield* Clock.currentTimeMillis + const connectionLastedMs = now - connectedAt + consecutiveFailures = connectionLastedMs >= STABLE_CONNECTION_MS ? 1 : consecutiveFailures + 1 + + const message = Result.isFailure(outcome) + ? outcome.failure.message + : "subscription ended unexpectedly" + const unauthorized = Result.isFailure(outcome) ? outcome.failure.unauthorized : false + + yield* deps.enqueueReport( + new RailwayResultReport({ + targetId: target.id, + kind: "logs", + at: now, + error: `log stream disconnected: ${message}`, + ...(unauthorized ? { unauthorized: true } : {}), + }), + ) + const delayMs = reconnectDelayMs(consecutiveFailures) + yield* Effect.logWarning("Railway log stream disconnected, reconnecting").pipe( + Effect.annotateLogs({ + targetId: target.id, + orgId: target.orgId, + error: message, + delayMs, + unauthorized, + }), + ) + yield* Effect.sleep(Duration.millis(delayMs)) + } + }) as Effect.Effect diff --git a/apps/railway-connector/src/MetricsPoller.ts b/apps/railway-connector/src/MetricsPoller.ts new file mode 100644 index 000000000..7ea3b95c4 --- /dev/null +++ b/apps/railway-connector/src/MetricsPoller.ts @@ -0,0 +1,227 @@ +/** + * Per-(target × service) metrics polling: one `metrics` GraphQL query per + * service per interval requesting all measurements at once, converted to + * OTLP gauges and pushed through the ingest gateway. Delays adapt to + * Railway's rate-limit headers (see RailwayGraphql.nextPollDelayMs). + */ +import { Clock, Duration, Effect, Result, type Semaphore } from "effect" +import { RailwayResultReport, type InternalRailwayTarget } from "@maple/domain/http" +import { + buildMetricsExportRequest, + decodeRailwayMetrics, + RAILWAY_MEASUREMENTS, +} from "./otlp/metrics" +import { nextPollDelayMs, type RailwayGraphqlShape } from "./RailwayGraphql" +import type { OtlpIngestShape } from "./OtlpIngest" + +/** Never ask Railway for more than this much history in one poll window. */ +export const MAX_WINDOW_MS = Duration.toMillis(Duration.hours(1)) + +export const METRICS_QUERY = ` +query mapleServiceMetrics($environmentId: String!, $serviceId: String!, $startDate: DateTime!, $endDate: DateTime!, $sampleRateSeconds: Int, $measurements: [MetricMeasurement!]!) { + metrics(environmentId: $environmentId, serviceId: $serviceId, startDate: $startDate, endDate: $endDate, sampleRateSeconds: $sampleRateSeconds, measurements: $measurements) { + measurement + values { + ts + value + } + } +} +` + +/** + * The poll window `[startMs, endMs]`: resumes from the watermark, clamped so + * a long-disabled target does not request unbounded history. + */ +export const pollWindow = ( + nowMs: number, + watermarkMs: number | null, + intervalMs: number, +): { readonly startMs: number; readonly endMs: number } => { + const start = watermarkMs ?? nowMs - intervalMs + return { startMs: Math.max(start, nowMs - MAX_WINDOW_MS), endMs: nowMs } +} + +/** Deterministic per-loop start delay in `[0, baseMs)` (FNV-1a) so the services of one token don't poll in a synchronized burst. */ +export const initialJitterMs = (key: string, baseMs: number): number => { + if (baseMs <= 0) return 0 + let hash = 0x811c9dc5 + for (let i = 0; i < key.length; i++) { + hash ^= key.charCodeAt(i) + hash = Math.imul(hash, 0x01000193) + } + return (hash >>> 0) % baseMs +} + +export interface MetricsPollerDeps { + readonly graphql: RailwayGraphqlShape + readonly otlp: Pick + readonly enqueueReport: (report: RailwayResultReport) => Effect.Effect + /** Global cap on concurrent Railway polls across all targets. */ + readonly semaphore: Semaphore.Semaphore +} + +/** Run one service's metrics poll loop forever. */ +export const runMetricsPoller = ( + target: InternalRailwayTarget, + service: { readonly id: string; readonly name: string }, + deps: MetricsPollerDeps, +): Effect.Effect => { + const baseMs = target.metricsIntervalSeconds * 1000 + + interface PollOutcome { + readonly error: string | null + readonly unauthorized: boolean + readonly rateLimited: boolean + readonly retryAfterMs: number | null + readonly remaining: number | null + } + + return Effect.gen(function* () { + let watermarkMs = target.metricsWatermarkMs + + const pollOnce: Effect.Effect = deps.semaphore.withPermits(1)( + Effect.gen(function* () { + const nowMs = yield* Clock.currentTimeMillis + const { startMs, endMs } = pollWindow(nowMs, watermarkMs, baseMs) + + const result = yield* Effect.result( + deps.graphql.query(target.token, { + query: METRICS_QUERY, + variables: { + environmentId: target.environmentId, + serviceId: service.id, + startDate: new Date(startMs).toISOString(), + endDate: new Date(endMs).toISOString(), + sampleRateSeconds: Math.max(60, target.metricsIntervalSeconds), + measurements: [...RAILWAY_MEASUREMENTS], + }, + }), + ) + + if (Result.isFailure(result)) { + const error = result.failure + yield* deps.enqueueReport( + new RailwayResultReport({ + targetId: target.id, + kind: "metrics", + at: nowMs, + error: error.message, + ...(error.unauthorized ? { unauthorized: true } : {}), + ...(error.rateLimited ? { rateLimited: true } : {}), + }), + ) + return { + error: error.message, + unauthorized: error.unauthorized, + rateLimited: error.rateLimited, + retryAfterMs: error.retryAfterMs, + remaining: null, + } satisfies PollOutcome + } + + const series = decodeRailwayMetrics(result.success.data) + const batch = buildMetricsExportRequest( + series, + { + serviceName: service.name, + serviceId: service.id, + projectId: target.projectId, + projectName: target.projectName, + environmentId: target.environmentId, + environmentName: target.environmentName, + }, + watermarkMs, + ) + + if (batch.request !== null) { + const sent = yield* Effect.result(deps.otlp.sendMetrics(target.ingestKey, batch.request)) + if (Result.isFailure(sent)) { + // The watermark stays put so the samples are re-fetched next poll + // (dedupe happens via the watermark filter on conversion). + yield* deps.enqueueReport( + new RailwayResultReport({ + targetId: target.id, + kind: "metrics", + at: nowMs, + error: sent.failure.message, + }), + ) + return { + error: sent.failure.message, + unauthorized: false, + rateLimited: false, + retryAfterMs: null, + remaining: result.success.rate.remaining, + } satisfies PollOutcome + } + } + + if (batch.maxSampleMs !== null && (watermarkMs === null || batch.maxSampleMs > watermarkMs)) { + watermarkMs = batch.maxSampleMs + } + yield* deps.enqueueReport( + new RailwayResultReport({ + targetId: target.id, + kind: "metrics", + at: nowMs, + error: null, + recordsIngested: batch.dataPointCount, + ...(watermarkMs !== null ? { newMetricsWatermarkMs: watermarkMs } : {}), + }), + ) + return { + error: null, + unauthorized: false, + rateLimited: false, + retryAfterMs: null, + remaining: result.success.rate.remaining, + } satisfies PollOutcome + }).pipe( + Effect.withSpan("railway.poll_metrics", { + attributes: { + orgId: target.orgId, + targetId: target.id, + serviceId: service.id, + serviceName: service.name, + intervalSeconds: target.metricsIntervalSeconds, + }, + }), + ), + ) + + let consecutiveRateLimits = 0 + // De-sync the services sharing this token so polls spread across the + // interval instead of bursting together against the hourly budget. + yield* Effect.sleep(Duration.millis(initialJitterMs(`${target.id}:${service.id}`, baseMs))) + while (true) { + const startedAt = yield* Clock.currentTimeMillis + const outcome = yield* pollOnce + const elapsedMs = (yield* Clock.currentTimeMillis) - startedAt + + if (outcome.error !== null) { + yield* Effect.logWarning("Railway metrics poll failed").pipe( + Effect.annotateLogs({ + targetId: target.id, + orgId: target.orgId, + serviceId: service.id, + error: outcome.error, + }), + ) + } + + consecutiveRateLimits = outcome.rateLimited ? consecutiveRateLimits + 1 : 0 + const delayMs = nextPollDelayMs({ + baseMs, + rateLimited: outcome.rateLimited, + retryAfterMs: outcome.retryAfterMs, + remaining: outcome.remaining, + consecutiveRateLimits: Math.max(0, consecutiveRateLimits - 1), + }) + // Happy-path cadence stays start-to-start; backoff runs the full delay + // from poll end so Retry-After is honored. + const sleepMs = outcome.rateLimited ? delayMs : Math.max(0, delayMs - elapsedMs) + yield* Effect.sleep(Duration.millis(sleepMs)) + } + }) as Effect.Effect +} diff --git a/apps/railway-connector/src/OtlpIngest.ts b/apps/railway-connector/src/OtlpIngest.ts new file mode 100644 index 000000000..859aeb6d4 --- /dev/null +++ b/apps/railway-connector/src/OtlpIngest.ts @@ -0,0 +1,80 @@ +import { Context, Effect, Layer, Schema } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" +import type { OtlpLogsExportRequest } from "./otlp/logs" +import type { OtlpMetricsExportRequest } from "./otlp/metrics" +import { RailwayConnectorEnv } from "./Env" + +export class OtlpIngestError extends Schema.TaggedErrorClass()( + "@maple/railway-connector/OtlpIngestError", + { + message: Schema.String, + status: Schema.NullOr(Schema.Number), + }, +) {} + +export interface OtlpIngestShape { + /** + * Send an OTLP/JSON logs export through the Maple ingest gateway, + * authenticated with the target org's public ingest key — so the data is + * metered for billing and routed to the org's warehouse (Tinybird or + * self-managed ClickHouse) like any customer OTLP traffic. + */ + readonly sendLogs: ( + ingestKey: string, + request: OtlpLogsExportRequest, + ) => Effect.Effect + /** Same, for a metrics export. */ + readonly sendMetrics: ( + ingestKey: string, + request: OtlpMetricsExportRequest, + ) => Effect.Effect +} + +export class OtlpIngest extends Context.Service()( + "@maple/railway-connector/OtlpIngest", + { + make: Effect.gen(function* () { + const env = yield* RailwayConnectorEnv + const client = yield* HttpClient.HttpClient + + const send = Effect.fn("OtlpIngest.send")(function* ( + path: "/v1/logs" | "/v1/metrics", + ingestKey: string, + request: unknown, + ) { + const httpRequest = HttpClientRequest.post(`${env.MAPLE_INGEST_URL}${path}`, { + headers: { authorization: `Bearer ${ingestKey}` }, + }).pipe(HttpClientRequest.bodyText(JSON.stringify(request), "application/json")) + + const response = yield* client.execute(httpRequest).pipe( + Effect.mapError( + (error) => + new OtlpIngestError({ + message: `ingest gateway unreachable: ${error.message}`, + status: null, + }), + ), + ) + if (response.status < 200 || response.status >= 300) { + const text = yield* response.text.pipe(Effect.orElseSucceed(() => "")) + return yield* Effect.fail( + new OtlpIngestError({ + message: + response.status === 402 + ? `ingest gateway rejected ${path}: billing limit reached (HTTP 402): ${text.slice(0, 200)}` + : `ingest gateway returned HTTP ${response.status} for ${path}: ${text.slice(0, 200)}`, + status: response.status, + }), + ) + } + }) + + return { + sendLogs: (ingestKey, request) => send("/v1/logs", ingestKey, request), + sendMetrics: (ingestKey, request) => send("/v1/metrics", ingestKey, request), + } satisfies OtlpIngestShape + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/railway-connector/src/RailwayGraphql.test.ts b/apps/railway-connector/src/RailwayGraphql.test.ts new file mode 100644 index 000000000..248324a1f --- /dev/null +++ b/apps/railway-connector/src/RailwayGraphql.test.ts @@ -0,0 +1,99 @@ +import { assert, describe, it } from "@effect/vitest" +import { MAX_BACKOFF_MS, nextPollDelayMs, RATE_LOW_WATER } from "./RailwayGraphql" + +const BASE = 300_000 + +describe("nextPollDelayMs", () => { + it("returns the base interval on the happy path", () => { + assert.strictEqual( + nextPollDelayMs({ + baseMs: BASE, + rateLimited: false, + retryAfterMs: null, + remaining: 500, + consecutiveRateLimits: 0, + }), + BASE, + ) + }) + + it("stretches the interval when the remaining budget runs low", () => { + assert.strictEqual( + nextPollDelayMs({ + baseMs: 60_000, + rateLimited: false, + retryAfterMs: null, + remaining: RATE_LOW_WATER - 1, + consecutiveRateLimits: 0, + }), + 240_000, + ) + // The stretch still respects the global backoff cap. + assert.strictEqual( + nextPollDelayMs({ + baseMs: BASE, + rateLimited: false, + retryAfterMs: null, + remaining: RATE_LOW_WATER - 1, + consecutiveRateLimits: 0, + }), + MAX_BACKOFF_MS, + ) + // A missing header is not treated as low budget. + assert.strictEqual( + nextPollDelayMs({ + baseMs: BASE, + rateLimited: false, + retryAfterMs: null, + remaining: null, + consecutiveRateLimits: 0, + }), + BASE, + ) + }) + + it("escalates exponentially on consecutive rate limits, honoring Retry-After", () => { + const first = nextPollDelayMs({ + baseMs: 60_000, + rateLimited: true, + retryAfterMs: null, + remaining: 0, + consecutiveRateLimits: 0, + }) + assert.strictEqual(first, 120_000) + + const second = nextPollDelayMs({ + baseMs: 60_000, + rateLimited: true, + retryAfterMs: null, + remaining: 0, + consecutiveRateLimits: 1, + }) + assert.strictEqual(second, 240_000) + + // Retry-After wins when longer than the exponential step. + assert.strictEqual( + nextPollDelayMs({ + baseMs: 60_000, + rateLimited: true, + retryAfterMs: 600_000, + remaining: 0, + consecutiveRateLimits: 0, + }), + 600_000, + ) + }) + + it("caps backoff at MAX_BACKOFF_MS", () => { + assert.strictEqual( + nextPollDelayMs({ + baseMs: 300_000, + rateLimited: true, + retryAfterMs: 3_600_000, + remaining: 0, + consecutiveRateLimits: 10, + }), + MAX_BACKOFF_MS, + ) + }) +}) diff --git a/apps/railway-connector/src/RailwayGraphql.ts b/apps/railway-connector/src/RailwayGraphql.ts new file mode 100644 index 000000000..8378f5549 --- /dev/null +++ b/apps/railway-connector/src/RailwayGraphql.ts @@ -0,0 +1,231 @@ +/** + * HTTP executor for Railway's public GraphQL API with rate-limit accounting. + * + * Railway rate-limits by the CUSTOMER's plan (100 requests/hour free, 1,000 + * hobby, 10,000 pro) and the plan is not queryable, so the poller reads the + * `X-RateLimit-Remaining` / `Retry-After` response headers and stretches poll + * delays when the budget runs low instead of assuming a tier. + */ +import { Context, Duration, Effect, Layer, Schema } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" +import { RailwayConnectorEnv } from "./Env" + +export class RailwayRequestError extends Schema.TaggedErrorClass()( + "@maple/railway-connector/RailwayRequestError", + { + message: Schema.String, + status: Schema.NullOr(Schema.Number), + unauthorized: Schema.Boolean, + rateLimited: Schema.Boolean, + retryAfterMs: Schema.NullOr(Schema.Number), + }, +) {} + +export interface RailwayRateInfo { + /** Requests left in the current window per `X-RateLimit-Remaining`; null when absent. */ + readonly remaining: number | null +} + +export interface RailwayGraphqlResult { + readonly data: unknown + readonly rate: RailwayRateInfo +} + +/** Below this many remaining requests the pollers stretch their delays. */ +export const RATE_LOW_WATER = 25 +/** Upper bound on backoff so a target keeps probing for recovery. */ +export const MAX_BACKOFF_MS = Duration.toMillis(Duration.minutes(15)) + +/** + * The delay before a target's next metrics poll. The happy path returns the + * configured interval; a rate-limited attempt escalates exponentially — + * honoring `Retry-After` when longer — and a low remaining budget stretches + * the interval 4× so many services on one token stay inside the hourly + * window. Capped at {@link MAX_BACKOFF_MS}. + */ +export const nextPollDelayMs = ({ + baseMs, + rateLimited, + retryAfterMs, + remaining, + consecutiveRateLimits, +}: { + readonly baseMs: number + readonly rateLimited: boolean + readonly retryAfterMs: number | null + readonly remaining: number | null + readonly consecutiveRateLimits: number +}): number => { + if (rateLimited) { + const exponential = baseMs * 2 ** (consecutiveRateLimits + 1) + return Math.min(MAX_BACKOFF_MS, Math.max(exponential, retryAfterMs ?? 0)) + } + if (remaining !== null && remaining < RATE_LOW_WATER) { + return Math.min(MAX_BACKOFF_MS, baseMs * 4) + } + return baseMs +} + +const parseRemaining = (value: string | undefined): number | null => { + if (value === undefined) return null + const parsed = Number(value.trim()) + return Number.isFinite(parsed) && parsed >= 0 ? parsed : null +} + +const parseRetryAfterMs = (value: string | undefined): number | null => { + if (value === undefined) return null + const seconds = Number(value.trim()) + return Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : null +} + +const graphqlErrorMessages = (errors: unknown): string | null => { + if (!Array.isArray(errors) || errors.length === 0) return null + const messages = errors + .map((error) => + typeof error === "object" && error !== null && typeof (error as { message?: unknown }).message === "string" + ? ((error as { message: string }).message) + : null, + ) + .filter((message): message is string => message !== null) + return messages.length > 0 ? messages.join("; ") : "Railway API returned GraphQL errors" +} + +const isUnauthorizedMessage = (message: string): boolean => /not authorized|unauthorized/i.test(message) + +export interface RailwayGraphqlShape { + /** + * POST one GraphQL operation with the target's Railway token. GraphQL-level + * errors surface as `RailwayRequestError` (auth-shaped messages flagged + * `unauthorized`); HTTP 429 surfaces `rateLimited` with `Retry-After`. + */ + readonly query: ( + token: string, + request: { readonly query: string; readonly variables?: Record }, + ) => Effect.Effect +} + +export class RailwayGraphql extends Context.Service()( + "@maple/railway-connector/RailwayGraphql", + { + make: Effect.gen(function* () { + const env = yield* RailwayConnectorEnv + const client = yield* HttpClient.HttpClient + + const REQUEST_TIMEOUT = Duration.seconds(30) + + const query = Effect.fn("RailwayGraphql.query")(function* ( + token: string, + request: { readonly query: string; readonly variables?: Record }, + ) { + const httpRequest = HttpClientRequest.post(env.RAILWAY_GRAPHQL_HTTP_URL, { + headers: { authorization: `Bearer ${token}` }, + }).pipe( + HttpClientRequest.bodyText( + JSON.stringify({ + query: request.query, + ...(request.variables === undefined ? {} : { variables: request.variables }), + }), + "application/json", + ), + ) + + const response = yield* client.execute(httpRequest).pipe( + Effect.timeout(REQUEST_TIMEOUT), + Effect.mapError( + (error) => + new RailwayRequestError({ + message: `Railway API unreachable: ${error.message}`, + status: null, + unauthorized: false, + rateLimited: false, + retryAfterMs: null, + }), + ), + ) + + const rate: RailwayRateInfo = { + remaining: parseRemaining(response.headers["x-ratelimit-remaining"]), + } + + if (response.status === 429) { + return yield* Effect.fail( + new RailwayRequestError({ + message: "Railway API rate limit reached", + status: 429, + unauthorized: false, + rateLimited: true, + retryAfterMs: parseRetryAfterMs(response.headers["retry-after"]), + }), + ) + } + if (response.status === 401 || response.status === 403) { + return yield* Effect.fail( + new RailwayRequestError({ + message: "Railway rejected the API token", + status: response.status, + unauthorized: true, + rateLimited: false, + retryAfterMs: null, + }), + ) + } + + const text = yield* response.text.pipe( + Effect.mapError( + (error) => + new RailwayRequestError({ + message: `Railway API response unreadable: ${error.message}`, + status: response.status, + unauthorized: false, + rateLimited: false, + retryAfterMs: null, + }), + ), + ) + + if (response.status >= 400) { + return yield* Effect.fail( + new RailwayRequestError({ + message: `Railway API returned HTTP ${response.status}: ${text.slice(0, 200)}`, + status: response.status, + unauthorized: false, + rateLimited: false, + retryAfterMs: null, + }), + ) + } + + const parsed = yield* Effect.try({ + try: () => JSON.parse(text) as { data?: unknown; errors?: unknown }, + catch: () => + new RailwayRequestError({ + message: "Railway API returned invalid JSON", + status: response.status, + unauthorized: false, + rateLimited: false, + retryAfterMs: null, + }), + }) + + const errorMessage = graphqlErrorMessages(parsed.errors) + if (errorMessage !== null) { + return yield* Effect.fail( + new RailwayRequestError({ + message: errorMessage, + status: response.status, + unauthorized: isUnauthorizedMessage(errorMessage), + rateLimited: false, + retryAfterMs: null, + }), + ) + } + + return { data: parsed.data ?? null, rate } satisfies RailwayGraphqlResult + }) + + return { query } satisfies RailwayGraphqlShape + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/railway-connector/src/RailwaySubscriptionClient.test.ts b/apps/railway-connector/src/RailwaySubscriptionClient.test.ts new file mode 100644 index 000000000..3aecab017 --- /dev/null +++ b/apps/railway-connector/src/RailwaySubscriptionClient.test.ts @@ -0,0 +1,151 @@ +import { assert, describe, it } from "@effect/vitest" +import { Effect, Fiber, Result } from "effect" +import { + ENVIRONMENT_LOGS_SUBSCRIPTION, + runGraphqlWsSubscription, + type WebSocketLike, +} from "./RailwaySubscriptionClient" + +/** In-memory WebSocket double driving the graphql-transport-ws handshake. */ +class FakeSocket implements WebSocketLike { + readonly sent: Array> = [] + closed: { code?: number; reason?: string } | null = null + private listeners = new Map void>>() + + send(data: string): void { + this.sent.push(JSON.parse(data) as Record) + } + + close(code?: number, reason?: string): void { + this.closed = { ...(code !== undefined ? { code } : {}), ...(reason !== undefined ? { reason } : {}) } + } + + addEventListener(type: string, listener: (event: never) => void): void { + const existing = this.listeners.get(type) ?? [] + this.listeners.set(type, [...existing, listener]) + } + + emit(type: "open"): void + emit(type: "message", event: { data: unknown }): void + emit(type: "close", event: { code?: number; reason?: string }): void + emit(type: "error"): void + emit(type: string, event?: unknown): void { + for (const listener of this.listeners.get(type) ?? []) { + ;(listener as (event: unknown) => void)(event) + } + } + + serverSays(message: Record): void { + this.emit("message", { data: JSON.stringify(message) }) + } +} + +const start = (socket: FakeSocket, onNext: (data: unknown) => void) => + Effect.gen(function* () { + const fiber = yield* Effect.forkChild( + Effect.result( + runGraphqlWsSubscription({ + wsUrl: "wss://backboard.example/graphql/v2", + token: "railway-token", + query: ENVIRONMENT_LOGS_SUBSCRIPTION, + variables: { environmentId: "env-1" }, + onNext, + webSocketFactory: () => socket, + }), + ), + ) + // Let the callback register its listeners before the fake emits events. + yield* Effect.yieldNow + return fiber + }) + +describe("runGraphqlWsSubscription", () => { + it.effect("handshakes, subscribes, and pumps next payloads", () => + Effect.gen(function* () { + const received: unknown[] = [] + const socket = new FakeSocket() + const fiber = yield* start(socket, (data) => received.push(data)) + + socket.emit("open") + assert.strictEqual(socket.sent[0]!.type, "connection_init") + const initPayload = socket.sent[0]!.payload as Record + assert.strictEqual(initPayload.authorization, "Bearer railway-token") + + socket.serverSays({ type: "connection_ack" }) + assert.strictEqual(socket.sent[1]!.type, "subscribe") + const subscribePayload = socket.sent[1]!.payload as { query: string; variables: unknown } + assert.include(subscribePayload.query, "environmentLogs") + assert.deepStrictEqual(subscribePayload.variables, { environmentId: "env-1" }) + + socket.serverSays({ + id: "1", + type: "next", + payload: { data: { environmentLogs: [{ message: "hi" }] } }, + }) + assert.deepStrictEqual(received, [{ environmentLogs: [{ message: "hi" }] }]) + + // Server pings are answered with pongs to keep the connection alive. + socket.serverSays({ type: "ping" }) + assert.strictEqual(socket.sent.at(-1)!.type, "pong") + + yield* Fiber.interrupt(fiber) + }), + ) + + it.effect("fails when the server rejects the subscription, flagging auth errors", () => + Effect.gen(function* () { + const socket = new FakeSocket() + const fiber = yield* start(socket, () => {}) + + socket.emit("open") + socket.serverSays({ type: "connection_ack" }) + socket.serverSays({ id: "1", type: "error", payload: [{ message: "Not Authorized" }] }) + + const result = yield* Fiber.join(fiber) + assert.isTrue(Result.isFailure(result)) + if (Result.isFailure(result)) { + assert.isTrue(result.failure.unauthorized) + assert.include(result.failure.message, "Not Authorized") + } + }), + ) + + it.effect("fails on close and on server-initiated complete so the caller reconnects", () => + Effect.gen(function* () { + const closedSocket = new FakeSocket() + const closedFiber = yield* start(closedSocket, () => {}) + closedSocket.emit("open") + closedSocket.serverSays({ type: "connection_ack" }) + closedSocket.emit("close", { code: 1006, reason: "" }) + const closedResult = yield* Fiber.join(closedFiber) + assert.isTrue(Result.isFailure(closedResult)) + if (Result.isFailure(closedResult)) { + assert.include(closedResult.failure.message, "code 1006") + assert.isFalse(closedResult.failure.unauthorized) + } + + const completedSocket = new FakeSocket() + const completedFiber = yield* start(completedSocket, () => {}) + completedSocket.emit("open") + completedSocket.serverSays({ type: "connection_ack" }) + completedSocket.serverSays({ id: "1", type: "complete" }) + const completedResult = yield* Fiber.join(completedFiber) + assert.isTrue(Result.isFailure(completedResult)) + if (Result.isFailure(completedResult)) { + assert.include(completedResult.failure.message, "completed") + } + }), + ) + + it.effect("interruption closes the socket", () => + Effect.gen(function* () { + const socket = new FakeSocket() + const fiber = yield* start(socket, () => {}) + socket.emit("open") + socket.serverSays({ type: "connection_ack" }) + + yield* Fiber.interrupt(fiber) + assert.isNotNull(socket.closed) + }), + ) +}) diff --git a/apps/railway-connector/src/RailwaySubscriptionClient.ts b/apps/railway-connector/src/RailwaySubscriptionClient.ts new file mode 100644 index 000000000..2687565d3 --- /dev/null +++ b/apps/railway-connector/src/RailwaySubscriptionClient.ts @@ -0,0 +1,271 @@ +/** + * Minimal graphql-transport-ws client for Railway's GraphQL subscriptions + * (`wss://backboard.railway.com/graphql/v2`), hand-rolled over the runtime + * WebSocket so interruption stays Effect-native and no ws dependency is + * needed (Bun ships a spec-compliant WebSocket). + * + * Protocol (https://github.com/enisdenjo/graphql-ws/blob/master/PROTOCOL.md): + * `connection_init` (auth in the payload) → `connection_ack` → `subscribe` → + * stream of `next` / `error` / `complete`; server `ping` answered with `pong`. + */ +import { Effect, Schema } from "effect" + +export class RailwaySubscriptionError extends Schema.TaggedErrorClass()( + "@maple/railway-connector/RailwaySubscriptionError", + { + message: Schema.String, + unauthorized: Schema.Boolean, + }, +) {} + +/** The subset of the WebSocket interface the client uses (factory-injectable for tests). */ +export interface WebSocketLike { + send(data: string): void + close(code?: number, reason?: string): void + addEventListener(type: "open", listener: () => void): void + addEventListener(type: "message", listener: (event: { data: unknown }) => void): void + addEventListener(type: "close", listener: (event: { code?: number; reason?: string }) => void): void + addEventListener(type: "error", listener: () => void): void +} + +export type WebSocketFactory = (url: string, protocol: string) => WebSocketLike + +const CONNECTION_ACK_TIMEOUT_MS = 15_000 +const SUBSCRIPTION_ID = "1" + +interface ServerMessage { + readonly type: string + readonly id?: string + readonly payload?: unknown +} + +const parseServerMessage = (data: unknown): ServerMessage | null => { + if (typeof data !== "string") return null + try { + const parsed = JSON.parse(data) as unknown + if (typeof parsed !== "object" || parsed === null) return null + const message = parsed as Record + if (typeof message.type !== "string") return null + return { + type: message.type, + ...(typeof message.id === "string" ? { id: message.id } : {}), + ...("payload" in message ? { payload: message.payload } : {}), + } + } catch { + return null + } +} + +const errorPayloadMessage = (payload: unknown): string => { + if (Array.isArray(payload)) { + const messages = payload + .map((error) => + typeof error === "object" && + error !== null && + typeof (error as { message?: unknown }).message === "string" + ? (error as { message: string }).message + : null, + ) + .filter((message): message is string => message !== null) + if (messages.length > 0) return messages.join("; ") + } + return "subscription error" +} + +const isUnauthorizedMessage = (message: string): boolean => /not authorized|unauthorized|forbidden/i.test(message) + +export interface SubscriptionOptions { + readonly wsUrl: string + readonly token: string + readonly query: string + readonly variables: Record + /** + * Called synchronously for every entry in a `next` payload's `data`. The + * callback must not throw; buffer + backpressure live in the caller. + */ + readonly onNext: (data: unknown) => void + /** Injectable for tests; defaults to the runtime WebSocket. */ + readonly webSocketFactory?: WebSocketFactory +} + +const defaultFactory: WebSocketFactory = (url, protocol) => + new WebSocket(url, [protocol]) as unknown as WebSocketLike + +/** + * Open the socket, run the handshake, and pump `next` payloads into `onNext` + * until the server errors, completes, or the connection drops — all of which + * fail the effect so the caller's reconnect loop takes over. Interruption + * closes the socket. This effect never succeeds. + */ +export const runGraphqlWsSubscription = ( + options: SubscriptionOptions, +): Effect.Effect => + Effect.callback((resume, signal) => { + const factory = options.webSocketFactory ?? defaultFactory + let socket: WebSocketLike + let settled = false + + const fail = (error: RailwaySubscriptionError) => { + if (settled) return + settled = true + resume(Effect.fail(error)) + } + + try { + socket = factory(options.wsUrl, "graphql-transport-ws") + } catch (error) { + resume( + Effect.fail( + new RailwaySubscriptionError({ + message: error instanceof Error ? error.message : "WebSocket construction failed", + unauthorized: false, + }), + ), + ) + return + } + + let acked = false + const ackTimer = setTimeout(() => { + fail( + new RailwaySubscriptionError({ + message: `no connection_ack within ${CONNECTION_ACK_TIMEOUT_MS}ms`, + unauthorized: false, + }), + ) + try { + socket.close(1000) + } catch { + // already closed + } + }, CONNECTION_ACK_TIMEOUT_MS) + + socket.addEventListener("open", () => { + // Railway accepts the bearer in the init payload; both spellings are + // sent because community clients disagree on the expected key. + socket.send( + JSON.stringify({ + type: "connection_init", + payload: { + authorization: `Bearer ${options.token}`, + Authorization: `Bearer ${options.token}`, + }, + }), + ) + }) + + socket.addEventListener("message", (event) => { + const message = parseServerMessage(event.data) + if (message === null) return + + switch (message.type) { + case "connection_ack": { + if (acked) return + acked = true + clearTimeout(ackTimer) + socket.send( + JSON.stringify({ + id: SUBSCRIPTION_ID, + type: "subscribe", + payload: { query: options.query, variables: options.variables }, + }), + ) + return + } + case "ping": { + socket.send(JSON.stringify({ type: "pong" })) + return + } + case "next": { + if (message.id !== SUBSCRIPTION_ID) return + const payload = message.payload + if (typeof payload === "object" && payload !== null && "data" in payload) { + options.onNext((payload as { data: unknown }).data) + } + return + } + case "error": { + if (message.id !== SUBSCRIPTION_ID) return + const text = errorPayloadMessage(message.payload) + fail( + new RailwaySubscriptionError({ + message: `subscription rejected: ${text}`, + unauthorized: isUnauthorizedMessage(text), + }), + ) + try { + socket.close(1000) + } catch { + // already closed + } + return + } + case "complete": { + if (message.id !== SUBSCRIPTION_ID) return + // A live log stream should never complete — treat it as a drop so + // the caller reconnects. + fail( + new RailwaySubscriptionError({ + message: "subscription completed by server", + unauthorized: false, + }), + ) + try { + socket.close(1000) + } catch { + // already closed + } + return + } + default: + return + } + }) + + socket.addEventListener("close", (event) => { + clearTimeout(ackTimer) + const reason = event.reason && event.reason.length > 0 ? event.reason : `code ${event.code ?? "?"}` + fail( + new RailwaySubscriptionError({ + message: `connection closed: ${reason}`, + unauthorized: + event.code === 4401 || event.code === 4403 || isUnauthorizedMessage(event.reason ?? ""), + }), + ) + }) + + socket.addEventListener("error", () => { + clearTimeout(ackTimer) + fail(new RailwaySubscriptionError({ message: "connection error", unauthorized: false })) + }) + + signal.addEventListener("abort", () => { + clearTimeout(ackTimer) + try { + socket.close(1000, "interrupted") + } catch { + // already closed + } + }) + }) + +/** Railway's environment-wide runtime log stream (all services in the environment). */ +export const ENVIRONMENT_LOGS_SUBSCRIPTION = ` +subscription mapleEnvironmentLogs($environmentId: String!) { + environmentLogs(environmentId: $environmentId) { + timestamp + message + severity + tags { + projectId + environmentId + serviceId + deploymentId + } + attributes { + key + value + } + } +} +` diff --git a/apps/railway-connector/src/Scheduler.test.ts b/apps/railway-connector/src/Scheduler.test.ts new file mode 100644 index 000000000..5b4e030b4 --- /dev/null +++ b/apps/railway-connector/src/Scheduler.test.ts @@ -0,0 +1,119 @@ +import { assert, describe, it } from "@effect/vitest" +import { Effect, Schema } from "effect" +import { InternalRailwayTarget, RailwayResultReport } from "@maple/domain/http" +import { loopFingerprint, loopKey, loopsForTarget, sendResultsInChunks } from "./Scheduler" + +const decodeTarget = Schema.decodeUnknownSync(InternalRailwayTarget) + +const target = decodeTarget({ + id: "11111111-1111-4111-8111-111111111111", + orgId: "org_1", + connectionId: "22222222-2222-4222-8222-222222222222", + tokenType: "account", + token: "railway-token", + projectId: "proj-1", + projectName: "maple-demo", + environmentId: "env-1", + environmentName: "production", + services: [ + { id: "svc-1", name: "api" }, + { id: "svc-2", name: "worker" }, + ], + logsEnabled: true, + metricsEnabled: true, + metricsIntervalSeconds: 300, + logWatermarkMs: null, + metricsWatermarkMs: null, + ingestKey: "maple_pk_test", +}) + +const report = (at: number) => + new RailwayResultReport({ + targetId: target.id, + kind: "metrics", + at, + error: null, + }) + +describe("loopsForTarget", () => { + it("expands to one log loop plus one metrics loop per service", () => { + const loops = loopsForTarget(target) + assert.deepStrictEqual(loops.map(loopKey), [ + `${target.id}:logs`, + `${target.id}:metrics:svc-1`, + `${target.id}:metrics:svc-2`, + ]) + }) + + it("honors the logs/metrics toggles", () => { + const logsOnly = decodeTarget({ ...target, token: target.token, metricsEnabled: false }) + assert.deepStrictEqual(loopsForTarget(logsOnly).map(loopKey), [`${target.id}:logs`]) + + const metricsOnly = decodeTarget({ ...target, logsEnabled: false }) + assert.strictEqual(loopsForTarget(metricsOnly).length, 2) + }) +}) + +describe("loopFingerprint", () => { + it("changes when the token, interval, or services change — but not on watermarks", () => { + const loops = loopsForTarget(target) + const logsLoop = loops[0]! + const metricsLoop = loops[1]! + + const watermarked = decodeTarget({ ...target, logWatermarkMs: 99, metricsWatermarkMs: 42 }) + assert.strictEqual(loopFingerprint(logsLoop), loopFingerprint(loopsForTarget(watermarked)[0]!)) + assert.strictEqual( + loopFingerprint(metricsLoop), + loopFingerprint(loopsForTarget(watermarked)[1]!), + ) + + const rotated = decodeTarget({ ...target, token: "new-token" }) + assert.notStrictEqual(loopFingerprint(logsLoop), loopFingerprint(loopsForTarget(rotated)[0]!)) + + const renamed = decodeTarget({ + ...target, + services: [ + { id: "svc-1", name: "renamed" }, + { id: "svc-2", name: "worker" }, + ], + }) + assert.notStrictEqual(loopFingerprint(logsLoop), loopFingerprint(loopsForTarget(renamed)[0]!)) + + const retimed = decodeTarget({ ...target, metricsIntervalSeconds: 600 }) + assert.notStrictEqual( + loopFingerprint(metricsLoop), + loopFingerprint(loopsForTarget(retimed)[1]!), + ) + // The logs loop does not restart on a metrics interval change. + assert.strictEqual(loopFingerprint(logsLoop), loopFingerprint(loopsForTarget(retimed)[0]!)) + }) +}) + +describe("sendResultsInChunks", () => { + it.effect("sends everything in order and reports nothing unsent", () => + Effect.gen(function* () { + const sent: number[] = [] + const outcome = yield* sendResultsInChunks( + [report(1), report(2), report(3)], + 2, + (chunk) => Effect.sync(() => void sent.push(chunk.length)), + ) + assert.deepStrictEqual(sent, [2, 1]) + assert.strictEqual(outcome.unsent.length, 0) + assert.isNull(outcome.error) + }), + ) + + it.effect("stops at the first failed chunk and returns the remainder", () => + Effect.gen(function* () { + let calls = 0 + const outcome = yield* sendResultsInChunks([report(1), report(2), report(3)], 1, () => { + calls += 1 + return calls === 2 ? Effect.fail(new Error("boom")) : Effect.void + }) + assert.strictEqual(outcome.unsent.length, 2) + assert.strictEqual(outcome.unsent[0]!.at, 2) + assert.strictEqual(outcome.error?.message, "boom") + }), + ) +}) diff --git a/apps/railway-connector/src/Scheduler.ts b/apps/railway-connector/src/Scheduler.ts new file mode 100644 index 000000000..162102d7a --- /dev/null +++ b/apps/railway-connector/src/Scheduler.ts @@ -0,0 +1,255 @@ +/** + * The connector's reconcile loop (cloned from apps/scraper's ScrapeScheduler): + * refresh the target list from the api on an interval, keep one log-stream + * fiber per target and one metrics-poll fiber per (target, service), and + * flush buffered result reports back to the api periodically. + */ +import { Clock, Context, Duration, Effect, Fiber, Layer, Ref, Result, Schedule, Semaphore } from "effect" +import type { InternalRailwayTarget, RailwayResultReport } from "@maple/domain/http" +import { ApiClient, type ApiRequestError } from "./ApiClient" +import { RailwayConnectorEnv } from "./Env" +import { runLogStream } from "./LogStream" +import { runMetricsPoller } from "./MetricsPoller" +import { OtlpIngest } from "./OtlpIngest" +import { RailwayGraphql } from "./RailwayGraphql" + +interface SchedulerStats { + readonly activeLoops: number + readonly lastReconcileAt: number | null + readonly pendingResults: number +} + +export interface RailwaySchedulerShape { + /** + * Run the connector forever: reconcile the target list on an interval, + * keep the per-target fibers alive, flush result reports periodically. + * Only exits on interruption. + */ + readonly run: Effect.Effect + readonly stats: Effect.Effect +} + +const RESULTS_FLUSH_INTERVAL = Duration.seconds(10) +/** Cap the report buffer so an unreachable API cannot grow memory unboundedly. */ +const MAX_BUFFERED_RESULTS = 10_000 +/** Max reports per `railway-results` POST (keeps single POSTs Worker-friendly). */ +const RESULTS_FLUSH_CHUNK_SIZE = 1_000 + +/** + * Send `results` to `send` in chunks, stopping at the first failed chunk. + * Returns the results that were NOT delivered so the caller re-buffers just + * those; `unsent` is empty when the whole batch went through. + */ +export const sendResultsInChunks = ( + results: ReadonlyArray, + chunkSize: number, + send: (chunk: ReadonlyArray) => Effect.Effect, +): Effect.Effect<{ readonly unsent: ReadonlyArray; readonly error: E | null }> => + Effect.gen(function* () { + for (let index = 0; index < results.length; index += chunkSize) { + const chunk = results.slice(index, index + chunkSize) + const outcome = yield* Effect.result(send(chunk)) + if (Result.isFailure(outcome)) return { unsent: results.slice(index), error: outcome.failure } + } + return { unsent: [], error: null } + }) + +export type LoopDescriptor = + | { readonly kind: "logs"; readonly target: InternalRailwayTarget } + | { + readonly kind: "metrics" + readonly target: InternalRailwayTarget + readonly service: { readonly id: string; readonly name: string } + } + +/** Fiber-map key: one log stream per target, one metrics loop per service. */ +export const loopKey = (descriptor: LoopDescriptor): string => + descriptor.kind === "logs" + ? `${descriptor.target.id}:logs` + : `${descriptor.target.id}:metrics:${descriptor.service.id}` + +/** + * Restart a loop when anything affecting its output changes. Watermarks are + * deliberately EXCLUDED — they advance on every report and each running loop + * tracks its own in memory; including them would restart every fiber on every + * reconcile. + */ +export const loopFingerprint = (descriptor: LoopDescriptor): string => { + const target = descriptor.target + const common = [ + target.token, + target.ingestKey, + target.orgId, + target.projectId, + target.projectName, + target.environmentId, + target.environmentName, + ] + if (descriptor.kind === "logs") { + // The services map drives per-service resource attribution of log lines. + return JSON.stringify([ + ...common, + "logs", + descriptor.target.services.map((service) => [service.id, service.name]), + ]) + } + return JSON.stringify([ + ...common, + "metrics", + descriptor.service.id, + descriptor.service.name, + target.metricsIntervalSeconds, + ]) +} + +/** Expand a target into the loops it should be running. */ +export const loopsForTarget = (target: InternalRailwayTarget): LoopDescriptor[] => [ + ...(target.logsEnabled ? [{ kind: "logs", target } satisfies LoopDescriptor] : []), + ...(target.metricsEnabled + ? target.services.map( + (service) => + ({ kind: "metrics", target, service: { id: service.id, name: service.name } }) satisfies LoopDescriptor, + ) + : []), +] + +interface LoopEntry { + readonly fingerprint: string + readonly fiber: Fiber.Fiber +} + +export class RailwayScheduler extends Context.Service()( + "@maple/railway-connector/Scheduler", + { + make: Effect.gen(function* () { + const env = yield* RailwayConnectorEnv + const api = yield* ApiClient + const otlp = yield* OtlpIngest + const graphql = yield* RailwayGraphql + + const semaphore = yield* Semaphore.make(env.RAILWAY_CONNECTOR_CONCURRENCY) + const resultsRef = yield* Ref.make>([]) + const fibersRef = yield* Ref.make(new Map()) + const lastReconcileRef = yield* Ref.make(null) + + const enqueueReport = (report: RailwayResultReport) => + Ref.update(resultsRef, (buffered) => + buffered.length >= MAX_BUFFERED_RESULTS + ? [...buffered.slice(1), report] + : [...buffered, report], + ) + + const runLoop = (descriptor: LoopDescriptor): Effect.Effect => + descriptor.kind === "logs" + ? runLogStream(descriptor.target, { + wsUrl: env.RAILWAY_GRAPHQL_WS_URL, + otlp, + enqueueReport, + }) + : runMetricsPoller(descriptor.target, descriptor.service, { + graphql, + otlp, + enqueueReport, + semaphore, + }) + + const reconcile = Effect.gen(function* () { + const targets = yield* api.listTargets() + const current = yield* Ref.get(fibersRef) + const next = new Map() + + // Collapse to one descriptor per key (last wins) so duplicate rows + // can never leak untracked fibers across reconciles. + const deduped = new Map() + for (const target of targets) { + for (const descriptor of loopsForTarget(target)) { + deduped.set(loopKey(descriptor), descriptor) + } + } + + yield* Effect.forEach( + deduped.entries(), + ([key, descriptor]) => + Effect.gen(function* () { + const fingerprint = loopFingerprint(descriptor) + const existing = current.get(key) + if (existing && existing.fingerprint === fingerprint) { + next.set(key, existing) + return + } + if (existing) yield* Fiber.interrupt(existing.fiber) + const fiber = yield* Effect.forkChild(runLoop(descriptor)) + next.set(key, { fingerprint, fiber }) + }), + { discard: true }, + ) + + yield* Effect.forEach( + current, + ([key, entry]) => (next.has(key) ? Effect.void : Fiber.interrupt(entry.fiber)), + { discard: true }, + ) + + yield* Ref.set(fibersRef, next) + yield* Ref.set(lastReconcileRef, yield* Clock.currentTimeMillis) + yield* Effect.annotateCurrentSpan({ activeLoops: next.size, targets: targets.length }) + }).pipe( + Effect.withSpan("railway.reconcile"), + // A failed list fetch keeps the current fibers running untouched. + Effect.catch((error) => + Effect.logWarning("Failed to refresh Railway target list").pipe( + Effect.annotateLogs({ error: error.message }), + ), + ), + ) + + const flushResults = Effect.gen(function* () { + const results = yield* Ref.getAndSet(resultsRef, []) + if (results.length === 0) return + const { unsent, error } = yield* sendResultsInChunks( + results, + RESULTS_FLUSH_CHUNK_SIZE, + api.reportResults, + ) + if (unsent.length > 0) { + yield* Ref.update(resultsRef, (buffered) => + [...unsent, ...buffered].slice(-MAX_BUFFERED_RESULTS), + ) + yield* Effect.logWarning("Failed to report Railway results").pipe( + Effect.annotateLogs({ + error: error?.message ?? "unknown", + bufferedResults: unsent.length, + }), + ) + } + }).pipe(Effect.withSpan("railway.flush_results")) + + const run = Effect.gen(function* () { + yield* Effect.forkChild( + flushResults.pipe(Effect.repeat(Schedule.spaced(RESULTS_FLUSH_INTERVAL))), + ) + return yield* reconcile.pipe( + Effect.repeat( + Schedule.spaced(Duration.seconds(env.RAILWAY_CONNECTOR_RECONCILE_INTERVAL_SECONDS)), + ), + Effect.flatMap(() => Effect.never), + ) + }) as Effect.Effect + + const stats = Effect.gen(function* () { + const fibers = yield* Ref.get(fibersRef) + const lastReconcileAt = yield* Ref.get(lastReconcileRef) + const pending = yield* Ref.get(resultsRef) + return { + activeLoops: fibers.size, + lastReconcileAt, + pendingResults: pending.length, + } satisfies SchedulerStats + }) + + return { run, stats } satisfies RailwaySchedulerShape + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/apps/railway-connector/src/main.ts b/apps/railway-connector/src/main.ts new file mode 100644 index 000000000..0a0bf0917 --- /dev/null +++ b/apps/railway-connector/src/main.ts @@ -0,0 +1,72 @@ +#!/usr/bin/env bun +/** + * The Maple Railway connector: a small standalone service that pulls + * customer telemetry from Railway's public GraphQL API. + * + * It polls the Maple API for enabled Railway targets (org-connected + * project/environments), streams each environment's runtime logs over a + * graphql-transport-ws subscription, polls per-service resource metrics + * (CPU/memory/network/disk), converts both to OTLP/JSON, and pushes them + * through the Maple ingest gateway with each org's public ingest key so the + * data is billed and warehouse-routed like customer OTLP. Outcomes and + * watermarks are reported back to the API. A `/health` endpoint serves the + * Railway healthcheck. + */ +import { BunRuntime } from "@effect/platform-bun" +import { Maple } from "@maple-dev/effect-sdk/server" +import { Effect, Layer } from "effect" +import { FetchHttpClient } from "effect/unstable/http" +import { ApiClient } from "./ApiClient" +import { RailwayConnectorEnv } from "./Env" +import { OtlpIngest } from "./OtlpIngest" +import { RailwayGraphql } from "./RailwayGraphql" +import { RailwayScheduler } from "./Scheduler" + +const TelemetryLayer = Maple.layer({ + serviceName: "railway-connector", + serviceNamespace: "backend", + repositoryUrl: "https://github.com/Makisuo/maple", + shutdownTimeout: "3 seconds", +}) + +const MainLayer = RailwayScheduler.layer.pipe( + Layer.provide(Layer.mergeAll(ApiClient.layer, OtlpIngest.layer, RailwayGraphql.layer)), + Layer.provideMerge(RailwayConnectorEnv.layer), + Layer.provide(FetchHttpClient.layer), +) + +const healthServer = Effect.gen(function* () { + const env = yield* RailwayConnectorEnv + const scheduler = yield* RailwayScheduler + + const server = yield* Effect.acquireRelease( + Effect.sync(() => + Bun.serve({ + port: env.PORT, + hostname: "0.0.0.0", + fetch: async (request) => { + const url = new URL(request.url) + if (url.pathname === "/health") { + const stats = await Effect.runPromise(scheduler.stats) + return new Response(JSON.stringify({ status: "ok", ...stats }), { + headers: { "content-type": "application/json" }, + }) + } + return new Response("maple-railway-connector", { status: 404 }) + }, + }), + ), + (running) => Effect.promise(() => running.stop()), + ) + + yield* Effect.logInfo("Health endpoint listening").pipe(Effect.annotateLogs({ port: server.port })) +}) + +const program = Effect.gen(function* () { + yield* healthServer + const scheduler = yield* RailwayScheduler + yield* Effect.logInfo("Maple Railway connector starting") + return yield* scheduler.run +}) + +program.pipe(Effect.scoped, Effect.provide(MainLayer), Effect.provide(TelemetryLayer), BunRuntime.runMain) diff --git a/apps/railway-connector/src/otlp/common.ts b/apps/railway-connector/src/otlp/common.ts new file mode 100644 index 000000000..2072be2dd --- /dev/null +++ b/apps/railway-connector/src/otlp/common.ts @@ -0,0 +1,49 @@ +/** + * Shared OTLP/JSON building blocks for the Railway connector. + * + * JSON shape contract — pinned by the Rust deserializer in `apps/ingest` + * (`opentelemetry-proto` `with-serde`; see the `scraper_contract` test in + * `apps/ingest/src/telemetry.rs`): camelCase field names, flattened oneofs + * (`asDouble`, `stringValue`), and `timeUnixNano`/`startTimeUnixNano` as + * STRINGS (custom u64-from-string). + */ + +export interface OtlpKeyValue { + readonly key: string + readonly value: { readonly stringValue: string } +} + +export interface RailwayResourceContext { + /** Railway service name → `service.name` (drives the Maple service views). */ + readonly serviceName: string + readonly serviceId: string + readonly projectId: string + readonly projectName: string | null + readonly environmentId: string + readonly environmentName: string | null +} + +export const attribute = (key: string, value: string): OtlpKeyValue => ({ + key, + value: { stringValue: value }, +}) + +/** + * One OTLP resource per Railway service so pulled telemetry shows up as + * first-class services in Maple. The gateway strips/stamps `maple_org_id` + * and `maple_ingest_source` itself — only the railway identity goes here. + */ +export const buildResourceAttributes = (context: RailwayResourceContext): OtlpKeyValue[] => [ + attribute("service.name", context.serviceName), + ...(context.projectName === null ? [] : [attribute("service.namespace", context.projectName)]), + attribute("cloud.provider", "railway"), + attribute("railway.project.id", context.projectId), + ...(context.projectName === null ? [] : [attribute("railway.project.name", context.projectName)]), + attribute("railway.environment.id", context.environmentId), + ...(context.environmentName === null + ? [] + : [attribute("railway.environment.name", context.environmentName)]), + attribute("railway.service.id", context.serviceId), +] + +export const msToUnixNano = (ms: number): string => `${Math.round(ms)}000000` diff --git a/apps/railway-connector/src/otlp/logs.test.ts b/apps/railway-connector/src/otlp/logs.test.ts new file mode 100644 index 000000000..ce9af48e9 --- /dev/null +++ b/apps/railway-connector/src/otlp/logs.test.ts @@ -0,0 +1,140 @@ +import { assert, describe, it } from "@effect/vitest" +import { + buildLogsExportRequest, + decodeRailwayLogEntry, + severityNumberFor, + type RailwayLogEntry, +} from "./logs" + +const context = { + projectId: "proj-1", + projectName: "maple-demo", + environmentId: "env-1", + environmentName: "production", + services: [ + { id: "svc-1", name: "api" }, + { id: "svc-2", name: "worker" }, + ], + observedAtMs: 1_750_000_100_000, +} + +const entry = (overrides: Partial): RailwayLogEntry => ({ + timestampMs: 1_750_000_000_000, + message: "hello", + severity: "info", + serviceId: "svc-1", + deploymentId: "dep-1", + attributes: [], + ...overrides, +}) + +describe("decodeRailwayLogEntry", () => { + it("decodes a full subscription entry", () => { + const decoded = decodeRailwayLogEntry({ + timestamp: "2026-06-15T12:00:00.000Z", + message: "GET /health 200", + severity: "info", + tags: { serviceId: "svc-1", deploymentId: "dep-9", environmentId: "env-1" }, + attributes: [ + { key: "level", value: "info" }, + { key: "requestId", value: "abc" }, + ], + }) + assert.isNotNull(decoded) + assert.strictEqual(decoded!.timestampMs, Date.parse("2026-06-15T12:00:00.000Z")) + assert.strictEqual(decoded!.message, "GET /health 200") + assert.strictEqual(decoded!.serviceId, "svc-1") + assert.strictEqual(decoded!.deploymentId, "dep-9") + assert.strictEqual(decoded!.attributes.length, 2) + }) + + it("degrades missing tags/severity/attributes instead of dropping the line", () => { + const decoded = decodeRailwayLogEntry({ timestamp: 1_750_000_000_000, message: "bare line" }) + assert.isNotNull(decoded) + assert.isNull(decoded!.severity) + assert.isNull(decoded!.serviceId) + assert.deepStrictEqual(decoded!.attributes, []) + }) + + it("drops entries with no message or unparseable timestamp", () => { + assert.isNull(decodeRailwayLogEntry({ timestamp: "2026-06-15T12:00:00Z" })) + assert.isNull(decodeRailwayLogEntry({ timestamp: "not-a-date", message: "x" })) + assert.isNull(decodeRailwayLogEntry("not an object")) + }) +}) + +describe("severityNumberFor", () => { + it("maps Railway severities onto OTLP severity numbers", () => { + assert.strictEqual(severityNumberFor("debug"), 5) + assert.strictEqual(severityNumberFor("info"), 9) + assert.strictEqual(severityNumberFor("warn"), 13) + assert.strictEqual(severityNumberFor("WARNING"), 13) + assert.strictEqual(severityNumberFor("error"), 17) + assert.strictEqual(severityNumberFor("err"), 17) + assert.strictEqual(severityNumberFor(null), 0) + assert.strictEqual(severityNumberFor("weird"), 0) + }) +}) + +describe("buildLogsExportRequest", () => { + it("groups entries into one resource per Railway service", () => { + const request = buildLogsExportRequest( + [ + entry({ serviceId: "svc-1", message: "from api" }), + entry({ serviceId: "svc-2", message: "from worker", severity: "error" }), + entry({ serviceId: "svc-1", message: "api again" }), + ], + context, + ) + assert.isNotNull(request) + assert.strictEqual(request!.resourceLogs.length, 2) + + const api = request!.resourceLogs.find((resource) => + resource.resource.attributes.some( + (attr) => attr.key === "service.name" && attr.value.stringValue === "api", + ), + )! + assert.strictEqual(api.scopeLogs[0]!.logRecords.length, 2) + const attrs = Object.fromEntries( + api.resource.attributes.map((attr) => [attr.key, attr.value.stringValue]), + ) + assert.strictEqual(attrs["cloud.provider"], "railway") + assert.strictEqual(attrs["service.namespace"], "maple-demo") + assert.strictEqual(attrs["railway.project.id"], "proj-1") + assert.strictEqual(attrs["railway.environment.name"], "production") + assert.strictEqual(attrs["railway.service.id"], "svc-1") + + const worker = request!.resourceLogs.find((resource) => + resource.resource.attributes.some( + (attr) => attr.key === "service.name" && attr.value.stringValue === "worker", + ), + )! + const record = worker.scopeLogs[0]!.logRecords[0]! + assert.strictEqual(record.severityNumber, 17) + assert.strictEqual(record.body.stringValue, "from worker") + // Rust deserializer contract: nanosecond timestamps are strings. + assert.strictEqual(record.timeUnixNano, "1750000000000000000") + assert.isTrue( + record.attributes.some( + (attr) => attr.key === "railway.deployment.id" && attr.value.stringValue === "dep-1", + ), + ) + }) + + it("attributes unknown and missing service ids without dropping records", () => { + const request = buildLogsExportRequest( + [entry({ serviceId: "svc-unknown" }), entry({ serviceId: null })], + context, + ) + assert.isNotNull(request) + const names = request!.resourceLogs.map( + (resource) => + resource.resource.attributes.find((attr) => attr.key === "service.name")!.value.stringValue, + ) + assert.includeMembers(names, ["railway:svc-unknown", "railway-environment:production"]) + }) + + it("returns null for an empty batch", () => { + assert.isNull(buildLogsExportRequest([], context)) + }) +}) diff --git a/apps/railway-connector/src/otlp/logs.ts b/apps/railway-connector/src/otlp/logs.ts new file mode 100644 index 000000000..acd668f73 --- /dev/null +++ b/apps/railway-connector/src/otlp/logs.ts @@ -0,0 +1,190 @@ +/** + * Converts Railway `environmentLogs` subscription entries into an OTLP/JSON + * `ExportLogsServiceRequest` for the Maple ingest gateway (`POST /v1/logs`). + */ +import { + attribute, + buildResourceAttributes, + msToUnixNano, + type OtlpKeyValue, + type RailwayResourceContext, +} from "./common" + +/** A tolerant parse of one Railway log entry (see decodeRailwayLogEntry). */ +export interface RailwayLogEntry { + /** Epoch milliseconds. */ + readonly timestampMs: number + readonly message: string + /** Railway's severity string (`info`, `warn`, `error`, …) or null. */ + readonly severity: string | null + /** Railway service the line came from; null for environment-level entries. */ + readonly serviceId: string | null + readonly deploymentId: string | null + /** Structured attributes Railway extracted from the line. */ + readonly attributes: ReadonlyArray<{ readonly key: string; readonly value: string }> +} + +export interface OtlpLogRecord { + readonly timeUnixNano: string + readonly observedTimeUnixNano: string + readonly severityNumber: number + readonly severityText: string + readonly body: { readonly stringValue: string } + readonly attributes: ReadonlyArray +} + +export interface OtlpLogsExportRequest { + readonly resourceLogs: ReadonlyArray<{ + readonly resource: { readonly attributes: ReadonlyArray } + readonly scopeLogs: ReadonlyArray<{ + readonly scope: { readonly name: string } + readonly logRecords: ReadonlyArray + }> + }> +} + +const SCOPE_NAME = "railway.environment_logs" + +/** + * OTLP severity numbers (spec: 1-4 TRACE, 5-8 DEBUG, 9-12 INFO, 13-16 WARN, + * 17-20 ERROR, 21-24 FATAL; 0 unspecified). + */ +export const severityNumberFor = (severity: string | null): number => { + if (severity === null) return 0 + switch (severity.toLowerCase()) { + case "trace": + return 1 + case "debug": + return 5 + case "info": + return 9 + case "warn": + case "warning": + return 13 + case "err": + case "error": + return 17 + case "fatal": + case "critical": + return 21 + default: + return 0 + } +} + +/** + * Tolerant decode of a subscription `next` payload entry. Railway's schema + * carries `timestamp` (RFC3339), `message`, `severity`, `tags { serviceId, + * deploymentId, … }`, and `attributes [{ key, value }]`; anything missing + * degrades instead of dropping the line (only an unparseable timestamp or a + * missing message discards it). + */ +export const decodeRailwayLogEntry = (raw: unknown): RailwayLogEntry | null => { + if (typeof raw !== "object" || raw === null) return null + const entry = raw as Record + + const message = typeof entry.message === "string" ? entry.message : null + if (message === null) return null + + const timestampMs = + typeof entry.timestamp === "string" + ? Date.parse(entry.timestamp) + : typeof entry.timestamp === "number" + ? entry.timestamp + : Number.NaN + if (!Number.isFinite(timestampMs)) return null + + const tags = + typeof entry.tags === "object" && entry.tags !== null ? (entry.tags as Record) : {} + + const attributes: Array<{ key: string; value: string }> = [] + if (Array.isArray(entry.attributes)) { + for (const item of entry.attributes) { + if (typeof item !== "object" || item === null) continue + const pair = item as Record + if (typeof pair.key === "string" && typeof pair.value === "string") { + attributes.push({ key: pair.key, value: pair.value }) + } + } + } + + return { + timestampMs, + message, + severity: typeof entry.severity === "string" ? entry.severity : null, + serviceId: typeof tags.serviceId === "string" ? tags.serviceId : null, + deploymentId: typeof tags.deploymentId === "string" ? tags.deploymentId : null, + attributes, + } +} + +export const buildLogRecord = (entry: RailwayLogEntry, observedAtMs: number): OtlpLogRecord => ({ + timeUnixNano: msToUnixNano(entry.timestampMs), + observedTimeUnixNano: msToUnixNano(observedAtMs), + severityNumber: severityNumberFor(entry.severity), + severityText: entry.severity ?? "", + body: { stringValue: entry.message }, + attributes: [ + ...(entry.deploymentId === null ? [] : [attribute("railway.deployment.id", entry.deploymentId)]), + ...entry.attributes.map((pair) => attribute(pair.key, pair.value)), + ], +}) + +export interface LogBatchContext { + readonly projectId: string + readonly projectName: string | null + readonly environmentId: string + readonly environmentName: string | null + /** Environment services (id → name) for per-service resource attribution. */ + readonly services: ReadonlyArray<{ readonly id: string; readonly name: string }> + readonly observedAtMs: number +} + +/** + * Group a batch of entries into one export request with one resource per + * Railway service. Entries with no/unknown serviceId attribute to a synthetic + * environment-level service so nothing is silently dropped; unknown service + * ids degrade to `railway:` until the next discovery refresh names them. + */ +export const buildLogsExportRequest = ( + entries: ReadonlyArray, + context: LogBatchContext, +): OtlpLogsExportRequest | null => { + if (entries.length === 0) return null + + const nameByServiceId = new Map(context.services.map((service) => [service.id, service.name])) + const byServiceId = new Map() + for (const entry of entries) { + const key = entry.serviceId ?? "" + const bucket = byServiceId.get(key) + if (bucket === undefined) byServiceId.set(key, [entry]) + else bucket.push(entry) + } + + const environmentLabel = context.environmentName ?? context.environmentId + + return { + resourceLogs: [...byServiceId.entries()].map(([serviceId, serviceEntries]) => { + const resourceContext: RailwayResourceContext = { + serviceName: + serviceId === "" + ? `railway-environment:${environmentLabel}` + : (nameByServiceId.get(serviceId) ?? `railway:${serviceId}`), + serviceId: serviceId === "" ? "environment" : serviceId, + projectId: context.projectId, + projectName: context.projectName, + environmentId: context.environmentId, + environmentName: context.environmentName, + } + return { + resource: { attributes: buildResourceAttributes(resourceContext) }, + scopeLogs: [ + { + scope: { name: SCOPE_NAME }, + logRecords: serviceEntries.map((entry) => buildLogRecord(entry, context.observedAtMs)), + }, + ], + } + }), + } +} diff --git a/apps/railway-connector/src/otlp/metrics.test.ts b/apps/railway-connector/src/otlp/metrics.test.ts new file mode 100644 index 000000000..33b5dd0c7 --- /dev/null +++ b/apps/railway-connector/src/otlp/metrics.test.ts @@ -0,0 +1,125 @@ +import { assert, describe, it } from "@effect/vitest" +import { buildMetricsExportRequest, decodeRailwayMetrics } from "./metrics" + +const context = { + serviceName: "api", + serviceId: "svc-1", + projectId: "proj-1", + projectName: "maple-demo", + environmentId: "env-1", + environmentName: "production", +} + +describe("decodeRailwayMetrics", () => { + it("decodes series with epoch-second timestamps", () => { + const series = decodeRailwayMetrics({ + metrics: [ + { + measurement: "CPU_USAGE", + values: [ + { ts: 1_750_000_000, value: 0.42 }, + { ts: 1_750_000_300, value: 0.55 }, + ], + }, + ], + }) + assert.strictEqual(series.length, 1) + assert.strictEqual(series[0]!.measurement, "CPU_USAGE") + assert.strictEqual(series[0]!.points[0]!.tsMs, 1_750_000_000_000) + }) + + it("tolerates ms timestamps, malformed points, and unknown shapes", () => { + const series = decodeRailwayMetrics({ + metrics: [ + { + measurement: "MEMORY_USAGE_GB", + values: [ + { ts: 1_750_000_000_000, value: 0.5 }, + { ts: "bad", value: 1 }, + { value: 2 }, + null, + ], + }, + { measurement: 42, values: [] }, + "garbage", + ], + }) + assert.strictEqual(series.length, 1) + assert.strictEqual(series[0]!.points.length, 1) + assert.strictEqual(series[0]!.points[0]!.tsMs, 1_750_000_000_000) + + assert.deepStrictEqual(decodeRailwayMetrics(null), []) + assert.deepStrictEqual(decodeRailwayMetrics({ metrics: "nope" }), []) + }) +}) + +describe("buildMetricsExportRequest", () => { + it("converts measurements to gauges with unit scaling", () => { + const result = buildMetricsExportRequest( + [ + { measurement: "CPU_USAGE", points: [{ tsMs: 1_750_000_000_000, value: 0.42 }] }, + { measurement: "MEMORY_USAGE_GB", points: [{ tsMs: 1_750_000_000_000, value: 0.5 }] }, + { measurement: "UNKNOWN_THING", points: [{ tsMs: 1_750_000_000_000, value: 1 }] }, + ], + context, + null, + ) + assert.isNotNull(result.request) + assert.strictEqual(result.dataPointCount, 2) + assert.strictEqual(result.maxSampleMs, 1_750_000_000_000) + + const metrics = result.request!.resourceMetrics[0]!.scopeMetrics[0]!.metrics + const cpu = metrics.find((metric) => metric.name === "railway.cpu.usage")! + assert.strictEqual(cpu.unit, "{cpu}") + assert.strictEqual(cpu.gauge.dataPoints[0]!.asDouble, 0.42) + assert.strictEqual(cpu.gauge.dataPoints[0]!.timeUnixNano, "1750000000000000000") + + // GB measurements land as bytes. + const memory = metrics.find((metric) => metric.name === "railway.memory.usage")! + assert.strictEqual(memory.unit, "By") + assert.strictEqual(memory.gauge.dataPoints[0]!.asDouble, 500_000_000) + + const attrs = Object.fromEntries( + result.request!.resourceMetrics[0]!.resource.attributes.map((attr) => [ + attr.key, + attr.value.stringValue, + ]), + ) + assert.strictEqual(attrs["service.name"], "api") + assert.strictEqual(attrs["railway.service.id"], "svc-1") + assert.strictEqual(attrs["cloud.provider"], "railway") + }) + + it("filters samples at or below the watermark and reports the new max", () => { + const result = buildMetricsExportRequest( + [ + { + measurement: "CPU_USAGE", + points: [ + { tsMs: 1_000, value: 0.1 }, + { tsMs: 2_000, value: 0.2 }, + { tsMs: 3_000, value: 0.3 }, + ], + }, + ], + context, + 2_000, + ) + assert.strictEqual(result.dataPointCount, 1) + assert.strictEqual(result.maxSampleMs, 3_000) + assert.strictEqual( + result.request!.resourceMetrics[0]!.scopeMetrics[0]!.metrics[0]!.gauge.dataPoints[0]!.asDouble, + 0.3, + ) + }) + + it("returns a null request when everything is filtered", () => { + const result = buildMetricsExportRequest( + [{ measurement: "CPU_USAGE", points: [{ tsMs: 1_000, value: 0.1 }] }], + context, + 5_000, + ) + assert.isNull(result.request) + assert.strictEqual(result.dataPointCount, 0) + }) +}) diff --git a/apps/railway-connector/src/otlp/metrics.ts b/apps/railway-connector/src/otlp/metrics.ts new file mode 100644 index 000000000..410d34042 --- /dev/null +++ b/apps/railway-connector/src/otlp/metrics.ts @@ -0,0 +1,209 @@ +/** + * Converts Railway `metrics` GraphQL query results into an OTLP/JSON + * `ExportMetricsServiceRequest` of gauges for the Maple ingest gateway + * (`POST /v1/metrics`). Gauges land in the `metrics_gauge` datasource. + */ +import { + buildResourceAttributes, + msToUnixNano, + type OtlpKeyValue, + type RailwayResourceContext, +} from "./common" + +/** The measurements polled for every service, all in one query. */ +export const RAILWAY_MEASUREMENTS = [ + "CPU_USAGE", + "CPU_LIMIT", + "MEMORY_USAGE_GB", + "MEMORY_LIMIT_GB", + "NETWORK_RX_GB", + "NETWORK_TX_GB", + "DISK_USAGE_GB", +] as const + +export type RailwayMeasurement = (typeof RAILWAY_MEASUREMENTS)[number] + +const GB = 1_000_000_000 + +/** + * Railway measurement → OTLP gauge. GB-denominated measurements convert to + * bytes so dashboards can format them with the standard byte units. + * NETWORK_RX/TX arrive as windowed totals, not lifetime counters, so they + * stay gauges rather than cumulative sums. + */ +const MEASUREMENT_SPECS: Record< + RailwayMeasurement, + { readonly name: string; readonly unit: string; readonly description: string; readonly scale: number } +> = { + CPU_USAGE: { + name: "railway.cpu.usage", + unit: "{cpu}", + description: "vCPU cores in use by the Railway service", + scale: 1, + }, + CPU_LIMIT: { + name: "railway.cpu.limit", + unit: "{cpu}", + description: "vCPU core limit of the Railway service", + scale: 1, + }, + MEMORY_USAGE_GB: { + name: "railway.memory.usage", + unit: "By", + description: "Memory in use by the Railway service", + scale: GB, + }, + MEMORY_LIMIT_GB: { + name: "railway.memory.limit", + unit: "By", + description: "Memory limit of the Railway service", + scale: GB, + }, + NETWORK_RX_GB: { + name: "railway.network.rx", + unit: "By", + description: "Network bytes received by the Railway service", + scale: GB, + }, + NETWORK_TX_GB: { + name: "railway.network.tx", + unit: "By", + description: "Network bytes transmitted by the Railway service", + scale: GB, + }, + DISK_USAGE_GB: { + name: "railway.disk.usage", + unit: "By", + description: "Disk in use by the Railway service", + scale: GB, + }, +} + +export interface RailwayMetricSeries { + readonly measurement: string + /** Samples as (epoch ms, value). */ + readonly points: ReadonlyArray<{ readonly tsMs: number; readonly value: number }> +} + +/** + * Tolerant decode of the `metrics` query result: `[{ measurement, values: + * [{ ts, value }] }]` with `ts` in epoch seconds (Railway) or milliseconds. + * Unknown measurements and malformed points are skipped, never fatal. + */ +export const decodeRailwayMetrics = (raw: unknown): RailwayMetricSeries[] => { + if (typeof raw !== "object" || raw === null) return [] + const data = raw as Record + const metrics = data.metrics + if (!Array.isArray(metrics)) return [] + + const series: RailwayMetricSeries[] = [] + for (const item of metrics) { + if (typeof item !== "object" || item === null) continue + const entry = item as Record + if (typeof entry.measurement !== "string" || !Array.isArray(entry.values)) continue + + const points: Array<{ tsMs: number; value: number }> = [] + for (const value of entry.values) { + if (typeof value !== "object" || value === null) continue + const point = value as Record + if (typeof point.ts !== "number" || typeof point.value !== "number") continue + if (!Number.isFinite(point.ts) || !Number.isFinite(point.value)) continue + // Railway timestamps are epoch seconds; tolerate ms just in case. + const tsMs = point.ts > 10_000_000_000 ? point.ts : point.ts * 1000 + points.push({ tsMs, value: point.value }) + } + series.push({ measurement: entry.measurement, points }) + } + return series +} + +interface OtlpNumberDataPoint { + readonly attributes: ReadonlyArray + readonly startTimeUnixNano: string + readonly timeUnixNano: string + readonly asDouble: number +} + +export interface OtlpMetricsExportRequest { + readonly resourceMetrics: ReadonlyArray<{ + readonly resource: { readonly attributes: ReadonlyArray } + readonly scopeMetrics: ReadonlyArray<{ + readonly scope: { readonly name: string } + readonly metrics: ReadonlyArray<{ + readonly name: string + readonly description: string + readonly unit: string + readonly gauge: { readonly dataPoints: ReadonlyArray } + }> + }> + }> +} + +const SCOPE_NAME = "railway.metrics" + +export interface MetricsBatchResult { + readonly request: OtlpMetricsExportRequest | null + readonly dataPointCount: number + /** Epoch ms of the newest sample converted — the next window's watermark. */ + readonly maxSampleMs: number | null +} + +/** + * Build one gauge export for a single Railway service. Samples at or before + * `afterMs` are dropped (already ingested in a previous window). + */ +export const buildMetricsExportRequest = ( + series: ReadonlyArray, + context: RailwayResourceContext, + afterMs: number | null, +): MetricsBatchResult => { + const metrics: Array<{ + name: string + description: string + unit: string + gauge: { dataPoints: OtlpNumberDataPoint[] } + }> = [] + let dataPointCount = 0 + let maxSampleMs: number | null = null + + for (const entry of series) { + const spec = MEASUREMENT_SPECS[entry.measurement as RailwayMeasurement] + if (spec === undefined) continue + + const dataPoints: OtlpNumberDataPoint[] = [] + for (const point of entry.points) { + if (afterMs !== null && point.tsMs <= afterMs) continue + dataPoints.push({ + attributes: [], + startTimeUnixNano: msToUnixNano(point.tsMs), + timeUnixNano: msToUnixNano(point.tsMs), + asDouble: point.value * spec.scale, + }) + if (maxSampleMs === null || point.tsMs > maxSampleMs) maxSampleMs = point.tsMs + } + if (dataPoints.length === 0) continue + + dataPointCount += dataPoints.length + metrics.push({ + name: spec.name, + description: spec.description, + unit: spec.unit, + gauge: { dataPoints }, + }) + } + + if (metrics.length === 0) return { request: null, dataPointCount: 0, maxSampleMs } + + return { + request: { + resourceMetrics: [ + { + resource: { attributes: buildResourceAttributes(context) }, + scopeMetrics: [{ scope: { name: SCOPE_NAME }, metrics }], + }, + ], + }, + dataPointCount, + maxSampleMs, + } +} diff --git a/apps/railway-connector/tsconfig.json b/apps/railway-connector/tsconfig.json new file mode 100644 index 000000000..d4f8de109 --- /dev/null +++ b/apps/railway-connector/tsconfig.json @@ -0,0 +1,24 @@ +{ + "include": ["src/**/*.ts", "src/**/*.d.ts"], + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022", "DOM"], + "types": ["bun", "node"], + "moduleResolution": "bundler", + "resolveJsonModule": true, + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "skipLibCheck": true, + "strict": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true, + "plugins": [ + { + "name": "@effect/language-service", + "reportSuggestionsAsWarningsInTsc": true + } + ] + } +} diff --git a/apps/railway-connector/vitest.config.ts b/apps/railway-connector/vitest.config.ts new file mode 100644 index 000000000..bf33f2b60 --- /dev/null +++ b/apps/railway-connector/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config" + +export default defineConfig({ + test: { + environment: "node", + include: ["src/**/*.test.ts"], + }, +}) diff --git a/apps/web/src/components/icons/index.ts b/apps/web/src/components/icons/index.ts index 9a6f2e2ea..304765278 100644 --- a/apps/web/src/components/icons/index.ts +++ b/apps/web/src/components/icons/index.ts @@ -113,6 +113,7 @@ export { PlusIcon } from "./plus" export { PostgresIcon } from "./postgres" export { PrometheusIcon } from "./prometheus" export { RabbitmqIcon } from "./rabbitmq" +export { RailwayIcon } from "./railway" export { PriorityBarsIcon, PRIORITY_LABEL } from "./priority-bars" export type { PriorityLevel } from "./priority-bars" export { RedisIcon } from "./redis" diff --git a/apps/web/src/components/icons/railway.tsx b/apps/web/src/components/icons/railway.tsx new file mode 100644 index 000000000..5abca845d --- /dev/null +++ b/apps/web/src/components/icons/railway.tsx @@ -0,0 +1,23 @@ +import type { IconProps } from "./icon" + +// Source: simple-icons (MIT) — https://simpleicons.org/icons/railway +// Railway's mark is monochrome — rendered in currentColor so the catalog can +// theme it via iconClassName (like GitHub/PlanetScale). +function RailwayIcon({ size = 24, className, ...props }: IconProps) { + return ( + + ) +} + +export { RailwayIcon } diff --git a/apps/web/src/components/integrations/integration-catalog.tsx b/apps/web/src/components/integrations/integration-catalog.tsx index f07ad16c7..188576c2f 100644 --- a/apps/web/src/components/integrations/integration-catalog.tsx +++ b/apps/web/src/components/integrations/integration-catalog.tsx @@ -9,12 +9,20 @@ import { HazelIcon, PlanetScaleIcon, PrometheusIcon, + RailwayIcon, WarpStreamIcon, } from "@/components/icons" import { Result, useAtomValue } from "@/lib/effect-atom" import { MapleApiAtomClient } from "@/lib/services/common/atom-client" -export type IntegrationId = "cloudflare" | "prometheus" | "planetscale" | "warpstream" | "hazel" | "github" +export type IntegrationId = + | "cloudflare" + | "prometheus" + | "planetscale" + | "warpstream" + | "hazel" + | "github" + | "railway" /** * Third-party brand accents for the icon-plate wash — no app token applies. @@ -75,6 +83,17 @@ const CATALOG: ReadonlyArray = [ accent: "#E52344", docsUrl: "https://maple.dev/docs/integrations/warpstream", }, + { + id: "railway", + name: "Railway", + description: + "Paste a Railway API token — Maple streams service logs and collects CPU, memory, network, and disk metrics.", + icon: RailwayIcon, + // Railway's mark is monochrome — neutral wash that works in both themes. + accent: "#8B8B8B", + iconClassName: "text-foreground", + docsUrl: "https://maple.dev/docs/integrations/railway", + }, { id: "hazel", name: "Hazel", @@ -129,6 +148,11 @@ export function useIntegrationStatuses(): Partial null) .orElse(() => STATUS_UNAVAILABLE) + const railway: CardStatus | null = Result.builder(railwayResult) + .onSuccess((status): CardStatus => { + if (!status.connected) return NOT_CONNECTED + if (status.lastValidationError) return { label: "Reconnect needed", variant: "warning" } + if (status.targetCount === 0) return { label: "Connected", variant: "success" } + return { + label: `${status.targetCount} environment${status.targetCount === 1 ? "" : "s"}`, + variant: status.erroredTargetCount > 0 ? "warning" : "success", + } + }) + .onInitial(() => null) + .orElse(() => STATUS_UNAVAILABLE) + return { cloudflare, prometheus: scrapeStatus("prometheus"), @@ -188,6 +225,7 @@ export function useIntegrationStatuses(): Partial, fallback: string): string => { + if (Exit.isSuccess(exit)) return fallback + const first = Cause.prettyErrors(exit.cause)[0] + return first?.message && first.message.length > 0 ? first.message : fallback +} + +export function RailwayIntegrationCard() { + const statusResult = useAtomValue( + MapleApiAtomClient.query("railway", "status", { + reactivityKeys: ["railwayIntegrationStatus"], + }), + ) + + const status = Result.builder(statusResult) + .onSuccess((value) => value) + .orElse(() => null) + + if (status === null) { + return ( +
+ + Loading Railway integration… +
+ ) + } + + if (!status.connected) return + + return +} + +function RailwayConnectForm({ reconnect = false }: { reconnect?: boolean }) { + const [token, setToken] = useState("") + const [busy, setBusy] = useState(false) + const [error, setError] = useState(null) + const connect = useAtomSet(MapleApiAtomClient.mutation("railway", "connect"), { + mode: "promiseExit", + }) + + async function handleConnect() { + if (token.trim().length === 0) return + setBusy(true) + setError(null) + const result = await connect({ + payload: new RailwayConnectRequest({ token: token.trim() }), + reactivityKeys: REACTIVITY, + }) + setBusy(false) + if (Exit.isSuccess(result)) { + setToken("") + toast.success(reconnect ? "Railway token updated" : "Railway connected") + } else { + setError(failureMessage(result, "Failed to connect Railway")) + } + } + + const form = ( +
+
+ setToken(event.target.value)} + placeholder="Railway API token" + autoComplete="off" + onKeyDown={(event) => { + if (event.key === "Enter") void handleConnect() + }} + /> + +
+ {error ?

{error}

: null} +
+ ) + + if (reconnect) return form + + return ( + + {form} + + ) +} + +function RailwayConnectedView({ + accountName, + tokenType, + lastValidationError, +}: { + accountName: string | null + tokenType: "account" | "workspace" | null + lastValidationError: string | null +}) { + const [busy, setBusy] = useState(false) + const [showReconnect, setShowReconnect] = useState(false) + const disconnect = useAtomSet(MapleApiAtomClient.mutation("railway", "disconnect"), { + mode: "promiseExit", + }) + + async function handleDisconnect() { + setBusy(true) + const result = await disconnect({ reactivityKeys: REACTIVITY }) + setBusy(false) + if (Exit.isSuccess(result)) toast.success("Railway disconnected") + else toast.error("Failed to disconnect Railway") + } + + return ( +
+ {lastValidationError ? ( + + + + {lastValidationError} — paste a fresh token below to resume ingestion. + + + ) : null} + +
+ +
+
+
+

Railway

+ + {lastValidationError ? "Reconnect needed" : "Connected"} + +
+

+ {accountName ? ( + <> + Connected as {accountName} + {tokenType ? ` (${tokenType} token)` : null}. + + ) : ( + <>Connected{tokenType ? ` with a ${tokenType} token` : null}. + )}{" "} + Pick the environments to ingest below. +

+
+
+ + +
+ {showReconnect ? : null} +
+
+ + +
+ ) +} + +function RailwayEnvironmentBoard() { + const discoveryResult = useAtomValue( + MapleApiAtomClient.query("railway", "discovery", { + reactivityKeys: ["railwayTargets", "railwayIntegrationStatus"], + }), + ) + const targetsResult = useAtomValue( + MapleApiAtomClient.query("railway", "listTargets", { + reactivityKeys: ["railwayTargets"], + }), + ) + const refreshTargets = useAtomRefresh( + MapleApiAtomClient.query("railway", "listTargets", { reactivityKeys: ["railwayTargets"] }), + ) + + const targets = Result.builder(targetsResult) + .onSuccess((response) => response.targets) + .orElse(() => [] as ReadonlyArray) + const targetById = new Map(targets.map((target) => [target.id, target])) + + return Result.builder(discoveryResult) + .onSuccess((discovery) => { + if (discovery.projects.length === 0) { + return ( +
+ No Railway projects are visible with this token. +
+ ) + } + return ( +
+ {discovery.projects.map((project) => ( + + ))} +
+ ) + }) + .onInitial(() => ( +
+ + Discovering Railway projects… +
+ )) + .orElse(() => ( + + + + Failed to load Railway projects — the token may have been revoked. Try replacing it above. + + + )) +} + +function RailwayProjectSection({ + project, + targetById, + onChanged, +}: { + project: RailwayDiscoveredProject + targetById: Map + onChanged: () => void +}) { + return ( +
+
+ {project.name} + + {project.environments.length} environment{project.environments.length === 1 ? "" : "s"} + +
+
+ {project.environments.map((environment) => ( + + ))} +
+
+ ) +} + +function RailwayEnvironmentRow({ + projectId, + environment, + target, + onChanged, +}: { + projectId: string + environment: RailwayDiscoveredEnvironment + target: RailwayTargetResponse | undefined + onChanged: () => void +}) { + const [busy, setBusy] = useState(false) + const createTarget = useAtomSet(MapleApiAtomClient.mutation("railway", "createTarget"), { + mode: "promiseExit", + }) + const updateTarget = useAtomSet(MapleApiAtomClient.mutation("railway", "updateTarget"), { + mode: "promiseExit", + }) + const deleteTarget = useAtomSet(MapleApiAtomClient.mutation("railway", "deleteTarget"), { + mode: "promiseExit", + }) + + const ingesting = target !== undefined && target.enabled + + async function handleToggleIngest(next: boolean) { + setBusy(true) + if (target === undefined) { + const result = await createTarget({ + payload: new CreateRailwayTargetRequest({ + projectId, + environmentId: environment.id, + }), + reactivityKeys: REACTIVITY, + }) + if (Exit.isSuccess(result)) toast.success(`Ingesting ${environment.name}`) + else toast.error(failureMessage(result, "Failed to enable ingestion")) + } else { + const result = await updateTarget({ + params: { targetId: target.id }, + payload: new UpdateRailwayTargetRequest({ enabled: next }), + reactivityKeys: REACTIVITY, + }) + if (!Exit.isSuccess(result)) toast.error(failureMessage(result, "Failed to update target")) + } + setBusy(false) + onChanged() + } + + async function handleToggleSignal(field: "logsEnabled" | "metricsEnabled", next: boolean) { + if (target === undefined) return + setBusy(true) + const result = await updateTarget({ + params: { targetId: target.id }, + payload: new UpdateRailwayTargetRequest( + field === "logsEnabled" ? { logsEnabled: next } : { metricsEnabled: next }, + ), + reactivityKeys: REACTIVITY, + }) + if (!Exit.isSuccess(result)) toast.error(failureMessage(result, "Failed to update target")) + setBusy(false) + onChanged() + } + + async function handleRemove() { + if (target === undefined) return + setBusy(true) + const result = await deleteTarget({ + params: { targetId: target.id }, + reactivityKeys: REACTIVITY, + }) + if (Exit.isSuccess(result)) toast.success(`Stopped ingesting ${environment.name}`) + else toast.error(failureMessage(result, "Failed to remove target")) + setBusy(false) + onChanged() + } + + return ( +
+
+
+
+ {environment.name} + + {environment.services.length} service{environment.services.length === 1 ? "" : "s"} + + {target?.lastError ? ( + + Error + + ) : null} +
+ {target ? ( +
+ + Logs:{" "} + {target.lastLogAt ? formatRelativeTime(target.lastLogAt) : "no data yet"} + + + Metrics:{" "} + {target.lastMetricsAt ? formatRelativeTime(target.lastMetricsAt) : "no data yet"} + +
+ ) : null} + {target?.lastError ? ( +

+ {target.lastError} +

+ ) : null} +
+
+ {ingesting ? ( + <> + + + + + ) : ( + + )} +
+
+
+ ) +} diff --git a/apps/web/src/routes/integrations.tsx b/apps/web/src/routes/integrations.tsx index 7c841b75e..de9119bea 100644 --- a/apps/web/src/routes/integrations.tsx +++ b/apps/web/src/routes/integrations.tsx @@ -6,6 +6,7 @@ import { DashboardLayout } from "@/components/layout/dashboard-layout" import { CloudflareAccountCard } from "@/components/integrations/cloudflare-account-card" import { GithubIntegrationCard } from "@/components/integrations/github-integration-card" import { HazelIntegrationCard } from "@/components/integrations/hazel-integration-card" +import { RailwayIntegrationCard } from "@/components/integrations/railway-integration-card" import { IntegrationCatalog, IntegrationIconPlate, @@ -22,7 +23,15 @@ import { ArrowLeftIcon, CircleInfoIcon, ExternalLinkIcon } from "@/components/ic const IntegrationsSearch = Schema.Struct({ integration: Schema.optional( - Schema.Literals(["cloudflare", "prometheus", "planetscale", "warpstream", "hazel", "github"]), + Schema.Literals([ + "cloudflare", + "prometheus", + "planetscale", + "warpstream", + "hazel", + "github", + "railway", + ]), ), }) @@ -96,6 +105,8 @@ function IntegrationsPage() { ) : integration === "github" ? ( + ) : integration === "railway" ? ( + ) : integration === "planetscale" ? ( ) : ( diff --git a/bun.lock b/bun.lock index a1a5cd8d7..af1ed4102 100644 --- a/bun.lock +++ b/bun.lock @@ -247,6 +247,23 @@ "typescript": "^6.0.3", }, }, + "apps/railway-connector": { + "name": "@maple/railway-connector", + "dependencies": { + "@effect/platform-bun": "catalog:effect", + "@maple-dev/effect-sdk": "workspace:*", + "@maple/domain": "workspace:*", + "effect": "catalog:effect", + }, + "devDependencies": { + "@effect/language-service": "catalog:effect", + "@effect/vitest": "4.0.0-beta.93", + "@types/bun": "^1.3.11", + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:", + }, + }, "apps/scraper": { "name": "@maple/scraper", "dependencies": { @@ -395,7 +412,7 @@ }, "lib/effect-sdk": { "name": "@maple-dev/effect-sdk", - "version": "0.5.0", + "version": "0.7.0", "dependencies": { "std-env": "^4.0.0", }, @@ -429,7 +446,7 @@ }, "packages/browser": { "name": "@maple-dev/browser", - "version": "0.1.0", + "version": "0.3.0", "dependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-trace-otlp-http": "^0.205.0", @@ -1512,6 +1529,8 @@ "@maple/query-engine": ["@maple/query-engine@workspace:packages/query-engine"], + "@maple/railway-connector": ["@maple/railway-connector@workspace:apps/railway-connector"], + "@maple/scraper": ["@maple/scraper@workspace:apps/scraper"], "@maple/ui": ["@maple/ui@workspace:packages/ui"], diff --git a/docs/railway-integration.md b/docs/railway-integration.md new file mode 100644 index 000000000..c0243870b --- /dev/null +++ b/docs/railway-integration.md @@ -0,0 +1,63 @@ +# Railway integration (internal architecture) + +Customer-facing docs: `apps/landing/src/content/docs/integrations/railway.md`. This page is the +operator/contributor map of how the integration works. + +## Why pull-based + +Railway has no log drain and no webhook for logs; the only programmatic access is the public +GraphQL API (`https://backboard.railway.com/graphql/v2`, WebSocket subscriptions on the same host). +So Maple pulls: the org pastes a workspace/account token, and a Maple-run connector streams logs +and polls metrics with it. + +## Pieces + +| Piece | Where | Role | +| --- | --- | --- | +| `railway_connections` / `railway_targets` | `packages/db/src/schema/railway.ts` | One AES-256-GCM-encrypted token per org; one row per ingested (project, environment) with health + watermark columns. | +| `RailwayIntegrationService` | `apps/api/src/services/RailwayIntegrationService.ts` | Connect (validate via `me`, falling back to the projects query for workspace tokens), discovery, target CRUD, the internal work list, result/watermark recording. | +| Public routes | `apps/api/src/routes/railway.http.ts` | `/api/railway/*` (MapleApi `railway` group). Mutations are admin-gated. | +| Internal routes | `apps/api/src/routes/railway-internal.http.ts` | `GET /api/internal/railway-targets` + `POST /api/internal/railway-results`, authenticated with `SD_INTERNAL_TOKEN` (same trust as scraper-internal). Decrypted tokens and org `maple_pk_*` ingest keys ride ONLY this wire. | +| Connector | `apps/railway-connector/` | Standalone Bun/Effect process (clone of `apps/scraper`'s scheduler shape). One `environmentLogs` graphql-transport-ws subscription per target + one metrics poll loop per (target, service). Pushes OTLP/JSON to the ingest gateway with the org's public key → billed + warehouse-routed like customer OTLP. **No changes to `apps/ingest` or `apps/alerting`.** | +| UI | `apps/web/src/components/integrations/railway-integration-card.tsx` | Paste-token connect, environment picker, per-target health. | + +## Data mapping + +- Logs: `service.name` = Railway service name (resolved from the environment's cached service + list; unknown ids degrade to `railway:`, environment-level lines to + `railway-environment:`), severity string → OTLP severityNumber, `railway.deployment.id` + + Railway's structured attributes as log attributes. +- Metrics: gauges `railway.cpu.usage/limit` ({cpu}), `railway.memory.usage/limit`, + `railway.disk.usage`, `railway.network.rx/tx` (bytes; GB values scaled ×1e9). Land in the + existing `metrics_gauge` datasource — no Tinybird changes. +- Both carry `cloud.provider=railway`, `railway.project/environment/service` ids+names, + `service.namespace` = project name. The gateway stamps `maple_org_id` itself. + +## Operational notes + +- **Single replica.** The connector has no lease coordination — two replicas would double-ingest + every log line. Keep the Railway service at 1 replica; add a lease column (see + `cloudflare_analytics_state.leaseUntil`) if HA is ever needed. +- **Rate limits are the customer's** (100/1,000/10,000 requests/hour by plan, not queryable). + Poll delays honor `X-RateLimit-Remaining` (4× stretch under 25 remaining) and `Retry-After` + (exponential backoff, 15 min cap). See `nextPollDelayMs` in + `apps/railway-connector/src/RailwayGraphql.ts`. +- **Watermarks**: the api advances `log_watermark_at`/`metrics_watermark_at` monotonically from + result reports. Log reconnects dedupe against the watermark; gaps during a disconnect are lossy + (documented) — metrics backfill up to 1 h. +- **Token revocation**: any Railway 401 (poll or subscription) reports `unauthorized: true`; the + api stamps `last_validation_error` on the connection and the card shows "Reconnect needed". +- **Billing**: pushes go through the ingest gateway with the org's public key, so Autumn metering + and 402 limits apply. The connector drops log batches on 402 instead of retrying forever. +- Deploy: new Railway service in Maple's own project — config file + `apps/railway-connector/railway.json`, root directory at the repo root, env `MAPLE_API_URL`, + `SD_INTERNAL_TOKEN`, `MAPLE_INGEST_URL` (+ optional `RAILWAY_CONNECTOR_*`, `PORT`, default 3476). + +## v1 boundaries (deliberate) + +- Workspace/account tokens only (project tokens use a different header and can't enumerate). +- Runtime logs only — no build/deploy logs, no HTTP edge logs. +- One Railway connection per org (unique index; relax later if needed). +- The `environmentLogs` subscription shape was written against Railway's public schema as used by + community egress tools; if Railway's schema drifts, the tolerant decoders drop what they can't + parse and target health surfaces the errors. diff --git a/packages/db/drizzle/0012_aromatic_wallow.sql b/packages/db/drizzle/0012_aromatic_wallow.sql new file mode 100644 index 000000000..1054d2824 --- /dev/null +++ b/packages/db/drizzle/0012_aromatic_wallow.sql @@ -0,0 +1,46 @@ +CREATE TABLE "railway_connections" ( + "id" text PRIMARY KEY NOT NULL, + "org_id" text NOT NULL, + "token_type" text NOT NULL, + "token_ciphertext" text NOT NULL, + "token_iv" text NOT NULL, + "token_tag" text NOT NULL, + "external_account_name" text, + "connected_by_user_id" text NOT NULL, + "enabled" boolean DEFAULT true NOT NULL, + "last_validated_at" timestamp with time zone, + "last_validation_error" text, + "last_validation_error_at" timestamp with time zone, + "created_at" timestamp with time zone NOT NULL, + "updated_at" timestamp with time zone NOT NULL +); +--> statement-breakpoint +CREATE TABLE "railway_targets" ( + "id" text PRIMARY KEY NOT NULL, + "org_id" text NOT NULL, + "connection_id" text NOT NULL, + "project_id" text NOT NULL, + "project_name" text, + "environment_id" text NOT NULL, + "environment_name" text, + "enabled" boolean DEFAULT true NOT NULL, + "logs_enabled" boolean DEFAULT true NOT NULL, + "metrics_enabled" boolean DEFAULT true NOT NULL, + "metrics_interval_seconds" integer DEFAULT 300 NOT NULL, + "services_json" text, + "services_discovered_at" timestamp with time zone, + "log_watermark_at" timestamp with time zone, + "metrics_watermark_at" timestamp with time zone, + "last_log_at" timestamp with time zone, + "last_metrics_at" timestamp with time zone, + "last_success_at" timestamp with time zone, + "last_error" text, + "last_error_at" timestamp with time zone, + "created_at" timestamp with time zone NOT NULL, + "updated_at" timestamp with time zone NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX "railway_connections_org_idx" ON "railway_connections" USING btree ("org_id");--> statement-breakpoint +CREATE UNIQUE INDEX "railway_targets_org_project_env_idx" ON "railway_targets" USING btree ("org_id","project_id","environment_id");--> statement-breakpoint +CREATE INDEX "railway_targets_org_idx" ON "railway_targets" USING btree ("org_id");--> statement-breakpoint +CREATE INDEX "railway_targets_connection_idx" ON "railway_targets" USING btree ("connection_id"); \ No newline at end of file diff --git a/packages/db/drizzle/meta/0012_snapshot.json b/packages/db/drizzle/meta/0012_snapshot.json new file mode 100644 index 000000000..998db378f --- /dev/null +++ b/packages/db/drizzle/meta/0012_snapshot.json @@ -0,0 +1,6121 @@ +{ + "id": "f81b9f7a-7c9a-486e-b2f6-2fd56148cf94", + "prevId": "8c4b199f-f711-4b87-961a-1672eb536e58", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.ai_triage_runs": { + "name": "ai_triage_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_kind": { + "name": "incident_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "context_json": { + "name": "context_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "result_json": { + "name": "result_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "ai_triage_runs_incident_idx": { + "name": "ai_triage_runs_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "ai_triage_runs_org_issue_idx": { + "name": "ai_triage_runs_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "ai_triage_runs_org_created_idx": { + "name": "ai_triage_runs_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ai_triage_settings": { + "name": "ai_triage_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "max_runs_per_day": { + "name": "max_runs_per_day", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 20 + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_delivery_events": { + "name": "alert_delivery_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "destination_id": { + "name": "destination_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delivery_key": { + "name": "delivery_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt_number": { + "name": "attempt_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scheduled_at": { + "name": "scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claim_expires_at": { + "name": "claim_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempted_at": { + "name": "attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "provider_message": { + "name": "provider_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_reference": { + "name": "provider_reference", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_delivery_events_org_idx": { + "name": "alert_delivery_events_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_org_incident_idx": { + "name": "alert_delivery_events_org_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_due_idx": { + "name": "alert_delivery_events_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scheduled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_claim_idx": { + "name": "alert_delivery_events_claim_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "claim_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scheduled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_delivery_attempt_idx": { + "name": "alert_delivery_events_delivery_attempt_idx", + "columns": [ + { + "expression": "delivery_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "attempt_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_destinations": { + "name": "alert_destinations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "config_json": { + "name": "config_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "secret_ciphertext": { + "name": "secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_iv": { + "name": "secret_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_tag": { + "name": "secret_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_tested_at": { + "name": "last_tested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_test_error": { + "name": "last_test_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_destinations_org_idx": { + "name": "alert_destinations_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_destinations_org_enabled_idx": { + "name": "alert_destinations_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_destinations_org_name_idx": { + "name": "alert_destinations_org_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_incidents": { + "name": "alert_incidents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_key": { + "name": "incident_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rule_name": { + "name": "rule_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_key": { + "name": "group_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "comparator": { + "name": "comparator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "threshold": { + "name": "threshold", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "threshold_upper": { + "name": "threshold_upper", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_observed_value": { + "name": "last_observed_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_delivered_event_type": { + "name": "last_delivered_event_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_notified_at": { + "name": "last_notified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error_issue_id": { + "name": "error_issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_incidents_org_idx": { + "name": "alert_incidents_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_org_status_idx": { + "name": "alert_incidents_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_org_rule_idx": { + "name": "alert_incidents_org_rule_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "rule_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_org_issue_idx": { + "name": "alert_incidents_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "error_issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_incident_key_idx": { + "name": "alert_incidents_incident_key_idx", + "columns": [ + { + "expression": "incident_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_rule_states": { + "name": "alert_rule_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_key": { + "name": "group_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'__total__'" + }, + "consecutive_breaches": { + "name": "consecutive_breaches", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "consecutive_healthy": { + "name": "consecutive_healthy", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_value": { + "name": "last_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_rule_states_org_idx": { + "name": "alert_rule_states_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "alert_rule_states_org_id_rule_id_group_key_pk": { + "name": "alert_rule_states_org_id_rule_id_group_key_pk", + "columns": [ + "org_id", + "rule_id", + "group_key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_rules": { + "name": "alert_rules", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notification_template_json": { + "name": "notification_template_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_names_json": { + "name": "service_names_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "exclude_service_names_json": { + "name": "exclude_service_names_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "tags_json": { + "name": "tags_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "comparator": { + "name": "comparator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "threshold": { + "name": "threshold", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "threshold_upper": { + "name": "threshold_upper", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "window_minutes": { + "name": "window_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "minimum_sample_count": { + "name": "minimum_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "consecutive_breaches_required": { + "name": "consecutive_breaches_required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "consecutive_healthy_required": { + "name": "consecutive_healthy_required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "renotify_interval_minutes": { + "name": "renotify_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 30 + }, + "metric_name": { + "name": "metric_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metric_type": { + "name": "metric_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metric_aggregation": { + "name": "metric_aggregation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "apdex_threshold_ms": { + "name": "apdex_threshold_ms", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "query_builder_draft_json": { + "name": "query_builder_draft_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "raw_query_sql": { + "name": "raw_query_sql", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "group_by": { + "name": "group_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "destination_ids_json": { + "name": "destination_ids_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "query_spec_json": { + "name": "query_spec_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "reducer": { + "name": "reducer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sample_count_strategy": { + "name": "sample_count_strategy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "no_data_behavior": { + "name": "no_data_behavior", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_scheduled_at": { + "name": "last_scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_rules_org_idx": { + "name": "alert_rules_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_rules_org_enabled_idx": { + "name": "alert_rules_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_rules_org_name_idx": { + "name": "alert_rules_org_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.anomaly_detector_settings": { + "name": "anomaly_detector_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sensitivity": { + "name": "sensitivity", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'normal'" + }, + "muted_signals_json": { + "name": "muted_signals_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "last_tick_at": { + "name": "last_tick_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.anomaly_detector_states": { + "name": "anomaly_detector_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detector_key": { + "name": "detector_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_env": { + "name": "deployment_env", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consecutive_breaches": { + "name": "consecutive_breaches", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "consecutive_healthy": { + "name": "consecutive_healthy", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_value": { + "name": "last_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "baseline_median": { + "name": "baseline_median", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "open_incident_id": { + "name": "open_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_resolved_at": { + "name": "last_resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_incident_id": { + "name": "last_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "anomaly_detector_states_org_idx": { + "name": "anomaly_detector_states_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_detector_states_open_incident_idx": { + "name": "anomaly_detector_states_open_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "open_incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_detector_states_evaluated_idx": { + "name": "anomaly_detector_states_evaluated_idx", + "columns": [ + { + "expression": "last_evaluated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "anomaly_detector_states_org_id_detector_key_pk": { + "name": "anomaly_detector_states_org_id_detector_key_pk", + "columns": [ + "org_id", + "detector_key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.anomaly_incidents": { + "name": "anomaly_incidents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detector_key": { + "name": "detector_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_env": { + "name": "deployment_env", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_issue_id": { + "name": "error_issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "opened_value": { + "name": "opened_value", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "baseline_median": { + "name": "baseline_median", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "baseline_sigma": { + "name": "baseline_sigma", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "threshold_value": { + "name": "threshold_value", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "last_observed_value": { + "name": "last_observed_value", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "resolve_reason": { + "name": "resolve_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "triage_status": { + "name": "triage_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprints_json": { + "name": "fingerprints_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "reopen_count": { + "name": "reopen_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_reopened_at": { + "name": "last_reopened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "anomaly_incidents_org_status_idx": { + "name": "anomaly_incidents_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_org_triggered_idx": { + "name": "anomaly_incidents_org_triggered_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_triggered_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_org_detector_idx": { + "name": "anomaly_incidents_org_detector_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "detector_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_org_issue_idx": { + "name": "anomaly_incidents_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "error_issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_prefix": { + "name": "key_prefix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'standard'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by_email": { + "name": "created_by_email", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + { + "expression": "key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_keys_org_id_idx": { + "name": "api_keys_org_id_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloudflare_analytics_state": { + "name": "cloudflare_analytics_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dataset": { + "name": "dataset", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "zone_id": { + "name": "zone_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "watermark_at": { + "name": "watermark_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "backfill_at": { + "name": "backfill_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "settings_json": { + "name": "settings_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "settings_fetched_at": { + "name": "settings_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "quantiles_available": { + "name": "quantiles_available", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "discovered_at": { + "name": "discovered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "live_scripts_json": { + "name": "live_scripts_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error_at": { + "name": "last_error_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cf_analytics_state_org_dataset_zone_idx": { + "name": "cf_analytics_state_org_dataset_zone_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dataset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "zone_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cf_analytics_state_org_idx": { + "name": "cf_analytics_state_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloudflare_logpush_connectors": { + "name": "cloudflare_logpush_connectors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dataset": { + "name": "dataset", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'http_requests'" + }, + "secret_ciphertext": { + "name": "secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_iv": { + "name": "secret_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_tag": { + "name": "secret_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_hash": { + "name": "secret_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_received_at": { + "name": "last_received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_rotated_at": { + "name": "secret_rotated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cloudflare_logpush_connectors_org_idx": { + "name": "cloudflare_logpush_connectors_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cloudflare_logpush_connectors_org_enabled_idx": { + "name": "cloudflare_logpush_connectors_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cloudflare_logpush_connectors_secret_hash_unique": { + "name": "cloudflare_logpush_connectors_secret_hash_unique", + "columns": [ + { + "expression": "secret_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dashboard_versions": { + "name": "dashboard_versions", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dashboard_id": { + "name": "dashboard_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version_number": { + "name": "version_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "change_kind": { + "name": "change_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "change_summary": { + "name": "change_summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_version_id": { + "name": "source_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "dashboard_versions_org_dashboard_idx": { + "name": "dashboard_versions_org_dashboard_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_versions_org_dashboard_version_unq": { + "name": "dashboard_versions_org_dashboard_version_unq", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "dashboard_versions_org_id_id_pk": { + "name": "dashboard_versions_org_id_id_pk", + "columns": [ + "org_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dashboards": { + "name": "dashboards", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "dashboards_org_updated_idx": { + "name": "dashboards_org_updated_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboards_org_name_idx": { + "name": "dashboards_org_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "dashboards_org_id_id_pk": { + "name": "dashboards_org_id_id_pk", + "columns": [ + "org_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.digest_subscriptions": { + "name": "digest_subscriptions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "day_of_week": { + "name": "day_of_week", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "last_sent_at": { + "name": "last_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_attempted_at": { + "name": "last_attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "digest_subscriptions_org_user_idx": { + "name": "digest_subscriptions_org_user_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "digest_subscriptions_org_enabled_idx": { + "name": "digest_subscriptions_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.actors": { + "name": "actors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agent_name": { + "name": "agent_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "capabilities_json": { + "name": "capabilities_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_active_at": { + "name": "last_active_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "actors_org_user_idx": { + "name": "actors_org_user_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "actors_org_agent_name_idx": { + "name": "actors_org_agent_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "agent_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "actors_org_type_idx": { + "name": "actors_org_type_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_incidents": { + "name": "error_incidents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_incidents_org_issue_idx": { + "name": "error_incidents_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_incidents_org_status_idx": { + "name": "error_incidents_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issue_events": { + "name": "error_issue_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_state": { + "name": "from_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "to_state": { + "name": "to_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issue_events_issue_idx": { + "name": "error_issue_events_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_events_actor_idx": { + "name": "error_issue_events_actor_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_events_type_idx": { + "name": "error_issue_events_type_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issue_states": { + "name": "error_issue_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_observed_occurrence_at": { + "name": "last_observed_occurrence_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "open_incident_id": { + "name": "open_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issue_states_org_idx": { + "name": "error_issue_states_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "error_issue_states_org_id_issue_id_pk": { + "name": "error_issue_states_org_id_issue_id_pk", + "columns": [ + "org_id", + "issue_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issues": { + "name": "error_issues", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'error'" + }, + "source_ref_json": { + "name": "source_ref_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_type": { + "name": "exception_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_message": { + "name": "exception_message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error_label": { + "name": "error_label", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "top_frame": { + "name": "top_frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_state": { + "name": "workflow_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'triage'" + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "severity_source": { + "name": "severity_source", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "assigned_actor_id": { + "name": "assigned_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_holder_actor_id": { + "name": "lease_holder_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "resolved_by_actor_id": { + "name": "resolved_by_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snooze_until": { + "name": "snooze_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issues_org_fp_idx": { + "name": "error_issues_org_fp_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_workflow_idx": { + "name": "error_issues_org_workflow_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_severity_idx": { + "name": "error_issues_org_severity_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "severity", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_last_seen_idx": { + "name": "error_issues_org_last_seen_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_seen_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_assignee_idx": { + "name": "error_issues_org_assignee_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "assigned_actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_lease_expiry_idx": { + "name": "error_issues_lease_expiry_idx", + "columns": [ + { + "expression": "lease_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_notification_policies": { + "name": "error_notification_policies", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "destination_ids_json": { + "name": "destination_ids_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "notify_on_first_seen": { + "name": "notify_on_first_seen", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "notify_on_regression": { + "name": "notify_on_regression", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "notify_on_resolve": { + "name": "notify_on_resolve", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "notify_on_transition_in_review": { + "name": "notify_on_transition_in_review", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "notify_on_transition_done": { + "name": "notify_on_transition_done", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "notify_on_claim": { + "name": "notify_on_claim", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "min_occurrence_count": { + "name": "min_occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'warning'" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.issue_escalation_policies": { + "name": "issue_escalation_policies", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "rules_json": { + "name": "rules_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.issue_escalations": { + "name": "issue_escalations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "issue_escalations_dedupe_idx": { + "name": "issue_escalations_dedupe_idx", + "columns": [ + { + "expression": "dedupe_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "issue_escalations_due_idx": { + "name": "issue_escalations_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "issue_escalations_org_issue_idx": { + "name": "issue_escalations_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.investigations": { + "name": "investigations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'investigating'" + }, + "seeded_by": { + "name": "seeded_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user'" + }, + "subject_json": { + "name": "subject_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "incident_kind": { + "name": "incident_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "report_json": { + "name": "report_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "diagnosed_at": { + "name": "diagnosed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "investigations_incident_idx": { + "name": "investigations_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"investigations\".\"incident_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_org_created_idx": { + "name": "investigations_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_org_issue_idx": { + "name": "investigations_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_org_status_idx": { + "name": "investigations_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_auth_states": { + "name": "oauth_auth_states", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "initiated_by_user_id": { + "name": "initiated_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uri": { + "name": "redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "return_to": { + "name": "return_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_verifier": { + "name": "code_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_auth_states_expires_idx": { + "name": "oauth_auth_states_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_connections": { + "name": "oauth_connections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_user_id": { + "name": "external_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_user_email": { + "name": "external_user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_account_name": { + "name": "external_account_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "connected_by_user_id": { + "name": "connected_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "access_token_ciphertext": { + "name": "access_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_iv": { + "name": "access_token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_tag": { + "name": "access_token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refresh_token_ciphertext": { + "name": "refresh_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token_iv": { + "name": "refresh_token_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token_tag": { + "name": "refresh_token_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_connections_org_provider_idx": { + "name": "oauth_connections_org_provider_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_connections_org_idx": { + "name": "oauth_connections_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_onboarding_state": { + "name": "org_onboarding_state", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_data_requested": { + "name": "demo_data_requested", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "checklist_dismissed_at": { + "name": "checklist_dismissed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "first_data_received_at": { + "name": "first_data_received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "welcome_email_sent_at": { + "name": "welcome_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "connect_nudge_email_sent_at": { + "name": "connect_nudge_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "stalled_email_sent_at": { + "name": "stalled_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "activation_email_sent_at": { + "name": "activation_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_ingest_attribute_mappings": { + "name": "org_ingest_attribute_mappings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_context": { + "name": "source_context", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_key": { + "name": "target_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "org_ingest_attribute_mappings_org_idx": { + "name": "org_ingest_attribute_mappings_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_recommendation_issues": { + "name": "org_recommendation_issues", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "number": { + "name": "number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "recommendation_key": { + "name": "recommendation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "canonical_key": { + "name": "canonical_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "usage_count": { + "name": "usage_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "opened_at": { + "name": "opened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "org_recommendation_issues_org_idx": { + "name": "org_recommendation_issues_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_recommendation_issues_org_key_idx": { + "name": "org_recommendation_issues_org_key_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "recommendation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_ingest_keys": { + "name": "org_ingest_keys", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_key_hash": { + "name": "public_key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_ciphertext": { + "name": "private_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_iv": { + "name": "private_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_tag": { + "name": "private_key_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_hash": { + "name": "private_key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_rotated_at": { + "name": "public_rotated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "private_rotated_at": { + "name": "private_rotated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "org_ingest_keys_public_key_unique": { + "name": "org_ingest_keys_public_key_unique", + "columns": [ + { + "expression": "public_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_ingest_keys_public_key_hash_unique": { + "name": "org_ingest_keys_public_key_hash_unique", + "columns": [ + { + "expression": "public_key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_ingest_keys_private_key_hash_unique": { + "name": "org_ingest_keys_private_key_hash_unique", + "columns": [ + { + "expression": "private_key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "org_ingest_keys_org_id_pk": { + "name": "org_ingest_keys_org_id_pk", + "columns": [ + "org_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_ingest_sampling_policies": { + "name": "org_ingest_sampling_policies", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "trace_sample_ratio": { + "name": "trace_sample_ratio", + "type": "double precision", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "always_keep_error_spans": { + "name": "always_keep_error_spans", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "always_keep_slow_spans_ms": { + "name": "always_keep_slow_spans_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_clickhouse_settings": { + "name": "org_clickhouse_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ch_url": { + "name": "ch_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ch_user": { + "name": "ch_user", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ch_password_ciphertext": { + "name": "ch_password_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ch_password_iv": { + "name": "ch_password_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ch_password_tag": { + "name": "ch_password_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ch_database": { + "name": "ch_database", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schema_version": { + "name": "schema_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "org_clickhouse_settings_org_id_pk": { + "name": "org_clickhouse_settings_org_id_pk", + "columns": [ + "org_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_clickhouse_schema_apply_runs": { + "name": "org_clickhouse_schema_apply_runs", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_instance_id": { + "name": "workflow_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "current_migration": { + "name": "current_migration", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "steps_total": { + "name": "steps_total", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "steps_done": { + "name": "steps_done", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "applied_versions": { + "name": "applied_versions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "skipped": { + "name": "skipped", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "org_clickhouse_schema_apply_runs_org_id_pk": { + "name": "org_clickhouse_schema_apply_runs_org_id_pk", + "columns": [ + "org_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.railway_connections": { + "name": "railway_connections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_type": { + "name": "token_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_account_name": { + "name": "external_account_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "connected_by_user_id": { + "name": "connected_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_validated_at": { + "name": "last_validated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_validation_error": { + "name": "last_validation_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_validation_error_at": { + "name": "last_validation_error_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "railway_connections_org_idx": { + "name": "railway_connections_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.railway_targets": { + "name": "railway_targets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_name": { + "name": "project_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "environment_name": { + "name": "environment_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "logs_enabled": { + "name": "logs_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "metrics_enabled": { + "name": "metrics_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "metrics_interval_seconds": { + "name": "metrics_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 300 + }, + "services_json": { + "name": "services_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "services_discovered_at": { + "name": "services_discovered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "log_watermark_at": { + "name": "log_watermark_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "metrics_watermark_at": { + "name": "metrics_watermark_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_log_at": { + "name": "last_log_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_metrics_at": { + "name": "last_metrics_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error_at": { + "name": "last_error_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "railway_targets_org_project_env_idx": { + "name": "railway_targets_org_project_env_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "railway_targets_org_idx": { + "name": "railway_targets_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "railway_targets_connection_idx": { + "name": "railway_targets_connection_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scrape_target_checks": { + "name": "scrape_target_checks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "byDefault", + "name": "scrape_target_checks_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sub_target_key": { + "name": "sub_target_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "samples_scraped": { + "name": "samples_scraped", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "samples_post_relabel": { + "name": "samples_post_relabel", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "scrape_target_checks_target_checked_idx": { + "name": "scrape_target_checks_target_checked_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scrape_target_checks_target_id_scrape_targets_id_fk": { + "name": "scrape_target_checks_target_id_scrape_targets_id_fk", + "tableFrom": "scrape_target_checks", + "tableTo": "scrape_targets", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scrape_targets": { + "name": "scrape_targets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'prometheus'" + }, + "discovery_config_json": { + "name": "discovery_config_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scrape_interval_seconds": { + "name": "scrape_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 15 + }, + "labels_json": { + "name": "labels_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "auth_credentials_ciphertext": { + "name": "auth_credentials_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_credentials_iv": { + "name": "auth_credentials_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_credentials_tag": { + "name": "auth_credentials_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_scrape_at": { + "name": "last_scrape_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_scrape_error": { + "name": "last_scrape_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "scrape_targets_org_idx": { + "name": "scrape_targets_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scrape_targets_org_enabled_idx": { + "name": "scrape_targets_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_commits": { + "name": "vcs_commits", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha": { + "name": "sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author_name": { + "name": "author_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_email": { + "name": "author_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_login": { + "name": "author_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_avatar_url": { + "name": "author_avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authored_at": { + "name": "authored_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "committed_at": { + "name": "committed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_commits_repo_sha_idx": { + "name": "vcs_commits_repo_sha_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sha", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_commits_org_sha_idx": { + "name": "vcs_commits_org_sha_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sha", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_installations": { + "name": "vcs_installations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_installation_id": { + "name": "external_installation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_login": { + "name": "account_login", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_account_id": { + "name": "external_account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_avatar_url": { + "name": "account_avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "repository_selection": { + "name": "repository_selection", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'all'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "suspended_at": { + "name": "suspended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "installed_by_user_id": { + "name": "installed_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_installations_provider_external_idx": { + "name": "vcs_installations_provider_external_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_installations_org_idx": { + "name": "vcs_installations_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_repositories": { + "name": "vcs_repositories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installation_id": { + "name": "installation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_repo_id": { + "name": "external_repo_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'main'" + }, + "tracked_branch": { + "name": "tracked_branch", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_archived": { + "name": "is_archived", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_repositories_org_repo_idx": { + "name": "vcs_repositories_org_repo_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_repo_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_repositories_org_idx": { + "name": "vcs_repositories_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_repositories_installation_idx": { + "name": "vcs_repositories_installation_idx", + "columns": [ + { + "expression": "installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_repository_branches": { + "name": "vcs_repository_branches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "head_sha": { + "name": "head_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_repository_branches_repo_name_idx": { + "name": "vcs_repository_branches_repo_name_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_repository_branches_org_idx": { + "name": "vcs_repository_branches_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index 0fae9f28b..a01124244 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -1,90 +1,97 @@ { - "version": "7", - "dialect": "postgresql", - "entries": [ - { - "idx": 0, - "version": "7", - "when": 1781306960498, - "tag": "0000_windy_raza", - "breakpoints": true - }, - { - "idx": 1, - "version": "7", - "when": 1782245542758, - "tag": "0001_naive_scalphunter", - "breakpoints": true - }, - { - "idx": 2, - "version": "7", - "when": 1782656894999, - "tag": "0002_bizarre_triathlon", - "breakpoints": true - }, - { - "idx": 3, - "version": "7", - "when": 1782992101248, - "tag": "0003_backfill_github_commit_avatars", - "breakpoints": true - }, - { - "idx": 4, - "version": "7", - "when": 1783200000000, - "tag": "0004_premium_korg", - "breakpoints": true - }, - { - "idx": 5, - "version": "7", - "when": 1783200001000, - "tag": "0005_loud_pyro", - "breakpoints": true - }, - { - "idx": 6, - "version": "7", - "when": 1783200002000, - "tag": "0006_natural_marvel_apes", - "breakpoints": true - }, - { - "idx": 7, - "version": "7", - "when": 1783200003000, - "tag": "0007_slippery_winter_soldier", - "breakpoints": true - }, - { - "idx": 8, - "version": "7", - "when": 1783200004000, - "tag": "0008_elite_butterfly", - "breakpoints": true - }, - { - "idx": 9, - "version": "7", - "when": 1783200005000, - "tag": "0009_electric_publication", - "breakpoints": true - }, - { - "idx": 10, - "version": "7", - "when": 1783377967610, - "tag": "0010_huge_dexter_bennett", - "breakpoints": true - }, - { - "idx": 11, - "version": "7", - "when": 1783377967700, - "tag": "0011_electric_publication_wave1", - "breakpoints": true - } - ] -} + "version": "7", + "dialect": "postgresql", + "entries": [ + { + "idx": 0, + "version": "7", + "when": 1781306960498, + "tag": "0000_windy_raza", + "breakpoints": true + }, + { + "idx": 1, + "version": "7", + "when": 1782245542758, + "tag": "0001_naive_scalphunter", + "breakpoints": true + }, + { + "idx": 2, + "version": "7", + "when": 1782656894999, + "tag": "0002_bizarre_triathlon", + "breakpoints": true + }, + { + "idx": 3, + "version": "7", + "when": 1782992101248, + "tag": "0003_backfill_github_commit_avatars", + "breakpoints": true + }, + { + "idx": 4, + "version": "7", + "when": 1783200000000, + "tag": "0004_premium_korg", + "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1783200001000, + "tag": "0005_loud_pyro", + "breakpoints": true + }, + { + "idx": 6, + "version": "7", + "when": 1783200002000, + "tag": "0006_natural_marvel_apes", + "breakpoints": true + }, + { + "idx": 7, + "version": "7", + "when": 1783200003000, + "tag": "0007_slippery_winter_soldier", + "breakpoints": true + }, + { + "idx": 8, + "version": "7", + "when": 1783200004000, + "tag": "0008_elite_butterfly", + "breakpoints": true + }, + { + "idx": 9, + "version": "7", + "when": 1783200005000, + "tag": "0009_electric_publication", + "breakpoints": true + }, + { + "idx": 10, + "version": "7", + "when": 1783377967610, + "tag": "0010_huge_dexter_bennett", + "breakpoints": true + }, + { + "idx": 11, + "version": "7", + "when": 1783377967700, + "tag": "0011_electric_publication_wave1", + "breakpoints": true + }, + { + "idx": 12, + "version": "7", + "when": 1783677908382, + "tag": "0012_aromatic_wallow", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index 3be7b5e4a..a4896ecde 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -17,5 +17,6 @@ export * from "./org-ingest-keys" export * from "./org-ingest-sampling-policies" export * from "./org-clickhouse-settings" export * from "./org-clickhouse-schema-apply-runs" +export * from "./railway" export * from "./scrape-targets" export * from "./vcs" diff --git a/packages/db/src/schema/railway.ts b/packages/db/src/schema/railway.ts new file mode 100644 index 000000000..bcb85117d --- /dev/null +++ b/packages/db/src/schema/railway.ts @@ -0,0 +1,75 @@ +import { boolean, index, integer, pgTable, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core" + +// One Railway API connection per org (v1). The pasted workspace/account token is stored with the +// same AES-256-GCM envelope convention as oauth_connections; it is only ever decrypted for the +// GraphQL discovery calls in the api and for the internal railway-connector work list. +export const railwayConnections = pgTable( + "railway_connections", + { + id: text("id").notNull().primaryKey(), + orgId: text("org_id").notNull(), + // "account" | "workspace" ("project" reserved for a later version — project tokens use a + // different auth header and cannot enumerate projects). + tokenType: text("token_type").notNull(), + tokenCiphertext: text("token_ciphertext").notNull(), + tokenIv: text("token_iv").notNull(), + tokenTag: text("token_tag").notNull(), + externalAccountName: text("external_account_name"), + connectedByUserId: text("connected_by_user_id").notNull(), + enabled: boolean("enabled").notNull().default(true), + lastValidatedAt: timestamp("last_validated_at", { withTimezone: true, mode: "date" }), + // Set when Railway rejects the token (401) — surfaced on the integration card as a + // "reconnect" prompt since Railway tokens have no refresh flow. + lastValidationError: text("last_validation_error"), + lastValidationErrorAt: timestamp("last_validation_error_at", { withTimezone: true, mode: "date" }), + createdAt: timestamp("created_at", { withTimezone: true, mode: "date" }).notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true, mode: "date" }).notNull(), + }, + (table) => [uniqueIndex("railway_connections_org_idx").on(table.orgId)], +) + +// One row per (project, environment) the org chose to ingest. Logs use a single environment-wide +// GraphQL subscription; metrics are polled per service from the cached services_json list. The +// watermark columns anchor gap backfill after a subscription reconnect and the next metrics +// window; health columns feed the integration card. +export const railwayTargets = pgTable( + "railway_targets", + { + id: text("id").notNull().primaryKey(), + orgId: text("org_id").notNull(), + connectionId: text("connection_id").notNull(), + projectId: text("project_id").notNull(), + projectName: text("project_name"), + environmentId: text("environment_id").notNull(), + environmentName: text("environment_name"), + enabled: boolean("enabled").notNull().default(true), + logsEnabled: boolean("logs_enabled").notNull().default(true), + metricsEnabled: boolean("metrics_enabled").notNull().default(true), + metricsIntervalSeconds: integer("metrics_interval_seconds").notNull().default(300), + // JSON array of { id, name } for the environment's services, discovered by the api and + // refreshed on a TTL — the connector polls metrics per service from this list. + servicesJson: text("services_json"), + servicesDiscoveredAt: timestamp("services_discovered_at", { withTimezone: true, mode: "date" }), + // Newest log timestamp delivered — reconnect backfill anchor. + logWatermarkAt: timestamp("log_watermark_at", { withTimezone: true, mode: "date" }), + // End of the newest metrics window ingested. + metricsWatermarkAt: timestamp("metrics_watermark_at", { withTimezone: true, mode: "date" }), + lastLogAt: timestamp("last_log_at", { withTimezone: true, mode: "date" }), + lastMetricsAt: timestamp("last_metrics_at", { withTimezone: true, mode: "date" }), + lastSuccessAt: timestamp("last_success_at", { withTimezone: true, mode: "date" }), + lastError: text("last_error"), + lastErrorAt: timestamp("last_error_at", { withTimezone: true, mode: "date" }), + createdAt: timestamp("created_at", { withTimezone: true, mode: "date" }).notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true, mode: "date" }).notNull(), + }, + (table) => [ + uniqueIndex("railway_targets_org_project_env_idx").on(table.orgId, table.projectId, table.environmentId), + index("railway_targets_org_idx").on(table.orgId), + index("railway_targets_connection_idx").on(table.connectionId), + ], +) + +export type RailwayConnectionRow = typeof railwayConnections.$inferSelect +export type RailwayConnectionInsert = typeof railwayConnections.$inferInsert +export type RailwayTargetRow = typeof railwayTargets.$inferSelect +export type RailwayTargetInsert = typeof railwayTargets.$inferInsert diff --git a/packages/domain/src/http/api.ts b/packages/domain/src/http/api.ts index 0aea37099..b6bf3568c 100644 --- a/packages/domain/src/http/api.ts +++ b/packages/domain/src/http/api.ts @@ -19,6 +19,7 @@ import { OnboardingApiGroup } from "./onboarding" import { OrgClickHouseSettingsApiGroup } from "./org-clickhouse-settings" import { OrganizationsApiGroup } from "./organizations" import { QueryEngineApiGroup } from "./query-engine" +import { RailwayApiGroup } from "./railway" import { RecommendationIssuesApiGroup } from "./recommendation-issues" import { ScrapeTargetsApiGroup } from "./scrape-targets" import { SessionReplaysApiGroup } from "./session-replay" @@ -47,6 +48,7 @@ export class MapleApi extends HttpApi.make("MapleApi") .add(OrgClickHouseSettingsApiGroup) .add(OrganizationsApiGroup) .add(QueryEngineApiGroup) + .add(RailwayApiGroup) .add(RecommendationIssuesApiGroup) .add(ScrapeTargetsApiGroup) .add(SessionReplaysApiGroup) diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index 091e53886..399a7cf64 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -21,6 +21,8 @@ export * from "./org-clickhouse-settings" export * from "./organizations" export * from "../primitives" export * from "./query-engine" +export * from "./railway" +export * from "./railway-internal" export * from "./recommendation-issues" export * from "./scrape-targets" export * from "./scraper-internal" diff --git a/packages/domain/src/http/railway-internal.ts b/packages/domain/src/http/railway-internal.ts new file mode 100644 index 000000000..fb0ee7f2f --- /dev/null +++ b/packages/domain/src/http/railway-internal.ts @@ -0,0 +1,71 @@ +import { Schema } from "effect" +import { + RailwayConnectionId, + RailwayMetricsIntervalSeconds, + RailwayTargetId, + RailwayTokenType, +} from "../primitives" + +/** + * Internal contract between the apps/api Railway integration store and the + * standalone railway-connector (apps/railway-connector). Both endpoints are + * authenticated with the `SD_INTERNAL_TOKEN` bearer — the decrypted Railway + * token only ever transits this internal wire. + */ + +export class InternalRailwayService extends Schema.Class("InternalRailwayService")({ + id: Schema.String, + name: Schema.String, +}) {} + +export class InternalRailwayTarget extends Schema.Class("InternalRailwayTarget")({ + id: RailwayTargetId, + orgId: Schema.String, + connectionId: RailwayConnectionId, + tokenType: RailwayTokenType, + /** Decrypted Railway API token — internal wire only, never persisted by the connector. */ + token: Schema.String, + projectId: Schema.String, + projectName: Schema.NullOr(Schema.String), + environmentId: Schema.String, + environmentName: Schema.NullOr(Schema.String), + services: Schema.Array(InternalRailwayService), + logsEnabled: Schema.Boolean, + metricsEnabled: Schema.Boolean, + metricsIntervalSeconds: RailwayMetricsIntervalSeconds, + /** Epoch ms of the newest log delivered — reconnect/backfill anchor. Null = start live. */ + logWatermarkMs: Schema.NullOr(Schema.Number), + /** Epoch ms of the end of the newest metrics window ingested. Null = start at now − interval. */ + metricsWatermarkMs: Schema.NullOr(Schema.Number), + /** + * The org's public ingest key (`maple_pk_*`). The connector pushes converted + * logs/metrics through the ingest gateway with this key so the data is billed + * and routed (Tinybird vs self-managed ClickHouse) exactly like customer OTLP. + */ + ingestKey: Schema.String, +}) {} + +export const InternalRailwayTargetList = Schema.Array(InternalRailwayTarget) + +export class RailwayResultReport extends Schema.Class("RailwayResultReport")({ + targetId: RailwayTargetId, + kind: Schema.Literals(["logs", "metrics"]), + /** Epoch milliseconds at which the work (flush / poll) completed. */ + at: Schema.Number, + /** Null on success; pretty-printed failure otherwise. */ + error: Schema.NullOr(Schema.String), + /** True when Railway answered 401 — the api marks the connection as needing a reconnect. */ + unauthorized: Schema.optionalKey(Schema.Boolean), + /** True when the report reflects a rate-limited attempt (Retry-After honored). */ + rateLimited: Schema.optionalKey(Schema.Boolean), + /** Log records or metric data points successfully pushed to ingest. */ + recordsIngested: Schema.optionalKey(Schema.Number), + /** Records dropped due to buffer overflow / billing limits since the last report. */ + recordsDropped: Schema.optionalKey(Schema.Number), + /** New log watermark (epoch ms) — the api advances it monotonically. */ + newLogWatermarkMs: Schema.optionalKey(Schema.Number), + /** New metrics watermark (epoch ms) — the api advances it monotonically. */ + newMetricsWatermarkMs: Schema.optionalKey(Schema.Number), +}) {} + +export const RailwayResultReportList = Schema.Array(RailwayResultReport) diff --git a/packages/domain/src/http/railway.ts b/packages/domain/src/http/railway.ts new file mode 100644 index 000000000..3b968c809 --- /dev/null +++ b/packages/domain/src/http/railway.ts @@ -0,0 +1,258 @@ +import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi" +import { Schema } from "effect" +import { + IsoDateTimeString, + RailwayConnectionId, + RailwayMetricsIntervalSeconds, + RailwayTargetId, + RailwayTokenType, +} from "../primitives" +import { Authorization } from "./current-tenant" + +/** + * Railway integration: an org pastes a Railway workspace/account API token; + * Maple discovers projects → environments → services and pulls runtime logs + * (GraphQL subscription) + resource metrics (metrics query) via the + * standalone railway-connector, pushing them through the org's own ingest + * endpoint as OTLP. + */ + +export class RailwayIntegrationStatus extends Schema.Class( + "RailwayIntegrationStatus", +)({ + connected: Schema.Boolean, + connectionId: Schema.NullOr(RailwayConnectionId), + tokenType: Schema.NullOr(RailwayTokenType), + accountName: Schema.NullOr(Schema.String), + enabled: Schema.Boolean, + lastValidatedAt: Schema.NullOr(IsoDateTimeString), + /** Set when Railway rejected the token (401) — the card prompts a reconnect. */ + lastValidationError: Schema.NullOr(Schema.String), + targetCount: Schema.Number, + erroredTargetCount: Schema.Number, +}) {} + +export class RailwayConnectRequest extends Schema.Class("RailwayConnectRequest")({ + token: Schema.String, +}) {} + +export class RailwayDisconnectResponse extends Schema.Class( + "RailwayDisconnectResponse", +)({ + disconnected: Schema.Boolean, +}) {} + +export class RailwayDiscoveredService extends Schema.Class( + "RailwayDiscoveredService", +)({ + id: Schema.String, + name: Schema.String, +}) {} + +export class RailwayDiscoveredEnvironment extends Schema.Class( + "RailwayDiscoveredEnvironment", +)({ + id: Schema.String, + name: Schema.String, + services: Schema.Array(RailwayDiscoveredService), + /** Id of the existing railway_targets row covering this environment, if any. */ + targetId: Schema.NullOr(RailwayTargetId), +}) {} + +export class RailwayDiscoveredProject extends Schema.Class( + "RailwayDiscoveredProject", +)({ + id: Schema.String, + name: Schema.String, + environments: Schema.Array(RailwayDiscoveredEnvironment), +}) {} + +export class RailwayDiscoveryResponse extends Schema.Class( + "RailwayDiscoveryResponse", +)({ + projects: Schema.Array(RailwayDiscoveredProject), +}) {} + +export class RailwayTargetResponse extends Schema.Class("RailwayTargetResponse")({ + id: RailwayTargetId, + projectId: Schema.String, + projectName: Schema.NullOr(Schema.String), + environmentId: Schema.String, + environmentName: Schema.NullOr(Schema.String), + enabled: Schema.Boolean, + logsEnabled: Schema.Boolean, + metricsEnabled: Schema.Boolean, + metricsIntervalSeconds: RailwayMetricsIntervalSeconds, + serviceCount: Schema.Number, + lastLogAt: Schema.NullOr(IsoDateTimeString), + lastMetricsAt: Schema.NullOr(IsoDateTimeString), + lastSuccessAt: Schema.NullOr(IsoDateTimeString), + lastError: Schema.NullOr(Schema.String), + lastErrorAt: Schema.NullOr(IsoDateTimeString), + createdAt: IsoDateTimeString, + updatedAt: IsoDateTimeString, +}) {} + +export class RailwayTargetsListResponse extends Schema.Class( + "RailwayTargetsListResponse", +)({ + targets: Schema.Array(RailwayTargetResponse), +}) {} + +export class CreateRailwayTargetRequest extends Schema.Class( + "CreateRailwayTargetRequest", +)({ + projectId: Schema.String, + environmentId: Schema.String, + logsEnabled: Schema.optionalKey(Schema.Boolean), + metricsEnabled: Schema.optionalKey(Schema.Boolean), + metricsIntervalSeconds: Schema.optionalKey(RailwayMetricsIntervalSeconds), +}) {} + +export class UpdateRailwayTargetRequest extends Schema.Class( + "UpdateRailwayTargetRequest", +)({ + enabled: Schema.optionalKey(Schema.Boolean), + logsEnabled: Schema.optionalKey(Schema.Boolean), + metricsEnabled: Schema.optionalKey(Schema.Boolean), + metricsIntervalSeconds: Schema.optionalKey(RailwayMetricsIntervalSeconds), +}) {} + +export class RailwayTargetDeleteResponse extends Schema.Class( + "RailwayTargetDeleteResponse", +)({ + id: RailwayTargetId, +}) {} + +export class RailwayPersistenceError extends Schema.TaggedErrorClass()( + "@maple/http/errors/RailwayPersistenceError", + { + message: Schema.String, + }, + { httpApiStatus: 503 }, +) {} + +export class RailwayValidationError extends Schema.TaggedErrorClass()( + "@maple/http/errors/RailwayValidationError", + { + message: Schema.String, + }, + { httpApiStatus: 400 }, +) {} + +export class RailwayEncryptionError extends Schema.TaggedErrorClass()( + "@maple/http/errors/RailwayEncryptionError", + { + message: Schema.String, + }, + { httpApiStatus: 500 }, +) {} + +/** Railway's GraphQL API failed or rejected the stored token. */ +export class RailwayUpstreamError extends Schema.TaggedErrorClass()( + "@maple/http/errors/RailwayUpstreamError", + { + message: Schema.String, + }, + { httpApiStatus: 502 }, +) {} + +export class RailwayNotFoundError extends Schema.TaggedErrorClass()( + "@maple/http/errors/RailwayNotFoundError", + { + message: Schema.String, + }, + { httpApiStatus: 404 }, +) {} + +export class RailwayForbiddenError extends Schema.TaggedErrorClass()( + "@maple/http/errors/RailwayForbiddenError", + { + message: Schema.String, + }, + { httpApiStatus: 403 }, +) {} + +export class RailwayApiGroup extends HttpApiGroup.make("railway") + .add( + HttpApiEndpoint.get("status", "/status", { + success: RailwayIntegrationStatus, + error: RailwayPersistenceError, + }), + ) + .add( + HttpApiEndpoint.post("connect", "/connect", { + payload: RailwayConnectRequest, + success: RailwayIntegrationStatus, + error: [ + RailwayValidationError, + RailwayUpstreamError, + RailwayPersistenceError, + RailwayEncryptionError, + RailwayForbiddenError, + ], + }), + ) + .add( + HttpApiEndpoint.post("disconnect", "/disconnect", { + success: RailwayDisconnectResponse, + error: [RailwayPersistenceError, RailwayForbiddenError], + }), + ) + .add( + HttpApiEndpoint.get("discovery", "/discovery", { + success: RailwayDiscoveryResponse, + error: [ + RailwayNotFoundError, + RailwayUpstreamError, + RailwayPersistenceError, + RailwayEncryptionError, + ], + }), + ) + .add( + HttpApiEndpoint.get("listTargets", "/targets", { + success: RailwayTargetsListResponse, + error: RailwayPersistenceError, + }), + ) + .add( + HttpApiEndpoint.post("createTarget", "/targets", { + payload: CreateRailwayTargetRequest, + success: RailwayTargetResponse, + error: [ + RailwayNotFoundError, + RailwayValidationError, + RailwayUpstreamError, + RailwayPersistenceError, + RailwayEncryptionError, + RailwayForbiddenError, + ], + }), + ) + .add( + HttpApiEndpoint.patch("updateTarget", "/targets/:targetId", { + params: { + targetId: RailwayTargetId, + }, + payload: UpdateRailwayTargetRequest, + success: RailwayTargetResponse, + error: [ + RailwayNotFoundError, + RailwayValidationError, + RailwayPersistenceError, + RailwayForbiddenError, + ], + }), + ) + .add( + HttpApiEndpoint.delete("deleteTarget", "/targets/:targetId", { + params: { + targetId: RailwayTargetId, + }, + success: RailwayTargetDeleteResponse, + error: [RailwayNotFoundError, RailwayPersistenceError, RailwayForbiddenError], + }), + ) + .prefix("/api/railway") + .middleware(Authorization) {} diff --git a/packages/domain/src/primitives.ts b/packages/domain/src/primitives.ts index 8f3fc26c2..811965553 100644 --- a/packages/domain/src/primitives.ts +++ b/packages/domain/src/primitives.ts @@ -84,6 +84,12 @@ export type ApiKeyId = Schema.Schema.Type export const ScrapeTargetId = MapleUuidId("@maple/ScrapeTargetId", "Scrape Target ID") export type ScrapeTargetId = Schema.Schema.Type +export const RailwayConnectionId = MapleUuidId("@maple/RailwayConnectionId", "Railway Connection ID") +export type RailwayConnectionId = Schema.Schema.Type + +export const RailwayTargetId = MapleUuidId("@maple/RailwayTargetId", "Railway Target ID") +export type RailwayTargetId = Schema.Schema.Type + export const AlertDestinationId = MapleUuidId("@maple/AlertDestinationId", "Alert Destination ID") export type AlertDestinationId = Schema.Schema.Type @@ -161,6 +167,26 @@ export const ScrapeTargetType = Schema.Literals(["prometheus", "planetscale"]).a }) export type ScrapeTargetType = Schema.Schema.Type +export const RailwayTokenType = Schema.Literals(["account", "workspace"]).annotate({ + identifier: "@maple/RailwayTokenType", + title: "Railway Token Type", +}) +export type RailwayTokenType = Schema.Schema.Type + +/** Poll interval bounds reflect Railway's per-plan rate limits (100 RPH on the free plan). */ +export const RailwayMetricsIntervalSeconds = Schema.Number.check( + Schema.isInt(), + Schema.isGreaterThanOrEqualTo(60), + Schema.isLessThanOrEqualTo(3600), +).pipe( + Schema.brand("@maple/RailwayMetricsIntervalSeconds"), + Schema.annotate({ + identifier: "@maple/RailwayMetricsIntervalSeconds", + title: "Railway Metrics Interval Seconds", + }), +) +export type RailwayMetricsIntervalSeconds = Schema.Schema.Type + export const IngestAttributeMappingId = MapleUuidId( "@maple/IngestAttributeMappingId", "Ingest Attribute Mapping ID",