From b0a5676267e897a64b34bd958ac833bd6560b89d Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 10:40:56 +0200 Subject: [PATCH 01/23] feat: add code graphs --- apps/api/Dockerfile | 8 + apps/api/package.json | 1 + .../lib/__tests__/graph-file-proxy.test.ts | 177 + .../src/lib/__tests__/repository-url.test.ts | 58 + apps/api/src/lib/chat.ts | 10 +- apps/api/src/lib/connector-access.ts | 120 + apps/api/src/lib/connectors.ts | 231 + apps/api/src/lib/graph-file-proxy.ts | 102 + apps/api/src/lib/graph-route.ts | 126 +- apps/api/src/lib/graph-upload-file-type.ts | 5 +- apps/api/src/lib/repository-url.ts | 290 + apps/api/src/lib/source-reference.ts | 17 +- apps/api/src/lib/team-chat.ts | 3 +- .../graph-archive-upload-route.test.ts | 340 +- apps/api/src/routes/connector-webhooks.ts | 209 + apps/api/src/routes/connectors.ts | 534 ++ apps/api/src/routes/graph.ts | 411 +- apps/api/src/server.ts | 5 + .../connectors/[connectorId]/connect/page.tsx | 10 + .../(app)/connectors/github/callback/page.tsx | 13 + .../github/install/callback/page.tsx | 24 + .../app/(app)/connectors/github/new/page.tsx | 14 + .../app/(app)/connectors/gitlab/new/page.tsx | 14 + apps/frontend/app/(app)/connectors/page.tsx | 5 + .../components/connectors/ConnectorPages.tsx | 807 ++ apps/frontend/lib/api/connectors.test.ts | 193 + apps/frontend/lib/api/connectors.ts | 169 + apps/frontend/lib/api/index.ts | 39 + apps/worker/Dockerfile | 14 +- .../lib/__tests__/code-manifest.test.ts | 280 + apps/worker/lib/code-file-metadata.ts | 1 + apps/worker/lib/code-manifest.ts | 128 + apps/worker/lib/code-repository-finalizer.ts | 142 + apps/worker/lib/file-content-source.ts | 205 + apps/worker/lib/file-processing-state.ts | 39 + apps/worker/lib/regenerate-descriptions.ts | 26 +- apps/worker/lib/save-graph.ts | 535 ++ apps/worker/package.json | 1 + apps/worker/worker.ts | 6 + apps/worker/workflows/process-code-file.ts | 232 + apps/worker/workflows/process-file.test.ts | 115 + apps/worker/workflows/process-file.ts | 553 +- apps/worker/workflows/process-files-spec.ts | 6 + .../workflows/sync-repository-graph-spec.ts | 19 + .../workflows/sync-repository-graph.test.ts | 360 + .../worker/workflows/sync-repository-graph.ts | 495 + bun.lock | 39 +- .../migration.sql | 14 + .../snapshot.json | 6823 ++++++++++++++ .../migration.sql | 20 + .../snapshot.json | 6876 ++++++++++++++ .../20260614105716_tricky_plazm/migration.sql | 115 + .../20260614105716_tricky_plazm/snapshot.json | 8283 +++++++++++++++++ .../src/__tests__/relationship-tools.test.ts | 181 + packages/ai/src/tools/correction.ts | 10 +- packages/ai/src/tools/entity.ts | 11 +- packages/ai/src/tools/relationship.ts | 61 +- packages/ai/src/tools/source.ts | 10 +- packages/connectors/package.json | 24 + .../src/__tests__/credentials.test.ts | 63 + .../connectors/src/__tests__/github.test.ts | 205 + .../connectors/src/__tests__/gitlab.test.ts | 229 + packages/connectors/src/code-files.ts | 20 + packages/connectors/src/credentials.ts | 130 + packages/connectors/src/github.ts | 472 + packages/connectors/src/gitlab.ts | 348 + packages/connectors/src/index.ts | 5 + packages/connectors/src/types.ts | 121 + packages/connectors/src/webhooks.ts | 34 + packages/connectors/tsconfig.json | 23 + packages/contracts/src/routes.ts | 156 + packages/contracts/src/source.ts | 6 + packages/db/package.json | 1 + .../db/src/__tests__/migration-compat.test.ts | 216 + packages/db/src/source-validity.ts | 40 + packages/db/src/tables/connectors.ts | 170 + packages/db/src/tables/graph.ts | 38 + packages/graph/package.json | 7 + packages/graph/src/__tests__/dedupe.test.ts | 37 + packages/graph/src/__tests__/extract.test.ts | 49 + .../graph/src/__tests__/file-type.test.ts | 26 + packages/graph/src/chunking/factory.ts | 1 + .../src/code/__tests__/repository.test.ts | 916 ++ packages/graph/src/code/file-path.ts | 7 + packages/graph/src/code/identity.ts | 23 + packages/graph/src/code/imports.ts | 638 ++ packages/graph/src/code/metadata.ts | 77 + packages/graph/src/code/parser.ts | 70 + packages/graph/src/code/repository.ts | 597 ++ packages/graph/src/code/syntax.ts | 259 + packages/graph/src/code/types.ts | 121 + packages/graph/src/dedupe.ts | 28 +- packages/graph/src/file-type.ts | 7 + packages/graph/src/index.ts | 2 + packages/graph/src/loader/factory.ts | 2 + packages/graph/src/merge.ts | 28 +- packages/graph/src/relationship-key.ts | 26 + plans/001-align-migration-snapshot.md | 153 + ...eturn-none-for-missing-same-entity-path.md | 152 + .../003-validate-repository-import-models.md | 171 + plans/004-sanitize-repository-url-errors.md | 178 + ...oute-code-uploads-through-code-workflow.md | 226 + .../006-support-external-github-code-files.md | 356 + ...7-version-code-sources-with-valid-until.md | 431 + ...ovider-connectors-for-repository-graphs.md | 731 ++ ...ncremental-connector-repository-updates.md | 348 + plans/README.md | 74 + 107 files changed, 36999 insertions(+), 578 deletions(-) create mode 100644 apps/api/src/lib/__tests__/graph-file-proxy.test.ts create mode 100644 apps/api/src/lib/__tests__/repository-url.test.ts create mode 100644 apps/api/src/lib/connector-access.ts create mode 100644 apps/api/src/lib/connectors.ts create mode 100644 apps/api/src/lib/repository-url.ts create mode 100644 apps/api/src/routes/connector-webhooks.ts create mode 100644 apps/api/src/routes/connectors.ts create mode 100644 apps/frontend/app/(app)/connectors/[connectorId]/connect/page.tsx create mode 100644 apps/frontend/app/(app)/connectors/github/callback/page.tsx create mode 100644 apps/frontend/app/(app)/connectors/github/install/callback/page.tsx create mode 100644 apps/frontend/app/(app)/connectors/github/new/page.tsx create mode 100644 apps/frontend/app/(app)/connectors/gitlab/new/page.tsx create mode 100644 apps/frontend/app/(app)/connectors/page.tsx create mode 100644 apps/frontend/components/connectors/ConnectorPages.tsx create mode 100644 apps/frontend/lib/api/connectors.test.ts create mode 100644 apps/frontend/lib/api/connectors.ts create mode 100644 apps/worker/lib/__tests__/code-manifest.test.ts create mode 100644 apps/worker/lib/code-file-metadata.ts create mode 100644 apps/worker/lib/code-manifest.ts create mode 100644 apps/worker/lib/code-repository-finalizer.ts create mode 100644 apps/worker/lib/file-content-source.ts create mode 100644 apps/worker/lib/file-processing-state.ts create mode 100644 apps/worker/lib/save-graph.ts create mode 100644 apps/worker/workflows/process-code-file.ts create mode 100644 apps/worker/workflows/process-file.test.ts create mode 100644 apps/worker/workflows/sync-repository-graph-spec.ts create mode 100644 apps/worker/workflows/sync-repository-graph.test.ts create mode 100644 apps/worker/workflows/sync-repository-graph.ts create mode 100644 migrations/20260613184716_sturdy_triton/migration.sql create mode 100644 migrations/20260613184716_sturdy_triton/snapshot.json create mode 100644 migrations/20260613201908_mature_emma_frost/migration.sql create mode 100644 migrations/20260613201908_mature_emma_frost/snapshot.json create mode 100644 migrations/20260614105716_tricky_plazm/migration.sql create mode 100644 migrations/20260614105716_tricky_plazm/snapshot.json create mode 100644 packages/ai/src/__tests__/relationship-tools.test.ts create mode 100644 packages/connectors/package.json create mode 100644 packages/connectors/src/__tests__/credentials.test.ts create mode 100644 packages/connectors/src/__tests__/github.test.ts create mode 100644 packages/connectors/src/__tests__/gitlab.test.ts create mode 100644 packages/connectors/src/code-files.ts create mode 100644 packages/connectors/src/credentials.ts create mode 100644 packages/connectors/src/github.ts create mode 100644 packages/connectors/src/gitlab.ts create mode 100644 packages/connectors/src/index.ts create mode 100644 packages/connectors/src/types.ts create mode 100644 packages/connectors/src/webhooks.ts create mode 100644 packages/connectors/tsconfig.json create mode 100644 packages/db/src/source-validity.ts create mode 100644 packages/db/src/tables/connectors.ts create mode 100644 packages/graph/src/__tests__/file-type.test.ts create mode 100644 packages/graph/src/code/__tests__/repository.test.ts create mode 100644 packages/graph/src/code/file-path.ts create mode 100644 packages/graph/src/code/identity.ts create mode 100644 packages/graph/src/code/imports.ts create mode 100644 packages/graph/src/code/metadata.ts create mode 100644 packages/graph/src/code/parser.ts create mode 100644 packages/graph/src/code/repository.ts create mode 100644 packages/graph/src/code/syntax.ts create mode 100644 packages/graph/src/code/types.ts create mode 100644 packages/graph/src/relationship-key.ts create mode 100644 plans/001-align-migration-snapshot.md create mode 100644 plans/002-return-none-for-missing-same-entity-path.md create mode 100644 plans/003-validate-repository-import-models.md create mode 100644 plans/004-sanitize-repository-url-errors.md create mode 100644 plans/005-route-code-uploads-through-code-workflow.md create mode 100644 plans/006-support-external-github-code-files.md create mode 100644 plans/007-version-code-sources-with-valid-until.md create mode 100644 plans/008-provider-connectors-for-repository-graphs.md create mode 100644 plans/009-incremental-connector-repository-updates.md create mode 100644 plans/README.md diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 333d7f5c..1ca9b105 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -2,6 +2,13 @@ FROM oven/bun:1 AS build WORKDIR /app +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + g++ \ + make \ + python3 \ + && rm -rf /var/lib/apt/lists/* + # Cache workspace dependency installation COPY package.json bun.lock ./ COPY patches patches @@ -59,6 +66,7 @@ RUN apt-get update \ brotli \ bzip2 \ ca-certificates \ + git \ ghostscript \ gzip \ libarchive-tools \ diff --git a/apps/api/package.json b/apps/api/package.json index c25a5fcc..fc19f06e 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -16,6 +16,7 @@ "@kiwi/ai": "workspace:*", "@kiwi/auth": "workspace:*", "@kiwi/contracts": "workspace:*", + "@kiwi/connectors": "workspace:*", "@kiwi/db": "workspace:*", "@kiwi/files": "workspace:*", "@kiwi/graph": "workspace:*", diff --git a/apps/api/src/lib/__tests__/graph-file-proxy.test.ts b/apps/api/src/lib/__tests__/graph-file-proxy.test.ts new file mode 100644 index 00000000..1bc778e3 --- /dev/null +++ b/apps/api/src/lib/__tests__/graph-file-proxy.test.ts @@ -0,0 +1,177 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; + +let selectedFile: unknown; +let bindingRow: unknown = null; +let metadataCalls = 0; +let streamCalls = 0; +let selectCallCount = 0; +const providerReadCalls: Array<{ path: string; commitSha: string }> = []; + +function createSelectQuery() { + const chain = { + from: () => chain, + innerJoin: () => chain, + where: () => chain, + limit: () => { + selectCallCount += 1; + if (selectCallCount === 1) { + return selectedFile ? [selectedFile] : []; + } + return bindingRow ? [bindingRow] : []; + }, + }; + return chain; +} + +mock.module("@kiwi/db", () => ({ + db: { + select: () => createSelectQuery(), + }, +})); + +mock.module("@kiwi/files", () => ({ + deleteFile: async () => undefined, + getFileMetadata: async () => { + metadataCalls += 1; + return { size: 12, type: "text/plain", lastModified: null }; + }, + getFileStream: async () => { + streamCalls += 1; + return { content: new Blob(["hello world\n"]).stream(), size: 12, type: "text/plain" }; + }, + listFiles: async () => [], + putGraphFile: async () => ({ key: "key", type: "text/plain" }), +})); + +mock.module("../connectors", () => ({ + createProviderClient: async () => ({ + readFile: async (_repository: unknown, path: string, commitSha: string) => { + providerReadCalls.push({ path, commitSha }); + return "export const older = true;\n"; + }, + }), +})); + +// Dynamic import is required so module mocks are installed before graph-file-proxy is evaluated. +const { getGraphFileProxyResponse } = await import("../graph-file-proxy"); + +describe("graph file proxy", () => { + beforeEach(() => { + metadataCalls = 0; + streamCalls = 0; + selectCallCount = 0; + providerReadCalls.length = 0; + bindingRow = null; + selectedFile = { + key: "graphs/graph-1/file-1.txt", + name: "file.txt", + mimeType: "text/plain", + storageKind: "internal", + externalProvider: null, + externalUrl: null, + repositoryBindingId: null, + metadata: null, + }; + }); + + test("streams internal files from S3", async () => { + const result = await getGraphFileProxyResponse({ + graphId: "graph-1", + fileId: "file-1", + request: new Request("http://localhost/file"), + bucket: "bucket", + }); + + expect(result.status).toBe("ok"); + expect(metadataCalls).toBe(1); + expect(streamCalls).toBe(1); + if (result.status === "ok") { + expect(result.response.status).toBe(200); + } + }); + + test("redirects external GitHub files to immutable HTML URLs without S3 calls", async () => { + selectedFile = { + key: "external:github:acme/widgets@commit-1:src/index.ts", + name: "widgets/src/index.ts", + mimeType: "text/plain", + storageKind: "external", + externalProvider: "github", + externalUrl: "https://raw.githubusercontent.com/acme/widgets/commit-1/src/index.ts", + repositoryBindingId: null, + metadata: JSON.stringify({ + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-1", + path: "src/index.ts", + external: { + provider: "github", + rawUrl: "https://raw.githubusercontent.com/acme/widgets/commit-1/src/index.ts", + htmlUrl: "https://github.com/acme/widgets/blob/commit-1/src/index.ts", + }, + }), + }; + + const result = await getGraphFileProxyResponse({ + graphId: "graph-1", + fileId: "file-1", + request: new Request("http://localhost/file"), + bucket: "bucket", + }); + + expect(result.status).toBe("ok"); + expect(metadataCalls).toBe(0); + expect(streamCalls).toBe(0); + expect(providerReadCalls).toEqual([]); + if (result.status === "ok") { + expect(result.response.status).toBe(307); + expect(result.response.headers.get("location")).toBe( + "https://github.com/acme/widgets/blob/commit-1/src/index.ts" + ); + } + }); + + test("reads connector-backed files at the row commit without S3 access", async () => { + selectedFile = { + key: "connector:binding-1:commit-2:src/index.ts", + name: "widgets/src/index.ts", + mimeType: "text/plain", + storageKind: "external", + externalProvider: "github", + externalUrl: "https://raw.githubusercontent.com/acme/widgets/commit-2/src/index.ts", + repositoryBindingId: "binding-1", + metadata: JSON.stringify({ + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-1", + path: "src/index.ts", + }), + }; + bindingRow = { + binding: { + providerRepositoryId: "1", + repositoryFullName: "acme/widgets", + repositoryHtmlUrl: "https://github.com/acme/widgets", + branch: "main", + }, + installation: { status: "active" }, + connector: { provider: "github", status: "active" }, + }; + + const result = await getGraphFileProxyResponse({ + graphId: "graph-1", + fileId: "file-1", + request: new Request("http://localhost/file"), + bucket: "bucket", + }); + + expect(result.status).toBe("ok"); + expect(metadataCalls).toBe(0); + expect(streamCalls).toBe(0); + expect(providerReadCalls).toEqual([{ path: "src/index.ts", commitSha: "commit-1" }]); + if (result.status === "ok") { + expect(result.response.status).toBe(200); + await expect(result.response.text()).resolves.toBe("export const older = true;\n"); + } + }); +}); diff --git a/apps/api/src/lib/__tests__/repository-url.test.ts b/apps/api/src/lib/__tests__/repository-url.test.ts new file mode 100644 index 00000000..e8c5a4d5 --- /dev/null +++ b/apps/api/src/lib/__tests__/repository-url.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, test } from "bun:test"; +import { isSupportedCodePath } from "@kiwi/graph/code/file-path"; +import { buildGitHubExternalCodeFile, normalizeRepositoryUrl } from "../repository-url"; + +describe("repository URL helpers", () => { + test("normalizes supported repository roots", () => { + expect(normalizeRepositoryUrl(" https://github.com/owner/repo ")).toEqual({ + url: "https://github.com/owner/repo.git", + name: "repo", + }); + expect(normalizeRepositoryUrl("https://gitlab.com/group/subgroup/repo.git?tab=readme#main")).toEqual({ + url: "https://gitlab.com/group/subgroup/repo.git", + name: "repo", + }); + expect(normalizeRepositoryUrl("https://bitbucket.org/team/repo.git")).toEqual({ + url: "https://bitbucket.org/team/repo.git", + name: "repo", + }); + }); + + test("rejects unsafe or non-root repository URLs", () => { + expect(() => normalizeRepositoryUrl("http://github.com/owner/repo")).toThrow("HTTPS"); + expect(() => normalizeRepositoryUrl("https://token@github.com/owner/repo")).toThrow("credentials"); + expect(() => normalizeRepositoryUrl("https://example.com/owner/repo")).toThrow("host is not supported"); + expect(() => normalizeRepositoryUrl("https://github.com/owner/repo/tree/main")).toThrow("repository root"); + expect(() => normalizeRepositoryUrl("https://github.com/owner")).toThrow("owner and repository"); + expect(() => normalizeRepositoryUrl("https://gitlab.com/group/repo/-/tree/main")).toThrow("repository root"); + }); + + test("recognizes supported code file paths without matching generated directories", () => { + expect(isSupportedCodePath("src/index.ts")).toBe(true); + expect(isSupportedCodePath("src/component.tsx")).toBe(true); + expect(isSupportedCodePath("src/script.js")).toBe(true); + expect(isSupportedCodePath("src/README.md")).toBe(false); + }); + + test("builds immutable GitHub external code links", () => { + expect( + buildGitHubExternalCodeFile({ + repositoryUrl: "https://github.com/owner/repo.git", + commitSha: "abc123", + path: "src/nested/my file.ts", + }) + ).toEqual({ + provider: "github", + rawUrl: "https://raw.githubusercontent.com/owner/repo/abc123/src/nested/my%20file.ts", + htmlUrl: "https://github.com/owner/repo/blob/abc123/src/nested/my%20file.ts", + key: "external:github:owner/repo@abc123:src/nested/my file.ts", + }); + expect( + buildGitHubExternalCodeFile({ + repositoryUrl: "https://gitlab.com/group/repo.git", + commitSha: "abc123", + path: "src/index.ts", + }) + ).toBeNull(); + }); +}); diff --git a/apps/api/src/lib/chat.ts b/apps/api/src/lib/chat.ts index a3d4caf4..b365aff3 100644 --- a/apps/api/src/lib/chat.ts +++ b/apps/api/src/lib/chat.ts @@ -28,6 +28,7 @@ import { chatTable, messageTable, type ChatMessage } from "@kiwi/db/tables/chats import { getDefaultOrganizationId } from "@kiwi/auth/server"; import { organizationPromptsTable, teamPromptsTable, userPromptsTable } from "@kiwi/db/tables/auth"; import { filesTable, graphPromptsTable, processRunsTable, sourcesTable, textUnitTable } from "@kiwi/db/tables/graph"; +import { currentSourcePredicate, visibleFilePredicate } from "@kiwi/db/source-validity"; import { error as logError, warn as logWarn } from "@kiwi/logger"; import { Result } from "better-result"; import { and, asc, desc, eq, inArray, isNotNull, isNull, sql } from "drizzle-orm"; @@ -707,7 +708,14 @@ export async function enrichCitation(graphId: string, sourceId: string): Promise .from(sourcesTable) .innerJoin(textUnitTable, eq(textUnitTable.id, sourcesTable.textUnitId)) .innerJoin(filesTable, eq(filesTable.id, textUnitTable.fileId)) - .where(and(eq(sourcesTable.id, sourceId), eq(filesTable.graphId, graphId))) + .where( + and( + eq(sourcesTable.id, sourceId), + eq(filesTable.graphId, graphId), + currentSourcePredicate(sourcesTable), + visibleFilePredicate(filesTable) + ) + ) .limit(1); if (!row) { diff --git a/apps/api/src/lib/connector-access.ts b/apps/api/src/lib/connector-access.ts new file mode 100644 index 00000000..ef8e10ee --- /dev/null +++ b/apps/api/src/lib/connector-access.ts @@ -0,0 +1,120 @@ +import { db } from "@kiwi/db"; +import { + connectorInstallationsTable, + connectorsTable, + repositoryGraphBindingsTable, + type ConnectorProvider, +} from "@kiwi/db/tables/connectors"; +import { graphTable } from "@kiwi/db/tables/graph"; +import { and, eq } from "drizzle-orm"; +import type { AuthUser } from "../middleware/auth"; +import { API_ERROR_CODES } from "../types"; +import { assertCanCreateTopLevelGraph, assertCanViewGraphWithRootOwner } from "./graph-access"; +import { requireOrganizationAdmin, requireTeamGraphCreateAccess } from "./team-access"; + +export type ConnectorRow = typeof connectorsTable.$inferSelect; +export type ConnectorInstallationRow = typeof connectorInstallationsTable.$inferSelect; +export type RepositoryGraphBindingRow = typeof repositoryGraphBindingsTable.$inferSelect; + +export async function requireConnector(id: string): Promise { + const [connector] = await db.select().from(connectorsTable).where(eq(connectorsTable.id, id)).limit(1); + if (!connector) { + throw new Error(API_ERROR_CODES.GRAPH_NOT_FOUND); + } + return connector; +} + +export async function requireActiveConnector(id: string, provider?: ConnectorProvider): Promise { + const connector = await requireConnector(id); + if (connector.status !== "active" || (provider && connector.provider !== provider)) { + throw new Error(API_ERROR_CODES.FORBIDDEN); + } + return connector; +} + +export async function assertCanManageConnectorOwner(user: AuthUser, input: { organizationId?: string; teamId?: string }) { + if (input.teamId) { + return requireTeamGraphCreateAccess(user, input.teamId); + } + + if (input.organizationId) { + return requireOrganizationAdmin(user, input.organizationId); + } + + return assertCanCreateTopLevelGraph(user); +} + +export async function assertCanUseInstallation(user: AuthUser, installationId: string): Promise { + const [installation] = await db + .select() + .from(connectorInstallationsTable) + .where(eq(connectorInstallationsTable.id, installationId)) + .limit(1); + + if (!installation || installation.status !== "active") { + throw new Error(API_ERROR_CODES.FORBIDDEN); + } + + await assertCanManageConnectorOwner(user, { + organizationId: installation.organizationId ?? undefined, + teamId: installation.teamId ?? undefined, + }); + return installation; +} + +export async function assertCanViewBinding(user: AuthUser, bindingId: string) { + const [row] = await db + .select({ binding: repositoryGraphBindingsTable, graph: graphTable }) + .from(repositoryGraphBindingsTable) + .innerJoin(graphTable, eq(graphTable.id, repositoryGraphBindingsTable.graphId)) + .where(eq(repositoryGraphBindingsTable.id, bindingId)) + .limit(1); + + if (!row) { + throw new Error(API_ERROR_CODES.GRAPH_NOT_FOUND); + } + + await assertCanViewGraphWithRootOwner(user, row.binding.graphId); + return row; +} + +export async function assertCanSyncBinding(user: AuthUser, bindingId: string) { + const row = await assertCanViewBinding(user, bindingId); + await assertCanManageConnectorOwner(user, { + organizationId: row.graph.organizationId ?? undefined, + teamId: row.graph.teamId ?? undefined, + }); + if (!row.binding.webhookEnabled) { + throw new Error(API_ERROR_CODES.FORBIDDEN); + } + return row; +} + +export async function loadConnectorBindingGraph(bindingId: string) { + const [row] = await db + .select({ + binding: repositoryGraphBindingsTable, + installation: connectorInstallationsTable, + connector: connectorsTable, + graph: graphTable, + }) + .from(repositoryGraphBindingsTable) + .innerJoin( + connectorInstallationsTable, + eq(connectorInstallationsTable.id, repositoryGraphBindingsTable.connectorInstallationId) + ) + .innerJoin(connectorsTable, eq(connectorsTable.id, connectorInstallationsTable.connectorId)) + .innerJoin(graphTable, eq(graphTable.id, repositoryGraphBindingsTable.graphId)) + .where(eq(repositoryGraphBindingsTable.id, bindingId)) + .limit(1); + + return row ?? null; +} + +export function visibleInstallationWhere(user: AuthUser, connectorId: string) { + if (user.isSystemAdmin) { + return eq(connectorInstallationsTable.connectorId, connectorId); + } + + return and(eq(connectorInstallationsTable.connectorId, connectorId), eq(connectorInstallationsTable.status, "active")); +} diff --git a/apps/api/src/lib/connectors.ts b/apps/api/src/lib/connectors.ts new file mode 100644 index 00000000..bce95657 --- /dev/null +++ b/apps/api/src/lib/connectors.ts @@ -0,0 +1,231 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; +import { + createGitHubClient, + createGitHubInstallationToken, + createGitLabClient, + decryptConnectorCredentials, + decryptConnectorSecret, + encryptConnectorCredentials, + encryptConnectorSecret, + type ConnectorProvider, + type ConnectorSecretPayload, + type GitHubConnectorCredentials, + type GitLabConnectorCredentials, + type GitLabInstallationCredentials, + type ProviderBranch, + type ProviderRepository, + type ProviderRepositoryClient, +} from "@kiwi/connectors"; +import type { connectorsTable, connectorInstallationsTable } from "@kiwi/db/tables/connectors"; +import { env } from "../env"; + +const frontendUrl = + env.TRUSTED_ORIGINS?.split(",") + .map((origin) => origin.trim()) + .find(Boolean) ?? "http://localhost:3000"; + +const STATE_VERSION = "v1"; +const STATE_TTL_MS = 10 * 60 * 1000; + +type ConnectorRow = typeof connectorsTable.$inferSelect; +type InstallationRow = typeof connectorInstallationsTable.$inferSelect; + +export type ConnectorState = { + purpose: "github-manifest" | "github-installation" | "gitlab-oauth"; + userId: string; + connectorId?: string; + organizationId?: string; + teamId?: string; + createdAt: number; +}; + +export type PublicConnector = { + id: string; + provider: ConnectorProvider; + name: string; + slug: string; + status: string; + appId: string | null; + clientId: string | null; + createdAt: string | null; + updatedAt: string | null; +}; + +export function toPublicConnector(row: ConnectorRow): PublicConnector { + return { + id: row.id, + provider: row.provider as ConnectorProvider, + name: row.name, + slug: row.slug, + status: row.status, + appId: row.appId, + clientId: row.clientId, + createdAt: row.createdAt?.toISOString() ?? null, + updatedAt: row.updatedAt?.toISOString() ?? null, + }; +} + +export function toPublicInstallation(row: InstallationRow) { + return { + id: row.id, + connectorId: row.connectorId, + provider: row.provider as ConnectorProvider, + providerInstallationId: row.providerInstallationId, + providerAccountLogin: row.providerAccountLogin, + providerAccountType: row.providerAccountType, + organizationId: row.organizationId, + teamId: row.teamId, + repositorySelection: row.repositorySelection, + status: row.status, + createdAt: row.createdAt?.toISOString() ?? null, + updatedAt: row.updatedAt?.toISOString() ?? null, + }; +} + +export function encryptCredentials(value: ConnectorSecretPayload): string { + return encryptConnectorCredentials(value, env.AUTH_SECRET); +} + +export function encryptSecret(value: string): string { + return encryptConnectorSecret(value, env.AUTH_SECRET); +} + +export function decryptSecret(value: string): string { + return decryptConnectorSecret(value, env.AUTH_SECRET); +} + +function isGitHubConnectorCredentials(value: ConnectorSecretPayload): value is GitHubConnectorCredentials { + return "provider" in value && value.provider === "github"; +} + +function isGitLabConnectorCredentials(value: ConnectorSecretPayload): value is GitLabConnectorCredentials { + return "provider" in value && value.provider === "gitlab" && "baseUrl" in value; +} + +function isGitLabInstallationCredentials(value: ConnectorSecretPayload): value is GitLabInstallationCredentials { + return "provider" in value && value.provider === "gitlab" && "accessToken" in value; +} + +export function signConnectorState(state: Omit): string { + const payload = Buffer.from(JSON.stringify({ ...state, createdAt: Date.now() })).toString("base64url"); + const signature = createHmac("sha256", env.AUTH_SECRET).update(payload).digest("base64url"); + return `${STATE_VERSION}.${payload}.${signature}`; +} + +export function verifyConnectorState(value: string, purpose: ConnectorState["purpose"], userId?: string): ConnectorState | null { + const [version, payload, signature] = value.split("."); + if (version !== STATE_VERSION || !payload || !signature) { + return null; + } + + const expected = createHmac("sha256", env.AUTH_SECRET).update(payload).digest("base64url"); + const actualBuffer = Buffer.from(signature); + const expectedBuffer = Buffer.from(expected); + if (actualBuffer.length !== expectedBuffer.length || !timingSafeEqual(actualBuffer, expectedBuffer)) { + return null; + } + + const parsed = JSON.parse(Buffer.from(payload, "base64url").toString("utf8")) as ConnectorState; + if (parsed.purpose !== purpose || Date.now() - parsed.createdAt > STATE_TTL_MS) { + return null; + } + + if (userId && parsed.userId !== userId) { + return null; + } + + return parsed; +} + +export function createManifestUrl(state: string, name: string): string { + const manifest = { + name, + url: frontendUrl, + hook_attributes: { + url: `${env.API_URL ?? "http://localhost:4321"}/connectors/webhooks/github`, + }, + redirect_url: `${frontendUrl}/connectors/github/callback`, + setup_url: `${frontendUrl}/connectors/github/install/callback`, + public: false, + default_permissions: { + contents: "read", + metadata: "read", + }, + default_events: ["push", "installation", "installation_repositories"], + }; + const url = new URL("https://github.com/settings/apps/new"); + url.searchParams.set("state", state); + url.searchParams.set("manifest", JSON.stringify(manifest)); + return url.href; +} + +export async function exchangeGitHubManifestCode(code: string) { + const response = await fetch(`https://api.github.com/app-manifests/${encodeURIComponent(code)}/conversions`, { + method: "POST", + headers: { + Accept: "application/vnd.github+json", + "User-Agent": "kiwi-connectors", + "X-GitHub-Api-Version": "2022-11-28", + }, + }); + + if (!response.ok) { + throw new Error("Invalid connector manifest code"); + } + + return (await response.json()) as { + id: number | string; + slug?: string; + name: string; + client_id?: string; + client_secret?: string; + pem: string; + webhook_secret?: string; + }; +} + +export async function createProviderClient(connector: ConnectorRow, installation: InstallationRow): Promise { + const credentials = decryptConnectorCredentials(connector.encryptedCredentials, env.AUTH_SECRET); + if (connector.provider === "github") { + if (!isGitHubConnectorCredentials(credentials)) { + throw new Error("Invalid connector credentials"); + } + const token = await createGitHubInstallationToken({ + credentials, + installationId: installation.providerInstallationId, + }); + return createGitHubClient({ installationToken: token.token }); + } + + if (!isGitLabConnectorCredentials(credentials)) { + throw new Error("Invalid connector credentials"); + } + const installationCredentials = installation.encryptedCredentials + ? decryptConnectorCredentials(installation.encryptedCredentials, env.AUTH_SECRET) + : null; + if (!installationCredentials || !isGitLabInstallationCredentials(installationCredentials)) { + throw new Error("Invalid connector installation credentials"); + } + return createGitLabClient({ + baseUrl: credentials.baseUrl, + accessToken: installationCredentials.accessToken, + }); +} + +export async function listProviderRepositories(connector: ConnectorRow, installation: InstallationRow): Promise { + return (await createProviderClient(connector, installation)).listRepositories(); +} + +export async function listProviderBranches( + connector: ConnectorRow, + installation: InstallationRow, + repositoryId: string +): Promise { + const client = await createProviderClient(connector, installation); + const repositories = await client.listRepositories(); + const repository = repositories.find((candidate) => candidate.id === repositoryId || candidate.fullName === repositoryId); + if (!repository) { + throw new Error("Repository not found"); + } + return client.listBranches(repository); +} diff --git a/apps/api/src/lib/graph-file-proxy.ts b/apps/api/src/lib/graph-file-proxy.ts index 3c15315e..1a5902fa 100644 --- a/apps/api/src/lib/graph-file-proxy.ts +++ b/apps/api/src/lib/graph-file-proxy.ts @@ -1,13 +1,22 @@ +import type { ConnectorProvider, ProviderRepository } from "@kiwi/connectors"; import { and, eq } from "drizzle-orm"; import { db } from "@kiwi/db"; +import { connectorInstallationsTable, connectorsTable, repositoryGraphBindingsTable } from "@kiwi/db/tables/connectors"; import { filesTable } from "@kiwi/db/tables/graph"; import { getFileMetadata, getFileStream } from "@kiwi/files"; +import { parseCodeFileMetadata } from "@kiwi/graph/code/metadata"; +import { createProviderClient } from "./connectors"; import { contentDispositionForFile, contentDispositionHeader, parseByteRange } from "./file-proxy"; export type GraphFileProxyRecord = { key: string; name: string; mimeType: string; + storageKind: string; + externalProvider: string | null; + externalUrl: string | null; + repositoryBindingId: string | null; + metadata: string | null; }; export type GraphFileProxyResult = @@ -42,6 +51,11 @@ export async function loadGraphFileForProxy(graphId: string, fileId: string): Pr key: filesTable.key, name: filesTable.name, mimeType: filesTable.mimeType, + storageKind: filesTable.storageKind, + externalProvider: filesTable.externalProvider, + externalUrl: filesTable.externalUrl, + repositoryBindingId: filesTable.repositoryBindingId, + metadata: filesTable.metadata, }) .from(filesTable) .where(and(eq(filesTable.graphId, graphId), eq(filesTable.id, fileId), eq(filesTable.deleted, false))) @@ -62,6 +76,56 @@ export async function getGraphFileProxyResponse(options: { return { status: "not_found" }; } + if (file.storageKind === "external") { + if (file.repositoryBindingId) { + const content = await readConnectorFile(file.repositoryBindingId, file.metadata); + if (content === null) { + return { status: "not_found" }; + } + const bytes = new TextEncoder().encode(content); + const range = parseByteRange(options.request.headers.get("range"), bytes.byteLength); + if (range === "invalid") { + return { status: "invalid_range", size: bytes.byteLength }; + } + const body = range ? bytes.slice(range.start, range.end + 1) : bytes; + const headers = new Headers({ + "Accept-Ranges": "bytes", + "Cache-Control": "private, no-cache", + "Content-Length": String(body.byteLength), + "Content-Type": file.mimeType || "text/plain; charset=utf-8", + "X-Content-Type-Options": "nosniff", + }); + if (range) { + headers.set("Content-Range", `bytes ${range.start}-${range.end}/${bytes.byteLength}`); + } + return { + status: "ok", + response: new Response(options.head ? null : body, { status: range ? 206 : 200, headers }), + }; + } + + if (file.externalProvider !== "github" || !file.externalUrl) { + return { status: "not_found" }; + } + + const metadata = parseCodeFileMetadata(file.metadata); + if (!metadata?.external || metadata.external.provider !== "github") { + return { status: "not_found" }; + } + + return { + status: "ok", + response: new Response(null, { + status: 307, + headers: { + "Cache-Control": "private, no-cache", + Location: metadata.external.htmlUrl, + "X-Content-Type-Options": "nosniff", + }, + }), + }; + } + const metadata = await getFileMetadata(file.key, options.bucket); if (!metadata) { return { status: "not_found" }; @@ -117,3 +181,41 @@ export async function getGraphFileProxyResponse(options: { }), }; } + +async function readConnectorFile(bindingId: string, metadataValue: string | null): Promise { + const metadata = parseCodeFileMetadata(metadataValue); + if (!metadata) { + return null; + } + + const [row] = await db + .select({ + binding: repositoryGraphBindingsTable, + installation: connectorInstallationsTable, + connector: connectorsTable, + }) + .from(repositoryGraphBindingsTable) + .innerJoin( + connectorInstallationsTable, + eq(connectorInstallationsTable.id, repositoryGraphBindingsTable.connectorInstallationId) + ) + .innerJoin(connectorsTable, eq(connectorsTable.id, connectorInstallationsTable.connectorId)) + .where(eq(repositoryGraphBindingsTable.id, bindingId)) + .limit(1); + + if (!row || row.connector.status !== "active" || row.installation.status !== "active") { + return null; + } + + const client = await createProviderClient(row.connector, row.installation); + const repository: ProviderRepository = { + provider: row.connector.provider as ConnectorProvider, + id: row.binding.providerRepositoryId, + fullName: row.binding.repositoryFullName, + name: row.binding.repositoryFullName.split("/").at(-1) ?? row.binding.repositoryFullName, + htmlUrl: row.binding.repositoryHtmlUrl, + defaultBranch: row.binding.branch, + private: true, + }; + return client.readFile(repository, metadata.path, metadata.commitSha); +} diff --git a/apps/api/src/lib/graph-route.ts b/apps/api/src/lib/graph-route.ts index 0c9c2cf6..42a07610 100644 --- a/apps/api/src/lib/graph-route.ts +++ b/apps/api/src/lib/graph-route.ts @@ -15,7 +15,7 @@ import { and, asc, eq, inArray, sql } from "drizzle-orm"; import { env } from "../env"; import { API_ERROR_CODES, errorResponse } from "../types"; import type { GraphDetailFileRecord, GraphFileRecord, GraphListItem, GraphRecentChatItem } from "../types/routes"; -import { type GraphRecord } from "./graph-access"; +import { selectGraphFields, type GraphRecord } from "./graph-access"; import { buildDeleteStepProgress, buildProcessStepProgress } from "./process-progress"; import { findActiveDeleteGraphFilesProgress, findProcessDescriptionProgress } from "./workflow-progress"; import type { GraphFileType } from "./graph-file-type"; @@ -37,6 +37,11 @@ export type UploadedFile = { mimeType: string; key: string; checksum?: string; + metadata?: string; + storageKind?: "internal" | "external"; + externalUrl?: string; + externalProvider?: string; + repositoryBindingId?: string; }; export type CreatedFileRecord = GraphFileRecord; export type GraphFileRow = Omit & { @@ -226,6 +231,17 @@ function buildTimeEstimate( }; } +function textArray(values: readonly string[]) { + if (values.length === 0) { + return sql`ARRAY[]::text[]`; + } + + return sql`ARRAY[${sql.join( + values.map((value) => sql`${value}`), + sql`, ` + )}]::text[]`; +} + function textList(values: readonly string[]) { return sql.join( values.map((value) => sql`${value}`), @@ -298,7 +314,8 @@ export const restoreGraphFileChangeFailure = async ( previousGraph: GraphRecord, addedFileIds: string[], uploadedKeys: string[], - processRunId?: string + processRunId?: string, + supersededFileIds: string[] = [] ) => { const failedDeletes = await cleanupUploadedKeys(uploadedKeys); @@ -311,6 +328,9 @@ export const restoreGraphFileChangeFailure = async ( if (addedFileIds.length > 0) { await tx.delete(filesTable).where(inArray(filesTable.id, addedFileIds)); } + if (supersededFileIds.length > 0) { + await tx.update(filesTable).set({ deleted: false }).where(inArray(filesTable.id, supersededFileIds)); + } await tx .update(graphTable) @@ -342,6 +362,108 @@ export const restoreGraphFileChangeFailure = async ( } }; +export async function commitGraphFileUploads(options: { + graph: GraphRecord; + uploadedFiles: UploadedFile[]; + supersedeRepositoryUrls?: string[]; +}): Promise<{ graph: GraphRecord; addedFiles: CreatedFileRecord[]; processRunId?: string; supersededFileIds: string[] }> { + const result = await db.transaction(async (tx) => { + const uploadedFileIds = options.uploadedFiles.map((file) => file.id); + const supersededFiles = + options.supersedeRepositoryUrls && options.supersedeRepositoryUrls.length > 0 + ? await tx + .update(filesTable) + .set({ deleted: true }) + .where( + and( + eq(filesTable.graphId, options.graph.id), + eq(filesTable.type, "code"), + eq(filesTable.deleted, false), + sql`${filesTable.id} <> ALL(${textArray(uploadedFileIds)})`, + sql`(${filesTable.metadata}::jsonb ->> 'repositoryUrl') = ANY(${textArray( + options.supersedeRepositoryUrls + )})` + ) + ) + .returning({ id: filesTable.id }) + : []; + + const insertedFiles = await tx + .insert(filesTable) + .values( + options.uploadedFiles.map((file) => ({ + id: file.id, + graphId: options.graph.id, + name: file.name, + size: file.size, + type: file.type, + mimeType: file.mimeType, + key: file.key, + storageKind: file.storageKind, + externalUrl: file.externalUrl, + externalProvider: file.externalProvider, + repositoryBindingId: file.repositoryBindingId, + checksum: file.checksum, + metadata: file.metadata, + })) + ) + .onConflictDoNothing() + .returning(selectFileFields); + + if (insertedFiles.length === 0) { + if (supersededFiles.length > 0) { + throw new Error("Repository snapshot did not insert replacement files"); + } + + return { + graph: options.graph, + addedFiles: insertedFiles, + processRunId: undefined, + supersededFileIds: [], + }; + } + + const [updatedGraph] = await tx + .update(graphTable) + .set({ state: "updating" }) + .where(eq(graphTable.id, options.graph.id)) + .returning(selectGraphFields); + + const [processRun] = await tx + .insert(processRunsTable) + .values({ + graphId: options.graph.id, + status: "pending", + }) + .returning({ id: processRunsTable.id }); + if (!processRun) { + throw new Error("Failed to create process run"); + } + + await tx.insert(processRunFilesTable).values( + insertedFiles.map((file) => ({ + processRunId: processRun.id, + fileId: file.id, + })) + ); + + return { + graph: updatedGraph ?? options.graph, + addedFiles: insertedFiles, + processRunId: processRun.id, + supersededFileIds: supersededFiles.map((file) => file.id), + }; + }); + + const addedKeys = new Set(result.addedFiles.map((file) => file.key)); + const skippedKeys = options.uploadedFiles.map((file) => file.key).filter((key) => !addedKeys.has(key)); + if (skippedKeys.length > 0) { + await cleanupUploadedKeys(skippedKeys); + } + + return result; +} + export function mapGraphError(statusFn: StatusFn, error: unknown) { if (!(error instanceof Error)) { return statusFn(500, errorResponse("Internal server error", API_ERROR_CODES.INTERNAL_SERVER_ERROR)); diff --git a/apps/api/src/lib/graph-upload-file-type.ts b/apps/api/src/lib/graph-upload-file-type.ts index c38ce20a..55829028 100644 --- a/apps/api/src/lib/graph-upload-file-type.ts +++ b/apps/api/src/lib/graph-upload-file-type.ts @@ -8,6 +8,9 @@ export type FileWithChecksum = { export type SupportedFileWithChecksum = FileWithChecksum & { type: GraphFileType; }; +type UploadModelFile = { + type: GraphFileType; +}; export type UploadFileTypeCheck = | { ok: true; files: SupportedFileWithChecksum[] } | { ok: false; fileName: string; message: string }; @@ -37,7 +40,7 @@ export function unsupportedUploadResponse(statusFn: StatusFn, check: Extract segment.trim()) + .filter(Boolean); + if (segments.length < 2) { + throw new RepositoryUrlError("validation", "Repository URL must include an owner and repository name"); + } + + if ((host === "github.com" || host === "bitbucket.org") && segments.length !== 2) { + throw new RepositoryUrlError("validation", "Repository URL must point to a repository root"); + } + + if (host === "gitlab.com" && segments.includes("-")) { + throw new RepositoryUrlError("validation", "Repository URL must point to a repository root"); + } + + const repositoryName = segments[segments.length - 1]!.replace(/\.git$/u, "") || segments[segments.length - 1]!; + parsed.hash = ""; + parsed.search = ""; + parsed.username = ""; + parsed.password = ""; + + const normalizedPath = `/${segments.join("/").replace(/\.git$/u, "")}.git`; + return { + url: `https://${host}${normalizedPath}`, + name: repositoryName, + }; +} + +type GitHubRepositoryParts = { + owner: string; + repo: string; +}; + +export type GitHubExternalCodeFile = { + provider: "github"; + rawUrl: string; + htmlUrl: string; + key: string; +}; + +function githubRepositoryParts(repositoryUrl: string): GitHubRepositoryParts | null { + const parsed = new URL(repositoryUrl); + if (parsed.protocol !== "https:" || parsed.hostname !== "github.com") { + return null; + } + + const segments = parsed.pathname + .split("/") + .map((segment) => segment.trim()) + .filter(Boolean); + if (segments.length !== 2) { + return null; + } + + return { + owner: segments[0]!, + repo: segments[1]!.replace(/\.git$/u, ""), + }; +} + +function encodePathSegments(value: string): string { + return value.split("/").map(encodeURIComponent).join("/"); +} + +export function buildGitHubExternalCodeFile(options: { + repositoryUrl: string; + commitSha: string; + path: string; +}): GitHubExternalCodeFile | null { + const parts = githubRepositoryParts(options.repositoryUrl); + if (!parts) { + return null; + } + + const owner = encodeURIComponent(parts.owner); + const repo = encodeURIComponent(parts.repo); + const commitSha = encodeURIComponent(options.commitSha); + const filePath = encodePathSegments(options.path); + const keyPath = options.path.replaceAll("\\", "/"); + + return { + provider: "github", + rawUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${commitSha}/${filePath}`, + htmlUrl: `https://github.com/${owner}/${repo}/blob/${commitSha}/${filePath}`, + key: `external:github:${parts.owner}/${parts.repo}@${options.commitSha}:${keyPath}`, + }; +} + +export async function loadRepositoryFromUrl(input: string): Promise { + const repository = normalizeRepositoryUrl(input); + const tempDir = await mkdtemp(path.join(tmpdir(), "kiwi-repository-")); + const repoPath = path.join(tempDir, "repo"); + + try { + await runGit(["clone", "--depth", "1", "--", repository.url, repoPath], tempDir); + const commitSha = (await runGit(["rev-parse", "HEAD"], repoPath)).trim(); + const listedFiles = (await runGit(["ls-files", "-z"], repoPath)).split("\0").filter(Boolean); + const files: RepositorySourceFile[] = []; + const repoRoot = path.resolve(repoPath); + let totalBytes = 0; + + for (const filePath of listedFiles) { + if (!shouldLoadCodePath(filePath)) { + continue; + } + + const absolutePath = path.resolve(repoPath, filePath); + if (!absolutePath.startsWith(`${repoRoot}${path.sep}`)) { + continue; + } + + const info = await stat(absolutePath); + if (!info.isFile() || info.size > MAX_REPOSITORY_CODE_FILE_BYTES) { + continue; + } + + if (files.length + 1 > MAX_REPOSITORY_CODE_FILES) { + throw new RepositoryUrlError("limit", "Repository contains too many supported code files"); + } + + if (totalBytes + info.size > MAX_REPOSITORY_CODE_BYTES) { + throw new RepositoryUrlError("limit", "Repository contains too much supported code"); + } + + const content = await readFile(absolutePath, "utf8"); + files.push({ path: filePath, content, size: info.size }); + totalBytes += info.size; + } + + return { + url: repository.url, + name: repository.name, + commitSha, + files, + }; + } finally { + await rm(tempDir, { recursive: true, force: true }); + } +} + +function shouldLoadCodePath(filePath: string): boolean { + const normalized = filePath.replaceAll("\\", "/"); + if (!isSupportedCodePath(normalized)) { + return false; + } + + return normalized.split("/").every((segment) => !SKIPPED_PATH_SEGMENTS.has(segment)); +} + +function runGit(args: string[], cwd: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn("git", args, { + cwd, + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + GIT_TERMINAL_PROMPT: "0", + }, + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let stdoutBytes = 0; + let stderrBytes = 0; + let settled = false; + + const finish = (callback: () => void) => { + if (settled) { + return; + } + + settled = true; + clearTimeout(timeout); + callback(); + }; + + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + finish(() => + reject( + new RepositoryUrlError("load", "Repository could not be loaded", { + cause: new Error("Repository git command timed out"), + }) + ) + ); + }, GIT_COMMAND_TIMEOUT_MS); + + child.stdout.on("data", (chunk: Buffer) => { + stdoutBytes += chunk.byteLength; + if (stdoutBytes <= MAX_GIT_OUTPUT_BYTES) { + stdout.push(chunk); + } + }); + child.stderr.on("data", (chunk: Buffer) => { + stderrBytes += chunk.byteLength; + if (stderrBytes <= MAX_GIT_OUTPUT_BYTES) { + stderr.push(chunk); + } + }); + child.on("error", (error) => + finish(() => reject(new RepositoryUrlError("load", "Repository could not be loaded", { cause: error }))) + ); + child.on("close", (code) => { + if (code === 0) { + finish(() => resolve(Buffer.concat(stdout).toString("utf8"))); + return; + } + + finish(() => { + const stderrText = Buffer.concat(stderr).toString("utf8").trim(); + reject( + new RepositoryUrlError("load", "Repository could not be loaded", { + cause: new Error(stderrText || "Repository git command failed"), + }) + ); + }); + }); + }); +} diff --git a/apps/api/src/lib/source-reference.ts b/apps/api/src/lib/source-reference.ts index 4723eb00..607f1841 100644 --- a/apps/api/src/lib/source-reference.ts +++ b/apps/api/src/lib/source-reference.ts @@ -1,6 +1,7 @@ import { and, eq, inArray } from "drizzle-orm"; import { db } from "@kiwi/db"; import { filesTable, sourcesTable, textUnitTable } from "@kiwi/db/tables/graph"; +import { currentSourcePredicate, visibleFilePredicate } from "@kiwi/db/source-validity"; import { getFile } from "@kiwi/files"; import type { SourceReferenceBatchSuccessData } from "@kiwi/contracts"; import { env } from "../env"; @@ -103,7 +104,14 @@ async function loadSourceReferenceRow(graphId: string, sourceId: string): Promis .from(sourcesTable) .innerJoin(textUnitTable, eq(textUnitTable.id, sourcesTable.textUnitId)) .innerJoin(filesTable, eq(filesTable.id, textUnitTable.fileId)) - .where(and(eq(sourcesTable.id, sourceId), eq(filesTable.graphId, graphId), eq(filesTable.deleted, false))) + .where( + and( + eq(sourcesTable.id, sourceId), + eq(filesTable.graphId, graphId), + currentSourcePredicate(sourcesTable), + visibleFilePredicate(filesTable) + ) + ) .limit(1); return row ? normalizeSourceReferenceRow(row) : null; @@ -116,7 +124,12 @@ async function loadSourceReferenceRows(graphId: string, sourceIds: string[]): Pr .innerJoin(textUnitTable, eq(textUnitTable.id, sourcesTable.textUnitId)) .innerJoin(filesTable, eq(filesTable.id, textUnitTable.fileId)) .where( - and(inArray(sourcesTable.id, sourceIds), eq(filesTable.graphId, graphId), eq(filesTable.deleted, false)) + and( + inArray(sourcesTable.id, sourceIds), + eq(filesTable.graphId, graphId), + currentSourcePredicate(sourcesTable), + visibleFilePredicate(filesTable) + ) ); return rows.map(normalizeSourceReferenceRow); diff --git a/apps/api/src/lib/team-chat.ts b/apps/api/src/lib/team-chat.ts index ea02f43a..f2468b54 100644 --- a/apps/api/src/lib/team-chat.ts +++ b/apps/api/src/lib/team-chat.ts @@ -14,6 +14,7 @@ import { createRequestInformationSection } from "@kiwi/ai/prompts/request-info.p import { db } from "@kiwi/db"; import type { ChatMessage } from "@kiwi/db/tables/chats"; import { filesTable, graphTable, sourcesTable, textUnitTable } from "@kiwi/db/tables/graph"; +import { currentSourcePredicate, visibleFilePredicate } from "@kiwi/db/source-validity"; import { generateText, stepCountIs, tool, type ModelMessage, type ToolSet } from "ai"; import { and, asc, eq, inArray, isNull } from "drizzle-orm"; import { z } from "zod"; @@ -511,7 +512,7 @@ export async function enrichTeamCitation( .from(sourcesTable) .innerJoin(textUnitTable, eq(textUnitTable.id, sourcesTable.textUnitId)) .innerJoin(filesTable, eq(filesTable.id, textUnitTable.fileId)) - .where(eq(sourcesTable.id, sourceId)) + .where(and(eq(sourcesTable.id, sourceId), currentSourcePredicate(sourcesTable), visibleFilePredicate(filesTable))) .limit(1); if (!row) { diff --git a/apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts b/apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts index a74112dc..b18bca20 100644 --- a/apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts +++ b/apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts @@ -3,7 +3,38 @@ import { Elysia } from "elysia"; const uploadedFiles: Array<{ graphId: string; fileId: string; name: string }> = []; let archiveExpansionMode: "success" | "limit" = "success"; -const workflowInputs: Array<{ graphId: string; fileIds: string[]; processRunId: string }> = []; +const workflowInputs: Array<{ + graphId: string; + fileIds: string[]; + processRunId: string; + code?: { kind: "repository" }; +}> = []; +const loadedUrls: string[] = []; +const insertedFileValues: Array<{ + name: string; + type: string; + mimeType: string; + key: string; + storageKind?: string; + externalProvider?: string; + externalUrl?: string; + metadata?: string; + checksum?: string; +}> = []; +const existingChecksumRows: Array<{ checksum: string }> = []; +let repositoryLoadMode: "success" | "limit-error" | "git-error" = "success"; + +class RepositoryUrlError extends Error { + constructor( + public readonly kind: "validation" | "limit" | "load", + message: string, + options?: { cause?: unknown } + ) { + super(message, options); + this.name = "RepositoryUrlError"; + } +} +let uploadModelMode: "success" | "missing" = "success"; const graphRow = { id: "graph-1", @@ -32,6 +63,7 @@ function graphFileRows( mimeType: string; key: string; checksum: string; + metadata?: string; }> ) { return values.map((file) => ({ @@ -43,6 +75,7 @@ function graphFileRows( mimeType: file.mimeType, key: file.key, checksum: file.checksum, + metadata: file.metadata, deleted: false, })); } @@ -63,6 +96,7 @@ function insertReturning(values: unknown) { mimeType: string; key: string; checksum: string; + metadata?: string; }> ); } @@ -86,7 +120,7 @@ const db = { }), select: () => ({ from: () => ({ - where: async () => [], + where: async () => existingChecksumRows, }), }), transaction: async (callback: (tx: typeof transactionDb) => Promise) => callback(transactionDb), @@ -96,7 +130,25 @@ const transactionDb = { insert: () => ({ values: (values: unknown) => ({ onConflictDoNothing: () => ({ - returning: () => insertReturning(values), + returning: () => { + if (Array.isArray(values)) { + insertedFileValues.push( + ...(values as Array<{ + name: string; + type: string; + mimeType: string; + key: string; + storageKind?: string; + externalProvider?: string; + externalUrl?: string; + metadata?: string; + checksum?: string; + }>) + ); + } + + return insertReturning(values); + }, }), returning: () => insertReturning(values), }), @@ -182,6 +234,79 @@ mock.module("../../lib/archive-upload", () => ({ }, })); +mock.module("../../lib/graph-upload-file-type", () => ({ + inferSupportedUploadedFiles: ( + files: Array<{ + file: File; + checksum: string; + }> + ) => ({ + ok: true, + files: files.map((file) => ({ + ...file, + type: file.file.name.endsWith(".ts") ? "code" : "text", + })), + }), + unsupportedUploadResponse: (statusFn: (code: number, body: unknown) => unknown) => + statusFn(415, { status: "error", code: "UNSUPPORTED_FILE_TYPE" }), + assertConfiguredUploadModels: async () => { + if (uploadModelMode === "missing") { + throw new Error("MODEL_NOT_CONFIGURED"); + } + }, +})); + +mock.module("../../lib/repository-url", () => ({ + MAX_REPOSITORY_URLS: 5, + RepositoryUrlError, + buildGitHubExternalCodeFile: ({ + repositoryUrl, + commitSha, + path, + }: { + repositoryUrl: string; + commitSha: string; + path: string; + }) => + repositoryUrl === "https://github.com/acme/widgets.git" + ? { + provider: "github", + rawUrl: `https://raw.githubusercontent.com/acme/widgets/${commitSha}/${path}`, + htmlUrl: `https://github.com/acme/widgets/blob/${commitSha}/${path}`, + key: `external:github:acme/widgets@${commitSha}:${path}`, + } + : null, + loadRepositoryFromUrl: async (url: string) => { + loadedUrls.push(url); + if (repositoryLoadMode === "limit-error") { + throw new RepositoryUrlError("limit", "Repository contains too many supported code files"); + } + if (repositoryLoadMode === "git-error") { + throw new RepositoryUrlError("load", "Repository could not be loaded", { + cause: new Error("fatal: could not read Username for 'https://github.com': terminal prompts disabled"), + }); + } + + return { + url: "https://github.com/acme/widgets.git", + name: "widgets", + commitSha: "commit-1", + files: [ + { + path: "src/index.ts", + content: "import { helper } from './helper';\nexport function main() { return helper(); }\n", + size: 75, + }, + { + path: "src/helper.ts", + content: "export function helper() { return 1; }\n", + size: 38, + }, + ], + }; + }, +})); + mock.module("../../lib/graph", () => ({ collectGraphClosure: async () => [], })); @@ -237,7 +362,12 @@ describe("graph route archive uploads", () => { beforeEach(() => { uploadedFiles.length = 0; workflowInputs.length = 0; + loadedUrls.length = 0; + insertedFileValues.length = 0; + existingChecksumRows.length = 0; archiveExpansionMode = "success"; + repositoryLoadMode = "success"; + uploadModelMode = "success"; }); test("creates one graph file and workflow input per extracted archive file", async () => { @@ -281,6 +411,32 @@ describe("graph route archive uploads", () => { ]); }); + test("stores direct source uploads as code files", async () => { + const form = new FormData(); + form.append("files", new File(["export const value = 1;\n"], "src/index.ts", { type: "text/plain" })); + + const response = await app().handle( + new Request("http://localhost/graphs/graph-1/files", { + method: "POST", + body: form, + }) + ); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.status).toBe("success"); + expect(insertedFileValues).toHaveLength(1); + expect(insertedFileValues[0].name).toBe("src/index.ts"); + expect(insertedFileValues[0].type).toBe("code"); + expect(workflowInputs).toEqual([ + { + graphId: "graph-1", + fileIds: body.data.addedFiles.map((file: { id: string }) => file.id), + processRunId: "process-run-1", + }, + ]); + }); + test("returns upload limit response when create archive expansion exceeds limits", async () => { archiveExpansionMode = "limit"; @@ -318,4 +474,182 @@ describe("graph route archive uploads", () => { expect(uploadedFiles).toEqual([]); expect(workflowInputs).toEqual([]); }); + + test("adds repository URL code files with metadata and code workflow input", async () => { + const response = await app().handle( + new Request("http://localhost/graphs/graph-1/urls", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + urls: [" https://github.com/acme/widgets ", "https://github.com/acme/widgets"], + }), + }) + ); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.status).toBe("success"); + expect(loadedUrls).toEqual(["https://github.com/acme/widgets"]); + expect(uploadedFiles).toEqual([]); + expect(insertedFileValues.map((file) => file.type)).toEqual(["code", "code"]); + expect(insertedFileValues.map((file) => file.mimeType)).toEqual(["text/plain", "text/plain"]); + expect(insertedFileValues.map((file) => file.storageKind)).toEqual(["external", "external"]); + expect(insertedFileValues.map((file) => file.externalProvider)).toEqual(["github", "github"]); + expect(insertedFileValues.map((file) => file.externalUrl)).toEqual([ + "https://raw.githubusercontent.com/acme/widgets/commit-1/src/index.ts", + "https://raw.githubusercontent.com/acme/widgets/commit-1/src/helper.ts", + ]); + expect(insertedFileValues.map((file) => file.key)).toEqual([ + "external:github:acme/widgets@commit-1:src/index.ts", + "external:github:acme/widgets@commit-1:src/helper.ts", + ]); + expect(insertedFileValues.map((file) => JSON.parse(file.metadata ?? "{}"))).toEqual([ + { + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-1", + path: "src/index.ts", + external: { + provider: "github", + rawUrl: "https://raw.githubusercontent.com/acme/widgets/commit-1/src/index.ts", + htmlUrl: "https://github.com/acme/widgets/blob/commit-1/src/index.ts", + }, + }, + { + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-1", + path: "src/helper.ts", + external: { + provider: "github", + rawUrl: "https://raw.githubusercontent.com/acme/widgets/commit-1/src/helper.ts", + htmlUrl: "https://github.com/acme/widgets/blob/commit-1/src/helper.ts", + }, + }, + ]); + expect(workflowInputs).toEqual([ + { + graphId: "graph-1", + fileIds: body.data.addedFiles.map((file: { id: string }) => file.id), + processRunId: "process-run-1", + code: { kind: "repository" }, + }, + ]); + }); + + test("validates repository URL models before upload or workflow side effects", async () => { + uploadModelMode = "missing"; + + const response = await app().handle( + new Request("http://localhost/graphs/graph-1/urls", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ urls: ["https://github.com/acme/widgets"] }), + }) + ); + const body = await response.json(); + + expect(response.status).toBe(400); + expect(body.status).toBe("error"); + expect(body.code).toBe("MODEL_NOT_CONFIGURED"); + expect(uploadedFiles).toEqual([]); + expect(insertedFileValues).toEqual([]); + expect(workflowInputs).toEqual([]); + }); + + test("creates latest repository snapshot files even when earlier checksums already exist", async () => { + const duplicateContent = "import { helper } from './helper';\nexport function main() { return helper(); }\n"; + const hashBuffer = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(duplicateContent)); + existingChecksumRows.push({ + checksum: [...new Uint8Array(hashBuffer)].map((byte) => byte.toString(16).padStart(2, "0")).join(""), + }); + + const response = await app().handle( + new Request("http://localhost/graphs/graph-1/urls", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ urls: ["https://github.com/acme/widgets"] }), + }) + ); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.status).toBe("success"); + expect(body.data.addedFiles.map((file: { name: string }) => file.name)).toEqual([ + "widgets/src/index.ts", + "widgets/src/helper.ts", + ]); + expect(uploadedFiles).toEqual([]); + expect(workflowInputs[0]?.code).toEqual({ kind: "repository" }); + }); + + test("still enqueues repository workflow when every URL file matches an older checksum", async () => { + for (const content of [ + "import { helper } from './helper';\nexport function main() { return helper(); }\n", + "export function helper() { return 1; }\n", + ]) { + const hashBuffer = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(content)); + existingChecksumRows.push({ + checksum: [...new Uint8Array(hashBuffer)].map((byte) => byte.toString(16).padStart(2, "0")).join(""), + }); + } + + const response = await app().handle( + new Request("http://localhost/graphs/graph-1/urls", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ urls: ["https://github.com/acme/widgets"] }), + }) + ); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.data.addedFiles.map((file: { name: string }) => file.name)).toEqual([ + "widgets/src/index.ts", + "widgets/src/helper.ts", + ]); + expect(body.data.workflowRunId).toBe("workflow-1"); + expect(uploadedFiles).toEqual([]); + expect(workflowInputs).toHaveLength(1); + }); + + test("maps repository loader limits to upload limit responses", async () => { + repositoryLoadMode = "limit-error"; + + const response = await app().handle( + new Request("http://localhost/graphs/graph-1/urls", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ urls: ["https://github.com/acme/widgets"] }), + }) + ); + const body = await response.json(); + + expect(response.status).toBe(413); + expect(body.code).toBe("UPLOAD_LIMIT_EXCEEDED"); + expect(uploadedFiles).toEqual([]); + expect(workflowInputs).toEqual([]); + }); + + test("sanitizes repository git failures in client responses", async () => { + repositoryLoadMode = "git-error"; + + const response = await app().handle( + new Request("http://localhost/graphs/graph-1/urls", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ urls: ["https://github.com/acme/widgets"] }), + }) + ); + const body = await response.json(); + const responseText = JSON.stringify(body); + + expect(response.status).toBe(400); + expect(body.code).toBe("UNSUPPORTED_FILE_TYPE"); + expect(body.message).toBe("Repository could not be loaded"); + expect(responseText).not.toContain("fatal:"); + expect(responseText).not.toContain("terminal prompts disabled"); + expect(uploadedFiles).toEqual([]); + expect(workflowInputs).toEqual([]); + }); }); diff --git a/apps/api/src/routes/connector-webhooks.ts b/apps/api/src/routes/connector-webhooks.ts new file mode 100644 index 00000000..f517f7c4 --- /dev/null +++ b/apps/api/src/routes/connector-webhooks.ts @@ -0,0 +1,209 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; +import { db } from "@kiwi/db"; +import { + connectorWebhookEventsTable, + connectorsTable, + repositoryGraphBindingsTable, + type ConnectorProvider, +} from "@kiwi/db/tables/connectors"; +import { syncRepositoryGraphSpec } from "@kiwi/worker/sync-repository-graph-spec"; +import { and, eq, or } from "drizzle-orm"; +import Elysia from "elysia"; +import { decryptSecret } from "../lib/connectors"; +import { ow } from "../openworkflow"; +import { API_ERROR_CODES, errorResponse, successResponse } from "../types"; + +type NormalizedPush = { + eventName: string; + deliveryId: string; + providerRepositoryId: string | null; + repositoryFullName: string | null; + branch: string | null; + commitSha: string | null; + deleted: boolean; +}; + +const ZERO_SHA = /^0+$/; + +function equalSecret(left: string, right: string) { + const leftBuffer = Buffer.from(left); + const rightBuffer = Buffer.from(right); + return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer); +} + +function verifyGitHubSignature(rawBody: string, signatureHeader: string | null, secret: string) { + if (!signatureHeader?.startsWith("sha256=")) { + return false; + } + const expected = `sha256=${createHmac("sha256", secret).update(rawBody).digest("hex")}`; + return equalSecret(signatureHeader, expected); +} + +function normalizeBranch(ref: unknown) { + if (typeof ref !== "string" || !ref.startsWith("refs/heads/")) { + return null; + } + return ref.slice("refs/heads/".length); +} + +function normalizeGitHub(payload: Record, eventName: string, deliveryId: string): NormalizedPush { + const repository = payload.repository && typeof payload.repository === "object" ? payload.repository : null; + const repo = repository as { id?: unknown; full_name?: unknown } | null; + const commitSha = typeof payload.after === "string" ? payload.after : null; + return { + eventName, + deliveryId, + providerRepositoryId: repo?.id === undefined ? null : String(repo.id), + repositoryFullName: typeof repo?.full_name === "string" ? repo.full_name : null, + branch: normalizeBranch(payload.ref), + commitSha, + deleted: !commitSha || ZERO_SHA.test(commitSha), + }; +} + +function normalizeGitLab(payload: Record, eventName: string, deliveryId: string): NormalizedPush { + const project = payload.project && typeof payload.project === "object" ? payload.project : null; + const repo = project as { id?: unknown; path_with_namespace?: unknown } | null; + const commitSha = typeof payload.after === "string" ? payload.after : null; + return { + eventName, + deliveryId, + providerRepositoryId: repo?.id === undefined ? null : String(repo.id), + repositoryFullName: typeof repo?.path_with_namespace === "string" ? repo.path_with_namespace : null, + branch: normalizeBranch(payload.ref), + commitSha, + deleted: !commitSha || ZERO_SHA.test(commitSha), + }; +} + +async function findVerifiedConnector(provider: ConnectorProvider, request: Request, rawBody: string) { + const connectors = await db + .select() + .from(connectorsTable) + .where(and(eq(connectorsTable.provider, provider), eq(connectorsTable.status, "active"))); + + for (const connector of connectors) { + const secret = decryptSecret(connector.webhookSecretEncrypted); + if (provider === "github") { + if (verifyGitHubSignature(rawBody, request.headers.get("x-hub-signature-256"), secret)) { + return connector; + } + } else if (equalSecret(request.headers.get("x-gitlab-token") ?? "", secret)) { + return connector; + } + } + + return null; +} + +async function handleWebhook(provider: ConnectorProvider, request: Request) { + const rawBody = await request.text(); + const connector = await findVerifiedConnector(provider, request, rawBody); + if (!connector) { + return new Response(JSON.stringify(errorResponse("Invalid webhook signature", API_ERROR_CODES.FORBIDDEN)), { + status: 403, + headers: { "Content-Type": "application/json" }, + }); + } + + const payload = JSON.parse(rawBody) as Record; + const eventName = + provider === "github" + ? request.headers.get("x-github-event") || "unknown" + : request.headers.get("x-gitlab-event") || String(payload.event_name ?? "unknown"); + const deliveryId = + provider === "github" + ? request.headers.get("x-github-delivery") || "missing" + : request.headers.get("x-gitlab-webhook-uuid") || `${String(payload.event_name ?? "event")}:${String(payload.after ?? Date.now())}`; + const normalized = provider === "github" ? normalizeGitHub(payload, eventName, deliveryId) : normalizeGitLab(payload, eventName, deliveryId); + const isPush = provider === "github" ? eventName === "push" : eventName === "Push Hook" || normalized.eventName === "push"; + + let bindings: Array = []; + if ( + isPush && + normalized.branch && + normalized.commitSha && + !normalized.deleted && + (normalized.providerRepositoryId || normalized.repositoryFullName) + ) { + const repositoryWhere = + normalized.providerRepositoryId && normalized.repositoryFullName + ? or( + eq(repositoryGraphBindingsTable.providerRepositoryId, normalized.providerRepositoryId), + eq(repositoryGraphBindingsTable.repositoryFullName, normalized.repositoryFullName) + ) + : normalized.providerRepositoryId + ? eq(repositoryGraphBindingsTable.providerRepositoryId, normalized.providerRepositoryId) + : eq(repositoryGraphBindingsTable.repositoryFullName, normalized.repositoryFullName!); + bindings = await db + .select() + .from(repositoryGraphBindingsTable) + .where( + and( + eq(repositoryGraphBindingsTable.provider, provider), + eq(repositoryGraphBindingsTable.branch, normalized.branch), + eq(repositoryGraphBindingsTable.webhookEnabled, true), + repositoryWhere + ) + ); + } + + const status = !isPush || normalized.deleted || bindings.length === 0 ? "ignored" : "enqueued"; + const [ledger] = await db + .insert(connectorWebhookEventsTable) + .values({ + connectorId: connector.id, + provider, + deliveryId, + eventName, + providerRepositoryId: normalized.providerRepositoryId, + branch: normalized.branch, + commitSha: normalized.commitSha, + status, + }) + .onConflictDoNothing({ + target: [ + connectorWebhookEventsTable.connectorId, + connectorWebhookEventsTable.provider, + connectorWebhookEventsTable.deliveryId, + ], + }) + .returning(); + + if (!ledger) { + return new Response(JSON.stringify(successResponse({ status: "duplicate" })), { + status: 202, + headers: { "Content-Type": "application/json" }, + }); + } + + if (status === "enqueued" && normalized.commitSha) { + for (const binding of bindings) { + await db + .update(repositoryGraphBindingsTable) + .set({ lastSeenCommitSha: normalized.commitSha, syncStatus: "pending", syncErrorCode: null }) + .where(eq(repositoryGraphBindingsTable.id, binding.id)); + await ow.runWorkflow(syncRepositoryGraphSpec, { + bindingId: binding.id, + reason: "webhook", + commitSha: normalized.commitSha, + deliveryId, + }); + } + } + + return new Response(JSON.stringify(successResponse({ status, enqueued: status === "enqueued" ? bindings.length : 0 })), { + status: 202, + headers: { "Content-Type": "application/json" }, + }); +} + +export const connectorWebhookRoute = new Elysia().post("/connectors/webhooks/:provider", async ({ params, request }) => { + if (params.provider !== "github" && params.provider !== "gitlab") { + return new Response(JSON.stringify(errorResponse("Unsupported provider", API_ERROR_CODES.INVALID_CHAT_REQUEST)), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + } + return handleWebhook(params.provider, request); +}); diff --git a/apps/api/src/routes/connectors.ts b/apps/api/src/routes/connectors.ts new file mode 100644 index 00000000..d1225457 --- /dev/null +++ b/apps/api/src/routes/connectors.ts @@ -0,0 +1,534 @@ +import { ulid } from "ulid"; +import { db } from "@kiwi/db"; +import { + connectorInstallationsTable, + connectorsTable, + repositoryGraphBindingsTable, + type ConnectorProvider, +} from "@kiwi/db/tables/connectors"; +import { graphTable } from "@kiwi/db/tables/graph"; +import { syncRepositoryGraphSpec } from "@kiwi/worker/sync-repository-graph-spec"; +import { Result } from "better-result"; +import { and, asc, eq } from "drizzle-orm"; +import Elysia from "elysia"; +import z from "zod"; +import { assertCanCreateTeamGraph, assertCanCreateTopLevelGraph } from "../lib/graph-access"; +import { + assertCanSyncBinding, + assertCanUseInstallation, + assertCanViewBinding, + requireActiveConnector, +} from "../lib/connector-access"; +import { + createManifestUrl, + encryptCredentials, + encryptSecret, + exchangeGitHubManifestCode, + listProviderBranches, + listProviderRepositories, + signConnectorState, + toPublicConnector, + toPublicInstallation, + verifyConnectorState, +} from "../lib/connectors"; +import { ow } from "../openworkflow"; +import { authMiddleware, type AuthUser } from "../middleware/auth"; +import { API_ERROR_CODES, errorResponse, successResponse } from "../types"; + +const githubManifestStartSchema = z.object({ + name: z.string().trim().min(1), +}); + +const gitLabCreateSchema = z.object({ + name: z.string().trim().min(1), + slug: z.string().trim().min(1), + baseUrl: z.string().trim().url(), + clientId: z.string().trim().min(1), + clientSecret: z.string().trim().min(1), + webhookSecret: z.string().trim().min(1), +}); + +const patchConnectorSchema = z.object({ + name: z.string().trim().min(1).optional(), + status: z.enum(["active", "disabled"]).optional(), + webhookSecret: z.string().trim().min(1).optional(), +}); + +const connectQuerySchema = z.object({ + organizationId: z.string().trim().min(1).optional(), + teamId: z.string().trim().min(1).optional(), +}); + +const installCallbackQuerySchema = z.object({ + state: z.string().trim().min(1), + installation_id: z.string().trim().min(1), + setup_action: z.string().trim().optional(), +}); + +const repositoryQuerySchema = z.object({ + installationId: z.string().trim().min(1), +}); + +const connectorOwnerSchema = z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("organization") }), + z.object({ kind: z.literal("team"), teamId: z.string().trim().min(1) }), +]); + +const createRepositoryGraphSchema = z.object({ + connectorInstallationId: z.string().trim().min(1), + repositoryId: z.string().trim().min(1), + repositoryFullName: z.string().trim().min(1), + repositoryHtmlUrl: z.string().trim().url(), + branch: z.string().trim().min(1), + name: z.string().trim().min(1), + owner: connectorOwnerSchema, +}); + +type RouteStatus = (code: number, body: unknown) => unknown; + +function mapConnectorError(status: RouteStatus, error: unknown) { + if (!(error instanceof Error)) { + return status(500, errorResponse("Internal server error", API_ERROR_CODES.INTERNAL_SERVER_ERROR)); + } + if (error.message === API_ERROR_CODES.UNAUTHORIZED) { + return status(401, errorResponse("Unauthorized", API_ERROR_CODES.UNAUTHORIZED)); + } + if (error.message === API_ERROR_CODES.FORBIDDEN) { + return status(403, errorResponse("Forbidden", API_ERROR_CODES.FORBIDDEN)); + } + if (error.message === API_ERROR_CODES.GRAPH_NOT_FOUND) { + return status(404, errorResponse("Not found", API_ERROR_CODES.GRAPH_NOT_FOUND)); + } + return status(400, errorResponse(error.message || "Invalid connector request", API_ERROR_CODES.INVALID_CHAT_REQUEST)); +} + +async function runConnectorAction(options: { + user: AuthUser | null | undefined; + status: RouteStatus; + action: (user: AuthUser) => Promise; +}) { + if (!options.user) { + return options.status(401, errorResponse("Unauthorized", API_ERROR_CODES.UNAUTHORIZED)); + } + const result = await Result.tryPromise(async () => options.action(options.user!)); + if (result.isErr()) { + return mapConnectorError(options.status, result.error); + } + return options.status(200, successResponse(result.value)); +} + +function assertSystemAdmin(user: AuthUser) { + if (!user.isSystemAdmin) { + throw new Error(API_ERROR_CODES.FORBIDDEN); + } +} + +function repositoryMatches(repository: { id: string; fullName: string }, requestedId: string) { + return repository.id === requestedId || repository.fullName === requestedId; +} + +function toBindingResponse(binding: typeof repositoryGraphBindingsTable.$inferSelect) { + return { + id: binding.id, + graphId: binding.graphId, + connectorInstallationId: binding.connectorInstallationId, + provider: binding.provider as ConnectorProvider, + providerRepositoryId: binding.providerRepositoryId, + repositoryFullName: binding.repositoryFullName, + repositoryHtmlUrl: binding.repositoryHtmlUrl, + branch: binding.branch, + lastSeenCommitSha: binding.lastSeenCommitSha, + lastSyncedCommitSha: binding.lastSyncedCommitSha, + syncStatus: binding.syncStatus, + syncErrorCode: binding.syncErrorCode, + webhookEnabled: binding.webhookEnabled, + createdAt: binding.createdAt?.toISOString() ?? null, + updatedAt: binding.updatedAt?.toISOString() ?? null, + }; +} + +export const connectorRoute = new Elysia({ prefix: "/connectors" }) + .use(authMiddleware) + .get("/", async ({ status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + const rows = await db.select().from(connectorsTable).orderBy(asc(connectorsTable.name)); + return rows.filter((row) => currentUser.isSystemAdmin || row.status === "active").map(toPublicConnector); + }, + }) + ) + .post( + "/github/manifest/start", + async ({ body, status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + assertSystemAdmin(currentUser); + const state = signConnectorState({ purpose: "github-manifest", userId: currentUser.id }); + return { state, manifestUrl: createManifestUrl(state, body.name.trim()) }; + }, + }), + { body: githubManifestStartSchema } + ) + .get( + "/github/manifest/callback", + async ({ query, status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + assertSystemAdmin(currentUser); + if (!verifyConnectorState(query.state, "github-manifest", currentUser.id)) { + throw new Error(API_ERROR_CODES.FORBIDDEN); + } + const app = await exchangeGitHubManifestCode(query.code); + const slug = app.slug ?? `github-${String(app.id)}`; + const [connector] = await db + .insert(connectorsTable) + .values({ + provider: "github", + name: app.name, + slug, + appId: String(app.id), + appSlug: slug, + clientId: app.client_id ?? null, + encryptedCredentials: encryptCredentials({ + provider: "github", + appId: String(app.id), + privateKeyPem: app.pem, + clientId: app.client_id, + clientSecret: app.client_secret, + }), + webhookSecretEncrypted: encryptSecret(app.webhook_secret ?? ulid()), + createdByUserId: currentUser.id, + }) + .returning(); + return toPublicConnector(connector); + }, + }), + { query: z.object({ code: z.string().trim().min(1), state: z.string().trim().min(1) }) } + ) + .post( + "/gitlab", + async ({ body, status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + assertSystemAdmin(currentUser); + const [connector] = await db + .insert(connectorsTable) + .values({ + provider: "gitlab", + name: body.name, + slug: body.slug, + status: "disabled", + appId: body.clientId, + clientId: body.clientId, + encryptedCredentials: encryptCredentials({ + provider: "gitlab", + baseUrl: body.baseUrl, + clientId: body.clientId, + clientSecret: body.clientSecret, + }), + webhookSecretEncrypted: encryptSecret(body.webhookSecret), + createdByUserId: currentUser.id, + }) + .returning(); + return toPublicConnector(connector); + }, + }), + { body: gitLabCreateSchema } + ) + .patch( + "/:id", + async ({ params, body, status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + assertSystemAdmin(currentUser); + const [connector] = await db + .update(connectorsTable) + .set({ + ...(body.name ? { name: body.name } : {}), + ...(body.status ? { status: body.status } : {}), + ...(body.webhookSecret ? { webhookSecretEncrypted: encryptSecret(body.webhookSecret) } : {}), + }) + .where(eq(connectorsTable.id, params.id)) + .returning(); + if (!connector) { + throw new Error(API_ERROR_CODES.GRAPH_NOT_FOUND); + } + return toPublicConnector(connector); + }, + }), + { body: patchConnectorSchema } + ) + .get( + "/:id/connect", + async ({ params, query, status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + const connector = await requireActiveConnector(params.id); + if (query.teamId) { + await assertCanCreateTeamGraph(currentUser, query.teamId); + } else { + await assertCanCreateTopLevelGraph(currentUser); + } + const state = signConnectorState({ + purpose: connector.provider === "github" ? "github-installation" : "gitlab-oauth", + userId: currentUser.id, + connectorId: connector.id, + organizationId: query.organizationId ?? currentUser.activeOrganizationId ?? undefined, + teamId: query.teamId, + }); + if (connector.provider === "github") { + if (!connector.appSlug) { + throw new Error(API_ERROR_CODES.FORBIDDEN); + } + return { + redirectUrl: `https://github.com/apps/${connector.appSlug}/installations/new?state=${encodeURIComponent(state)}`, + }; + } + throw new Error("GitLab connector installations are disabled until OAuth flow support lands."); + }, + }), + { query: connectQuerySchema } + ) + .get( + "/github/install/callback", + async ({ query, status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + const state = verifyConnectorState(query.state, "github-installation", currentUser.id); + if (!state?.connectorId) { + throw new Error(API_ERROR_CODES.FORBIDDEN); + } + const connector = await requireActiveConnector(state.connectorId, "github"); + if (state.teamId) { + await assertCanCreateTeamGraph(currentUser, state.teamId); + } else { + await assertCanCreateTopLevelGraph(currentUser); + } + const [installation] = await db + .insert(connectorInstallationsTable) + .values({ + connectorId: connector.id, + provider: "github", + providerInstallationId: query.installation_id, + providerAccountLogin: query.installation_id, + providerAccountType: "organization", + organizationId: state.organizationId ?? currentUser.activeOrganizationId, + teamId: state.teamId ?? null, + installedByUserId: currentUser.id, + repositorySelection: query.setup_action ?? "unknown", + }) + .onConflictDoUpdate({ + target: [ + connectorInstallationsTable.connectorId, + connectorInstallationsTable.providerInstallationId, + connectorInstallationsTable.organizationId, + connectorInstallationsTable.teamId, + ], + set: { status: "active", installedByUserId: currentUser.id }, + }) + .returning(); + return toPublicInstallation(installation); + }, + }), + { query: installCallbackQuerySchema } + ) + .get("/:id/installations", async ({ params, status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + await requireActiveConnector(params.id); + const rows = await db + .select() + .from(connectorInstallationsTable) + .where( + and( + eq(connectorInstallationsTable.connectorId, params.id), + eq(connectorInstallationsTable.status, "active") + ) + ) + .orderBy(asc(connectorInstallationsTable.providerAccountLogin)); + const visible = []; + for (const row of rows) { + const allowed = await Result.tryPromise(async () => assertCanUseInstallation(currentUser, row.id)); + if (!allowed.isErr()) { + visible.push(toPublicInstallation(row)); + } + } + return visible; + }, + }) + ) + .get( + "/:id/repositories", + async ({ params, query, status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + const installation = await assertCanUseInstallation(currentUser, query.installationId); + const connector = await requireActiveConnector(params.id, installation.provider as ConnectorProvider); + const repositories = await listProviderRepositories(connector, installation); + return repositories.map((repository) => ({ + id: repository.id, + provider: repository.provider, + fullName: repository.fullName, + name: repository.name, + htmlUrl: repository.htmlUrl, + defaultBranch: repository.defaultBranch, + private: repository.private, + })); + }, + }), + { query: repositoryQuerySchema } + ) + .get( + "/:id/repositories/:repositoryId/branches", + async ({ params, query, status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + const installation = await assertCanUseInstallation(currentUser, query.installationId); + const connector = await requireActiveConnector(params.id, installation.provider as ConnectorProvider); + const branches = await listProviderBranches(connector, installation, params.repositoryId); + return branches.map((branch) => ({ name: branch.name, commitSha: branch.commitSha })); + }, + }), + { query: repositoryQuerySchema } + ) + .post( + "/:id/repository-graphs", + async ({ params, body, status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + const installation = await assertCanUseInstallation(currentUser, body.connectorInstallationId); + if (body.owner.kind === "team") { + if (installation.teamId !== body.owner.teamId) { + throw new Error(API_ERROR_CODES.FORBIDDEN); + } + await assertCanCreateTeamGraph(currentUser, body.owner.teamId); + } else { + if (installation.teamId !== null) { + throw new Error(API_ERROR_CODES.FORBIDDEN); + } + await assertCanCreateTopLevelGraph(currentUser); + } + + const connector = await requireActiveConnector(params.id, installation.provider as ConnectorProvider); + const repositories = await listProviderRepositories(connector, installation); + const repository = repositories.find((candidate) => repositoryMatches(candidate, body.repositoryId)); + if (!repository) { + throw new Error(API_ERROR_CODES.GRAPH_NOT_FOUND); + } + + const branches = await listProviderBranches(connector, installation, repository.id); + const branch = branches.find((candidate) => candidate.name === body.branch); + if (!branch) { + throw new Error(API_ERROR_CODES.GRAPH_NOT_FOUND); + } + + const created = await db.transaction(async (tx) => { + const [graph] = await tx + .insert(graphTable) + .values({ + organizationId: installation.organizationId, + teamId: body.owner.kind === "team" ? body.owner.teamId : null, + name: body.name, + description: null, + state: "updating", + type: "code", + }) + .returning(); + + const [binding] = await tx + .insert(repositoryGraphBindingsTable) + .values({ + graphId: graph.id, + connectorInstallationId: installation.id, + provider: connector.provider, + providerRepositoryId: repository.id, + repositoryFullName: repository.fullName, + repositoryHtmlUrl: repository.htmlUrl, + branch: branch.name, + lastSeenCommitSha: branch.commitSha, + syncStatus: "pending", + }) + .returning(); + + return { graph, binding }; + }); + + try { + const handle = await ow.runWorkflow(syncRepositoryGraphSpec, { + bindingId: created.binding.id, + reason: "initial", + commitSha: branch.commitSha, + }); + return { + graph: created.graph, + binding: toBindingResponse(created.binding), + workflowRunId: handle.workflowRun.id, + }; + } catch (error) { + await Promise.all([ + db + .update(repositoryGraphBindingsTable) + .set({ syncStatus: "failed", syncErrorCode: "enqueue_failed" }) + .where(eq(repositoryGraphBindingsTable.id, created.binding.id)), + db.update(graphTable).set({ state: "ready" }).where(eq(graphTable.id, created.graph.id)), + ]); + throw error; + } + }, + }), + { body: createRepositoryGraphSchema } + ); + +export const repositoryGraphBindingRoute = new Elysia({ prefix: "/repository-graph-bindings" }) + .use(authMiddleware) + .get("/:id", async ({ params, status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + const { binding } = await assertCanViewBinding(currentUser, params.id); + return toBindingResponse(binding); + }, + }) + ) + .post("/:id/sync", async ({ params, status, user }) => + runConnectorAction({ + user, + status, + action: async (currentUser) => { + const { binding } = await assertCanSyncBinding(currentUser, params.id); + const [updatedBinding] = await db + .update(repositoryGraphBindingsTable) + .set({ syncStatus: "pending", syncErrorCode: null }) + .where(eq(repositoryGraphBindingsTable.id, binding.id)) + .returning(); + const handle = await ow.runWorkflow(syncRepositoryGraphSpec, { bindingId: binding.id, reason: "manual" }); + return { + binding: toBindingResponse(updatedBinding ?? binding), + workflowRunId: handle.workflowRun.id, + }; + }, + }) + ); diff --git a/apps/api/src/routes/graph.ts b/apps/api/src/routes/graph.ts index dae2e89a..863eeec4 100644 --- a/apps/api/src/routes/graph.ts +++ b/apps/api/src/routes/graph.ts @@ -7,6 +7,7 @@ import { db } from "@kiwi/db"; import { filesTable, graphTable, processRunFilesTable, processRunsTable } from "@kiwi/db/tables/graph"; import { teamTable } from "@kiwi/db/tables/auth"; import { deleteFile, listFiles, putGraphFile } from "@kiwi/files"; +import { serializeCodeFileMetadata } from "@kiwi/graph/code/metadata"; import { error as logError } from "@kiwi/logger"; import { deleteGraphFilesSpec } from "@kiwi/worker/delete-graph-files-spec"; import { processFilesSpec } from "@kiwi/worker/process-files-spec"; @@ -15,6 +16,13 @@ import { chunk } from "../lib/array"; import { expandArchiveUploadFiles } from "../lib/archive-upload"; import { collectGraphClosure } from "../lib/graph"; import { listAccessibleGraphs } from "../lib/graph-list"; +import { + buildGitHubExternalCodeFile, + loadRepositoryFromUrl, + MAX_REPOSITORY_URLS, + RepositoryUrlError, + type LoadedRepository, +} from "../lib/repository-url"; import { cancelActiveFileProcessingWorkflowRuns, cancelActiveGraphWorkflowRuns } from "../lib/workflow-cancellation"; import { assertCanCreateTopLevelGraph, @@ -30,6 +38,7 @@ import { import { cleanupUploadedKeys, cleanupFailedGraphCreation, + commitGraphFileUploads, mapGraphError, toGraphFileRecord, assertConfiguredUploadModels, @@ -64,6 +73,12 @@ type NewGraphOwner = graphId: string; }; +type RepositoryUploadSource = { + repository: LoadedRepository; + file: LoadedRepository["files"][number]; + checksum: string; +}; + function archiveUploadResponse( statusFn: (code: number, body: unknown) => unknown, expanded: { ok: false; kind: "unsupported" | "limit"; fileName: string; message: string } @@ -91,6 +106,41 @@ async function getGraphOwnerModelOrganizationId(owner: NewGraphOwner) { return rootOwner.organizationId; } +async function contentChecksum(content: string): Promise { + const hashBuffer = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(content)); + + return [...new Uint8Array(hashBuffer)].map((byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +function getRepositoryUrlError(error: unknown): RepositoryUrlError | undefined { + if (error instanceof RepositoryUrlError) { + return error; + } + + if (error instanceof Error && error.cause instanceof RepositoryUrlError) { + return error.cause; + } + + return undefined; +} + +function repositoryUrlErrorResponse(statusFn: (code: number, body: unknown) => unknown, error: unknown) { + const repositoryError = getRepositoryUrlError(error); + if (!repositoryError) { + return statusFn( + 400, + errorResponse("Repository could not be loaded", API_ERROR_CODES.UNSUPPORTED_FILE_TYPE) + ); + } + + if (repositoryError.kind === "limit") { + return statusFn(413, errorResponse(repositoryError.message, API_ERROR_CODES.UPLOAD_LIMIT_EXCEEDED)); + } + + const message = repositoryError.kind === "validation" ? repositoryError.message : "Repository could not be loaded"; + return statusFn(400, errorResponse(message, API_ERROR_CODES.UNSUPPORTED_FILE_TYPE)); +} + export const graphRoute = new Elysia({ prefix: "/graphs" }) .use(authMiddleware) .get( @@ -528,6 +578,272 @@ export const graphRoute = new Elysia({ prefix: "/graphs" }) }), } ) + .post( + "/:id/urls", + async ({ body, params, user, status }) => { + if (!user) { + return status(401, { + status: "error", + message: "Unauthorized", + code: "UNAUTHORIZED", + }); + } + + const accessResult = await Result.tryPromise(async () => assertCanManageGraphFiles(user, params.id)); + if (accessResult.isErr()) { + return mapGraphError(status, accessResult.error); + } + const existingGraph = accessResult.value; + + const urls = [...new Set(body.urls.map((url) => url.trim()).filter(Boolean))]; + if (urls.length === 0) { + return status(400, { + status: "error", + message: "No changes requested", + code: API_ERROR_CODES.NO_CHANGES, + }); + } + + if (urls.length > MAX_REPOSITORY_URLS) { + return status( + 413, + errorResponse( + `At most ${MAX_REPOSITORY_URLS} repository URLs can be processed at once`, + API_ERROR_CODES.UPLOAD_LIMIT_EXCEEDED + ) + ); + } + + const repositoriesResult = await Result.tryPromise(async () => { + const repositories: LoadedRepository[] = []; + for (const url of urls) { + repositories.push(await loadRepositoryFromUrl(url)); + } + + return repositories; + }); + if (repositoriesResult.isErr()) { + return repositoryUrlErrorResponse(status, repositoriesResult.error); + } + const repositories = repositoriesResult.value; + + const seenSnapshotChecksums = new Set(); + const repositorySources: RepositoryUploadSource[] = []; + + for (const repository of repositories) { + for (const file of repository.files) { + const checksum = await contentChecksum(file.content); + if (seenSnapshotChecksums.has(checksum)) { + continue; + } + + seenSnapshotChecksums.add(checksum); + repositorySources.push({ repository, file, checksum }); + } + } + + if (repositorySources.length === 0) { + return status(200, { + status: "success", + data: { + graph: existingGraph, + addedFiles: [], + workflowRunId: null, + }, + }); + } + + const repositoryModelResult = await Result.tryPromise(async () => { + await assertConfiguredUploadModels({ + organizationId: await getGraphOwnerModelOrganizationId({ + ownerMode: "graph", + graphId: existingGraph.id, + }), + files: repositorySources.map(() => ({ type: "code" as const })), + secret: env.AUTH_SECRET, + }); + }); + if (repositoryModelResult.isErr()) { + const error = + repositoryModelResult.error instanceof Error && repositoryModelResult.error.cause instanceof Error + ? repositoryModelResult.error.cause + : repositoryModelResult.error; + return mapGraphError(status, error); + } + + const uploadedFiles: UploadedFile[] = []; + try { + for (const source of repositorySources) { + const fileId = ulid(); + const name = `${source.repository.name}/${source.file.path}`; + const external = buildGitHubExternalCodeFile({ + repositoryUrl: source.repository.url, + commitSha: source.repository.commitSha, + path: source.file.path, + }); + + if (external) { + uploadedFiles.push({ + id: fileId, + name, + size: source.file.size, + type: "code", + mimeType: "text/plain", + key: external.key, + storageKind: "external", + externalProvider: external.provider, + externalUrl: external.rawUrl, + checksum: source.checksum, + metadata: serializeCodeFileMetadata({ + repositoryUrl: source.repository.url, + repositoryName: source.repository.name, + commitSha: source.repository.commitSha, + path: source.file.path, + external: { + provider: external.provider, + rawUrl: external.rawUrl, + htmlUrl: external.htmlUrl, + }, + }), + }); + continue; + } + + const upload = await putGraphFile( + existingGraph.id, + fileId, + name, + source.file.content, + env.S3_BUCKET + ); + + uploadedFiles.push({ + id: fileId, + name, + size: source.file.size, + type: "code", + mimeType: "text/plain", + key: upload.key, + storageKind: "internal", + checksum: source.checksum, + metadata: serializeCodeFileMetadata({ + repositoryUrl: source.repository.url, + repositoryName: source.repository.name, + commitSha: source.repository.commitSha, + path: source.file.path, + }), + }); + } + } catch (uploadError) { + const failedDeletes = await cleanupUploadedKeys( + uploadedFiles.filter((file) => file.storageKind !== "external").map((file) => file.key) + ); + + logError("graph repository URL add failed during file upload", { + graphId: existingGraph.id, + uploadedKeyCount: uploadedFiles.length, + failedS3CleanupCount: failedDeletes, + error: uploadError, + }); + + return status(500, { + status: "error", + message: "Internal server error", + code: "INTERNAL_SERVER_ERROR", + }); + } + + let result: Awaited>; + + try { + result = await commitGraphFileUploads({ + graph: existingGraph, + uploadedFiles, + supersedeRepositoryUrls: repositories.map((repository) => repository.url), + }); + } catch (dbPatchError) { + const failedDeletes = await cleanupUploadedKeys( + uploadedFiles.filter((file) => file.storageKind !== "external").map((file) => file.key) + ); + + logError("graph repository URL add failed during database update", { + graphId: existingGraph.id, + uploadedKeyCount: uploadedFiles.length, + failedS3CleanupCount: failedDeletes, + error: dbPatchError, + }); + + return status(500, { + status: "error", + message: "Internal server error", + code: "INTERNAL_SERVER_ERROR", + }); + } + + if (result.addedFiles.length === 0) { + return status(200, { + status: "success", + data: { + graph: result.graph, + addedFiles: result.addedFiles, + workflowRunId: null, + }, + }); + } + + try { + if (!result.processRunId) { + throw new Error("Missing process run id"); + } + + const handle = await ow.runWorkflow(processFilesSpec, { + graphId: existingGraph.id, + fileIds: result.addedFiles.map((file) => file.id), + processRunId: result.processRunId, + code: { kind: "repository" }, + }); + + return status(200, { + status: "success", + data: { + graph: result.graph, + addedFiles: result.addedFiles, + workflowRunId: handle.workflowRun.id, + }, + }); + } catch (enqueueError) { + await restoreGraphFileChangeFailure( + existingGraph.id, + existingGraph, + result.addedFiles.map((file) => file.id), + uploadedFiles.filter((file) => file.storageKind !== "external").map((file) => file.key), + result.processRunId, + result.supersededFileIds + ); + + logError("graph repository URL add failed during workflow enqueue", { + graphId: existingGraph.id, + uploadedKeyCount: uploadedFiles.length, + addedFileCount: result.addedFiles.length, + error: enqueueError, + }); + + return status(500, { + status: "error", + message: "Internal server error", + code: "INTERNAL_SERVER_ERROR", + }); + } + }, + { + params: t.Object({ + id: t.String(), + }), + body: t.Object({ + urls: t.Array(t.String()), + }), + } + ) .post( "/:id/files", async ({ body, params, user, status }) => { @@ -649,77 +965,10 @@ export const graphRoute = new Elysia({ prefix: "/graphs" }) }); } - let graph = existingGraph; - let addedFiles: CreatedFileRecord[] = []; - let processRunId: string | undefined; + let result: Awaited>; try { - const result = await db.transaction(async (tx) => { - const insertedFiles = await tx - .insert(filesTable) - .values( - uploadedFiles.map((file) => ({ - id: file.id, - graphId: existingGraph.id, - name: file.name, - size: file.size, - type: file.type, - mimeType: file.mimeType, - key: file.key, - checksum: file.checksum, - })) - ) - .onConflictDoNothing() - .returning(selectFileFields); - - if (insertedFiles.length === 0) { - return { - graph: existingGraph, - addedFiles: insertedFiles, - processRunId: undefined, - }; - } - - const [updatedGraph] = await tx - .update(graphTable) - .set({ state: "updating" }) - .where(eq(graphTable.id, existingGraph.id)) - .returning(selectGraphFields); - - const [processRun] = await tx - .insert(processRunsTable) - .values({ - graphId: existingGraph.id, - status: "pending", - }) - .returning({ id: processRunsTable.id }); - if (!processRun) { - throw new Error("Failed to create process run"); - } - - await tx.insert(processRunFilesTable).values( - insertedFiles.map((file) => ({ - processRunId: processRun.id, - fileId: file.id, - })) - ); - - return { - graph: updatedGraph ?? existingGraph, - addedFiles: insertedFiles, - processRunId: processRun.id, - }; - }); - - graph = result.graph; - addedFiles = result.addedFiles; - processRunId = result.processRunId; - - const addedKeys = new Set(addedFiles.map((file) => file.key)); - const skippedKeys = uploadedFiles.map((file) => file.key).filter((key) => !addedKeys.has(key)); - if (skippedKeys.length > 0) { - await cleanupUploadedKeys(skippedKeys); - } + result = await commitGraphFileUploads({ graph: existingGraph, uploadedFiles }); } catch (dbPatchError) { const failedDeletes = await cleanupUploadedKeys(uploadedFiles.map((file) => file.key)); @@ -737,33 +986,33 @@ export const graphRoute = new Elysia({ prefix: "/graphs" }) }); } - if (addedFiles.length === 0) { + if (result.addedFiles.length === 0) { return status(200, { status: "success", data: { - graph, - addedFiles, + graph: result.graph, + addedFiles: result.addedFiles, workflowRunId: null, }, }); } try { - if (!processRunId) { + if (!result.processRunId) { throw new Error("Missing process run id"); } const handle = await ow.runWorkflow(processFilesSpec, { graphId: existingGraph.id, - fileIds: addedFiles.map((file) => file.id), - processRunId, + fileIds: result.addedFiles.map((file) => file.id), + processRunId: result.processRunId, }); return status(200, { status: "success", data: { - graph, - addedFiles, + graph: result.graph, + addedFiles: result.addedFiles, workflowRunId: handle.workflowRun.id, }, }); @@ -771,15 +1020,15 @@ export const graphRoute = new Elysia({ prefix: "/graphs" }) await restoreGraphFileChangeFailure( existingGraph.id, existingGraph, - addedFiles.map((file) => file.id), + result.addedFiles.map((file) => file.id), uploadedFiles.map((file) => file.key), - processRunId + result.processRunId ); logError("graph file add failed during workflow enqueue", { graphId: existingGraph.id, uploadedKeyCount: uploadedFiles.length, - addedFileCount: addedFiles.length, + addedFileCount: result.addedFiles.length, error: enqueueError, }); @@ -819,6 +1068,7 @@ export const graphRoute = new Elysia({ prefix: "/graphs" }) const [file] = await db .select({ id: filesTable.id, + type: filesTable.type, status: filesTable.status, processStep: filesTable.processStep, processErrorCode: filesTable.processErrorCode, @@ -906,6 +1156,7 @@ export const graphRoute = new Elysia({ prefix: "/graphs" }) graphId: existingGraph.id, fileIds: [file.id], processRunId: retry.runId, + ...(file.type === "code" ? { code: { kind: "repository" as const } } : {}), }); return status(200, { diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index ca7eec95..6f17bb39 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -11,6 +11,8 @@ import { chatRoute } from "./routes/chat"; import { chatLibraryRoute } from "./routes/chat-library"; import { fileTypesRoute } from "./routes/file-types"; import { graphFilesRoute } from "./routes/graph-files"; +import { connectorRoute, repositoryGraphBindingRoute } from "./routes/connectors"; +import { connectorWebhookRoute } from "./routes/connector-webhooks"; import { graphRoute } from "./routes/graph"; import { graphSuggestionsRoute } from "./routes/graph-suggestions"; import { mcpRoute } from "./routes/mcp"; @@ -51,6 +53,7 @@ const app = new Elysia({ ) ) .use(mcpRoute) + .use(connectorWebhookRoute) .use(authMiddleware) .use(authRoute) .use(chatRoute) @@ -58,6 +61,8 @@ const app = new Elysia({ .use(fileTypesRoute) .use(graphFilesRoute) .use(graphSuggestionsRoute) + .use(connectorRoute) + .use(repositoryGraphBindingRoute) .use(graphRoute) .use(modelsRoute) .use(promptsRoute) diff --git a/apps/frontend/app/(app)/connectors/[connectorId]/connect/page.tsx b/apps/frontend/app/(app)/connectors/[connectorId]/connect/page.tsx new file mode 100644 index 00000000..579f91d7 --- /dev/null +++ b/apps/frontend/app/(app)/connectors/[connectorId]/connect/page.tsx @@ -0,0 +1,10 @@ +import { ConnectorConnectPage } from "@/components/connectors/ConnectorPages"; + +type Props = { + params: Promise<{ connectorId: string }>; +}; + +export default async function ConnectConnectorPage({ params }: Props) { + const { connectorId } = await params; + return ; +} diff --git a/apps/frontend/app/(app)/connectors/github/callback/page.tsx b/apps/frontend/app/(app)/connectors/github/callback/page.tsx new file mode 100644 index 00000000..89f43af5 --- /dev/null +++ b/apps/frontend/app/(app)/connectors/github/callback/page.tsx @@ -0,0 +1,13 @@ +import { GitHubConnectorCallbackPage } from "@/components/connectors/ConnectorPages"; + +export default async function GitHubConnectorCallback({ + searchParams, +}: { + searchParams: Promise<{ code?: string | string[]; state?: string | string[] }>; +}) { + const params = await searchParams; + const code = typeof params.code === "string" ? params.code : ""; + const state = typeof params.state === "string" ? params.state : ""; + + return ; +} diff --git a/apps/frontend/app/(app)/connectors/github/install/callback/page.tsx b/apps/frontend/app/(app)/connectors/github/install/callback/page.tsx new file mode 100644 index 00000000..af84647d --- /dev/null +++ b/apps/frontend/app/(app)/connectors/github/install/callback/page.tsx @@ -0,0 +1,24 @@ +import { GitHubConnectorInstallCallbackPage } from "@/components/connectors/ConnectorPages"; + +export default async function GitHubConnectorInstallCallback({ + searchParams, +}: { + searchParams: Promise<{ + installation_id?: string | string[]; + setup_action?: string | string[]; + state?: string | string[]; + }>; +}) { + const params = await searchParams; + const installationId = typeof params.installation_id === "string" ? params.installation_id : ""; + const setupAction = typeof params.setup_action === "string" ? params.setup_action : ""; + const state = typeof params.state === "string" ? params.state : ""; + + return ( + + ); +} diff --git a/apps/frontend/app/(app)/connectors/github/new/page.tsx b/apps/frontend/app/(app)/connectors/github/new/page.tsx new file mode 100644 index 00000000..be7821ac --- /dev/null +++ b/apps/frontend/app/(app)/connectors/github/new/page.tsx @@ -0,0 +1,14 @@ +import { hasRole } from "@kiwi/auth/permissions"; +import { forbidden } from "next/navigation"; + +import { GitHubConnectorNewPage } from "@/components/connectors/ConnectorPages"; +import { getServerSession } from "@/lib/auth/get-server-session"; + +export default async function NewGitHubConnectorPage() { + const session = await getServerSession(); + if (!hasRole((session?.user as { role?: string })?.role ?? null, "admin")) { + forbidden(); + } + + return ; +} diff --git a/apps/frontend/app/(app)/connectors/gitlab/new/page.tsx b/apps/frontend/app/(app)/connectors/gitlab/new/page.tsx new file mode 100644 index 00000000..dc3e1bb8 --- /dev/null +++ b/apps/frontend/app/(app)/connectors/gitlab/new/page.tsx @@ -0,0 +1,14 @@ +import { hasRole } from "@kiwi/auth/permissions"; +import { forbidden } from "next/navigation"; + +import { GitLabConnectorNewPage } from "@/components/connectors/ConnectorPages"; +import { getServerSession } from "@/lib/auth/get-server-session"; + +export default async function NewGitLabConnectorPage() { + const session = await getServerSession(); + if (!hasRole((session?.user as { role?: string })?.role ?? null, "admin")) { + forbidden(); + } + + return ; +} diff --git a/apps/frontend/app/(app)/connectors/page.tsx b/apps/frontend/app/(app)/connectors/page.tsx new file mode 100644 index 00000000..4722281b --- /dev/null +++ b/apps/frontend/app/(app)/connectors/page.tsx @@ -0,0 +1,5 @@ +import { ConnectorListPage } from "@/components/connectors/ConnectorPages"; + +export default function ConnectorsPage() { + return ; +} diff --git a/apps/frontend/components/connectors/ConnectorPages.tsx b/apps/frontend/components/connectors/ConnectorPages.tsx new file mode 100644 index 00000000..d5595b12 --- /dev/null +++ b/apps/frontend/components/connectors/ConnectorPages.tsx @@ -0,0 +1,807 @@ +"use client"; + +import { + completeGitHubConnectorInstallation, + completeGitHubConnectorManifest, + createGitLabConnector, + createRepositoryGraph, + fetchConnectorBranches, + fetchConnectorInstallations, + fetchConnectorRepositories, + fetchConnectors, + startConnectorConnect, + startGitHubConnectorManifest, + type ConnectorBranchRecord, + type ConnectorInstallationRecord, + type ConnectorRepositoryRecord, +} from "@/lib/api"; +import { ORGANIZATION_GROUP_ID } from "@/lib/api/projects"; +import { useGroupsWithProjects } from "@/hooks/use-data"; +import { useApiClient } from "@/providers/ApiClientProvider"; +import { useAuth } from "@/providers/AuthProvider"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { ExternalLink, Loader2, Plug, RefreshCw } from "lucide-react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useEffect, useMemo, useState, type ReactNode } from "react"; +import { toast } from "sonner"; + +const connectorQueryKeys = { + connectors: ["connectors"] as const, + installations: (connectorId: string) => ["connectors", connectorId, "installations"] as const, + repositories: (connectorId: string, installationId: string) => + ["connectors", connectorId, "installations", installationId, "repositories"] as const, + branches: (connectorId: string, installationId: string, repositoryId: string) => + ["connectors", connectorId, "installations", installationId, "repositories", repositoryId, "branches"] as const, +}; + +const EMPTY_INSTALLATIONS: ConnectorInstallationRecord[] = []; +const EMPTY_REPOSITORIES: ConnectorRepositoryRecord[] = []; +const EMPTY_BRANCHES: ConnectorBranchRecord[] = []; + + +export function ConnectorListPage() { + const apiClient = useApiClient(); + const { isAdmin, isSystemAdmin } = useAuth(); + const { data: groups = [] } = useGroupsWithProjects(); + const canManageGraphs = + isAdmin || groups.some((group) => group.scope === "organization" || group.role === "admin" || group.role === "moderator"); + const { data: connectors = [], isLoading, error } = useQuery({ + queryKey: connectorQueryKeys.connectors, + queryFn: () => fetchConnectors(apiClient), + }); + + return ( +
+
+
+
+

Repository connectors

+

+ Connect provider apps to create private repository graphs and keep them synced. +

+
+ {isSystemAdmin ? ( +
+ + +
+ ) : null} +
+ + {error ? ( + + + Unable to load connectors + Please try again later. + + + ) : null} + + {isLoading ? ( + + + Loading connectors… + + + ) : null} + + {!isLoading && connectors.length === 0 ? ( + + + No connectors configured + + {isSystemAdmin + ? "Create a GitHub or GitLab connector before managers can add repository graphs." + : "Ask a system administrator to configure a repository connector."} + + + + ) : null} + +
+ {connectors.map((connector) => ( + + + + {connector.name} + + {connector.provider === "github" ? "GitHub" : "GitLab"} connector + + +
+ + Slug: {connector.slug} + + + {connector.status} + +
+
+ {canManageGraphs && connector.status === "active" ? ( + + ) : null} + {isSystemAdmin ? ( + + ) : null} +
+ + + ))} +
+
+
+ ); +} + +export function GitHubConnectorNewPage() { + const apiClient = useApiClient(); + const [name, setName] = useState("KIWI GitHub Connector"); + const manifestMutation = useMutation({ + mutationFn: () => startGitHubConnectorManifest(apiClient, { name: name.trim() }), + onSuccess: ({ manifestUrl }) => { + window.location.assign(manifestUrl); + }, + onError: () => toast.error("Unable to start GitHub manifest flow."), + }); + const disabled = manifestMutation.isPending || name.trim().length === 0; + + return ( +
+
+

Create GitHub connector

+

+ Start a GitHub App manifest flow for this KIWI instance. GitHub will return the app credentials to KIWI. +

+
+ + + Required permissions + The generated app requests only read access and push events. + + +
    +
  • Contents: read, so KIWI can ingest repository files.
    • +
  • Metadata: read, so KIWI can list repositories and branches.
    • +
  • Push webhook, so KIWI can enqueue a sync when the selected branch changes.
    • +
+ + + + + Manifest starter + Name the connector before continuing to GitHub. + + +
{ + event.preventDefault(); + if (!disabled) manifestMutation.mutate(); + }} + > +
+ + setName(event.target.value)} + autoComplete="off" + /> +
+ +
+ + +
+ ); +} + +type GitHubConnectorCallbackPageProps = { + code: string; + state: string; +}; + +export function GitHubConnectorCallbackPage({ code, state }: GitHubConnectorCallbackPageProps) { + const apiClient = useApiClient(); + const router = useRouter(); + const [error, setError] = useState(null); + + useEffect(() => { + if (!code || !state) { + setError("Missing GitHub manifest callback parameters."); + return; + } + + let cancelled = false; + void completeGitHubConnectorManifest(apiClient, { code, state }) + .then((connector) => { + if (cancelled) return; + router.replace(`/connectors/${connector.id}/connect`); + }) + .catch(() => { + if (cancelled) return; + setError("Unable to finish GitHub connector creation."); + toast.error("Unable to finish GitHub connector creation."); + }); + + return () => { + cancelled = true; + }; + }, [apiClient, code, router, state]); + + return ( + + + Finishing GitHub connector setup + + {error ?? "KIWI is exchanging the GitHub manifest callback and preparing the connector."} + + + {!error ? ( + + Redirecting… + + ) : null} + + ); +} + +type GitHubConnectorInstallCallbackPageProps = { + installationId: string; + setupAction: string; + state: string; +}; + +export function GitHubConnectorInstallCallbackPage({ + installationId, + setupAction, + state, +}: GitHubConnectorInstallCallbackPageProps) { + const apiClient = useApiClient(); + const router = useRouter(); + const [error, setError] = useState(null); + + useEffect(() => { + if (!installationId || !state) { + setError("Missing GitHub installation callback parameters."); + return; + } + + let cancelled = false; + void completeGitHubConnectorInstallation(apiClient, { + installation_id: installationId, + setup_action: setupAction || undefined, + state, + }) + .then((installation) => { + if (cancelled) return; + router.replace(`/connectors/${installation.connectorId}/connect`); + }) + .catch(() => { + if (cancelled) return; + setError("Unable to finish the GitHub installation flow."); + toast.error("Unable to finish the GitHub installation flow."); + }); + + return () => { + cancelled = true; + }; + }, [apiClient, installationId, router, setupAction, state]); + + return ( + + + Finishing GitHub installation + + {error ?? "KIWI is storing the GitHub installation before returning you to connector setup."} + + + {!error ? ( + + Redirecting… + + ) : null} + + ); +} + +export function GitLabConnectorNewPage() { + const apiClient = useApiClient(); + const queryClient = useQueryClient(); + const [name, setName] = useState("KIWI GitLab Connector"); + const [baseUrl, setBaseUrl] = useState("https://gitlab.com"); + const [clientId, setClientId] = useState(""); + const [clientSecret, setClientSecret] = useState(""); + const [webhookSecret, setWebhookSecret] = useState(""); + + const createMutation = useMutation({ + mutationFn: () => + createGitLabConnector(apiClient, { + name: name.trim(), + baseUrl: baseUrl.trim(), + clientId: clientId.trim(), + clientSecret, + webhookSecret, + }), + onSuccess: () => { + setClientSecret(""); + setWebhookSecret(""); + queryClient.invalidateQueries({ queryKey: connectorQueryKeys.connectors }); + toast.success("GitLab connector saved in disabled state until OAuth install flow is available."); + window.location.assign("/connectors"); + }, + onError: () => toast.error("Unable to save GitLab connector."), + }); + const disabled = + createMutation.isPending || + name.trim().length === 0 || + baseUrl.trim().length === 0 || + clientId.trim().length === 0 || + clientSecret.length === 0 || + webhookSecret.length === 0; + + return ( +
+
+

Create GitLab connector

+

+ Register a GitLab application manually, then store its credentials encrypted in KIWI. +

+
+ + + Required GitLab access + Use the narrowest application scopes that allow repository reads and push hooks. + + +
    +
  • Read repository access for file ingestion.
    • +
  • API access only when needed to list projects, branches, or configure push webhooks.
    • +
  • A push webhook token that KIWI can verify without exposing it back to the browser.
    • +
+ + + + + Connector settings + Secrets are submitted once and cleared from the form after saving. + + +
{ + event.preventDefault(); + if (!disabled) createMutation.mutate(); + }} + > +
+
+ + setName(event.target.value)} /> +
+
+ + setBaseUrl(event.target.value)} + placeholder="https://gitlab.com" + /> +
+
+ + setClientId(event.target.value)} + autoComplete="off" + /> +
+
+ + setClientSecret(event.target.value)} + type="password" + autoComplete="new-password" + /> +
+
+ + setWebhookSecret(event.target.value)} + type="password" + autoComplete="new-password" + /> +
+
+ +
+ + +
+ ); +} + +type ConnectorConnectPageProps = { + connectorId: string; +}; + +export function ConnectorConnectPage({ connectorId }: ConnectorConnectPageProps) { + const apiClient = useApiClient(); + const router = useRouter(); + const { isAdmin } = useAuth(); + const { data: groups = [], isLoading: groupsLoading } = useGroupsWithProjects(); + const { data: connectors = [] } = useQuery({ + queryKey: connectorQueryKeys.connectors, + queryFn: () => fetchConnectors(apiClient), + }); + const connector = connectors.find((item) => item.id === connectorId) ?? null; + const manageableOwners = useMemo(() => { + const owners = groups.filter( + (group) => group.scope === "organization" || group.role === "admin" || group.role === "moderator" + ); + if (isAdmin && !owners.some((group) => group.scope === "organization")) { + return [ + { + id: ORGANIZATION_GROUP_ID, + name: "Organization", + role: "admin" as const, + scope: "organization" as const, + projects: [], + }, + ...owners, + ]; + } + return owners; + }, [groups, isAdmin]); + const canManageGraphs = manageableOwners.length > 0; + const connectorActive = connector?.status === "active"; + const [ownerValue, setOwnerValue] = useState(""); + const [installationId, setInstallationId] = useState(""); + const [repositoryId, setRepositoryId] = useState(""); + const [branchName, setBranchName] = useState(""); + const [graphName, setGraphName] = useState(""); + + useEffect(() => { + if (!ownerValue && manageableOwners.length > 0) setOwnerValue(manageableOwners[0].id); + }, [manageableOwners, ownerValue]); + + const installationsQuery = useQuery({ + queryKey: connectorQueryKeys.installations(connectorId), + queryFn: () => fetchConnectorInstallations(apiClient, connectorId), + enabled: canManageGraphs && connectorActive, + }); + const installations = installationsQuery.data ?? EMPTY_INSTALLATIONS; + + useEffect(() => { + if (!installationId && installations.length > 0) setInstallationId(installations[0].id); + }, [installationId, installations]); + + const repositoriesQuery = useQuery({ + queryKey: connectorQueryKeys.repositories(connectorId, installationId), + queryFn: () => fetchConnectorRepositories(apiClient, connectorId, installationId), + enabled: canManageGraphs && connectorActive && installationId.length > 0, + }); + const repositories = repositoriesQuery.data ?? EMPTY_REPOSITORIES; + const selectedRepository = repositories.find((repository) => repository.id === repositoryId) ?? null; + + useEffect(() => { + if (repositories.length === 0) { + if (repositoryId) setRepositoryId(""); + return; + } + if (!repositories.some((repository) => repository.id === repositoryId)) { + const defaultRepository = repositories.find((repository) => repository.defaultBranch) ?? repositories[0]; + setRepositoryId(defaultRepository.id); + } + }, [repositories, repositoryId]); + + const branchesQuery = useQuery({ + queryKey: connectorQueryKeys.branches(connectorId, installationId, repositoryId), + queryFn: () => fetchConnectorBranches(apiClient, connectorId, installationId, repositoryId), + enabled: canManageGraphs && connectorActive && installationId.length > 0 && repositoryId.length > 0, + }); + const branches = branchesQuery.data ?? EMPTY_BRANCHES; + + useEffect(() => { + if (branches.length === 0) { + if (branchName) setBranchName(""); + return; + } + if (!branches.some((branch) => branch.name === branchName)) { + const defaultBranch = selectedRepository?.defaultBranch; + setBranchName(branches.find((branch) => branch.name === defaultBranch)?.name ?? branches[0].name); + } + }, [branches, branchName, selectedRepository?.defaultBranch]); + + useEffect(() => { + if (!graphName && selectedRepository) setGraphName(selectedRepository.name); + }, [graphName, selectedRepository]); + + + const connectMutation = useMutation({ + mutationFn: () => + startConnectorConnect(apiClient, connectorId, { + ...(ownerValue === ORGANIZATION_GROUP_ID ? {} : { teamId: ownerValue }), + }), + onSuccess: ({ redirectUrl }) => { + window.location.assign(redirectUrl); + }, + onError: () => toast.error("Unable to open the provider installation flow."), + }); + const createMutation = useMutation({ + mutationFn: async () => { + if (!selectedRepository) throw new Error("Repository is required"); + const owner = + ownerValue === ORGANIZATION_GROUP_ID + ? { kind: "organization" as const } + : { kind: "team" as const, teamId: ownerValue }; + return createRepositoryGraph(apiClient, connectorId, { + connectorInstallationId: installationId, + repositoryId: selectedRepository.id, + repositoryFullName: selectedRepository.fullName, + repositoryHtmlUrl: selectedRepository.htmlUrl, + branch: branchName, + name: graphName.trim() || selectedRepository.name, + owner, + }); + }, + onSuccess: ({ graph }) => { + const routeGroupId = ownerValue === ORGANIZATION_GROUP_ID ? ORGANIZATION_GROUP_ID : ownerValue; + router.push(`/${routeGroupId}/${graph.graphId ?? graph.id}`); + }, + onError: () => toast.error("Unable to create repository graph."), + }); + + const disabled = + createMutation.isPending || + !canManageGraphs || + !connectorActive || + ownerValue.length === 0 || + installationId.length === 0 || + repositoryId.length === 0 || + branchName.length === 0 || + !selectedRepository; + const installDisabled = + connectMutation.isPending || !connectorActive || ownerValue.length === 0 || connector?.provider !== "github"; + + if (!canManageGraphs && groupsLoading) { + return ( + + + Loading connector access… + + + ); + } + + if (!canManageGraphs) { + return ( + + + Repository connector access denied + You need manager access to connect repositories. + + + ); + } + + if (connector && !connectorActive) { + return ( + + + {connector.name} is disabled + + {connector.provider === "gitlab" + ? "GitLab connectors are saved in a disabled state until OAuth installation support is available." + : "Enable this connector before connecting repositories."} + + + + ); + } + + return ( +
+
+

Connect repository

+

+ {connector ? `${connector.provider === "github" ? "GitHub" : "GitLab"} · ${connector.name}` : "Choose an installation, repository, and branch."} +

+
+ + + + Repository graph + Create a graph from a provider repository without exposing provider credentials. + + + {connector?.provider === "github" ? ( +
+
+

Need a GitHub installation?

+

+ Install or refresh the GitHub App for the selected owner before choosing a repository. +

+
+ +
+ ) : null} + +
{ + event.preventDefault(); + if (!disabled) createMutation.mutate(); + }} + > +
+ + {manageableOwners.map((group) => ( + + {group.name} ({group.scope}) + + ))} + + + { + setInstallationId(value); + setRepositoryId(""); + setBranchName(""); + }} + disabled={installations.length === 0} + > + {installations.map((installation) => ( + + {installation.providerAccountLogin} ({installation.repositorySelection}) + + ))} + + + { + setRepositoryId(value); + setBranchName(""); + const repository = repositories.find((item) => item.id === value); + if (repository) setGraphName(repository.name); + }} + disabled={repositories.length === 0 || repositoriesQuery.isLoading} + > + {repositories.map((repository) => ( + + {repository.fullName} + {repository.private ? " · private" : ""} + + ))} + + + + {branches.map((branch) => ( + + {branch.name} + {branch.name === selectedRepository?.defaultBranch ? " · default" : ""} + + ))} + +
+ +
+ + setGraphName(event.target.value)} + placeholder={selectedRepository?.name ?? "Repository graph"} + /> +
+ + + + + +
+ +
+ ); +} + + +function SelectField({ + label, + value, + onValueChange, + disabled, + children, +}: { + label: string; + value: string; + onValueChange: (value: string) => void; + disabled?: boolean; + children: ReactNode; +}) { + return ( +
+ + +
+ ); +} + +function ConnectorSelectionState({ + repositories, + branches, + repositoriesLoading, + branchesLoading, +}: { + repositories: ConnectorRepositoryRecord[]; + branches: ConnectorBranchRecord[]; + repositoriesLoading: boolean; + branchesLoading: boolean; +}) { + if (repositoriesLoading || branchesLoading) { + return

Loading provider metadata…

; + } + if (repositories.length === 0) { + return

No repositories are available for this installation.

; + } + if (branches.length === 0) { + return

No branches are available for this repository.

; + } + return null; +} diff --git a/apps/frontend/lib/api/connectors.test.ts b/apps/frontend/lib/api/connectors.test.ts new file mode 100644 index 00000000..be5c0e27 --- /dev/null +++ b/apps/frontend/lib/api/connectors.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, test, vi } from "vitest"; +import type { KiwiApiClient } from "./client"; +import { + completeGitHubConnectorInstallation, + completeGitHubConnectorManifest, + createGitLabConnector, + createRepositoryGraph, + fetchConnectorBranches, + fetchConnectorInstallations, + fetchConnectorRepositories, + fetchConnectors, + startConnectorConnect, + startGitHubConnectorManifest, + syncRepositoryGraphBinding, +} from "./connectors"; + +const connector = { + id: "connector-1", + provider: "github" as const, + name: "GitHub", + slug: "github", + status: "active" as const, + appId: "123", + clientId: null, + createdAt: "2026-06-14T00:00:00.000Z", + updatedAt: "2026-06-14T00:00:00.000Z", +}; + +const installation = { + id: "installation-1", + connectorId: connector.id, + provider: "github" as const, + providerInstallationId: "987", + providerAccountLogin: "kiwi-org", + providerAccountType: "organization" as const, + organizationId: "org-1", + teamId: null, + repositorySelection: "selected" as const, + status: "active" as const, + createdAt: connector.createdAt, + updatedAt: connector.updatedAt, +}; + +const repository = { + provider: "github" as const, + id: "repo/1", + fullName: "kiwi/app", + name: "app", + htmlUrl: "https://github.com/kiwi/app", + defaultBranch: "main", + private: true, +}; + +const branch = { name: "main", commitSha: "abc123" }; + +const binding = { + id: "binding-1", + graphId: "graph-1", + connectorInstallationId: installation.id, + provider: "github" as const, + providerRepositoryId: repository.id, + repositoryFullName: repository.fullName, + repositoryHtmlUrl: repository.htmlUrl, + branch: branch.name, + lastSeenCommitSha: branch.commitSha, + lastSyncedCommitSha: null, + syncStatus: "pending" as const, + syncErrorCode: null, + webhookEnabled: true, + createdAt: connector.createdAt, + updatedAt: connector.updatedAt, +}; + +describe("connector API helpers", () => { + test("fetches connectors", async () => { + const get = vi.fn(async () => ({ status: "success" as const, data: [connector] })); + const client = { baseURL: "/api", get } as unknown as KiwiApiClient; + + await expect(fetchConnectors(client)).resolves.toEqual([connector]); + expect(get).toHaveBeenCalledWith("/connectors"); + }); + + test("starts the GitHub manifest flow", async () => { + const input = { name: "GitHub" }; + const data = { manifestUrl: "https://github.com/settings/apps/new", state: "state-1" }; + const post = vi.fn(async () => ({ status: "success" as const, data })); + const client = { baseURL: "/api", post } as unknown as KiwiApiClient; + + await expect(startGitHubConnectorManifest(client, input)).resolves.toEqual(data); + expect(post).toHaveBeenCalledWith("/connectors/github/manifest/start", input); + }); + + test("creates a GitLab connector without rendering returned secrets", async () => { + const input = { + name: "GitLab", + baseUrl: "https://gitlab.com", + clientId: "client-id", + clientSecret: "client-secret", + webhookSecret: "webhook-secret", + }; + const post = vi.fn(async () => ({ status: "success" as const, data: { ...connector, provider: "gitlab" as const } })); + const client = { baseURL: "/api", post } as unknown as KiwiApiClient; + + await expect(createGitLabConnector(client, input)).resolves.toMatchObject({ provider: "gitlab" }); + expect(post).toHaveBeenCalledWith("/connectors/gitlab", input); + }); + + test("completes GitHub connector callbacks through API routes", async () => { + const get = vi + .fn() + .mockResolvedValueOnce({ status: "success" as const, data: connector }) + .mockResolvedValueOnce({ status: "success" as const, data: installation }); + const client = { baseURL: "/api", get } as unknown as KiwiApiClient; + + await expect(completeGitHubConnectorManifest(client, { code: "code-1", state: "state-1" })).resolves.toEqual( + connector + ); + await expect( + completeGitHubConnectorInstallation(client, { + state: "state-2", + installation_id: "installation-1", + setup_action: "install", + }) + ).resolves.toEqual(installation); + expect(get).toHaveBeenNthCalledWith(1, "/connectors/github/manifest/callback?code=code-1&state=state-1"); + expect(get).toHaveBeenNthCalledWith( + 2, + "/connectors/github/install/callback?state=state-2&installation_id=installation-1&setup_action=install" + ); + }); + + test("starts connector installation flow for a managed owner", async () => { + const get = vi.fn(async () => ({ status: "success" as const, data: { redirectUrl: "https://github.com/apps/kiwi/install" } })); + const client = { baseURL: "/api", get } as unknown as KiwiApiClient; + + await expect(startConnectorConnect(client, connector.id, { teamId: "team-1" })).resolves.toEqual({ + redirectUrl: "https://github.com/apps/kiwi/install", + }); + expect(get).toHaveBeenCalledWith("/connectors/connector-1/connect?teamId=team-1"); + }); + + test("fetches installation repositories and encoded branches", async () => { + const get = vi + .fn() + .mockResolvedValueOnce({ status: "success" as const, data: [installation] }) + .mockResolvedValueOnce({ status: "success" as const, data: [repository] }) + .mockResolvedValueOnce({ status: "success" as const, data: [branch] }); + const client = { baseURL: "/api", get } as unknown as KiwiApiClient; + + await expect(fetchConnectorInstallations(client, connector.id)).resolves.toEqual([installation]); + await expect(fetchConnectorRepositories(client, connector.id, installation.id)).resolves.toEqual([repository]); + await expect(fetchConnectorBranches(client, connector.id, installation.id, repository.id)).resolves.toEqual([branch]); + expect(get).toHaveBeenNthCalledWith(1, "/connectors/connector-1/installations"); + expect(get).toHaveBeenNthCalledWith(2, "/connectors/connector-1/repositories?installationId=installation-1"); + expect(get).toHaveBeenNthCalledWith( + 3, + "/connectors/connector-1/repositories/repo%2F1/branches?installationId=installation-1" + ); + }); + + test("creates and syncs repository graphs", async () => { + const graph = { + id: "row-1", + name: "app", + description: null, + organizationId: "org-1", + teamId: null, + userId: null, + graphId: "graph-1", + hidden: false, + state: "updating" as const, + }; + const input = { + connectorInstallationId: installation.id, + repositoryId: repository.id, + repositoryFullName: repository.fullName, + repositoryHtmlUrl: repository.htmlUrl, + branch: branch.name, + name: "app", + owner: { kind: "organization" as const }, + }; + const post = vi + .fn() + .mockResolvedValueOnce({ status: "success" as const, data: { graph, binding, workflowRunId: "run-1" } }) + .mockResolvedValueOnce({ status: "success" as const, data: { binding, workflowRunId: "run-2" } }); + const client = { baseURL: "/api", post } as unknown as KiwiApiClient; + + await expect(createRepositoryGraph(client, connector.id, input)).resolves.toEqual({ graph, binding, workflowRunId: "run-1" }); + await expect(syncRepositoryGraphBinding(client, binding.id)).resolves.toEqual({ binding, workflowRunId: "run-2" }); + expect(post).toHaveBeenNthCalledWith(1, "/connectors/connector-1/repository-graphs", input); + expect(post).toHaveBeenNthCalledWith(2, "/repository-graph-bindings/binding-1/sync"); + }); +}); diff --git a/apps/frontend/lib/api/connectors.ts b/apps/frontend/lib/api/connectors.ts new file mode 100644 index 00000000..cbe2cfb1 --- /dev/null +++ b/apps/frontend/lib/api/connectors.ts @@ -0,0 +1,169 @@ +import type { + ApiResponse, + ConnectorBranchListResponse, + ConnectorBranchRecord, + ConnectorConnectStartResponse, + ConnectorInstallationListResponse, + ConnectorInstallationRecord, + ConnectorListResponse, + ConnectorRecord, + ConnectorRepositoryListResponse, + ConnectorRepositoryRecord, + GitHubConnectorManifestStartInput, + GitHubConnectorManifestStartResponse, + GitHubConnectorManifestStartSuccessData, + GitLabConnectorCreateInput, + GitLabConnectorCreateResponse, + RepositoryGraphBindingResponse, + RepositoryGraphBindingRecord, + RepositoryGraphBindingSyncResponse, + RepositoryGraphBindingSyncSuccessData, + RepositoryGraphCreateInput, + RepositoryGraphCreateResponse, + RepositoryGraphCreateSuccessData, +} from "@kiwi/contracts"; + +import { unwrapApiResponse, type KiwiApiClient } from "./client"; + +export async function fetchConnectors(client: KiwiApiClient): Promise { + const response = await client.get("/connectors"); + return unwrapApiResponse(response); +} + +export async function startGitHubConnectorManifest( + client: KiwiApiClient, + input: GitHubConnectorManifestStartInput +): Promise { + const response = await client.post("/connectors/github/manifest/start", input); + return unwrapApiResponse(response); +} + +export async function completeGitHubConnectorManifest( + client: KiwiApiClient, + input: { code: string; state: string } +): Promise { + const params = new URLSearchParams(input); + const response = await client.get< + ApiResponse + >(`/connectors/github/manifest/callback?${params.toString()}`); + return unwrapApiResponse(response); +} + +export async function completeGitHubConnectorInstallation( + client: KiwiApiClient, + input: { state: string; installation_id: string; setup_action?: string } +): Promise { + const params = new URLSearchParams(); + params.set("state", input.state); + params.set("installation_id", input.installation_id); + if (input.setup_action) { + params.set("setup_action", input.setup_action); + } + const response = await client.get< + ApiResponse + >(`/connectors/github/install/callback?${params.toString()}`); + return unwrapApiResponse(response); +} + +export async function createGitLabConnector( + client: KiwiApiClient, + input: GitLabConnectorCreateInput +): Promise { + const response = await client.post("/connectors/gitlab", input); + return unwrapApiResponse(response); +} +export async function startConnectorConnect( + client: KiwiApiClient, + connectorId: string, + input: { organizationId?: string; teamId?: string } +): Promise<{ redirectUrl: string }> { + const params = new URLSearchParams(); + if (input.organizationId) { + params.set("organizationId", input.organizationId); + } + if (input.teamId) { + params.set("teamId", input.teamId); + } + const response = await client.get( + `/connectors/${encodeURIComponent(connectorId)}/connect?${params.toString()}` + ); + return unwrapApiResponse(response); +} + +export async function fetchConnectorInstallations( + client: KiwiApiClient, + connectorId: string +): Promise { + const response = await client.get( + `/connectors/${encodeURIComponent(connectorId)}/installations` + ); + return unwrapApiResponse(response); +} + +export async function fetchConnectorRepositories( + client: KiwiApiClient, + connectorId: string, + installationId: string +): Promise { + const params = new URLSearchParams({ installationId }); + const response = await client.get( + `/connectors/${encodeURIComponent(connectorId)}/repositories?${params.toString()}` + ); + return unwrapApiResponse(response); +} + +export async function fetchConnectorBranches( + client: KiwiApiClient, + connectorId: string, + installationId: string, + repositoryId: string +): Promise { + const params = new URLSearchParams({ installationId }); + const response = await client.get( + `/connectors/${encodeURIComponent(connectorId)}/repositories/${encodeURIComponent(repositoryId)}/branches?${params.toString()}` + ); + return unwrapApiResponse(response); +} + +export async function createRepositoryGraph( + client: KiwiApiClient, + connectorId: string, + input: RepositoryGraphCreateInput +): Promise { + const response = await client.post( + `/connectors/${encodeURIComponent(connectorId)}/repository-graphs`, + input + ); + return unwrapApiResponse(response); +} + +export async function fetchRepositoryGraphBinding( + client: KiwiApiClient, + bindingId: string +): Promise { + const response = await client.get( + `/repository-graph-bindings/${encodeURIComponent(bindingId)}` + ); + return unwrapApiResponse(response); +} + +export async function syncRepositoryGraphBinding( + client: KiwiApiClient, + bindingId: string +): Promise { + const response = await client.post( + `/repository-graph-bindings/${encodeURIComponent(bindingId)}/sync` + ); + return unwrapApiResponse(response); +} + +export type { + ConnectorBranchRecord, + ConnectorInstallationRecord, + ConnectorRecord, + ConnectorRepositoryRecord, + GitHubConnectorManifestStartInput, + GitLabConnectorCreateInput, + RepositoryGraphBindingRecord, + RepositoryGraphCreateInput, +}; diff --git a/apps/frontend/lib/api/index.ts b/apps/frontend/lib/api/index.ts index d98f5183..ad020328 100644 --- a/apps/frontend/lib/api/index.ts +++ b/apps/frontend/lib/api/index.ts @@ -43,6 +43,30 @@ export type { SourceReferenceResponse, TextUnitResponse, } from "@kiwi/contracts"; +export type { + ConnectorBranchListResponse, + ConnectorBranchRecord, + ConnectorInstallationListResponse, + ConnectorInstallationRecord, + ConnectorListResponse, + ConnectorProvider, + ConnectorRecord, + ConnectorRepositoryListResponse, + ConnectorRepositoryRecord, + GitHubConnectorManifestStartInput, + GitHubConnectorManifestStartResponse, + GitHubConnectorManifestStartSuccessData, + GitLabConnectorCreateInput, + GitLabConnectorCreateResponse, + RepositoryGraphBindingRecord, + RepositoryGraphBindingResponse, + RepositoryGraphBindingSyncResponse, + RepositoryGraphBindingSyncSuccessData, + RepositoryGraphCreateInput, + RepositoryGraphCreateResponse, + RepositoryGraphCreateSuccessData, +} from "@kiwi/contracts"; + // Re-export team API helpers under the existing frontend group names. export { @@ -57,6 +81,21 @@ export { updateGroup, updateGroupUsers, } from "./groups"; +export { + completeGitHubConnectorInstallation, + completeGitHubConnectorManifest, + createGitLabConnector, + createRepositoryGraph, + fetchConnectorBranches, + fetchConnectorInstallations, + fetchConnectorRepositories, + fetchConnectors, + fetchRepositoryGraphBinding, + startConnectorConnect, + startGitHubConnectorManifest, + syncRepositoryGraphBinding, +} from "./connectors"; + // Re-export projects API export { diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index 1658a25e..a4e2e056 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -1,7 +1,13 @@ FROM oven/bun:1 RUN apt-get update \ - && apt-get install -y --no-install-recommends ghostscript \ + && apt-get install -y --no-install-recommends \ + g++ \ + ghostscript \ + make \ + node-gyp \ + nodejs \ + python3 \ && rm -rf /var/lib/apt/lists/* WORKDIR /app @@ -21,6 +27,12 @@ COPY packages/logger/package.json packages/logger/package.json RUN bun install --frozen-lockfile +RUN arch="$(node -p 'process.arch')" \ + && cd packages/graph/node_modules/tree-sitter \ + && CXXFLAGS="-std=c++20" node-gyp rebuild \ + && mkdir -p "prebuilds/linux-${arch}" \ + && cp build/Release/tree_sitter_runtime_binding.node "prebuilds/linux-${arch}/tree-sitter.node" + COPY apps/worker/index.ts apps/worker/index.ts COPY apps/worker/env.ts apps/worker/env.ts COPY apps/worker/worker.ts apps/worker/worker.ts diff --git a/apps/worker/lib/__tests__/code-manifest.test.ts b/apps/worker/lib/__tests__/code-manifest.test.ts new file mode 100644 index 00000000..5fed9cc6 --- /dev/null +++ b/apps/worker/lib/__tests__/code-manifest.test.ts @@ -0,0 +1,280 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; + +type MockFileRow = { + id: string; + name: string; + key: string; + storageKind: string; + externalUrl: string | null; + externalProvider: string | null; + repositoryBindingId: string | null; + metadata: string; +}; + +const baseRows: MockFileRow[] = [ + { + id: "file-new", + name: "widgets/src/index.ts", + key: "key-new", + storageKind: "internal", + externalUrl: null, + externalProvider: null, + repositoryBindingId: null, + metadata: JSON.stringify({ + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-1", + path: "src/index.ts", + }), + }, + { + id: "file-existing", + name: "widgets/src/helper.ts", + key: "key-existing", + storageKind: "internal", + externalUrl: null, + externalProvider: null, + repositoryBindingId: null, + metadata: JSON.stringify({ + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-1", + path: "src/helper.ts", + }), + }, + { + id: "file-other", + name: "other/src/helper.ts", + key: "key-other", + storageKind: "internal", + externalUrl: null, + externalProvider: null, + repositoryBindingId: null, + metadata: JSON.stringify({ + repositoryUrl: "https://github.com/acme/other.git", + repositoryName: "other", + commitSha: "commit-1", + path: "src/helper.ts", + }), + }, +]; + +const fileContents: Record = { + "key-new": "import { helper } from './helper';\nexport function main() { return helper(); }\n", + "key-existing": "export function helper() { return 1; }\n", + "key-other": "export function helper() { return 2; }\n", + "key-connector-new": "import { shared } from './shared';\nexport const main = () => shared;\n", + "key-connector-existing": "export const shared = 1;\n", + "key-connector-other-binding": "export const shared = 2;\n", +}; + +let rows = [...baseRows]; +let selectedFileIds = ["file-new"]; +let selectCallCount = 0; +let uploadedManifest: unknown; + +mock.module("@kiwi/db", () => ({ + db: { + select: () => ({ + from: () => ({ + where: async () => { + selectCallCount += 1; + return selectCallCount === 1 ? rows.filter((row) => selectedFileIds.includes(row.id)) : rows; + }, + }), + }), + }, +})); + +mock.module("@kiwi/files", () => ({ + getFile: async (key: string) => { + const content = fileContents[key]; + return content ? { content } : null; + }, + putNamedFile: async (_name: string, content: string) => { + uploadedManifest = JSON.parse(content); + return { key: "manifest-key" }; + }, +})); + +mock.module("../../env", () => ({ + env: { S3_BUCKET: "test-bucket" }, +})); + +// Dynamic import is required so module mocks and env are installed before worker modules are evaluated. +const { prepareCodeManifest } = await import("../code-manifest"); +const { readFileContentSource } = await import("../file-content-source"); + +describe("prepareCodeManifest", () => { + beforeEach(() => { + selectCallCount = 0; + uploadedManifest = undefined; + selectedFileIds = ["file-new"]; + rows = [...baseRows]; + }); + + test("includes active files from the same repository commit as selected files", async () => { + const key = await prepareCodeManifest({ graphId: "graph-1", fileIds: ["file-new"], processRunId: "run-1" }); + + expect(key).toBe("manifest-key"); + expect((uploadedManifest as { files: Array<{ path: string }> }).files.map((file) => file.path).sort()).toEqual([ + "src/helper.ts", + "src/index.ts", + ]); + }); + + test("includes unchanged connector siblings from the same binding across mixed commits", async () => { + rows = [ + { + id: "file-connector-new", + name: "widgets/src/index.ts", + key: "key-connector-new", + storageKind: "internal", + externalUrl: null, + externalProvider: null, + repositoryBindingId: "binding-1", + metadata: JSON.stringify({ + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-2", + path: "src/index.ts", + }), + }, + { + id: "file-connector-existing", + name: "widgets/src/shared.ts", + key: "key-connector-existing", + storageKind: "internal", + externalUrl: null, + externalProvider: null, + repositoryBindingId: "binding-1", + metadata: JSON.stringify({ + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-1", + path: "src/shared.ts", + }), + }, + { + id: "file-connector-other-binding", + name: "widgets/src/shared.ts", + key: "key-connector-other-binding", + storageKind: "internal", + externalUrl: null, + externalProvider: null, + repositoryBindingId: "binding-2", + metadata: JSON.stringify({ + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-1", + path: "src/shared.ts", + }), + }, + ]; + selectedFileIds = ["file-connector-new"]; + + const key = await prepareCodeManifest({ graphId: "graph-1", fileIds: selectedFileIds, processRunId: "run-1" }); + + expect(key).toBe("manifest-key"); + expect((uploadedManifest as { files: Array<{ path: string }> }).files.map((file) => file.path).sort()).toEqual([ + "src/index.ts", + "src/shared.ts", + ]); + expect((uploadedManifest as { files: Array<{ fileId: string }> }).files.map((file) => file.fileId).sort()).toEqual([ + "file-connector-existing", + "file-connector-new", + ]); + }); + + test("includes external GitHub files from the same repository commit", async () => { + rows = [ + ...baseRows, + { + id: "file-external", + name: "widgets/src/external.ts", + key: "external:github:acme/widgets@commit-1:src/external.ts", + storageKind: "external", + externalUrl: "https://raw.githubusercontent.com/acme/widgets/commit-1/src/external.ts", + externalProvider: "github", + repositoryBindingId: null, + metadata: JSON.stringify({ + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-1", + path: "src/external.ts", + }), + }, + ]; + globalThis.fetch = (async () => + new Response("export const external = true;\n", { + headers: { "content-type": "text/plain" }, + })) as unknown as typeof fetch; + + const key = await prepareCodeManifest({ graphId: "graph-1", fileIds: ["file-new"], processRunId: "run-1" }); + + expect(key).toBe("manifest-key"); + expect((uploadedManifest as { files: Array<{ path: string }> }).files.map((file) => file.path).sort()).toEqual([ + "src/external.ts", + "src/helper.ts", + "src/index.ts", + ]); + }); +}); + +describe("readFileContentSource", () => { + test("reads internal S3 content", async () => { + await expect(readFileContentSource({ kind: "internal", key: "key-new" })).resolves.toBe( + "import { helper } from './helper';\nexport function main() { return helper(); }\n" + ); + }); + + test("fetches external GitHub raw content", async () => { + globalThis.fetch = (async () => + new Response("export const value = 1;\n", { + headers: { "content-type": "text/plain" }, + })) as unknown as typeof fetch; + + await expect( + readFileContentSource({ + kind: "external", + provider: "github", + url: "https://raw.githubusercontent.com/acme/widgets/commit-1/src/index.ts", + }) + ).resolves.toBe("export const value = 1;\n"); + }); + + test("rejects non-GitHub external URLs before fetching", async () => { + let fetched = false; + globalThis.fetch = (async () => { + fetched = true; + return new Response("nope"); + }) as unknown as typeof fetch; + + await expect( + readFileContentSource({ + kind: "external", + provider: "github", + url: "https://example.com/acme/widgets/commit-1/src/index.ts", + }) + ).rejects.toThrow("Unsupported external file source"); + expect(fetched).toBe(false); + }); + + test("rejects oversized external responses", async () => { + globalThis.fetch = (async () => + new Response("too large", { + headers: { + "content-type": "text/plain", + "content-length": String(2 * 1024 * 1024 + 1), + }, + })) as unknown as typeof fetch; + + await expect( + readFileContentSource({ + kind: "external", + provider: "github", + url: "https://raw.githubusercontent.com/acme/widgets/commit-1/src/index.ts", + }) + ).rejects.toThrow("too large"); + }); +}); diff --git a/apps/worker/lib/code-file-metadata.ts b/apps/worker/lib/code-file-metadata.ts new file mode 100644 index 00000000..386820b7 --- /dev/null +++ b/apps/worker/lib/code-file-metadata.ts @@ -0,0 +1 @@ +export { parseCodeFileMetadata, serializeCodeFileMetadata, type CodeFileMetadata } from "@kiwi/graph/code/metadata"; diff --git a/apps/worker/lib/code-manifest.ts b/apps/worker/lib/code-manifest.ts new file mode 100644 index 00000000..ad5d9910 --- /dev/null +++ b/apps/worker/lib/code-manifest.ts @@ -0,0 +1,128 @@ +import { and, eq, inArray } from "drizzle-orm"; +import { db } from "@kiwi/db"; +import { filesTable } from "@kiwi/db/tables/graph"; +import { getFile, putNamedFile } from "@kiwi/files"; +import { buildCodeRepositoryManifest } from "@kiwi/graph/code/repository"; +import type { CodeRepositoryFile, CodeRepositoryManifest } from "@kiwi/graph/code/repository"; +import { env } from "../env"; +import { parseCodeFileMetadata } from "./code-file-metadata"; +import { fileContentSourceFromRow, readFileContentSource } from "./file-content-source"; + +type ManifestFileRow = { + id: string; + name: string; + key: string; + metadata: string | null; + storageKind: string; + externalUrl: string | null; + externalProvider: string | null; + repositoryBindingId: string | null; +}; + +export async function prepareCodeManifest(options: { + graphId: string; + fileIds: string[]; + processRunId?: string; +}): Promise { + if (options.fileIds.length === 0) { + return undefined; + } + + const selectedRows = await db + .select({ + id: filesTable.id, + name: filesTable.name, + key: filesTable.key, + metadata: filesTable.metadata, + storageKind: filesTable.storageKind, + externalUrl: filesTable.externalUrl, + externalProvider: filesTable.externalProvider, + repositoryBindingId: filesTable.repositoryBindingId, + }) + .from(filesTable) + .where( + and( + eq(filesTable.graphId, options.graphId), + eq(filesTable.type, "code"), + eq(filesTable.deleted, false), + inArray(filesTable.id, options.fileIds) + ) + ); + + if (selectedRows.length === 0) { + return undefined; + } + + const selectedScopes = new Set(selectedRows.map(repositoryManifestScopeKey).filter((scope): scope is string => scope !== null)); + const rows = selectedScopes.size + ? await db + .select({ + id: filesTable.id, + name: filesTable.name, + key: filesTable.key, + metadata: filesTable.metadata, + storageKind: filesTable.storageKind, + externalUrl: filesTable.externalUrl, + externalProvider: filesTable.externalProvider, + repositoryBindingId: filesTable.repositoryBindingId, + }) + .from(filesTable) + .where(and(eq(filesTable.graphId, options.graphId), eq(filesTable.type, "code"), eq(filesTable.deleted, false))) + : selectedRows; + + const files: CodeRepositoryFile[] = []; + for (const row of rows) { + const metadata = parseCodeFileMetadata(row.metadata); + const scope = repositoryManifestScopeKey(row); + if (metadata && selectedScopes.size > 0 && (!scope || !selectedScopes.has(scope))) { + continue; + } + + const content = await readFileContentSource(fileContentSourceFromRow(row)); + if (!metadata || content === null) { + continue; + } + + files.push({ + fileId: row.id, + content, + ...metadata, + }); + } + + if (files.length === 0) { + return undefined; + } + + const manifest = buildCodeRepositoryManifest(files); + const uploaded = await putNamedFile( + "code-manifest.json", + JSON.stringify(manifest), + `graphs/${options.graphId}/process-runs/${options.processRunId ?? "adhoc"}`, + env.S3_BUCKET + ); + + return uploaded.key; +} + +function repositoryManifestScopeKey(row: Pick): string | null { + if (row.repositoryBindingId) { + return `binding:${row.repositoryBindingId}`; + } + + const metadata = parseCodeFileMetadata(row.metadata); + if (!metadata) { + return null; + } + + return `repository:${metadata.repositoryUrl}\0${metadata.commitSha}`; +} + +export async function loadCodeManifest(key: string): Promise { + const manifest = await getFile(key, env.S3_BUCKET, "json"); + if (!manifest) { + throw new Error(`Failed to load code manifest from ${key}`); + } + + return manifest.content; +} diff --git a/apps/worker/lib/code-repository-finalizer.ts b/apps/worker/lib/code-repository-finalizer.ts new file mode 100644 index 00000000..56265c73 --- /dev/null +++ b/apps/worker/lib/code-repository-finalizer.ts @@ -0,0 +1,142 @@ +import { db } from "@kiwi/db"; +import { currentSourcePredicate, currentSourceSql } from "@kiwi/db/source-validity"; +import { filesTable, sourcesTable, textUnitTable } from "@kiwi/db/tables/graph"; +import { and, eq, inArray, isNotNull, sql } from "drizzle-orm"; +import { parseCodeFileMetadata } from "./code-file-metadata"; +import { textArray } from "./sql"; + +export type RepositoryFileMetadataRow = { + id: string; + metadata: string | null; +}; + +export type RepositoryFinalizationTargets = { + repositoryUrls: string[]; + olderFileIds: string[]; +}; + +export type RepositorySourceInvalidationResult = { + entityIds: string[]; + relationshipIds: string[]; +}; + +export function resolveRepositoryFinalizationTargets( + latestRows: RepositoryFileMetadataRow[], + candidateRows: RepositoryFileMetadataRow[] +): RepositoryFinalizationTargets { + const latestFileIds = new Set(latestRows.map((row) => row.id)); + const repositoryUrls = [ + ...new Set( + latestRows + .map((row) => parseCodeFileMetadata(row.metadata)?.repositoryUrl) + .filter((url): url is string => url !== undefined) + ), + ]; + if (repositoryUrls.length === 0) { + return { repositoryUrls: [], olderFileIds: [] }; + } + + const repositoryUrlSet = new Set(repositoryUrls); + const olderFileIds = candidateRows + .filter((row) => !latestFileIds.has(row.id)) + .filter((row) => { + const metadata = parseCodeFileMetadata(row.metadata); + return metadata !== null && repositoryUrlSet.has(metadata.repositoryUrl); + }) + .map((row) => row.id); + + return { repositoryUrls, olderFileIds }; +} + +export async function invalidateSupersededRepositorySources(options: { + graphId: string; + latestFileIds?: string[]; + retiredFileIds?: string[]; +}): Promise { + if (options.retiredFileIds !== undefined) { + return invalidateRepositorySourceTargets({ + graphId: options.graphId, + retiredFileIds: options.retiredFileIds, + markDeleted: true, + }); + } + + if (!options.latestFileIds || options.latestFileIds.length === 0) { + return { entityIds: [], relationshipIds: [] }; + } + + return db.transaction(async (tx) => { + const latestRows = await tx + .select({ id: filesTable.id, metadata: filesTable.metadata }) + .from(filesTable) + .where(and(eq(filesTable.graphId, options.graphId), inArray(filesTable.id, options.latestFileIds))); + const candidateRows = await tx + .select({ id: filesTable.id, metadata: filesTable.metadata }) + .from(filesTable) + .where(and(eq(filesTable.graphId, options.graphId), eq(filesTable.type, "code"))); + const targets = resolveRepositoryFinalizationTargets(latestRows, candidateRows); + return invalidateRepositorySources(tx, options.graphId, targets.olderFileIds, false); + }); +} + +async function invalidateRepositorySourceTargets(options: { + graphId: string; + retiredFileIds: string[]; + markDeleted: boolean; +}): Promise { + if (options.retiredFileIds.length === 0) { + return { entityIds: [], relationshipIds: [] }; + } + + return db.transaction(async (tx) => + invalidateRepositorySources(tx, options.graphId, options.retiredFileIds, options.markDeleted) + ); +} + +async function invalidateRepositorySources( + tx: Parameters[0]>[0], + graphId: string, + retiredFileIds: string[], + markDeleted: boolean +): Promise { + if (retiredFileIds.length === 0) { + return { entityIds: [], relationshipIds: [] }; + } + + if (markDeleted) { + await tx + .update(filesTable) + .set({ deleted: true }) + .where(and(eq(filesTable.graphId, graphId), eq(filesTable.deleted, false), inArray(filesTable.id, retiredFileIds))); + } + + const affectedEntityRows = await tx + .selectDistinct({ id: sourcesTable.entityId }) + .from(sourcesTable) + .innerJoin(textUnitTable, eq(textUnitTable.id, sourcesTable.textUnitId)) + .where(and(inArray(textUnitTable.fileId, retiredFileIds), currentSourcePredicate(), isNotNull(sourcesTable.entityId))); + const affectedRelationshipRows = await tx + .selectDistinct({ id: sourcesTable.relationshipId }) + .from(sourcesTable) + .innerJoin(textUnitTable, eq(textUnitTable.id, sourcesTable.textUnitId)) + .where( + and( + inArray(textUnitTable.fileId, retiredFileIds), + currentSourcePredicate(), + isNotNull(sourcesTable.relationshipId) + ) + ); + + await tx.execute(sql` + UPDATE sources source + SET valid_until = NOW() + FROM text_units text_unit + WHERE source.text_unit_id = text_unit.id + AND text_unit.file_id = ANY(${textArray(retiredFileIds)}) + AND ${currentSourceSql("source")} + `); + return { + entityIds: affectedEntityRows.map((row) => row.id).filter((id): id is string => id !== null), + relationshipIds: affectedRelationshipRows.map((row) => row.id).filter((id): id is string => id !== null), + }; +} diff --git a/apps/worker/lib/file-content-source.ts b/apps/worker/lib/file-content-source.ts new file mode 100644 index 00000000..0bc04714 --- /dev/null +++ b/apps/worker/lib/file-content-source.ts @@ -0,0 +1,205 @@ +import { + createGitHubClient, + createGitHubInstallationToken, + createGitLabClient, + decryptConnectorCredentials, + type ConnectorSecretPayload, + type GitHubConnectorCredentials, + type GitLabConnectorCredentials, + type GitLabInstallationCredentials, + type ProviderRepository, +} from "@kiwi/connectors"; +import { db } from "@kiwi/db"; +import { connectorInstallationsTable, connectorsTable, repositoryGraphBindingsTable } from "@kiwi/db/tables/connectors"; +import { getFile } from "@kiwi/files"; +import { eq } from "drizzle-orm"; +import { env } from "../env"; +import { parseCodeFileMetadata } from "./code-file-metadata"; + +export type FileContentSource = + | { kind: "internal"; key: string } + | { kind: "external"; provider: "github"; url: string; metadata?: string | null } + | { kind: "connector"; bindingId: string; provider: "github" | "gitlab"; metadata?: string | null }; + +const MAX_CODE_FILE_BYTES = 2 * 1024 * 1024; + +export function fileContentSourceFromRow(row: { + key: string; + storageKind?: string | null; + externalProvider?: string | null; + externalUrl?: string | null; + repositoryBindingId?: string | null; + metadata?: string | null; +}): FileContentSource { + if (row.storageKind === "external") { + if (row.repositoryBindingId) { + if (row.externalProvider !== "github" && row.externalProvider !== "gitlab") { + throw new Error("Unsupported connector file source"); + } + return { + kind: "connector", + provider: row.externalProvider, + bindingId: row.repositoryBindingId, + metadata: row.metadata, + }; + } + + if (row.externalProvider !== "github" || !row.externalUrl) { + throw new Error("Unsupported external file source"); + } + + return { kind: "external", provider: "github", url: row.externalUrl, metadata: row.metadata }; + } + + return { kind: "internal", key: row.key }; +} + +export async function readFileContentSource(source: FileContentSource): Promise { + if (source.kind === "internal") { + const file = await getFile(source.key, env.S3_BUCKET, "text"); + return file?.content ?? null; + } + + if (source.kind === "connector") { + return readConnectorFile(source.bindingId, source.metadata); + } + + return readExternalGitHubFile(source.url); +} + +function isGitHubConnectorCredentials(value: ConnectorSecretPayload): value is GitHubConnectorCredentials { + return "provider" in value && value.provider === "github"; +} + +function isGitLabConnectorCredentials(value: ConnectorSecretPayload): value is GitLabConnectorCredentials { + return "provider" in value && value.provider === "gitlab" && "baseUrl" in value; +} + +function isGitLabInstallationCredentials(value: ConnectorSecretPayload): value is GitLabInstallationCredentials { + return "provider" in value && value.provider === "gitlab" && "accessToken" in value; +} + +async function readConnectorFile(bindingId: string, metadataValue?: string | null): Promise { + const metadata = parseCodeFileMetadata(metadataValue); + if (!metadata) { + return null; + } + + const [row] = await db + .select({ + binding: repositoryGraphBindingsTable, + installation: connectorInstallationsTable, + connector: connectorsTable, + }) + .from(repositoryGraphBindingsTable) + .innerJoin( + connectorInstallationsTable, + eq(connectorInstallationsTable.id, repositoryGraphBindingsTable.connectorInstallationId) + ) + .innerJoin(connectorsTable, eq(connectorsTable.id, connectorInstallationsTable.connectorId)) + .where(eq(repositoryGraphBindingsTable.id, bindingId)) + .limit(1); + + if (!row || row.connector.status !== "active" || row.installation.status !== "active") { + return null; + } + + const connectorCredentials = decryptConnectorCredentials(row.connector.encryptedCredentials, env.AUTH_SECRET); + if (row.connector.provider === "github") { + if (!isGitHubConnectorCredentials(connectorCredentials)) { + return null; + } + const installationToken = await createGitHubInstallationToken({ + credentials: connectorCredentials, + installationId: row.installation.providerInstallationId, + }); + const client = createGitHubClient({ installationToken: installationToken.token }); + const repository: ProviderRepository = { + provider: "github", + id: row.binding.providerRepositoryId, + fullName: row.binding.repositoryFullName, + name: row.binding.repositoryFullName.split("/").at(-1) ?? row.binding.repositoryFullName, + htmlUrl: row.binding.repositoryHtmlUrl, + defaultBranch: row.binding.branch, + private: true, + }; + return client.readFile(repository, metadata.path, metadata.commitSha); + } + + if (!isGitLabConnectorCredentials(connectorCredentials)) { + return null; + } + const installationCredentials = row.installation.encryptedCredentials + ? decryptConnectorCredentials(row.installation.encryptedCredentials, env.AUTH_SECRET) + : null; + if (!installationCredentials || !isGitLabInstallationCredentials(installationCredentials)) { + return null; + } + const client = createGitLabClient({ + baseUrl: connectorCredentials.baseUrl, + accessToken: installationCredentials.accessToken, + }); + const repository: ProviderRepository = { + provider: "gitlab", + id: row.binding.providerRepositoryId, + fullName: row.binding.repositoryFullName, + name: row.binding.repositoryFullName.split("/").at(-1) ?? row.binding.repositoryFullName, + htmlUrl: row.binding.repositoryHtmlUrl, + defaultBranch: row.binding.branch, + private: true, + }; + return client.readFile(repository, metadata.path, metadata.commitSha); +} + +async function readExternalGitHubFile(url: string): Promise { + const parsed = new URL(url); + if (parsed.protocol !== "https:" || parsed.hostname !== "raw.githubusercontent.com") { + throw new Error("Unsupported external file source"); + } + + const response = await fetch(parsed, { redirect: "manual" }); + const responseUrl = new URL(response.url || parsed.href); + if (responseUrl.protocol !== "https:" || responseUrl.hostname !== "raw.githubusercontent.com") { + throw new Error("Unsupported external file source"); + } + + if (!response.ok) { + throw new Error("External file content not found"); + } + + const contentType = response.headers.get("content-type") ?? ""; + if (!contentType.toLowerCase().startsWith("text/")) { + throw new Error("External file content is not text"); + } + + const contentLength = response.headers.get("content-length"); + if (contentLength && Number(contentLength) > MAX_CODE_FILE_BYTES) { + throw new Error("External file content is too large"); + } + + if (!response.body) { + throw new Error("External file content is empty"); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let totalBytes = 0; + let content = ""; + + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + + totalBytes += value.byteLength; + if (totalBytes > MAX_CODE_FILE_BYTES) { + throw new Error("External file content is too large"); + } + + content += decoder.decode(value, { stream: true }); + } + + content += decoder.decode(); + return content; +} diff --git a/apps/worker/lib/file-processing-state.ts b/apps/worker/lib/file-processing-state.ts new file mode 100644 index 00000000..e8ee167b --- /dev/null +++ b/apps/worker/lib/file-processing-state.ts @@ -0,0 +1,39 @@ +import { db } from "@kiwi/db"; +import { filesTable, type FileProcessStatus, type FileProcessStep } from "@kiwi/db/tables/graph"; +import type { FileProcessErrorCode } from "@kiwi/contracts/routes"; +import { eq } from "drizzle-orm"; + +export async function updateFileProcessingState( + fileId: string, + processStep: FileProcessStep, + status: FileProcessStatus, + processErrorCode?: FileProcessErrorCode | null +) { + await db + .update(filesTable) + .set({ + processStep, + status, + ...(processErrorCode !== undefined + ? { processErrorCode } + : status === "failed" + ? {} + : { processErrorCode: null }), + }) + .where(eq(filesTable.id, fileId)); +} + +export async function stopIfFileDeleted(fileId: string) { + const [file] = await db + .select({ deleted: filesTable.deleted }) + .from(filesTable) + .where(eq(filesTable.id, fileId)) + .limit(1); + + if (file?.deleted) { + await updateFileProcessingState(fileId, "completed", "processed"); + return true; + } + + return false; +} diff --git a/apps/worker/lib/regenerate-descriptions.ts b/apps/worker/lib/regenerate-descriptions.ts index f51f6115..3c610220 100644 --- a/apps/worker/lib/regenerate-descriptions.ts +++ b/apps/worker/lib/regenerate-descriptions.ts @@ -1,5 +1,6 @@ import { db } from "@kiwi/db"; -import { entityTable, relationshipTable, sourcesTable } from "@kiwi/db/tables/graph"; +import { entityTable, filesTable, relationshipTable, sourcesTable, textUnitTable } from "@kiwi/db/tables/graph"; +import { unexpiredSourcePredicate, visibleFilePredicate } from "@kiwi/db/source-validity"; import { and, eq, inArray, isNotNull, sql } from "drizzle-orm"; import { embed, embedMany } from "ai"; import { withAiSlot } from "@kiwi/ai"; @@ -50,6 +51,7 @@ export async function updateSourceEmbeddingsBatch( )} ) AS batch(id, embedding) WHERE source.id = batch.id + AND source.valid_until IS NULL `); } @@ -85,12 +87,14 @@ async function regenerateDescriptions( description: string; embedding: number[]; sourceEmbeddings: SourceEmbeddingUpdate[]; - }) => Promise + }) => Promise, + deactivate: (id: string) => Promise ) { for (const chunk of chunkItems(items, DESCRIPTION_BATCH_SIZE)) { await Promise.all( chunk.map(async (item) => { if (item.sources.length === 0) { + await deactivate(item.id); return; } @@ -152,13 +156,17 @@ export async function regenerateEntities(graphId: string, entityIds: string[], c description: sourcesTable.description, }) .from(sourcesTable) + .innerJoin(textUnitTable, eq(textUnitTable.id, sourcesTable.textUnitId)) + .innerJoin(filesTable, eq(filesTable.id, textUnitTable.fileId)) .where( and( inArray( sourcesTable.entityId, entities.map((entity) => entity.id) ), - isNotNull(sourcesTable.entityId) + isNotNull(sourcesTable.entityId), + unexpiredSourcePredicate(sourcesTable), + visibleFilePredicate(filesTable) ) ); @@ -185,6 +193,9 @@ export async function regenerateEntities(graphId: string, entityIds: string[], c await updateSourceEmbeddingsBatch(tx, sourceEmbeddings); }); + }, + async (id) => { + await db.update(entityTable).set({ active: false }).where(eq(entityTable.id, id)); } ); } @@ -216,13 +227,17 @@ export async function regenerateRelationships(graphId: string, relationshipIds: description: sourcesTable.description, }) .from(sourcesTable) + .innerJoin(textUnitTable, eq(textUnitTable.id, sourcesTable.textUnitId)) + .innerJoin(filesTable, eq(filesTable.id, textUnitTable.fileId)) .where( and( inArray( sourcesTable.relationshipId, relationships.map((relationship) => relationship.id) ), - isNotNull(sourcesTable.relationshipId) + isNotNull(sourcesTable.relationshipId), + unexpiredSourcePredicate(sourcesTable), + visibleFilePredicate(filesTable) ) ); @@ -261,6 +276,9 @@ export async function regenerateRelationships(graphId: string, relationshipIds: await updateSourceEmbeddingsBatch(tx, sourceEmbeddings); }); + }, + async (id) => { + await db.update(relationshipTable).set({ active: false }).where(eq(relationshipTable.id, id)); } ); } diff --git a/apps/worker/lib/save-graph.ts b/apps/worker/lib/save-graph.ts new file mode 100644 index 00000000..5b929c2c --- /dev/null +++ b/apps/worker/lib/save-graph.ts @@ -0,0 +1,535 @@ +import { db } from "@kiwi/db"; +import { entityTable, filesTable, relationshipTable, sourcesTable, textUnitTable } from "@kiwi/db/tables/graph"; +import { currentSourceSql, unexpiredSourcePredicate, visibleFilePredicate, visibleFileSql } from "@kiwi/db/source-validity"; +import type { Graph } from "@kiwi/graph"; +import { eq, sql, and } from "drizzle-orm"; +import { chunkItems } from "./chunk"; +import { EMPTY_VECTOR_SQL, entityCompactNameKey, textArray } from "./sql"; +import { toTextUnitRows } from "./text-unit-rows"; + +const DEFAULT_RELATIONSHIP_KIND = "RELATED"; + +type GraphSaveTransaction = Parameters[0]>[0]; + +export type GraphSaveResult = { + summary: { + units: number; + entities: number; + relationships: number; + }; + duration: number; + metrics: { + insertUnitsDuration: number; + insertEntitiesDuration: number; + insertRelationshipsDuration: number; + dedupeEntitiesDuration: number; + dedupeRelationshipsDuration: number; + invalidateStaleSourcesDuration: number; + }; +}; + +export async function collectPendingDescriptionTargets(graphId: string) { + const newEntities = await db + .select({ id: entityTable.id, name: entityTable.name }) + .from(entityTable) + .where(and(eq(entityTable.graphId, graphId), eq(entityTable.active, false))); + + const updatedEntityRows = await db + .selectDistinct({ + id: entityTable.id, + name: entityTable.name, + description: entityTable.description, + }) + .from(entityTable) + .innerJoin(sourcesTable, eq(sourcesTable.entityId, entityTable.id)) + .innerJoin(textUnitTable, eq(textUnitTable.id, sourcesTable.textUnitId)) + .innerJoin(filesTable, eq(filesTable.id, textUnitTable.fileId)) + .where( + and( + eq(entityTable.graphId, graphId), + eq(entityTable.active, true), + eq(sourcesTable.active, false), + unexpiredSourcePredicate(sourcesTable), + visibleFilePredicate(filesTable) + ) + ); + const updatedEntities = Array.from(new Map(updatedEntityRows.map((entity) => [entity.id, entity])).values()); + + const newRelationships = await db + .select({ id: relationshipTable.id, sourceId: relationshipTable.sourceId, targetId: relationshipTable.targetId }) + .from(relationshipTable) + .where(and(eq(relationshipTable.graphId, graphId), eq(relationshipTable.active, false))); + + const updatedRelationshipRows = await db + .selectDistinct({ + id: relationshipTable.id, + sourceId: relationshipTable.sourceId, + targetId: relationshipTable.targetId, + description: relationshipTable.description, + }) + .from(relationshipTable) + .innerJoin(sourcesTable, eq(sourcesTable.relationshipId, relationshipTable.id)) + .innerJoin(textUnitTable, eq(textUnitTable.id, sourcesTable.textUnitId)) + .innerJoin(filesTable, eq(filesTable.id, textUnitTable.fileId)) + .where( + and( + eq(relationshipTable.graphId, graphId), + eq(relationshipTable.active, true), + eq(sourcesTable.active, false), + unexpiredSourcePredicate(sourcesTable), + visibleFilePredicate(filesTable) + ) + ); + const updatedRelationships = Array.from( + new Map(updatedRelationshipRows.map((relationship) => [relationship.id, relationship])).values() + ); + + return { + entityIds: [...newEntities.map((entity) => entity.id), ...updatedEntities.map((entity) => entity.id)], + relationshipIds: [ + ...newRelationships.map((relationship) => relationship.id), + ...updatedRelationships.map((relationship) => relationship.id), + ], + }; +} + +export async function saveGraphToDatabase(graphId: string, graph: Graph): Promise { + const start = performance.now(); + const rows = buildGraphRows(graphId, graph); + + const metrics = await db.transaction(async (tx) => { + const insertMetrics = await insertGraphRows(tx, rows); + const dedupeEntitiesDuration = await measureDuration(() => + dedupeEntityRows(tx, graphId, rows.insertedEntityIds) + ); + const dedupeRelationshipsDuration = await measureDuration(() => + dedupeRelationshipRows(tx, graphId, rows.insertedRelationshipIds) + ); + const invalidateStaleSourcesDuration = await measureDuration(() => + invalidateStaleCurrentCodeSources(tx, graphId, rows.insertedSourceIds) + ); + + return { + ...insertMetrics, + dedupeEntitiesDuration, + dedupeRelationshipsDuration, + invalidateStaleSourcesDuration, + }; + }); + + return { + summary: { + units: graph.units.length, + entities: graph.entities.length, + relationships: graph.relationships.length, + }, + duration: performance.now() - start, + metrics, + }; +} + +function buildGraphRows(graphId: string, graph: Graph) { + const unitRows = toTextUnitRows(graph.units); + const entityRows = graph.entities.map((entity) => ({ + id: entity.id, + graphId, + active: false, + name: entity.name, + description: "", + type: entity.type, + embedding: EMPTY_VECTOR_SQL, + })); + const sourceRows = [ + ...graph.entities.flatMap((entity) => + entity.sources.map((source) => ({ + id: source.id, + entityId: entity.id, + relationshipId: null, + textUnitId: source.unitId, + active: false, + description: source.description, + sourceChunkIds: source.sourceChunkIds ?? [], + embedding: EMPTY_VECTOR_SQL, + })) + ), + ...graph.relationships.flatMap((relationship) => + relationship.sources.map((source) => ({ + id: source.id, + entityId: null, + relationshipId: relationship.id, + textUnitId: source.unitId, + active: false, + description: source.description, + sourceChunkIds: source.sourceChunkIds ?? [], + embedding: EMPTY_VECTOR_SQL, + })) + ), + ]; + const relationshipRows = graph.relationships.map((relationship) => ({ + id: relationship.id, + active: false, + sourceId: relationship.sourceId, + targetId: relationship.targetId, + graphId, + kind: relationship.kind ?? DEFAULT_RELATIONSHIP_KIND, + directed: relationship.directed === true, + rank: relationship.strength, + description: "", + embedding: EMPTY_VECTOR_SQL, + })); + + const insertedEntityIds = entityRows.map((entity) => entity.id); + const insertedRelationshipIds = relationshipRows.map((relationship) => relationship.id); + const insertedSourceIds = sourceRows.map((source) => source.id); + + return { unitRows, entityRows, relationshipRows, sourceRows, insertedEntityIds, insertedRelationshipIds, insertedSourceIds }; +} + +async function measureDuration(work: () => Promise): Promise { + const start = performance.now(); + await work(); + return performance.now() - start; +} + +async function insertGraphRows(tx: GraphSaveTransaction, rows: ReturnType) { + const insertUnitsDuration = await measureDuration(async () => { + for (const chunk of chunkItems(rows.unitRows)) { + await tx + .insert(textUnitTable) + .values(chunk) + .onConflictDoUpdate({ + target: textUnitTable.id, + set: { + fileId: sql`excluded.file_id`, + text: sql`excluded.text`, + startPage: sql`excluded.start_page`, + endPage: sql`excluded.end_page`, + chunks: sql`excluded.chunks`, + updatedAt: sql`NOW()`, + }, + }); + } + }); + + const insertEntitiesDuration = await measureDuration(async () => { + for (const chunk of chunkItems(rows.entityRows)) { + await tx.insert(entityTable).values(chunk).onConflictDoNothing(); + } + }); + + const insertRelationshipsDuration = await measureDuration(async () => { + for (const chunk of chunkItems(rows.relationshipRows)) { + await tx.insert(relationshipTable).values(chunk).onConflictDoNothing(); + } + for (const chunk of chunkItems(rows.sourceRows)) { + await tx.insert(sourcesTable).values(chunk).onConflictDoNothing(); + } + }); + + return { insertUnitsDuration, insertEntitiesDuration, insertRelationshipsDuration }; +} + +async function dedupeEntityRows(tx: GraphSaveTransaction, graphId: string, insertedEntityIds: string[]) { + if (insertedEntityIds.length === 0) { + return; + } + + const entityIds = textArray(insertedEntityIds); + const candidateNameKeySql = sql.raw(entityCompactNameKey("candidate.name")); + const seededNameKeySql = sql.raw(entityCompactNameKey("seed.name")); + + await tx.execute(sql` + WITH seeded_keys AS ( + SELECT DISTINCT seed.type, ${seededNameKeySql} AS normalized_name + FROM entities seed + WHERE seed.graph_id = ${graphId} + AND seed.id = ANY(${entityIds}) + ), duplicates AS ( + SELECT + candidate.id, + first_value(candidate.id) OVER ( + PARTITION BY candidate.graph_id, candidate.type, ${candidateNameKeySql} + ORDER BY candidate.active DESC, candidate.id ASC + ) AS canonical_id + FROM entities candidate + JOIN seeded_keys seeded + ON seeded.type = candidate.type + AND seeded.normalized_name = ${candidateNameKeySql} + WHERE candidate.graph_id = ${graphId} + ) + UPDATE sources source + SET entity_id = duplicates.canonical_id + FROM duplicates + WHERE source.entity_id = duplicates.id + AND duplicates.id <> duplicates.canonical_id + `); + + await tx.execute(sql` + WITH seeded_keys AS ( + SELECT DISTINCT seed.type, ${seededNameKeySql} AS normalized_name + FROM entities seed + WHERE seed.graph_id = ${graphId} + AND seed.id = ANY(${entityIds}) + ), duplicates AS ( + SELECT + candidate.id, + first_value(candidate.id) OVER ( + PARTITION BY candidate.graph_id, candidate.type, ${candidateNameKeySql} + ORDER BY candidate.active DESC, candidate.id ASC + ) AS canonical_id + FROM entities candidate + JOIN seeded_keys seeded + ON seeded.type = candidate.type + AND seeded.normalized_name = ${candidateNameKeySql} + WHERE candidate.graph_id = ${graphId} + ) + UPDATE relationships relationship + SET source_id = duplicates.canonical_id + FROM duplicates + WHERE relationship.source_id = duplicates.id + AND relationship.graph_id = ${graphId} + AND duplicates.id <> duplicates.canonical_id + `); + + await tx.execute(sql` + WITH seeded_keys AS ( + SELECT DISTINCT seed.type, ${seededNameKeySql} AS normalized_name + FROM entities seed + WHERE seed.graph_id = ${graphId} + AND seed.id = ANY(${entityIds}) + ), duplicates AS ( + SELECT + candidate.id, + first_value(candidate.id) OVER ( + PARTITION BY candidate.graph_id, candidate.type, ${candidateNameKeySql} + ORDER BY candidate.active DESC, candidate.id ASC + ) AS canonical_id + FROM entities candidate + JOIN seeded_keys seeded + ON seeded.type = candidate.type + AND seeded.normalized_name = ${candidateNameKeySql} + WHERE candidate.graph_id = ${graphId} + ) + UPDATE relationships relationship + SET target_id = duplicates.canonical_id + FROM duplicates + WHERE relationship.target_id = duplicates.id + AND relationship.graph_id = ${graphId} + AND duplicates.id <> duplicates.canonical_id + `); + + await tx.execute(sql` + WITH seeded_keys AS ( + SELECT DISTINCT seed.type, ${seededNameKeySql} AS normalized_name + FROM entities seed + WHERE seed.graph_id = ${graphId} + AND seed.id = ANY(${entityIds}) + ), duplicates AS ( + SELECT + candidate.id, + first_value(candidate.id) OVER ( + PARTITION BY candidate.graph_id, candidate.type, ${candidateNameKeySql} + ORDER BY candidate.active DESC, candidate.id ASC + ) AS canonical_id + FROM entities candidate + JOIN seeded_keys seeded + ON seeded.type = candidate.type + AND seeded.normalized_name = ${candidateNameKeySql} + WHERE candidate.graph_id = ${graphId} + ) + DELETE FROM entities entity + USING duplicates + WHERE entity.id = duplicates.id + AND duplicates.id <> duplicates.canonical_id + `); +} + +async function dedupeRelationshipRows(tx: GraphSaveTransaction, graphId: string, insertedRelationshipIds: string[]) { + await tx.execute(sql` + DELETE FROM relationships + WHERE graph_id = ${graphId} + AND directed = false + AND source_id = target_id + `); + + if (insertedRelationshipIds.length === 0) { + return; + } + + const relationshipIds = textArray(insertedRelationshipIds); + + await tx.execute(sql` + WITH seeded_pairs AS ( + SELECT DISTINCT + relationship.kind, + relationship.directed, + CASE WHEN relationship.directed THEN relationship.source_id ELSE least(relationship.source_id, relationship.target_id) END AS pair_source_id, + CASE WHEN relationship.directed THEN relationship.target_id ELSE greatest(relationship.source_id, relationship.target_id) END AS pair_target_id + FROM relationships relationship + WHERE relationship.graph_id = ${graphId} + AND relationship.id = ANY(${relationshipIds}) + ), duplicates AS ( + SELECT + relationship.id, + CASE WHEN relationship.directed THEN relationship.source_id ELSE least(relationship.source_id, relationship.target_id) END AS canonical_source_id, + CASE WHEN relationship.directed THEN relationship.target_id ELSE greatest(relationship.source_id, relationship.target_id) END AS canonical_target_id, + first_value(relationship.id) OVER ( + PARTITION BY relationship.graph_id, + relationship.kind, + relationship.directed, + CASE WHEN relationship.directed THEN relationship.source_id ELSE least(relationship.source_id, relationship.target_id) END, + CASE WHEN relationship.directed THEN relationship.target_id ELSE greatest(relationship.source_id, relationship.target_id) END + ORDER BY relationship.active DESC, relationship.id ASC + ) AS canonical_id, + max(relationship.rank) OVER ( + PARTITION BY relationship.graph_id, + relationship.kind, + relationship.directed, + CASE WHEN relationship.directed THEN relationship.source_id ELSE least(relationship.source_id, relationship.target_id) END, + CASE WHEN relationship.directed THEN relationship.target_id ELSE greatest(relationship.source_id, relationship.target_id) END + ) AS canonical_rank + FROM relationships relationship + JOIN seeded_pairs seeded + ON seeded.kind = relationship.kind + AND seeded.directed = relationship.directed + AND seeded.pair_source_id = CASE WHEN relationship.directed THEN relationship.source_id ELSE least(relationship.source_id, relationship.target_id) END + AND seeded.pair_target_id = CASE WHEN relationship.directed THEN relationship.target_id ELSE greatest(relationship.source_id, relationship.target_id) END + WHERE relationship.graph_id = ${graphId} + ) + UPDATE relationships relationship + SET source_id = duplicates.canonical_source_id, + target_id = duplicates.canonical_target_id, + rank = CASE + WHEN relationship.id = duplicates.canonical_id THEN duplicates.canonical_rank + ELSE relationship.rank + END, + updated_at = NOW() + FROM duplicates + WHERE relationship.id = duplicates.id + AND ( + relationship.source_id <> duplicates.canonical_source_id + OR relationship.target_id <> duplicates.canonical_target_id + OR ( + relationship.id = duplicates.canonical_id + AND relationship.rank <> duplicates.canonical_rank + ) + ) + `); + + await tx.execute(sql` + WITH seeded_pairs AS ( + SELECT DISTINCT + relationship.kind, + relationship.directed, + CASE WHEN relationship.directed THEN relationship.source_id ELSE least(relationship.source_id, relationship.target_id) END AS pair_source_id, + CASE WHEN relationship.directed THEN relationship.target_id ELSE greatest(relationship.source_id, relationship.target_id) END AS pair_target_id + FROM relationships relationship + WHERE relationship.graph_id = ${graphId} + AND relationship.id = ANY(${relationshipIds}) + ), duplicates AS ( + SELECT + relationship.id, + first_value(relationship.id) OVER ( + PARTITION BY relationship.graph_id, + relationship.kind, + relationship.directed, + CASE WHEN relationship.directed THEN relationship.source_id ELSE least(relationship.source_id, relationship.target_id) END, + CASE WHEN relationship.directed THEN relationship.target_id ELSE greatest(relationship.source_id, relationship.target_id) END + ORDER BY relationship.active DESC, relationship.id ASC + ) AS canonical_id + FROM relationships relationship + JOIN seeded_pairs seeded + ON seeded.kind = relationship.kind + AND seeded.directed = relationship.directed + AND seeded.pair_source_id = CASE WHEN relationship.directed THEN relationship.source_id ELSE least(relationship.source_id, relationship.target_id) END + AND seeded.pair_target_id = CASE WHEN relationship.directed THEN relationship.target_id ELSE greatest(relationship.source_id, relationship.target_id) END + WHERE relationship.graph_id = ${graphId} + ) + UPDATE sources source + SET relationship_id = duplicates.canonical_id + FROM duplicates + WHERE source.relationship_id = duplicates.id + AND duplicates.id <> duplicates.canonical_id + `); + + await tx.execute(sql` + WITH seeded_pairs AS ( + SELECT DISTINCT + relationship.kind, + relationship.directed, + CASE WHEN relationship.directed THEN relationship.source_id ELSE least(relationship.source_id, relationship.target_id) END AS pair_source_id, + CASE WHEN relationship.directed THEN relationship.target_id ELSE greatest(relationship.source_id, relationship.target_id) END AS pair_target_id + FROM relationships relationship + WHERE relationship.graph_id = ${graphId} + AND relationship.id = ANY(${relationshipIds}) + ), duplicates AS ( + SELECT + relationship.id, + first_value(relationship.id) OVER ( + PARTITION BY relationship.graph_id, + relationship.kind, + relationship.directed, + CASE WHEN relationship.directed THEN relationship.source_id ELSE least(relationship.source_id, relationship.target_id) END, + CASE WHEN relationship.directed THEN relationship.target_id ELSE greatest(relationship.source_id, relationship.target_id) END + ORDER BY relationship.active DESC, relationship.id ASC + ) AS canonical_id + FROM relationships relationship + JOIN seeded_pairs seeded + ON seeded.kind = relationship.kind + AND seeded.directed = relationship.directed + AND seeded.pair_source_id = CASE WHEN relationship.directed THEN relationship.source_id ELSE least(relationship.source_id, relationship.target_id) END + AND seeded.pair_target_id = CASE WHEN relationship.directed THEN relationship.target_id ELSE greatest(relationship.source_id, relationship.target_id) END + WHERE relationship.graph_id = ${graphId} + ) + DELETE FROM relationships relationship + USING duplicates + WHERE relationship.id = duplicates.id + AND duplicates.id <> duplicates.canonical_id + `); +} + +async function invalidateStaleCurrentCodeSources(tx: GraphSaveTransaction, graphId: string, insertedSourceIds: string[]) { + if (insertedSourceIds.length === 0) { + return; + } + + const sourceIds = textArray(insertedSourceIds); + + await tx.execute(sql` + WITH new_code_sources AS ( + SELECT DISTINCT source.id, source.entity_id, source.relationship_id + FROM sources source + INNER JOIN text_units text_unit ON text_unit.id = source.text_unit_id + INNER JOIN files file ON file.id = text_unit.file_id + WHERE source.id = ANY(${sourceIds}) + AND source.valid_until IS NULL + AND file.graph_id = ${graphId} + AND ${visibleFileSql("file")} + AND file.file_type = 'code' + ), stale_sources AS ( + SELECT old_source.id + FROM sources old_source + INNER JOIN text_units old_text_unit ON old_text_unit.id = old_source.text_unit_id + INNER JOIN files old_file ON old_file.id = old_text_unit.file_id + INNER JOIN new_code_sources new_source + ON ( + new_source.entity_id IS NOT NULL + AND old_source.entity_id = new_source.entity_id + ) + OR ( + new_source.relationship_id IS NOT NULL + AND old_source.relationship_id = new_source.relationship_id + ) + WHERE ${currentSourceSql("old_source")} + AND ${visibleFileSql("old_file")} + AND old_file.graph_id = ${graphId} + AND old_file.file_type = 'code' + AND old_source.id <> ALL(${sourceIds}) + ) + UPDATE sources source + SET valid_until = NOW(), + updated_at = NOW() + FROM stale_sources + WHERE source.id = stale_sources.id + `); +} diff --git a/apps/worker/package.json b/apps/worker/package.json index fdfb290e..a49dfc0e 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -15,6 +15,7 @@ "dependencies": { "@kiwi/ai": "workspace:*", "@kiwi/contracts": "workspace:*", + "@kiwi/connectors": "workspace:*", "@kiwi/db": "workspace:*", "@kiwi/files": "workspace:*", "@kiwi/graph": "workspace:*", diff --git a/apps/worker/worker.ts b/apps/worker/worker.ts index e58c6553..71720dbf 100644 --- a/apps/worker/worker.ts +++ b/apps/worker/worker.ts @@ -108,14 +108,18 @@ async function startWorkerProcess() { { deleteProjectFile }, { deleteGraphFiles }, { processFile, processFiles }, + { processCodeFile }, { updateDescriptions }, + { syncRepositoryGraph }, ] = await Promise.all([ import("."), import("./env"), import("./workflows/delete-file"), import("./workflows/delete-graph-files"), import("./workflows/process-file"), + import("./workflows/process-code-file"), import("./workflows/update-descriptions"), + import("./workflows/sync-repository-graph"), ]); const parentPid = process.ppid; @@ -123,9 +127,11 @@ async function startWorkerProcess() { ow.implementWorkflow(processFiles.spec, processFiles.fn); ow.implementWorkflow(processFile.spec, processFile.fn); + ow.implementWorkflow(processCodeFile.spec, processCodeFile.fn); ow.implementWorkflow(deleteProjectFile.spec, deleteProjectFile.fn); ow.implementWorkflow(deleteGraphFiles.spec, deleteGraphFiles.fn); ow.implementWorkflow(updateDescriptions.spec, updateDescriptions.fn); + ow.implementWorkflow(syncRepositoryGraph.spec, syncRepositoryGraph.fn); const worker = ow.newWorker({ concurrency }); let shuttingDown = false; diff --git a/apps/worker/workflows/process-code-file.ts b/apps/worker/workflows/process-code-file.ts new file mode 100644 index 00000000..2f264999 --- /dev/null +++ b/apps/worker/workflows/process-code-file.ts @@ -0,0 +1,232 @@ +import { db } from "@kiwi/db"; +import { filesTable, processStatsTable } from "@kiwi/db/tables/graph"; +import { and, eq } from "drizzle-orm"; +import { defineWorkflow } from "openworkflow"; +import z from "zod"; +import { estimateToken } from "@kiwi/ai"; +import type { Graph } from "@kiwi/graph"; +import type { CodeRepositoryFile } from "@kiwi/graph/code/repository"; +import { getFile, putNamedFile } from "@kiwi/files"; +import { error as logError } from "@kiwi/logger"; +import { env } from "../env"; +import { deleteGraphFileProcessingArtifacts, getGraphFileArtifactPaths } from "../lib/derived-files"; +import { classifyFileProcessError } from "../lib/file-process-error"; +import { loadCodeManifest } from "../lib/code-manifest"; +import { fileContentSourceFromRow, readFileContentSource } from "../lib/file-content-source"; +import { parseCodeFileMetadata } from "../lib/code-file-metadata"; +import { updateFileProcessingState, stopIfFileDeleted } from "../lib/file-processing-state"; +import { saveGraphToDatabase } from "../lib/save-graph"; + +const FILE_DELETED = "__file_deleted__" as const; + +type ProcessFileRow = typeof filesTable.$inferSelect; + +function workflowError(error: unknown) { + if (error instanceof Error) { + return new Error(error.message, { cause: error }); + } + + return new Error("Workflow failed", { cause: error }); +} + +function codeRepositoryFileFromRow(file: ProcessFileRow, content: string): CodeRepositoryFile { + const metadata = parseCodeFileMetadata(file.metadata); + + return { + fileId: file.id, + repositoryUrl: metadata?.repositoryUrl ?? `graph:${file.graphId}`, + repositoryName: metadata?.repositoryName ?? "code", + commitSha: metadata?.commitSha ?? "unknown", + path: metadata?.path ?? file.name, + content, + }; +} + +export const processCodeFile = defineWorkflow( + { + name: "process-code-file", + version: "1.0.0", + retryPolicy: { + initialInterval: "1s", + backoffCoefficient: 2, + maximumInterval: "30s", + maximumAttempts: 3, + }, + schema: z.object({ + graphId: z.string(), + fileId: z.string(), + codeManifestKey: z.string().optional(), + }), + }, + async ({ input, step, run }) => { + try { + const [fileData] = await step.run({ name: "get-code-file-data" }, async () => { + return db + .select() + .from(filesTable) + .where(and(eq(filesTable.graphId, input.graphId), eq(filesTable.id, input.fileId))) + .limit(1); + }); + + if (!fileData) { + return; + } + + if (fileData.deleted) { + await updateFileProcessingState(input.fileId, "completed", "processed"); + return; + } + + const paths = getGraphFileArtifactPaths({ + graphId: input.graphId, + fileId: input.fileId, + fileKey: fileData.key, + }); + + const baseFile = await step.run({ name: "preprocess-code-file" }, async () => { + if (await stopIfFileDeleted(input.fileId)) { + return FILE_DELETED; + } + + await updateFileProcessingState(input.fileId, "preprocessing", "processing"); + const start = performance.now(); + const source = await readFileContentSource(fileContentSourceFromRow(fileData)); + if (source === null) { + throw new Error(`Failed to load file ${fileData.key}`); + } + + if (source.trim() === "") { + throw new Error("No readable text found in file"); + } + + const repositoryFile = codeRepositoryFileFromRow(fileData, source); + const tokenCount = estimateToken(source); + + await db + .update(filesTable) + .set({ tokenCount, loader: "repository", chunker: "ast" }) + .where(eq(filesTable.id, input.fileId)); + + return { + repositoryFile, + duration: performance.now() - start, + tokenCount, + }; + }); + if (baseFile === FILE_DELETED) { + return; + } + + const graphResult = await step.run({ name: "build-code-graph" }, async () => { + if (await stopIfFileDeleted(input.fileId)) { + return FILE_DELETED; + } + + await updateFileProcessingState(input.fileId, "extracting", "processing"); + const start = performance.now(); + const { buildCodeFileGraph, buildCodeRepositoryManifest } = await import("@kiwi/graph/code/repository"); + const manifest = input.codeManifestKey + ? await loadCodeManifest(input.codeManifestKey) + : buildCodeRepositoryManifest([baseFile.repositoryFile]); + const graph = buildCodeFileGraph(baseFile.repositoryFile, manifest); + const uploadedGraph = await putNamedFile( + "graph.json", + JSON.stringify(graph), + paths.processingPrefix, + env.S3_BUCKET + ); + + return { + graphKey: uploadedGraph.key, + duration: performance.now() - start, + }; + }); + if (graphResult === FILE_DELETED) { + return; + } + + const saveGraphResult = await step.run({ name: "save-code-graph" }, async () => { + if (await stopIfFileDeleted(input.fileId)) { + return FILE_DELETED; + } + + await updateFileProcessingState(input.fileId, "saving", "processing"); + const loadedGraph = await getFile(graphResult.graphKey, env.S3_BUCKET, "json"); + if (!loadedGraph) { + throw new Error(`Failed to load graph from ${graphResult.graphKey}`); + } + + const saveResult = await saveGraphToDatabase(input.graphId, loadedGraph.content); + + return { + summary: { + fileId: input.fileId, + ...saveResult.summary, + }, + duration: saveResult.duration, + metrics: saveResult.metrics, + }; + }); + if (saveGraphResult === FILE_DELETED) { + return; + } + + const statsResult = await step.run({ name: "store-code-process-stats" }, async () => { + if (await stopIfFileDeleted(input.fileId)) { + return FILE_DELETED; + } + + try { + await db.insert(processStatsTable).values({ + totalTime: baseFile.duration + graphResult.duration + saveGraphResult.duration, + files: 1, + fileSizes: fileData.size, + fileType: "code", + tokenCount: baseFile.tokenCount, + }); + } catch (error) { + logError("failed to store code file process stats", { + graphId: input.graphId, + fileId: input.fileId, + error, + }); + } + }); + if (statsResult === FILE_DELETED) { + return; + } + + await step.run({ name: "mark-code-file-complete" }, async () => { + await updateFileProcessingState(input.fileId, "completed", "processed"); + }); + + await step.run({ name: "cleanup-code-processing-artifacts" }, async () => { + try { + return await deleteGraphFileProcessingArtifacts({ + graphId: input.graphId, + fileId: input.fileId, + fileKey: fileData.key, + bucket: env.S3_BUCKET, + }); + } catch (error) { + logError("failed to cleanup code processing artifacts", { + graphId: input.graphId, + fileId: input.fileId, + error, + }); + return { deletedKeyCount: 0 }; + } + }); + + return saveGraphResult.summary; + } catch (error) { + if (run.retryTerminal) { + await updateFileProcessingState(input.fileId, "failed", "failed", classifyFileProcessError(error)); + } else { + await updateFileProcessingState(input.fileId, "pending", "processing", null); + } + + throw workflowError(error); + } + } +); diff --git a/apps/worker/workflows/process-file.test.ts b/apps/worker/workflows/process-file.test.ts new file mode 100644 index 00000000..7659df15 --- /dev/null +++ b/apps/worker/workflows/process-file.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, test } from "bun:test"; + +process.env.AUTH_SECRET = "test-secret"; +process.env.S3_ACCESS_KEY_ID = "test-access-key"; +process.env.S3_SECRET_ACCESS_KEY = "test-secret-key"; +process.env.S3_ENDPOINT = "http://localhost:9000"; +process.env.S3_REGION = "us-east-1"; +process.env.S3_BUCKET = "test-bucket"; +process.env.DATABASE_URL = "postgres://user:pass@localhost:5432/kiwi"; +process.env.DATABASE_DIRECT_URL = "postgres://user:pass@localhost:5432/kiwi"; + +// Dynamic import is required so this test can seed worker env vars before env.ts is evaluated. +const { fileProcessingWorkflow, shouldAbortRepositoryBatch, shouldFinalizeRepositoryBatch } = await import( + "./process-file" +); +const { resolveRepositoryFinalizationTargets } = await import("../lib/code-repository-finalizer"); + +describe("fileProcessingWorkflow", () => { + test("routes mixed batch children by stored file type", () => { + const codeWorkflow = fileProcessingWorkflow("graph-1", "code-file", "code", "manifest-key"); + const textWorkflow = fileProcessingWorkflow("graph-1", "text-file", "text", "manifest-key"); + + expect(codeWorkflow.spec.name).toBe("process-code-file"); + expect(codeWorkflow.input).toEqual({ + graphId: "graph-1", + fileId: "code-file", + codeManifestKey: "manifest-key", + }); + expect(textWorkflow.spec.name).toBe("process-file"); + expect(textWorkflow.input).toEqual({ + graphId: "graph-1", + fileId: "text-file", + }); + }); +}); + +describe("repository batch guards", () => { + test("finalizes repository batches only after every child workflow succeeds", () => { + expect( + shouldFinalizeRepositoryBatch( + { kind: "repository", retiredFileIds: ["old-file"] }, + [{ status: "fulfilled", value: undefined }] + ) + ).toBe(true); + expect(shouldFinalizeRepositoryBatch({ kind: "repository", retiredFileIds: ["old-file"] }, [])).toBe(true); + expect( + shouldFinalizeRepositoryBatch( + { kind: "repository", retiredFileIds: ["old-file"] }, + [ + { status: "fulfilled", value: undefined }, + { status: "rejected", reason: new Error("failed") }, + ] + ) + ).toBe(false); + expect(shouldFinalizeRepositoryBatch(undefined, [{ status: "fulfilled", value: undefined }])).toBe(false); + }); + + test("aborts incremental repository batches when any child workflow fails", () => { + expect( + shouldAbortRepositoryBatch( + { kind: "repository", retiredFileIds: [] }, + [ + { status: "fulfilled", value: undefined }, + { status: "rejected", reason: new Error("failed") }, + ] + ) + ).toBe(true); + expect( + shouldAbortRepositoryBatch( + { kind: "repository" }, + [ + { status: "fulfilled", value: undefined }, + { status: "rejected", reason: new Error("failed") }, + ] + ) + ).toBe(false); + }); +}); + +describe("resolveRepositoryFinalizationTargets", () => { + test("targets older files for the same repository URL and preserves other repositories", () => { + const latestMetadata = JSON.stringify({ + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-2", + path: "src/index.ts", + }); + const oldMetadata = JSON.stringify({ + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-1", + path: "src/removed.ts", + }); + const otherMetadata = JSON.stringify({ + repositoryUrl: "https://github.com/acme/other.git", + repositoryName: "other", + commitSha: "commit-1", + path: "src/index.ts", + }); + + expect( + resolveRepositoryFinalizationTargets( + [{ id: "new-file", metadata: latestMetadata }], + [ + { id: "new-file", metadata: latestMetadata }, + { id: "old-file", metadata: oldMetadata }, + { id: "other-file", metadata: otherMetadata }, + ] + ) + ).toEqual({ + repositoryUrls: ["https://github.com/acme/widgets.git"], + olderFileIds: ["old-file"], + }); + }); +}); diff --git a/apps/worker/workflows/process-file.ts b/apps/worker/workflows/process-file.ts index f5d486b6..c73d9e75 100644 --- a/apps/worker/workflows/process-file.ts +++ b/apps/worker/workflows/process-file.ts @@ -1,26 +1,15 @@ import { db } from "@kiwi/db"; -import { - type FileProcessStatus, - type FileProcessStep, - entityTable, - filesTable, - graphTable, - processRunsTable, - processStatsTable, - relationshipTable, - sourcesTable, - textUnitTable, -} from "@kiwi/db/tables/graph"; +import { filesTable, graphTable, processRunsTable, processStatsTable } from "@kiwi/db/tables/graph"; import { and, eq, inArray, sql } from "drizzle-orm"; import { defineWorkflow } from "openworkflow"; import z from "zod"; import { S3Loader } from "@kiwi/graph/loader/s3"; import { createGraphChunker } from "@kiwi/graph/chunker/factory"; -import type { FileProcessErrorCode } from "@kiwi/contracts/routes"; import { env } from "../env"; -import { type Graph, type GraphFile, type LoadedGraphDocument, type Unit } from "@kiwi/graph"; +import type { Graph, GraphFile, LoadedGraphDocument, Unit } from "@kiwi/graph"; import { dedupe } from "@kiwi/graph/dedupe"; import { coerceGraphFileType } from "@kiwi/graph/file-type"; +import type { GraphFileType } from "@kiwi/graph/file-type"; import { loadGraphDocument } from "@kiwi/graph/loader/document"; import { createDetectedGraphLoader, detectGraphLoaderFileFormat } from "@kiwi/graph/loader/factory"; import { mergeGraphs } from "@kiwi/graph/merge"; @@ -30,7 +19,6 @@ import { resolveGraphModelOrganizationId } from "@kiwi/ai/models"; import { getFile, putNamedFile } from "@kiwi/files"; import { error as logError } from "@kiwi/logger"; import { createWorkerClient } from "../lib/ai"; -import { EMPTY_VECTOR_SQL, entityCompactNameKey, textArray } from "../lib/sql"; import { chunkItems } from "../lib/chunk"; import { processFilesSpec } from "./process-files-spec"; import { deleteGraphFileProcessingArtifacts, getGraphFileArtifactPaths } from "../lib/derived-files"; @@ -38,9 +26,13 @@ import { buildMetadata, buildMetadataExcerpt } from "../lib/metadata"; import { updateDescriptionsSpec } from "./update-descriptions-spec"; import { DESCRIPTION_BATCH_SIZE } from "../lib/description-workflow"; import { getFileTypeProcessingConfig } from "../lib/file-type-config"; -import { toTextUnitRows } from "../lib/text-unit-rows"; import { requireReadableContentText } from "../lib/readable-text"; import { classifyFileProcessError } from "../lib/file-process-error"; +import { collectPendingDescriptionTargets, saveGraphToDatabase } from "../lib/save-graph"; +import { prepareCodeManifest } from "../lib/code-manifest"; +import { invalidateSupersededRepositorySources } from "../lib/code-repository-finalizer"; +import { updateFileProcessingState, stopIfFileDeleted } from "../lib/file-processing-state"; +import { processCodeFile } from "./process-code-file"; const FILE_DELETED = "__file_deleted__" as const; const NO_RETRY = { maximumAttempts: 1 } as const; @@ -54,39 +46,46 @@ function workflowError(error: unknown) { return new Error("Workflow failed", { cause: error }); } -async function updateFileProcessingState( +export function fileProcessingWorkflow( + graphId: string, fileId: string, - processStep: FileProcessStep, - status: FileProcessStatus, - processErrorCode?: FileProcessErrorCode | null + fileType: GraphFileType | undefined, + codeManifestKey?: string ) { - await db - .update(filesTable) - .set({ - processStep, - status, - ...(processErrorCode !== undefined - ? { processErrorCode } - : status === "failed" - ? {} - : { processErrorCode: null }), - }) - .where(eq(filesTable.id, fileId)); + return fileType === "code" + ? { + spec: processCodeFile.spec, + input: { + graphId, + fileId, + ...(codeManifestKey ? { codeManifestKey } : {}), + }, + } + : { + spec: processFile.spec, + input: { + graphId, + fileId, + }, + }; } -async function stopIfFileDeleted(fileId: string) { - const [file] = await db - .select({ deleted: filesTable.deleted }) - .from(filesTable) - .where(eq(filesTable.id, fileId)) - .limit(1); - - if (file?.deleted) { - await updateFileProcessingState(fileId, "completed", "processed"); - return true; - } +export function shouldAbortRepositoryBatch( + code: { kind: "repository"; retiredFileIds?: string[] } | undefined, + results: PromiseSettledResult[] +) { + return code?.kind === "repository" && code.retiredFileIds !== undefined && results.some((result) => result.status === "rejected"); +} - return false; +export function shouldFinalizeRepositoryBatch( + code: { kind: "repository"; retiredFileIds?: string[] } | undefined, + results: PromiseSettledResult[] +) { + return ( + code?.kind === "repository" && + results.every((result) => result.status === "fulfilled") && + (results.length > 0 || (code.retiredFileIds?.length ?? 0) > 0) + ); } export const processFiles = defineWorkflow(processFilesSpec, async ({ input, step, run }) => { @@ -118,86 +117,81 @@ export const processFiles = defineWorkflow(processFilesSpec, async ({ input, ste } }); - const fileResults = await Promise.allSettled( - input.fileIds.map((fileId) => - step.runWorkflow(processFile.spec, { - graphId: input.graphId, - fileId, + const fileTypeRows = await step.run({ name: "load-file-types" }, async () => { + if (input.fileIds.length === 0) { + return []; + } + + return db + .select({ + id: filesTable.id, + type: filesTable.type, }) - ) + .from(filesTable) + .where(and(eq(filesTable.graphId, input.graphId), inArray(filesTable.id, input.fileIds))); + }); + const fileTypes = new Map(fileTypeRows.map((file) => [file.id, coerceGraphFileType(file.type)])); + const hasCodeFiles = [...fileTypes.values()].some((type) => type === "code"); + + const codeManifestKey = + input.code && hasCodeFiles + ? await step.run({ name: "prepare-code-manifest" }, async () => + prepareCodeManifest({ + graphId: input.graphId, + fileIds: input.fileIds.filter((fileId) => fileTypes.get(fileId) === "code"), + processRunId: input.processRunId, + }) + ) + : undefined; + + const fileResults = await Promise.allSettled( + input.fileIds.map((fileId) => { + const workflow = fileProcessingWorkflow(input.graphId, fileId, fileTypes.get(fileId), codeManifestKey); + return step.runWorkflow(workflow.spec, workflow.input); + }) ); if (fileResults.length > 0 && fileResults.every((result) => result.status === "rejected")) { throw new Error(`All ${fileResults.length} file processing workflows failed`); } - - const descriptions = await step.run({ name: "generate-descriptions" }, async () => { - const newEntities = await db - .select({ id: entityTable.id, name: entityTable.name }) - .from(entityTable) - .where(and(eq(entityTable.graphId, input.graphId), eq(entityTable.active, false))); - - const updatedEntityRows = await db - .selectDistinct({ - id: entityTable.id, - name: entityTable.name, - description: entityTable.description, - }) - .from(entityTable) - .innerJoin(sourcesTable, eq(sourcesTable.entityId, entityTable.id)) - .where( - and( - eq(entityTable.graphId, input.graphId), - eq(entityTable.active, true), - eq(sourcesTable.active, false) - ) - ); - const updatedEntities = Array.from( - new Map(updatedEntityRows.map((entity) => [entity.id, entity])).values() + if (shouldAbortRepositoryBatch(input.code, fileResults)) { + throw new Error( + `${fileResults.filter((result) => result.status === "rejected").length} repository file processing workflows failed` ); + } - const entityIds = [ - ...newEntities.map((entity) => entity.id), - ...updatedEntities.map((entity) => entity.id), - ]; - - const newRelationships = await db - .select({ - id: relationshipTable.id, - sourceId: relationshipTable.sourceId, - targetId: relationshipTable.targetId, - }) - .from(relationshipTable) - .where(and(eq(relationshipTable.graphId, input.graphId), eq(relationshipTable.active, false))); - - const updatedRelationshipRows = await db - .selectDistinct({ - id: relationshipTable.id, - sourceId: relationshipTable.sourceId, - targetId: relationshipTable.targetId, - description: relationshipTable.description, - }) - .from(relationshipTable) - .innerJoin(sourcesTable, eq(sourcesTable.relationshipId, relationshipTable.id)) - .where( - and( - eq(relationshipTable.graphId, input.graphId), - eq(relationshipTable.active, true), - eq(sourcesTable.active, false) - ) - ); - const updatedRelationships = Array.from( - new Map(updatedRelationshipRows.map((relationship) => [relationship.id, relationship])).values() + if (shouldFinalizeRepositoryBatch(input.code, fileResults)) { + const invalidated = await step.run({ name: "finalize-repository-snapshot" }, async () => + invalidateSupersededRepositorySources( + input.code?.retiredFileIds !== undefined + ? { + graphId: input.graphId, + retiredFileIds: input.code.retiredFileIds, + } + : { + graphId: input.graphId, + latestFileIds: input.fileIds.filter((fileId) => fileTypes.get(fileId) === "code"), + } + ) ); - const relationshipIds = [ - ...newRelationships.map((relationship) => relationship.id), - ...updatedRelationships.map((relationship) => relationship.id), - ]; + await Promise.all([ + ...chunkItems(invalidated.entityIds, DESCRIPTION_BATCH_SIZE).map((entityIds) => + step.runWorkflow(updateDescriptionsSpec, { + graphId: input.graphId, + entityIds, + }) + ), + ...chunkItems(invalidated.relationshipIds, DESCRIPTION_BATCH_SIZE).map((relationshipIds) => + step.runWorkflow(updateDescriptionsSpec, { + graphId: input.graphId, + relationshipIds, + }) + ), + ]); + } - return { - entityIds, - relationshipIds, - }; + const descriptions = await step.run({ name: "generate-descriptions" }, async () => { + return collectPendingDescriptionTargets(input.graphId); }); await Promise.all([ @@ -501,7 +495,6 @@ export const processFile = defineWorkflow( } await updateFileProcessingState(input.fileId, "deduplicating", "processing"); - const start = performance.now(); await updateFileProcessingState(input.fileId, "saving", "processing"); @@ -511,337 +504,15 @@ export const processFile = defineWorkflow( } const graph = loadedGraph.content; - const unitRows = toTextUnitRows(graph.units); - const entityRows = graph.entities.map((entity) => ({ - id: entity.id, - graphId: input.graphId, - active: false, - name: entity.name, - description: "", - type: entity.type, - embedding: EMPTY_VECTOR_SQL, - })); - const sourceRows = [ - ...graph.entities.flatMap((entity) => - entity.sources.map((source) => ({ - id: source.id, - entityId: entity.id, - relationshipId: null, - textUnitId: source.unitId, - active: false, - description: source.description, - sourceChunkIds: source.sourceChunkIds ?? [], - embedding: EMPTY_VECTOR_SQL, - })) - ), - ...graph.relationships.flatMap((relationship) => - relationship.sources.map((source) => ({ - id: source.id, - entityId: null, - relationshipId: relationship.id, - textUnitId: source.unitId, - active: false, - description: source.description, - sourceChunkIds: source.sourceChunkIds ?? [], - embedding: EMPTY_VECTOR_SQL, - })) - ), - ]; - const relationshipRows = graph.relationships.map((relationship) => ({ - id: relationship.id, - active: false, - sourceId: relationship.sourceId, - targetId: relationship.targetId, - graphId: input.graphId, - rank: relationship.strength, - description: "", - embedding: EMPTY_VECTOR_SQL, - })); - - const insertedEntityIds = entityRows.map((entity) => entity.id); - const insertedRelationshipIds = relationshipRows.map((relationship) => relationship.id); - - const metrics = await db.transaction(async (tx) => { - const insertUnitsStart = performance.now(); - for (const chunk of chunkItems(unitRows)) { - await tx - .insert(textUnitTable) - .values(chunk) - .onConflictDoUpdate({ - target: textUnitTable.id, - set: { - fileId: sql`excluded.file_id`, - text: sql`excluded.text`, - startPage: sql`excluded.start_page`, - endPage: sql`excluded.end_page`, - chunks: sql`excluded.chunks`, - updatedAt: sql`NOW()`, - }, - }); - } - const insertUnitsDuration = performance.now() - insertUnitsStart; - - const insertEntitiesStart = performance.now(); - for (const chunk of chunkItems(entityRows)) { - await tx.insert(entityTable).values(chunk).onConflictDoNothing(); - } - const insertEntitiesDuration = performance.now() - insertEntitiesStart; - - const insertRelationshipsStart = performance.now(); - for (const chunk of chunkItems(relationshipRows)) { - await tx.insert(relationshipTable).values(chunk).onConflictDoNothing(); - } - for (const chunk of chunkItems(sourceRows)) { - await tx.insert(sourcesTable).values(chunk).onConflictDoNothing(); - } - const insertRelationshipsDuration = performance.now() - insertRelationshipsStart; - - const dedupeEntitiesStart = performance.now(); - if (insertedEntityIds.length > 0) { - const entityIds = textArray(insertedEntityIds); - const candidateNameKeySql = sql.raw(entityCompactNameKey("candidate.name")); - const seededNameKeySql = sql.raw(entityCompactNameKey("seed.name")); - - await tx.execute(sql` - WITH seeded_keys AS ( - SELECT DISTINCT seed.type, ${seededNameKeySql} AS normalized_name - FROM entities seed - WHERE seed.graph_id = ${input.graphId} - AND seed.id = ANY(${entityIds}) - ), duplicates AS ( - SELECT - candidate.id, - first_value(candidate.id) OVER ( - PARTITION BY candidate.graph_id, candidate.type, ${candidateNameKeySql} - ORDER BY candidate.active DESC, candidate.id ASC - ) AS canonical_id - FROM entities candidate - JOIN seeded_keys seeded - ON seeded.type = candidate.type - AND seeded.normalized_name = ${candidateNameKeySql} - WHERE candidate.graph_id = ${input.graphId} - ) - UPDATE sources source - SET entity_id = duplicates.canonical_id - FROM duplicates - WHERE source.entity_id = duplicates.id - AND duplicates.id <> duplicates.canonical_id - `); - - await tx.execute(sql` - WITH seeded_keys AS ( - SELECT DISTINCT seed.type, ${seededNameKeySql} AS normalized_name - FROM entities seed - WHERE seed.graph_id = ${input.graphId} - AND seed.id = ANY(${entityIds}) - ), duplicates AS ( - SELECT - candidate.id, - first_value(candidate.id) OVER ( - PARTITION BY candidate.graph_id, candidate.type, ${candidateNameKeySql} - ORDER BY candidate.active DESC, candidate.id ASC - ) AS canonical_id - FROM entities candidate - JOIN seeded_keys seeded - ON seeded.type = candidate.type - AND seeded.normalized_name = ${candidateNameKeySql} - WHERE candidate.graph_id = ${input.graphId} - ) - UPDATE relationships relationship - SET source_id = duplicates.canonical_id - FROM duplicates - WHERE relationship.source_id = duplicates.id - AND relationship.graph_id = ${input.graphId} - AND duplicates.id <> duplicates.canonical_id - `); - - await tx.execute(sql` - WITH seeded_keys AS ( - SELECT DISTINCT seed.type, ${seededNameKeySql} AS normalized_name - FROM entities seed - WHERE seed.graph_id = ${input.graphId} - AND seed.id = ANY(${entityIds}) - ), duplicates AS ( - SELECT - candidate.id, - first_value(candidate.id) OVER ( - PARTITION BY candidate.graph_id, candidate.type, ${candidateNameKeySql} - ORDER BY candidate.active DESC, candidate.id ASC - ) AS canonical_id - FROM entities candidate - JOIN seeded_keys seeded - ON seeded.type = candidate.type - AND seeded.normalized_name = ${candidateNameKeySql} - WHERE candidate.graph_id = ${input.graphId} - ) - UPDATE relationships relationship - SET target_id = duplicates.canonical_id - FROM duplicates - WHERE relationship.target_id = duplicates.id - AND relationship.graph_id = ${input.graphId} - AND duplicates.id <> duplicates.canonical_id - `); - - await tx.execute(sql` - WITH seeded_keys AS ( - SELECT DISTINCT seed.type, ${seededNameKeySql} AS normalized_name - FROM entities seed - WHERE seed.graph_id = ${input.graphId} - AND seed.id = ANY(${entityIds}) - ), duplicates AS ( - SELECT - candidate.id, - first_value(candidate.id) OVER ( - PARTITION BY candidate.graph_id, candidate.type, ${candidateNameKeySql} - ORDER BY candidate.active DESC, candidate.id ASC - ) AS canonical_id - FROM entities candidate - JOIN seeded_keys seeded - ON seeded.type = candidate.type - AND seeded.normalized_name = ${candidateNameKeySql} - WHERE candidate.graph_id = ${input.graphId} - ) - DELETE FROM entities entity - USING duplicates - WHERE entity.id = duplicates.id - AND duplicates.id <> duplicates.canonical_id - `); - } - const dedupeEntitiesDuration = performance.now() - dedupeEntitiesStart; - - const dedupeRelationshipsStart = performance.now(); - await tx.execute(sql` - DELETE FROM relationships - WHERE graph_id = ${input.graphId} - AND source_id = target_id - `); - - if (insertedRelationshipIds.length > 0) { - const relationshipIds = textArray(insertedRelationshipIds); - - await tx.execute(sql` - WITH seeded_pairs AS ( - SELECT DISTINCT - least(relationship.source_id, relationship.target_id) AS pair_source_id, - greatest(relationship.source_id, relationship.target_id) AS pair_target_id - FROM relationships relationship - WHERE relationship.graph_id = ${input.graphId} - AND relationship.id = ANY(${relationshipIds}) - ), duplicates AS ( - SELECT - relationship.id, - least(relationship.source_id, relationship.target_id) AS canonical_source_id, - greatest(relationship.source_id, relationship.target_id) AS canonical_target_id, - first_value(relationship.id) OVER ( - PARTITION BY relationship.graph_id, least(relationship.source_id, relationship.target_id), greatest(relationship.source_id, relationship.target_id) - ORDER BY relationship.active DESC, relationship.id ASC - ) AS canonical_id, - max(relationship.rank) OVER ( - PARTITION BY relationship.graph_id, least(relationship.source_id, relationship.target_id), greatest(relationship.source_id, relationship.target_id) - ) AS canonical_rank - FROM relationships relationship - JOIN seeded_pairs seeded - ON seeded.pair_source_id = least(relationship.source_id, relationship.target_id) - AND seeded.pair_target_id = greatest(relationship.source_id, relationship.target_id) - WHERE relationship.graph_id = ${input.graphId} - ) - UPDATE relationships relationship - SET source_id = duplicates.canonical_source_id, - target_id = duplicates.canonical_target_id, - rank = CASE - WHEN relationship.id = duplicates.canonical_id THEN duplicates.canonical_rank - ELSE relationship.rank - END, - updated_at = NOW() - FROM duplicates - WHERE relationship.id = duplicates.id - AND ( - relationship.source_id <> duplicates.canonical_source_id - OR relationship.target_id <> duplicates.canonical_target_id - OR ( - relationship.id = duplicates.canonical_id - AND relationship.rank <> duplicates.canonical_rank - ) - ) - `); - - await tx.execute(sql` - WITH seeded_pairs AS ( - SELECT DISTINCT - least(relationship.source_id, relationship.target_id) AS pair_source_id, - greatest(relationship.source_id, relationship.target_id) AS pair_target_id - FROM relationships relationship - WHERE relationship.graph_id = ${input.graphId} - AND relationship.id = ANY(${relationshipIds}) - ), duplicates AS ( - SELECT - relationship.id, - first_value(relationship.id) OVER ( - PARTITION BY relationship.graph_id, least(relationship.source_id, relationship.target_id), greatest(relationship.source_id, relationship.target_id) - ORDER BY relationship.active DESC, relationship.id ASC - ) AS canonical_id - FROM relationships relationship - JOIN seeded_pairs seeded - ON seeded.pair_source_id = least(relationship.source_id, relationship.target_id) - AND seeded.pair_target_id = greatest(relationship.source_id, relationship.target_id) - WHERE relationship.graph_id = ${input.graphId} - ) - UPDATE sources source - SET relationship_id = duplicates.canonical_id - FROM duplicates - WHERE source.relationship_id = duplicates.id - AND duplicates.id <> duplicates.canonical_id - `); - - await tx.execute(sql` - WITH seeded_pairs AS ( - SELECT DISTINCT - least(relationship.source_id, relationship.target_id) AS pair_source_id, - greatest(relationship.source_id, relationship.target_id) AS pair_target_id - FROM relationships relationship - WHERE relationship.graph_id = ${input.graphId} - AND relationship.id = ANY(${relationshipIds}) - ), duplicates AS ( - SELECT - relationship.id, - first_value(relationship.id) OVER ( - PARTITION BY relationship.graph_id, least(relationship.source_id, relationship.target_id), greatest(relationship.source_id, relationship.target_id) - ORDER BY relationship.active DESC, relationship.id ASC - ) AS canonical_id - FROM relationships relationship - JOIN seeded_pairs seeded - ON seeded.pair_source_id = least(relationship.source_id, relationship.target_id) - AND seeded.pair_target_id = greatest(relationship.source_id, relationship.target_id) - WHERE relationship.graph_id = ${input.graphId} - ) - DELETE FROM relationships relationship - USING duplicates - WHERE relationship.id = duplicates.id - AND duplicates.id <> duplicates.canonical_id - `); - } - const dedupeRelationshipsDuration = performance.now() - dedupeRelationshipsStart; - - return { - insertUnitsDuration, - insertEntitiesDuration, - insertRelationshipsDuration, - dedupeEntitiesDuration, - dedupeRelationshipsDuration, - }; - }); - - const duration = performance.now() - start; + const saveResult = await saveGraphToDatabase(input.graphId, graph); return { summary: { fileId: input.fileId, - units: graph.units.length, - entities: graph.entities.length, - relationships: graph.relationships.length, + ...saveResult.summary, }, - duration, - metrics, + duration: saveResult.duration, + metrics: saveResult.metrics, }; }); if (saveGraphResult === FILE_DELETED) { diff --git a/apps/worker/workflows/process-files-spec.ts b/apps/worker/workflows/process-files-spec.ts index f3effac4..922987bd 100644 --- a/apps/worker/workflows/process-files-spec.ts +++ b/apps/worker/workflows/process-files-spec.ts @@ -14,5 +14,11 @@ export const processFilesSpec = defineWorkflowSpec({ graphId: z.string(), fileIds: z.array(z.string()), processRunId: z.string().optional(), + code: z + .object({ + kind: z.literal("repository"), + retiredFileIds: z.array(z.string()).optional(), + }) + .optional(), }), }); diff --git a/apps/worker/workflows/sync-repository-graph-spec.ts b/apps/worker/workflows/sync-repository-graph-spec.ts new file mode 100644 index 00000000..48f4b658 --- /dev/null +++ b/apps/worker/workflows/sync-repository-graph-spec.ts @@ -0,0 +1,19 @@ +import { defineWorkflowSpec } from "openworkflow"; +import z from "zod"; + +export const syncRepositoryGraphSpec = defineWorkflowSpec({ + name: "sync-repository-graph", + version: "1.0.0", + retryPolicy: { + initialInterval: "5s", + backoffCoefficient: 2, + maximumInterval: "1m", + maximumAttempts: 3, + }, + schema: z.object({ + bindingId: z.string(), + reason: z.enum(["initial", "webhook", "manual"]), + commitSha: z.string().optional(), + deliveryId: z.string().optional(), + }), +}); diff --git a/apps/worker/workflows/sync-repository-graph.test.ts b/apps/worker/workflows/sync-repository-graph.test.ts new file mode 100644 index 00000000..36591651 --- /dev/null +++ b/apps/worker/workflows/sync-repository-graph.test.ts @@ -0,0 +1,360 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import type { ProviderRepositoryChange } from "@kiwi/connectors"; + +type SelectResult = + | { kind: "limit"; value: unknown[] } + | { kind: "where"; value: unknown[] }; + +const insertedFileValues: Array> = []; +const processWorkflowInputs: Array> = []; +const deleteWorkflowInputs: Array> = []; +const bindingUpdates: Array> = []; +const compareCalls: Array<{ fromCommitSha: string; toCommitSha: string }> = []; +const readFileCalls: Array<{ path: string; commitSha: string }> = []; + +let selectResults: SelectResult[] = []; +let processFilesError: Error | null = null; +let compareChanges: ProviderRepositoryChange[] = []; +let readFileContents: Record = {}; +let loadSnapshotCalls = 0; + +function createSelectQuery() { + const result = selectResults.shift(); + if (!result) { + throw new Error("Unexpected select call"); + } + + const chain = { + from: () => chain, + innerJoin: () => chain, + where: () => (result.kind === "where" ? result.value : chain), + limit: () => (result.kind === "limit" ? result.value : result.value), + }; + return chain; +} + +const transactionDb = { + update: () => ({ + set: (values: Record) => ({ + where: async () => { + bindingUpdates.push(values); + return undefined; + }, + }), + }), + insert: () => ({ + values: (values: unknown) => { + if ( + Array.isArray(values) && + values.every((value) => typeof value === "object" && value !== null && "processRunId" in value && "fileId" in value) + ) { + return undefined; + } + + return { + onConflictDoNothing: () => ({ + returning: () => { + if (!Array.isArray(values)) { + throw new Error("Expected file rows"); + } + insertedFileValues.push(...(values as Array>)); + return (values as Array<{ id: string }>).map((value) => ({ id: value.id })); + }, + }), + returning: () => [{ id: "process-run-1" }], + }; + }, + }), +}; + +mock.module("@kiwi/db", () => ({ + db: { + select: () => createSelectQuery(), + update: () => ({ + set: (values: Record) => ({ + where: async () => { + bindingUpdates.push(values); + return undefined; + }, + }), + }), + transaction: async (callback: (tx: typeof transactionDb) => Promise) => callback(transactionDb), + }, +})); + +mock.module("@kiwi/connectors", () => ({ + ConnectorProviderError: class ConnectorProviderError extends Error { + constructor(public readonly kind: string, message: string) { + super(message); + this.name = "ConnectorProviderError"; + } + }, + MAX_REPOSITORY_CODE_BYTES: 100_000, + MAX_REPOSITORY_CODE_FILES: 100, + createGitHubClient: () => ({ + provider: "github", + listRepositories: async () => [], + listBranches: async () => [{ name: "main", commitSha: "commit-new" }], + loadRepositorySnapshot: async () => { + loadSnapshotCalls += 1; + return { + repository: { + provider: "github", + id: "1", + fullName: "acme/widgets", + name: "widgets", + htmlUrl: "https://github.com/acme/widgets", + defaultBranch: "main", + private: true, + }, + branch: { name: "main", commitSha: "commit-new" }, + commitSha: "commit-new", + files: [], + }; + }, + compareRepository: async (_repository: unknown, fromCommitSha: string, toCommitSha: string) => { + compareCalls.push({ fromCommitSha, toCommitSha }); + return { fromCommitSha, toCommitSha, changes: compareChanges }; + }, + readFile: async (_repository: unknown, path: string, commitSha: string) => { + readFileCalls.push({ path, commitSha }); + const content = readFileContents[path]; + if (!content) { + throw new Error(`Missing mocked content for ${path}`); + } + return content; + }, + }), + createGitHubInstallationToken: async () => ({ token: "installation-token", expiresAt: "2026-01-01T01:00:00Z" }), + createGitLabClient: () => { + throw new Error("GitLab client was not expected"); + }, + decryptConnectorCredentials: () => ({ provider: "github", appId: "app-1", privateKeyPem: "pem" }), + normalizeGitLabBaseUrl: (value: string) => value.replace(/\/+$/, ""), +})); + +mock.module("../env", () => ({ + env: { + AUTH_SECRET: "test-secret", + }, +})); + +function bindingRow(lastSyncedCommitSha: string | null) { + return { + binding: { + id: "binding-1", + graphId: "graph-1", + connectorInstallationId: "installation-1", + providerRepositoryId: "1", + repositoryFullName: "acme/widgets", + repositoryHtmlUrl: "https://github.com/acme/widgets", + branch: "main", + webhookEnabled: true, + lastSeenCommitSha: lastSyncedCommitSha, + lastSyncedCommitSha, + }, + installation: { + id: "installation-1", + connectorId: "connector-1", + providerInstallationId: "99", + encryptedCredentials: null, + status: "active", + }, + connector: { + id: "connector-1", + provider: "github", + encryptedCredentials: "encrypted", + status: "active", + }, + graph: { + id: "graph-1", + state: "ready", + }, + }; +} + +function activeFile(id: string, commitSha: string, path: string, size: number) { + return { + id, + size, + metadata: JSON.stringify({ + repositoryUrl: "https://github.com/acme/widgets", + repositoryName: "acme/widgets", + commitSha, + path, + }), + }; +} + +async function runWorkflow(input: { bindingId: string; reason: "manual" | "webhook" | "initial"; commitSha?: string }) { + return syncRepositoryGraph.fn({ + input, + step: { + run: async (_config: { name: string }, fn: () => Promise | unknown) => fn(), + runWorkflow: async (spec: { name: string }, workflowInput?: Record) => { + if (spec.name === "process-files") { + processWorkflowInputs.push(workflowInput ?? {}); + if (processFilesError) { + throw processFilesError; + } + return undefined; + } + if (spec.name === "delete-file") { + deleteWorkflowInputs.push(workflowInput ?? {}); + return undefined; + } + throw new Error(`Unexpected workflow ${spec.name}`); + }, + }, + version: null, + run: { + id: "workflow-run-1", + workflowName: "sync-repository-graph", + createdAt: new Date("2026-01-01T00:00:00Z"), + startedAt: new Date("2026-01-01T00:00:00Z"), + retryAttempt: 1, + retryMaxAttempts: 1, + retryTerminal: true, + }, + }); +} + +// Dynamic import is required so module mocks are installed before the workflow module is evaluated. +const { syncRepositoryGraph } = await import("./sync-repository-graph"); + +describe("syncRepositoryGraph", () => { + beforeEach(() => { + insertedFileValues.length = 0; + processWorkflowInputs.length = 0; + deleteWorkflowInputs.length = 0; + bindingUpdates.length = 0; + compareCalls.length = 0; + readFileCalls.length = 0; + selectResults = []; + processFilesError = null; + compareChanges = []; + readFileContents = {}; + loadSnapshotCalls = 0; + }); + + test("processes only changed supported connector files", async () => { + compareChanges = [{ status: "modified", newPath: "src/index.ts" }]; + readFileContents = { + "src/index.ts": "export const next = shared;\n", + }; + selectResults = [ + { kind: "limit", value: [bindingRow("commit-old")] }, + { + kind: "where", + value: [ + activeFile("old-file", "commit-old", "src/index.ts", 25), + activeFile("shared-file", "commit-old", "src/shared.ts", 22), + ], + }, + ]; + + const result = await runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" }); + + expect(result).toMatchObject({ commitSha: "commit-new", fileCount: 1 }); + expect(loadSnapshotCalls).toBe(0); + expect(compareCalls).toEqual([{ fromCommitSha: "commit-old", toCommitSha: "commit-new" }]); + expect(readFileCalls).toEqual([{ path: "src/index.ts", commitSha: "commit-new" }]); + expect(insertedFileValues.map((row) => row.name)).toEqual(["src/index.ts"]); + expect(processWorkflowInputs).toHaveLength(1); + expect(processWorkflowInputs[0]).toMatchObject({ + graphId: "graph-1", + processRunId: "process-run-1", + code: { kind: "repository", retiredFileIds: ["old-file"] }, + }); + expect(String(insertedFileValues[0]?.checksum)).toStartWith("commit-new:src/index.ts:"); + }); + + test("finalizes delete-only connector deltas without inserting new files", async () => { + compareChanges = [{ status: "deleted", oldPath: "src/removed.ts" }]; + selectResults = [ + { kind: "limit", value: [bindingRow("commit-old")] }, + { + kind: "where", + value: [ + activeFile("removed-file", "commit-old", "src/removed.ts", 20), + activeFile("shared-file", "commit-old", "src/shared.ts", 22), + ], + }, + ]; + + const result = await runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" }); + + expect(result).toMatchObject({ commitSha: "commit-new", fileCount: 0 }); + expect(readFileCalls).toEqual([]); + expect(insertedFileValues).toEqual([]); + expect(processWorkflowInputs).toEqual([ + { + graphId: "graph-1", + fileIds: [], + code: { kind: "repository", retiredFileIds: ["removed-file"] }, + }, + ]); + }); + + test("skips processing when no supported code paths changed", async () => { + compareChanges = []; + selectResults = [ + { kind: "limit", value: [bindingRow("commit-old")] }, + { + kind: "where", + value: [activeFile("old-file", "commit-old", "src/index.ts", 25)], + }, + ]; + + const result = await runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" }); + + expect(result).toMatchObject({ commitSha: "commit-new", fileCount: 0 }); + expect(processWorkflowInputs).toEqual([]); + expect(insertedFileValues).toEqual([]); + expect(bindingUpdates).toContainEqual({ + syncStatus: "synced", + lastSeenCommitSha: "commit-new", + lastSyncedCommitSha: "commit-new", + syncErrorCode: null, + }); + }); + + test("rolls back inserted files when incremental processing fails", async () => { + compareChanges = [{ status: "modified", newPath: "src/index.ts" }]; + readFileContents = { + "src/index.ts": "export const next = shared;\n", + }; + processFilesError = new Error("child failure"); + selectResults = [ + { kind: "limit", value: [bindingRow("commit-old")] }, + { + kind: "where", + value: [ + activeFile("old-file", "commit-old", "src/index.ts", 25), + activeFile("shared-file", "commit-old", "src/shared.ts", 22), + ], + }, + ]; + + await expect(runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" })).rejects.toThrow( + "child failure" + ); + expect(deleteWorkflowInputs).toEqual([{ graphId: "graph-1", fileId: insertedFileValues[0]?.id }]); + }); + + test("rejects duplicate normalized paths from provider deltas", async () => { + compareChanges = [ + { status: "modified", newPath: "src/index.ts" }, + { status: "deleted", oldPath: "src/index.ts" }, + ]; + selectResults = [ + { kind: "limit", value: [bindingRow("commit-old")] }, + { kind: "where", value: [activeFile("old-file", "commit-old", "src/index.ts", 25)] }, + ]; + + await expect(runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" })).rejects.toThrow( + "Provider delta contained duplicate path src/index.ts" + ); + expect(processWorkflowInputs).toEqual([]); + }); +}); diff --git a/apps/worker/workflows/sync-repository-graph.ts b/apps/worker/workflows/sync-repository-graph.ts new file mode 100644 index 00000000..24292dda --- /dev/null +++ b/apps/worker/workflows/sync-repository-graph.ts @@ -0,0 +1,495 @@ +import { createHash } from "node:crypto"; +import { + ConnectorProviderError, + MAX_REPOSITORY_CODE_BYTES, + MAX_REPOSITORY_CODE_FILES, + createGitHubClient, + createGitHubInstallationToken, + createGitLabClient, + decryptConnectorCredentials, + normalizeGitLabBaseUrl, +} from "@kiwi/connectors"; +import type { + ConnectorSecretPayload, + GitHubConnectorCredentials, + GitLabConnectorCredentials, + GitLabInstallationCredentials, + ProviderCodeFile, + ProviderRepository, + ProviderRepositoryChange, + ProviderRepositoryClient, + ProviderRepositorySnapshot, +} from "@kiwi/connectors"; +import { db } from "@kiwi/db"; +import { + connectorInstallationsTable, + connectorsTable, + connectorWebhookEventsTable, + repositoryGraphBindingsTable, +} from "@kiwi/db/tables/connectors"; +import { filesTable, graphTable, processRunFilesTable, processRunsTable } from "@kiwi/db/tables/graph"; +import { serializeCodeFileMetadata } from "@kiwi/graph/code/metadata"; +import { and, eq } from "drizzle-orm"; +import { defineWorkflow } from "openworkflow"; +import { parseCodeFileMetadata } from "../lib/code-file-metadata"; +import { env } from "../env"; +import { deleteFileSpec } from "./delete-file-spec"; +import { processFilesSpec } from "./process-files-spec"; +import { syncRepositoryGraphSpec } from "./sync-repository-graph-spec"; + +type BindingGraphRow = { + binding: typeof repositoryGraphBindingsTable.$inferSelect; + installation: typeof connectorInstallationsTable.$inferSelect; + connector: typeof connectorsTable.$inferSelect; + graph: typeof graphTable.$inferSelect; +}; + +type ProviderClientContext = { + client: ProviderRepositoryClient; + gitLabBaseUrl?: string; +}; + +type Snapshot = ProviderRepositorySnapshot & { + files: ProviderCodeFile[]; +}; + +type ActiveBindingFile = { + id: string; + size: number; + path: string; +}; + +type IncrementalSyncPlan = { + newPaths: string[]; + retiredFileIds: string[]; +}; + +type InsertedRepositoryFiles = { + fileIds: string[]; + processRunId: string; +}; + +async function loadBindingGraph(bindingId: string): Promise { + const [row] = await db + .select({ + binding: repositoryGraphBindingsTable, + installation: connectorInstallationsTable, + connector: connectorsTable, + graph: graphTable, + }) + .from(repositoryGraphBindingsTable) + .innerJoin( + connectorInstallationsTable, + eq(connectorInstallationsTable.id, repositoryGraphBindingsTable.connectorInstallationId) + ) + .innerJoin(connectorsTable, eq(connectorsTable.id, connectorInstallationsTable.connectorId)) + .innerJoin(graphTable, eq(graphTable.id, repositoryGraphBindingsTable.graphId)) + .where(eq(repositoryGraphBindingsTable.id, bindingId)) + .limit(1); + return row ?? null; +} + +function isGitHubConnectorCredentials(value: ConnectorSecretPayload): value is GitHubConnectorCredentials { + return "provider" in value && value.provider === "github"; +} + +function isGitLabConnectorCredentials(value: ConnectorSecretPayload): value is GitLabConnectorCredentials { + return "provider" in value && value.provider === "gitlab" && "baseUrl" in value; +} + +function isGitLabInstallationCredentials(value: ConnectorSecretPayload): value is GitLabInstallationCredentials { + return "provider" in value && value.provider === "gitlab" && "accessToken" in value; +} + +function repositoryFromBinding(row: BindingGraphRow): ProviderRepository { + return { + provider: row.connector.provider as ProviderRepository["provider"], + id: row.binding.providerRepositoryId, + fullName: row.binding.repositoryFullName, + name: row.binding.repositoryFullName.split("/").at(-1) ?? row.binding.repositoryFullName, + htmlUrl: row.binding.repositoryHtmlUrl, + defaultBranch: row.binding.branch, + private: true, + }; +} + +async function createProviderClient(row: BindingGraphRow): Promise { + const connectorCredentials = decryptConnectorCredentials(row.connector.encryptedCredentials, env.AUTH_SECRET); + if (row.connector.provider === "github") { + if (!isGitHubConnectorCredentials(connectorCredentials)) { + throw new Error("Invalid connector credentials"); + } + const token = await createGitHubInstallationToken({ + credentials: connectorCredentials, + installationId: row.installation.providerInstallationId, + }); + return { + client: createGitHubClient({ installationToken: token.token }), + }; + } + + if (!isGitLabConnectorCredentials(connectorCredentials)) { + throw new Error("Invalid connector credentials"); + } + const installationCredentials = row.installation.encryptedCredentials + ? decryptConnectorCredentials(row.installation.encryptedCredentials, env.AUTH_SECRET) + : null; + if (!installationCredentials || !isGitLabInstallationCredentials(installationCredentials)) { + throw new Error("Invalid connector installation credentials"); + } + return { + client: createGitLabClient({ + baseUrl: connectorCredentials.baseUrl, + accessToken: installationCredentials.accessToken, + }), + gitLabBaseUrl: connectorCredentials.baseUrl, + }; +} + +async function resolveTargetCommitSha( + row: BindingGraphRow, + client: ProviderRepositoryClient, + inputCommitSha?: string +): Promise { + if (inputCommitSha) { + return inputCommitSha; + } + + const branch = (await client.listBranches(repositoryFromBinding(row))).find((candidate) => candidate.name === row.binding.branch); + if (!branch) { + throw new ConnectorProviderError("not-found", "Repository branch was not found"); + } + + return branch.commitSha; +} + +async function loadSnapshot(row: BindingGraphRow, client: ProviderRepositoryClient, commitSha: string): Promise { + return client.loadRepositorySnapshot(repositoryFromBinding(row), row.binding.branch, commitSha) as Promise; +} + +async function loadActiveBindingFiles(bindingId: string): Promise> { + const rows = await db + .select({ + id: filesTable.id, + size: filesTable.size, + metadata: filesTable.metadata, + }) + .from(filesTable) + .where(and(eq(filesTable.repositoryBindingId, bindingId), eq(filesTable.type, "code"), eq(filesTable.deleted, false))); + + const filesByPath = new Map(); + for (const row of rows) { + const metadata = parseCodeFileMetadata(row.metadata); + if (!metadata) { + throw new Error(`Active binding file ${row.id} is missing repository metadata`); + } + if (filesByPath.has(metadata.path)) { + throw new Error(`Repository binding ${bindingId} has multiple active rows for ${metadata.path}`); + } + filesByPath.set(metadata.path, { + id: row.id, + size: row.size, + path: metadata.path, + }); + } + + return filesByPath; +} + +function planIncrementalChanges( + activeFiles: Map, + changes: ProviderRepositoryChange[] +): IncrementalSyncPlan { + const seenPaths = new Set(); + const newPaths: string[] = []; + const retiredFileIds = new Set(); + + const trackPath = (path: string) => { + if (seenPaths.has(path)) { + throw new Error(`Provider delta contained duplicate path ${path}`); + } + seenPaths.add(path); + }; + + const retirePath = (path: string) => { + const activeFile = activeFiles.get(path); + if (activeFile) { + retiredFileIds.add(activeFile.id); + } + }; + + for (const change of changes) { + switch (change.status) { + case "added": + case "modified": + trackPath(change.newPath); + newPaths.push(change.newPath); + retirePath(change.newPath); + break; + case "deleted": + trackPath(change.oldPath); + retirePath(change.oldPath); + break; + case "renamed": + trackPath(change.oldPath); + trackPath(change.newPath); + retirePath(change.oldPath); + retirePath(change.newPath); + newPaths.push(change.newPath); + break; + } + } + + return { + newPaths, + retiredFileIds: [...retiredFileIds], + }; +} + +function buildConnectorFile( + row: BindingGraphRow, + context: ProviderClientContext, + commitSha: string, + path: string, + content: string +): ProviderCodeFile { + const rawUrl = + row.connector.provider === "github" + ? `https://raw.githubusercontent.com/${row.binding.repositoryFullName}/${commitSha}/${path}` + : context.gitLabBaseUrl + ? `${normalizeGitLabBaseUrl(context.gitLabBaseUrl)}/api/v4/projects/${encodeURIComponent(row.binding.providerRepositoryId)}/repository/files/${encodeURIComponent(path)}/raw?ref=${encodeURIComponent(commitSha)}` + : undefined; + + return { + path, + size: Buffer.byteLength(content, "utf8"), + checksum: createHash("sha256").update(content, "utf8").digest("hex"), + htmlUrl: + row.connector.provider === "github" + ? `${row.binding.repositoryHtmlUrl}/blob/${commitSha}/${path}` + : `${row.binding.repositoryHtmlUrl}/-/blob/${commitSha}/${path}`, + ...(rawUrl ? { rawUrl } : {}), + content, + }; +} + +function assertBindingSnapshotLimits( + activeFiles: Map, + retiredFileIds: string[], + newFiles: ProviderCodeFile[] +) { + const retiredIds = new Set(retiredFileIds); + const totalFileCount = [...activeFiles.values()].filter((file) => !retiredIds.has(file.id)).length + newFiles.length; + if (totalFileCount > MAX_REPOSITORY_CODE_FILES) { + throw new ConnectorProviderError("limit", "Repository contains too many supported code files"); + } + + const totalBytes = + [...activeFiles.values()].reduce((sum, file) => sum + (retiredIds.has(file.id) ? 0 : file.size), 0) + + newFiles.reduce((sum, file) => sum + file.size, 0); + if (totalBytes > MAX_REPOSITORY_CODE_BYTES) { + throw new ConnectorProviderError("limit", "Repository contains too much supported code"); + } +} + +function fileRows(row: BindingGraphRow, files: ProviderCodeFile[], commitSha: string) { + return files.map((file) => ({ + graphId: row.binding.graphId, + name: file.path, + size: file.size, + type: "code", + mimeType: "text/plain", + key: `connector:${row.binding.id}:${commitSha}:${file.path}`, + storageKind: "external", + externalUrl: file.rawUrl ?? file.htmlUrl, + externalProvider: row.connector.provider, + repositoryBindingId: row.binding.id, + checksum: `${commitSha}:${file.path}:${file.checksum}`, + metadata: serializeCodeFileMetadata({ + repositoryUrl: row.binding.repositoryHtmlUrl, + repositoryName: row.binding.repositoryFullName, + commitSha, + path: file.path, + external: + row.connector.provider === "github" && file.rawUrl + ? { provider: "github", rawUrl: file.rawUrl, htmlUrl: file.htmlUrl } + : undefined, + }), + id: crypto.randomUUID(), + })); +} + +async function insertRepositoryFiles( + row: BindingGraphRow, + files: ProviderCodeFile[], + commitSha: string +): Promise { + return db.transaction(async (tx) => { + await tx + .update(repositoryGraphBindingsTable) + .set({ syncStatus: "syncing", lastSeenCommitSha: commitSha, syncErrorCode: null }) + .where(eq(repositoryGraphBindingsTable.id, row.binding.id)); + + const insertedFiles = await tx + .insert(filesTable) + .values(fileRows(row, files, commitSha)) + .onConflictDoNothing() + .returning({ id: filesTable.id }); + if (insertedFiles.length !== files.length) { + throw new Error("Failed to insert all repository files"); + } + + const [processRun] = await tx + .insert(processRunsTable) + .values({ graphId: row.binding.graphId, status: "pending" }) + .returning({ id: processRunsTable.id }); + if (!processRun) { + throw new Error("Failed to create process run"); + } + + await tx.insert(processRunFilesTable).values( + insertedFiles.map((file) => ({ + processRunId: processRun.id, + fileId: file.id, + })) + ); + return { + fileIds: insertedFiles.map((file) => file.id), + processRunId: processRun.id, + }; + }); +} + +async function markWebhookDuplicate(provider: typeof connectorsTable.$inferSelect.provider, deliveryId: string) { + await db + .update(connectorWebhookEventsTable) + .set({ status: "duplicate" }) + .where(and(eq(connectorWebhookEventsTable.provider, provider), eq(connectorWebhookEventsTable.deliveryId, deliveryId))); +} + +async function markBindingSynced(bindingId: string, commitSha: string) { + await db + .update(repositoryGraphBindingsTable) + .set({ syncStatus: "synced", lastSeenCommitSha: commitSha, lastSyncedCommitSha: commitSha, syncErrorCode: null }) + .where(eq(repositoryGraphBindingsTable.id, bindingId)); +} + +export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async ({ input, step }) => { + const row = await step.run({ name: "load-binding" }, async () => loadBindingGraph(input.bindingId)); + if (!row || row.connector.status !== "active" || row.installation.status !== "active" || !row.binding.webhookEnabled) { + return { skipped: true }; + } + + const context = await step.run({ name: "create-provider-client" }, async () => createProviderClient(row)); + const commitSha = await step.run({ name: "resolve-target-commit" }, async () => + resolveTargetCommitSha(row, context.client, input.commitSha) + ); + + if (row.binding.lastSyncedCommitSha === commitSha) { + if (input.deliveryId) { + await step.run({ name: "mark-webhook-duplicate" }, async () => + markWebhookDuplicate(row.connector.provider, input.deliveryId!) + ); + } + return { skipped: true, commitSha }; + } + + if (!row.binding.lastSyncedCommitSha) { + const snapshot = await step.run({ name: "load-provider-snapshot" }, async () => + loadSnapshot(row, context.client, commitSha) + ); + if (snapshot.files.length === 0) { + await step.run({ name: "mark-empty-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); + return { commitSha, fileCount: 0 }; + } + + const created = await step.run({ name: "commit-external-files" }, async () => + insertRepositoryFiles(row, snapshot.files, commitSha) + ); + try { + await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: created.fileIds, + processRunId: created.processRunId, + code: { kind: "repository", retiredFileIds: [] }, + }); + } catch (error) { + await Promise.all( + created.fileIds.map((fileId) => + step.runWorkflow(deleteFileSpec, { + graphId: row.binding.graphId, + fileId, + }) + ) + ); + throw error; + } + + await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); + return { commitSha, fileCount: created.fileIds.length }; + } + + const activeFiles = await step.run({ name: "load-active-binding-files" }, async () => + loadActiveBindingFiles(row.binding.id) + ); + const delta = await step.run({ name: "compare-provider-commits" }, async () => + context.client.compareRepository(repositoryFromBinding(row), row.binding.lastSyncedCommitSha!, commitSha) + ); + const plan = planIncrementalChanges(activeFiles, delta.changes); + if (plan.newPaths.length === 0 && plan.retiredFileIds.length === 0) { + await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); + return { commitSha, fileCount: 0 }; + } + + const changedFiles = + plan.newPaths.length > 0 + ? await step.run({ name: "load-changed-files" }, async () => + Promise.all( + plan.newPaths.map(async (path) => + buildConnectorFile( + row, + context, + commitSha, + path, + await context.client.readFile(repositoryFromBinding(row), path, commitSha) + ) + ) + ) + ) + : []; + assertBindingSnapshotLimits(activeFiles, plan.retiredFileIds, changedFiles); + + if (changedFiles.length > 0) { + const created = await step.run({ name: "commit-external-files" }, async () => + insertRepositoryFiles(row, changedFiles, commitSha) + ); + try { + await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: created.fileIds, + processRunId: created.processRunId, + code: { kind: "repository", retiredFileIds: plan.retiredFileIds }, + }); + } catch (error) { + await Promise.all( + created.fileIds.map((fileId) => + step.runWorkflow(deleteFileSpec, { + graphId: row.binding.graphId, + fileId, + }) + ) + ); + throw error; + } + await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); + return { commitSha, fileCount: created.fileIds.length }; + } + + await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: [], + code: { kind: "repository", retiredFileIds: plan.retiredFileIds }, + }); + await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); + return { commitSha, fileCount: 0 }; +}); diff --git a/bun.lock b/bun.lock index ddf718d7..411bca06 100644 --- a/bun.lock +++ b/bun.lock @@ -25,6 +25,7 @@ "@elysiajs/cors": "^1.4.2", "@kiwi/ai": "workspace:*", "@kiwi/auth": "workspace:*", + "@kiwi/connectors": "workspace:*", "@kiwi/contracts": "workspace:*", "@kiwi/db": "workspace:*", "@kiwi/files": "workspace:*", @@ -126,6 +127,7 @@ "name": "@kiwi/worker", "dependencies": { "@kiwi/ai": "workspace:*", + "@kiwi/connectors": "workspace:*", "@kiwi/contracts": "workspace:*", "@kiwi/db": "workspace:*", "@kiwi/files": "workspace:*", @@ -191,6 +193,15 @@ "typescript": "catalog:", }, }, + "packages/connectors": { + "name": "@kiwi/connectors", + "devDependencies": { + "@types/bun": "catalog:", + }, + "peerDependencies": { + "typescript": "catalog:", + }, + }, "packages/contracts": { "name": "@kiwi/contracts", "dependencies": { @@ -241,6 +252,7 @@ "@kiwi/files": "workspace:*", "@libpdf/core": "^0.3.6", "@platformatic/vfs": "^0.4.0", + "@tree-sitter-grammars/tree-sitter-zig": "^1.1.2", "@xmldom/xmldom": "^0.9.10", "ai": "catalog:", "dom-serializer": "^3.1.1", @@ -249,6 +261,11 @@ "js-tiktoken": "^1.0.21", "jszip": "^3.10.1", "pdf-to-img": "^6.1.0", + "tree-sitter": "^0.25.0", + "tree-sitter-c": "^0.24.1", + "tree-sitter-javascript": "^0.25.0", + "tree-sitter-rust": "^0.24.0", + "tree-sitter-typescript": "^0.23.2", "ulid": "^3.0.2", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", "zod": "catalog:", @@ -612,6 +629,8 @@ "@kiwi/auth": ["@kiwi/auth@workspace:packages/auth"], + "@kiwi/connectors": ["@kiwi/connectors@workspace:packages/connectors"], + "@kiwi/contracts": ["@kiwi/contracts@workspace:packages/contracts"], "@kiwi/db": ["@kiwi/db@workspace:packages/db"], @@ -1238,6 +1257,8 @@ "@tokenizer/token": ["@tokenizer/token@0.3.0", "", {}, "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="], + "@tree-sitter-grammars/tree-sitter-zig": ["@tree-sitter-grammars/tree-sitter-zig@1.1.2", "", { "dependencies": { "node-addon-api": "^8.3.0", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.22.1" }, "optionalPeers": ["tree-sitter"] }, "sha512-J0L31HZ2isy3F5zb2g5QWQOv2r/pbruQNL9ADhuQv2pn5BQOzxt80WcEJaYXBeuJ8GHxVT42slpCna8k1c8LOw=="], + "@ts-morph/common": ["@ts-morph/common@0.27.0", "", { "dependencies": { "fast-glob": "^3.3.3", "minimatch": "^10.0.1", "path-browserify": "^1.0.1" } }, "sha512-Wf29UqxWDpc+i61k3oIOzcUfQt79PIT9y/MWfAGlrkjg6lBC1hwDECLXPVJAhWjiGbfBCxZd65F/LIZF3+jeJQ=="], "@turbo/darwin-64": ["@turbo/darwin-64@2.9.16", "", { "os": "darwin", "cpu": "x64" }, "sha512-jLjApWTSNd7JZ5JaLYfelW1ytnGQOvB7ivl+2RD1xQvJTbi8I9gBjzcga7tDZVPyaxpl10YTfJt3BrYXR18KDw=="], @@ -2104,12 +2125,14 @@ "nitro": ["nitro@3.0.260311-beta", "", { "dependencies": { "consola": "^3.4.2", "crossws": "^0.4.4", "db0": "^0.3.4", "env-runner": "^0.1.6", "h3": "^2.0.1-rc.16", "hookable": "^6.0.1", "nf3": "^0.3.11", "ocache": "^0.1.2", "ofetch": "^2.0.0-alpha.3", "ohash": "^2.0.11", "rolldown": "^1.0.0-rc.8", "srvx": "^0.11.9", "unenv": "^2.0.0-rc.24", "unstorage": "^2.0.0-alpha.6" }, "peerDependencies": { "dotenv": "*", "giget": "*", "jiti": "^2.6.1", "rollup": "^4.59.0", "vite": "^7 || ^8 || >=8.0.0-0", "xml2js": "^0.6.2", "zephyr-agent": "^0.1.15" }, "optionalPeers": ["dotenv", "giget", "jiti", "rollup", "vite", "xml2js", "zephyr-agent"], "bin": { "nitro": "dist/cli/index.mjs" } }, "sha512-0o0fJ9LUh4WKUqJNX012jyieUOtMCnadkNDWr0mHzdraoHpJP/1CGNefjRyZyMXSpoJfwoWdNEZu2iGf35TUvQ=="], - "node-addon-api": ["node-addon-api@7.1.1", "", {}, "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ=="], + "node-addon-api": ["node-addon-api@8.8.0", "", {}, "sha512-c5Ko1fZJIJmzhFIkhRN76WTq+fC6tWnGy9CXA0fA+XygsWZmEwG8vmbkNqxMyoaa0Tin4djul49NzdVcJJcjeA=="], "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], "node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], + "node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="], + "node-readable-to-web-readable-stream": ["node-readable-to-web-readable-stream@0.4.2", "", {}, "sha512-/cMZNI34v//jUTrI+UIo4ieHAB5EZRY/+7OmXZgBxaWBMcW2tGdceIw06RFxWxrKZ5Jp3sI2i5TsRo+CBhtVLQ=="], "node-releases": ["node-releases@2.0.38", "", {}, "sha512-3qT/88Y3FbH/Kx4szpQQ4HzUbVrHPKTLVpVocKiLfoYvw9XSGOX2FmD2d6DrXbVYyAQTF2HeF6My8jmzx7/CRw=="], @@ -2502,6 +2525,16 @@ "tr46": ["tr46@6.0.0", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw=="], + "tree-sitter": ["tree-sitter@0.25.0", "", { "dependencies": { "node-addon-api": "^8.3.0", "node-gyp-build": "^4.8.4" } }, "sha512-PGZZzFW63eElZJDe/b/R/LbsjDDYJa5UEjLZJB59RQsMX+fo0j54fqBPn1MGKav/QNa0JR0zBiVaikYDWCj5KQ=="], + + "tree-sitter-c": ["tree-sitter-c@0.24.1", "", { "dependencies": { "node-addon-api": "^8.3.1", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.22.4" }, "optionalPeers": ["tree-sitter"] }, "sha512-lkYwWN3SRecpvaeqmFKkuPNR3ZbtnvHU+4XAEEkJdrp3JfSp2pBrhXOtvfsENUneye76g889Y0ddF2DM0gEDpA=="], + + "tree-sitter-javascript": ["tree-sitter-javascript@0.25.0", "", { "dependencies": { "node-addon-api": "^8.3.1", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.25.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-1fCbmzAskZkxcZzN41sFZ2br2iqTYP3tKls1b/HKGNPQUVOpsUxpmGxdN/wMqAk3jYZnYBR1dd/y/0avMeU7dw=="], + + "tree-sitter-rust": ["tree-sitter-rust@0.24.0", "", { "dependencies": { "node-addon-api": "^8.2.2", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.22.1" }, "optionalPeers": ["tree-sitter"] }, "sha512-NWemUDf629Tfc90Y0Z55zuwPCAHkLxWnMf2RznYu4iBkkrQl2o/CHGB7Cr52TyN5F1DAx8FmUnDtCy9iUkXZEQ=="], + + "tree-sitter-typescript": ["tree-sitter-typescript@0.23.2", "", { "dependencies": { "node-addon-api": "^8.2.2", "node-gyp-build": "^4.8.2", "tree-sitter-javascript": "^0.23.1" }, "peerDependencies": { "tree-sitter": "^0.21.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-e04JUUKxTT53/x3Uq1zIL45DoYKVfHH4CZqwgZhPg5qYROl5nQjV+85ruFzFGZxu+QeFVbRTPDRnqL9UbU4VeA=="], + "trim-lines": ["trim-lines@3.0.1", "", {}, "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg=="], "trough": ["trough@2.2.0", "", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="], @@ -2692,6 +2725,8 @@ "@openworkflow/dashboard/react-dom": ["react-dom@19.2.5", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.5" } }, "sha512-J5bAZz+DXMMwW/wV3xzKke59Af6CHY7G4uYLN1OvBcKEsWOs4pQExj86BBKamxl/Ik5bx9whOrvBlSDfWzgSag=="], + "@parcel/watcher/node-addon-api": ["node-addon-api@7.1.1", "", {}, "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ=="], + "@reduxjs/toolkit/immer": ["immer@11.1.6", "", {}, "sha512-uwrF08UBQfxk49i9WcUeCx045wjB1zXEHNJmbYHPVVspxmjwSeWCoKbB8DEIvs3XkBJV6lcRAyLaWJ2+u3MMCw=="], "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.10.0", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" }, "bundled": true }, "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw=="], @@ -2832,6 +2867,8 @@ "strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], + "tree-sitter-typescript/tree-sitter-javascript": ["tree-sitter-javascript@0.23.1", "", { "dependencies": { "node-addon-api": "^8.2.2", "node-gyp-build": "^4.8.2" }, "peerDependencies": { "tree-sitter": "^0.21.1" }, "optionalPeers": ["tree-sitter"] }, "sha512-/bnhbrTD9frUYHQTiYnPcxyHORIw157ERBa6dqzaKxvR/x3PC4Yzd+D1pZIMS6zNg2v3a8BZ0oK7jHqsQo9fWA=="], + "whatwg-encoding/iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], "wrap-ansi/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], diff --git a/migrations/20260613184716_sturdy_triton/migration.sql b/migrations/20260613184716_sturdy_triton/migration.sql new file mode 100644 index 00000000..022e6a8f --- /dev/null +++ b/migrations/20260613184716_sturdy_triton/migration.sql @@ -0,0 +1,14 @@ +ALTER TABLE "relationships" ADD COLUMN IF NOT EXISTS "kind" text DEFAULT 'RELATED' NOT NULL; +ALTER TABLE "relationships" ADD COLUMN IF NOT EXISTS "directed" boolean DEFAULT false NOT NULL; + +INSERT INTO "file_type_configs" ("id", "organization_id", "file_type", "loader", "chunker", "chunk_size", "document_mode") +SELECT + 'ftc_' || "organization"."id" || '_code', + "organization"."id", + 'code', + 'text', + 'semantic', + 2000, + NULL +FROM "organization" +ON CONFLICT ("organization_id", "file_type") DO NOTHING; diff --git a/migrations/20260613184716_sturdy_triton/snapshot.json b/migrations/20260613184716_sturdy_triton/snapshot.json new file mode 100644 index 00000000..6b88b345 --- /dev/null +++ b/migrations/20260613184716_sturdy_triton/snapshot.json @@ -0,0 +1,6823 @@ +{ + "id": "9da60ec1-81fc-46b4-be3e-0b3b2debf5f1", + "prevIds": [ + "b9a9f2fb-03a6-4d2b-a459-a922887c2436" + ], + "version": "8", + "dialect": "postgres", + "ddl": [ + { + "isRlsEnabled": true, + "name": "account", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "apikey", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "invitation", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "member", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "organization_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "organization", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "session", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "team_member_roles", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "teamMember", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "team_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "team", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "user_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "user", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "verification", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "chats", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "messages", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "file_type_configs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "entities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "files", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graph_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graphs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graph_updates", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "process_run_files", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "process_runs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "process_stats", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "relationships", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "sources", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "text_units", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "models", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graph_suggestions", + "entityType": "tables", + "schema": "public" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "accountId", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "providerId", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "accessToken", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refreshToken", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "accessTokenExpiresAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refreshTokenExpiresAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "scope", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "idToken", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "password", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "config_id", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "start", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prefix", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "key", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reference_id", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refill_interval", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refill_amount", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_refill_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rate_limit_enabled", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rate_limit_time_window", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rate_limit_max", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "request_count", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remaining", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_request", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "permissions", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organizationId", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "email", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "teamId", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expiresAt", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "inviterId", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organizationId", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'member'", + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "systemRoleProvisioned", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "logo", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expiresAt", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ipAddress", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userAgent", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "impersonatedBy", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activeOrganizationId", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activeTeamId", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_member_id", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'member'", + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "teamId", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organizationId", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "email", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "emailVerified", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "image", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banned", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banReason", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banExpires", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "identifier", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "value", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expiresAt", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'graph'", + "generated": null, + "identity": null, + "name": "scope", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "project_id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "pinned_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "archived_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chat_id", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "parts", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tokens_per_second", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "time_to_first_token", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "input_tokens", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "output_tokens", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "total_tokens", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_type", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "loader", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunker", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunk_size", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "document_mode", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "active", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "vector(4096)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('simple', coalesce(name, '')), 'A') || setweight(to_tsvector('simple', coalesce(description, '')), 'B')", + "type": "stored" + }, + "identity": null, + "name": "search_tsv", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_size", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_type", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "mime_type", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_key", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "checksum", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "deleted", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'processing'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "process_step", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "process_error_code", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "token_count", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "loader", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunker", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunk_size", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "document_mode", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'ready'", + "generated": null, + "identity": null, + "name": "state", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "hidden", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "update_type", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "json", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "update_message", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "process_run_id", + "entityType": "columns", + "schema": "public", + "table": "process_run_files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_id", + "entityType": "columns", + "schema": "public", + "table": "process_run_files" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "process_run_files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "completed_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "total_time", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "files", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "file_sizes", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'unknown'", + "generated": null, + "identity": null, + "name": "file_type", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "token_count", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "active", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "target_id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'RELATED'", + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "directed", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "rank", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "vector(4096)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('simple', coalesce(description, '')), 'A')", + "type": "stored" + }, + "identity": null, + "name": "search_tsv", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "entity_id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "relationship_id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "text_unit_id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "active", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "json", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "source_chunk_ids", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "vector(4096)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('simple', coalesce(description, '')), 'A')", + "type": "stored" + }, + "identity": null, + "name": "search_tsv", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_id", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "text", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "start_page", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "end_page", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "json", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "chunks", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "model_id", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "display_name", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "adapter", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_model", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "250000", + "generated": null, + "identity": null, + "name": "context_window", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "encrypted_credentials", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "is_default", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "entity_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reference", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "suggestion", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "suggested_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chat_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "message_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_source_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_at", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "providerId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "account_user_provider_idx", + "entityType": "indexes", + "schema": "public", + "table": "account" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "email", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_email_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "role", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_role_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "teamId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_team_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "member_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "member_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "member_organization_user_unique", + "entityType": "indexes", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "organization_prompts_organization_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "organization_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "slug", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "organization_slug_idx", + "entityType": "indexes", + "schema": "public", + "table": "organization" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "teamId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_member_team_idx", + "entityType": "indexes", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_member_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "teamId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_member_team_user_unique", + "entityType": "indexes", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_prompts_team_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "team_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "team_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "user_prompts_user_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "user_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "project_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_project_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_team_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "project_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "(\"pinned_at\" is null)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"archived_at\" IS NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_project_archived_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "(\"pinned_at\" is null)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"archived_at\" IS NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_team_archived_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "title", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "chats_title_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "chat_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_chat_messages_chat_id_id", + "entityType": "indexes", + "schema": "public", + "table": "messages" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "chat_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "role", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_chat_messages_chat_role_status_id", + "entityType": "indexes", + "schema": "public", + "table": "messages" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "file_type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "file_type_configs_organization_file_type_unique", + "entityType": "indexes", + "schema": "public", + "table": "file_type_configs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "entities_graph_active_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "entities_graph_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "entities_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "embedding", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "vector_cosine_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "diskann", + "concurrently": false, + "name": "entities_embedding_diskann_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "checksum", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"deleted\" = false AND \"checksum\" IS NOT NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_checksum_active_unique", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "files_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_active_created_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "file_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_active_key_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_prompts_graph_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_organization_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_team_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_user_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_graph_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "graphs_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"graph_id\" IS NULL AND \"team_id\" IS NULL AND \"hidden\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_visible_root_organization_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"graph_id\" IS NULL AND \"hidden\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_visible_root_team_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "file_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "process_run_files_file_idx", + "entityType": "indexes", + "schema": "public", + "table": "process_run_files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "process_runs_graph_status_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "process_runs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "source_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_source_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "target_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_target_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "description", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "relationships_description_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "embedding", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "vector_cosine_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "diskann", + "concurrently": false, + "name": "relationships_embedding_diskann_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "entity_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_entity_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "relationship_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_relationship_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "text_unit_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_text_unit_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "description", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "sources_description_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "embedding", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "vector_cosine_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "diskann", + "concurrently": false, + "name": "sources_embedding_diskann_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "file_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "text_units_file_idx", + "entityType": "indexes", + "schema": "public", + "table": "text_units" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "model_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "models_organization_model_id_unique", + "entityType": "indexes", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"is_default\" = true", + "with": "", + "method": "btree", + "concurrently": false, + "name": "models_organization_type_default_unique", + "entityType": "indexes", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "models_organization_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_suggestions_graph_status_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_suggestions_source_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "entity_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_suggestions_entity_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "account_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "account" + }, + { + "nameExplicit": false, + "columns": [ + "organizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "invitation_organizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": false, + "columns": [ + "teamId" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "invitation_teamId_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": false, + "columns": [ + "inviterId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "invitation_inviterId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": false, + "columns": [ + "organizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "member_organizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "member_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "organization_prompts_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "organization_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "session_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "impersonatedBy" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "session_impersonatedBy_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "activeOrganizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "session_activeOrganizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "activeTeamId" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "session_activeTeamId_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "team_member_id" + ], + "schemaTo": "public", + "tableTo": "teamMember", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "team_member_roles_team_member_id_teamMember_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "team_member_roles" + }, + { + "nameExplicit": false, + "columns": [ + "teamId" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "teamMember_teamId_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "teamMember_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": false, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "team_prompts_team_id_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "team_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "organizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "team_organizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": false, + "columns": [ + "user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "user_prompts_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "user_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "chats_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": false, + "columns": [ + "project_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "chats_project_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": false, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "chats_team_id_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": false, + "columns": [ + "chat_id" + ], + "schemaTo": "public", + "tableTo": "chats", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "messages_chat_id_chats_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "messages" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "file_type_configs_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "file_type_configs" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "entities_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "files_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_prompts_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_team_id_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + "team_id", + "organization_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id", + "organizationId" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_team_organization_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_updates_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_updates" + }, + { + "nameExplicit": false, + "columns": [ + "process_run_id" + ], + "schemaTo": "public", + "tableTo": "process_runs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "process_run_files_process_run_id_process_runs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "process_run_files" + }, + { + "nameExplicit": false, + "columns": [ + "file_id" + ], + "schemaTo": "public", + "tableTo": "files", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "process_run_files_file_id_files_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "process_run_files" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "process_runs_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "process_runs" + }, + { + "nameExplicit": false, + "columns": [ + "source_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "relationships_source_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": false, + "columns": [ + "target_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "relationships_target_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "relationships_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": false, + "columns": [ + "entity_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "sources_entity_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": false, + "columns": [ + "relationship_id" + ], + "schemaTo": "public", + "tableTo": "relationships", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "sources_relationship_id_relationships_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": false, + "columns": [ + "text_unit_id" + ], + "schemaTo": "public", + "tableTo": "text_units", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "sources_text_unit_id_text_units_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": false, + "columns": [ + "file_id" + ], + "schemaTo": "public", + "tableTo": "files", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "text_units_file_id_files_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "text_units" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "models_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "source_id" + ], + "schemaTo": "public", + "tableTo": "sources", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_source_id_sources_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "entity_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_entity_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "suggested_by_user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_suggested_by_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "chat_id" + ], + "schemaTo": "public", + "tableTo": "chats", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_chat_id_chats_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "message_id" + ], + "schemaTo": "public", + "tableTo": "messages", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_message_id_messages_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "applied_by_user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_applied_by_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "applied_source_id" + ], + "schemaTo": "public", + "tableTo": "sources", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_applied_source_id_sources_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "columns": [ + "process_run_id", + "file_id" + ], + "nameExplicit": true, + "name": "process_run_files_pk", + "entityType": "pks", + "schema": "public", + "table": "process_run_files" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "account_pkey", + "schema": "public", + "table": "account", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "apikey_pkey", + "schema": "public", + "table": "apikey", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "invitation_pkey", + "schema": "public", + "table": "invitation", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "member_pkey", + "schema": "public", + "table": "member", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "organization_prompts_pkey", + "schema": "public", + "table": "organization_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "organization_pkey", + "schema": "public", + "table": "organization", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "session_pkey", + "schema": "public", + "table": "session", + "entityType": "pks" + }, + { + "columns": [ + "team_member_id" + ], + "nameExplicit": false, + "name": "team_member_roles_pkey", + "schema": "public", + "table": "team_member_roles", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "teamMember_pkey", + "schema": "public", + "table": "teamMember", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "team_prompts_pkey", + "schema": "public", + "table": "team_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "team_pkey", + "schema": "public", + "table": "team", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "user_prompts_pkey", + "schema": "public", + "table": "user_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "user_pkey", + "schema": "public", + "table": "user", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "verification_pkey", + "schema": "public", + "table": "verification", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "chats_pkey", + "schema": "public", + "table": "chats", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "messages_pkey", + "schema": "public", + "table": "messages", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "file_type_configs_pkey", + "schema": "public", + "table": "file_type_configs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "entities_pkey", + "schema": "public", + "table": "entities", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "files_pkey", + "schema": "public", + "table": "files", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graph_prompts_pkey", + "schema": "public", + "table": "graph_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graphs_pkey", + "schema": "public", + "table": "graphs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graph_updates_pkey", + "schema": "public", + "table": "graph_updates", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "process_runs_pkey", + "schema": "public", + "table": "process_runs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "process_stats_pkey", + "schema": "public", + "table": "process_stats", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "relationships_pkey", + "schema": "public", + "table": "relationships", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "sources_pkey", + "schema": "public", + "table": "sources", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "text_units_pkey", + "schema": "public", + "table": "text_units", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "models_pkey", + "schema": "public", + "table": "models", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graph_suggestions_pkey", + "schema": "public", + "table": "graph_suggestions", + "entityType": "pks" + }, + { + "nameExplicit": true, + "columns": [ + "id", + "organizationId" + ], + "nullsNotDistinct": false, + "name": "team_id_organization_unique", + "entityType": "uniques", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": false, + "columns": [ + "id" + ], + "nullsNotDistinct": false, + "name": "apikey_id_key", + "schema": "public", + "table": "apikey", + "entityType": "uniques" + }, + { + "nameExplicit": false, + "columns": [ + "slug" + ], + "nullsNotDistinct": false, + "name": "organization_slug_key", + "schema": "public", + "table": "organization", + "entityType": "uniques" + }, + { + "nameExplicit": false, + "columns": [ + "token" + ], + "nullsNotDistinct": false, + "name": "session_token_key", + "schema": "public", + "table": "session", + "entityType": "uniques" + }, + { + "nameExplicit": false, + "columns": [ + "email" + ], + "nullsNotDistinct": false, + "name": "user_email_key", + "schema": "public", + "table": "user", + "entityType": "uniques" + }, + { + "value": "\n (\n \"scope\" = 'graph'\n AND \"project_id\" IS NOT NULL\n AND \"team_id\" IS NULL\n )\n OR\n (\n \"scope\" = 'team'\n AND \"project_id\" IS NULL\n AND \"team_id\" IS NOT NULL\n )\n ", + "name": "chats_scope_target_check", + "entityType": "checks", + "schema": "public", + "table": "chats" + }, + { + "value": "jsonb_typeof(\"parts\") = 'array'", + "name": "chat_messages_parts_array_check", + "entityType": "checks", + "schema": "public", + "table": "messages" + }, + { + "value": "(((\"organization_id\" IS NOT NULL)::int + (\"user_id\" IS NOT NULL)::int + (\"graph_id\" IS NOT NULL)::int) = 1)", + "name": "graphs_single_owner_check", + "entityType": "checks", + "schema": "public", + "table": "graphs" + }, + { + "value": "\"team_id\" IS NULL OR \"organization_id\" IS NOT NULL", + "name": "graphs_team_requires_organization_check", + "entityType": "checks", + "schema": "public", + "table": "graphs" + }, + { + "value": "((\"start_page\" IS NULL AND \"end_page\" IS NULL) OR (\"start_page\" IS NOT NULL AND \"end_page\" IS NOT NULL AND \"start_page\" >= 1 AND \"end_page\" >= \"start_page\"))", + "name": "text_units_page_span_check", + "entityType": "checks", + "schema": "public", + "table": "text_units" + }, + { + "value": "\n (\n \"kind\" = 'source_correction'\n AND \"source_id\" IS NOT NULL\n AND \"entity_id\" IS NULL\n )\n OR\n (\n \"kind\" = 'entity_addition'\n AND \"source_id\" IS NULL\n AND \"entity_id\" IS NOT NULL\n )\n ", + "name": "graph_suggestions_target_check", + "entityType": "checks", + "schema": "public", + "table": "graph_suggestions" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/migrations/20260613201908_mature_emma_frost/migration.sql b/migrations/20260613201908_mature_emma_frost/migration.sql new file mode 100644 index 00000000..bdc2ab4f --- /dev/null +++ b/migrations/20260613201908_mature_emma_frost/migration.sql @@ -0,0 +1,20 @@ +ALTER TABLE "files" ADD COLUMN "storage_kind" text DEFAULT 'internal' NOT NULL; +ALTER TABLE "files" ADD COLUMN "external_url" text; +ALTER TABLE "files" ADD COLUMN "external_provider" text; + +ALTER TABLE "files" ADD CONSTRAINT "files_storage_origin_check" CHECK ( + ( + "storage_kind" = 'internal' + AND "external_url" IS NULL + AND "external_provider" IS NULL + ) + OR ( + "storage_kind" = 'external' + AND "external_url" IS NOT NULL + AND "external_provider" IS NOT NULL + ) +); + +ALTER TABLE "files" ADD CONSTRAINT "files_external_provider_check" CHECK ( + "external_provider" IS NULL OR "external_provider" = 'github' +); \ No newline at end of file diff --git a/migrations/20260613201908_mature_emma_frost/snapshot.json b/migrations/20260613201908_mature_emma_frost/snapshot.json new file mode 100644 index 00000000..bf0f9b39 --- /dev/null +++ b/migrations/20260613201908_mature_emma_frost/snapshot.json @@ -0,0 +1,6876 @@ +{ + "id": "e7424156-436b-42ec-adee-bdb5dd5d397a", + "prevIds": [ + "9da60ec1-81fc-46b4-be3e-0b3b2debf5f1" + ], + "version": "8", + "dialect": "postgres", + "ddl": [ + { + "isRlsEnabled": true, + "name": "account", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "apikey", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "invitation", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "member", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "organization_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "organization", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "session", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "team_member_roles", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "teamMember", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "team_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "team", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "user_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "user", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "verification", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "chats", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "messages", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "file_type_configs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "entities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "files", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graph_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graphs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graph_updates", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "process_run_files", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "process_runs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "process_stats", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "relationships", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "sources", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "text_units", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "models", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graph_suggestions", + "entityType": "tables", + "schema": "public" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "accountId", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "providerId", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "accessToken", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refreshToken", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "accessTokenExpiresAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refreshTokenExpiresAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "scope", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "idToken", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "password", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "config_id", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "start", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prefix", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "key", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reference_id", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refill_interval", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refill_amount", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_refill_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rate_limit_enabled", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rate_limit_time_window", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rate_limit_max", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "request_count", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remaining", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_request", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "permissions", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organizationId", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "email", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "teamId", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expiresAt", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "inviterId", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organizationId", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'member'", + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "systemRoleProvisioned", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "logo", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expiresAt", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ipAddress", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userAgent", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "impersonatedBy", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activeOrganizationId", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activeTeamId", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_member_id", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'member'", + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "teamId", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organizationId", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "email", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "emailVerified", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "image", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banned", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banReason", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banExpires", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "identifier", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "value", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expiresAt", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'graph'", + "generated": null, + "identity": null, + "name": "scope", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "project_id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "pinned_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "archived_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chat_id", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "parts", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tokens_per_second", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "time_to_first_token", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "input_tokens", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "output_tokens", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "total_tokens", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_type", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "loader", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunker", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunk_size", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "document_mode", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "active", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "vector(4096)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('simple', coalesce(name, '')), 'A') || setweight(to_tsvector('simple', coalesce(description, '')), 'B')", + "type": "stored" + }, + "identity": null, + "name": "search_tsv", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_size", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_type", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "mime_type", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_key", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'internal'", + "generated": null, + "identity": null, + "name": "storage_kind", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_url", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_provider", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "checksum", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "deleted", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'processing'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "process_step", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "process_error_code", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "token_count", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "loader", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunker", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunk_size", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "document_mode", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'ready'", + "generated": null, + "identity": null, + "name": "state", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "hidden", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "update_type", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "json", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "update_message", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "process_run_id", + "entityType": "columns", + "schema": "public", + "table": "process_run_files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_id", + "entityType": "columns", + "schema": "public", + "table": "process_run_files" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "process_run_files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "completed_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "total_time", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "files", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "file_sizes", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'unknown'", + "generated": null, + "identity": null, + "name": "file_type", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "token_count", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "active", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "target_id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'RELATED'", + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "directed", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "rank", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "vector(4096)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('simple', coalesce(description, '')), 'A')", + "type": "stored" + }, + "identity": null, + "name": "search_tsv", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "entity_id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "relationship_id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "text_unit_id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "active", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "json", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "source_chunk_ids", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "vector(4096)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('simple', coalesce(description, '')), 'A')", + "type": "stored" + }, + "identity": null, + "name": "search_tsv", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_id", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "text", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "start_page", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "end_page", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "json", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "chunks", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "model_id", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "display_name", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "adapter", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_model", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "250000", + "generated": null, + "identity": null, + "name": "context_window", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "encrypted_credentials", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "is_default", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "entity_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reference", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "suggestion", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "suggested_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chat_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "message_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_source_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_at", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "providerId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "account_user_provider_idx", + "entityType": "indexes", + "schema": "public", + "table": "account" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "email", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_email_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "role", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_role_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "teamId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_team_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "member_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "member_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "member_organization_user_unique", + "entityType": "indexes", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "organization_prompts_organization_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "organization_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "slug", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "organization_slug_idx", + "entityType": "indexes", + "schema": "public", + "table": "organization" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "teamId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_member_team_idx", + "entityType": "indexes", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_member_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "teamId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_member_team_user_unique", + "entityType": "indexes", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_prompts_team_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "team_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "team_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "user_prompts_user_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "user_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "project_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_project_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_team_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "project_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "(\"pinned_at\" is null)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"archived_at\" IS NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_project_archived_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "(\"pinned_at\" is null)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"archived_at\" IS NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_team_archived_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "title", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "chats_title_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "chat_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_chat_messages_chat_id_id", + "entityType": "indexes", + "schema": "public", + "table": "messages" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "chat_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "role", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_chat_messages_chat_role_status_id", + "entityType": "indexes", + "schema": "public", + "table": "messages" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "file_type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "file_type_configs_organization_file_type_unique", + "entityType": "indexes", + "schema": "public", + "table": "file_type_configs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "entities_graph_active_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "entities_graph_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "entities_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "embedding", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "vector_cosine_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "diskann", + "concurrently": false, + "name": "entities_embedding_diskann_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "checksum", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"deleted\" = false AND \"checksum\" IS NOT NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_checksum_active_unique", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "files_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_active_created_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "file_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_active_key_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_prompts_graph_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_organization_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_team_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_user_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_graph_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "graphs_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"graph_id\" IS NULL AND \"team_id\" IS NULL AND \"hidden\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_visible_root_organization_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"graph_id\" IS NULL AND \"hidden\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_visible_root_team_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "file_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "process_run_files_file_idx", + "entityType": "indexes", + "schema": "public", + "table": "process_run_files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "process_runs_graph_status_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "process_runs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "source_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_source_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "target_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_target_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "description", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "relationships_description_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "embedding", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "vector_cosine_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "diskann", + "concurrently": false, + "name": "relationships_embedding_diskann_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "entity_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_entity_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "relationship_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_relationship_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "text_unit_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_text_unit_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "description", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "sources_description_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "embedding", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "vector_cosine_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "diskann", + "concurrently": false, + "name": "sources_embedding_diskann_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "file_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "text_units_file_idx", + "entityType": "indexes", + "schema": "public", + "table": "text_units" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "model_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "models_organization_model_id_unique", + "entityType": "indexes", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"is_default\" = true", + "with": "", + "method": "btree", + "concurrently": false, + "name": "models_organization_type_default_unique", + "entityType": "indexes", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "models_organization_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_suggestions_graph_status_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_suggestions_source_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "entity_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_suggestions_entity_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "account_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "account" + }, + { + "nameExplicit": false, + "columns": [ + "organizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "invitation_organizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": false, + "columns": [ + "teamId" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "invitation_teamId_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": false, + "columns": [ + "inviterId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "invitation_inviterId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": false, + "columns": [ + "organizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "member_organizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "member_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "organization_prompts_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "organization_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "session_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "impersonatedBy" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "session_impersonatedBy_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "activeOrganizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "session_activeOrganizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "activeTeamId" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "session_activeTeamId_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "team_member_id" + ], + "schemaTo": "public", + "tableTo": "teamMember", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "team_member_roles_team_member_id_teamMember_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "team_member_roles" + }, + { + "nameExplicit": false, + "columns": [ + "teamId" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "teamMember_teamId_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "teamMember_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": false, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "team_prompts_team_id_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "team_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "organizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "team_organizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": false, + "columns": [ + "user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "user_prompts_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "user_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "chats_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": false, + "columns": [ + "project_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "chats_project_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": false, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "chats_team_id_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": false, + "columns": [ + "chat_id" + ], + "schemaTo": "public", + "tableTo": "chats", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "messages_chat_id_chats_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "messages" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "file_type_configs_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "file_type_configs" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "entities_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "files_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_prompts_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_team_id_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + "team_id", + "organization_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id", + "organizationId" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_team_organization_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_updates_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_updates" + }, + { + "nameExplicit": false, + "columns": [ + "process_run_id" + ], + "schemaTo": "public", + "tableTo": "process_runs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "process_run_files_process_run_id_process_runs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "process_run_files" + }, + { + "nameExplicit": false, + "columns": [ + "file_id" + ], + "schemaTo": "public", + "tableTo": "files", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "process_run_files_file_id_files_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "process_run_files" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "process_runs_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "process_runs" + }, + { + "nameExplicit": false, + "columns": [ + "source_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "relationships_source_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": false, + "columns": [ + "target_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "relationships_target_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "relationships_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": false, + "columns": [ + "entity_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "sources_entity_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": false, + "columns": [ + "relationship_id" + ], + "schemaTo": "public", + "tableTo": "relationships", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "sources_relationship_id_relationships_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": false, + "columns": [ + "text_unit_id" + ], + "schemaTo": "public", + "tableTo": "text_units", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "sources_text_unit_id_text_units_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": false, + "columns": [ + "file_id" + ], + "schemaTo": "public", + "tableTo": "files", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "text_units_file_id_files_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "text_units" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "models_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "source_id" + ], + "schemaTo": "public", + "tableTo": "sources", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_source_id_sources_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "entity_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_entity_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "suggested_by_user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_suggested_by_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "chat_id" + ], + "schemaTo": "public", + "tableTo": "chats", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_chat_id_chats_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "message_id" + ], + "schemaTo": "public", + "tableTo": "messages", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_message_id_messages_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "applied_by_user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_applied_by_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "applied_source_id" + ], + "schemaTo": "public", + "tableTo": "sources", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_applied_source_id_sources_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "columns": [ + "process_run_id", + "file_id" + ], + "nameExplicit": true, + "name": "process_run_files_pk", + "entityType": "pks", + "schema": "public", + "table": "process_run_files" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "account_pkey", + "schema": "public", + "table": "account", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "apikey_pkey", + "schema": "public", + "table": "apikey", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "invitation_pkey", + "schema": "public", + "table": "invitation", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "member_pkey", + "schema": "public", + "table": "member", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "organization_prompts_pkey", + "schema": "public", + "table": "organization_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "organization_pkey", + "schema": "public", + "table": "organization", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "session_pkey", + "schema": "public", + "table": "session", + "entityType": "pks" + }, + { + "columns": [ + "team_member_id" + ], + "nameExplicit": false, + "name": "team_member_roles_pkey", + "schema": "public", + "table": "team_member_roles", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "teamMember_pkey", + "schema": "public", + "table": "teamMember", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "team_prompts_pkey", + "schema": "public", + "table": "team_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "team_pkey", + "schema": "public", + "table": "team", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "user_prompts_pkey", + "schema": "public", + "table": "user_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "user_pkey", + "schema": "public", + "table": "user", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "verification_pkey", + "schema": "public", + "table": "verification", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "chats_pkey", + "schema": "public", + "table": "chats", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "messages_pkey", + "schema": "public", + "table": "messages", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "file_type_configs_pkey", + "schema": "public", + "table": "file_type_configs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "entities_pkey", + "schema": "public", + "table": "entities", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "files_pkey", + "schema": "public", + "table": "files", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graph_prompts_pkey", + "schema": "public", + "table": "graph_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graphs_pkey", + "schema": "public", + "table": "graphs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graph_updates_pkey", + "schema": "public", + "table": "graph_updates", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "process_runs_pkey", + "schema": "public", + "table": "process_runs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "process_stats_pkey", + "schema": "public", + "table": "process_stats", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "relationships_pkey", + "schema": "public", + "table": "relationships", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "sources_pkey", + "schema": "public", + "table": "sources", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "text_units_pkey", + "schema": "public", + "table": "text_units", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "models_pkey", + "schema": "public", + "table": "models", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graph_suggestions_pkey", + "schema": "public", + "table": "graph_suggestions", + "entityType": "pks" + }, + { + "nameExplicit": true, + "columns": [ + "id", + "organizationId" + ], + "nullsNotDistinct": false, + "name": "team_id_organization_unique", + "entityType": "uniques", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": false, + "columns": [ + "id" + ], + "nullsNotDistinct": false, + "name": "apikey_id_key", + "schema": "public", + "table": "apikey", + "entityType": "uniques" + }, + { + "nameExplicit": false, + "columns": [ + "slug" + ], + "nullsNotDistinct": false, + "name": "organization_slug_key", + "schema": "public", + "table": "organization", + "entityType": "uniques" + }, + { + "nameExplicit": false, + "columns": [ + "token" + ], + "nullsNotDistinct": false, + "name": "session_token_key", + "schema": "public", + "table": "session", + "entityType": "uniques" + }, + { + "nameExplicit": false, + "columns": [ + "email" + ], + "nullsNotDistinct": false, + "name": "user_email_key", + "schema": "public", + "table": "user", + "entityType": "uniques" + }, + { + "value": "\n (\n \"scope\" = 'graph'\n AND \"project_id\" IS NOT NULL\n AND \"team_id\" IS NULL\n )\n OR\n (\n \"scope\" = 'team'\n AND \"project_id\" IS NULL\n AND \"team_id\" IS NOT NULL\n )\n ", + "name": "chats_scope_target_check", + "entityType": "checks", + "schema": "public", + "table": "chats" + }, + { + "value": "jsonb_typeof(\"parts\") = 'array'", + "name": "chat_messages_parts_array_check", + "entityType": "checks", + "schema": "public", + "table": "messages" + }, + { + "value": "(((\"organization_id\" IS NOT NULL)::int + (\"user_id\" IS NOT NULL)::int + (\"graph_id\" IS NOT NULL)::int) = 1)", + "name": "graphs_single_owner_check", + "entityType": "checks", + "schema": "public", + "table": "graphs" + }, + { + "value": "\"team_id\" IS NULL OR \"organization_id\" IS NOT NULL", + "name": "graphs_team_requires_organization_check", + "entityType": "checks", + "schema": "public", + "table": "graphs" + }, + { + "value": "((\"start_page\" IS NULL AND \"end_page\" IS NULL) OR (\"start_page\" IS NOT NULL AND \"end_page\" IS NOT NULL AND \"start_page\" >= 1 AND \"end_page\" >= \"start_page\"))", + "name": "text_units_page_span_check", + "entityType": "checks", + "schema": "public", + "table": "text_units" + }, + { + "value": "\n (\n \"kind\" = 'source_correction'\n AND \"source_id\" IS NOT NULL\n AND \"entity_id\" IS NULL\n )\n OR\n (\n \"kind\" = 'entity_addition'\n AND \"source_id\" IS NULL\n AND \"entity_id\" IS NOT NULL\n )\n ", + "name": "graph_suggestions_target_check", + "entityType": "checks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "value": "\n (\n \"storage_kind\" = 'internal'\n AND \"external_url\" IS NULL\n AND \"external_provider\" IS NULL\n )\n OR (\n \"storage_kind\" = 'external'\n AND \"external_url\" IS NOT NULL\n AND \"external_provider\" IS NOT NULL\n )\n ", + "name": "files_storage_origin_check", + "entityType": "checks", + "schema": "public", + "table": "files" + }, + { + "value": "\"external_provider\" IS NULL OR \"external_provider\" = 'github'", + "name": "files_external_provider_check", + "entityType": "checks", + "schema": "public", + "table": "files" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/migrations/20260614105716_tricky_plazm/migration.sql b/migrations/20260614105716_tricky_plazm/migration.sql new file mode 100644 index 00000000..d1a67328 --- /dev/null +++ b/migrations/20260614105716_tricky_plazm/migration.sql @@ -0,0 +1,115 @@ +ALTER TABLE "sources" ADD COLUMN IF NOT EXISTS "valid_until" timestamp with time zone; + +CREATE INDEX IF NOT EXISTS "sources_current_id_idx" + ON "sources" ("id") + WHERE "active" = true AND "valid_until" IS NULL; + +CREATE INDEX IF NOT EXISTS "sources_entity_current_id_idx" + ON "sources" ("entity_id", "id") + WHERE "active" = true AND "valid_until" IS NULL AND "entity_id" IS NOT NULL; + +CREATE INDEX IF NOT EXISTS "sources_relationship_current_id_idx" + ON "sources" ("relationship_id", "id") + WHERE "active" = true AND "valid_until" IS NULL AND "relationship_id" IS NOT NULL; + +CREATE TABLE IF NOT EXISTS "connectors" ( + "id" text PRIMARY KEY NOT NULL, + "provider" text NOT NULL, + "name" text NOT NULL, + "slug" text NOT NULL, + "status" text DEFAULT 'active' NOT NULL, + "app_id" text, + "app_slug" text, + "client_id" text, + "encrypted_credentials" text NOT NULL, + "webhook_secret_encrypted" text NOT NULL, + "created_by_user_id" text, + "created_at" timestamp with time zone DEFAULT now(), + "updated_at" timestamp with time zone DEFAULT now(), + CONSTRAINT "connectors_slug_unique" UNIQUE("slug"), + CONSTRAINT "connectors_created_by_user_id_user_id_fk" FOREIGN KEY ("created_by_user_id") REFERENCES "public"."user"("id") ON DELETE set null, + CONSTRAINT "connectors_provider_check" CHECK ("connectors"."provider" in ('github', 'gitlab')), + CONSTRAINT "connectors_status_check" CHECK ("connectors"."status" in ('draft', 'active', 'disabled')) +); + +CREATE TABLE IF NOT EXISTS "connector_installations" ( + "id" text PRIMARY KEY NOT NULL, + "connector_id" text NOT NULL, + "provider" text NOT NULL, + "provider_installation_id" text NOT NULL, + "provider_account_login" text NOT NULL, + "provider_account_type" text, + "organization_id" text, + "team_id" text, + "installed_by_user_id" text, + "encrypted_credentials" text, + "repository_selection" text DEFAULT 'unknown' NOT NULL, + "status" text DEFAULT 'active' NOT NULL, + "created_at" timestamp with time zone DEFAULT now(), + "updated_at" timestamp with time zone DEFAULT now(), + CONSTRAINT "connector_installations_connector_id_connectors_id_fk" FOREIGN KEY ("connector_id") REFERENCES "public"."connectors"("id") ON DELETE cascade, + CONSTRAINT "connector_installations_organization_id_organization_id_fk" FOREIGN KEY ("organization_id") REFERENCES "public"."organization"("id") ON DELETE cascade, + CONSTRAINT "connector_installations_team_id_team_id_fk" FOREIGN KEY ("team_id") REFERENCES "public"."team"("id") ON DELETE cascade, + CONSTRAINT "connector_installations_installed_by_user_id_user_id_fk" FOREIGN KEY ("installed_by_user_id") REFERENCES "public"."user"("id") ON DELETE set null, + CONSTRAINT "connector_installations_provider_scope_unique" UNIQUE("connector_id", "provider_installation_id", "organization_id", "team_id"), + CONSTRAINT "connector_installations_provider_check" CHECK ("connector_installations"."provider" in ('github', 'gitlab')), + CONSTRAINT "connector_installations_status_check" CHECK ("connector_installations"."status" in ('active', 'disabled')), + CONSTRAINT "connector_installations_owner_scope_check" CHECK (("organization_id" is not null and "team_id" is null) or ("organization_id" is not null and "team_id" is not null)) +); + +CREATE TABLE IF NOT EXISTS "repository_graph_bindings" ( + "id" text PRIMARY KEY NOT NULL, + "graph_id" text NOT NULL, + "connector_installation_id" text NOT NULL, + "provider" text NOT NULL, + "provider_repository_id" text NOT NULL, + "repository_full_name" text NOT NULL, + "repository_html_url" text NOT NULL, + "branch" text NOT NULL, + "last_seen_commit_sha" text, + "last_synced_commit_sha" text, + "sync_status" text DEFAULT 'pending' NOT NULL, + "sync_error_code" text, + "webhook_enabled" boolean DEFAULT true NOT NULL, + "created_at" timestamp with time zone DEFAULT now(), + "updated_at" timestamp with time zone DEFAULT now(), + CONSTRAINT "repository_graph_bindings_graph_id_graphs_id_fk" FOREIGN KEY ("graph_id") REFERENCES "public"."graphs"("id") ON DELETE cascade, + CONSTRAINT "repository_graph_bindings_connector_installation_id_fk" FOREIGN KEY ("connector_installation_id") REFERENCES "public"."connector_installations"("id") ON DELETE restrict, + CONSTRAINT "repository_graph_bindings_provider_check" CHECK ("repository_graph_bindings"."provider" in ('github', 'gitlab')), + CONSTRAINT "repository_graph_bindings_sync_status_check" CHECK ("repository_graph_bindings"."sync_status" in ('pending', 'syncing', 'synced', 'failed')) +); + +CREATE TABLE IF NOT EXISTS "connector_webhook_events" ( + "id" text PRIMARY KEY NOT NULL, + "connector_id" text NOT NULL, + "provider" text NOT NULL, + "delivery_id" text NOT NULL, + "event_name" text NOT NULL, + "provider_repository_id" text, + "branch" text, + "commit_sha" text, + "status" text NOT NULL, + "error_code" text, + "created_at" timestamp with time zone DEFAULT now(), + CONSTRAINT "connector_webhook_events_connector_id_connectors_id_fk" FOREIGN KEY ("connector_id") REFERENCES "public"."connectors"("id") ON DELETE cascade, + CONSTRAINT "connector_webhook_events_provider_check" CHECK ("connector_webhook_events"."provider" in ('github', 'gitlab')), + CONSTRAINT "connector_webhook_events_status_check" CHECK ("connector_webhook_events"."status" in ('ignored', 'enqueued', 'duplicate', 'failed')) +); + +ALTER TABLE "files" ADD COLUMN IF NOT EXISTS "repository_binding_id" text; +ALTER TABLE "files" DROP CONSTRAINT IF EXISTS "files_external_provider_check"; +ALTER TABLE "files" ADD CONSTRAINT "files_external_provider_check" CHECK ("external_provider" IS NULL OR "external_provider" in ('github', 'gitlab')); +ALTER TABLE "files" DROP CONSTRAINT IF EXISTS "files_repository_binding_id_repository_graph_bindings_id_fk"; +ALTER TABLE "files" ADD CONSTRAINT "files_repository_binding_id_repository_graph_bindings_id_fk" FOREIGN KEY ("repository_binding_id") REFERENCES "public"."repository_graph_bindings"("id") ON DELETE set null; + +CREATE INDEX IF NOT EXISTS "connectors_provider_status_idx" ON "connectors" ("provider", "status"); +CREATE INDEX IF NOT EXISTS "connector_installations_connector_status_idx" ON "connector_installations" ("connector_id", "status"); +CREATE INDEX IF NOT EXISTS "connector_installations_organization_idx" ON "connector_installations" ("organization_id"); +CREATE INDEX IF NOT EXISTS "connector_installations_team_idx" ON "connector_installations" ("team_id"); +CREATE UNIQUE INDEX IF NOT EXISTS "repository_graph_bindings_graph_unique" ON "repository_graph_bindings" ("graph_id"); +CREATE UNIQUE INDEX IF NOT EXISTS "repository_graph_bindings_repository_branch_unique" ON "repository_graph_bindings" ("connector_installation_id", "provider_repository_id", "branch"); +CREATE INDEX IF NOT EXISTS "repository_graph_bindings_provider_repo_branch_idx" ON "repository_graph_bindings" ("provider", "provider_repository_id", "branch"); +CREATE INDEX IF NOT EXISTS "repository_graph_bindings_installation_status_idx" ON "repository_graph_bindings" ("connector_installation_id", "sync_status"); +CREATE UNIQUE INDEX IF NOT EXISTS "connector_webhook_events_delivery_unique" ON "connector_webhook_events" ("connector_id", "provider", "delivery_id"); +CREATE INDEX IF NOT EXISTS "connector_webhook_events_binding_lookup_idx" ON "connector_webhook_events" ("provider", "provider_repository_id", "branch"); +CREATE INDEX IF NOT EXISTS "files_repository_binding_active_idx" ON "files" ("repository_binding_id", "created_at", "id") WHERE "deleted" = false; \ No newline at end of file diff --git a/migrations/20260614105716_tricky_plazm/snapshot.json b/migrations/20260614105716_tricky_plazm/snapshot.json new file mode 100644 index 00000000..cbcc90c7 --- /dev/null +++ b/migrations/20260614105716_tricky_plazm/snapshot.json @@ -0,0 +1,8283 @@ +{ + "id": "9b6b85c6-0014-4a15-bb55-fd27e7e3b3d2", + "prevIds": [ + "e7424156-436b-42ec-adee-bdb5dd5d397a" + ], + "version": "8", + "dialect": "postgres", + "ddl": [ + { + "isRlsEnabled": true, + "name": "account", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "apikey", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "invitation", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "member", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "organization_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "organization", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "session", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "team_member_roles", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "teamMember", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "team_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "team", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "user_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "user", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "verification", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "chats", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "messages", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "file_type_configs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "entities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "files", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graph_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graphs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graph_updates", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "process_run_files", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "process_runs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "process_stats", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "relationships", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "sources", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "text_units", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "models", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graph_suggestions", + "entityType": "tables", + "schema": "public" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "accountId", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "providerId", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "accessToken", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refreshToken", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "accessTokenExpiresAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refreshTokenExpiresAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "scope", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "idToken", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "password", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "config_id", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "start", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prefix", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "key", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reference_id", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refill_interval", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refill_amount", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_refill_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rate_limit_enabled", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rate_limit_time_window", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rate_limit_max", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "request_count", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remaining", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_request", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "permissions", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organizationId", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "email", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "teamId", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expiresAt", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "inviterId", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organizationId", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'member'", + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "systemRoleProvisioned", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "logo", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expiresAt", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ipAddress", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userAgent", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "impersonatedBy", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activeOrganizationId", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activeTeamId", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_member_id", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'member'", + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "teamId", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organizationId", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "email", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "emailVerified", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "image", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banned", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banReason", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banExpires", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "identifier", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "value", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expiresAt", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'graph'", + "generated": null, + "identity": null, + "name": "scope", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "project_id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "pinned_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "archived_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chat_id", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "parts", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tokens_per_second", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "time_to_first_token", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "input_tokens", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "output_tokens", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "total_tokens", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_type", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "loader", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunker", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunk_size", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "document_mode", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "active", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "vector(4096)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('simple', coalesce(name, '')), 'A') || setweight(to_tsvector('simple', coalesce(description, '')), 'B')", + "type": "stored" + }, + "identity": null, + "name": "search_tsv", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_size", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_type", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "mime_type", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_key", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'internal'", + "generated": null, + "identity": null, + "name": "storage_kind", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_url", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_provider", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "checksum", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "deleted", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'processing'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "process_step", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "process_error_code", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "token_count", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "loader", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunker", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunk_size", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "document_mode", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'ready'", + "generated": null, + "identity": null, + "name": "state", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "hidden", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "update_type", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "json", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "update_message", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "process_run_id", + "entityType": "columns", + "schema": "public", + "table": "process_run_files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_id", + "entityType": "columns", + "schema": "public", + "table": "process_run_files" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "process_run_files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "completed_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "total_time", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "files", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "file_sizes", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'unknown'", + "generated": null, + "identity": null, + "name": "file_type", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "token_count", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "active", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "target_id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'RELATED'", + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "directed", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "rank", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "vector(4096)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('simple', coalesce(description, '')), 'A')", + "type": "stored" + }, + "identity": null, + "name": "search_tsv", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "entity_id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "relationship_id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "text_unit_id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "active", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "valid_until", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "json", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "source_chunk_ids", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "vector(4096)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('simple', coalesce(description, '')), 'A')", + "type": "stored" + }, + "identity": null, + "name": "search_tsv", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_id", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "text", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "start_page", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "end_page", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "json", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "chunks", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "model_id", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "display_name", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "adapter", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_model", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "250000", + "generated": null, + "identity": null, + "name": "context_window", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "encrypted_credentials", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "is_default", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "entity_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reference", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "suggestion", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "suggested_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chat_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "message_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_source_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_at", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "providerId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "account_user_provider_idx", + "entityType": "indexes", + "schema": "public", + "table": "account" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "email", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_email_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "role", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_role_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "teamId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_team_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "member_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "member_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "member_organization_user_unique", + "entityType": "indexes", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "organization_prompts_organization_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "organization_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "slug", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "organization_slug_idx", + "entityType": "indexes", + "schema": "public", + "table": "organization" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "teamId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_member_team_idx", + "entityType": "indexes", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_member_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "teamId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_member_team_user_unique", + "entityType": "indexes", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_prompts_team_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "team_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "team_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "user_prompts_user_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "user_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "project_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_project_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_team_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "project_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "(\"pinned_at\" is null)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"archived_at\" IS NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_project_archived_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "(\"pinned_at\" is null)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"archived_at\" IS NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_team_archived_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "title", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "chats_title_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "chat_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_chat_messages_chat_id_id", + "entityType": "indexes", + "schema": "public", + "table": "messages" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "chat_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "role", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_chat_messages_chat_role_status_id", + "entityType": "indexes", + "schema": "public", + "table": "messages" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "file_type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "file_type_configs_organization_file_type_unique", + "entityType": "indexes", + "schema": "public", + "table": "file_type_configs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "entities_graph_active_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "entities_graph_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "entities_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "embedding", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "vector_cosine_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "diskann", + "concurrently": false, + "name": "entities_embedding_diskann_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "checksum", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"deleted\" = false AND \"checksum\" IS NOT NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_checksum_active_unique", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "files_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_active_created_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "file_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_active_key_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_prompts_graph_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_organization_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_team_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_user_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_graph_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "graphs_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"graph_id\" IS NULL AND \"team_id\" IS NULL AND \"hidden\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_visible_root_organization_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"graph_id\" IS NULL AND \"hidden\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_visible_root_team_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "file_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "process_run_files_file_idx", + "entityType": "indexes", + "schema": "public", + "table": "process_run_files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "process_runs_graph_status_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "process_runs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "source_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_source_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "target_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_target_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "description", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "relationships_description_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "embedding", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "vector_cosine_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "diskann", + "concurrently": false, + "name": "relationships_embedding_diskann_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "entity_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_entity_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "relationship_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_relationship_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"active\" = true AND \"valid_until\" IS NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_current_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "entity_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"active\" = true AND \"valid_until\" IS NULL AND \"entity_id\" IS NOT NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_entity_current_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "relationship_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"active\" = true AND \"valid_until\" IS NULL AND \"relationship_id\" IS NOT NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_relationship_current_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "text_unit_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_text_unit_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "description", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "sources_description_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "embedding", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "vector_cosine_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "diskann", + "concurrently": false, + "name": "sources_embedding_diskann_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "file_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "text_units_file_idx", + "entityType": "indexes", + "schema": "public", + "table": "text_units" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "model_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "models_organization_model_id_unique", + "entityType": "indexes", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"is_default\" = true", + "with": "", + "method": "btree", + "concurrently": false, + "name": "models_organization_type_default_unique", + "entityType": "indexes", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "models_organization_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_suggestions_graph_status_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_suggestions_source_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "entity_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_suggestions_entity_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "account_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "account" + }, + { + "nameExplicit": false, + "columns": [ + "organizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "invitation_organizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": false, + "columns": [ + "teamId" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "invitation_teamId_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": false, + "columns": [ + "inviterId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "invitation_inviterId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": false, + "columns": [ + "organizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "member_organizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "member_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "organization_prompts_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "organization_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "session_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "impersonatedBy" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "session_impersonatedBy_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "activeOrganizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "session_activeOrganizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "activeTeamId" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "session_activeTeamId_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "team_member_id" + ], + "schemaTo": "public", + "tableTo": "teamMember", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "team_member_roles_team_member_id_teamMember_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "team_member_roles" + }, + { + "nameExplicit": false, + "columns": [ + "teamId" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "teamMember_teamId_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "teamMember_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": false, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "team_prompts_team_id_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "team_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "organizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "team_organizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": false, + "columns": [ + "user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "user_prompts_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "user_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "chats_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": false, + "columns": [ + "project_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "chats_project_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": false, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "chats_team_id_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": false, + "columns": [ + "chat_id" + ], + "schemaTo": "public", + "tableTo": "chats", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "messages_chat_id_chats_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "messages" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "file_type_configs_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "file_type_configs" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "entities_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "files_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_prompts_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_team_id_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + "team_id", + "organization_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id", + "organizationId" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_team_organization_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_updates_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_updates" + }, + { + "nameExplicit": false, + "columns": [ + "process_run_id" + ], + "schemaTo": "public", + "tableTo": "process_runs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "process_run_files_process_run_id_process_runs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "process_run_files" + }, + { + "nameExplicit": false, + "columns": [ + "file_id" + ], + "schemaTo": "public", + "tableTo": "files", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "process_run_files_file_id_files_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "process_run_files" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "process_runs_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "process_runs" + }, + { + "nameExplicit": false, + "columns": [ + "source_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "relationships_source_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": false, + "columns": [ + "target_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "relationships_target_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "relationships_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": false, + "columns": [ + "entity_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "sources_entity_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": false, + "columns": [ + "relationship_id" + ], + "schemaTo": "public", + "tableTo": "relationships", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "sources_relationship_id_relationships_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": false, + "columns": [ + "text_unit_id" + ], + "schemaTo": "public", + "tableTo": "text_units", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "sources_text_unit_id_text_units_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": false, + "columns": [ + "file_id" + ], + "schemaTo": "public", + "tableTo": "files", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "text_units_file_id_files_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "text_units" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "models_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "source_id" + ], + "schemaTo": "public", + "tableTo": "sources", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_source_id_sources_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "entity_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_entity_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "suggested_by_user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_suggested_by_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "chat_id" + ], + "schemaTo": "public", + "tableTo": "chats", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_chat_id_chats_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "message_id" + ], + "schemaTo": "public", + "tableTo": "messages", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_message_id_messages_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "applied_by_user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_applied_by_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "applied_source_id" + ], + "schemaTo": "public", + "tableTo": "sources", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_applied_source_id_sources_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "columns": [ + "process_run_id", + "file_id" + ], + "nameExplicit": true, + "name": "process_run_files_pk", + "entityType": "pks", + "schema": "public", + "table": "process_run_files" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "account_pkey", + "schema": "public", + "table": "account", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "apikey_pkey", + "schema": "public", + "table": "apikey", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "invitation_pkey", + "schema": "public", + "table": "invitation", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "member_pkey", + "schema": "public", + "table": "member", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "organization_prompts_pkey", + "schema": "public", + "table": "organization_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "organization_pkey", + "schema": "public", + "table": "organization", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "session_pkey", + "schema": "public", + "table": "session", + "entityType": "pks" + }, + { + "columns": [ + "team_member_id" + ], + "nameExplicit": false, + "name": "team_member_roles_pkey", + "schema": "public", + "table": "team_member_roles", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "teamMember_pkey", + "schema": "public", + "table": "teamMember", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "team_prompts_pkey", + "schema": "public", + "table": "team_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "team_pkey", + "schema": "public", + "table": "team", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "user_prompts_pkey", + "schema": "public", + "table": "user_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "user_pkey", + "schema": "public", + "table": "user", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "verification_pkey", + "schema": "public", + "table": "verification", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "chats_pkey", + "schema": "public", + "table": "chats", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "messages_pkey", + "schema": "public", + "table": "messages", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "file_type_configs_pkey", + "schema": "public", + "table": "file_type_configs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "entities_pkey", + "schema": "public", + "table": "entities", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "files_pkey", + "schema": "public", + "table": "files", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graph_prompts_pkey", + "schema": "public", + "table": "graph_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graphs_pkey", + "schema": "public", + "table": "graphs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graph_updates_pkey", + "schema": "public", + "table": "graph_updates", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "process_runs_pkey", + "schema": "public", + "table": "process_runs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "process_stats_pkey", + "schema": "public", + "table": "process_stats", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "relationships_pkey", + "schema": "public", + "table": "relationships", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "sources_pkey", + "schema": "public", + "table": "sources", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "text_units_pkey", + "schema": "public", + "table": "text_units", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "models_pkey", + "schema": "public", + "table": "models", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graph_suggestions_pkey", + "schema": "public", + "table": "graph_suggestions", + "entityType": "pks" + }, + { + "nameExplicit": true, + "columns": [ + "id", + "organizationId" + ], + "nullsNotDistinct": false, + "name": "team_id_organization_unique", + "entityType": "uniques", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": false, + "columns": [ + "id" + ], + "nullsNotDistinct": false, + "name": "apikey_id_key", + "schema": "public", + "table": "apikey", + "entityType": "uniques" + }, + { + "nameExplicit": false, + "columns": [ + "slug" + ], + "nullsNotDistinct": false, + "name": "organization_slug_key", + "schema": "public", + "table": "organization", + "entityType": "uniques" + }, + { + "nameExplicit": false, + "columns": [ + "token" + ], + "nullsNotDistinct": false, + "name": "session_token_key", + "schema": "public", + "table": "session", + "entityType": "uniques" + }, + { + "nameExplicit": false, + "columns": [ + "email" + ], + "nullsNotDistinct": false, + "name": "user_email_key", + "schema": "public", + "table": "user", + "entityType": "uniques" + }, + { + "value": "\n (\n \"scope\" = 'graph'\n AND \"project_id\" IS NOT NULL\n AND \"team_id\" IS NULL\n )\n OR\n (\n \"scope\" = 'team'\n AND \"project_id\" IS NULL\n AND \"team_id\" IS NOT NULL\n )\n ", + "name": "chats_scope_target_check", + "entityType": "checks", + "schema": "public", + "table": "chats" + }, + { + "value": "jsonb_typeof(\"parts\") = 'array'", + "name": "chat_messages_parts_array_check", + "entityType": "checks", + "schema": "public", + "table": "messages" + }, + { + "value": "(((\"organization_id\" IS NOT NULL)::int + (\"user_id\" IS NOT NULL)::int + (\"graph_id\" IS NOT NULL)::int) = 1)", + "name": "graphs_single_owner_check", + "entityType": "checks", + "schema": "public", + "table": "graphs" + }, + { + "value": "\"team_id\" IS NULL OR \"organization_id\" IS NOT NULL", + "name": "graphs_team_requires_organization_check", + "entityType": "checks", + "schema": "public", + "table": "graphs" + }, + { + "value": "((\"start_page\" IS NULL AND \"end_page\" IS NULL) OR (\"start_page\" IS NOT NULL AND \"end_page\" IS NOT NULL AND \"start_page\" >= 1 AND \"end_page\" >= \"start_page\"))", + "name": "text_units_page_span_check", + "entityType": "checks", + "schema": "public", + "table": "text_units" + }, + { + "value": "\n (\n \"kind\" = 'source_correction'\n AND \"source_id\" IS NOT NULL\n AND \"entity_id\" IS NULL\n )\n OR\n (\n \"kind\" = 'entity_addition'\n AND \"source_id\" IS NULL\n AND \"entity_id\" IS NOT NULL\n )\n ", + "name": "graph_suggestions_target_check", + "entityType": "checks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "value": "\n (\n \"storage_kind\" = 'internal'\n AND \"external_url\" IS NULL\n AND \"external_provider\" IS NULL\n )\n OR (\n \"storage_kind\" = 'external'\n AND \"external_url\" IS NOT NULL\n AND \"external_provider\" IS NOT NULL\n )\n ", + "name": "files_storage_origin_check", + "entityType": "checks", + "schema": "public", + "table": "files" + }, + { + "isRlsEnabled": true, + "name": "connectors", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "connector_installations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "repository_graph_bindings", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "connector_webhook_events", + "entityType": "tables", + "schema": "public" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'active'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_id", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_slug", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "client_id", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "encrypted_credentials", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "webhook_secret_encrypted", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connector_id", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_installation_id", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_account_login", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_account_type", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "installed_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "encrypted_credentials", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'unknown'", + "generated": null, + "identity": null, + "name": "repository_selection", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'active'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connector_installation_id", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_repository_id", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repository_full_name", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repository_html_url", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "branch", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_seen_commit_sha", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_synced_commit_sha", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "sync_status", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "sync_error_code", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "webhook_enabled", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connector_id", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "delivery_id", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "event_name", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_repository_id", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "branch", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "commit_sha", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error_code", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repository_binding_id", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "name": "connectors_provider_check", + "entityType": "checks", + "schema": "public", + "table": "connectors" + }, + { + "name": "connectors_status_check", + "entityType": "checks", + "schema": "public", + "table": "connectors" + }, + { + "name": "connector_installations_provider_check", + "entityType": "checks", + "schema": "public", + "table": "connector_installations" + }, + { + "name": "connector_installations_status_check", + "entityType": "checks", + "schema": "public", + "table": "connector_installations" + }, + { + "name": "connector_installations_owner_scope_check", + "entityType": "checks", + "schema": "public", + "table": "connector_installations" + }, + { + "name": "repository_graph_bindings_provider_check", + "entityType": "checks", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "name": "repository_graph_bindings_sync_status_check", + "entityType": "checks", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "name": "connector_webhook_events_provider_check", + "entityType": "checks", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "name": "connector_webhook_events_status_check", + "entityType": "checks", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "name": "files_external_provider_check", + "entityType": "checks", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "slug", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connectors_slug_unique", + "entityType": "indexes", + "schema": "public", + "table": "connectors" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connectors_provider_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "connectors" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider_installation_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_installations_provider_scope_unique", + "entityType": "indexes", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_installations_connector_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_installations_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_installations_team_idx", + "entityType": "indexes", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "repository_graph_bindings_graph_unique", + "entityType": "indexes", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector_installation_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider_repository_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "branch", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "repository_graph_bindings_repository_branch_unique", + "entityType": "indexes", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider_repository_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "branch", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "repository_graph_bindings_provider_repo_branch_idx", + "entityType": "indexes", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector_installation_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "sync_status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "repository_graph_bindings_installation_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "delivery_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_webhook_events_delivery_unique", + "entityType": "indexes", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider_repository_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "branch", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_webhook_events_binding_lookup_idx", + "entityType": "indexes", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "repository_binding_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_repository_binding_active_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + "created_by_user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "connectors_created_by_user_id_user_id_fk", + "entityType": "fks", + "schema": "public", + "table": "connectors" + }, + { + "nameExplicit": true, + "columns": [ + "connector_id" + ], + "schemaTo": "public", + "tableTo": "connectors", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "connector_installations_connector_id_connectors_id_fk", + "entityType": "fks", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "connector_installations_organization_id_organization_id_fk", + "entityType": "fks", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "connector_installations_team_id_team_id_fk", + "entityType": "fks", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + "installed_by_user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "connector_installations_installed_by_user_id_user_id_fk", + "entityType": "fks", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "repository_graph_bindings_graph_id_graphs_id_fk", + "entityType": "fks", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "nameExplicit": true, + "columns": [ + "connector_installation_id" + ], + "schemaTo": "public", + "tableTo": "connector_installations", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "RESTRICT", + "name": "repository_graph_bindings_connector_installation_id_fk", + "entityType": "fks", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "nameExplicit": true, + "columns": [ + "connector_id" + ], + "schemaTo": "public", + "tableTo": "connectors", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "connector_webhook_events_connector_id_connectors_id_fk", + "entityType": "fks", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "nameExplicit": true, + "columns": [ + "repository_binding_id" + ], + "schemaTo": "public", + "tableTo": "repository_graph_bindings", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "files_repository_binding_id_repository_graph_bindings_id_fk", + "entityType": "fks", + "schema": "public", + "table": "files" + } + ], + "renames": [] +} diff --git a/packages/ai/src/__tests__/relationship-tools.test.ts b/packages/ai/src/__tests__/relationship-tools.test.ts new file mode 100644 index 00000000..a3f27102 --- /dev/null +++ b/packages/ai/src/__tests__/relationship-tools.test.ts @@ -0,0 +1,181 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; + +type Row = Record; + +const selectResults: Row[][] = []; + +function nextSelectResult() { + const result = selectResults.shift(); + if (!result) { + throw new Error("Unexpected select query"); + } + + return result; +} + +function queryResult() { + return { + then: (resolve: (value: Row[]) => unknown, reject: (error: unknown) => unknown) => + Promise.resolve().then(nextSelectResult).then(resolve, reject), + limit: () => nextSelectResult(), + orderBy: () => ({ + limit: () => nextSelectResult(), + }), + }; +} + +mock.module("@kiwi/db", () => ({ + db: { + select: () => ({ + from: () => ({ + where: () => queryResult(), + }), + }), + }, +})); + +mock.module("@kiwi/logger", () => ({ + debug: () => undefined, + error: () => undefined, + info: () => undefined, + warn: () => undefined, +})); + +const { getNeighboursTool, getPathBetweenTool } = await import("../tools/relationship"); + +async function executeTool(tool: { execute?: (input: unknown) => Promise }, input: unknown) { + if (!tool.execute) { + throw new Error("Tool has no execute function"); + } + + return tool.execute(input); +} + +describe("relationship graph tools", () => { + beforeEach(() => { + selectResults.length = 0; + }); + + test("get_path_between_entities does not traverse directed relationships backwards", async () => { + selectResults.push([{ id: "relationship-ba", sourceId: "entity-b", targetId: "entity-a", directed: true }]); + + const output = await executeTool(getPathBetweenTool("graph-1"), { + sourceEntityId: "entity-a", + targetEntityId: "entity-b", + }); + + expect(output).toContain("none found"); + }); + + test("get_path_between_entities returns no path for missing same-entity requests", async () => { + selectResults.push([]); + + const output = await executeTool(getPathBetweenTool("graph-1"), { + sourceEntityId: "missing-entity", + targetEntityId: "missing-entity", + }); + + expect(output).toBe("## Path\n- none found within 5 hops"); + expect(output).not.toContain("- missing-entity,"); + }); + + test("get_path_between_entities returns zero-hop path for existing same-entity requests", async () => { + selectResults.push([{ id: "entity-a", name: "A", type: "CODE_FUNCTION" }]); + + const output = await executeTool(getPathBetweenTool("graph-1"), { + sourceEntityId: "entity-a", + targetEntityId: "entity-a", + }); + + expect(output).toBe("## Path\n- entity-a, A, CODE_FUNCTION"); + }); + + test("get_path_between_entities traverses directed relationships forward", async () => { + selectResults.push( + [{ id: "relationship-ab", sourceId: "entity-a", targetId: "entity-b", directed: true }], + [{ id: "relationship-bc", sourceId: "entity-b", targetId: "entity-c", directed: true }], + [ + { id: "entity-a", name: "A", type: "CODE_FUNCTION" }, + { id: "entity-b", name: "B", type: "CODE_FUNCTION" }, + { id: "entity-c", name: "C", type: "CODE_FUNCTION" }, + ], + [ + { + id: "relationship-ab", + sourceId: "entity-a", + targetId: "entity-b", + kind: "CALLS", + directed: true, + description: "A calls B.", + }, + { + id: "relationship-bc", + sourceId: "entity-b", + targetId: "entity-c", + kind: "CALLS", + directed: true, + description: "B calls C.", + }, + ] + ); + + const output = await executeTool(getPathBetweenTool("graph-1"), { + sourceEntityId: "entity-a", + targetEntityId: "entity-c", + }); + + expect(output).toContain("relationship-ab, entity-a -> entity-b, CALLS"); + expect(output).toContain("relationship-bc, entity-b -> entity-c, CALLS"); + }); + + test("get_path_between_entities still traverses undirected relationships both ways", async () => { + selectResults.push( + [{ id: "relationship-ba", sourceId: "entity-b", targetId: "entity-a", directed: false }], + [ + { id: "entity-a", name: "A", type: "CODE_FUNCTION" }, + { id: "entity-b", name: "B", type: "CODE_FUNCTION" }, + ], + [ + { + id: "relationship-ba", + sourceId: "entity-b", + targetId: "entity-a", + kind: "RELATED", + directed: false, + description: "A and B are related.", + }, + ] + ); + + const output = await executeTool(getPathBetweenTool("graph-1"), { + sourceEntityId: "entity-a", + targetEntityId: "entity-b", + }); + + expect(output).toContain("relationship-ba, entity-b -- entity-a, RELATED"); + }); + + test("get_entity_neighbours reports relationship kind and direction", async () => { + selectResults.push( + [ + { + id: "relationship-ab", + sourceId: "entity-a", + targetId: "entity-b", + kind: "CALLS", + directed: true, + description: "A calls B.", + rank: 0.8, + }, + ], + [{ id: "entity-b", name: "B", type: "CODE_FUNCTION", description: "Function B." }] + ); + + const output = await executeTool(getNeighboursTool("graph-1"), { + entityId: "entity-a", + limit: 10, + }); + + expect(output).toContain("relationship-ab, CALLS, outgoing, entity-a -> entity-b"); + }); +}); diff --git a/packages/ai/src/tools/correction.ts b/packages/ai/src/tools/correction.ts index 57deaaf9..70d4f66d 100644 --- a/packages/ai/src/tools/correction.ts +++ b/packages/ai/src/tools/correction.ts @@ -1,6 +1,7 @@ import { db } from "@kiwi/db"; import { graphSuggestionsTable } from "@kiwi/db/tables/suggestions"; import { entityTable, filesTable, sourcesTable, textUnitTable } from "@kiwi/db/tables/graph"; +import { currentSourcePredicate, visibleFilePredicate } from "@kiwi/db/source-validity"; import { and, eq } from "drizzle-orm"; import { tool } from "ai"; import { z } from "zod"; @@ -44,7 +45,14 @@ async function assertSourceInGraph(graphId: string, sourceId: string) { .from(sourcesTable) .innerJoin(textUnitTable, eq(textUnitTable.id, sourcesTable.textUnitId)) .innerJoin(filesTable, eq(filesTable.id, textUnitTable.fileId)) - .where(and(eq(sourcesTable.id, sourceId), eq(filesTable.graphId, graphId))) + .where( + and( + eq(sourcesTable.id, sourceId), + eq(filesTable.graphId, graphId), + currentSourcePredicate(sourcesTable), + visibleFilePredicate(filesTable) + ) + ) .limit(1); if (!source) { diff --git a/packages/ai/src/tools/entity.ts b/packages/ai/src/tools/entity.ts index af968034..d0089330 100644 --- a/packages/ai/src/tools/entity.ts +++ b/packages/ai/src/tools/entity.ts @@ -1,6 +1,7 @@ import { db } from "@kiwi/db"; import type { EmbeddingModelV3 } from "@ai-sdk/provider"; -import { entityTable, sourcesTable, textUnitTable } from "@kiwi/db/tables/graph"; +import { entityTable, filesTable, sourcesTable, textUnitTable } from "@kiwi/db/tables/graph"; +import { currentSourcePredicate, currentSourceSql, visibleFilePredicate, visibleFileSql } from "@kiwi/db/source-validity"; import { and, asc, cosineDistance, eq, exists, gt, inArray, sql } from "drizzle-orm"; import { embed, tool } from "ai"; import { withAiSlot } from "../concurrency"; @@ -66,7 +67,10 @@ function buildFileScopeExpression(fileIds: string[]) { select 1 from sources source inner join text_units text_unit on text_unit.id = source.text_unit_id + inner join files file on file.id = text_unit.file_id where source.entity_id = e.id + and ${currentSourceSql("source")} + and ${visibleFileSql("file")} and text_unit.file_id in (${sql.join( fileIds.map((fileId) => sql`${fileId}`), sql`, ` @@ -238,10 +242,13 @@ export const listEntitiesTool = (graphId: string, options: EntityToolOptions = { .select({ id: sourcesTable.id }) .from(sourcesTable) .innerJoin(textUnitTable, eq(textUnitTable.id, sourcesTable.textUnitId)) + .innerJoin(filesTable, eq(filesTable.id, textUnitTable.fileId)) .where( and( eq(sourcesTable.entityId, entityTable.id), - inArray(textUnitTable.fileId, fileIds) + currentSourcePredicate(sourcesTable), + inArray(textUnitTable.fileId, fileIds), + visibleFilePredicate(filesTable) ) ) ) diff --git a/packages/ai/src/tools/relationship.ts b/packages/ai/src/tools/relationship.ts index 23e149bc..d191f639 100644 --- a/packages/ai/src/tools/relationship.ts +++ b/packages/ai/src/tools/relationship.ts @@ -1,6 +1,7 @@ import { db } from "@kiwi/db"; import type { EmbeddingModelV3 } from "@ai-sdk/provider"; import { entityTable, relationshipTable } from "@kiwi/db/tables/graph"; +import { currentSourceSql, visibleFileSql } from "@kiwi/db/source-validity"; import { and, asc, cosineDistance, eq, gt, inArray, or, sql } from "drizzle-orm"; import { embed, tool } from "ai"; import { withAiSlot } from "../concurrency"; @@ -27,11 +28,23 @@ type SearchRelationshipRow = { targetId: string; sourceName: string; targetName: string; + kind: string; + directed: boolean; description: string; rank: number; score: number; }; +type RelationshipEdgeRow = { + id: string; + sourceId: string; + targetId: string; + description: string; + kind: string; + directed: boolean; + rank: number; +}; + function buildRelationshipKeywordBoostExpression(terms: string[]) { return greatest( terms.flatMap((term) => [ @@ -67,7 +80,10 @@ function buildRelationshipFileScopeExpression(fileIds: string[]) { select 1 from sources source inner join text_units text_unit on text_unit.id = source.text_unit_id + inner join files file on file.id = text_unit.file_id where source.relationship_id = r.id + and ${currentSourceSql("source")} + and ${visibleFileSql("file")} and text_unit.file_id in (${sql.join( fileIds.map((fileId) => sql`${fileId}`), sql`, ` @@ -83,23 +99,29 @@ function toSearchRelationshipRows(rows: Record[]): SearchRelati targetId: String(row.targetId ?? ""), sourceName: String(row.sourceName ?? "Unknown"), targetName: String(row.targetName ?? "Unknown"), + kind: String(row.kind ?? "RELATED"), + directed: row.directed === true, description: String(row.description ?? ""), rank: Number(row.rank ?? 0), score: Number(row.score ?? 0), })); } +function formatRelationshipDirection(row: Pick) { + return row.directed ? `${row.sourceId} -> ${row.targetId}` : `${row.sourceId} -- ${row.targetId}`; +} + function formatRelationshipList( rows: Array< Pick< SearchRelationshipRow, - "id" | "sourceId" | "targetId" | "sourceName" | "targetName" | "description" | "rank" + "id" | "sourceId" | "targetId" | "sourceName" | "targetName" | "kind" | "directed" | "description" | "rank" > > ) { return rows.map( (row) => - `- ${row.id}, ${row.sourceId} ${row.sourceName} -> ${row.targetId} ${row.targetName}, ${truncateWords(row.description) || "No description"}, rank ${row.rank}` + `- ${row.id}, ${formatRelationshipDirection(row)}, ${row.sourceName} to ${row.targetName}, ${row.kind}, ${truncateWords(row.description) || "No description"}, rank ${row.rank}` ); } @@ -178,6 +200,8 @@ export const searchRelationshipsTool = ( r.target_id as "targetId", coalesce(source_entity.name, 'Unknown') as "sourceName", coalesce(target_entity.name, 'Unknown') as "targetName", + r.kind, + r.directed, r.description, r.rank, ${semanticScore} as semantic_score, @@ -197,6 +221,8 @@ export const searchRelationshipsTool = ( ranked."targetId", ranked."sourceName", ranked."targetName", + ranked.kind, + ranked.directed, ranked.description, ranked.rank, ranked.score @@ -278,6 +304,8 @@ export const getRelationshipsTool = (graphId: string) => id: relationshipTable.id, sourceId: relationshipTable.sourceId, targetId: relationshipTable.targetId, + kind: relationshipTable.kind, + directed: relationshipTable.directed, description: relationshipTable.description, rank: relationshipTable.rank, }) @@ -309,7 +337,7 @@ export const getRelationshipsTool = (graphId: string) => const source = entityMap.get(row.sourceId); const target = entityMap.get(row.targetId); - return `- ${row.id}, ${row.sourceId} ${source?.name ?? "Unknown"} -> ${row.targetId} ${target?.name ?? "Unknown"}, ${description || "No description"}, rank ${row.rank}`; + return `- ${row.id}, ${formatRelationshipDirection(row)}, ${source?.name ?? "Unknown"} to ${target?.name ?? "Unknown"}, ${row.kind}, ${description || "No description"}, rank ${row.rank}`; }) : ["- none"]), ...(hasMore && items.length > 0 ? [``, `Next cursor: ${items[items.length - 1]?.id}`] : []), @@ -353,6 +381,8 @@ export const getNeighboursTool = (graphId: string) => id: relationshipTable.id, sourceId: relationshipTable.sourceId, targetId: relationshipTable.targetId, + kind: relationshipTable.kind, + directed: relationshipTable.directed, description: relationshipTable.description, rank: relationshipTable.rank, }) @@ -387,8 +417,9 @@ export const getNeighboursTool = (graphId: string) => const neighbour = entityMap.get(neighbourId); const relationship = truncateWords(row.description, 30); const description = truncateWords(neighbour?.description ?? "", 20); + const direction = row.sourceId === entityId ? "outgoing" : "incoming"; - return `- ${neighbourId}, ${neighbour?.name ?? "Unknown"}, ${neighbour?.type ?? "Unknown"}, ${description || "No description"}; via ${row.id}, ${relationship || "No relationship description"}, rank ${row.rank}`; + return `- ${neighbourId}, ${neighbour?.name ?? "Unknown"}, ${neighbour?.type ?? "Unknown"}, ${description || "No description"}; via ${row.id}, ${row.kind}, ${row.directed ? direction : "undirected"}, ${formatRelationshipDirection(row)}, ${relationship || "No relationship description"}, rank ${row.rank}`; }) : ["- none"]), ...(hasMore && items.length > 0 ? [``, `Next cursor: ${items[items.length - 1]?.id}`] : []), @@ -430,10 +461,11 @@ export const getPathBetweenTool = (graphId: string) => .where(and(eq(entityTable.graphId, graphId), eq(entityTable.id, sourceEntityId))) .limit(1); - return [ - "## Path", - `- ${entity?.id ?? sourceEntityId}, ${entity?.name ?? "Unknown"}, ${entity?.type ?? "Unknown"}`, - ].join("\n"); + if (!entity) { + return "## Path\n- none found within 5 hops"; + } + + return ["## Path", `- ${entity.id}, ${entity.name}, ${entity.type}`].join("\n"); } const maxDepth = 5; @@ -451,6 +483,7 @@ export const getPathBetweenTool = (graphId: string) => id: relationshipTable.id, sourceId: relationshipTable.sourceId, targetId: relationshipTable.targetId, + directed: relationshipTable.directed, }) .from(relationshipTable) .where( @@ -476,7 +509,11 @@ export const getPathBetweenTool = (graphId: string) => nextFrontier.push(relationship.targetId); } - if (frontier.includes(relationship.targetId) && !visited.has(relationship.sourceId)) { + if ( + !relationship.directed && + frontier.includes(relationship.targetId) && + !visited.has(relationship.sourceId) + ) { visited.add(relationship.sourceId); previous.set(relationship.sourceId, { entityId: relationship.targetId, @@ -521,6 +558,10 @@ export const getPathBetweenTool = (graphId: string) => ? await db .select({ id: relationshipTable.id, + sourceId: relationshipTable.sourceId, + targetId: relationshipTable.targetId, + kind: relationshipTable.kind, + directed: relationshipTable.directed, description: relationshipTable.description, }) .from(relationshipTable) @@ -538,7 +579,7 @@ export const getPathBetweenTool = (graphId: string) => const relationship = relationshipMap.get(pathRelationshipIds[index]!); const description = truncateWords(relationship?.description ?? "", 30); lines.push( - `- ${pathRelationshipIds[index]}, ${description || "No relationship description"}` + `- ${pathRelationshipIds[index]}, ${relationship ? formatRelationshipDirection(relationship) : "Unknown direction"}, ${relationship?.kind ?? "RELATED"}, ${description || "No relationship description"}` ); } } diff --git a/packages/ai/src/tools/source.ts b/packages/ai/src/tools/source.ts index ad9ce424..1f40bca7 100644 --- a/packages/ai/src/tools/source.ts +++ b/packages/ai/src/tools/source.ts @@ -1,6 +1,7 @@ import { db } from "@kiwi/db"; import type { EmbeddingModelV3 } from "@ai-sdk/provider"; import { filesTable, sourcesTable, textUnitTable } from "@kiwi/db/tables/graph"; +import { currentSourcePredicate, currentSourceSql, visibleFilePredicate, visibleFileSql } from "@kiwi/db/source-validity"; import { and, asc, cosineDistance, eq, gt, inArray, or, sql, type SQL } from "drizzle-orm"; import { embed, tool } from "ai"; import { withAiSlot } from "../concurrency"; @@ -203,8 +204,7 @@ async function getScopedSources( onConsideredFileIds?.(fileIds); if (terms.length === 0) { - const clauses = [eq(sourcesTable.active, true), eq(filesTable.graphId, graphId)]; - + const clauses = [currentSourcePredicate(sourcesTable), eq(filesTable.graphId, graphId), visibleFilePredicate(filesTable)]; if (cursor) { clauses.push(gt(sourcesTable.id, cursor)); } @@ -306,7 +306,8 @@ async function getScopedSources( from sources source inner join text_units text_unit on text_unit.id = source.text_unit_id inner join files file on file.id = text_unit.file_id - where source.active = true + where ${currentSourceSql("source")} + and ${visibleFileSql("file")} and file.graph_id = ${graphId} ${fileScope} ${subjectScope} @@ -459,8 +460,9 @@ export const getSourceFileMetadataTool = (graphId: string, options: SourceToolOp .innerJoin(filesTable, eq(filesTable.id, textUnitTable.fileId)) .where( and( - eq(sourcesTable.active, true), + currentSourcePredicate(sourcesTable), eq(filesTable.graphId, graphId), + visibleFilePredicate(filesTable), inArray(sourcesTable.id, ids) ) ); diff --git a/packages/connectors/package.json b/packages/connectors/package.json new file mode 100644 index 00000000..7a32d9b9 --- /dev/null +++ b/packages/connectors/package.json @@ -0,0 +1,24 @@ +{ + "name": "@kiwi/connectors", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts", + "./credentials": "./src/credentials.ts", + "./types": "./src/types.ts", + "./github": "./src/github.ts", + "./gitlab": "./src/gitlab.ts", + "./webhooks": "./src/webhooks.ts" + }, + "scripts": { + "build": "bunx tsc --noEmit", + "test": "bun test" + }, + "dependencies": {}, + "devDependencies": { + "@types/bun": "catalog:" + }, + "peerDependencies": { + "typescript": "catalog:" + } +} diff --git a/packages/connectors/src/__tests__/credentials.test.ts b/packages/connectors/src/__tests__/credentials.test.ts new file mode 100644 index 00000000..78538265 --- /dev/null +++ b/packages/connectors/src/__tests__/credentials.test.ts @@ -0,0 +1,63 @@ +import { createCipheriv, hkdfSync, randomBytes } from "node:crypto"; +import { describe, expect, test } from "bun:test"; +import { + decryptConnectorCredentials, + decryptConnectorSecret, + encryptConnectorCredentials, + encryptConnectorSecret, +} from "../credentials"; + +const AUTH_SECRET = "unit-test-auth-secret"; + +describe("connector credentials", () => { + test("round-trips connector credentials and webhook secrets", () => { + const encrypted = encryptConnectorCredentials( + { + provider: "github", + appId: "123", + privateKeyPem: "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----", + clientId: "client-id", + clientSecret: "client-secret", + webhookSecret: "webhook-secret", + }, + AUTH_SECRET + ); + + expect(decryptConnectorCredentials(encrypted, AUTH_SECRET)).toEqual({ + provider: "github", + appId: "123", + privateKeyPem: "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----", + clientId: "client-id", + clientSecret: "client-secret", + webhookSecret: "webhook-secret", + }); + + expect(decryptConnectorSecret(encryptConnectorSecret("hook", AUTH_SECRET), AUTH_SECRET)).toBe("hook"); + }); + + test("rejects invalid versions and wrong secrets", () => { + const encrypted = encryptConnectorCredentials({ provider: "gitlab", accessToken: "token" }, AUTH_SECRET); + + expect(() => decryptConnectorCredentials(encrypted.replace("v1:", "v0:"), AUTH_SECRET)).toThrow( + "Invalid connector credentials" + ); + expect(() => decryptConnectorCredentials(encrypted, "wrong-secret")).toThrow("Invalid connector credentials"); + }); + + test("rejects invalid decrypted shapes", () => { + const encrypted = encryptUnchecked({ provider: "github", appId: "123" }, AUTH_SECRET); + + expect(() => decryptConnectorCredentials(encrypted, AUTH_SECRET)).toThrow("Invalid connector credentials"); + expect(() => encryptConnectorCredentials({ provider: "gitlab", accessToken: "" } as never, AUTH_SECRET)).toThrow( + "Invalid connector credentials" + ); + }); +}); + +function encryptUnchecked(value: unknown, secret: string): string { + const iv = randomBytes(12); + const key = Buffer.from(hkdfSync("sha256", secret, "kiwi-connector-credentials:v1", "connector-credential-encryption", 32)); + const cipher = createCipheriv("aes-256-gcm", key, iv, { authTagLength: 16 }); + const ciphertext = Buffer.concat([cipher.update(Buffer.from(JSON.stringify(value), "utf8")), cipher.final()]); + return ["v1", iv.toString("base64url"), cipher.getAuthTag().toString("base64url"), ciphertext.toString("base64url")].join(":"); +} diff --git a/packages/connectors/src/__tests__/github.test.ts b/packages/connectors/src/__tests__/github.test.ts new file mode 100644 index 00000000..895deb56 --- /dev/null +++ b/packages/connectors/src/__tests__/github.test.ts @@ -0,0 +1,205 @@ +import { createHmac, generateKeyPairSync } from "node:crypto"; +import { describe, expect, test } from "bun:test"; +import { + createGitHubAppJwt, + createGitHubClient, + createGitHubInstallationToken, + loadGitHubRepositorySnapshot, + normalizeGitHubWebhookEvent, + verifyGitHubWebhookSignature, +} from "../github"; +import type { FetchLike, ProviderRepository } from "../types"; + +const GITHUB_REPOSITORY: ProviderRepository = { + provider: "github", + id: "1", + fullName: "acme/app", + name: "app", + htmlUrl: "https://github.com/acme/app", + defaultBranch: "main", + private: true, +}; + +describe("GitHub connector", () => { + test("creates app JWT and installation tokens", async () => { + const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); + const privateKeyPem = privateKey.export({ type: "pkcs8", format: "pem" }).toString(); + const jwt = createGitHubAppJwt({ appId: "42", privateKeyPem, now: new Date("2026-01-01T00:00:00Z") }); + expect(jwt.split(".")).toHaveLength(3); + + const calls: RequestInit[] = []; + const fetchImpl: FetchLike = async (_input, init) => { + calls.push(init ?? {}); + return jsonResponse({ token: "installation-token", expires_at: "2026-01-01T01:00:00Z" }); + }; + + await expect( + createGitHubInstallationToken({ + credentials: { provider: "github", appId: "42", privateKeyPem }, + installationId: "99", + apiBaseUrl: "https://github.test", + fetch: fetchImpl, + now: new Date("2026-01-01T00:00:00Z"), + }) + ).resolves.toEqual({ token: "installation-token", expiresAt: "2026-01-01T01:00:00Z" }); + expect(calls[0]?.method).toBe("POST"); + expect(String((calls[0]?.headers as Record).authorization)).toMatch(/^Bearer /); + expect(calls[0]?.body).toBe(JSON.stringify({ permissions: { contents: "read", metadata: "read" } })); + }); + + test("lists repositories and branches with pagination", async () => { + const urls: string[] = []; + const fetchImpl: FetchLike = async (input) => { + urls.push(String(input)); + if (String(input).includes("/installation/repositories")) { + return jsonResponse({ + repositories: [ + { + id: 1, + full_name: "acme/app", + name: "app", + html_url: "https://github.com/acme/app", + default_branch: "main", + private: true, + }, + ], + }); + } + return jsonResponse([{ name: "main", commit: { sha: "commit-sha" } }]); + }; + const client = createGitHubClient({ installationToken: "token", apiBaseUrl: "https://github.test", fetch: fetchImpl }); + + await expect(client.listRepositories()).resolves.toEqual([GITHUB_REPOSITORY]); + await expect(client.listBranches(GITHUB_REPOSITORY)).resolves.toEqual([{ name: "main", commitSha: "commit-sha" }]); + expect(urls).toContain("https://github.test/installation/repositories?per_page=100&page=1"); + expect(urls).toContain("https://github.test/repos/acme/app/branches?per_page=100&page=1"); + }); + + test("loads supported repository snapshots through tree and blob APIs", async () => { + const calls: string[] = []; + const fetchImpl: FetchLike = async (input) => { + const url = String(input); + calls.push(url); + if (url.endsWith("/branches/main")) { + return jsonResponse({ commit: { sha: "commit-sha", commit: { tree: { sha: "tree-sha" } } } }); + } + if (url.includes("/git/trees/tree-sha")) { + return jsonResponse({ + tree: [ + { type: "blob", path: "src/index.ts", sha: "blob-sha", size: 18 }, + { type: "blob", path: "dist/generated.ts", sha: "skip-sha", size: 10 }, + { type: "blob", path: "README.md", sha: "readme-sha", size: 10 }, + ], + }); + } + if (url.includes("/git/blobs/blob-sha")) { + return jsonResponse({ encoding: "base64", content: Buffer.from("export const ok = 1;", "utf8").toString("base64") }); + } + throw new Error(`unexpected URL ${url}`); + }; + + await expect( + loadGitHubRepositorySnapshot({ + installationToken: "token", + apiBaseUrl: "https://github.test", + fetch: fetchImpl, + repository: GITHUB_REPOSITORY, + branch: "main", + }) + ).resolves.toEqual({ + repository: GITHUB_REPOSITORY, + branch: { name: "main", commitSha: "commit-sha" }, + commitSha: "commit-sha", + files: [ + { + path: "src/index.ts", + size: 20, + checksum: "blob-sha", + htmlUrl: "https://github.com/acme/app/blob/commit-sha/src/index.ts", + rawUrl: "https://raw.githubusercontent.com/acme/app/commit-sha/src/index.ts", + content: "export const ok = 1;", + }, + ], + }); + expect(calls.some((url) => url.includes("skip-sha"))).toBe(false); + expect(calls.some((url) => url.includes("readme-sha"))).toBe(false); + }); + + test("normalizes incremental compare changes for supported code paths", async () => { + const client = createGitHubClient({ + installationToken: "token", + apiBaseUrl: "https://github.test", + fetch: async (input) => { + expect(String(input)).toBe("https://github.test/repos/acme/app/compare/commit-old...commit-new"); + return jsonResponse({ + status: "ahead", + files: [ + { filename: "src/modified.ts", status: "modified" }, + { filename: "src/added.ts", status: "added" }, + { filename: "src/deleted.ts", status: "removed" }, + { filename: "src/renamed.ts", status: "renamed", previous_filename: "src/old-name.ts" }, + { filename: "README.md", status: "renamed", previous_filename: "src/renamed-away.ts" }, + { filename: "README.md", status: "modified" }, + ], + }); + }, + }); + + await expect(client.compareRepository(GITHUB_REPOSITORY, "commit-old", "commit-new")).resolves.toEqual({ + fromCommitSha: "commit-old", + toCommitSha: "commit-new", + changes: [ + { status: "modified", newPath: "src/modified.ts" }, + { status: "added", newPath: "src/added.ts" }, + { status: "deleted", oldPath: "src/deleted.ts" }, + { status: "renamed", oldPath: "src/old-name.ts", newPath: "src/renamed.ts" }, + { status: "deleted", oldPath: "src/renamed-away.ts" }, + ], + }); + }); + + test("rejects GitHub compare responses that are not safe to apply incrementally", async () => { + const client = createGitHubClient({ + installationToken: "token", + apiBaseUrl: "https://github.test", + fetch: async () => jsonResponse({ status: "diverged", files: [] }), + }); + + await expect(client.compareRepository(GITHUB_REPOSITORY, "commit-old", "commit-new")).rejects.toThrow( + "GitHub compare response is invalid" + ); + }); + + test("verifies webhooks and normalizes push events", () => { + const body = JSON.stringify({ ref: "refs/heads/main" }); + const signature = `sha256=${createHmac("sha256", "secret").update(body).digest("hex")}`; + + expect(verifyGitHubWebhookSignature({ body, webhookSecret: "secret", signatureHeader: signature })).toBe(true); + expect(verifyGitHubWebhookSignature({ body, webhookSecret: "secret", signatureHeader: "sha256=bad" })).toBe(false); + expect( + normalizeGitHubWebhookEvent({ + eventName: "push", + deliveryId: "delivery", + payload: { + ref: "refs/heads/main", + after: "commit-sha", + repository: { id: 1, full_name: "acme/app" }, + installation: { id: 99 }, + }, + }) + ).toMatchObject({ + provider: "github", + deliveryId: "delivery", + eventName: "push", + repositoryId: "1", + repositoryFullName: "acme/app", + branch: "main", + commitSha: "commit-sha", + installationId: "99", + }); + }); +}); + +function jsonResponse(value: unknown, init?: ResponseInit): Response { + return new Response(JSON.stringify(value), { status: 200, ...init, headers: { "content-type": "application/json", ...init?.headers } }); +} diff --git a/packages/connectors/src/__tests__/gitlab.test.ts b/packages/connectors/src/__tests__/gitlab.test.ts new file mode 100644 index 00000000..cb30ecbe --- /dev/null +++ b/packages/connectors/src/__tests__/gitlab.test.ts @@ -0,0 +1,229 @@ +import { describe, expect, test } from "bun:test"; +import { + createGitLabClient, + loadGitLabRepositorySnapshot, + normalizeGitLabBaseUrl, + normalizeGitLabWebhookEvent, + verifyGitLabWebhookToken, +} from "../gitlab"; +import type { FetchLike, ProviderRepository } from "../types"; + +const GITLAB_REPOSITORY: ProviderRepository = { + provider: "gitlab", + id: "7", + fullName: "acme/app", + name: "app", + htmlUrl: "https://gitlab.test/acme/app", + defaultBranch: "main", + private: true, +}; + +describe("GitLab connector", () => { + test("normalizes base URLs", () => { + expect(normalizeGitLabBaseUrl("https://gitlab.example.com///")).toBe("https://gitlab.example.com"); + expect(normalizeGitLabBaseUrl("https://gitlab.example.com/root/")).toBe("https://gitlab.example.com/root"); + expect(() => normalizeGitLabBaseUrl("ftp://gitlab.example.com")).toThrow("GitLab base URL must use HTTP or HTTPS"); + }); + + test("lists projects and branches", async () => { + const urls: string[] = []; + const fetchImpl: FetchLike = async (input) => { + const url = String(input); + urls.push(url); + if (url.includes("/projects?")) { + return jsonResponse([ + { + id: 7, + path_with_namespace: "acme/app", + name: "app", + web_url: "https://gitlab.test/acme/app", + default_branch: "main", + visibility: "private", + }, + ]); + } + return jsonResponse([{ name: "main", commit: { id: "commit-sha" } }]); + }; + const client = createGitLabClient({ baseUrl: "https://gitlab.test", accessToken: "token", fetch: fetchImpl }); + + await expect(client.listRepositories()).resolves.toEqual([GITLAB_REPOSITORY]); + await expect(client.listBranches(GITLAB_REPOSITORY)).resolves.toEqual([{ name: "main", commitSha: "commit-sha" }]); + expect(urls).toContain("https://gitlab.test/api/v4/projects?membership=true&per_page=100&page=1"); + expect(urls).toContain("https://gitlab.test/api/v4/projects/7/repository/branches?per_page=100&page=1"); + }); + + test("loads supported repository snapshots through tree and file APIs", async () => { + const calls: string[] = []; + const fetchImpl: FetchLike = async (input) => { + const url = String(input); + calls.push(url); + if (url.endsWith("/repository/branches/main")) { + return jsonResponse({ commit: { id: "commit-sha" } }); + } + if (url.includes("/repository/tree?")) { + return jsonResponse([ + { type: "blob", id: "file-sha", path: "src/index.ts" }, + { type: "blob", id: "generated-sha", path: "node_modules/pkg/index.ts" }, + { type: "blob", id: "readme-sha", path: "README.md" }, + ]); + } + if (url.includes("/repository/files/src%2Findex.ts/raw?")) { + return new Response("export const ok = 1;", { status: 200, headers: { "content-length": "20" } }); + } + throw new Error(`unexpected URL ${url}`); + }; + + await expect( + loadGitLabRepositorySnapshot({ + baseUrl: "https://gitlab.test", + accessToken: "token", + fetch: fetchImpl, + repository: GITLAB_REPOSITORY, + branch: "main", + }) + ).resolves.toEqual({ + repository: GITLAB_REPOSITORY, + branch: { name: "main", commitSha: "commit-sha" }, + commitSha: "commit-sha", + files: [ + { + path: "src/index.ts", + size: 20, + checksum: "file-sha", + htmlUrl: "https://gitlab.test/acme/app/-/blob/commit-sha/src/index.ts", + rawUrl: "https://gitlab.test/api/v4/projects/7/repository/files/src%2Findex.ts/raw?ref=commit-sha", + content: "export const ok = 1;", + }, + ], + }); + expect(calls.some((url) => url.includes("generated-sha"))).toBe(false); + expect(calls.some((url) => url.includes("readme-sha"))).toBe(false); + }); + + test("normalizes incremental compare changes for supported code paths", async () => { + const client = createGitLabClient({ + baseUrl: "https://gitlab.test", + accessToken: "token", + fetch: async (input) => { + expect(String(input)).toBe( + "https://gitlab.test/api/v4/projects/7/repository/compare?from=commit-old&to=commit-new&straight=true" + ); + return jsonResponse({ + compare_timeout: false, + diffs: [ + { + old_path: "src/modified.ts", + new_path: "src/modified.ts", + new_file: false, + renamed_file: false, + deleted_file: false, + }, + { + old_path: "src/added.ts", + new_path: "src/added.ts", + new_file: true, + renamed_file: false, + deleted_file: false, + }, + { + old_path: "src/deleted.ts", + new_path: "src/deleted.ts", + new_file: false, + renamed_file: false, + deleted_file: true, + }, + { + old_path: "src/old-name.ts", + new_path: "src/renamed.ts", + new_file: false, + renamed_file: true, + deleted_file: false, + }, + { + old_path: "src/renamed-away.ts", + new_path: "README.md", + new_file: false, + renamed_file: true, + deleted_file: false, + }, + { + old_path: "README.md", + new_path: "README.md", + new_file: false, + renamed_file: false, + deleted_file: false, + }, + ], + }); + }, + }); + + await expect(client.compareRepository(GITLAB_REPOSITORY, "commit-old", "commit-new")).resolves.toEqual({ + fromCommitSha: "commit-old", + toCommitSha: "commit-new", + changes: [ + { status: "modified", newPath: "src/modified.ts" }, + { status: "added", newPath: "src/added.ts" }, + { status: "deleted", oldPath: "src/deleted.ts" }, + { status: "renamed", oldPath: "src/old-name.ts", newPath: "src/renamed.ts" }, + { status: "deleted", oldPath: "src/renamed-away.ts" }, + ], + }); + }); + + test("rejects GitLab compare timeouts", async () => { + const client = createGitLabClient({ + baseUrl: "https://gitlab.test", + accessToken: "token", + fetch: async () => jsonResponse({ compare_timeout: true, diffs: [] }), + }); + + await expect(client.compareRepository(GITLAB_REPOSITORY, "commit-old", "commit-new")).rejects.toThrow( + "GitLab compare response is invalid" + ); + }); + + test("rejects malformed GitLab compare responses", async () => { + const client = createGitLabClient({ + baseUrl: "https://gitlab.test", + accessToken: "token", + fetch: async () => + jsonResponse({ + compare_timeout: false, + diffs: [{ old_path: "src/index.ts", new_path: "src/index.ts" }], + }), + }); + + await expect(client.compareRepository(GITLAB_REPOSITORY, "commit-old", "commit-new")).rejects.toThrow( + "GitLab compare response is invalid" + ); + }); + + test("verifies webhook tokens and normalizes push events", () => { + expect(verifyGitLabWebhookToken({ webhookSecret: "secret", tokenHeader: "secret" })).toBe(true); + expect(verifyGitLabWebhookToken({ webhookSecret: "secret", tokenHeader: "wrong" })).toBe(false); + expect( + normalizeGitLabWebhookEvent({ + eventName: "push", + deliveryId: "delivery", + payload: { + ref: "refs/heads/main", + after: "commit-sha", + project: { id: 7, path_with_namespace: "acme/app" }, + }, + }) + ).toMatchObject({ + provider: "gitlab", + deliveryId: "delivery", + eventName: "push", + repositoryId: "7", + repositoryFullName: "acme/app", + branch: "main", + commitSha: "commit-sha", + }); + }); +}); + +function jsonResponse(value: unknown, init?: ResponseInit): Response { + return new Response(JSON.stringify(value), { status: 200, ...init, headers: { "content-type": "application/json", ...init?.headers } }); +} diff --git a/packages/connectors/src/code-files.ts b/packages/connectors/src/code-files.ts new file mode 100644 index 00000000..c5fa6471 --- /dev/null +++ b/packages/connectors/src/code-files.ts @@ -0,0 +1,20 @@ +const SUPPORTED_CODE_EXTENSIONS: Record = { + ".js": true, + ".jsx": true, + ".ts": true, + ".tsx": true, + ".mts": true, + ".cts": true, +}; + +export function isSupportedCodePath(filePath: string): boolean { + const normalized = filePath.replaceAll("\\", "/"); + const lastSlash = normalized.lastIndexOf("/"); + const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized; + const extensionStart = fileName.lastIndexOf("."); + if (extensionStart < 0) { + return false; + } + + return SUPPORTED_CODE_EXTENSIONS[fileName.slice(extensionStart).toLowerCase()] === true; +} diff --git a/packages/connectors/src/credentials.ts b/packages/connectors/src/credentials.ts new file mode 100644 index 00000000..87e447a1 --- /dev/null +++ b/packages/connectors/src/credentials.ts @@ -0,0 +1,130 @@ +import { createCipheriv, createDecipheriv, hkdfSync, randomBytes } from "node:crypto"; +import type { ConnectorCredentials, ConnectorInstallationCredentials } from "./types"; + +const ENCRYPTION_VERSION = "v1"; +const ENCRYPTION_ALGORITHM = "aes-256-gcm"; +const ENCRYPTION_KEY_SALT = "kiwi-connector-credentials:v1"; +const ENCRYPTION_KEY_INFO = "connector-credential-encryption"; +const IV_BYTE_LENGTH = 12; +const AUTH_TAG_BYTE_LENGTH = 16; + +export type ConnectorSecretPayload = ConnectorCredentials | ConnectorInstallationCredentials | { secret: string }; + +export function encryptConnectorCredentials(credentials: ConnectorSecretPayload, secret: string): string { + assertValidConnectorCredentials(credentials); + const iv = randomBytes(IV_BYTE_LENGTH); + const cipher = createCipheriv(ENCRYPTION_ALGORITHM, deriveEncryptionKey(secret), iv, { + authTagLength: AUTH_TAG_BYTE_LENGTH, + }); + const plaintext = Buffer.from(JSON.stringify(credentials), "utf8"); + const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]); + const authTag = cipher.getAuthTag(); + + return [ENCRYPTION_VERSION, encodeBase64Url(iv), encodeBase64Url(authTag), encodeBase64Url(ciphertext)].join(":"); +} + +export function decryptConnectorCredentials(value: string, secret: string): ConnectorSecretPayload { + const [version, rawIv, rawAuthTag, rawCiphertext, extra] = value.split(":"); + if (version !== ENCRYPTION_VERSION || !rawIv || !rawAuthTag || !rawCiphertext || extra !== undefined) { + throw new Error("Invalid connector credentials"); + } + + try { + const decipher = createDecipheriv(ENCRYPTION_ALGORITHM, deriveEncryptionKey(secret), decodeBase64Url(rawIv), { + authTagLength: AUTH_TAG_BYTE_LENGTH, + }); + decipher.setAuthTag(decodeBase64Url(rawAuthTag)); + const plaintext = Buffer.concat([decipher.update(decodeBase64Url(rawCiphertext)), decipher.final()]).toString( + "utf8" + ); + const parsed = JSON.parse(plaintext) as ConnectorSecretPayload; + assertValidConnectorCredentials(parsed); + return parsed; + } catch (error) { + throw new Error("Invalid connector credentials", { cause: error }); + } +} + +export function encryptConnectorSecret(secretValue: string, secret: string): string { + return encryptConnectorCredentials({ secret: secretValue }, secret); +} + +export function decryptConnectorSecret(value: string, secret: string): string { + const payload = decryptConnectorCredentials(value, secret); + if (!isSecretPayload(payload)) { + throw new Error("Invalid connector secret"); + } + return payload.secret; +} + +export function assertValidConnectorCredentials(value: unknown): asserts value is ConnectorSecretPayload { + if (!isObject(value)) { + throw new Error("Invalid connector credentials"); + } + + if (isSecretPayload(value)) { + return; + } + + if (value.provider === "github") { + if (hasNonEmptyString(value, "appId") && hasNonEmptyString(value, "privateKeyPem")) { + assertOptionalString(value, "clientId"); + assertOptionalString(value, "clientSecret"); + assertOptionalString(value, "webhookSecret"); + return; + } + if (hasNonEmptyString(value, "installationId")) { + return; + } + } + + if (value.provider === "gitlab") { + if ( + hasNonEmptyString(value, "baseUrl") && + hasNonEmptyString(value, "clientId") && + hasNonEmptyString(value, "clientSecret") + ) { + assertOptionalString(value, "webhookSecret"); + return; + } + if (hasNonEmptyString(value, "accessToken")) { + assertOptionalString(value, "refreshToken"); + assertOptionalString(value, "expiresAt"); + return; + } + } + + throw new Error("Invalid connector credentials"); +} + +function deriveEncryptionKey(secret: string): Buffer { + return Buffer.from(hkdfSync("sha256", secret, ENCRYPTION_KEY_SALT, ENCRYPTION_KEY_INFO, 32)); +} + +function encodeBase64Url(value: Buffer): string { + return value.toString("base64url"); +} + +function decodeBase64Url(value: string): Buffer { + return Buffer.from(value, "base64url"); +} + +function isSecretPayload(value: object): value is { secret: string } { + return hasNonEmptyString(value, "secret"); +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function hasNonEmptyString(value: object, key: string): boolean { + const record = value as Record; + return typeof record[key] === "string" && record[key].trim().length > 0; +} + +function assertOptionalString(value: object, key: string) { + const record = value as Record; + if (record[key] !== undefined && typeof record[key] !== "string") { + throw new Error("Invalid connector credentials"); + } +} diff --git a/packages/connectors/src/github.ts b/packages/connectors/src/github.ts new file mode 100644 index 00000000..adba8735 --- /dev/null +++ b/packages/connectors/src/github.ts @@ -0,0 +1,472 @@ +import { createSign } from "node:crypto"; +import { isSupportedCodePath } from "./code-files"; +import type { + FetchLike, + GitHubConnectorCredentials, + NormalizedWebhookEvent, + ProviderBranch, + ProviderCodeFile, + ProviderRepository, + ProviderRepositoryClient, + ProviderRepositoryDelta, + ProviderRepositorySnapshot, +} from "./types"; +import { ConnectorProviderError } from "./types"; +import { branchNameFromGitRef, verifyHmacSha256Signature } from "./webhooks"; + +export const GITHUB_API_BASE_URL = "https://api.github.com"; +export const MAX_REPOSITORY_CODE_FILES = 1_000; +export const MAX_REPOSITORY_CODE_BYTES = 100 * 1024 * 1024; +export const MAX_REPOSITORY_CODE_FILE_BYTES = 2 * 1024 * 1024; + +const GITHUB_JWT_TTL_SECONDS = 9 * 60; +const SKIPPED_PATH_SEGMENTS: Record = { + ".git": true, + ".next": true, + ".turbo": true, + build: true, + coverage: true, + dist: true, + node_modules: true, + out: true, + vendor: true, +}; + +type GitHubClientOptions = { + installationToken: string; + apiBaseUrl?: string; + fetch?: FetchLike; +}; + +type InstallationTokenOptions = { + credentials: GitHubConnectorCredentials; + installationId: string; + apiBaseUrl?: string; + fetch?: FetchLike; + now?: Date; +}; + +type SnapshotOptions = GitHubClientOptions & { + repository: ProviderRepository; + branch: string; + commitSha?: string; +}; + +export function createGitHubAppJwt(options: { + appId: string; + privateKeyPem: string; + now?: Date; + expiresInSeconds?: number; +}): string { + const nowSeconds = Math.floor((options.now?.getTime() ?? Date.now()) / 1_000); + const header = encodeJsonBase64Url({ alg: "RS256", typ: "JWT" }); + const payload = encodeJsonBase64Url({ + iat: nowSeconds - 60, + exp: nowSeconds + (options.expiresInSeconds ?? GITHUB_JWT_TTL_SECONDS), + iss: options.appId, + }); + const signingInput = `${header}.${payload}`; + const signature = createSign("RSA-SHA256").update(signingInput).sign(options.privateKeyPem, "base64url"); + return `${signingInput}.${signature}`; +} + +export async function createGitHubInstallationToken(options: InstallationTokenOptions): Promise<{ + token: string; + expiresAt: string; +}> { + const apiBaseUrl = normalizeApiBaseUrl(options.apiBaseUrl ?? GITHUB_API_BASE_URL); + const response = await (options.fetch ?? fetch)( + `${apiBaseUrl}/app/installations/${encodeURIComponent(options.installationId)}/access_tokens`, + { + method: "POST", + headers: githubHeaders( + createGitHubAppJwt({ + appId: options.credentials.appId, + privateKeyPem: options.credentials.privateKeyPem, + now: options.now, + }) + ), + body: JSON.stringify({ permissions: { contents: "read", metadata: "read" } }), + } + ); + const json = await readJson(response); + if (!response.ok || !isObject(json) || typeof json.token !== "string" || typeof json.expires_at !== "string") { + throw new ConnectorProviderError("auth", "GitHub installation token request failed"); + } + return { token: json.token, expiresAt: json.expires_at }; +} + +export function createGitHubClient(options: GitHubClientOptions): ProviderRepositoryClient { + return { + provider: "github", + async listRepositories() { + return listGitHubInstallationRepositories(options); + }, + async listBranches(repository) { + return listGitHubBranches({ ...options, repository }); + }, + async loadRepositorySnapshot(repository, branch, commitSha) { + return loadGitHubRepositorySnapshot({ ...options, repository, branch, commitSha }); + }, + async compareRepository(repository, fromCommitSha, toCommitSha) { + return compareGitHubRepository({ ...options, repository, fromCommitSha, toCommitSha }); + }, + async readFile(repository, path, commitSha) { + return readGitHubRepositoryFile({ ...options, repository, path, commitSha }); + }, + }; +} + +export async function listGitHubInstallationRepositories(options: GitHubClientOptions): Promise { + const repositories: ProviderRepository[] = []; + for (let page = 1; ; page += 1) { + const apiBaseUrl = normalizeApiBaseUrl(options.apiBaseUrl ?? GITHUB_API_BASE_URL); + const json = await getGitHubJson( + `${apiBaseUrl}/installation/repositories?per_page=100&page=${page}`, + options.installationToken, + options.fetch + ); + if (!isObject(json) || !Array.isArray(json.repositories)) { + throw new ConnectorProviderError("provider", "GitHub repository response is invalid"); + } + for (const repo of json.repositories) { + repositories.push(mapGitHubRepository(repo)); + } + if (json.repositories.length < 100) { + return repositories; + } + } +} + +export async function listGitHubBranches(options: GitHubClientOptions & { repository: ProviderRepository }): Promise { + const [owner, repo] = splitFullName(options.repository.fullName); + const branches: ProviderBranch[] = []; + for (let page = 1; ; page += 1) { + const apiBaseUrl = normalizeApiBaseUrl(options.apiBaseUrl ?? GITHUB_API_BASE_URL); + const json = await getGitHubJson( + `${apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/branches?per_page=100&page=${page}`, + options.installationToken, + options.fetch + ); + if (!Array.isArray(json)) { + throw new ConnectorProviderError("provider", "GitHub branches response is invalid"); + } + for (const branch of json) { + if (isObject(branch) && typeof branch.name === "string" && isObject(branch.commit) && typeof branch.commit.sha === "string") { + branches.push({ name: branch.name, commitSha: branch.commit.sha }); + } + } + if (json.length < 100) { + return branches; + } + } +} + +export async function loadGitHubRepositorySnapshot(options: SnapshotOptions): Promise { + const [owner, repo] = splitFullName(options.repository.fullName); + const apiBaseUrl = normalizeApiBaseUrl(options.apiBaseUrl ?? GITHUB_API_BASE_URL); + let treeSha: string; + let branch: ProviderBranch; + if (options.commitSha) { + const commitJson = await getGitHubJson( + `${apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/git/commits/${encodeURIComponent(options.commitSha)}`, + options.installationToken, + options.fetch + ); + if (!isObject(commitJson) || !isObject(commitJson.tree) || typeof commitJson.tree.sha !== "string") { + throw new ConnectorProviderError("not-found", "GitHub commit was not found"); + } + treeSha = commitJson.tree.sha; + branch = { name: options.branch, commitSha: options.commitSha }; + } else { + const branchJson = await getGitHubJson( + `${apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/branches/${encodeURIComponent(options.branch)}`, + options.installationToken, + options.fetch + ); + if (!isObject(branchJson) || !isObject(branchJson.commit) || typeof branchJson.commit.sha !== "string") { + throw new ConnectorProviderError("not-found", "GitHub branch was not found"); + } + treeSha = + isObject(branchJson.commit.commit) && + isObject(branchJson.commit.commit.tree) && + typeof branchJson.commit.commit.tree.sha === "string" + ? branchJson.commit.commit.tree.sha + : branchJson.commit.sha; + branch = { name: options.branch, commitSha: branchJson.commit.sha }; + } + const treeJson = await getGitHubJson( + `${apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/git/trees/${encodeURIComponent(treeSha)}?recursive=1`, + options.installationToken, + options.fetch + ); + if (!isObject(treeJson) || !Array.isArray(treeJson.tree)) { + throw new ConnectorProviderError("provider", "GitHub tree response is invalid"); + } + + const files: ProviderCodeFile[] = []; + let totalBytes = 0; + for (const item of treeJson.tree) { + if (!isGitHubBlob(item) || !shouldLoadCodePath(item.path) || item.size > MAX_REPOSITORY_CODE_FILE_BYTES) { + continue; + } + if (files.length + 1 > MAX_REPOSITORY_CODE_FILES) { + throw new ConnectorProviderError("limit", "Repository contains too many supported code files"); + } + if (totalBytes + item.size > MAX_REPOSITORY_CODE_BYTES) { + throw new ConnectorProviderError("limit", "Repository contains too much supported code"); + } + const blobJson = await getGitHubJson( + `${apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/git/blobs/${encodeURIComponent(item.sha)}`, + options.installationToken, + options.fetch + ); + if (!isObject(blobJson) || typeof blobJson.content !== "string" || blobJson.encoding !== "base64") { + throw new ConnectorProviderError("provider", "GitHub blob response is invalid"); + } + const content = Buffer.from(blobJson.content.replaceAll("\n", ""), "base64").toString("utf8"); + const size = Buffer.byteLength(content, "utf8"); + if (size > MAX_REPOSITORY_CODE_FILE_BYTES || totalBytes + size > MAX_REPOSITORY_CODE_BYTES) { + throw new ConnectorProviderError("limit", "Repository contains too much supported code"); + } + files.push({ + path: item.path, + size, + checksum: item.sha, + htmlUrl: `https://github.com/${options.repository.fullName}/blob/${branch.commitSha}/${item.path}`, + rawUrl: `https://raw.githubusercontent.com/${options.repository.fullName}/${branch.commitSha}/${item.path}`, + content, + }); + totalBytes += size; + } + + return { repository: options.repository, branch, commitSha: branch.commitSha, files }; +} +export async function compareGitHubRepository( + options: GitHubClientOptions & { + repository: ProviderRepository; + fromCommitSha: string; + toCommitSha: string; + } +): Promise { + const [owner, repo] = splitFullName(options.repository.fullName); + const apiBaseUrl = normalizeApiBaseUrl(options.apiBaseUrl ?? GITHUB_API_BASE_URL); + const json = await getGitHubJson( + `${apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/compare/${encodeURIComponent(options.fromCommitSha)}...${encodeURIComponent(options.toCommitSha)}`, + options.installationToken, + options.fetch + ); + if ( + !isObject(json) || + (json.status !== "ahead" && json.status !== "identical") || + !Array.isArray(json.files) + ) { + throw new ConnectorProviderError("provider", "GitHub compare response is invalid"); + } + + const changes: ProviderRepositoryDelta["changes"] = []; + for (const entry of json.files) { + if (!isGitHubCompareFile(entry)) { + throw new ConnectorProviderError("provider", "GitHub compare response is invalid"); + } + + switch (entry.status) { + case "added": + case "copied": + if (shouldLoadCodePath(entry.filename)) { + changes.push({ status: "added", newPath: entry.filename }); + } + break; + case "modified": + case "changed": + if (shouldLoadCodePath(entry.filename)) { + changes.push({ status: "modified", newPath: entry.filename }); + } + break; + case "removed": + if (shouldLoadCodePath(entry.filename)) { + changes.push({ status: "deleted", oldPath: entry.filename }); + } + break; + case "renamed": { + const oldSupported = shouldLoadCodePath(entry.previous_filename); + const newSupported = shouldLoadCodePath(entry.filename); + if (oldSupported && newSupported) { + changes.push({ status: "renamed", oldPath: entry.previous_filename, newPath: entry.filename }); + } else if (oldSupported) { + changes.push({ status: "deleted", oldPath: entry.previous_filename }); + } else if (newSupported) { + changes.push({ status: "added", newPath: entry.filename }); + } + break; + } + case "unchanged": + break; + default: + throw new ConnectorProviderError("provider", "GitHub compare response is invalid"); + } + } + + return { + fromCommitSha: options.fromCommitSha, + toCommitSha: options.toCommitSha, + changes, + }; +} + +export async function readGitHubRepositoryFile(options: GitHubClientOptions & { + repository: ProviderRepository; + path: string; + commitSha: string; +}): Promise { + const [owner, repo] = splitFullName(options.repository.fullName); + const apiBaseUrl = normalizeApiBaseUrl(options.apiBaseUrl ?? GITHUB_API_BASE_URL); + const response = await (options.fetch ?? fetch)( + `${apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/contents/${encodePathSegments(options.path)}?ref=${encodeURIComponent(options.commitSha)}`, + { + headers: { + ...githubHeaders(options.installationToken), + Accept: "application/vnd.github.raw+json", + }, + } + ); + + if (!response.ok) { + throw new ConnectorProviderError("not-found", "GitHub file was not found"); + } + + const contentLength = response.headers.get("content-length"); + if (contentLength && Number(contentLength) > MAX_REPOSITORY_CODE_FILE_BYTES) { + throw new ConnectorProviderError("limit", "Repository file is too large"); + } + + const content = await response.text(); + if (Buffer.byteLength(content, "utf8") > MAX_REPOSITORY_CODE_FILE_BYTES) { + throw new ConnectorProviderError("limit", "Repository file is too large"); + } + + return content; +} + +export function verifyGitHubWebhookSignature(options: { + body: string | Buffer | Uint8Array; + webhookSecret: string; + signatureHeader: string | null | undefined; +}): boolean { + return verifyHmacSha256Signature({ + body: options.body, + secret: options.webhookSecret, + signatureHeader: options.signatureHeader, + prefix: "sha256=", + }); +} + +export function normalizeGitHubWebhookEvent(options: { + eventName: string; + deliveryId: string; + payload: unknown; +}): NormalizedWebhookEvent { + const payload = isObject(options.payload) ? options.payload : {}; + const repository = isObject(payload.repository) ? payload.repository : null; + const installation = isObject(payload.installation) ? payload.installation : null; + return { + provider: "github", + deliveryId: options.deliveryId, + eventName: options.eventName, + repositoryId: repository && (typeof repository.id === "string" || typeof repository.id === "number") ? String(repository.id) : null, + repositoryFullName: repository && typeof repository.full_name === "string" ? repository.full_name : null, + branch: branchNameFromGitRef(payload.ref), + commitSha: typeof payload.after === "string" ? payload.after : null, + installationId: installation && (typeof installation.id === "string" || typeof installation.id === "number") ? String(installation.id) : undefined, + raw: options.payload, + }; +} + +async function getGitHubJson(url: string, token: string, fetchImpl: FetchLike | undefined): Promise { + const response = await (fetchImpl ?? fetch)(url, { headers: githubHeaders(token) }); + const json = await readJson(response); + if (!response.ok) { + throw new ConnectorProviderError(response.status === 404 ? "not-found" : "provider", "GitHub API request failed"); + } + return json; +} + +function githubHeaders(token: string): Record { + return { + accept: "application/vnd.github+json", + authorization: `Bearer ${token}`, + "content-type": "application/json", + "x-github-api-version": "2022-11-28", + }; +} + +function mapGitHubRepository(value: unknown): ProviderRepository { + if (!isObject(value) || typeof value.full_name !== "string" || typeof value.name !== "string" || (typeof value.id !== "string" && typeof value.id !== "number")) { + throw new ConnectorProviderError("provider", "GitHub repository response is invalid"); + } + return { + provider: "github", + id: String(value.id), + fullName: value.full_name, + name: value.name, + htmlUrl: typeof value.html_url === "string" ? value.html_url : `https://github.com/${value.full_name}`, + defaultBranch: typeof value.default_branch === "string" ? value.default_branch : null, + private: value.private === true, + }; +} + +function splitFullName(fullName: string): [string, string] { + const [owner, repo, ...extra] = fullName.split("/"); + if (!owner || !repo || extra.length > 0) { + throw new ConnectorProviderError("validation", "Repository full name is invalid"); + } + return [owner, repo]; +} + +function encodePathSegments(value: string): string { + return value.split("/").map(encodeURIComponent).join("/"); +} + +async function readJson(response: Response): Promise { + const text = await response.text(); + return text.length === 0 ? null : JSON.parse(text); +} + +function shouldLoadCodePath(filePath: string): boolean { + const normalized = filePath.replaceAll("\\", "/"); + return isSupportedCodePath(normalized) && normalized.split("/").every((segment) => SKIPPED_PATH_SEGMENTS[segment] !== true); +} + +function normalizeApiBaseUrl(value: string): string { + return value.replace(/\/+$/, ""); +} + +function encodeJsonBase64Url(value: unknown): string { + return Buffer.from(JSON.stringify(value), "utf8").toString("base64url"); +} + +function isGitHubBlob(value: unknown): value is { path: string; sha: string; size: number } { + return ( + isObject(value) && + value.type === "blob" && + typeof value.path === "string" && + typeof value.sha === "string" && + typeof value.size === "number" + ); +} +function isGitHubCompareFile(value: unknown): value is { + filename: string; + status: "added" | "changed" | "copied" | "modified" | "removed" | "renamed" | "unchanged"; + previous_filename: string; +} { + return ( + isObject(value) && + typeof value.filename === "string" && + typeof value.status === "string" && + (value.status !== "renamed" || typeof value.previous_filename === "string") + ); +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/connectors/src/gitlab.ts b/packages/connectors/src/gitlab.ts new file mode 100644 index 00000000..4f9c6e1a --- /dev/null +++ b/packages/connectors/src/gitlab.ts @@ -0,0 +1,348 @@ +import { isSupportedCodePath } from "./code-files"; +import type { + FetchLike, + NormalizedWebhookEvent, + ProviderBranch, + ProviderCodeFile, + ProviderRepository, + ProviderRepositoryClient, + ProviderRepositoryDelta, + ProviderRepositorySnapshot, +} from "./types"; +import { ConnectorProviderError } from "./types"; +import { branchNameFromGitRef, verifySharedSecretToken } from "./webhooks"; +import { MAX_REPOSITORY_CODE_BYTES, MAX_REPOSITORY_CODE_FILES, MAX_REPOSITORY_CODE_FILE_BYTES } from "./github"; + +const SKIPPED_PATH_SEGMENTS: Record = { + ".git": true, + ".next": true, + ".turbo": true, + build: true, + coverage: true, + dist: true, + node_modules: true, + out: true, + vendor: true, +}; + +type GitLabClientOptions = { + baseUrl: string; + accessToken: string; + fetch?: FetchLike; +}; + +type SnapshotOptions = GitLabClientOptions & { + repository: ProviderRepository; + branch: string; + commitSha?: string; +}; + +export function normalizeGitLabBaseUrl(value: string): string { + const parsed = new URL(value.trim()); + if (parsed.protocol !== "https:" && parsed.protocol !== "http:") { + throw new ConnectorProviderError("validation", "GitLab base URL must use HTTP or HTTPS"); + } + parsed.pathname = parsed.pathname.replace(/\/+$/, ""); + parsed.search = ""; + parsed.hash = ""; + return parsed.toString().replace(/\/+$/, ""); +} + +export function createGitLabClient(options: GitLabClientOptions): ProviderRepositoryClient { + return { + provider: "gitlab", + async listRepositories() { + return listGitLabProjects(options); + }, + async listBranches(repository) { + return listGitLabBranches({ ...options, repository }); + }, + async loadRepositorySnapshot(repository, branch, commitSha) { + return loadGitLabRepositorySnapshot({ ...options, repository, branch, commitSha }); + }, + async compareRepository(repository, fromCommitSha, toCommitSha) { + return compareGitLabRepository({ ...options, repository, fromCommitSha, toCommitSha }); + }, + async readFile(repository, path, commitSha) { + return readGitLabRepositoryFile({ ...options, repository, path, commitSha }); + }, + }; +} + +export async function listGitLabProjects(options: GitLabClientOptions): Promise { + const repositories: ProviderRepository[] = []; + for (let page = 1; ; page += 1) { + const json = await getGitLabJson( + `${gitLabApiBase(options.baseUrl)}/projects?membership=true&per_page=100&page=${page}`, + options.accessToken, + options.fetch + ); + if (!Array.isArray(json)) { + throw new ConnectorProviderError("provider", "GitLab projects response is invalid"); + } + for (const project of json) { + repositories.push(mapGitLabProject(project)); + } + if (json.length < 100) { + return repositories; + } + } +} + +export async function listGitLabBranches(options: GitLabClientOptions & { repository: ProviderRepository }): Promise { + const branches: ProviderBranch[] = []; + for (let page = 1; ; page += 1) { + const json = await getGitLabJson( + `${gitLabApiBase(options.baseUrl)}/projects/${encodeURIComponent(options.repository.id)}/repository/branches?per_page=100&page=${page}`, + options.accessToken, + options.fetch + ); + if (!Array.isArray(json)) { + throw new ConnectorProviderError("provider", "GitLab branches response is invalid"); + } + for (const branch of json) { + if (isObject(branch) && typeof branch.name === "string" && isObject(branch.commit) && typeof branch.commit.id === "string") { + branches.push({ name: branch.name, commitSha: branch.commit.id }); + } + } + if (json.length < 100) { + return branches; + } + } +} + +export async function loadGitLabRepositorySnapshot(options: SnapshotOptions): Promise { + let commitSha = options.commitSha; + if (!commitSha) { + const branchJson = await getGitLabJson( + `${gitLabApiBase(options.baseUrl)}/projects/${encodeURIComponent(options.repository.id)}/repository/branches/${encodeURIComponent(options.branch)}`, + options.accessToken, + options.fetch + ); + if (!isObject(branchJson) || !isObject(branchJson.commit) || typeof branchJson.commit.id !== "string") { + throw new ConnectorProviderError("not-found", "GitLab branch was not found"); + } + commitSha = branchJson.commit.id; + } + const branch: ProviderBranch = { name: options.branch, commitSha }; + const tree = await listGitLabTree(options, commitSha); + const files: ProviderCodeFile[] = []; + let totalBytes = 0; + + for (const item of tree) { + if (!isGitLabBlob(item) || !shouldLoadCodePath(item.path)) { + continue; + } + if (files.length + 1 > MAX_REPOSITORY_CODE_FILES) { + throw new ConnectorProviderError("limit", "Repository contains too many supported code files"); + } + const rawUrl = `${gitLabApiBase(options.baseUrl)}/projects/${encodeURIComponent(options.repository.id)}/repository/files/${encodeURIComponent(item.path)}/raw?ref=${encodeURIComponent(branch.commitSha)}`; + const content = await getGitLabText(rawUrl, options.accessToken, options.fetch); + const size = Buffer.byteLength(content, "utf8"); + if (size > MAX_REPOSITORY_CODE_FILE_BYTES) { + continue; + } + if (totalBytes + size > MAX_REPOSITORY_CODE_BYTES) { + throw new ConnectorProviderError("limit", "Repository contains too much supported code"); + } + files.push({ + path: item.path, + size, + checksum: item.id, + htmlUrl: `${options.repository.htmlUrl}/-/blob/${branch.commitSha}/${item.path}`, + rawUrl, + content, + }); + totalBytes += size; + } + + return { repository: options.repository, branch, commitSha: branch.commitSha, files }; +} +export async function compareGitLabRepository( + options: GitLabClientOptions & { + repository: ProviderRepository; + fromCommitSha: string; + toCommitSha: string; + } +): Promise { + const json = await getGitLabJson( + `${gitLabApiBase(options.baseUrl)}/projects/${encodeURIComponent(options.repository.id)}/repository/compare?from=${encodeURIComponent(options.fromCommitSha)}&to=${encodeURIComponent(options.toCommitSha)}&straight=true`, + options.accessToken, + options.fetch + ); + if (!isObject(json) || json.compare_timeout === true || !Array.isArray(json.diffs)) { + throw new ConnectorProviderError("provider", "GitLab compare response is invalid"); + } + + const changes: ProviderRepositoryDelta["changes"] = []; + for (const entry of json.diffs) { + if (!isGitLabCompareFile(entry)) { + throw new ConnectorProviderError("provider", "GitLab compare response is invalid"); + } + + if (entry.renamed_file) { + const oldSupported = shouldLoadCodePath(entry.old_path); + const newSupported = shouldLoadCodePath(entry.new_path); + if (oldSupported && newSupported) { + changes.push({ status: "renamed", oldPath: entry.old_path, newPath: entry.new_path }); + } else if (oldSupported) { + changes.push({ status: "deleted", oldPath: entry.old_path }); + } else if (newSupported) { + changes.push({ status: "added", newPath: entry.new_path }); + } + continue; + } + + if (entry.deleted_file) { + if (shouldLoadCodePath(entry.old_path)) { + changes.push({ status: "deleted", oldPath: entry.old_path }); + } + continue; + } + + if (entry.new_file) { + if (shouldLoadCodePath(entry.new_path)) { + changes.push({ status: "added", newPath: entry.new_path }); + } + continue; + } + + if (shouldLoadCodePath(entry.new_path)) { + changes.push({ status: "modified", newPath: entry.new_path }); + } + } + + return { + fromCommitSha: options.fromCommitSha, + toCommitSha: options.toCommitSha, + changes, + }; +} + +export async function readGitLabRepositoryFile(options: GitLabClientOptions & { + repository: ProviderRepository; + path: string; + commitSha: string; +}): Promise { + const rawUrl = `${gitLabApiBase(options.baseUrl)}/projects/${encodeURIComponent(options.repository.id)}/repository/files/${encodeURIComponent(options.path)}/raw?ref=${encodeURIComponent(options.commitSha)}`; + const content = await getGitLabText(rawUrl, options.accessToken, options.fetch); + if (Buffer.byteLength(content, "utf8") > MAX_REPOSITORY_CODE_FILE_BYTES) { + throw new ConnectorProviderError("limit", "Repository file is too large"); + } + return content; +} + +export function verifyGitLabWebhookToken(options: { + webhookSecret: string; + tokenHeader: string | null | undefined; +}): boolean { + return verifySharedSecretToken(options.tokenHeader, options.webhookSecret); +} + +export function normalizeGitLabWebhookEvent(options: { + eventName: string; + deliveryId: string; + payload: unknown; +}): NormalizedWebhookEvent { + const payload = isObject(options.payload) ? options.payload : {}; + const project = isObject(payload.project) ? payload.project : null; + return { + provider: "gitlab", + deliveryId: options.deliveryId, + eventName: options.eventName, + repositoryId: project && (typeof project.id === "string" || typeof project.id === "number") ? String(project.id) : null, + repositoryFullName: project && typeof project.path_with_namespace === "string" ? project.path_with_namespace : null, + branch: branchNameFromGitRef(payload.ref), + commitSha: typeof payload.after === "string" ? payload.after : null, + raw: options.payload, + }; +} + +async function listGitLabTree(options: SnapshotOptions, ref: string): Promise { + const entries: unknown[] = []; + for (let page = 1; ; page += 1) { + const json = await getGitLabJson( + `${gitLabApiBase(options.baseUrl)}/projects/${encodeURIComponent(options.repository.id)}/repository/tree?recursive=true&per_page=100&page=${page}&ref=${encodeURIComponent(ref)}`, + options.accessToken, + options.fetch + ); + if (!Array.isArray(json)) { + throw new ConnectorProviderError("provider", "GitLab tree response is invalid"); + } + entries.push(...json); + if (json.length < 100) { + return entries; + } + } +} + +async function getGitLabJson(url: string, token: string, fetchImpl: FetchLike | undefined): Promise { + const response = await (fetchImpl ?? fetch)(url, { headers: { authorization: `Bearer ${token}`, accept: "application/json" } }); + const text = await response.text(); + const json = text.length === 0 ? null : JSON.parse(text); + if (!response.ok) { + throw new ConnectorProviderError(response.status === 404 ? "not-found" : "provider", "GitLab API request failed"); + } + return json; +} + +async function getGitLabText(url: string, token: string, fetchImpl: FetchLike | undefined): Promise { + const response = await (fetchImpl ?? fetch)(url, { headers: { authorization: `Bearer ${token}` } }); + const contentLength = response.headers.get("content-length"); + if (contentLength && Number(contentLength) > MAX_REPOSITORY_CODE_FILE_BYTES) { + throw new ConnectorProviderError("limit", "Repository contains a code file that is too large"); + } + const text = await response.text(); + if (!response.ok) { + throw new ConnectorProviderError(response.status === 404 ? "not-found" : "provider", "GitLab raw file request failed"); + } + return text; +} + +function mapGitLabProject(value: unknown): ProviderRepository { + if (!isObject(value) || typeof value.path_with_namespace !== "string" || typeof value.name !== "string" || (typeof value.id !== "string" && typeof value.id !== "number")) { + throw new ConnectorProviderError("provider", "GitLab project response is invalid"); + } + return { + provider: "gitlab", + id: String(value.id), + fullName: value.path_with_namespace, + name: value.name, + htmlUrl: typeof value.web_url === "string" ? value.web_url : "", + defaultBranch: typeof value.default_branch === "string" ? value.default_branch : null, + private: value.visibility !== "public", + }; +} + +function gitLabApiBase(baseUrl: string): string { + return `${normalizeGitLabBaseUrl(baseUrl)}/api/v4`; +} + +function shouldLoadCodePath(filePath: string): boolean { + const normalized = filePath.replaceAll("\\", "/"); + return isSupportedCodePath(normalized) && normalized.split("/").every((segment) => SKIPPED_PATH_SEGMENTS[segment] !== true); +} + +function isGitLabBlob(value: unknown): value is { id: string; path: string } { + return isObject(value) && value.type === "blob" && typeof value.id === "string" && typeof value.path === "string"; +} +function isGitLabCompareFile(value: unknown): value is { + old_path: string; + new_path: string; + new_file: boolean; + renamed_file: boolean; + deleted_file: boolean; +} { + return ( + isObject(value) && + typeof value.old_path === "string" && + typeof value.new_path === "string" && + typeof value.new_file === "boolean" && + typeof value.renamed_file === "boolean" && + typeof value.deleted_file === "boolean" + ); +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/connectors/src/index.ts b/packages/connectors/src/index.ts new file mode 100644 index 00000000..ee348555 --- /dev/null +++ b/packages/connectors/src/index.ts @@ -0,0 +1,5 @@ +export * from "./credentials"; +export * from "./types"; +export * from "./github"; +export * from "./gitlab"; +export * from "./webhooks"; diff --git a/packages/connectors/src/types.ts b/packages/connectors/src/types.ts new file mode 100644 index 00000000..7a7f0a8c --- /dev/null +++ b/packages/connectors/src/types.ts @@ -0,0 +1,121 @@ +export type ConnectorProvider = "github" | "gitlab"; + +export type ProviderRepository = { + provider: ConnectorProvider; + id: string; + fullName: string; + name: string; + htmlUrl: string; + defaultBranch: string | null; + private: boolean; +}; + +export type ProviderBranch = { + name: string; + commitSha: string; +}; + +export type ProviderCodeFile = { + path: string; + size: number; + checksum: string; + htmlUrl: string; + rawUrl?: string; + content: string; +}; + +export type ProviderRepositorySnapshot = { + repository: ProviderRepository; + branch: ProviderBranch; + commitSha: string; + files: ProviderCodeFile[]; +}; +export type ProviderRepositoryChange = + | { + status: "added" | "modified"; + newPath: string; + } + | { + status: "deleted"; + oldPath: string; + } + | { + status: "renamed"; + oldPath: string; + newPath: string; + }; + +export type ProviderRepositoryDelta = { + fromCommitSha: string; + toCommitSha: string; + changes: ProviderRepositoryChange[]; +}; + +export type ConnectorCredentials = GitHubConnectorCredentials | GitLabConnectorCredentials; + +export type ConnectorInstallationCredentials = + | GitHubInstallationCredentials + | GitLabInstallationCredentials; + +export type GitHubConnectorCredentials = { + provider: "github"; + appId: string; + privateKeyPem: string; + clientId?: string; + clientSecret?: string; + webhookSecret?: string; +}; + +export type GitHubInstallationCredentials = { + provider: "github"; + installationId: string; +}; + +export type GitLabConnectorCredentials = { + provider: "gitlab"; + baseUrl: string; + clientId: string; + clientSecret: string; + webhookSecret?: string; +}; + +export type GitLabInstallationCredentials = { + provider: "gitlab"; + accessToken: string; + refreshToken?: string; + expiresAt?: string; +}; + +export type NormalizedWebhookEvent = { + provider: ConnectorProvider; + deliveryId: string; + eventName: string; + repositoryId: string | null; + repositoryFullName: string | null; + branch: string | null; + commitSha: string | null; + installationId?: string; + raw: unknown; +}; + +export type ProviderRepositoryClient = { + readonly provider: ConnectorProvider; + listRepositories(): Promise; + listBranches(repository: ProviderRepository): Promise; + loadRepositorySnapshot(repository: ProviderRepository, branch: string, commitSha?: string): Promise; + compareRepository(repository: ProviderRepository, fromCommitSha: string, toCommitSha: string): Promise; + readFile(repository: ProviderRepository, path: string, commitSha: string): Promise; +}; + +export type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise; + +export class ConnectorProviderError extends Error { + constructor( + public readonly kind: "auth" | "limit" | "not-found" | "provider" | "validation", + message: string, + options?: { cause?: unknown } + ) { + super(message, options); + this.name = "ConnectorProviderError"; + } +} diff --git a/packages/connectors/src/webhooks.ts b/packages/connectors/src/webhooks.ts new file mode 100644 index 00000000..fd61f16d --- /dev/null +++ b/packages/connectors/src/webhooks.ts @@ -0,0 +1,34 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; + +export function verifyHmacSha256Signature(options: { + body: string | Buffer | Uint8Array; + secret: string; + signatureHeader: string | null | undefined; + prefix: string; +}): boolean { + if (!options.signatureHeader || options.secret.length === 0) { + return false; + } + + const expected = `${options.prefix}${createHmac("sha256", options.secret).update(options.body).digest("hex")}`; + const received = Buffer.from(options.signatureHeader, "utf8"); + const expectedBytes = Buffer.from(expected, "utf8"); + return received.byteLength === expectedBytes.byteLength && timingSafeEqual(received, expectedBytes); +} + +export function verifySharedSecretToken(receivedToken: string | null | undefined, expectedToken: string): boolean { + if (!receivedToken || expectedToken.length === 0) { + return false; + } + + const received = Buffer.from(receivedToken, "utf8"); + const expected = Buffer.from(expectedToken, "utf8"); + return received.byteLength === expected.byteLength && timingSafeEqual(received, expected); +} + +export function branchNameFromGitRef(ref: unknown): string | null { + if (typeof ref !== "string" || !ref.startsWith("refs/heads/")) { + return null; + } + return ref.slice("refs/heads/".length) || null; +} diff --git a/packages/connectors/tsconfig.json b/packages/connectors/tsconfig.json new file mode 100644 index 00000000..56cd751c --- /dev/null +++ b/packages/connectors/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "lib": ["ESNext"], + "target": "ESNext", + "module": "Preserve", + "moduleDetection": "force", + "allowJs": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false, + "types": ["bun"] + }, + "exclude": ["**/__tests__/**", "**/*.test.ts", "**/*.spec.ts"] +} diff --git a/packages/contracts/src/routes.ts b/packages/contracts/src/routes.ts index c46500bb..2dd2104a 100644 --- a/packages/contracts/src/routes.ts +++ b/packages/contracts/src/routes.ts @@ -482,6 +482,162 @@ export type GraphSuggestionApplySuccessData = { workflowRunId: string | null; warnings?: string[]; }; +export type ConnectorProvider = "github" | "gitlab"; +export type ConnectorStatus = "draft" | "active" | "disabled"; +export type ConnectorInstallationStatus = "active" | "disabled" | "pending"; +export type ConnectorAccountType = "user" | "organization" | "group" | null; +export type ConnectorRepositorySelection = "all" | "selected" | "unknown"; +export type RepositoryGraphBindingSyncStatus = "pending" | "syncing" | "synced" | "failed"; + +export type ConnectorRecord = { + id: string; + provider: ConnectorProvider; + name: string; + slug: string; + status: ConnectorStatus; + appId: string | null; + clientId: string | null; + createdAt: string; + updatedAt: string; +}; + +export type ConnectorInstallationRecord = { + id: string; + connectorId: string; + provider: ConnectorProvider; + providerInstallationId: string; + providerAccountLogin: string; + providerAccountType: ConnectorAccountType; + organizationId: string | null; + teamId: string | null; + repositorySelection: ConnectorRepositorySelection; + status: ConnectorInstallationStatus; + createdAt: string; + updatedAt: string; +}; + +export type ConnectorRepositoryRecord = { + provider: ConnectorProvider; + id: string; + fullName: string; + name: string; + htmlUrl: string; + defaultBranch: string | null; + private: boolean; +}; + +export type ConnectorBranchRecord = { + name: string; + commitSha: string; +}; + +export type RepositoryGraphBindingRecord = { + id: string; + graphId: string; + connectorInstallationId: string; + provider: ConnectorProvider; + providerRepositoryId: string; + repositoryFullName: string; + repositoryHtmlUrl: string; + branch: string; + lastSeenCommitSha: string | null; + lastSyncedCommitSha: string | null; + syncStatus: RepositoryGraphBindingSyncStatus; + syncErrorCode: string | null; + webhookEnabled: boolean; + createdAt: string; + updatedAt: string; +}; + +export type GitHubConnectorManifestStartInput = { + name: string; +}; + +export type GitHubConnectorManifestStartSuccessData = { + manifestUrl: string; + state: string; +}; + +export type ConnectorConnectStartSuccessData = { + redirectUrl: string; +}; + +export type GitLabConnectorCreateInput = { + name: string; + baseUrl: string; + clientId: string; + clientSecret: string; + webhookSecret: string; +}; + +export type ConnectorOwnerScopeInput = + | { + kind: "organization"; + } + | { + kind: "team"; + teamId: string; + }; + +export type RepositoryGraphCreateInput = { + connectorInstallationId: string; + repositoryId: string; + repositoryFullName: string; + repositoryHtmlUrl: string; + branch: string; + name: string; + owner: ConnectorOwnerScopeInput; +}; + +export type RepositoryGraphCreateSuccessData = { + graph: GraphRecord; + binding: RepositoryGraphBindingRecord; + workflowRunId: string | null; +}; + +export type RepositoryGraphBindingSyncSuccessData = { + binding: RepositoryGraphBindingRecord; + workflowRunId: string | null; +}; + +export type ConnectorListResponse = ApiResponse; + +export type ConnectorConnectStartResponse = ApiResponse< + ConnectorConnectStartSuccessData, + "UNAUTHORIZED" | "FORBIDDEN" | "GRAPH_NOT_FOUND" | "INVALID_GRAPH_OWNER" | "INTERNAL_SERVER_ERROR" +>; +export type GitHubConnectorManifestStartResponse = ApiResponse< + GitHubConnectorManifestStartSuccessData, + "UNAUTHORIZED" | "FORBIDDEN" | "INVALID_NAME" | "INTERNAL_SERVER_ERROR" +>; +export type GitLabConnectorCreateResponse = ApiResponse< + ConnectorRecord, + "UNAUTHORIZED" | "FORBIDDEN" | "INVALID_NAME" | "INTERNAL_SERVER_ERROR" +>; +export type ConnectorInstallationListResponse = ApiResponse< + ConnectorInstallationRecord[], + "UNAUTHORIZED" | "FORBIDDEN" | "INTERNAL_SERVER_ERROR" +>; +export type ConnectorRepositoryListResponse = ApiResponse< + ConnectorRepositoryRecord[], + "UNAUTHORIZED" | "FORBIDDEN" | "INTERNAL_SERVER_ERROR" +>; +export type ConnectorBranchListResponse = ApiResponse< + ConnectorBranchRecord[], + "UNAUTHORIZED" | "FORBIDDEN" | "INTERNAL_SERVER_ERROR" +>; +export type RepositoryGraphCreateResponse = ApiResponse< + RepositoryGraphCreateSuccessData, + "UNAUTHORIZED" | "FORBIDDEN" | "TEAM_NOT_FOUND" | "INVALID_GRAPH_OWNER" | "INVALID_NAME" | "INTERNAL_SERVER_ERROR" +>; +export type RepositoryGraphBindingResponse = ApiResponse< + RepositoryGraphBindingRecord, + "UNAUTHORIZED" | "FORBIDDEN" | "GRAPH_NOT_FOUND" | "INVALID_GRAPH_OWNER" | "INTERNAL_SERVER_ERROR" +>; +export type RepositoryGraphBindingSyncResponse = ApiResponse< + RepositoryGraphBindingSyncSuccessData, + "UNAUTHORIZED" | "FORBIDDEN" | "GRAPH_NOT_FOUND" | "INVALID_GRAPH_OWNER" | "INTERNAL_SERVER_ERROR" +>; export type TeamCreateResponse = ApiResponse< TeamCreateSuccessData, diff --git a/packages/contracts/src/source.ts b/packages/contracts/src/source.ts index 36c07cc7..59bdb6e1 100644 --- a/packages/contracts/src/source.ts +++ b/packages/contracts/src/source.ts @@ -18,6 +18,12 @@ export type TextUnitSourceChunk = text: string; startPage: number | null; endPage: number | null; + filePath?: string; + language?: string; + startLine?: number; + endLine?: number; + startColumn?: number; + endColumn?: number; regions?: SourceChunkRegion[]; } | { diff --git a/packages/db/package.json b/packages/db/package.json index b936370c..3803ae9b 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -4,6 +4,7 @@ "type": "module", "exports": { "./utils": "./src/utils.ts", + "./source-validity": "./src/source-validity.ts", ".": "./src/index.ts", "./tables/*": "./src/tables/*.ts" }, diff --git a/packages/db/src/__tests__/migration-compat.test.ts b/packages/db/src/__tests__/migration-compat.test.ts index c81ef816..6ca801ea 100644 --- a/packages/db/src/__tests__/migration-compat.test.ts +++ b/packages/db/src/__tests__/migration-compat.test.ts @@ -12,6 +12,36 @@ const jsonFileTypeMigration = new URL( "../../../../migrations/20260613080110_public_piledriver/migration.sql", import.meta.url ); +const codeGraphMigration = new URL( + "../../../../migrations/20260613184716_sturdy_triton/migration.sql", + import.meta.url +); +const codeGraphSnapshot = new URL( + "../../../../migrations/20260613184716_sturdy_triton/snapshot.json", + import.meta.url +); +const externalFileStorageMigration = new URL( + "../../../../migrations/20260613201908_mature_emma_frost/migration.sql", + import.meta.url +); +const externalFileStorageSnapshot = new URL( + "../../../../migrations/20260613201908_mature_emma_frost/snapshot.json", + import.meta.url +); +const sourceValidityMigration = new URL( + "../../../../migrations/20260614105716_tricky_plazm/migration.sql", + import.meta.url +); +const sourceValiditySnapshot = new URL( + "../../../../migrations/20260614105716_tricky_plazm/snapshot.json", + import.meta.url +); +const saveGraphModule = new URL("../../../../apps/worker/lib/save-graph.ts", import.meta.url); +const regenerateDescriptionsModule = new URL( + "../../../../apps/worker/lib/regenerate-descriptions.ts", + import.meta.url +); +const sourceReferenceModule = new URL("../../../../apps/api/src/lib/source-reference.ts", import.meta.url); describe("source reference migration compatibility", () => { test("uses an expand-only migration for source chunk backfill", async () => { @@ -46,3 +76,189 @@ describe("json file type migration compatibility", () => { expect(sql).toContain('ON CONFLICT ("organization_id", "file_type") DO NOTHING'); }); }); + +describe("code graph migration compatibility", () => { + test("adds directed relationship metadata and seeds code file type configs idempotently", async () => { + const sql = await Bun.file(codeGraphMigration).text(); + + expect(sql).toContain("ADD COLUMN IF NOT EXISTS \"kind\" text DEFAULT 'RELATED' NOT NULL"); + expect(sql).toContain('ADD COLUMN IF NOT EXISTS "directed" boolean DEFAULT false NOT NULL'); + expect(sql).toContain("'code'"); + expect(sql).toContain("'text'"); + expect(sql).toContain("'semantic'"); + expect(sql).toContain('ON CONFLICT ("organization_id", "file_type") DO NOTHING'); + }); + + test("keeps relationship metadata columns in the migration snapshot", async () => { + const snapshot = (await Bun.file(codeGraphSnapshot).json()) as { + ddl: Array>; + }; + const relationshipColumns = snapshot.ddl.filter( + (entry) => entry.entityType === "columns" && entry.table === "relationships" + ); + + expect(relationshipColumns).toContainEqual( + expect.objectContaining({ + name: "kind", + type: "text", + notNull: true, + default: "'RELATED'", + }) + ); + expect(relationshipColumns).toContainEqual( + expect.objectContaining({ + name: "directed", + type: "boolean", + notNull: true, + default: "false", + }) + ); + }); +}); + +describe("external file storage migration compatibility", () => { + test("adds explicit storage origin columns and constraints", async () => { + const sql = await Bun.file(externalFileStorageMigration).text(); + + expect(sql).toContain("ADD COLUMN \"storage_kind\" text DEFAULT 'internal' NOT NULL"); + expect(sql).toContain('ADD COLUMN "external_url" text'); + expect(sql).toContain('ADD COLUMN "external_provider" text'); + expect(sql).toContain('ADD CONSTRAINT "files_storage_origin_check"'); + expect(sql).toContain(`"storage_kind" = 'internal'`); + expect(sql).toContain(`"storage_kind" = 'external'`); + expect(sql).toContain('ADD CONSTRAINT "files_external_provider_check"'); + expect(sql).toContain(`"external_provider" IS NULL OR "external_provider" = 'github'`); + }); + + test("keeps external storage columns in the migration snapshot", async () => { + const snapshot = (await Bun.file(externalFileStorageSnapshot).json()) as { + ddl: Array>; + }; + const fileColumns = snapshot.ddl.filter((entry) => entry.entityType === "columns" && entry.table === "files"); + const fileChecks = snapshot.ddl.filter((entry) => entry.entityType === "checks" && entry.table === "files"); + + expect(fileColumns).toContainEqual( + expect.objectContaining({ + name: "storage_kind", + type: "text", + notNull: true, + default: "'internal'", + }) + ); + expect(fileColumns).toContainEqual(expect.objectContaining({ name: "external_url", type: "text" })); + expect(fileColumns).toContainEqual(expect.objectContaining({ name: "external_provider", type: "text" })); + expect(fileChecks).toContainEqual(expect.objectContaining({ name: "files_storage_origin_check" })); + expect(fileChecks).toContainEqual(expect.objectContaining({ name: "files_external_provider_check" })); + }); +}); + +describe("source validity migration compatibility", () => { + test("adds source validity without deleting historical sources", async () => { + const sql = await Bun.file(sourceValidityMigration).text(); + + expect(sql).toContain('ADD COLUMN IF NOT EXISTS "valid_until" timestamp with time zone'); + expect(sql).toContain('"active" = true AND "valid_until" IS NULL'); + expect(sql).toContain('CREATE INDEX IF NOT EXISTS "sources_current_id_idx"'); + expect(sql).toContain('CREATE INDEX IF NOT EXISTS "sources_entity_current_id_idx"'); + expect(sql).toContain('CREATE INDEX IF NOT EXISTS "sources_relationship_current_id_idx"'); + expect(sql).not.toMatch(/DELETE\s+FROM\s+"sources"/i); + }); + + test("keeps valid_until, connector tables, and indexes in the migration snapshot", async () => { + const snapshot = (await Bun.file(sourceValiditySnapshot).json()) as { + ddl: Array>; + }; + const sourceColumns = snapshot.ddl.filter((entry) => entry.entityType === "columns" && entry.table === "sources"); + const sourceIndexes = snapshot.ddl.filter((entry) => entry.entityType === "indexes" && entry.table === "sources"); + const connectorTables = snapshot.ddl.filter((entry) => entry.entityType === "tables"); + const connectorColumns = snapshot.ddl.filter((entry) => entry.entityType === "columns"); + const connectorIndexes = snapshot.ddl.filter((entry) => entry.entityType === "indexes"); + const connectorChecks = snapshot.ddl.filter((entry) => entry.entityType === "checks"); + const connectorFks = snapshot.ddl.filter((entry) => entry.entityType === "fks"); + + expect(sourceColumns).toContainEqual( + expect.objectContaining({ + name: "valid_until", + type: "timestamp with time zone", + notNull: false, + }) + ); + expect(sourceIndexes).toContainEqual( + expect.objectContaining({ + name: "sources_current_id_idx", + where: '"active" = true AND "valid_until" IS NULL', + }) + ); + expect(sourceIndexes).toContainEqual(expect.objectContaining({ name: "sources_entity_current_id_idx" })); + expect(sourceIndexes).toContainEqual(expect.objectContaining({ name: "sources_relationship_current_id_idx" })); + expect(connectorTables).toContainEqual(expect.objectContaining({ name: "connectors" })); + expect(connectorTables).toContainEqual(expect.objectContaining({ name: "connector_installations" })); + expect(connectorTables).toContainEqual(expect.objectContaining({ name: "repository_graph_bindings" })); + expect(connectorTables).toContainEqual(expect.objectContaining({ name: "connector_webhook_events" })); + expect(connectorColumns).toContainEqual( + expect.objectContaining({ table: "files", name: "repository_binding_id", type: "text" }) + ); + expect(connectorColumns).toContainEqual( + expect.objectContaining({ table: "connectors", name: "encrypted_credentials", notNull: true }) + ); + expect(connectorColumns).toContainEqual( + expect.objectContaining({ table: "repository_graph_bindings", name: "sync_status", default: "'pending'" }) + ); + expect(connectorIndexes).toContainEqual(expect.objectContaining({ name: "connectors_provider_status_idx" })); + expect(connectorIndexes).toContainEqual( + expect.objectContaining({ name: "repository_graph_bindings_graph_unique", isUnique: true }) + ); + expect(connectorIndexes).toContainEqual( + expect.objectContaining({ name: "connector_webhook_events_delivery_unique", isUnique: true }) + ); + expect(connectorIndexes).toContainEqual(expect.objectContaining({ name: "files_repository_binding_active_idx" })); + expect(connectorChecks).toContainEqual(expect.objectContaining({ name: "connectors_provider_check" })); + expect(connectorChecks).toContainEqual( + expect.objectContaining({ name: "repository_graph_bindings_sync_status_check" }) + ); + expect(connectorFks).toContainEqual( + expect.objectContaining({ + name: "files_repository_binding_id_repository_graph_bindings_id_fk", + table: "files", + tableTo: "repository_graph_bindings", + }) + ); + }); + + test("adds connector tables and idempotent indexes to the migration sql", async () => { + const sql = await Bun.file(sourceValidityMigration).text(); + + expect(sql).toContain('CREATE TABLE IF NOT EXISTS "connectors"'); + expect(sql).toContain('CREATE TABLE IF NOT EXISTS "connector_installations"'); + expect(sql).toContain('CREATE TABLE IF NOT EXISTS "repository_graph_bindings"'); + expect(sql).toContain('CREATE TABLE IF NOT EXISTS "connector_webhook_events"'); + expect(sql).toContain('ADD COLUMN IF NOT EXISTS "repository_binding_id" text'); + expect(sql).toContain('ADD CONSTRAINT "files_repository_binding_id_repository_graph_bindings_id_fk"'); + expect(sql).toContain('CREATE UNIQUE INDEX IF NOT EXISTS "connector_webhook_events_delivery_unique"'); + expect(sql).toContain("\"external_provider\" IS NULL OR \"external_provider\" in ('github', 'gitlab')"); + }); + test("invalidates older code sources instead of deleting them", async () => { + const source = await Bun.file(saveGraphModule).text(); + + expect(source).toContain("SET valid_until = NOW()"); + expect(source).toContain("old_file.file_type = 'code'"); + expect(source).toContain("currentSourceSql(\"old_source\")"); + expect(source).not.toMatch(/DELETE\s+FROM\s+sources\s+source/i); + }); + + test("filters user-facing source references to current visible sources", async () => { + const source = await Bun.file(sourceReferenceModule).text(); + + expect(source).toContain("currentSourcePredicate(sourcesTable)"); + expect(source).toContain("visibleFilePredicate(filesTable)"); + }); + + test("deactivates descriptions with no unexpired visible sources", async () => { + const source = await Bun.file(regenerateDescriptionsModule).text(); + + expect(source).toContain("unexpiredSourcePredicate(sourcesTable)"); + expect(source).toContain("visibleFilePredicate(filesTable)"); + expect(source).toContain("set({ active: false })"); + expect(source).toContain("AND source.valid_until IS NULL"); + }); +}); diff --git a/packages/db/src/source-validity.ts b/packages/db/src/source-validity.ts new file mode 100644 index 00000000..07099062 --- /dev/null +++ b/packages/db/src/source-validity.ts @@ -0,0 +1,40 @@ +import { and, eq, isNull, sql, type SQL } from "drizzle-orm"; +import { filesTable, sourcesTable } from "./tables/graph"; + +type SourceValidityColumns = { + active: typeof sourcesTable.active; + validUntil: typeof sourcesTable.validUntil; +}; + +type FileVisibilityColumns = { + deleted: typeof filesTable.deleted; +}; + +export function currentSourcePredicate(source: SourceValidityColumns = sourcesTable): SQL { + return and(eq(source.active, true), isNull(source.validUntil))!; +} + +export function unexpiredSourcePredicate(source: Pick = sourcesTable): SQL { + return isNull(source.validUntil); +} + +export function visibleFilePredicate(file: FileVisibilityColumns = filesTable): SQL { + return eq(file.deleted, false); +} + +export function currentSourceSql(alias = "source"): SQL { + const quoted = quoteIdentifier(alias); + return sql.raw(`${quoted}."active" = true AND ${quoted}."valid_until" IS NULL`); +} + +export function visibleFileSql(alias = "file"): SQL { + return sql.raw(`${quoteIdentifier(alias)}."deleted" = false`); +} + +function quoteIdentifier(identifier: string) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)) { + throw new Error("Invalid SQL identifier"); + } + + return `"${identifier}"`; +} diff --git a/packages/db/src/tables/connectors.ts b/packages/db/src/tables/connectors.ts new file mode 100644 index 00000000..fc9f4f01 --- /dev/null +++ b/packages/db/src/tables/connectors.ts @@ -0,0 +1,170 @@ +import { sql } from "drizzle-orm"; +import { boolean, check, index, pgTable, text, timestamp, unique, uniqueIndex } from "drizzle-orm/pg-core"; +import { ulid } from "ulid"; +import { organizationTable, teamTable, userTable } from "./auth"; +import { graphTable } from "./graph"; + +export const CONNECTOR_PROVIDER_VALUES = ["github", "gitlab"] as const; +export type ConnectorProvider = (typeof CONNECTOR_PROVIDER_VALUES)[number]; + +export const CONNECTOR_STATUS_VALUES = ["draft", "active", "disabled"] as const; +export type ConnectorStatus = (typeof CONNECTOR_STATUS_VALUES)[number]; + +export const CONNECTOR_INSTALLATION_STATUS_VALUES = ["active", "disabled"] as const; +export type ConnectorInstallationStatus = (typeof CONNECTOR_INSTALLATION_STATUS_VALUES)[number]; + +export const REPOSITORY_GRAPH_SYNC_STATUS_VALUES = ["pending", "syncing", "synced", "failed"] as const; +export type RepositoryGraphSyncStatus = (typeof REPOSITORY_GRAPH_SYNC_STATUS_VALUES)[number]; + +export const CONNECTOR_WEBHOOK_EVENT_STATUS_VALUES = ["ignored", "enqueued", "duplicate", "failed"] as const; +export type ConnectorWebhookEventStatus = (typeof CONNECTOR_WEBHOOK_EVENT_STATUS_VALUES)[number]; + +export const connectorsTable = pgTable.withRLS( + "connectors", + { + id: text("id") + .primaryKey() + .$default(() => ulid()), + provider: text("provider", { enum: CONNECTOR_PROVIDER_VALUES }).notNull(), + name: text("name").notNull(), + slug: text("slug").notNull(), + status: text("status", { enum: CONNECTOR_STATUS_VALUES }).notNull().default("active"), + appId: text("app_id"), + appSlug: text("app_slug"), + clientId: text("client_id"), + encryptedCredentials: text("encrypted_credentials").notNull(), + webhookSecretEncrypted: text("webhook_secret_encrypted").notNull(), + createdByUserId: text("created_by_user_id").references(() => userTable.id, { onDelete: "set null" }), + createdAt: timestamp("created_at", { withTimezone: true, mode: "date" }).defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true, mode: "date" }) + .defaultNow() + .$onUpdate(() => sql`NOW()`), + }, + (table) => [ + uniqueIndex("connectors_slug_unique").on(table.slug), + index("connectors_provider_status_idx").on(table.provider, table.status), + check("connectors_provider_check", sql`${table.provider} in ('github', 'gitlab')`), + check("connectors_status_check", sql`${table.status} in ('draft', 'active', 'disabled')`), + ] +); + +export const connectorInstallationsTable = pgTable.withRLS( + "connector_installations", + { + id: text("id") + .primaryKey() + .$default(() => ulid()), + connectorId: text("connector_id") + .notNull() + .references(() => connectorsTable.id, { onDelete: "cascade" }), + provider: text("provider", { enum: CONNECTOR_PROVIDER_VALUES }).notNull(), + providerInstallationId: text("provider_installation_id").notNull(), + providerAccountLogin: text("provider_account_login").notNull(), + providerAccountType: text("provider_account_type"), + organizationId: text("organization_id").references(() => organizationTable.id, { onDelete: "cascade" }), + teamId: text("team_id").references(() => teamTable.id, { onDelete: "cascade" }), + installedByUserId: text("installed_by_user_id").references(() => userTable.id, { onDelete: "set null" }), + encryptedCredentials: text("encrypted_credentials"), + repositorySelection: text("repository_selection").notNull().default("unknown"), + status: text("status", { enum: CONNECTOR_INSTALLATION_STATUS_VALUES }).notNull().default("active"), + createdAt: timestamp("created_at", { withTimezone: true, mode: "date" }).defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true, mode: "date" }) + .defaultNow() + .$onUpdate(() => sql`NOW()`), + }, + (table) => [ + unique("connector_installations_provider_scope_unique").on( + table.connectorId, + table.providerInstallationId, + table.organizationId, + table.teamId + ), + index("connector_installations_connector_status_idx").on(table.connectorId, table.status), + index("connector_installations_organization_idx").on(table.organizationId), + index("connector_installations_team_idx").on(table.teamId), + check("connector_installations_provider_check", sql`${table.provider} in ('github', 'gitlab')`), + check("connector_installations_status_check", sql`${table.status} in ('active', 'disabled')`), + check( + "connector_installations_owner_scope_check", + sql`(${table.organizationId} is not null and ${table.teamId} is null) or (${table.organizationId} is not null and ${table.teamId} is not null)` + ), + ] +); + +export const repositoryGraphBindingsTable = pgTable.withRLS( + "repository_graph_bindings", + { + id: text("id") + .primaryKey() + .$default(() => ulid()), + graphId: text("graph_id") + .notNull() + .references(() => graphTable.id, { onDelete: "cascade" }), + connectorInstallationId: text("connector_installation_id") + .notNull() + .references(() => connectorInstallationsTable.id, { onDelete: "restrict" }), + provider: text("provider", { enum: CONNECTOR_PROVIDER_VALUES }).notNull(), + providerRepositoryId: text("provider_repository_id").notNull(), + repositoryFullName: text("repository_full_name").notNull(), + repositoryHtmlUrl: text("repository_html_url").notNull(), + branch: text("branch").notNull(), + lastSeenCommitSha: text("last_seen_commit_sha"), + lastSyncedCommitSha: text("last_synced_commit_sha"), + syncStatus: text("sync_status", { enum: REPOSITORY_GRAPH_SYNC_STATUS_VALUES }).notNull().default("pending"), + syncErrorCode: text("sync_error_code"), + webhookEnabled: boolean("webhook_enabled").notNull().default(true), + createdAt: timestamp("created_at", { withTimezone: true, mode: "date" }).defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true, mode: "date" }) + .defaultNow() + .$onUpdate(() => sql`NOW()`), + }, + (table) => [ + uniqueIndex("repository_graph_bindings_graph_unique").on(table.graphId), + uniqueIndex("repository_graph_bindings_repository_branch_unique").on( + table.connectorInstallationId, + table.providerRepositoryId, + table.branch + ), + index("repository_graph_bindings_provider_repo_branch_idx").on( + table.provider, + table.providerRepositoryId, + table.branch + ), + index("repository_graph_bindings_installation_status_idx").on(table.connectorInstallationId, table.syncStatus), + check("repository_graph_bindings_provider_check", sql`${table.provider} in ('github', 'gitlab')`), + check( + "repository_graph_bindings_sync_status_check", + sql`${table.syncStatus} in ('pending', 'syncing', 'synced', 'failed')` + ), + ] +); + +export const connectorWebhookEventsTable = pgTable.withRLS( + "connector_webhook_events", + { + id: text("id") + .primaryKey() + .$default(() => ulid()), + connectorId: text("connector_id") + .notNull() + .references(() => connectorsTable.id, { onDelete: "cascade" }), + provider: text("provider", { enum: CONNECTOR_PROVIDER_VALUES }).notNull(), + deliveryId: text("delivery_id").notNull(), + eventName: text("event_name").notNull(), + providerRepositoryId: text("provider_repository_id"), + branch: text("branch"), + commitSha: text("commit_sha"), + status: text("status", { enum: CONNECTOR_WEBHOOK_EVENT_STATUS_VALUES }).notNull(), + errorCode: text("error_code"), + createdAt: timestamp("created_at", { withTimezone: true, mode: "date" }).defaultNow(), + }, + (table) => [ + uniqueIndex("connector_webhook_events_delivery_unique").on(table.connectorId, table.provider, table.deliveryId), + index("connector_webhook_events_binding_lookup_idx").on(table.provider, table.providerRepositoryId, table.branch), + check("connector_webhook_events_provider_check", sql`${table.provider} in ('github', 'gitlab')`), + check( + "connector_webhook_events_status_check", + sql`${table.status} in ('ignored', 'enqueued', 'duplicate', 'failed')` + ), + ] +); diff --git a/packages/db/src/tables/graph.ts b/packages/db/src/tables/graph.ts index 42f5442d..dc9b9f4d 100644 --- a/packages/db/src/tables/graph.ts +++ b/packages/db/src/tables/graph.ts @@ -138,6 +138,10 @@ export const filesTable = pgTable.withRLS( type: text("file_type").notNull(), mimeType: text("mime_type").notNull(), key: text("file_key").notNull(), + storageKind: text("storage_kind").notNull().default("internal"), + externalUrl: text("external_url"), + externalProvider: text("external_provider"), + repositoryBindingId: text("repository_binding_id"), checksum: text("checksum"), deleted: boolean("deleted").default(false), status: text("status", { enum: FILE_PROCESS_STATUS_VALUES }).notNull().default("processing"), @@ -168,6 +172,28 @@ export const filesTable = pgTable.withRLS( index("files_graph_active_key_idx") .on(table.graphId, table.key) .where(sql`${table.deleted} = false`), + check( + "files_storage_origin_check", + sql` + ( + ${table.storageKind} = 'internal' + AND ${table.externalUrl} IS NULL + AND ${table.externalProvider} IS NULL + ) + OR ( + ${table.storageKind} = 'external' + AND ${table.externalUrl} IS NOT NULL + AND ${table.externalProvider} IS NOT NULL + ) + ` + ), + check( + "files_external_provider_check", + sql`${table.externalProvider} IS NULL OR ${table.externalProvider} in ('github', 'gitlab')` + ), + index("files_repository_binding_active_idx") + .on(table.repositoryBindingId, table.createdAt, table.id) + .where(sql`${table.deleted} = false`), ] ); @@ -245,6 +271,8 @@ export const relationshipTable = pgTable.withRLS( graphId: text("graph_id") .notNull() .references(() => graphTable.id, { onDelete: "cascade" }), + kind: text("kind").notNull().default("RELATED"), + directed: boolean("directed").notNull().default(false), rank: doublePrecision("rank").notNull().default(0), description: text("description").notNull(), embedding: vector("embedding", { dimensions: 4096 }).notNull(), @@ -276,6 +304,7 @@ export const sourcesTable = pgTable.withRLS( .notNull() .references(() => textUnitTable.id, { onDelete: "cascade" }), active: boolean("active").notNull().default(false), + validUntil: timestamp("valid_until", { withTimezone: true, mode: "date" }), description: text("description").notNull(), sourceChunkIds: json("source_chunk_ids") .$type() @@ -292,6 +321,15 @@ export const sourcesTable = pgTable.withRLS( index("sources_active_id_idx").on(table.active, table.id), index("sources_entity_active_id_idx").on(table.entityId, table.active, table.id), index("sources_relationship_active_id_idx").on(table.relationshipId, table.active, table.id), + index("sources_current_id_idx") + .on(table.id) + .where(sql`${table.active} = true AND ${table.validUntil} IS NULL`), + index("sources_entity_current_id_idx") + .on(table.entityId, table.id) + .where(sql`${table.active} = true AND ${table.validUntil} IS NULL AND ${table.entityId} IS NOT NULL`), + index("sources_relationship_current_id_idx") + .on(table.relationshipId, table.id) + .where(sql`${table.active} = true AND ${table.validUntil} IS NULL AND ${table.relationshipId} IS NOT NULL`), index("sources_text_unit_idx").on(table.textUnitId), index("sources_description_trgm_idx").using("gin", table.description.op("gin_trgm_ops")), index("sources_embedding_diskann_idx").using("diskann", table.embedding.op("vector_cosine_ops")), diff --git a/packages/graph/package.json b/packages/graph/package.json index c9fb6d04..3fd882d4 100644 --- a/packages/graph/package.json +++ b/packages/graph/package.json @@ -7,6 +7,7 @@ "./file-type": "./src/file-type.ts", "./loader/*": "./src/loader/*.ts", "./chunker/*": "./src/chunking/*.ts", + "./code/*": "./src/code/*.ts", "./lib/*": "./src/lib/*.ts", "./unit": "./src/unit.ts", "./merge": "./src/merge.ts", @@ -31,6 +32,12 @@ "js-tiktoken": "^1.0.21", "jszip": "^3.10.1", "pdf-to-img": "^6.1.0", + "tree-sitter": "^0.25.0", + "tree-sitter-c": "^0.24.1", + "tree-sitter-javascript": "^0.25.0", + "tree-sitter-rust": "^0.24.0", + "tree-sitter-typescript": "^0.23.2", + "@tree-sitter-grammars/tree-sitter-zig": "^1.1.2", "ulid": "^3.0.2", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", "zod": "catalog:" diff --git a/packages/graph/src/__tests__/dedupe.test.ts b/packages/graph/src/__tests__/dedupe.test.ts index daa7a813..7d120be0 100644 --- a/packages/graph/src/__tests__/dedupe.test.ts +++ b/packages/graph/src/__tests__/dedupe.test.ts @@ -284,4 +284,41 @@ describe("dedupe", () => { expect(deduped.entities).toHaveLength(2); }); + + test("does not merge relationships with different kinds", () => { + const graph: Graph = { + id: "graph-relationship-kinds", + units: [textUnit("unit-1", "file-1", "A imports and calls B.")], + entities: [ + { id: "entity-a", name: "A", type: "CODE_FUNCTION", description: "", sources: [] }, + { id: "entity-b", name: "B", type: "CODE_FUNCTION", description: "", sources: [] }, + ], + relationships: [ + { + id: "relationship-imports", + sourceId: "entity-a", + targetId: "entity-b", + kind: "IMPORTS", + directed: true, + strength: 0.9, + description: "", + sources: [], + }, + { + id: "relationship-calls", + sourceId: "entity-a", + targetId: "entity-b", + kind: "CALLS", + directed: true, + strength: 0.8, + description: "", + sources: [], + }, + ], + }; + + const deduped = dedupe(graph); + + expect(deduped.relationships.map((relationship) => relationship.kind).sort()).toEqual(["CALLS", "IMPORTS"]); + }); }); diff --git a/packages/graph/src/__tests__/extract.test.ts b/packages/graph/src/__tests__/extract.test.ts index 5eee900b..7373d924 100644 --- a/packages/graph/src/__tests__/extract.test.ts +++ b/packages/graph/src/__tests__/extract.test.ts @@ -330,4 +330,53 @@ describe("mergeGraphs", () => { expect(merged.entities).toEqual([]); expect(merged.relationships).toEqual([]); }); + + test("does not merge opposite directed relationships", () => { + const left: Graph = { + id: "graph-left-directed", + units: [textUnit("unit-left", "file-1", "A calls B.")], + entities: [ + { id: "entity-a-left", name: "A", type: "CODE_FUNCTION", description: "", sources: [] }, + { id: "entity-b-left", name: "B", type: "CODE_FUNCTION", description: "", sources: [] }, + ], + relationships: [ + { + id: "relationship-ab", + sourceId: "entity-a-left", + targetId: "entity-b-left", + kind: "CALLS", + directed: true, + strength: 0.8, + description: "", + sources: [], + }, + ], + }; + const right: Graph = { + id: "graph-right-directed", + units: [textUnit("unit-right", "file-1", "B calls A.")], + entities: [ + { id: "entity-a-right", name: "A", type: "CODE_FUNCTION", description: "", sources: [] }, + { id: "entity-b-right", name: "B", type: "CODE_FUNCTION", description: "", sources: [] }, + ], + relationships: [ + { + id: "relationship-ba", + sourceId: "entity-b-right", + targetId: "entity-a-right", + kind: "CALLS", + directed: true, + strength: 0.6, + description: "", + sources: [], + }, + ], + }; + + const merged = mergeGraphs(left, right); + + expect(merged.relationships).toHaveLength(2); + expect(merged.relationships.every((relationship) => relationship.kind === "CALLS")).toBe(true); + expect(merged.relationships.every((relationship) => relationship.directed === true)).toBe(true); + }); }); diff --git a/packages/graph/src/__tests__/file-type.test.ts b/packages/graph/src/__tests__/file-type.test.ts new file mode 100644 index 00000000..d12e12e1 --- /dev/null +++ b/packages/graph/src/__tests__/file-type.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, test } from "bun:test"; +import { inferGraphFileType } from "../file-type"; + +function file(name: string, type = "") { + return { name, type } as File; +} + +describe("inferGraphFileType", () => { + test("classifies supported source paths as code", () => { + expect(inferGraphFileType(file("src/index.ts"))).toBe("code"); + expect(inferGraphFileType(file("src/component.tsx"))).toBe("code"); + expect(inferGraphFileType(file("src/script.js"))).toBe("code"); + expect(inferGraphFileType(file("src/lib.rs"))).toBe("code"); + expect(inferGraphFileType(file("src/LIB.RS"))).toBe("code"); + expect(inferGraphFileType(file("src/main.zig"))).toBe("code"); + expect(inferGraphFileType(file("src/MAIN.ZIG"))).toBe("code"); + expect(inferGraphFileType(file("src/main.c"))).toBe("code"); + expect(inferGraphFileType(file("src/math.h"))).toBe("code"); + expect(inferGraphFileType(file("src/MATH.H"))).toBe("code"); + }); + + test("keeps text and structured formats out of code classification", () => { + expect(inferGraphFileType(file("README.md"))).toBe("text"); + expect(inferGraphFileType(file("data.json", "application/json"))).toBe("json"); + }); +}); diff --git a/packages/graph/src/chunking/factory.ts b/packages/graph/src/chunking/factory.ts index 3429aba6..c8e1b099 100644 --- a/packages/graph/src/chunking/factory.ts +++ b/packages/graph/src/chunking/factory.ts @@ -58,6 +58,7 @@ export const DEFAULT_FILE_TYPE_CHUNKING: Record ({ + fileId: input.fileId, + repositoryUrl: "https://github.com/acme/widgets.git", + repositoryName: "widgets", + commitSha: "commit-1", + path: input.path, + content: input.content, +}); +const repositoryPrefix = "https://github.com/acme/widgets.git"; + +function relationshipKinds(graph: ReturnType) { + return graph.relationships.map((relationship) => relationship.kind).sort(); +} + +function entityByName(graph: ReturnType, name: string) { + return graph.entities.find((entity) => entity.name === name); +} + +function hasRelationship( + graph: ReturnType, + kind: string, + sourceName: string, + targetName: string +) { + const source = entityByName(graph, sourceName); + const target = entityByName(graph, targetName); + return graph.relationships.some( + (relationship) => + relationship.kind === kind && relationship.sourceId === source?.id && relationship.targetId === target?.id + ); +} +function relationshipCount( + graph: ReturnType, + kind: string, + sourceName: string, + targetName: string +) { + const source = entityByName(graph, sourceName); + const target = entityByName(graph, targetName); + return graph.relationships.filter( + (relationship) => + relationship.kind === kind && relationship.sourceId === source?.id && relationship.targetId === target?.id + ).length; +} + +describe("code repository graph builder", () => { + test("builds code entities, directed relationships, and source metadata from TypeScript AST", () => { + const helper = baseFile({ + fileId: "file-helper", + path: "src/helper.ts", + content: "export function helper() {\n return 1;\n}\n", + }); + const index = baseFile({ + fileId: "file-index", + path: "src/index.ts", + content: [ + "import { helper } from './helper';", + "class Base {}", + "interface Runnable { run(): number }", + "export class Runner extends Base implements Runnable {", + " run() {", + " return helper();", + " }", + "}", + "export const makeRunner = () => new Runner();", + "export type RunnerId = string;", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([helper, index]); + + const graph = buildCodeFileGraph(index, manifest); + + expect(graph.entities.map((entity) => [entity.type, entity.name]).sort()).toContainEqual([ + "CODE_FILE", + `${repositoryPrefix}:src/index.ts`, + ]); + expect(graph.entities.map((entity) => [entity.type, entity.name]).sort()).toContainEqual([ + "CODE_CLASS", + `${repositoryPrefix}:src/index.ts#Runner`, + ]); + expect(graph.entities.map((entity) => [entity.type, entity.name]).sort()).toContainEqual([ + "CODE_INTERFACE", + `${repositoryPrefix}:src/index.ts#Runnable`, + ]); + expect(graph.entities.map((entity) => [entity.type, entity.name]).sort()).toContainEqual([ + "CODE_FUNCTION", + `${repositoryPrefix}:src/index.ts#makeRunner`, + ]); + expect(graph.entities.map((entity) => [entity.type, entity.name]).sort()).toContainEqual([ + "CODE_TYPE", + `${repositoryPrefix}:src/index.ts#RunnerId`, + ]); + expect(graph.entities.map((entity) => [entity.type, entity.name]).sort()).toContainEqual([ + "CODE_FUNCTION", + `${repositoryPrefix}:src/helper.ts#helper`, + ]); + expect(relationshipKinds(graph)).toContain("IMPORTS"); + expect(relationshipKinds(graph)).toContain("CALLS"); + expect(relationshipKinds(graph)).toContain("CONTAINS"); + expect(relationshipKinds(graph)).toContain("EXTENDS"); + expect(relationshipKinds(graph)).toContain("IMPLEMENTS"); + expect(graph.relationships.every((relationship) => relationship.directed === true)).toBe(true); + + const runEntity = graph.entities.find((entity) => entity.name === `${repositoryPrefix}:src/index.ts#Runner.run`); + expect(runEntity).toBeDefined(); + const runChunk = runEntity?.sources + .map((source) => graph.units.find((unit) => unit.id === source.unitId)?.chunks[0]) + .find(Boolean); + expect(runChunk).toMatchObject({ + type: "text", + filePath: "src/index.ts", + language: "typescript", + startLine: 5, + endLine: 7, + }); + expect(runChunk?.text).toContain("run() {"); + expect(runChunk?.text).toContain("helper()"); + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/index.ts#Runner.run`, + `${repositoryPrefix}:src/helper.ts#helper` + ) + ).toBe(true); + }); + + test("keeps named import resolution when reading manifests without explicit exports", () => { + const helper = baseFile({ + fileId: "file-legacy-helper", + path: "src/helper.ts", + content: "export function helper() { return 1; }\n", + }); + const consumer = baseFile({ + fileId: "file-legacy-consumer", + path: "src/legacy-consumer.ts", + content: "import { helper } from './helper';\nexport function run() { return helper(); }\n", + }); + const manifest = buildCodeRepositoryManifest([helper, consumer]); + const legacyManifest = { ...manifest }; + delete (legacyManifest as typeof legacyManifest & { exports?: unknown }).exports; + + const graph = buildCodeFileGraph(consumer, legacyManifest as typeof manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/legacy-consumer.ts#run`, + `${repositoryPrefix}:src/helper.ts#helper` + ) + ).toBe(true); + }); + + test("resolves index imports inside repository manifests", () => { + const dependency = baseFile({ + fileId: "file-module", + path: "src/lib/index.ts", + content: "export function dependency() { return 1; }\n", + }); + const consumer = baseFile({ + fileId: "file-consumer", + path: "src/consumer.ts", + content: "import { dependency } from './lib';\nexport function consumer() { return dependency(); }\n", + }); + const manifest = buildCodeRepositoryManifest([dependency, consumer]); + + const graph = buildCodeFileGraph(consumer, manifest); + + const dependencyEntity = graph.entities.find( + (entity) => entity.name === `${repositoryPrefix}:src/lib/index.ts#dependency` + ); + const consumerEntity = graph.entities.find( + (entity) => entity.name === `${repositoryPrefix}:src/consumer.ts#consumer` + ); + expect(dependencyEntity).toBeDefined(); + expect(consumerEntity).toBeDefined(); + expect( + graph.relationships.some( + (relationship) => + relationship.kind === "CALLS" && + relationship.sourceId === consumerEntity?.id && + relationship.targetId === dependencyEntity?.id + ) + ).toBe(true); + }); + test("resolves default namespace and barrel imports across files", () => { + const defaultHelper = baseFile({ + fileId: "file-default-helper", + path: "src/default-helper.ts", + content: "const helper = () => 1;\nexport default helper;\n", + }); + const helpers = baseFile({ + fileId: "file-helpers", + path: "src/helpers.ts", + content: "export function add() { return 1; }\n", + }); + const barrel = baseFile({ + fileId: "file-barrel", + path: "src/lib/index.ts", + content: "export { add } from '../helpers';\n", + }); + const consumer = baseFile({ + fileId: "file-default-consumer", + path: "src/default-consumer.ts", + content: [ + "import helper from './default-helper';", + "import * as helpers from './helpers';", + "import { add as addFromBarrel } from './lib';", + "export function run() {", + " return helper() + helpers.add() + addFromBarrel();", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([defaultHelper, helpers, barrel, consumer]); + const graph = buildCodeFileGraph(consumer, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/default-consumer.ts#run`, + `${repositoryPrefix}:src/default-helper.ts#helper` + ) + ).toBe(true); + expect( + relationshipCount( + graph, + "CALLS", + `${repositoryPrefix}:src/default-consumer.ts#run`, + `${repositoryPrefix}:src/helpers.ts#add` + ) + ).toBe(2); + expect( + hasRelationship(graph, "IMPORTS", `${repositoryPrefix}:src/default-consumer.ts`, `${repositoryPrefix}:src/lib/index.ts`) + ).toBe(true); + }); + + test("resolves explicit extensions export-star barrels and default reexports", () => { + const defaultHelper = baseFile({ + fileId: "file-default-helper-declaration", + path: "src/default-helper.ts", + content: "export default function helper() { return 1; }\n", + }); + const shared = baseFile({ + fileId: "file-shared", + path: "src/shared.ts", + content: "export function sharedHelper() { return 1; }\n", + }); + const barrel = baseFile({ + fileId: "file-export-star-barrel", + path: "src/lib/index.ts", + content: [ + "export { default as helper } from '../default-helper';", + "export * from '../shared';", + ].join("\n"), + }); + const consumer = baseFile({ + fileId: "file-explicit-extension-consumer", + path: "src/use-barrel.ts", + content: [ + "import { helper, sharedHelper } from './lib/index.ts';", + "export function run() {", + " return helper() + sharedHelper();", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([defaultHelper, shared, barrel, consumer]); + const graph = buildCodeFileGraph(consumer, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/use-barrel.ts#run`, + `${repositoryPrefix}:src/default-helper.ts#helper` + ) + ).toBe(true); + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/use-barrel.ts#run`, + `${repositoryPrefix}:src/shared.ts#sharedHelper` + ) + ).toBe(true); + }); + test("resolves non-relative package imports to external symbols", () => { + const consumer = baseFile({ + fileId: "file-external-consumer", + path: "src/external-consumer.ts", + content: [ + "import { debounce as delay } from 'lodash';", + "import * as React from 'react';", + "export function run() {", + " delay();", + " return React.useMemo(() => 1, []);", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([consumer]); + const graph = buildCodeFileGraph(consumer, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/external-consumer.ts#run`, + "widgets:external:lodash#debounce" + ) + ).toBe(true); + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/external-consumer.ts#run`, + "widgets:external:react#useMemo" + ) + ).toBe(true); + expect( + hasRelationship(graph, "IMPORTS", `${repositoryPrefix}:src/external-consumer.ts`, "widgets:external:lodash") + ).toBe(true); + expect( + hasRelationship(graph, "IMPORTS", `${repositoryPrefix}:src/external-consumer.ts`, "widgets:external:react") + ).toBe(true); + }); + test("prefers local definitions over imported ones when names overlap", () => { + const helper = baseFile({ + fileId: "file-shadow-helper", + path: "src/helper.ts", + content: "export function helper() { return 1; }\n", + }); + const consumer = baseFile({ + fileId: "file-shadow-consumer", + path: "src/shadow.ts", + content: [ + "import { helper } from './helper';", + "export function helperLocal() { return 2; }", + "export function run() {", + " return helperLocal();", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([helper, consumer]); + const graph = buildCodeFileGraph(consumer, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/shadow.ts#run`, + `${repositoryPrefix}:src/shadow.ts#helperLocal` + ) + ).toBe(true); + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/shadow.ts#run`, + `${repositoryPrefix}:src/helper.ts#helper` + ) + ).toBe(false); + }); + + test("resolves external default imports as callable symbols", () => { + const consumer = baseFile({ + fileId: "file-external-default-consumer", + path: "src/external-default.ts", + content: [ + "import nanoid from 'nanoid';", + "export function run() {", + " return nanoid();", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([consumer]); + const graph = buildCodeFileGraph(consumer, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/external-default.ts#run`, + "widgets:external:nanoid#default" + ) + ).toBe(true); + }); + + test("resolves member calls on external default imports", () => { + const consumer = baseFile({ + fileId: "file-external-default-member-consumer", + path: "src/external-default-member.ts", + content: [ + "import React from 'react';", + "export function run() {", + " return React.useMemo(() => 1, []);", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([consumer]); + const graph = buildCodeFileGraph(consumer, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/external-default-member.ts#run`, + "widgets:external:react#useMemo" + ) + ).toBe(true); + }); + + test("keeps nested namespace member paths for external modules", () => { + const consumer = baseFile({ + fileId: "file-external-namespace-consumer", + path: "src/external-namespace.ts", + content: [ + "import * as React from 'react';", + "export function run() {", + " return React.Children.only(null);", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([consumer]); + const graph = buildCodeFileGraph(consumer, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/external-namespace.ts#run`, + "widgets:external:react#Children.only" + ) + ).toBe(true); + }); + + test("resolves Rust fully qualified calls into nested modules", () => { + const mathRoot = baseFile({ + fileId: "file-rust-qualified-root", + path: "src/math/mod.rs", + content: "pub mod nested;\n", + }); + const nestedMath = baseFile({ + fileId: "file-rust-qualified-nested-module", + path: "src/math/nested.rs", + content: "pub fn add() -> i32 { 1 }\n", + }); + const main = baseFile({ + fileId: "file-rust-qualified-main", + path: "src/main.rs", + content: [ + "mod math;", + "pub fn run() -> i32 {", + " crate::math::nested::add()", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([mathRoot, nestedMath, main]); + const graph = buildCodeFileGraph(main, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/main.rs#run`, + `${repositoryPrefix}:src/math/nested.rs#add` + ) + ).toBe(true); + }); + + test("resolves Rust aliased module paths into nested modules", () => { + const mathRoot = baseFile({ + fileId: "file-rust-alias-root", + path: "src/math/mod.rs", + content: "pub mod nested;\n", + }); + const nestedMath = baseFile({ + fileId: "file-rust-alias-nested-module", + path: "src/math/nested.rs", + content: "pub fn add() -> i32 { 1 }\n", + }); + const main = baseFile({ + fileId: "file-rust-alias-main", + path: "src/main.rs", + content: [ + "mod math;", + "use crate::math as helpers;", + "pub fn run() -> i32 {", + " helpers::nested::add()", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([mathRoot, nestedMath, main]); + const graph = buildCodeFileGraph(main, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/main.rs#run`, + `${repositoryPrefix}:src/math/nested.rs#add` + ) + ).toBe(true); + }); + + test("keeps nested external Zig symbol paths", () => { + const main = baseFile({ + fileId: "file-zig-std-mem-main", + path: "src/std-mem.zig", + content: [ + 'const std = @import("std");', + "pub fn run() bool {", + ' return std.mem.eql(u8, "a", "b");', + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([main]); + const graph = buildCodeFileGraph(main, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/std-mem.zig#run`, + "widgets:external:std#mem.eql" + ) + ).toBe(true); + }); + + test("avoids ambiguous system include call resolution", () => { + const main = baseFile({ + fileId: "file-c-ambiguous-system-main", + path: "src/ambiguous-stdio.c", + content: [ + "#include ", + "#include ", + 'int run(void) { return printf("hi"); }', + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([main]); + const graph = buildCodeFileGraph(main, manifest); + + expect( + graph.relationships.some( + (relationship) => + relationship.kind === "CALLS" && + relationship.sourceId === entityByName(graph, `${repositoryPrefix}:src/ambiguous-stdio.c#run`)?.id + ) + ).toBe(false); + }); + + test("prefers local C declarations over wildcard external includes", () => { + const main = baseFile({ + fileId: "file-c-local-over-external-main", + path: "src/local-over-external.c", + content: [ + "#include ", + "int printf(void);", + "int run(void) { return printf(); }", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([main]); + const graph = buildCodeFileGraph(main, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/local-over-external.c#run`, + `${repositoryPrefix}:src/local-over-external.c#printf` + ) + ).toBe(true); + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/local-over-external.c#run`, + "widgets:external:stdio.h#printf" + ) + ).toBe(false); + }); + + test("resolves Rust fully qualified calls without aliases", () => { + const math = baseFile({ + fileId: "file-rust-qualified-math", + path: "src/math.rs", + content: "pub fn add() -> i32 { 1 }\n", + }); + const nested = baseFile({ + fileId: "file-rust-qualified-nested", + path: "src/nested/mod.rs", + content: [ + "pub fn run() -> i32 {", + " crate::math::add() + super::math::add()", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([math, nested]); + const graph = buildCodeFileGraph(nested, manifest); + + expect( + relationshipCount(graph, "CALLS", `${repositoryPrefix}:src/nested/mod.rs#run`, `${repositoryPrefix}:src/math.rs#add`) + ).toBe(2); + }); + + test("resolves Zig package imports to external symbols", () => { + const main = baseFile({ + fileId: "file-zig-std-main", + path: "src/std-main.zig", + content: [ + 'const std = @import("std");', + "pub fn run() void {", + ' std.debug.print("hi", .{});', + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([main]); + const graph = buildCodeFileGraph(main, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/std-main.zig#run`, + "widgets:external:std#debug.print" + ) + ).toBe(true); + expect(hasRelationship(graph, "IMPORTS", `${repositoryPrefix}:src/std-main.zig`, "widgets:external:std")).toBe(true); + }); + + test("resolves C system includes to external symbols", () => { + const main = baseFile({ + fileId: "file-c-stdio-main", + path: "src/stdio-main.c", + content: '#include \nint run(void) { printf("hi"); return 0; }\n', + }); + const manifest = buildCodeRepositoryManifest([main]); + const graph = buildCodeFileGraph(main, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/stdio-main.c#run`, + "widgets:external:stdio.h#printf" + ) + ).toBe(true); + expect( + hasRelationship(graph, "IMPORTS", `${repositoryPrefix}:src/stdio-main.c`, "widgets:external:stdio.h") + ).toBe(true); + }); + + test("resolves Rust imports and module calls across files", () => { + const math = baseFile({ + fileId: "file-rust-math", + path: "src/math.rs", + content: "pub fn add() -> i32 { 1 }\n", + }); + const main = baseFile({ + fileId: "file-rust-main", + path: "src/main.rs", + content: [ + "mod math;", + "use crate::math::add;", + "pub fn run() -> i32 {", + " add() + math::add()", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([math, main]); + const graph = buildCodeFileGraph(main, manifest); + + expect( + relationshipCount(graph, "CALLS", `${repositoryPrefix}:src/main.rs#run`, `${repositoryPrefix}:src/math.rs#add`) + ).toBe(2); + expect( + hasRelationship(graph, "IMPORTS", `${repositoryPrefix}:src/main.rs`, `${repositoryPrefix}:src/math.rs`) + ).toBe(true); + }); + + test("resolves Rust aliases super paths and modrs modules", () => { + const rootMath = baseFile({ + fileId: "file-rust-root-math", + path: "src/math/mod.rs", + content: "pub fn add() -> i32 { 1 }\n", + }); + const shared = baseFile({ + fileId: "file-rust-shared", + path: "src/shared.rs", + content: "pub fn scale() -> i32 { 2 }\n", + }); + const nested = baseFile({ + fileId: "file-rust-nested", + path: "src/nested/mod.rs", + content: [ + "use crate::math as helpers;", + "use super::shared::{scale as shared_scale};", + "pub fn run() -> i32 {", + " helpers::add() + shared_scale()", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([rootMath, shared, nested]); + const graph = buildCodeFileGraph(nested, manifest); + const runEntity = entityByName(graph, `${repositoryPrefix}:src/nested/mod.rs#run`); + const runChunk = runEntity?.sources + .map((source) => graph.units.find((unit) => unit.id === source.unitId)?.chunks[0]) + .find(Boolean); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/nested/mod.rs#run`, + `${repositoryPrefix}:src/math/mod.rs#add` + ) + ).toBe(true); + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/nested/mod.rs#run`, + `${repositoryPrefix}:src/shared.rs#scale` + ) + ).toBe(true); + expect(runChunk?.language).toBe("rust"); + }); + + test("resolves Zig imports across files", () => { + const helpers = baseFile({ + fileId: "file-zig-helpers", + path: "src/helpers.zig", + content: "pub fn add() i32 { return 1; }\n", + }); + const main = baseFile({ + fileId: "file-zig-main", + path: "src/main.zig", + content: [ + 'const helpers = @import("helpers.zig");', + "pub fn run() i32 {", + " return helpers.add();", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([helpers, main]); + const graph = buildCodeFileGraph(main, manifest); + const runEntity = entityByName(graph, `${repositoryPrefix}:src/main.zig#run`); + const runChunk = runEntity?.sources + .map((source) => graph.units.find((unit) => unit.id === source.unitId)?.chunks[0]) + .find(Boolean); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/main.zig#run`, + `${repositoryPrefix}:src/helpers.zig#add` + ) + ).toBe(true); + expect(runChunk?.language).toBe("zig"); + }); + + test("resolves Zig parent-relative imports", () => { + const helpers = baseFile({ + fileId: "file-zig-parent-helpers", + path: "src/helpers.zig", + content: "pub fn add() i32 { return 1; }\n", + }); + const main = baseFile({ + fileId: "file-zig-parent-main", + path: "src/app/main.zig", + content: [ + 'const helpers = @import("../helpers.zig");', + "pub fn run() i32 {", + " return helpers.add();", + "}", + ].join("\n"), + }); + const manifest = buildCodeRepositoryManifest([helpers, main]); + const graph = buildCodeFileGraph(main, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/app/main.zig#run`, + `${repositoryPrefix}:src/helpers.zig#add` + ) + ).toBe(true); + }); + + test("resolves C header includes across files", () => { + const header = baseFile({ + fileId: "file-c-header", + path: "src/math.h", + content: "int add(int left, int right);\n", + }); + const main = baseFile({ + fileId: "file-c-main", + path: "src/main.c", + content: '#include "math.h"\nint run(void) { return add(1, 2); }\n', + }); + const manifest = buildCodeRepositoryManifest([header, main]); + const graph = buildCodeFileGraph(main, manifest); + const runEntity = entityByName(graph, `${repositoryPrefix}:src/main.c#run`); + const runChunk = runEntity?.sources + .map((source) => graph.units.find((unit) => unit.id === source.unitId)?.chunks[0]) + .find(Boolean); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/main.c#run`, + `${repositoryPrefix}:src/math.h#add` + ) + ).toBe(true); + expect( + hasRelationship(graph, "IMPORTS", `${repositoryPrefix}:src/main.c`, `${repositoryPrefix}:src/math.h`) + ).toBe(true); + expect(runChunk?.language).toBe("c"); + }); + + test("resolves parent-relative C header includes", () => { + const header = baseFile({ + fileId: "file-c-nested-header", + path: "src/include/math.h", + content: "int add(int left, int right);\n", + }); + const main = baseFile({ + fileId: "file-c-nested-main", + path: "src/app/main.c", + content: '#include "../include/math.h"\nint run(void) { return add(1, 2); }\n', + }); + const manifest = buildCodeRepositoryManifest([header, main]); + const graph = buildCodeFileGraph(main, manifest); + + expect( + hasRelationship( + graph, + "CALLS", + `${repositoryPrefix}:src/app/main.c#run`, + `${repositoryPrefix}:src/include/math.h#add` + ) + ).toBe(true); + }); + + test("keeps same-named functions in different modules distinct after merge and dedupe", () => { + const alpha = baseFile({ + fileId: "file-alpha", + path: "src/alpha/math.ts", + content: "export function normalize() { return 'alpha'; }\n", + }); + const beta = baseFile({ + fileId: "file-beta", + path: "src/beta/math.ts", + content: "export function normalize() { return 'beta'; }\n", + }); + const manifest = buildCodeRepositoryManifest([alpha, beta]); + + const graph = dedupe(mergeGraphs([buildCodeFileGraph(alpha, manifest), buildCodeFileGraph(beta, manifest)])); + + expect( + graph.entities + .filter((entity) => entity.type === "CODE_FUNCTION" && entity.name.endsWith("#normalize")) + .map((entity) => entity.name) + .sort() + ).toEqual([ + `${repositoryPrefix}:src/alpha/math.ts#normalize`, + `${repositoryPrefix}:src/beta/math.ts#normalize`, + ]); + }); + + test("keeps same repository names from different URLs distinct after dedupe", () => { + const left = baseFile({ + fileId: "file-left", + path: "src/math.ts", + content: "export function normalize() { return 'left'; }\n", + }); + const right = { + ...baseFile({ + fileId: "file-right", + path: "src/math.ts", + content: "export function normalize() { return 'right'; }\n", + }), + repositoryUrl: "https://gitlab.com/acme/widgets.git", + }; + const manifest = buildCodeRepositoryManifest([left, right]); + + const graph = dedupe(mergeGraphs([buildCodeFileGraph(left, manifest), buildCodeFileGraph(right, manifest)])); + + expect( + graph.entities + .filter((entity) => entity.type === "CODE_FUNCTION" && entity.name.endsWith("src/math.ts#normalize")) + .map((entity) => entity.name) + .sort() + ).toEqual([ + "https://github.com/acme/widgets.git:src/math.ts#normalize", + "https://gitlab.com/acme/widgets.git:src/math.ts#normalize", + ]); + }); + + test("returns an empty graph for unsupported code paths", () => { + const file = baseFile({ fileId: "file-readme", path: "README.md", content: "# docs" }); + const graph = buildCodeFileGraph(file, buildCodeRepositoryManifest([file])); + + expect(graph.units).toEqual([]); + expect(graph.entities).toEqual([]); + expect(graph.relationships).toEqual([]); + }); +}); diff --git a/packages/graph/src/code/file-path.ts b/packages/graph/src/code/file-path.ts new file mode 100644 index 00000000..4c8983d9 --- /dev/null +++ b/packages/graph/src/code/file-path.ts @@ -0,0 +1,7 @@ +import path from "node:path"; + +const SUPPORTED_EXTENSIONS = new Set([".js", ".jsx", ".ts", ".tsx", ".mts", ".cts", ".rs", ".zig", ".c", ".h"]); + +export function isSupportedCodePath(filePath: string): boolean { + return SUPPORTED_EXTENSIONS.has(path.posix.extname(filePath).toLowerCase()); +} diff --git a/packages/graph/src/code/identity.ts b/packages/graph/src/code/identity.ts new file mode 100644 index 00000000..3bb87e7c --- /dev/null +++ b/packages/graph/src/code/identity.ts @@ -0,0 +1,23 @@ +import { createHash } from "node:crypto"; +import type { CodeManifestDefinition, CodeRepositoryFile } from "./types"; + +export function entityName(definition: CodeManifestDefinition): string { + return `${fileEntityName(definition)}#${definition.qualifiedName}`; +} + +export function fileEntityName(file: Pick): string { + return `${file.repositoryUrl}:${file.path}`; +} + +export function fileEntityId(repositoryUrl: string, commitSha: string, filePath: string): string { + return stableId("code_file", repositoryUrl, commitSha, filePath); +} + +export function definitionKey(filePath: string, simpleName: string): string { + return `${filePath}\0${simpleName}`; +} + +export function stableId(prefix: string, ...parts: string[]): string { + const hash = createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 32); + return `${prefix}_${hash}`; +} diff --git a/packages/graph/src/code/imports.ts b/packages/graph/src/code/imports.ts new file mode 100644 index 00000000..e3ce9f55 --- /dev/null +++ b/packages/graph/src/code/imports.ts @@ -0,0 +1,638 @@ +import path from "node:path"; +import { definitionKey } from "./identity"; +import type { + CodeManifestDefinition, + CodeManifestExport, + CodeManifestFile, + CodeLanguage, + ExportRecord, + ImportBinding, + ImportRecord, + ImportResolutionMode, + ParsedCodeFile, + TreeSitterNode, +} from "./types"; +import { childForField, fieldText, walk } from "./syntax"; + +const ECMASCRIPT_RESOLVE_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts", ".js", ".jsx"]; +const ZIG_RESOLVE_EXTENSIONS = [".zig"]; + +export function collectImports(root: TreeSitterNode, language: CodeLanguage): ImportRecord[] { + switch (language) { + case "javascript": + case "typescript": + case "tsx": + return collectEcmaScriptImports(root); + case "rust": + return collectRustImports(root); + case "zig": + return collectZigImports(root); + case "c": + return collectCImports(root); + } +} + +export function collectExports(file: ParsedCodeFile): ExportRecord[] { + if (!isEcmaScriptLanguage(file.language)) { + return []; + } + + const exports: ExportRecord[] = []; + walk(file.root, (node) => { + if (node.type !== "export_statement") { + return; + } + + exports.push(...parseExportStatement(node)); + }); + return exports; +} + +export function buildManifestExports( + files: ParsedCodeFile[], + manifestFiles: CodeManifestFile[], + definitions: CodeManifestDefinition[] +): CodeManifestExport[] { + const filesByPath = new Map(manifestFiles.map((file) => [file.path, file])); + const definitionsByPathAndName = new Map( + definitions.map((definition) => [definitionKey(definition.path, definition.simpleName), definition]) + ); + const rawExportsByPath = new Map(files.map((file) => [file.path, collectExports(file)])); + const cache = new Map(); + + const resolveExportedDefinition = (filePath: string, exportedName: string, stack: Set) => { + const exportMatch = resolveFileExports(filePath, stack).find((entry) => entry.exportedName === exportedName); + if (exportMatch) { + return exportMatch; + } + + return definitionsByPathAndName.get(definitionKey(filePath, exportedName)) ?? null; + }; + + const resolveFileExports = (filePath: string, stack: Set): CodeManifestExport[] => { + const cached = cache.get(filePath); + if (cached) { + return cached; + } + + if (stack.has(filePath)) { + return []; + } + stack.add(filePath); + + const resolved = new Map(); + for (const record of rawExportsByPath.get(filePath) ?? []) { + if (record.kind === "local") { + const definition = definitionsByPathAndName.get(definitionKey(filePath, record.localName)); + if (definition) { + resolved.set(record.exportedName, { + ...definition, + exportedName: record.exportedName, + exportedPath: filePath, + }); + } + continue; + } + + const targetPath = resolveImportTargetPath( + { + specifier: record.specifier, + resolutionMode: record.resolutionMode, + }, + filePath, + filesByPath + ); + if (!targetPath) { + continue; + } + + if (record.kind === "reexport") { + const definition = resolveExportedDefinition(targetPath, record.importedName, stack); + if (definition) { + resolved.set(record.exportedName, { + ...definition, + exportedName: record.exportedName, + exportedPath: filePath, + }); + } + continue; + } + + for (const definition of resolveFileExports(targetPath, stack)) { + if (definition.exportedName === "default") { + continue; + } + resolved.set(definition.exportedName, { + ...definition, + exportedPath: filePath, + }); + } + } + + const exports = [...resolved.values()]; + cache.set(filePath, exports); + stack.delete(filePath); + return exports; + }; + + for (const file of files) { + resolveFileExports(file.path, new Set()); + } + + return [...cache.values()].flat(); +} + +export function importLocalNames(importRecord: ImportRecord): string[] { + return [ + ...importRecord.namedImports.map((binding) => binding.local), + ...(importRecord.defaultImport ? [importRecord.defaultImport] : []), + ...(importRecord.namespaceImport ? [importRecord.namespaceImport] : []), + ]; +} + +export function parseHeritage(text: string): Array<{ kind: "EXTENDS" | "IMPLEMENTS"; name: string }> { + const items: Array<{ kind: "EXTENDS" | "IMPLEMENTS"; name: string }> = []; + const head = text.split("{")[0] ?? text; + const extendsMatch = head.match(/\bextends\s+([A-Za-z_$][\w$]*)/u)?.[1]; + if (extendsMatch) items.push({ kind: "EXTENDS", name: extendsMatch }); + + const implementsMatch = head.match(/\bimplements\s+([^{}]+)/u)?.[1]; + if (implementsMatch) { + for (const implemented of implementsMatch.split(",")) { + const name = implemented.trim().match(/^([A-Za-z_$][\w$]*)/u)?.[1]; + if (name) items.push({ kind: "IMPLEMENTS", name }); + } + } + + return items; +} + +export function resolveImportTargetPath( + importRecord: Pick, + currentFilePath: string, + filesByPath: ReadonlyMap +): string | null { + switch (importRecord.resolutionMode) { + case "relative": + return resolveRelativeImportPath(importRecord.specifier, currentFilePath, filesByPath, ECMASCRIPT_RESOLVE_EXTENSIONS); + case "zig": + return resolveRelativeImportPath(importRecord.specifier, currentFilePath, filesByPath, ZIG_RESOLVE_EXTENSIONS); + case "c-local": + return resolveLocalIncludePath(importRecord.specifier, currentFilePath, filesByPath); + case "rust": + return resolveRustImportPath(importRecord.specifier, currentFilePath, filesByPath); + case "external": + return null; + } +} + +function collectEcmaScriptImports(root: TreeSitterNode): ImportRecord[] { + const imports: ImportRecord[] = []; + walk(root, (node) => { + if (node.type !== "import_statement") return; + const parsed = parseImportStatement(node); + if (parsed) imports.push(parsed); + }); + return imports; +} + +function collectRustImports(root: TreeSitterNode): ImportRecord[] { + const imports: ImportRecord[] = []; + walk(root, (node) => { + if (node.type === "mod_item") { + const moduleName = fieldText(node, "name"); + if (moduleName) { + imports.push({ + node, + specifier: `self::${moduleName}`, + resolutionMode: "rust", + namespaceImport: moduleName, + namedImports: [], + }); + } + return; + } + + if (node.type !== "use_declaration") { + return; + } + + imports.push(...parseRustUseDeclaration(node)); + }); + return imports; +} + +function collectZigImports(root: TreeSitterNode): ImportRecord[] { + const imports: ImportRecord[] = []; + walk(root, (node) => { + if (node.type !== "variable_declaration") { + return; + } + + const localName = node.namedChild(0)?.text; + const builtin = node.namedChild(1); + const specifier = builtin ? zigImportSpecifier(builtin) : null; + if (!localName || !specifier) { + return; + } + + imports.push({ + node, + specifier, + resolutionMode: specifier.endsWith(".zig") ? "zig" : "external", + namespaceImport: localName, + namedImports: [], + }); + }); + return imports; +} + +function collectCImports(root: TreeSitterNode): ImportRecord[] { + const imports: ImportRecord[] = []; + walk(root, (node) => { + if (node.type !== "preproc_include") { + return; + } + + const rawPath = childForField(node, "path")?.text ?? ""; + if (!rawPath) { + return; + } + + const isLocalInclude = rawPath.startsWith("\""); + imports.push({ + node, + specifier: unquote(rawPath), + resolutionMode: isLocalInclude ? "c-local" : "external", + namedImports: [], + importAllDefinitions: true, + }); + }); + return imports; +} + +function parseImportStatement(node: TreeSitterNode): ImportRecord | null { + const specifier = parseStringSpecifier(node); + if (!specifier) return null; + const defaultImport = collectDefaultImport(node); + const namespaceImport = collectNamespaceImport(node); + + return { + node, + specifier, + resolutionMode: specifier.startsWith(".") ? "relative" : "external", + namedImports: collectNamedImports(node), + ...(defaultImport ? { defaultImport } : {}), + ...(namespaceImport ? { namespaceImport } : {}), + }; +} + +function parseExportStatement(node: TreeSitterNode): ExportRecord[] { + const declaration = childForField(node, "declaration"); + if (declaration) { + const names = collectDeclarationNames(declaration); + if (node.text.startsWith("export default ")) { + const localName = names[0]; + return localName + ? [ + { + node, + kind: "local", + exportedName: "default", + localName, + }, + ] + : []; + } + + return names.map((localName) => ({ + node, + kind: "local" as const, + exportedName: localName, + localName, + })); + } + + const value = childForField(node, "value"); + if (value?.type === "identifier" && node.text.startsWith("export default ")) { + return [ + { + node, + kind: "local", + exportedName: "default", + localName: value.text, + }, + ]; + } + + const specifier = parseStringSpecifier(node); + const exportClause = namedChildren(node).find((child) => child.type === "export_clause"); + if (exportClause && specifier) { + return collectExportClauseBindings(exportClause).map((binding) => ({ + node, + kind: "reexport" as const, + exportedName: binding.local, + importedName: binding.imported, + specifier, + resolutionMode: specifier.startsWith(".") ? "relative" : "external", + })); + } + + if (exportClause) { + return collectExportClauseBindings(exportClause).map((binding) => ({ + node, + kind: "local" as const, + exportedName: binding.local, + localName: binding.imported, + })); + } + + return specifier + ? [ + { + node, + kind: "export-all", + specifier, + resolutionMode: specifier.startsWith(".") ? "relative" : "external", + }, + ] + : []; +} + +function parseRustUseDeclaration(node: TreeSitterNode): ImportRecord[] { + const argument = childForField(node, "argument"); + if (!argument) { + return []; + } + + if (argument.type === "scoped_use_list") { + const basePath = childForField(argument, "path")?.text; + const useList = namedChildren(argument).find((child) => child.type === "use_list"); + if (!basePath || !useList) { + return []; + } + + return namedChildren(useList).flatMap((child) => { + if (child.type === "use_as_clause") { + return parseRustFlatUse(node, childForField(child, "path")?.text, fieldText(child, "alias"), basePath); + } + return parseRustFlatUse(node, child.text, undefined, basePath); + }); + } + + if (argument.type === "use_as_clause") { + return parseRustFlatUse(node, childForField(argument, "path")?.text, fieldText(argument, "alias")); + } + + return parseRustFlatUse(node, argument.text); +} + +function parseRustFlatUse( + node: TreeSitterNode, + pathText: string | undefined, + alias?: string | null, + basePath?: string +): ImportRecord[] { + if (!pathText) { + return []; + } + + const fullPath = basePath ? `${basePath}::${pathText}` : pathText; + const segments = fullPath.split("::").filter(Boolean); + if (segments.length === 0) { + return []; + } + + const resolutionMode = rustResolutionMode(segments[0] ?? ""); + const lastSegment = segments.at(-1); + if (!lastSegment) { + return []; + } + + if (resolutionMode === "external" && segments.length >= 2) { + return [ + { + node, + specifier: segments.slice(0, -1).join("::"), + resolutionMode, + namedImports: [{ imported: lastSegment, local: alias ?? lastSegment }], + }, + ]; + } + + if (segments.length >= 3) { + return [ + { + node, + specifier: segments.slice(0, -1).join("::"), + resolutionMode, + namedImports: [{ imported: lastSegment, local: alias ?? lastSegment }], + }, + ]; + } + + return [ + { + node, + specifier: fullPath, + resolutionMode, + namespaceImport: alias ?? lastSegment, + namedImports: [], + }, + ]; +} + +function parseStringSpecifier(node: TreeSitterNode): string | null { + const source = childForField(node, "source")?.text; + return source ? unquote(source) : null; +} + +function collectNamedImports(node: TreeSitterNode): ImportBinding[] { + const imports: ImportBinding[] = []; + walk(node, (candidate) => { + if (candidate.type !== "import_specifier") { + return; + } + + const imported = fieldText(candidate, "name") ?? candidate.text.split(/\s+as\s+/u)[0]?.trim(); + const local = fieldText(candidate, "alias") ?? candidate.text.split(/\s+as\s+/u)[1]?.trim() ?? imported; + if (imported && local) { + imports.push({ imported: imported.replace(/^type\s+/u, ""), local }); + } + }); + return imports; +} + +function collectExportClauseBindings(node: TreeSitterNode): ImportBinding[] { + const bindings: ImportBinding[] = []; + walk(node, (candidate) => { + if (candidate.type !== "export_specifier") { + return; + } + + const imported = fieldText(candidate, "name") ?? candidate.text.split(/\s+as\s+/u)[0]?.trim(); + const local = fieldText(candidate, "alias") ?? candidate.text.split(/\s+as\s+/u)[1]?.trim() ?? imported; + if (imported && local) { + bindings.push({ imported, local }); + } + }); + return bindings; +} + +function collectDeclarationNames(node: TreeSitterNode): string[] { + if (node.type === "function_declaration" || node.type === "class_declaration") { + const name = fieldText(node, "name"); + return name ? [name] : []; + } + + const names: string[] = []; + walk(node, (candidate) => { + if (candidate.type === "variable_declarator") { + const name = fieldText(candidate, "name"); + if (name) { + names.push(name); + } + } + }); + return names; +} + +function collectDefaultImport(node: TreeSitterNode): string | undefined { + const clause = namedChildren(node).find((child) => child.type === "import_clause"); + if (!clause) return undefined; + + for (let index = 0; index < clause.namedChildCount; index += 1) { + const child = clause.namedChild(index); + if (child?.type === "identifier") { + return child.text; + } + } + + return undefined; +} + +function collectNamespaceImport(node: TreeSitterNode): string | undefined { + let namespaceImport: string | undefined; + walk(node, (candidate) => { + if (candidate.type === "namespace_import") { + namespaceImport = fieldText(candidate, "name") ?? candidate.namedChild(0)?.text ?? undefined; + } + }); + return namespaceImport; +} + +function zigImportSpecifier(node: TreeSitterNode): string | null { + if (node.type !== "builtin_function" || node.namedChild(0)?.text !== "@import") { + return null; + } + + const argumentList = node.namedChild(1); + const stringNode = argumentList?.namedChild(0); + return stringNode ? unquote(stringNode.text) : null; +} + +function resolveRelativeImportPath( + specifier: string, + currentFilePath: string, + filesByPath: ReadonlyMap, + extensions: readonly string[] +): string | null { + const basePath = path.posix.normalize(path.posix.join(path.posix.dirname(currentFilePath), specifier)); + const candidates = dedupePaths([ + basePath, + ...extensions.map((extension) => (basePath.endsWith(extension) ? basePath : `${basePath}${extension}`)), + ...extensions.map((extension) => path.posix.join(basePath, `index${extension}`)), + ]); + + return candidates.find((candidate) => filesByPath.has(candidate)) ?? null; +} + +function resolveLocalIncludePath( + specifier: string, + currentFilePath: string, + filesByPath: ReadonlyMap +): string | null { + const candidate = path.posix.normalize(path.posix.join(path.posix.dirname(currentFilePath), specifier)); + return filesByPath.has(candidate) ? candidate : null; +} + +function resolveRustImportPath( + specifier: string, + currentFilePath: string, + filesByPath: ReadonlyMap +): string | null { + const segments = specifier.split("::").filter(Boolean); + if (segments.length === 0) { + return null; + } + + let baseDirectory = rustModuleDirectory(currentFilePath); + if (segments[0] === "crate") { + baseDirectory = rustCrateRootDirectory(currentFilePath); + segments.shift(); + } else if (segments[0] === "self") { + segments.shift(); + } else if (segments[0] === "super") { + while (segments[0] === "super") { + baseDirectory = path.posix.dirname(baseDirectory); + segments.shift(); + } + if (segments[0] === "self") { + segments.shift(); + } + } else { + return null; + } + + if (segments.length === 0) { + return null; + } + + const modulePath = path.posix.join(baseDirectory, ...segments); + const candidates = [`${modulePath}.rs`, path.posix.join(modulePath, "mod.rs")]; + return candidates.find((candidate) => filesByPath.has(candidate)) ?? null; +} + +function rustCrateRootDirectory(currentFilePath: string) { + const segments = currentFilePath.split("/"); + const sourceIndex = segments.lastIndexOf("src"); + if (sourceIndex === -1) { + return path.posix.dirname(currentFilePath); + } + return segments.slice(0, sourceIndex + 1).join("/"); +} + +function rustModuleDirectory(currentFilePath: string) { + const baseName = path.posix.basename(currentFilePath); + if (baseName === "main.rs" || baseName === "lib.rs" || baseName === "mod.rs") { + return path.posix.dirname(currentFilePath); + } + return path.posix.join(path.posix.dirname(currentFilePath), path.posix.basename(currentFilePath, ".rs")); +} + +function rustResolutionMode(segment: string): ImportResolutionMode { + return segment === "crate" || segment === "self" || segment === "super" ? "rust" : "external"; +} + +function isEcmaScriptLanguage(language: CodeLanguage) { + return language === "javascript" || language === "typescript" || language === "tsx"; +} + +function namedChildren(node: TreeSitterNode): TreeSitterNode[] { + const children: TreeSitterNode[] = []; + for (let index = 0; index < node.namedChildCount; index += 1) { + const child = node.namedChild(index); + if (child) { + children.push(child); + } + } + return children; +} + +function dedupePaths(paths: string[]) { + return [...new Set(paths)]; +} + +function unquote(value: string): string { + return value.replace(/^['"<]|[>'"]$/gu, ""); +} diff --git a/packages/graph/src/code/metadata.ts b/packages/graph/src/code/metadata.ts new file mode 100644 index 00000000..4b75b31b --- /dev/null +++ b/packages/graph/src/code/metadata.ts @@ -0,0 +1,77 @@ +import type { CodeRepositoryFile } from "./repository"; + +export type CodeFileExternalMetadata = { + provider: "github"; + rawUrl: string; + htmlUrl: string; +}; + +export type CodeFileMetadata = Omit & { + external?: CodeFileExternalMetadata; +}; + +export function serializeCodeFileMetadata(metadata: CodeFileMetadata): string { + return JSON.stringify(metadata); +} + + +function isAllowedExternalUrl(value: unknown, host: string): value is string { + if (typeof value !== "string") { + return false; + } + + try { + const url = new URL(value); + return url.protocol === "https:" && url.hostname === host; + } catch { + return false; + } +} + +export function parseCodeFileMetadata(value: string | null | undefined): CodeFileMetadata | null { + if (!value) { + return null; + } + + try { + const parsed = JSON.parse(value) as Partial; + if ( + typeof parsed.repositoryUrl !== "string" || + typeof parsed.repositoryName !== "string" || + typeof parsed.commitSha !== "string" || + typeof parsed.path !== "string" + ) { + return null; + } + + const external = + parsed.external && + typeof parsed.external === "object" && + "provider" in parsed.external && + parsed.external.provider === "github" && + "rawUrl" in parsed.external && + isAllowedExternalUrl(parsed.external.rawUrl, "raw.githubusercontent.com") && + "htmlUrl" in parsed.external && + isAllowedExternalUrl(parsed.external.htmlUrl, "github.com") + ? { + provider: parsed.external.provider, + rawUrl: parsed.external.rawUrl, + htmlUrl: parsed.external.htmlUrl, + } + : undefined; + + if (parsed.external && !external) { + return null; + } + + return { + repositoryUrl: parsed.repositoryUrl, + repositoryName: parsed.repositoryName, + commitSha: parsed.commitSha, + path: parsed.path, + ...(external ? { external } : {}), + }; + } catch { + return null; + } +} diff --git a/packages/graph/src/code/parser.ts b/packages/graph/src/code/parser.ts new file mode 100644 index 00000000..97105d2e --- /dev/null +++ b/packages/graph/src/code/parser.ts @@ -0,0 +1,70 @@ +import { createRequire } from "node:module"; +import path from "node:path"; +import type { CodeLanguage, CodeRepositoryFile, ParsedCodeFile, TreeSitterLanguage, TreeSitterParser } from "./types"; + +const require = createRequire(import.meta.url); +const Parser = require("tree-sitter") as { new (): TreeSitterParser }; +const C = require("tree-sitter-c") as TreeSitterLanguage; +const JavaScript = require("tree-sitter-javascript") as TreeSitterLanguage; +const Rust = require("tree-sitter-rust") as TreeSitterLanguage; +const TypeScript = require("tree-sitter-typescript") as { + typescript: TreeSitterLanguage; + tsx: TreeSitterLanguage; +}; +const Zig = require("@tree-sitter-grammars/tree-sitter-zig") as TreeSitterLanguage; + +export function parseCodeFile(file: CodeRepositoryFile): ParsedCodeFile[] { + const language = detectCodeLanguage(file.path); + if (!language) { + return []; + } + + const parser = new Parser(); + parser.setLanguage(languageGrammar(language)); + const tree = parser.parse(file.content); + if (!tree) { + return []; + } + + return [{ ...file, language, root: tree.rootNode }]; +} + +function detectCodeLanguage(filePath: string): CodeLanguage | null { + switch (path.posix.extname(filePath).toLowerCase()) { + case ".js": + return "javascript"; + case ".jsx": + case ".tsx": + return "tsx"; + case ".ts": + case ".mts": + case ".cts": + return "typescript"; + case ".rs": + return "rust"; + case ".zig": + return "zig"; + case ".c": + case ".h": + return "c"; + default: + return null; + } +} + +function languageGrammar(language: CodeLanguage): TreeSitterLanguage { + switch (language) { + case "javascript": + return JavaScript; + case "tsx": + return TypeScript.tsx; + case "typescript": + return TypeScript.typescript; + case "rust": + return Rust; + case "zig": + return Zig; + case "c": + return C; + } +} diff --git a/packages/graph/src/code/repository.ts b/packages/graph/src/code/repository.ts new file mode 100644 index 00000000..1a77725a --- /dev/null +++ b/packages/graph/src/code/repository.ts @@ -0,0 +1,597 @@ +import path from "node:path"; +import type { Entity, Graph, Relationship, Source, TextUnitSourceChunk, Unit } from ".."; +import { definitionKey, entityName, fileEntityId, fileEntityName, stableId } from "./identity"; +import { buildManifestExports, collectImports, parseHeritage, resolveImportTargetPath } from "./imports"; +import { parseCodeFile } from "./parser"; +import { callName, collectDefinitions, nodeSnippet, spanSize, walk } from "./syntax"; +import type { + CodeManifestDefinition, + CodeManifestExport, + CodeManifestFile, + CodeRepositoryFile, + CodeRepositoryManifest, + Definition, + ImportRecord, + ParsedCodeFile, + TreeSitterNode, +} from "./types"; + +export { isSupportedCodePath } from "./file-path"; +export type { CodeManifestDefinition, CodeManifestFile, CodeRepositoryFile, CodeRepositoryManifest } from "./types"; + +export function buildCodeRepositoryManifest(files: CodeRepositoryFile[]): CodeRepositoryManifest { + const parsedFiles = files.flatMap(parseCodeFile); + const manifestFiles = parsedFiles.map((file) => manifestFile(file)); + const definitions = parsedFiles.flatMap((file) => + collectDefinitions(file).map(({ node: _node, ...definition }) => definition) + ); + const exports = buildManifestExports(parsedFiles, manifestFiles, definitions); + + return { + files: manifestFiles, + definitions, + exports, + }; +} + +export function buildCodeFileGraph(file: CodeRepositoryFile, manifest: CodeRepositoryManifest): Graph { + const parsed = parseCodeFile(file)[0]; + if (!parsed) { + return { + id: stableId("code_graph", file.repositoryUrl, file.commitSha, file.path), + units: [], + entities: [], + relationships: [], + }; + } + + return new CodeFileGraphBuilder(parsed, manifest).build(); +} + +class CodeFileGraphBuilder { + private readonly units: Unit[] = []; + private readonly entitiesById = new Map(); + private readonly relationshipsById = new Map(); + private readonly definitions: Definition[]; + private readonly definitionsBySimpleName: Map; + private readonly manifestFilesByPath: Map; + private readonly manifestDefinitionsByPathAndName: Map; + private readonly manifestDefinitionsByPath: Map; + private readonly manifestExportsByPathAndName: Map; + private readonly importTargetsByLocalName = new Map(); + private readonly namespaceImportPathsByLocalName = new Map(); + private readonly externalNamespaceImportSpecifiersByLocalName = new Map(); + private readonly wildcardExternalImportSpecifiers = new Set(); + private fileEntity!: Entity; + + constructor(private readonly file: ParsedCodeFile, manifest: CodeRepositoryManifest) { + this.definitions = collectDefinitions(file); + this.definitionsBySimpleName = new Map( + this.definitions.map((definition) => [definition.simpleName, definition]) + ); + this.manifestFilesByPath = new Map(manifest.files.map((file) => [file.path, file])); + this.manifestDefinitionsByPathAndName = new Map( + manifest.definitions.map((definition) => [definitionKey(definition.path, definition.simpleName), definition]) + ); + this.manifestDefinitionsByPath = new Map(); + for (const definition of manifest.definitions) { + const definitions = this.manifestDefinitionsByPath.get(definition.path) ?? []; + definitions.push(definition); + this.manifestDefinitionsByPath.set(definition.path, definitions); + } + this.manifestExportsByPathAndName = new Map( + (manifest.exports ?? []).map((entry) => [definitionKey(entry.exportedPath, entry.exportedName), entry]) + ); + } + + build(): Graph { + this.fileEntity = this.addEntity({ + id: fileEntityId(this.file.repositoryUrl, this.file.commitSha, this.file.path), + name: fileEntityName(this.file), + type: "CODE_FILE", + source: this.sourceFor({ + description: `Source file ${this.file.path} in ${this.file.repositoryName} at ${this.file.commitSha}.`, + text: this.file.path, + }), + }); + + for (const definition of this.definitions) { + this.addDefinition(definition); + } + + this.indexImports(); + this.indexCallsAndHeritage(); + + return { + id: stableId("code_graph", this.file.repositoryUrl, this.file.commitSha, this.file.path), + units: this.units, + entities: [...this.entitiesById.values()], + relationships: [...this.relationshipsById.values()], + }; + } + + private addDefinition(definition: Definition) { + const entity = this.addEntity({ + id: definition.entityId, + name: entityName(definition), + type: definition.type, + source: this.sourceFor({ + node: definition.node, + description: [ + `Defines ${definition.type.toLowerCase()} ${definition.qualifiedName} in ${this.file.path}.`, + nodeSnippet(this.file, definition.node), + ].join("\n\n"), + }), + }); + + const parent = definition.parentQualifiedName + ? this.definitions.find((candidate) => candidate.qualifiedName === definition.parentQualifiedName) + : undefined; + + this.addRelationship({ + source: parent ? this.entityForManifestDefinition(parent) : this.fileEntity, + target: entity, + kind: "CONTAINS", + strength: 1, + node: definition.node, + description: parent + ? `${parent.qualifiedName} contains ${definition.qualifiedName}.` + : `${this.file.path} contains ${definition.qualifiedName}.`, + }); + } + + private indexImports() { + for (const importRecord of collectImports(this.file.root, this.file.language)) { + const targetPath = resolveImportTargetPath(importRecord, this.file.path, this.manifestFilesByPath); + const targetFile = targetPath ? this.manifestFilesByPath.get(targetPath) : undefined; + const targetEntity = targetFile ? this.fileEntityForManifest(targetFile) : this.externalModuleEntity(importRecord); + + this.addRelationship({ + source: this.fileEntity, + target: targetEntity, + kind: "IMPORTS", + strength: 0.9, + node: importRecord.node, + description: `${this.file.path} imports ${importRecord.specifier}.`, + }); + + if (!targetPath) { + if (importRecord.importAllDefinitions) { + this.wildcardExternalImportSpecifiers.add(importRecord.specifier); + } + + for (const binding of importRecord.namedImports) { + this.importTargetsByLocalName.set( + binding.local, + this.externalSymbolEntity(importRecord.specifier, binding.imported, importRecord.node) + ); + } + + if (importRecord.defaultImport) { + this.importTargetsByLocalName.set( + importRecord.defaultImport, + this.externalSymbolEntity(importRecord.specifier, "default", importRecord.node) + ); + this.externalNamespaceImportSpecifiersByLocalName.set( + importRecord.defaultImport, + importRecord.specifier + ); + } + + if (importRecord.namespaceImport) { + this.importTargetsByLocalName.set(importRecord.namespaceImport, targetEntity); + this.externalNamespaceImportSpecifiersByLocalName.set( + importRecord.namespaceImport, + importRecord.specifier + ); + } + continue; + } + + if (importRecord.importAllDefinitions) { + for (const definition of this.manifestDefinitionsByPath.get(targetPath) ?? []) { + this.importTargetsByLocalName.set(definition.simpleName, this.entityForManifestDefinition(definition)); + } + } + + for (const binding of importRecord.namedImports) { + const targetDefinition = this.resolveImportedDefinition(targetPath, binding.imported); + if (targetDefinition) { + this.importTargetsByLocalName.set(binding.local, this.entityForManifestDefinition(targetDefinition)); + } + } + + if (importRecord.defaultImport) { + const defaultTarget = this.resolveImportedDefinition(targetPath, "default"); + this.importTargetsByLocalName.set( + importRecord.defaultImport, + defaultTarget ? this.entityForManifestDefinition(defaultTarget) : targetEntity + ); + } + + if (importRecord.namespaceImport) { + this.namespaceImportPathsByLocalName.set(importRecord.namespaceImport, targetPath); + this.importTargetsByLocalName.set(importRecord.namespaceImport, targetEntity); + } + } + } + + private indexCallsAndHeritage() { + walk(this.file.root, (node) => { + if (node.type === "call_expression") { + this.collectCall(node); + return; + } + + if (node.type === "class_declaration" || node.type === "interface_declaration") { + this.collectHeritage(node); + } + }); + } + + private collectCall(node: TreeSitterNode) { + const callerDefinition = this.enclosingDefinition(node); + const caller = callerDefinition ? this.entityForManifestDefinition(callerDefinition) : this.fileEntity; + const calleeName = callName(node); + if (!calleeName) { + return; + } + + const target = this.resolveCallee(callerDefinition ?? null, calleeName); + if (!target) { + return; + } + + this.addRelationship({ + source: caller, + target, + kind: "CALLS", + strength: 0.8, + node, + description: `${caller.name} calls ${target.name}.`, + }); + } + + private collectHeritage(node: TreeSitterNode) { + const sourceDefinition = this.enclosingDefinition(node); + if (!sourceDefinition) { + return; + } + + for (const heritage of parseHeritage(node.text)) { + const target = this.resolveName(heritage.name); + if (!target) { + continue; + } + + this.addRelationship({ + source: this.entityForManifestDefinition(sourceDefinition), + target, + kind: heritage.kind, + strength: 0.9, + node, + description: `${entityName(sourceDefinition)} ${heritage.kind.toLowerCase()} ${target.name}.`, + }); + } + } + + private resolveCallee(caller: Definition | null, name: string): Entity | null { + if (name.includes(".")) { + const segments = name.split("."); + const objectName = segments[0]; + const memberPath = segments.slice(1).join("."); + const finalName = segments.at(-1); + + if ((objectName === "this" || objectName === "self") && finalName && caller) { + const className = caller.qualifiedName.split(".").at(0); + const method = this.definitions.find( + (definition) => definition.qualifiedName === `${className}.${finalName}` + ); + if (method) { + return this.entityForManifestDefinition(method); + } + } + + if (this.file.language === "rust") { + const rustTarget = this.resolveRustQualifiedCallee(name, finalName); + if (rustTarget) { + return rustTarget; + } + } + + if (objectName && finalName) { + const namespacePath = this.namespaceImportPathsByLocalName.get(objectName); + if (namespacePath) { + const importedDefinition = this.resolveImportedDefinition(namespacePath, finalName); + if (importedDefinition) { + return this.entityForManifestDefinition(importedDefinition); + } + } + + const externalSpecifier = this.externalNamespaceImportSpecifiersByLocalName.get(objectName); + if (externalSpecifier && memberPath) { + return this.externalSymbolEntity(externalSpecifier, memberPath); + } + } + + return objectName ? (this.importTargetsByLocalName.get(objectName) ?? null) : null; + } + + return this.resolveName(name); + } + + private resolveName(name: string): Entity | null { + const definition = this.definitionsBySimpleName.get(name); + if (definition) { + return this.entityForManifestDefinition(definition); + } + + const imported = this.importTargetsByLocalName.get(name); + if (imported) { + return imported; + } + + if (this.wildcardExternalImportSpecifiers.size === 1) { + return this.externalSymbolEntity([...this.wildcardExternalImportSpecifiers][0]!, name); + } + + return null; + } + + private resolveImportedDefinition(targetPath: string, importedName: string): CodeManifestDefinition | CodeManifestExport | null { + return ( + this.manifestExportsByPathAndName.get(definitionKey(targetPath, importedName)) ?? + this.manifestDefinitionsByPathAndName.get(definitionKey(targetPath, importedName)) ?? + null + ); + } + + private enclosingDefinition(node: TreeSitterNode): Definition | null { + return ( + this.definitions + .filter( + (definition) => + definition.node.startIndex <= node.startIndex && definition.node.endIndex >= node.endIndex + ) + .sort((left, right) => spanSize(left.node) - spanSize(right.node))[0] ?? null + ); + } + + private entityForManifestDefinition(definition: CodeManifestDefinition): Entity { + return this.addEntity({ + id: definition.entityId, + name: entityName(definition), + type: definition.type, + }); + } + + private fileEntityForManifest(file: CodeManifestFile): Entity { + return this.addEntity({ + id: file.entityId, + name: fileEntityName(file), + type: "CODE_FILE", + }); + } + + private externalModuleEntity(importRecord: ImportRecord): Entity { + return this.addEntity({ + id: stableId("code_external", this.file.repositoryUrl, this.file.commitSha, importRecord.specifier), + name: `${this.file.repositoryName}:external:${importRecord.specifier}`, + type: "CODE_EXTERNAL_MODULE", + source: this.sourceFor({ + node: importRecord.node, + description: `External module ${importRecord.specifier} imported by ${this.file.path}.`, + }), + }); + } + private externalSymbolEntity(specifier: string, symbolName: string, node?: TreeSitterNode): Entity { + return this.addEntity({ + id: stableId("code_external_symbol", this.file.repositoryUrl, this.file.commitSha, specifier, symbolName), + name: `${this.file.repositoryName}:external:${specifier}#${symbolName}`, + type: "CODE_EXTERNAL_SYMBOL", + ...(node + ? { + source: this.sourceFor({ + node, + description: `External symbol ${symbolName} from ${specifier} referenced by ${this.file.path}.`, + }), + } + : {}), + }); + } + + private resolveRustQualifiedCallee(name: string, finalName: string | undefined): Entity | null { + if (!finalName) { + return null; + } + + const pathPrefix = name.slice(0, -(finalName.length + 1)); + const targetPath = this.resolveRustQualifiedTargetPath(pathPrefix); + if (!targetPath) { + return null; + } + + const targetDefinition = this.resolveImportedDefinition(targetPath, finalName); + return targetDefinition ? this.entityForManifestDefinition(targetDefinition) : null; + } + + private resolveRustQualifiedTargetPath(pathPrefix: string): string | null { + if (!pathPrefix) { + return null; + } + + if (pathPrefix.startsWith("crate::") || pathPrefix.startsWith("self::") || pathPrefix.startsWith("super::")) { + return resolveImportTargetPath( + { specifier: pathPrefix, resolutionMode: "rust" }, + this.file.path, + this.manifestFilesByPath + ); + } + + const [rootSegment, ...nestedSegments] = pathPrefix.split("::"); + if (!rootSegment) { + return null; + } + + const basePath = this.namespaceImportPathsByLocalName.get(rootSegment); + if (!basePath) { + return null; + } + + return this.resolveRustNestedModulePath(basePath, nestedSegments); + } + + private resolveRustNestedModulePath(basePath: string, nestedSegments: string[]): string | null { + let currentPath = basePath; + for (const segment of nestedSegments) { + const moduleDirectory = + path.posix.basename(currentPath) === "mod.rs" + ? path.posix.dirname(currentPath) + : path.posix.join(path.posix.dirname(currentPath), path.posix.basename(currentPath, ".rs")); + const candidateBase = path.posix.join(moduleDirectory, segment); + const nextPath = + [candidateBase, `${candidateBase}.rs`, path.posix.join(candidateBase, "mod.rs")].find((candidate) => + this.manifestFilesByPath.has(candidate) + ) ?? null; + if (!nextPath) { + return null; + } + currentPath = nextPath; + } + + return currentPath; + } + + private addEntity(input: { id: string; name: string; type: string; source?: Source }): Entity { + const existing = this.entitiesById.get(input.id); + if (existing) { + if (input.source && !existing.sources.some((source) => source.id === input.source?.id)) { + existing.sources.push(input.source); + } + return existing; + } + + const entity = { + id: input.id, + name: input.name, + type: input.type, + description: "", + sources: input.source ? [input.source] : [], + } satisfies Entity; + this.entitiesById.set(entity.id, entity); + return entity; + } + + private addRelationship(input: { + source: Entity; + target: Entity; + kind: string; + strength: number; + node: TreeSitterNode; + description: string; + }) { + const id = stableId( + "code_relationship", + this.file.repositoryUrl, + this.file.commitSha, + this.file.path, + input.kind, + input.source.id, + input.target.id, + String(input.node.startIndex), + String(input.node.endIndex) + ); + const source = this.sourceFor({ + node: input.node, + description: input.description, + }); + const existing = this.relationshipsById.get(id); + if (existing) { + if (!existing.sources.some((candidate) => candidate.id === source.id)) { + existing.sources.push(source); + } + existing.strength = Math.max(existing.strength, input.strength); + return existing; + } + + const relationship = { + id, + sourceId: input.source.id, + targetId: input.target.id, + kind: input.kind, + directed: true, + strength: input.strength, + description: "", + sources: [source], + } satisfies Relationship; + this.relationshipsById.set(relationship.id, relationship); + return relationship; + } + + private sourceFor(input: { node?: TreeSitterNode; description: string; text?: string }): Source { + const unit = this.unitFor(input.node, input.text); + return { + id: stableId("code_source", unit.id, input.description), + unitId: unit.id, + description: input.description, + sourceChunkIds: [1], + }; + } + + private unitFor(node?: TreeSitterNode, text?: string): Unit { + const content = text ?? (node ? nodeSnippet(this.file, node) : this.file.path); + const id = stableId( + "code_unit", + this.file.repositoryUrl, + this.file.commitSha, + this.file.path, + String(node?.startIndex ?? 0), + String(node?.endIndex ?? 0), + content + ); + const existing = this.units.find((unit) => unit.id === id); + if (existing) { + return existing; + } + + const chunk = { + id: 1, + type: "text", + text: content, + startPage: null, + endPage: null, + filePath: this.file.path, + language: this.file.language, + ...(node + ? { + startLine: node.startPosition.row + 1, + endLine: node.endPosition.row + 1, + startColumn: node.startPosition.column + 1, + endColumn: node.endPosition.column + 1, + } + : {}), + } satisfies TextUnitSourceChunk; + const unit = { + id, + fileId: this.file.fileId, + content, + startPage: null, + endPage: null, + chunks: [chunk], + } satisfies Unit; + this.units.push(unit); + return unit; + } +} + +function manifestFile(file: ParsedCodeFile): CodeManifestFile { + return { + fileId: file.fileId, + repositoryUrl: file.repositoryUrl, + repositoryName: file.repositoryName, + commitSha: file.commitSha, + path: file.path, + language: file.language, + entityId: fileEntityId(file.repositoryUrl, file.commitSha, file.path), + }; +} diff --git a/packages/graph/src/code/syntax.ts b/packages/graph/src/code/syntax.ts new file mode 100644 index 00000000..8e4da7d6 --- /dev/null +++ b/packages/graph/src/code/syntax.ts @@ -0,0 +1,259 @@ +import type { Definition, ParsedCodeFile, TreeSitterNode } from "./types"; +import { stableId } from "./identity"; + +export function collectDefinitions(file: ParsedCodeFile): Definition[] { + const definitions: Definition[] = []; + + walk(file.root, (node) => { + const definition = definitionFromNode(file, node); + if (definition) { + definitions.push(definition); + } + }); + + return definitions; +} + +export function walk(node: TreeSitterNode, visit: (node: TreeSitterNode) => void) { + visit(node); + for (const child of namedChildren(node)) { + walk(child, visit); + } +} + +export function childForField(node: TreeSitterNode, fieldName: string): TreeSitterNode | null { + return node.childForFieldName(fieldName); +} + +export function fieldText(node: TreeSitterNode, fieldName: string): string | null { + return childForField(node, fieldName)?.text ?? null; +} + +export function spanSize(node: TreeSitterNode) { + return node.endIndex - node.startIndex; +} + +export function nodeSnippet(file: ParsedCodeFile, node: TreeSitterNode): string { + return file.content.slice(node.startIndex, node.endIndex).trimEnd(); +} + +export function callName(node: TreeSitterNode): string | null { + const callee = childForField(node, "function"); + if (!callee) return null; + if (callee.type === "identifier" || callee.type === "field_identifier" || callee.type === "property_identifier") { + return callee.text; + } + + return memberLikeName(callee); +} + +function definitionFromNode(file: ParsedCodeFile, node: TreeSitterNode): Definition | null { + switch (node.type) { + case "function_declaration": + case "generator_function_declaration": { + const simpleName = fieldText(node, "name"); + if (!simpleName) return null; + return definition(file, node, simpleName, simpleName, "CODE_FUNCTION"); + } + case "function_signature": { + const simpleName = fieldText(node, "name"); + if (!simpleName) return null; + return definition(file, node, simpleName, simpleName, "CODE_FUNCTION"); + } + case "function_signature_item": + case "function_item": { + const simpleName = fieldText(node, "name"); + if (!simpleName) return null; + const parentQualifiedName = + node.type === "function_signature_item" ? enclosingRustTraitName(node) : enclosingRustImplTypeName(node); + return parentQualifiedName + ? definition(file, node, simpleName, `${parentQualifiedName}.${simpleName}`, "CODE_METHOD", parentQualifiedName) + : definition(file, node, simpleName, simpleName, "CODE_FUNCTION"); + } + case "function_definition": { + const simpleName = cFunctionName(node); + if (!simpleName) return null; + return definition(file, node, simpleName, simpleName, "CODE_FUNCTION"); + } + case "declaration": { + const simpleName = cFunctionName(node); + if (!simpleName) return null; + return definition(file, node, simpleName, simpleName, "CODE_FUNCTION"); + } + case "class_declaration": { + const simpleName = fieldText(node, "name"); + if (!simpleName) return null; + return definition(file, node, simpleName, simpleName, "CODE_CLASS"); + } + case "struct_item": { + const simpleName = fieldText(node, "name"); + if (!simpleName) return null; + return definition(file, node, simpleName, simpleName, "CODE_CLASS"); + } + case "interface_declaration": + case "trait_item": { + const simpleName = fieldText(node, "name"); + if (!simpleName) return null; + return definition(file, node, simpleName, simpleName, "CODE_INTERFACE"); + } + case "type_alias_declaration": { + const simpleName = fieldText(node, "name"); + if (!simpleName) return null; + return definition(file, node, simpleName, simpleName, "CODE_TYPE"); + } + case "method_definition": { + const simpleName = fieldText(node, "name"); + const className = enclosingClassName(node); + if (!simpleName || !className) return null; + return definition(file, node, simpleName, `${className}.${simpleName}`, "CODE_METHOD", className); + } + case "variable_declarator": { + const value = childForField(node, "value"); + if (!value || !["arrow_function", "function", "class"].includes(value.type)) { + return null; + } + + const simpleName = fieldText(node, "name"); + if (!simpleName) return null; + return definition( + file, + node, + simpleName, + simpleName, + value.type === "class" ? "CODE_CLASS" : "CODE_FUNCTION" + ); + } + default: + return null; + } +} + +function definition( + file: ParsedCodeFile, + node: TreeSitterNode, + simpleName: string, + qualifiedName: string, + type: string, + parentQualifiedName?: string +): Definition { + return { + entityId: stableId("code_entity", file.repositoryUrl, file.commitSha, file.path, qualifiedName), + fileId: file.fileId, + path: file.path, + repositoryUrl: file.repositoryUrl, + repositoryName: file.repositoryName, + commitSha: file.commitSha, + simpleName, + qualifiedName, + type, + node, + ...(parentQualifiedName ? { parentQualifiedName } : {}), + }; +} + +function namedChildren(node: TreeSitterNode): TreeSitterNode[] { + const children: TreeSitterNode[] = []; + for (let index = 0; index < node.namedChildCount; index += 1) { + const child = node.namedChild(index); + if (child) children.push(child); + } + return children; +} + +function memberLikeName(node: TreeSitterNode): string | null { + if (node.type === "member_expression") { + const object = childForField(node, "object")?.text ?? node.namedChild(0)?.text; + const property = childForField(node, "property")?.text ?? node.namedChild(1)?.text; + return object && property ? `${object}.${property}` : (property ?? null); + } + + if (node.type === "field_expression") { + const object = childForField(node, "object")?.text ?? childForField(node, "value")?.text ?? node.namedChild(0)?.text; + const property = childForField(node, "property")?.text ?? childForField(node, "field")?.text ?? node.namedChild(1)?.text; + return object && property ? `${object}.${property}` : (property ?? null); + } + + if (node.type === "scoped_identifier") { + const object = childForField(node, "path")?.text ?? node.namedChild(0)?.text; + const property = childForField(node, "name")?.text ?? node.namedChild(1)?.text; + return object && property ? `${object}.${property}` : (property ?? null); + } + + return null; +} + +function cFunctionName(node: TreeSitterNode): string | null { + const declarator = childForField(node, "declarator"); + const functionDeclarator = findNodeByType(declarator, "function_declarator"); + if (!functionDeclarator) { + return null; + } + + return fieldText(functionDeclarator, "declarator") ?? firstIdentifier(functionDeclarator); +} + +function findNodeByType(node: TreeSitterNode | null, type: string): TreeSitterNode | null { + if (!node) { + return null; + } + if (node.type === type) { + return node; + } + + for (const child of namedChildren(node)) { + const match = findNodeByType(child, type); + if (match) { + return match; + } + } + + return null; +} + +function firstIdentifier(node: TreeSitterNode): string | null { + if (node.type === "identifier" || node.type === "field_identifier" || node.type === "type_identifier") { + return node.text; + } + + for (const child of namedChildren(node)) { + const match = firstIdentifier(child); + if (match) { + return match; + } + } + + return null; +} + +function enclosingClassName(node: TreeSitterNode): string | null { + let current = node.parent; + while (current) { + if (current.type === "class_declaration") { + return fieldText(current, "name"); + } + current = current.parent; + } + return null; +} + +function enclosingRustImplTypeName(node: TreeSitterNode): string | null { + let current = node.parent; + while (current) { + if (current.type === "impl_item") { + return fieldText(current, "type"); + } + current = current.parent; + } + return null; +} + +function enclosingRustTraitName(node: TreeSitterNode): string | null { + let current = node.parent; + while (current) { + if (current.type === "trait_item") { + return fieldText(current, "name"); + } + current = current.parent; + } + return null; +} diff --git a/packages/graph/src/code/types.ts b/packages/graph/src/code/types.ts new file mode 100644 index 00000000..b66b1795 --- /dev/null +++ b/packages/graph/src/code/types.ts @@ -0,0 +1,121 @@ +export type TreeSitterLanguage = unknown; + +export type TreeSitterPoint = { + row: number; + column: number; +}; + +export type TreeSitterNode = { + type: string; + text: string; + startIndex: number; + endIndex: number; + startPosition: TreeSitterPoint; + endPosition: TreeSitterPoint; + namedChildCount: number; + parent: TreeSitterNode | null; + namedChild: (index: number) => TreeSitterNode | null; + childForFieldName: (name: string) => TreeSitterNode | null; +}; + +export type TreeSitterTree = { + rootNode: TreeSitterNode; +}; + +export type TreeSitterParser = { + setLanguage: (language: TreeSitterLanguage) => void; + parse: (source: string) => TreeSitterTree | null; +}; + +export type CodeLanguage = "javascript" | "typescript" | "tsx" | "rust" | "zig" | "c"; + +export type ImportResolutionMode = "relative" | "zig" | "rust" | "c-local" | "external"; + +export type CodeRepositoryFile = { + fileId: string; + repositoryUrl: string; + repositoryName: string; + commitSha: string; + path: string; + content: string; +}; + +export type CodeManifestFile = { + fileId: string; + repositoryUrl: string; + repositoryName: string; + commitSha: string; + path: string; + language: CodeLanguage; + entityId: string; +}; + +export type CodeManifestDefinition = { + entityId: string; + fileId: string; + path: string; + repositoryUrl: string; + repositoryName: string; + commitSha: string; + simpleName: string; + qualifiedName: string; + type: string; +}; + +export type CodeManifestExport = CodeManifestDefinition & { + exportedName: string; + exportedPath: string; +}; + +export type CodeRepositoryManifest = { + files: CodeManifestFile[]; + definitions: CodeManifestDefinition[]; + exports: CodeManifestExport[]; +}; + +export type ParsedCodeFile = CodeRepositoryFile & { + language: CodeLanguage; + root: TreeSitterNode; +}; + +export type Definition = CodeManifestDefinition & { + node: TreeSitterNode; + parentQualifiedName?: string; +}; + +export type ImportBinding = { + imported: string; + local: string; +}; + +export type ImportRecord = { + node: TreeSitterNode; + specifier: string; + resolutionMode: ImportResolutionMode; + defaultImport?: string; + namespaceImport?: string; + namedImports: ImportBinding[]; + importAllDefinitions?: boolean; +}; + +export type ExportRecord = + | { + node: TreeSitterNode; + kind: "local"; + exportedName: string; + localName: string; + } + | { + node: TreeSitterNode; + kind: "reexport"; + exportedName: string; + importedName: string; + specifier: string; + resolutionMode: ImportResolutionMode; + } + | { + node: TreeSitterNode; + kind: "export-all"; + specifier: string; + resolutionMode: ImportResolutionMode; + }; diff --git a/packages/graph/src/dedupe.ts b/packages/graph/src/dedupe.ts index a3cc1748..d855c9d5 100644 --- a/packages/graph/src/dedupe.ts +++ b/packages/graph/src/dedupe.ts @@ -1,5 +1,6 @@ import { ulid } from "ulid"; import type { Entity, Graph, Relationship, Source, Unit } from "."; +import { DEFAULT_RELATIONSHIP_KIND, normalizedRelationshipEndpoints, relationshipKey } from "./relationship-key"; const exactOnlyTypes = new Set(["DATE", "FACT"]); const organizationSuffixes = new Set([ @@ -243,13 +244,6 @@ const mergeUnits = (units: Unit[]) => { return [...merged.values()]; }; -const buildRelationshipKey = (sourceId: string, targetId: string) => { - const normalizedSourceId = sourceId <= targetId ? sourceId : targetId; - const normalizedTargetId = sourceId <= targetId ? targetId : sourceId; - - return `${normalizedSourceId}::${normalizedTargetId}`; -}; - export function dedupe(graph: Graph): Graph { const parents = graph.entities.map((_, index) => index); @@ -321,9 +315,17 @@ export function dedupe(graph: Graph): Graph { continue; } - const normalizedSourceId = sourceId <= targetId ? sourceId : targetId; - const normalizedTargetId = sourceId <= targetId ? targetId : sourceId; - const key = buildRelationshipKey(normalizedSourceId, normalizedTargetId); + const endpoints = normalizedRelationshipEndpoints({ + sourceId, + targetId, + directed: relationship.directed, + }); + const key = relationshipKey({ + sourceId: endpoints.sourceId, + targetId: endpoints.targetId, + kind: relationship.kind, + directed: endpoints.directed, + }); const existingRelationship = relationshipMap.get(key); if (existingRelationship) { @@ -347,8 +349,10 @@ export function dedupe(graph: Graph): Graph { relationshipMap.set(key, { ...relationship, - sourceId: normalizedSourceId, - targetId: normalizedTargetId, + sourceId: endpoints.sourceId, + targetId: endpoints.targetId, + kind: relationship.kind ?? DEFAULT_RELATIONSHIP_KIND, + directed: endpoints.directed, description: (relationship.description ?? "") .trim() .replace(/[\r\n]+/g, " ") diff --git a/packages/graph/src/file-type.ts b/packages/graph/src/file-type.ts index 28906de9..7d46faaf 100644 --- a/packages/graph/src/file-type.ts +++ b/packages/graph/src/file-type.ts @@ -1,3 +1,5 @@ +import { isSupportedCodePath } from "./code/file-path"; + export const GRAPH_FILE_TYPES = [ "pdf", "doc", @@ -17,6 +19,7 @@ export const GRAPH_FILE_TYPES = [ "xml", "yaml", "toml", + "code", "text", ] as const; @@ -183,5 +186,9 @@ export function inferGraphFileType(file: Pick): GraphFile return "toml"; } + if (isSupportedCodePath(name)) { + return "code"; + } + return "text"; } diff --git a/packages/graph/src/index.ts b/packages/graph/src/index.ts index bf45342b..ad60a3f9 100644 --- a/packages/graph/src/index.ts +++ b/packages/graph/src/index.ts @@ -12,6 +12,8 @@ export type Relationship = { id: string; sourceId: string; targetId: string; + kind?: string; + directed?: boolean; strength: number; description?: string; sources: Source[]; diff --git a/packages/graph/src/loader/factory.ts b/packages/graph/src/loader/factory.ts index 868a5bdb..a2b2a2d6 100644 --- a/packages/graph/src/loader/factory.ts +++ b/packages/graph/src/loader/factory.ts @@ -69,6 +69,7 @@ export const DEFAULT_DOCUMENT_MODES: Record) => `${entity.name}::${entity.type}`; -const getRelationshipKey = (relationship: Pick) => { - const sourceId = relationship.sourceId <= relationship.targetId ? relationship.sourceId : relationship.targetId; - const targetId = relationship.sourceId <= relationship.targetId ? relationship.targetId : relationship.sourceId; - - return `${sourceId}::${targetId}`; -}; - export function mergeGraphs(graphs: Graph[]): Graph; export function mergeGraphs(left: Graph, right: Graph): Graph; export function mergeGraphs(input: Graph[] | Graph, right?: Graph): Graph { @@ -52,9 +46,17 @@ export function mergeGraphs(input: Graph[] | Graph, right?: Graph): Graph { continue; } - const normalizedSourceId = sourceId <= targetId ? sourceId : targetId; - const normalizedTargetId = sourceId <= targetId ? targetId : sourceId; - const key = getRelationshipKey({ sourceId: normalizedSourceId, targetId: normalizedTargetId }); + const endpoints = normalizedRelationshipEndpoints({ + sourceId, + targetId, + directed: relationship.directed, + }); + const key = relationshipKey({ + sourceId: endpoints.sourceId, + targetId: endpoints.targetId, + kind: relationship.kind, + directed: endpoints.directed, + }); const existingRelationship = mergedRelationships.get(key); if (existingRelationship) { @@ -70,8 +72,10 @@ export function mergeGraphs(input: Graph[] | Graph, right?: Graph): Graph { mergedRelationships.set(key, { ...relationship, - sourceId: normalizedSourceId, - targetId: normalizedTargetId, + sourceId: endpoints.sourceId, + targetId: endpoints.targetId, + kind: relationship.kind ?? DEFAULT_RELATIONSHIP_KIND, + directed: endpoints.directed, sources: [...relationship.sources], }); } diff --git a/packages/graph/src/relationship-key.ts b/packages/graph/src/relationship-key.ts new file mode 100644 index 00000000..1c1783b9 --- /dev/null +++ b/packages/graph/src/relationship-key.ts @@ -0,0 +1,26 @@ +import type { Relationship } from "."; + +export const DEFAULT_RELATIONSHIP_KIND = "RELATED"; + +export function normalizedRelationshipEndpoints( + relationship: Pick +): { sourceId: string; targetId: string; directed: boolean } { + const directed = relationship.directed === true; + const sourceId = directed + ? relationship.sourceId + : relationship.sourceId <= relationship.targetId + ? relationship.sourceId + : relationship.targetId; + const targetId = directed + ? relationship.targetId + : relationship.sourceId <= relationship.targetId + ? relationship.targetId + : relationship.sourceId; + + return { sourceId, targetId, directed }; +} + +export function relationshipKey(relationship: Pick) { + const endpoints = normalizedRelationshipEndpoints(relationship); + return `${relationship.kind ?? DEFAULT_RELATIONSHIP_KIND}::${endpoints.directed ? "directed" : "undirected"}::${endpoints.sourceId}::${endpoints.targetId}`; +} diff --git a/plans/001-align-migration-snapshot.md b/plans/001-align-migration-snapshot.md new file mode 100644 index 00000000..e1601d37 --- /dev/null +++ b/plans/001-align-migration-snapshot.md @@ -0,0 +1,153 @@ +# Plan 001: Align the relationship migration snapshot with the schema + +> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. +> +> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/db/src/tables/graph.ts migrations/20260613184716_sturdy_triton packages/db/src/__tests__/migration-compat.test.ts` +> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. + +## Status + +- **Priority**: P1 +- **Effort**: S +- **Risk**: LOW +- **Depends on**: none +- **Category**: migration +- **Planned at**: commit `1dea5eb77`, 2026-06-13 + +## Why this matters + +The branch adds `relationships.kind` and `relationships.directed` in both the Drizzle schema and the SQL migration, but the migration snapshot does not contain those columns. Drizzle uses snapshots as migration-generation state. If this lands stale, the next generated migration can diff from the wrong schema state and re-emit already-applied relationship column changes. + +## Current state + +Relevant files: + +- `packages/db/src/tables/graph.ts` — source Drizzle schema for `relationships`. +- `migrations/20260613184716_sturdy_triton/migration.sql` — SQL migration for the branch. +- `migrations/20260613184716_sturdy_triton/snapshot.json` — generated Drizzle snapshot that is currently stale. +- `packages/db/src/__tests__/migration-compat.test.ts` — migration compatibility tests. + +Current schema excerpt: + +```ts +// packages/db/src/tables/graph.ts:245-250 +graphId: text("graph_id") + .notNull() + .references(() => graphTable.id, { onDelete: "cascade" }), +kind: text("kind").notNull().default("RELATED"), +directed: boolean("directed").notNull().default(false), +rank: doublePrecision("rank").notNull().default(0), +``` + +Current migration excerpt: + +```sql +-- migrations/20260613184716_sturdy_triton/migration.sql:1-2 +ALTER TABLE "relationships" ADD COLUMN IF NOT EXISTS "kind" text DEFAULT 'RELATED' NOT NULL; +ALTER TABLE "relationships" ADD COLUMN IF NOT EXISTS "directed" boolean DEFAULT false NOT NULL; +``` + +Current stale snapshot excerpt: + +```json +// migrations/20260613184716_sturdy_triton/snapshot.json:3008-3021 +"name": "graph_id", +"entityType": "columns", +"schema": "public", +"table": "relationships" +}, +{ +"type": "double precision", +"notNull": true, +"default": "0", +"name": "rank", +"table": "relationships" +``` + +Repo conventions: + +- Use Bun from the repo root. +- Do not run `bun run db:migrate`. +- Do not hand-create a new migration. This plan fixes the existing branch migration metadata. +- Root verification commands available: `bun run test`, `bun run lint`. `bun run test` passed at planning time; `bun run lint` exited 0 with one existing frontend warning in `apps/frontend/components/theme/ThemePresetScript.tsx`. + +## Commands you will need + +| Purpose | Command | Expected on success | +|---|---|---| +| Tests | `bun run test` | exit 0; all workspace tests pass | +| Lint | `bun run lint` | exit 0; existing warning may remain, no new errors | + +## Scope + +**In scope**: + +- `migrations/20260613184716_sturdy_triton/snapshot.json` +- `packages/db/src/__tests__/migration-compat.test.ts` + +**Out of scope**: + +- Creating another migration directory. +- Running `bun run db:migrate`. +- Changing relationship application code. +- Editing older migration snapshots. + +## Git workflow + +- Branch name suggestion: `advisor/001-align-migration-snapshot`. +- Commit message style from repo history: Conventional Commits, e.g. `fix(db): align relationship migration snapshot`. +- Do not push or open a PR unless instructed. + +## Steps + +### Step 1: Add the missing snapshot columns + +Update `migrations/20260613184716_sturdy_triton/snapshot.json` so the `relationships` table has both columns between `graph_id` and `rank`, matching schema/migration semantics: + +- `kind`: type `text`, `notNull: true`, default `'RELATED'` in the snapshot's existing default representation for string defaults. +- `directed`: type `boolean`, `notNull: true`, default `false` in the snapshot's existing default representation for boolean defaults. + +Prefer regenerating the snapshot with the repo's Drizzle workflow if that produces only this snapshot correction. If regeneration wants to create another migration or broad unrelated changes, stop and hand-edit only these two column entries to match the snapshot format around nearby columns. + +**Verify**: `bun run test --filter @kiwi/db` → exit 0; DB package tests pass. + +### Step 2: Add a compatibility assertion for the snapshot + +In `packages/db/src/__tests__/migration-compat.test.ts`, extend the existing migration compatibility coverage for `20260613184716_sturdy_triton` so it checks that `snapshot.json` contains `relationships.kind` and `relationships.directed`. Keep this as a metadata regression test; do not assert fragile line numbers. + +**Verify**: `bun run test --filter @kiwi/db` → exit 0; the new assertion fails before Step 1 and passes after Step 1. + +### Step 3: Run repo checks + +Run the standard read-only checks from the repo root. + +**Verify**: `bun run test` → exit 0; all tests pass. + +**Verify**: `bun run lint` → exit 0; no new errors. + +## Test plan + +- Extend `packages/db/src/__tests__/migration-compat.test.ts`. +- Cover both new relationship columns in `snapshot.json`. +- Do not add a migration execution test; this is a snapshot/schema parity regression. + +## Done criteria + +- [ ] `migrations/20260613184716_sturdy_triton/snapshot.json` contains `relationships.kind` and `relationships.directed` with defaults/nullability matching `graph.ts` and `migration.sql`. +- [ ] `bun run test --filter @kiwi/db` exits 0. +- [ ] `bun run test` exits 0. +- [ ] `bun run lint` exits 0. +- [ ] No files outside the in-scope list are modified. +- [ ] `plans/README.md` status row updated. + +## STOP conditions + +Stop and report if: + +- Regenerating the snapshot creates a new migration directory or changes unrelated snapshots. +- The live schema/migration excerpts no longer match the current-state snippets above. +- The snapshot format for defaults is unclear after comparing nearby string/boolean defaults. + +## Maintenance notes + +Reviewers should check this before any follow-up Drizzle migration is generated. A stale snapshot is cheap to fix now and expensive to unwind after another migration is generated on top of it. diff --git a/plans/002-return-none-for-missing-same-entity-path.md b/plans/002-return-none-for-missing-same-entity-path.md new file mode 100644 index 00000000..5752ff65 --- /dev/null +++ b/plans/002-return-none-for-missing-same-entity-path.md @@ -0,0 +1,152 @@ +# Plan 002: Return no path for missing same-entity path requests + +> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. +> +> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/ai/src/tools/relationship.ts packages/ai/src/__tests__/relationship-tools.test.ts` +> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. + +## Status + +- **Priority**: P1 +- **Effort**: S +- **Risk**: LOW +- **Depends on**: none +- **Category**: bug +- **Planned at**: commit `1dea5eb77`, 2026-06-13 + +## Why this matters + +`get_path_between_entities` special-cases identical source and target IDs. It correctly queries the current graph for that entity, but when the entity is absent it still returns a one-line path with the caller-supplied ID and `Unknown` fields. That lets an invalid, deleted, or cross-graph ID look like a valid zero-hop path to the model. + +## Current state + +Relevant files: + +- `packages/ai/src/tools/relationship.ts` — relationship graph tools. +- `packages/ai/src/__tests__/relationship-tools.test.ts` — current mocked DB tests for path/neighbour behavior. + +Current behavior: + +```ts +// packages/ai/src/tools/relationship.ts:449-463 +if (sourceEntityId === targetEntityId) { + const [entity] = await db + .select({ + id: entityTable.id, + name: entityTable.name, + type: entityTable.type, + }) + .from(entityTable) + .where(and(eq(entityTable.graphId, graphId), eq(entityTable.id, sourceEntityId))) + .limit(1); + + return [ + "## Path", + `- ${entity?.id ?? sourceEntityId}, ${entity?.name ?? "Unknown"}, ${entity?.type ?? "Unknown"}`, + ].join("\n"); +} +``` + +Existing test style: + +```ts +// packages/ai/src/__tests__/relationship-tools.test.ts:44-52 +const { getNeighboursTool, getPathBetweenTool } = await import("../tools/relationship"); + +async function executeTool(tool: { execute?: (input: unknown) => Promise }, input: unknown) { + if (!tool.execute) { + throw new Error("Tool has no execute function"); + } + + return tool.execute(input); +} +``` + +Repo conventions: + +- Tests use `bun:test`, module mocks, and result text assertions. +- Keep tool output stable and compact; existing no-path wording is `## Path\n- none found within 5 hops`. +- Root verification commands available: `bun run test`, `bun run lint`. `bun run test` passed at planning time; `bun run lint` exited 0 with one existing frontend warning. + +## Commands you will need + +| Purpose | Command | Expected on success | +|---|---|---| +| Targeted tests | `bun test packages/ai/src/__tests__/relationship-tools.test.ts` | exit 0; relationship tool tests pass | +| Workspace tests | `bun run test` | exit 0; all workspace tests pass | +| Lint | `bun run lint` | exit 0; no new errors | + +## Scope + +**In scope**: + +- `packages/ai/src/tools/relationship.ts` +- `packages/ai/src/__tests__/relationship-tools.test.ts` + +**Out of scope**: + +- Changing BFS path depth or direction semantics. +- Changing relationship/neighbour output formats outside this missing-entity branch. +- Adding DB constraints; that is a separate pre-existing finding. + +## Git workflow + +- Branch name suggestion: `advisor/002-same-entity-path-not-found`. +- Commit message style: `fix(ai): return no path for missing entity`. +- Do not push or open a PR unless instructed. + +## Steps + +### Step 1: Add the failing test + +Add a test in `packages/ai/src/__tests__/relationship-tools.test.ts` for `sourceEntityId === targetEntityId` where the entity lookup returns no rows: + +- Push one empty select result into `selectResults`. +- Execute `getPathBetweenTool("graph-1")` with identical IDs. +- Assert the output does not contain the requested ID as a path line. +- Assert it returns a no-path response. Prefer reusing the existing wording `## Path\n- none found within 5 hops` unless you first introduce a small helper for no-path text. + +**Verify**: `bun test packages/ai/src/__tests__/relationship-tools.test.ts` → the new test fails before Step 2. + +### Step 2: Return no path when the graph-scoped entity is absent + +In `packages/ai/src/tools/relationship.ts`, change only the same-entity branch: + +- Keep the existing graph-scoped entity query. +- If `entity` is undefined, return the same no-path wording used by the normal not-found branch. +- If `entity` exists, keep the existing zero-hop output. + +**Verify**: `bun test packages/ai/src/__tests__/relationship-tools.test.ts` → exit 0; new and existing relationship tests pass. + +### Step 3: Run repo checks + +**Verify**: `bun run test` → exit 0. + +**Verify**: `bun run lint` → exit 0; no new errors. + +## Test plan + +- New unit test in `packages/ai/src/__tests__/relationship-tools.test.ts` for missing same-ID entity. +- Existing directed/undirected path tests must still pass. + +## Done criteria + +- [ ] Missing same-ID entity returns no path, not `Unknown` entity output. +- [ ] Existing same-ID entity still returns a one-line zero-hop path. +- [ ] `bun test packages/ai/src/__tests__/relationship-tools.test.ts` exits 0. +- [ ] `bun run test` exits 0. +- [ ] `bun run lint` exits 0. +- [ ] No files outside the in-scope list are modified. +- [ ] `plans/README.md` status row updated. + +## STOP conditions + +Stop and report if: + +- The relationship tool test harness no longer uses `selectResults` or `executeTool` as shown above. +- The tool output contract for no-path responses has changed elsewhere. +- Fixing this requires changing graph access or authorization code. + +## Maintenance notes + +This is defense-in-depth for model-facing tool output. Future graph tools should never turn missing graph-scoped DB rows into `Unknown` entities unless the caller explicitly requested a best-effort external ID echo. diff --git a/plans/003-validate-repository-import-models.md b/plans/003-validate-repository-import-models.md new file mode 100644 index 00000000..ee953455 --- /dev/null +++ b/plans/003-validate-repository-import-models.md @@ -0,0 +1,171 @@ +# Plan 003: Validate repository import models before upload and enqueue + +> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. +> +> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- apps/api/src/routes/graph.ts apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts apps/api/src/lib/graph-upload-file-type.ts` +> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. + +## Status + +- **Priority**: P1 +- **Effort**: S +- **Risk**: LOW +- **Depends on**: none +- **Category**: bug +- **Planned at**: commit `1dea5eb77`, 2026-06-13 + +## Why this matters + +The regular file-upload route validates required processing models before writing files to S3 or inserting DB rows. The new repository URL route skips that validation and can create code file rows plus enqueue a workflow that later fails in the worker if required graph processing models are unavailable. Repository imports should fail before side effects, just like normal uploads. + +## Current state + +Relevant files: + +- `apps/api/src/routes/graph.ts` — repository URL route and regular file upload route. +- `apps/api/src/lib/graph-upload-file-type.ts` — existing upload model assertion helper. +- `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` — route tests with mocked uploads/workflows. + +Repository route currently proceeds directly to uploads: + +```ts +// apps/api/src/routes/graph.ts:632-646 +if (repositorySources.length === 0) { + return status(200, { ... }); +} + +const uploadedFiles: UploadedFile[] = []; +try { + for (const source of repositorySources) { + const fileId = ulid(); +``` + +Regular upload path validates first: + +```ts +// apps/api/src/routes/graph.ts:847-858 +const uploadModelResult = await Result.tryPromise(async () => { + await assertConfiguredUploadModels({ + organizationId: await getGraphOwnerModelOrganizationId({ + ownerMode: "graph", + graphId: existingGraph.id, + }), + files: supportedUpload.files, + secret: env.AUTH_SECRET, + }); +}); +if (uploadModelResult.isErr()) { + return mapGraphError(status, uploadModelResult.error); +} +``` + +Existing repository route test expectations: + +```ts +// apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts:398-424 +expect(response.status).toBe(200); +expect(insertedFileValues.map((file) => file.type)).toEqual(["code", "code"]); +expect(workflowInputs).toEqual([ + { + graphId: "graph-1", + fileIds: body.data.addedFiles.map((file: { id: string }) => file.id), + processRunId: "process-run-1", + code: { kind: "repository" }, + }, +]); +``` + +Repo conventions: + +- API routes use `Result.tryPromise` and `mapGraphError` for expected async errors. +- Tests use `bun:test` and module-level arrays to assert no side effects. +- Root verification commands available: `bun run test`, `bun run lint`. `bun run test` passed at planning time; `bun run lint` exited 0 with one existing frontend warning. + +## Commands you will need + +| Purpose | Command | Expected on success | +|---|---|---| +| Targeted API tests | `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` | exit 0; graph route upload tests pass | +| Workspace tests | `bun run test` | exit 0; all workspace tests pass | +| Lint | `bun run lint` | exit 0; no new errors | + +## Scope + +**In scope**: + +- `apps/api/src/routes/graph.ts` +- `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` +- `apps/api/src/lib/graph-upload-file-type.ts` only if a tiny shared type/helper is needed + +**Out of scope**: + +- Worker model resolution. +- Repository cloning limits. +- Changing normal file upload behavior. +- Creating compatibility shims for old repository import behavior. + +## Git workflow + +- Branch name suggestion: `advisor/003-validate-repository-import-models`. +- Commit message style: `fix(api): validate repository import models`. +- Do not push or open a PR unless instructed. + +## Steps + +### Step 1: Add a failing route test for validation-before-side-effects + +In `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts`, extend the mocks so `assertConfiguredUploadModels` can be forced to fail for the repository URL route. Add a test that: + +- Sets the model assertion mock to reject with an error that `mapGraphError` can convert consistently. +- Calls `POST /graphs/graph-1/urls` with a valid repository URL. +- Asserts the response is an error. +- Asserts `uploadedFiles`, `insertedFileValues`, and `workflowInputs` are still empty. + +Use the existing arrays at lines 4-21 as the side-effect oracle. + +**Verify**: `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` → the new test fails before Step 2. + +### Step 2: Validate repository code imports before upload + +In `apps/api/src/routes/graph.ts`, after `repositorySources.length > 0` and before the `uploadedFiles` loop: + +- Build a minimal list of supported upload descriptors for the repository sources with `type: "code"`. +- Call `assertConfiguredUploadModels` with `organizationId` from `getGraphOwnerModelOrganizationId({ ownerMode: "graph", graphId: existingGraph.id })` and `secret: env.AUTH_SECRET`. +- Wrap with `Result.tryPromise` and return `mapGraphError(status, error)` on failure, matching the regular upload path. + +If `assertConfiguredUploadModels` intentionally only checks audio/video today, still add the call. It keeps repository imports aligned when code/text model requirements are added. + +**Verify**: `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` → exit 0. + +### Step 3: Run repo checks + +**Verify**: `bun run test` → exit 0. + +**Verify**: `bun run lint` → exit 0; no new errors. + +## Test plan + +- New route test for repository URL model-validation failure before upload/DB/workflow side effects. +- Existing repository URL success, duplicate, no-enqueue, and limit tests must continue passing. + +## Done criteria + +- [ ] Repository URL imports call the same model validation path before S3 upload and DB insert. +- [ ] Validation failure leaves `uploadedFiles`, `insertedFileValues`, and `workflowInputs` empty in the route test. +- [ ] `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` exits 0. +- [ ] `bun run test` exits 0. +- [ ] `bun run lint` exits 0. +- [ ] No files outside the in-scope list are modified. +- [ ] `plans/README.md` status row updated. + +## STOP conditions + +Stop and report if: + +- `assertConfiguredUploadModels` has been removed or changed to require browser `File` instances specifically. +- `getGraphOwnerModelOrganizationId` no longer accepts graph-owned uploads. +- The test harness cannot observe upload/DB/workflow side effects without broad rewrites. + +## Maintenance notes + +This keeps all graph file ingestion paths consistent. If code imports later require a text model explicitly, this route will already have the right pre-side-effect validation point. diff --git a/plans/004-sanitize-repository-url-errors.md b/plans/004-sanitize-repository-url-errors.md new file mode 100644 index 00000000..15e3210f --- /dev/null +++ b/plans/004-sanitize-repository-url-errors.md @@ -0,0 +1,178 @@ +# Plan 004: Sanitize repository URL loader errors returned to clients + +> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. +> +> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- apps/api/src/lib/repository-url.ts apps/api/src/routes/graph.ts apps/api/src/lib/__tests__/repository-url.test.ts apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` +> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. + +## Status + +- **Priority**: P2 +- **Effort**: S +- **Risk**: LOW +- **Depends on**: none +- **Category**: security +- **Planned at**: commit `1dea5eb77`, 2026-06-13 + +## Why this matters + +The repository URL loader rejects failed git commands with raw stderr. The route then returns `error.message` verbatim to the client. That leaks implementation-specific git/provider output and makes the API contract depend on git wording instead of stable KIWI error codes/messages. + +## Current state + +Relevant files: + +- `apps/api/src/lib/repository-url.ts` — URL normalization, git clone, file loading. +- `apps/api/src/routes/graph.ts` — maps repository loader errors to HTTP responses. +- `apps/api/src/lib/__tests__/repository-url.test.ts` — URL helper unit tests. +- `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` — route tests for repository URL error mapping. + +Raw git stderr source: + +```ts +// apps/api/src/lib/repository-url.ts:194-197 +finish(() => { + const stderrText = Buffer.concat(stderr).toString("utf8").trim(); + reject(new Error(stderrText || "Repository git command failed")); +}); +``` + +Verbatim HTTP response mapping: + +```ts +// apps/api/src/routes/graph.ts:109-116 +function repositoryUrlErrorResponse(statusFn: (code: number, body: unknown) => unknown, error: unknown) { + const message = error instanceof Error ? error.message : "Unsupported repository URL"; + + if (message.includes("too many") || message.includes("too much")) { + return statusFn(413, errorResponse(message, API_ERROR_CODES.UPLOAD_LIMIT_EXCEEDED)); + } + + return statusFn(400, errorResponse(message, API_ERROR_CODES.UNSUPPORTED_FILE_TYPE)); +} +``` + +Existing tests cover limits but not generic git failures: + +```ts +// apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts:478-493 +test("maps repository loader limits to upload limit responses", async () => { + repositoryLoadMode = "limit-error"; + ... + expect(response.status).toBe(413); + expect(body.code).toBe("UPLOAD_LIMIT_EXCEEDED"); + expect(uploadedFiles).toEqual([]); + expect(workflowInputs).toEqual([]); +}); +``` + +Repo conventions: + +- API error responses use `errorResponse(message, API_ERROR_CODES.X)`. +- Keep raw operational details in server logs, not response bodies. +- Never reproduce secret values in tests or fixtures. +- Root verification commands available: `bun run test`, `bun run lint`. `bun run test` passed at planning time; `bun run lint` exited 0 with one existing frontend warning. + +## Commands you will need + +| Purpose | Command | Expected on success | +|---|---|---| +| Repository helper tests | `bun test apps/api/src/lib/__tests__/repository-url.test.ts` | exit 0 | +| Route tests | `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` | exit 0 | +| Workspace tests | `bun run test` | exit 0 | +| Lint | `bun run lint` | exit 0; no new errors | + +## Scope + +**In scope**: + +- `apps/api/src/lib/repository-url.ts` +- `apps/api/src/routes/graph.ts` +- `apps/api/src/lib/__tests__/repository-url.test.ts` +- `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` + +**Out of scope**: + +- Changing allowed hosts or clone strategy. +- Adding aggregate repository import budgets. +- Logging secret values or raw credentials. +- Changing 413 behavior for known file/byte/count limits. + +## Git workflow + +- Branch name suggestion: `advisor/004-sanitize-repository-url-errors`. +- Commit message style: `fix(api): sanitize repository import errors`. +- Do not push or open a PR unless instructed. + +## Steps + +### Step 1: Introduce typed repository loader errors + +In `apps/api/src/lib/repository-url.ts`, add a small exported error class or discriminated helper so callers can distinguish: + +- validation errors with safe messages (`Repository URL must use HTTPS`, unsupported host, credentials, non-root URL), +- limit errors with safe messages (`too many`, `too much`), +- git/load failures with a generic safe client message. + +Do not expose raw stderr via `.message` for git command failures. If retaining stderr for logs, store it in a non-enumerable field or `cause`, and do not serialize it into responses. + +**Verify**: `bun test apps/api/src/lib/__tests__/repository-url.test.ts` → exit 0. + +### Step 2: Map only safe messages to responses + +Update `repositoryUrlErrorResponse` in `apps/api/src/routes/graph.ts`: + +- Return 413 only for typed limit errors. +- Return 400 and `UNSUPPORTED_FILE_TYPE` for invalid/unsupported repository inputs with safe validation messages. +- Return 400 and a generic message such as `Repository could not be loaded` for git/provider/load failures. +- If raw stderr is logged, log server-side only and keep values out of tests/plans. + +**Verify**: `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` → existing limit test still passes. + +### Step 3: Add route coverage for generic git failure sanitization + +Extend `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` with a repository loader mode that throws an error representing raw git stderr. Assert: + +- Response status is 400. +- Response code is `UNSUPPORTED_FILE_TYPE`. +- Response message is the generic sanitized text. +- The raw stderr string does not appear in the response body. +- `uploadedFiles` and `workflowInputs` stay empty. + +**Verify**: `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` → exit 0. + +### Step 4: Run repo checks + +**Verify**: `bun run test` → exit 0. + +**Verify**: `bun run lint` → exit 0; no new errors. + +## Test plan + +- Unit tests for repository URL validation should keep safe validation messages. +- Route test for generic git failure must prove raw stderr is not returned. +- Existing limit-error route test must continue returning 413. + +## Done criteria + +- [ ] Raw git stderr cannot flow into `error.message` returned by `repositoryUrlErrorResponse`. +- [ ] Limit errors still return `UPLOAD_LIMIT_EXCEEDED` with safe limit text. +- [ ] Generic clone/load failures return a stable sanitized client message. +- [ ] `bun test apps/api/src/lib/__tests__/repository-url.test.ts` exits 0. +- [ ] `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` exits 0. +- [ ] `bun run test` exits 0. +- [ ] `bun run lint` exits 0. +- [ ] No files outside the in-scope list are modified. +- [ ] `plans/README.md` status row updated. + +## STOP conditions + +Stop and report if: + +- Existing clients/tests intentionally depend on exact raw git stderr in the HTTP response. +- Typed error handling requires changing shared API error contracts outside the in-scope files. +- You encounter credentials in logs or tests; do not copy them, report the location only. + +## Maintenance notes + +Keep provider and git diagnostics observable in server logs, but keep client responses stable. This makes future clone strategy changes possible without breaking API consumers. diff --git a/plans/005-route-code-uploads-through-code-workflow.md b/plans/005-route-code-uploads-through-code-workflow.md new file mode 100644 index 00000000..b935b029 --- /dev/null +++ b/plans/005-route-code-uploads-through-code-workflow.md @@ -0,0 +1,226 @@ +# Plan 005: Route direct code uploads through the code workflow + +> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. +> +> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/graph/src/file-type.ts apps/api/src/lib/graph-upload-file-type.ts apps/api/src/routes/graph.ts apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts apps/worker/workflows/process-file.ts apps/worker/workflows/process-files-spec.ts apps/worker/workflows/process-code-file.ts packages/graph/src/code/file-path.ts packages/graph/src/__tests__` +> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. + +## Status + +- **Priority**: P2 +- **Effort**: M +- **Risk**: MED +- **Depends on**: none +- **Category**: bug +- **Planned at**: commit `1dea5eb77`, 2026-06-13 + +## Why this matters + +The branch adds a `code` graph file type and a dedicated `process-code-file` workflow, but ordinary uploads and archive-expanded source files still infer `.ts/.tsx/.js/.jsx/.mts/.cts` as `text`. Even if inference is fixed, the parent workflow currently routes all children through the code workflow only when the whole batch has `input.code`. Users uploading source files directly would miss code-specific graph extraction, while mixed batches risk being routed all-code or all-generic. + +## Current state + +Relevant files: + +- `packages/graph/src/file-type.ts` — declares `code` but does not infer code extensions. +- `packages/graph/src/code/file-path.ts` — existing supported-code path predicate used by repository URL imports. +- `apps/api/src/lib/graph-upload-file-type.ts` — direct upload type inference. +- `apps/api/src/routes/graph.ts` — file and repository URL routes enqueue `processFilesSpec`. +- `apps/worker/workflows/process-file.ts` — parent workflow chooses child workflow. +- `apps/worker/workflows/process-code-file.ts` — code-file workflow already handles rows without repository metadata using fallbacks. + +Current file type state: + +```ts +// packages/graph/src/file-type.ts:1-21 +export const GRAPH_FILE_TYPES = [ + "pdf", + ... + "toml", + "code", + "text", +] as const; +``` + +```ts +// packages/graph/src/file-type.ts:52-187 +export function inferGraphFileType(file: Pick): GraphFileType { + ... + if (mime === "application/toml" || mime === "text/toml" || ext === "toml") { + return "toml"; + } + + return "text"; +} +``` + +Direct upload caller: + +```ts +// apps/api/src/lib/graph-upload-file-type.ts:23-31 +export function inferSupportedUploadedFiles(files: FileWithChecksum[]): UploadFileTypeCheck { + const typedFiles: SupportedFileWithChecksum[] = []; + + for (const fileWithChecksum of files) { + const type = inferGraphFileType(fileWithChecksum.file); + typedFiles.push({ ...fileWithChecksum, type }); + } + + return { ok: true, files: typedFiles }; +} +``` + +Current workflow routing is batch-wide: + +```ts +// apps/worker/workflows/process-file.ts:47-56 +function fileProcessingWorkflow(input: { graphId: string; code?: { kind: "repository" } }, fileId: string, codeManifestKey?: string) { + return input.code + ? { + spec: processCodeFile.spec, + input: { + graphId: input.graphId, + fileId, + ...(codeManifestKey ? { codeManifestKey } : {}), + }, + } +``` + +Code workflow fallback for non-repository metadata: + +```ts +// apps/worker/workflows/process-code-file.ts:34-40 +return { + fileId: file.id, + repositoryUrl: metadata?.repositoryUrl ?? `graph:${file.graphId}`, + repositoryName: metadata?.repositoryName ?? "code", + commitSha: metadata?.commitSha ?? "unknown", + path: metadata?.path ?? file.name, + content, +}; +``` + +Repo conventions: + +- Keep changes minimal and local. +- Use existing `isSupportedCodePath` rather than duplicating extension lists. +- API route tests assert workflow inputs using arrays in `graph-archive-upload-route.test.ts`. +- Root verification commands available: `bun run test`, `bun run lint`. `bun run test` passed at planning time; `bun run lint` exited 0 with one existing frontend warning. + +## Commands you will need + +| Purpose | Command | Expected on success | +|---|---|---| +| Graph tests | `bun test packages/graph/src` | exit 0 | +| API route tests | `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` | exit 0 | +| Worker tests | `bun test apps/worker` | exit 0 | +| Workspace tests | `bun run test` | exit 0 | +| Lint | `bun run lint` | exit 0; no new errors | + +## Scope + +**In scope**: + +- `packages/graph/src/file-type.ts` +- `packages/graph/src/code/file-path.ts` only if exporting/reusing a helper requires a small adjustment +- `apps/api/src/lib/graph-upload-file-type.ts` +- `apps/api/src/routes/graph.ts` +- `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` +- `apps/worker/workflows/process-file.ts` +- `apps/worker/workflows/process-files-spec.ts` +- `apps/worker/workflows/process-code-file.ts` only if its input schema needs a small metadata/manfiest adjustment +- Existing tests under `packages/graph/src/__tests__` or `packages/graph/src/code/__tests__` + +**Out of scope**: + +- Adding support for languages beyond the existing `isSupportedCodePath` set. +- Changing repository URL cloning/loading behavior. +- Changing code graph AST extraction semantics. +- Rewriting process run status handling. + +## Git workflow + +- Branch name suggestion: `advisor/005-route-code-uploads-through-code-workflow`. +- Commit message style: `fix(graph): route code uploads through code workflow`. +- Do not push or open a PR unless instructed. + +## Steps + +### Step 1: Classify supported source files as `code` + +Update `inferGraphFileType` so paths accepted by `isSupportedCodePath` return `code` before falling back to `text`. Keep structured data formats (`json`, `csv`, `xml`, `yaml`, `toml`) classified as their existing types, not code. + +Add tests for at least: + +- `src/index.ts` → `code` +- `src/component.tsx` → `code` +- `src/script.js` → `code` +- `README.md` → `text` or existing behavior +- `data.json` remains `json` + +**Verify**: `bun test packages/graph/src` → exit 0; file type tests pass. + +### Step 2: Route child workflows by actual file type, not a batch-wide flag + +Change `apps/worker/workflows/process-file.ts` so `processFiles` can process mixed batches safely: + +- Query the selected file IDs and their `type` once near the start of the parent workflow, or inside `fileProcessingWorkflow` via a small precomputed map. +- Use `processCodeFile` only for files whose DB `type` is `code`. +- Use `processFile` for all other file types. +- Keep `codeManifestKey` optional and pass it only to code-file child workflows. +- Preserve repository URL behavior: repository imports still prepare a shared manifest and code files still receive it. + +Avoid a batch-wide `input.code` switch for child selection. Keeping the `code` field as a hint for manifest preparation is acceptable; removing it is acceptable only if all API callers and tests are migrated cleanly. + +**Verify**: `bun test apps/worker` → exit 0. + +### Step 3: Enqueue direct code uploads without breaking mixed file batches + +Update `apps/api/src/routes/graph.ts` only if needed after Step 2: + +- Direct file uploads should continue enqueueing `processFilesSpec` with all added file IDs. +- Repository URL uploads should continue passing enough information to prepare a repository manifest. +- Mixed upload batches containing one `.ts` file and one non-code file must not route the non-code file through `processCodeFile`. + +Add or extend route tests in `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` for direct or archive-expanded code files. Assert code files are inserted with `type: "code"`. If the test harness can observe child workflow selection only at parent input level, cover the worker selection in worker tests instead. + +**Verify**: `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` → exit 0. + +### Step 4: Run repo checks + +**Verify**: `bun run test` → exit 0. + +**Verify**: `bun run lint` → exit 0; no new errors. + +## Test plan + +- Graph/file-type test for supported code extensions and structured-format precedence. +- Worker test for mixed batch child workflow selection: code file uses `processCodeFile`, text/PDF/etc. file uses `processFile`. +- API route test that direct or archive-expanded `.ts` files are stored as `code`. +- Existing repository URL tests must still assert `code: { kind: "repository" }` or the new equivalent manifest hint. + +## Done criteria + +- [ ] Supported TypeScript/JavaScript source paths infer `GraphFileType` `code`. +- [ ] Direct and archive-expanded code uploads are inserted with `type: "code"`. +- [ ] Mixed batches route code files to `processCodeFile` and non-code files to `processFile`. +- [ ] Repository URL imports still build repository metadata and shared manifests. +- [ ] `bun test packages/graph/src` exits 0. +- [ ] `bun test apps/worker` exits 0. +- [ ] `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` exits 0. +- [ ] `bun run test` exits 0. +- [ ] `bun run lint` exits 0. +- [ ] No files outside the in-scope list are modified. +- [ ] `plans/README.md` status row updated. + +## STOP conditions + +Stop and report if: + +- `isSupportedCodePath` accepts extensions that should remain structured document types in product behavior. +- OpenWorkflow cannot dispatch child workflows dynamically based on a precomputed type map. +- Fixing mixed-batch routing requires changing process-run schema or adding new persistent status values. + +## Maintenance notes + +The important invariant after this plan: file type decides child workflow, repository metadata only improves cross-file code graph quality. Do not reintroduce a batch-wide switch that assumes every file in one upload batch has the same processing path. diff --git a/plans/006-support-external-github-code-files.md b/plans/006-support-external-github-code-files.md new file mode 100644 index 00000000..625eb7ed --- /dev/null +++ b/plans/006-support-external-github-code-files.md @@ -0,0 +1,356 @@ +# Plan 006: Support external GitHub code files without copying them to S3 + +> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. +> +> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/db/src/tables/graph.ts apps/api/src/routes/graph.ts apps/api/src/lib/repository-url.ts apps/api/src/lib/graph-file-proxy.ts apps/worker/workflows/process-code-file.ts apps/worker/lib/code-manifest.ts packages/graph/src/code/metadata.ts packages/files/src` +> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. + +## Status + +- **Priority**: P1 +- **Effort**: L +- **Risk**: MED +- **Depends on**: `003-validate-repository-import-models`, `004-sanitize-repository-url-errors`, ideally `005-route-code-uploads-through-code-workflow` +- **Category**: feature / storage architecture +- **Planned at**: commit `1dea5eb77`, 2026-06-13 + +## Why this matters + +Repository code imports currently create one `files` row per source file and upload every selected source file into S3. For GitHub repositories this is unnecessary: code content has a stable external origin when addressed by commit SHA and path. The product needs files that can be external so code imports can link to GitHub directly, avoid S3 duplication, and still feed the worker/code graph pipeline. + +The target invariant: **internal uploaded files use S3; external GitHub code files use immutable GitHub commit+path references and are fetched only when processing/proxying requires bytes.** + +## Current state + +Relevant files: + +- `packages/db/src/tables/graph.ts` — `files` table assumes every file has a non-null S3 key. +- `apps/api/src/routes/graph.ts` — repository URL route uploads each repository source to S3 before inserting DB rows. +- `apps/api/src/lib/repository-url.ts` — clones repository, enumerates supported files, returns content. +- `packages/graph/src/code/metadata.ts` — code metadata only stores repository URL/name/commit/path. +- `apps/worker/workflows/process-code-file.ts` — code workflow reads code content from S3 using `fileData.key`. +- `apps/worker/lib/code-manifest.ts` — repository manifest preparation reads every matching code file from S3. +- `apps/api/src/lib/graph-file-proxy.ts` — file proxy streams only S3 objects. + +Current `files` schema excerpt: + +```ts +// packages/db/src/tables/graph.ts:136-148 +name: text("name").notNull(), +size: integer("file_size").notNull(), +type: text("file_type").notNull(), +mimeType: text("mime_type").notNull(), +key: text("file_key").notNull(), +checksum: text("checksum"), +deleted: boolean("deleted").default(false), +status: text("status", { enum: FILE_PROCESS_STATUS_VALUES }).notNull().default("processing"), +processStep: text("process_step", { enum: FILE_PROCESS_STEP_VALUES }).notNull().default("pending"), +processErrorCode: text("process_error_code").$type(), +tokenCount: integer("token_count").notNull().default(0), +metadata: text("metadata"), +``` + +Current repository URL route copies to S3: + +```ts +// apps/api/src/routes/graph.ts:652-667 +const upload = await putGraphFile(existingGraph.id, fileId, source.file, env.S3_BUCKET); +uploadedFiles.push({ + graphId: existingGraph.id, + fileId, + name: source.name, + size: source.size, + type: "code", + mimeType: "text/plain", + key: upload.key, + checksum: source.checksum, + metadata: serializeCodeFileMetadata({ + repositoryUrl: source.repository.url, + repositoryName: source.repository.name, + commitSha: source.repository.commitSha, + path: source.path, + }), +}); +``` + +Current code workflow reads S3: + +```ts +// apps/worker/workflows/process-code-file.ts:79-93 +const paths = getGraphFileArtifactPaths({ + graphId: input.graphId, + fileId: input.fileId, + fileKey: fileData.key, +}); +... +const source = await getFile(fileData.key, env.S3_BUCKET, "text"); +if (!source) { + throw new Error("File content not found"); +} +``` + +Current code metadata: + +```ts +// packages/graph/src/code/metadata.ts:3-30 +export type CodeFileMetadata = Omit; +... +return { + repositoryUrl: parsed.repositoryUrl, + repositoryName: parsed.repositoryName, + commitSha: parsed.commitSha, + path: parsed.path, +}; +``` + +Repo conventions: + +- Run commands from the repo root. +- Do not run `bun run db:migrate`. +- Do not hand-create migrations first; for custom/manual migrations run `bun run db:generate --custom` before editing. +- Use `better-result` / `Result.tryPromise` in routes when mapping expected async errors. +- Keep changes minimal and local; no compatibility shims unless explicitly required. +- Root verification commands available: `bun run test`, `bun run lint`. At planning time, `bun run test` passed; `bun run lint` exited 0 with one existing frontend warning. + +## Design decision + +Use explicit file storage origin columns instead of overloading `file_key` or only `metadata`. + +Add these concepts: + +- `files.storage_kind`: enum-like text, `"internal" | "external"`, default `"internal"`, not null. +- `files.external_url`: nullable text. Canonical immutable raw-content URL for external files. +- `files.external_provider`: nullable text, initially only `"github"`. +- Keep `files.file_key` non-null for now, but for external GitHub code set it to a deterministic synthetic key: `external:github:/@:`. This preserves existing key-based DB lookups, dedupe UI references, and unique indexes while making it clear the key is not an S3 key. + +External GitHub code metadata must include enough to rebuild safe URLs without trusting arbitrary user input: + +```ts +type CodeFileMetadata = { + repositoryUrl: string; + repositoryName: string; + commitSha: string; + path: string; + external?: { + provider: "github"; + rawUrl: string; // https://raw.githubusercontent.com//// + htmlUrl: string; // https://github.com///blob// + }; +}; +``` + +Security invariant: external URLs are generated by server-side normalization from allowed GitHub repository URL + commit SHA + supported code path. Never accept arbitrary external URLs from request bodies in this plan. + +## Commands you will need + +| Purpose | Command | Expected on success | +|---|---|---| +| Generate migration scaffold | `bun run db:generate --custom` | creates a new migration scaffold to edit | +| DB tests | `bun test packages/db/src/__tests__/migration-compat.test.ts` | exit 0 | +| API repository tests | `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts apps/api/src/lib/__tests__/repository-url.test.ts` | exit 0 | +| Worker code tests | `bun test apps/worker/lib/__tests__/code-manifest.test.ts packages/graph/src/code/__tests__/repository.test.ts` | exit 0 | +| Workspace tests | `bun run test` | exit 0 | +| Lint | `bun run lint` | exit 0; no new errors | + +## Scope + +**In scope**: + +- `packages/db/src/tables/graph.ts` +- New migration under `migrations/` created by `bun run db:generate --custom` +- `packages/db/src/__tests__/migration-compat.test.ts` +- `packages/graph/src/code/metadata.ts` +- `apps/api/src/lib/repository-url.ts` +- `apps/api/src/routes/graph.ts` +- `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` +- `apps/api/src/lib/__tests__/repository-url.test.ts` +- `apps/api/src/lib/graph-file-proxy.ts` +- `apps/worker/workflows/process-code-file.ts` +- `apps/worker/lib/code-manifest.ts` +- `apps/worker/lib/__tests__/code-manifest.test.ts` +- A small shared helper module if needed for external file access, preferably near existing file/code helpers. + +**Out of scope**: + +- Accepting arbitrary external URLs from users. +- Supporting private GitHub repositories or GitHub API tokens. +- Supporting GitLab/Bitbucket external raw links. Keep existing non-GitHub behavior internal/S3 unless product explicitly expands scope. +- Externalizing PDFs/images/audio/video/documents. +- Removing S3 support for normal uploads. +- Solving repository clone budget limits; plan 006 can coexist with later partial-clone/no-clone work. + +## Git workflow + +- Branch name suggestion: `advisor/006-external-github-code-files`. +- Commit message style: `feat(files): support external github code files`. +- Do not push or open a PR unless instructed. + +## Steps + +### Step 1: Add file storage origin columns + +Run `bun run db:generate --custom` from the repo root. Edit the generated migration to add: + +- `storage_kind text NOT NULL DEFAULT 'internal'` on `files`. +- `external_url text` on `files`. +- `external_provider text` on `files`. +- A check constraint requiring internal files to have no external URL/provider and external files to have both. Suggested shape: + - internal: `storage_kind = 'internal' AND external_url IS NULL AND external_provider IS NULL` + - external: `storage_kind = 'external' AND external_url IS NOT NULL AND external_provider IS NOT NULL` +- A provider check limiting `external_provider` to `github` when not null. + +Update `packages/db/src/tables/graph.ts` with matching columns and checks. Keep `key` non-null. + +Add migration compatibility tests in `packages/db/src/__tests__/migration-compat.test.ts` for the new columns and checks. + +**Verify**: `bun test packages/db/src/__tests__/migration-compat.test.ts` → exit 0. + +### Step 2: Extend code metadata and GitHub URL helpers + +In `packages/graph/src/code/metadata.ts`, extend `CodeFileMetadata` parsing/serialization to preserve optional external GitHub link metadata. Validate all external fields strictly: + +- `provider === "github"` +- `rawUrl` is HTTPS, host `raw.githubusercontent.com` +- `htmlUrl` is HTTPS, host `github.com` + +In `apps/api/src/lib/repository-url.ts`, add helpers that only emit external links for GitHub repository URLs: + +- Parse normalized repo URL `https://github.com//.git`. +- Build immutable raw URL: `https://raw.githubusercontent.com////`. +- Build immutable HTML URL: `https://github.com///blob//`. +- Build synthetic key: `external:github:/@:`. + +Use `encodeURIComponent` only where URL path construction requires it; do not double-encode slashes in repository file paths. Add unit tests for paths with spaces and nested directories. + +**Verify**: `bun test apps/api/src/lib/__tests__/repository-url.test.ts` → exit 0. + +### Step 3: Stop uploading GitHub repository code files to S3 + +In `apps/api/src/routes/graph.ts`, change only the repository URL add path: + +- For GitHub repository sources, skip `putGraphFile`. +- Insert `files` rows with: + - `type: "code"` + - `mimeType: "text/plain"` + - `key: synthetic external key` + - `storageKind: "external"` + - `externalProvider: "github"` + - `externalUrl: raw GitHub URL` + - `checksum: source.checksum` if content was available during repository enumeration + - `metadata` including `external.rawUrl` and `external.htmlUrl` +- For non-GitHub providers, keep existing S3 upload behavior unless Step 2 chose to reject externalization for them explicitly. +- Ensure cleanup code does not call `deleteFile` for external synthetic keys. + +Update `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts`: + +- Existing repository URL test should expect zero S3 uploads for GitHub imports. +- Inserted file values should include `storageKind: "external"`, external provider/url, synthetic key, and code metadata with GitHub links. +- Workflow input should still enqueue `processFilesSpec` with `code: { kind: "repository" }` unless plan 005 already changed this contract. +- Duplicate checksum behavior should still skip already-present code files. + +**Verify**: `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` → exit 0. + +### Step 4: Add a safe external content reader for workers + +Add a small helper used by `process-code-file` and `code-manifest`: + +```ts +type FileContentSource = + | { kind: "internal"; key: string } + | { kind: "external"; provider: "github"; url: string }; +``` + +Behavior: + +- Internal source: use existing `getFile(key, env.S3_BUCKET, "text")`. +- External GitHub source: fetch `external_url` with `fetch` only after validating HTTPS host `raw.githubusercontent.com`. +- Enforce a hard response size limit consistent with repository code limits. Do not read unbounded responses. +- Require `2xx` status and text content; classify failures as file processing failures. +- Do not follow redirects to non-allowlisted hosts. If the runtime follows redirects automatically, set redirect handling explicitly and validate the final URL if the Fetch API exposes it. + +Use this helper in: + +- `apps/worker/workflows/process-code-file.ts` instead of direct `getFile(fileData.key, ...)`. +- `apps/worker/lib/code-manifest.ts` instead of direct `getFile(row.key, ...)`. + +Add tests for: + +- Internal S3 path still works. +- External GitHub URL is fetched and returned. +- Non-GitHub external URL is rejected before network fetch. +- Oversized external response is rejected. + +**Verify**: `bun test apps/worker/lib/__tests__/code-manifest.test.ts` → exit 0. + +### Step 5: Make file proxy external-aware + +Update `apps/api/src/lib/graph-file-proxy.ts` so external GitHub files do not call S3 metadata/stream APIs: + +- Load `storageKind`, `externalProvider`, `externalUrl`, `size`, `mimeType`, `name` with the file row. +- For `storageKind === "internal"`, keep existing behavior. +- For external GitHub code files, return either: + - a `302`/`307` redirect to the immutable GitHub HTML URL from metadata for browser viewing, or + - a proxied raw response fetched from `raw.githubusercontent.com` if current UI expects same-origin bytes. + +Prefer redirecting to the GitHub HTML URL for user-facing file open/download behavior. If the existing route requires byte-range preview, use proxied raw bytes for `Range` requests and keep the same headers. + +Add focused tests for external proxy behavior. Do not expose arbitrary external URLs. + +**Verify**: run the relevant API test file containing graph-file-proxy tests; if no file exists, add a focused test next to existing API lib tests and run it with `bun test `. + +### Step 6: Preserve source/citation behavior + +Check source references that include `file_key` still work with synthetic external keys: + +- `apps/api/src/lib/source-reference-record.ts` +- `apps/api/src/lib/source-reference.ts` +- `apps/api/src/routes/graph-files.ts` + +If UI/API output needs an external URL, add it explicitly to the response shape rather than overloading `file_key`. Keep existing `file_key` for backward-compatible identification inside this branch's clean cutover. + +**Verify**: run source-reference/graph-files tests if touched. + +### Step 7: Run repo checks + +**Verify**: `bun run test` → exit 0. + +**Verify**: `bun run lint` → exit 0; no new errors. + +## Test plan + +- DB migration compatibility test for `storage_kind`, `external_url`, `external_provider`, and checks. +- Repository URL helper tests for GitHub raw/html/synthetic-key generation. +- Route test proving GitHub repository code imports insert external file rows without S3 uploads. +- Worker tests for external GitHub content fetch and allowlist rejection. +- Proxy test for external GitHub file behavior. +- Existing repository URL duplicate/checksum tests must still pass. + +## Done criteria + +- [ ] `files` table supports explicit internal/external origin with constraints. +- [ ] GitHub repository URL imports create external code file rows and do not upload source contents to S3. +- [ ] External rows use immutable commit SHA raw/html GitHub links generated server-side. +- [ ] Worker code processing can read external GitHub code content safely. +- [ ] Code manifest preparation can include external GitHub files without S3 reads. +- [ ] File proxy/source reference behavior is defined for external files. +- [ ] Non-GitHub repository imports keep existing behavior or are explicitly rejected with a safe message; no silent partial externalization. +- [ ] `bun test packages/db/src/__tests__/migration-compat.test.ts` exits 0. +- [ ] `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts apps/api/src/lib/__tests__/repository-url.test.ts` exits 0. +- [ ] Worker/API proxy focused tests exit 0. +- [ ] `bun run test` exits 0. +- [ ] `bun run lint` exits 0. +- [ ] `plans/README.md` status row updated. + +## STOP conditions + +Stop and report if: + +- The product needs private GitHub repositories in the first version. This plan intentionally supports public immutable GitHub links only. +- The DB migration generator wants broad unrelated schema changes. +- Existing UI requires same-origin byte streaming for code files and cannot tolerate redirects to GitHub HTML URLs. +- External content fetch cannot enforce host allowlisting and response-size limits in the current runtime. +- Non-GitHub repository support must remain no-S3 in the same release; that expands provider-specific URL generation and needs a separate design decision. + +## Maintenance notes + +Do not hide external state only in `metadata`. Storage origin is a first-class file invariant because cleanup, proxying, worker reads, and dedupe all branch on it. Future providers should add provider-specific URL builders and allowlists; they should not accept arbitrary URLs from clients. diff --git a/plans/007-version-code-sources-with-valid-until.md b/plans/007-version-code-sources-with-valid-until.md new file mode 100644 index 00000000..fb737f82 --- /dev/null +++ b/plans/007-version-code-sources-with-valid-until.md @@ -0,0 +1,431 @@ +# Plan 007: Version code sources with `validUntil` so graph tools use only the latest repository snapshot + +> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. +> +> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/db/src/tables/graph.ts apps/api/src/routes/graph.ts apps/api/src/lib/graph-route.ts apps/worker/lib/save-graph.ts apps/worker/lib/regenerate-descriptions.ts apps/worker/workflows/process-file.ts apps/worker/workflows/process-code-file.ts packages/ai/src/tools/source.ts packages/ai/src/tools/entity.ts apps/api/src/lib/source-reference.ts apps/api/src/lib/chat.ts apps/api/src/lib/team-chat.ts apps/api/src/lib/graph-file-proxy.ts packages/graph/src/code/identity.ts packages/graph/src/code/repository.ts` +> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. + +## Status + +- **Priority**: P1 +- **Effort**: L +- **Risk**: HIGH +- **Depends on**: `005-route-code-uploads-through-code-workflow`, `006-support-external-github-code-files` +- **Category**: correctness / source history +- **Planned at**: commit `1dea5eb77`, 2026-06-13 + +## Why this matters + +Repository code imports are snapshots. Re-uploading the same repository at a newer commit should make the graph represent the latest code, not append new snippets to the old function evidence forever. + +Today source rows are append-only. When a function changes, the canonical deduped entity receives both old and new sources; description regeneration reads all sources for the entity, so the function description can mix stale and latest code. When a function is removed, the old entity can remain active because nothing invalidates its sources. + +The target invariant: **code entities and relationships use only current valid sources; historical sources stay in the database with `validUntil` for future change-history features.** + +## Current state + +Relevant files: + +- `packages/db/src/tables/graph.ts` — `sources` has `active` but no validity window. +- `apps/worker/lib/save-graph.ts` — inserts new sources and dedupes entities/relationships, but does not invalidate old sources. +- `apps/worker/lib/regenerate-descriptions.ts` — regenerates descriptions from every linked source, not only current sources. +- `packages/ai/src/tools/source.ts` — graph source tools filter `sources.active = true` but not source validity. +- `packages/ai/src/tools/entity.ts` — file-scoped entity listing checks for any linked source, not only valid sources. +- `apps/api/src/lib/source-reference.ts`, `apps/api/src/lib/chat.ts`, `apps/api/src/lib/team-chat.ts` — source ID lookup paths do not reject invalid sources. +- `apps/api/src/routes/graph.ts` and `apps/api/src/lib/graph-route.ts` — repository imports insert files by checksum and do not model a repository update/supersession. + +Current source schema: + +```ts +// packages/db/src/tables/graph.ts:291-314 +export const sourcesTable = pgTable.withRLS( + "sources", + { + id: text("id") + .primaryKey() + .$default(() => ulid()), + entityId: text("entity_id").references(() => entityTable.id, { onDelete: "cascade" }), + relationshipId: text("relationship_id").references(() => relationshipTable.id, { onDelete: "cascade" }), + textUnitId: text("text_unit_id") + .notNull() + .references(() => textUnitTable.id, { onDelete: "cascade" }), + active: boolean("active").notNull().default(false), + description: text("description").notNull(), + sourceChunkIds: json("source_chunk_ids") + .$type() + .notNull() + .default(sql`'[]'::json`), + embedding: vector("embedding", { dimensions: 4096 }).notNull(), + searchTsv: tsvector("search_tsv").generatedAlwaysAs(() => weightedTsvectorGenerated(["description"])), + createdAt: timestamp("created_at", { withTimezone: true, mode: "date" }).defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true, mode: "date" }) + .defaultNow() + .$onUpdate(() => sql`NOW()`), + }, +``` + +Current save behavior: + +```ts +// apps/worker/lib/save-graph.ts:118-143 +const sourceRows = [ + ...graph.entities.flatMap((entity) => + entity.sources.map((source) => ({ + id: source.id, + entityId: entity.id, + relationshipId: null, + textUnitId: source.unitId, + active: false, + description: source.description, + sourceChunkIds: source.sourceChunkIds ?? [], + embedding: EMPTY_VECTOR_SQL, + })) + ), + ...graph.relationships.flatMap((relationship) => + relationship.sources.map((source) => ({ + id: source.id, + entityId: null, + relationshipId: relationship.id, + textUnitId: source.unitId, + active: false, + description: source.description, + sourceChunkIds: source.sourceChunkIds ?? [], + embedding: EMPTY_VECTOR_SQL, + })) + ), +]; +``` + +```ts +// apps/worker/lib/save-graph.ts:199-200 +for (const chunk of chunkItems(rows.sourceRows)) { + await tx.insert(sourcesTable).values(chunk).onConflictDoNothing(); +} +``` + +Current description regeneration uses stale sources too: + +```ts +// apps/worker/lib/regenerate-descriptions.ts:148-163 +const sources = await db + .select({ + id: sourcesTable.id, + entityId: sourcesTable.entityId, + description: sourcesTable.description, + }) + .from(sourcesTable) + .where( + and( + inArray( + sourcesTable.entityId, + entities.map((entity) => entity.id) + ), + isNotNull(sourcesTable.entityId) + ) + ); +``` + +Current source tool validity is only `active`: + +```ts +// packages/ai/src/tools/source.ts:205-207 +if (terms.length === 0) { + const clauses = [eq(sourcesTable.active, true), eq(filesTable.graphId, graphId)]; +``` + +```ts +// packages/ai/src/tools/source.ts:306-310 +from sources source +inner join text_units text_unit on text_unit.id = source.text_unit_id +inner join files file on file.id = text_unit.file_id +where source.active = true + and file.graph_id = ${graphId} +``` + +Current code identity makes dedupe necessary across commits: + +```ts +// packages/graph/src/code/identity.ts:8-13 +export function fileEntityName(file: Pick): string { + return `${file.repositoryUrl}:${file.path}`; +} + +export function fileEntityId(repositoryUrl: string, commitSha: string, filePath: string): string { + return stableId("code_file", repositoryUrl, commitSha, filePath); +} +``` + +The entity name excludes commit, but the generated ID includes commit. `save-graph.ts` dedupes entities by graph/type/normalized name and rewrites new sources to the canonical active entity, so changed functions append new sources to the old canonical entity unless old sources are invalidated. + +Repo conventions: + +- Run commands from the repo root. +- Do not run `bun run db:migrate`. +- Do not hand-create migrations first; for custom/manual migrations run `bun run db:generate --custom` before editing. +- Use Drizzle schema + migration compatibility tests for schema changes. +- Keep changes local; prefer shared helpers for repeated validity predicates. +- Root verification commands available: `bun run test`, `bun run lint`. At planning time, `bun run test` passed; `bun run lint` exited 0 with one existing frontend warning. + +## Design decision + +Add `sources.valid_until` as the history boundary. Do not delete old code sources. + +Definition: + +- A **current source** is `sources.active = true AND sources.valid_until IS NULL`. +- A **pending current source** is `sources.active = false AND sources.valid_until IS NULL` and exists only between graph save and description/embedding activation. +- An **historical source** has `sources.valid_until IS NOT NULL`. Historical rows keep their text unit, chunk IDs, description, embedding, and active flag for future history queries, but current graph tools ignore them. + +This plan keeps `entities` and `relationships` as the current graph surface: + +- If a subject has one or more current sources, regenerate its description from current sources only and keep it active. +- If a code subject has no current sources after a repository update, deactivate it so graph tools stop returning removed functions/edges. + +## Commands you will need + +| Purpose | Command | Expected on success | +|---|---|---| +| Generate migration scaffold | `bun run db:generate --custom` | creates a new migration scaffold to edit | +| DB migration tests | `bun test packages/db/src/__tests__/migration-compat.test.ts` | exit 0 | +| Worker focused tests | `bun test apps/worker/lib apps/worker/workflows` | exit 0 | +| AI tool tests | `bun test packages/ai/src/__tests__` | exit 0 | +| API source tests | `bun test apps/api/src/lib/__tests__ apps/api/src/routes/__tests__` | exit 0 | +| Workspace tests | `bun run test` | exit 0 | +| Lint | `bun run lint` | exit 0; no new errors | + +## Scope + +**In scope**: + +- `packages/db/src/tables/graph.ts` +- New migration under `migrations/` created by `bun run db:generate --custom` +- `packages/db/src/__tests__/migration-compat.test.ts` +- `apps/worker/lib/save-graph.ts` +- `apps/worker/lib/regenerate-descriptions.ts` +- `apps/worker/workflows/process-file.ts` +- `apps/worker/workflows/process-code-file.ts` +- `apps/api/src/routes/graph.ts` +- `apps/api/src/lib/graph-route.ts` +- `packages/ai/src/tools/source.ts` +- `packages/ai/src/tools/entity.ts` +- `packages/ai/src/tools/correction.ts` if corrections can target stale source IDs +- `apps/api/src/lib/source-reference.ts` +- `apps/api/src/lib/chat.ts` +- `apps/api/src/lib/team-chat.ts` +- Tests for the touched modules + +**Out of scope**: + +- Building the webhook integration. +- Building a user-facing history UI. +- Arbitrary non-code document source versioning. +- Changing source IDs already emitted in old chat messages beyond making invalid IDs no longer resolve as current citations. +- Rewriting code entity identity to remove commit SHA from IDs. Dedupe by name remains the current bridge across commits. + +## Git workflow + +- Branch name suggestion: `advisor/007-version-code-sources-valid-until`. +- Commit message style: `feat(graph): version code sources with validUntil`. +- Do not push or open a PR unless instructed. + +## Steps + +### Step 1: Add `sources.valid_until` + +Run `bun run db:generate --custom` from the repo root. Edit the generated migration to add: + +```sql +ALTER TABLE "sources" ADD COLUMN "valid_until" timestamp with time zone; +``` + +Update `packages/db/src/tables/graph.ts`: + +- Add `validUntil: timestamp("valid_until", { withTimezone: true, mode: "date" })` to `sourcesTable`. +- Keep existing `active` semantics; do not replace it with `validUntil`. +- Add indexes for current source lookups. Suggested indexes: + - `sources_entity_current_id_idx` on `(entity_id, active, id)` where `valid_until IS NULL`. + - `sources_relationship_current_id_idx` on `(relationship_id, active, id)` where `valid_until IS NULL`. + - `sources_current_id_idx` on `(active, id)` where `valid_until IS NULL`. + +Extend `packages/db/src/__tests__/migration-compat.test.ts` so the migration and snapshot include `sources.valid_until` and the current-source indexes. + +**Verify**: `bun test packages/db/src/__tests__/migration-compat.test.ts` → exit 0. + +### Step 2: Centralize the current-source predicate + +Add a tiny helper where it can be imported without cycles. Preferred options: + +- `packages/db/src/source-validity.ts`, exporting Drizzle predicates/helpers, or +- local helpers in each package if the DB package should avoid query helpers. + +The helper must express: + +```ts +source.active = true AND source.validUntil IS NULL +``` + +For raw SQL in `packages/ai/src/tools/source.ts`, use the exact SQL predicate: + +```sql +source.active = true AND source.valid_until IS NULL +``` + +Do not use only `active` in any current source query after this plan. + +**Verify**: no command yet; this step is covered by later tests. + +### Step 3: Make repository imports represent a latest snapshot + +Update the repository URL add flow so uploading the same repository is an update, not an append-only import. + +In `apps/api/src/routes/graph.ts` / `apps/api/src/lib/graph-route.ts`: + +- Detect repository scope by normalized `repository.url`. +- For repository imports, do not skip files solely because their checksum already exists from a previous commit of the same repository. Latest snapshots need rows for unchanged files too, especially because external GitHub URLs include the latest commit SHA. +- Create a new file row for every supported file in the latest repository snapshot. +- Keep current duplicate protection for repeated URLs within the same request. +- Mark older non-deleted code file rows for the same graph + repository URL as `deleted = true` only after the new file rows and process run have been committed successfully. Do not call the delete-file workflow; old text units and sources must remain for history. +- If a DB transaction fails, do not mark old files deleted and clean up only internal S3 uploads. + +Implementation hint: repository URL is currently only inside `files.metadata`. If querying JSON text is too brittle, add a small repository import manifest table or include a stable repository scope field in code file metadata parsing before building the SQL. Keep this plan scoped to repository code imports only. + +**Verify**: route tests should prove a second import of `https://github.com/acme/widgets` creates new file rows for the latest commit and marks previous repository file rows deleted without deleting their text units/sources. + +### Step 4: Invalidate superseded sources when a code subject is refreshed + +Update `apps/worker/lib/save-graph.ts` after inserts and dedupe have rewritten new sources to canonical entity/relationship IDs: + +- Capture inserted source IDs from `rows.sourceRows`. +- Query those inserted source rows after dedupe to get their canonical `entityId` / `relationshipId` and the file metadata of their text unit. +- For inserted sources whose file is `type = 'code'`, set `valid_until = NOW()` on older current code sources for the same canonical `entityId` or `relationshipId`, excluding the newly inserted source IDs. +- Scope invalidation to code sources only by joining `sources -> text_units -> files` and requiring `files.file_type = 'code'`. +- Do not invalidate sources from PDFs/docs/manual suggestions that happen to mention the same entity. + +This handles the main case: same function or relationship appears in a changed file. The new source becomes the only current source for that function/edge; old snippets remain historical. + +**Verify**: add a worker/lib test that saves two code graphs for the same function with different snippets. After the second save + description activation, the first source has `validUntil` set and the second source remains current. + +### Step 5: Invalidate removed functions/edges at repository update finalization + +A function removed from the latest repository snapshot has no new source, so Step 4 will not touch it. Add a repository update finalizer in the parent code batch flow (`apps/worker/workflows/process-file.ts`) after all child code workflows for a repository snapshot complete successfully: + +- Identify repository URL and commit SHA for the current batch from code file metadata. +- Identify the latest file IDs in this process run. +- Find current code sources for the same graph + repository URL whose text unit belongs to older file rows not in the latest file ID set. +- Set `valid_until = NOW()` for those sources. +- Collect affected entity and relationship IDs. +- Trigger `updateDescriptionsSpec` for affected IDs so removed functions/edges are deactivated or descriptions rebuilt from remaining current sources. + +If any child code workflow fails terminally, do not run the repository finalizer. Keep old sources current until a complete latest snapshot is available. + +**Verify**: add a process-files workflow test or lower-level finalizer test covering a repository update where `old.ts#removedFunction` has no counterpart in the new file set; after finalization its source is invalid and the entity is inactive or absent from source tools. + +### Step 6: Regenerate descriptions from current sources only + +Update `apps/worker/lib/regenerate-descriptions.ts`: + +- Select only `sources.validUntil IS NULL` for entity and relationship source lists. +- `updateSourceEmbeddingsBatch` should only activate source IDs that still have `validUntil IS NULL`. +- If an entity/relationship has zero current sources: + - Set `active = false`. + - Clear or retain description consistently. Prefer clearing to `""` and setting embedding to `EMPTY_VECTOR_SQL` if the schema requires a non-null vector. + - Do not call the LLM for it. +- If it has current sources, regenerate from those current sources only and set active true. + +**Verify**: add tests for: + +- Entity description after update includes only the new source description. +- Entity with only invalidated sources becomes inactive. +- Relationship equivalent behavior. + +### Step 7: Filter graph tools and citation lookups to current sources + +Update current-source consumers: + +- `packages/ai/src/tools/source.ts` + - `get_entity_sources`, `get_relationship_sources`, semantic source search, and `get_source_file_metadata` must require `source.valid_until IS NULL` plus `source.active = true`. + - Add `file.deleted = false` where appropriate so superseded repository file rows do not leak through current tools. +- `packages/ai/src/tools/entity.ts` + - File-scoped `list_entities` subquery must require current sources, not any historical source. +- `packages/ai/src/tools/correction.ts` + - Reject correction suggestions for historical sources. +- `apps/api/src/lib/source-reference.ts` + - `loadSourceReference` and batch loading must reject historical sources for normal citation resolution. +- `apps/api/src/lib/chat.ts` and `apps/api/src/lib/team-chat.ts` + - Source ID lookups used by chat citation normalization must reject historical sources. + +Do not add history-query behavior in this plan. Historical sources should be retained but invisible to current tools. + +**Verify**: add/extend tests proving an invalidated source ID is not returned by source tools and does not resolve as a current citation. + +### Step 8: Keep history rows intact + +Confirm invalidation does not delete: + +- `sources` +- `text_units` +- old `files` rows for code snapshots + +Older file rows may be marked `deleted = true` for current file-list behavior, but text/source rows stay available for future history. Do not call `deleteProjectFile` for superseded repository snapshots. + +**Verify**: in tests, after repository update, old source/text unit rows still exist with `validUntil` set. + +### Step 9: Run repo checks + +**Verify**: `bun run test` → exit 0. + +**Verify**: `bun run lint` → exit 0; no new errors. + +## Test plan + +Add focused tests before broad checks: + +1. DB migration compatibility: + - `sources.valid_until` exists in migration and snapshot. + - Current-source indexes exist. +2. Save graph source invalidation: + - Same code function, changed snippet → old source `validUntil` set, new source current. + - Same relationship, changed snippet → old source `validUntil` set, new source current. +3. Repository update finalization: + - Removed function's only source is invalidated when a full newer snapshot succeeds. + - Finalizer does not run when any child workflow fails. +4. Description regeneration: + - Descriptions use only current sources. + - Entity/relationship with no current sources becomes inactive. +5. Tool filtering: + - `get_entity_sources` and `get_relationship_sources` never return invalidated sources. + - `get_source_file_metadata` ignores invalidated source IDs. + - File-scoped entity listing ignores invalidated sources. +6. API citation/source reference: + - Invalidated source IDs do not resolve as current citations. + +## Done criteria + +- [ ] `sources.valid_until` exists in Drizzle schema, migration SQL, and snapshot. +- [ ] Current-source queries require `active = true AND valid_until IS NULL`. +- [ ] Re-uploading a repository creates a latest snapshot rather than deduping away unchanged files by checksum alone. +- [ ] Changed functions invalidate previous code sources for the canonical function entity and store new current sources. +- [ ] Removed functions/edges from a successful latest repository snapshot stop appearing in graph tools. +- [ ] Entity/relationship descriptions are regenerated from current sources only. +- [ ] Historical source/text rows remain in the database for future history features. +- [ ] Current graph source tools and citation resolvers ignore invalidated sources. +- [ ] `bun test packages/db/src/__tests__/migration-compat.test.ts` exits 0. +- [ ] Worker focused tests exit 0. +- [ ] AI/API source focused tests exit 0. +- [ ] `bun run test` exits 0. +- [ ] `bun run lint` exits 0. +- [ ] `plans/README.md` status row updated. + +## STOP conditions + +Stop and report if: + +- Source history must preserve old GitHub commit URLs per source but the current schema only stores repository metadata on `files`. That may require source-level metadata before this plan is safe. +- Repository update processing cannot reliably know when a full snapshot has succeeded. Do not invalidate removed functions on partial success. +- Drizzle migration generation produces broad unrelated schema changes. +- Existing UI/API contracts require old source IDs from previous chat messages to remain resolvable as current citations. +- Any implementation would delete old sources/text units instead of marking `validUntil`. + +## Maintenance notes + +`active` means “embedded/generated and eligible if otherwise current.” `validUntil` means “superseded by a newer code snapshot.” Future history features should query `validUntil IS NOT NULL` explicitly and should not overload current source tools with historical data. diff --git a/plans/008-provider-connectors-for-repository-graphs.md b/plans/008-provider-connectors-for-repository-graphs.md new file mode 100644 index 00000000..278d2900 --- /dev/null +++ b/plans/008-provider-connectors-for-repository-graphs.md @@ -0,0 +1,731 @@ +# Plan 008: Provider connectors for private repository graphs and webhook-driven updates + +> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. +> +> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/db/src/tables apps/api/src/server.ts apps/api/src/routes apps/api/src/lib apps/worker/workflows apps/worker/lib packages/contracts/src apps/frontend/app apps/frontend/components apps/frontend/lib` +> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. + +## Status + +- **Priority**: P1 +- **Effort**: XL +- **Risk**: HIGH +- **Depends on**: `006-support-external-github-code-files`, `007-version-code-sources-with-valid-until` +- **Category**: feature / integrations / repository sync +- **Planned at**: commit `1dea5eb77`, 2026-06-13 + +## Why this matters + +The URL import path still starts from a public HTTPS repository URL and historically used `git clone`. That cannot cover private repositories cleanly and cannot keep a selected branch up to date. The product needs first-class provider connectors: + +1. A system admin creates an instance-level connector app for GitHub/GitLab. +2. A graph manager (organization admin, team admin, or team moderator) connects that app to repositories they control. +3. KIWI lists accessible repositories and branches through the provider API. +4. The user creates a graph from a selected repo + branch. +5. Provider push webhooks enqueue rebuilds when that branch moves, so the graph stays current. + +Plan 007 supplies the source invalidation model (`validUntil`) needed to replace old function evidence with latest-branch evidence instead of accumulating stale snippets. + +## Current state + +Relevant files: + +- `packages/auth/src/permissions.ts` — graph management permission vocabulary. +- `apps/api/src/lib/team-access.ts` — organization admins, team admins, and team moderators can create/manage team graphs. +- `apps/api/src/lib/graph-access.ts` — graph create/file-manage authorization helpers. +- `apps/api/src/routes/graph.ts` — graph creation and URL-based repository import. +- `apps/api/src/lib/repository-url.ts` — public URL repo loader, external GitHub URL helper, still provider-agnostic by URL. +- `apps/api/src/server.ts` — all current routes are mounted after `authMiddleware`; webhooks need a verified unauthenticated route before auth. +- `apps/worker/workflows/process-files-spec.ts` — parent workflow already accepts `code: { kind: "repository" }`. +- `apps/worker/worker.ts` — workflow registration point. +- `packages/db/src/tables/graph.ts` and `packages/db/src/tables/auth.ts` — graph/team/user schema. +- `packages/ai/src/models.ts` — existing AES-GCM-style encrypted credential storage pattern to reuse for connector secrets. +- `apps/frontend/app/(app)/settings/page.tsx` and `apps/frontend/components/settings/sections.tsx` — system-admin UI guard pattern. +- `apps/frontend/lib/api/client.ts` and `apps/frontend/lib/api/projects.ts` — frontend API client conventions. + +Current permission model: + +```ts +// packages/auth/src/permissions.ts:16-18 +group: ["create", "update", "delete", "view:all", "view", "add:user", "remove:user", "list:user"], +graph: ["view", "create", "update", "delete", "add:file", "delete:file", "list:file"], +chat: ["create"], +``` + +Current team graph manager rule: + +```ts +// apps/api/src/lib/team-access.ts:140-147 +export async function requireTeamGraphCreateAccess(user: AuthUser, teamId: string) { + const access = await requireTeamAccess(user, teamId); + if (access.organizationAdmin || access.role === "admin" || access.role === "moderator") { + return access; + } + + throw new Error(API_ERROR_CODES.FORBIDDEN); +} +``` + +Current graph creation owner resolution: + +```ts +// apps/api/src/routes/graph.ts:267-289 +const ownerResult = await Result.tryPromise(async () => { + if (body.teamId) { + const access = await assertCanCreateTeamGraph(user, body.teamId); + return { + ownerMode: "team" as const, + organizationId: access.team.organizationId, + teamId: body.teamId, + }; + } + + if (body.graphId) { + await assertCanCreateUnderParentGraph(user, body.graphId); + return { + ownerMode: "graph" as const, + graphId: body.graphId, + }; + } + + const access = await assertCanCreateTopLevelGraph(user); + return { + ownerMode: "organization" as const, + organizationId: access.organizationId, + }; +}); +``` + +Current URL repository add path enqueues code processing: + +```ts +// apps/api/src/routes/graph.ts:807-812 +const handle = await ow.runWorkflow(processFilesSpec, { + graphId: existingGraph.id, + fileIds: result.addedFiles.map((file) => file.id), + processRunId: result.processRunId, + code: { kind: "repository" }, +}); +``` + +Current repository loader still starts with a clone: + +```ts +// apps/api/src/lib/repository-url.ts:161-169 +export async function loadRepositoryFromUrl(input: string): Promise { + const repository = normalizeRepositoryUrl(input); + const tempDir = await mkdtemp(path.join(tmpdir(), "kiwi-repository-")); + const repoPath = path.join(tempDir, "repo"); + + try { + await runGit(["clone", "--depth", "1", "--", repository.url, repoPath], tempDir); + const commitSha = (await runGit(["rev-parse", "HEAD"], repoPath)).trim(); +``` + +Current route mount order means webhook routes must be special-cased: + +```ts +// apps/api/src/server.ts:53-61 +.use(mcpRoute) +.use(authMiddleware) +.use(authRoute) +.use(chatRoute) +... +.use(graphRoute) +``` + +Existing encryption pattern: + +```ts +// packages/ai/src/models.ts:162-174 +export function encryptModelCredentials(credentials: ModelCredentials, secret: string): string { + const iv = randomBytes(IV_BYTE_LENGTH); + const cipher = createCipheriv(ENCRYPTION_ALGORITHM, deriveEncryptionKey(secret), iv, { + authTagLength: AUTH_TAG_BYTE_LENGTH, + }); + ... +} + +export function decryptModelCredentials(value: string, secret: string): ModelCredentials { +``` + +Provider docs observed during planning: + +- GitHub Apps create installation access tokens with `POST /app/installations/{installation_id}/access_tokens`; tokens expire and can be scoped to repositories and permissions. `contents: read` is the critical permission for reading repository files. +- GitHub Apps list accessible repositories with `GET /installation/repositories` using an installation token. +- GitHub push webhooks include delivery IDs and push ref/after commit data; verify `X-Hub-Signature-256` with the app webhook secret. +- GitLab push hooks include `event_name: "push"`, `ref: "refs/heads/"`, `after`, project path/id, and headers such as `X-Gitlab-Event`, `X-Gitlab-Webhook-UUID`, and `X-Gitlab-Token`. + +Repo conventions: + +- Run commands from the repo root. +- Do not run `bun run db:migrate`. +- Do not hand-create migrations first; for custom/manual migrations run `bun run db:generate --custom` before editing. +- Use `Result.tryPromise` in API routes for expected async errors. +- Keep comments rare; prefer small helper modules over duplicated provider logic. +- Root verification commands available: `bun run test`, `bun run lint`. At planning time, `bun run test` passed; `bun run lint` exited 0 with one existing frontend warning. + +## Design decision + +Build a provider connector layer rather than extending the public URL import path. + +Core concepts: + +- **Connector**: instance-level provider app config created by a system admin. Example: one GitHub App for this KIWI instance. +- **Connector installation/account**: a provider authorization connected by a graph manager to a KIWI owner scope. GitHub installation ID; GitLab OAuth account/project-token context. +- **Repository binding**: one KIWI graph tracks one provider repository + one branch. This binding drives manual sync and webhook sync. +- **Webhook event ledger**: stores delivery IDs and enqueue results so retries are idempotent. + +Provider support for this plan: + +- GitHub: full end-to-end implementation with GitHub App manifest/create flow, installation flow, repo/branch listing, content fetch, and push webhook sync. +- GitLab: implement the same DB/API/provider interface and manual sysadmin connector config, plus OAuth/project listing and webhook verification if feasible. If GitLab project webhook creation needs a broader `api` scope or self-managed-instance behavior diverges, finish GitHub and leave GitLab behind a `disabled` connector state with explicit STOP/report rather than inventing an unsafe token flow. + +Security invariants: + +- Connector secrets, private keys, OAuth client secrets, webhook secrets, and refresh tokens are encrypted at rest with the existing `AUTH_SECRET`-derived pattern. Never log them. +- Webhook endpoints are unauthenticated but must verify provider signatures/tokens before reading business fields or enqueueing work. +- Users can only create repository graphs in scopes where they can already manage graphs: organization admin, team admin, team moderator. +- Do not accept arbitrary raw URLs from users. Repository content comes from provider API clients using stored connector authorization. +- Webhook processing is idempotent by provider delivery ID and by `(repositoryBindingId, commitSha)`. + +## Commands you will need + +| Purpose | Command | Expected on success | +|---|---|---| +| Generate migration scaffold | `bun run db:generate --custom` | creates a new migration scaffold to edit | +| DB migration tests | `bun test packages/db/src/__tests__/migration-compat.test.ts` | exit 0 | +| API connector tests | `bun test apps/api/src/routes/__tests__ apps/api/src/lib/__tests__` | exit 0 | +| Worker connector tests | `bun test apps/worker/lib apps/worker/workflows` | exit 0 | +| Frontend tests | `bun test apps/frontend` | exit 0 | +| Workspace tests | `bun run test` | exit 0 | +| Lint | `bun run lint` | exit 0; no new errors | + +## Scope + +**In scope**: + +- New connector DB tables and migration. +- Shared provider client/helpers, preferably a new `packages/connectors` workspace package if both API and worker need provider logic. +- New API routes under `/connectors` plus public webhook routes under `/connectors/:id/webhooks/:provider` mounted before auth middleware. +- GitHub App manifest setup UI at `/connectors/github/new` for system admins. +- Connector installation/connect UI at `/connectors/:id/connect` for graph managers. +- Repository/branch list endpoints. +- Graph creation from repository + branch. +- Worker workflow to sync a connector repository snapshot and enqueue/process code files without `git clone`. +- Push webhook handling that enqueues updates for matching branch bindings. +- Contract/frontend types and minimal UI to select connector, repo, branch, and owner scope. + +**Out of scope**: + +- Pull request preview graphs. +- User-facing history UI for old function versions. +- Write access to repositories. +- GitHub Checks/Statuses. +- Fine-grained per-file incremental graph updates; rebuild the selected branch snapshot and rely on plan 007 source invalidation. +- Supporting arbitrary provider URLs or Bitbucket. +- Moving existing URL import users to connectors automatically. + +## Git workflow + +- Branch name suggestion: `advisor/008-provider-connectors-repository-sync`. +- Commit message style: `feat(connectors): add repository graph sync`. +- Do not push or open a PR unless instructed. + +## Steps + +### Step 1: Add connector schema + +Run `bun run db:generate --custom` from the repo root. Add Drizzle tables in `packages/db/src/tables/connectors.ts` or the repo's preferred table module. Export them from the package index if needed. + +Suggested tables: + +#### `connectors` + +- `id text primary key` +- `provider text not null` enum-like: `github | gitlab` +- `name text not null` +- `slug text not null unique` +- `status text not null default 'active'` enum-like: `draft | active | disabled` +- `app_id text` — GitHub App ID or GitLab application ID. +- `client_id text` +- `encrypted_credentials text not null` — provider secrets; shape validated in code. +- `webhook_secret_encrypted text not null` +- `created_by_user_id text references user(id) on delete set null` +- timestamps + +#### `connector_installations` + +- `id text primary key` +- `connector_id text not null references connectors(id) on delete cascade` +- `provider text not null` +- `provider_installation_id text not null` — GitHub installation ID; GitLab account/project auth identifier. +- `provider_account_login text not null` +- `provider_account_type text` — `user | organization | group`. +- `organization_id text references organization(id) on delete cascade` +- `team_id text references team(id) on delete cascade` +- `installed_by_user_id text references user(id) on delete set null` +- `encrypted_credentials text` — GitLab OAuth refresh/access token if needed; null for GitHub App installations. +- `repository_selection text` — `all | selected | unknown`. +- `status text not null default 'active'` +- timestamps +- unique `(connector_id, provider_installation_id, organization_id, team_id)` +- check: either organization scope or team scope is present, not both unless team requires organization. For teams, require organization too if that matches existing graph team ownership. + +#### `repository_graph_bindings` + +- `id text primary key` +- `graph_id text not null references graphs(id) on delete cascade unique` +- `connector_installation_id text not null references connector_installations(id) on delete restrict` +- `provider text not null` +- `provider_repository_id text not null` +- `repository_full_name text not null` — `owner/repo` or GitLab `namespace/project`. +- `repository_html_url text not null` +- `branch text not null` +- `last_seen_commit_sha text` +- `last_synced_commit_sha text` +- `sync_status text not null default 'pending'` — `pending | syncing | synced | failed`. +- `sync_error_code text` +- `webhook_enabled boolean not null default true` +- timestamps +- unique `(connector_installation_id, provider_repository_id, branch)` if one branch graph per installation is desired; otherwise unique only on `graph_id` and let multiple graphs track the same branch. + +#### `connector_webhook_events` + +- `id text primary key` +- `connector_id text not null references connectors(id) on delete cascade` +- `provider text not null` +- `delivery_id text not null` +- `event_name text not null` +- `provider_repository_id text` +- `branch text` +- `commit_sha text` +- `status text not null` — `ignored | enqueued | duplicate | failed`. +- `error_code text` +- `created_at timestamp not null default now()` +- unique `(connector_id, provider, delivery_id)` + +Add migration compatibility tests for table/column/check/index presence. + +**Verify**: `bun test packages/db/src/__tests__/migration-compat.test.ts` → exit 0. + +### Step 2: Add encrypted connector credential helpers + +Create a small helper module, ideally in a new shared package if both API and worker consume it: + +- Reuse the `packages/ai/src/models.ts` encryption approach: HKDF from `AUTH_SECRET`, random IV, auth tag, versioned string. +- Supported secret shapes: + - GitHub connector credentials: `{ appId: string; privateKeyPem: string; clientId?: string; clientSecret?: string }` + - GitHub webhook secret: store separately or inside credentials, but keep a dedicated accessor for verification. + - GitLab connector credentials: `{ baseUrl: string; clientId: string; clientSecret: string }` + - GitLab installation credentials: `{ accessToken: string; refreshToken?: string; expiresAt?: string }` +- Validate decrypted shape before returning it. + +Do not export raw decrypted credentials beyond provider client factories. + +**Verify**: add unit tests for encrypt/decrypt, invalid version, invalid shape, and wrong secret failure. Run the new test file. + +### Step 3: Add provider client interfaces + +Create provider-neutral interfaces. Suggested package: `packages/connectors/src`. + +Types: + +```ts +export type ConnectorProvider = "github" | "gitlab"; + +export type ProviderRepository = { + provider: ConnectorProvider; + id: string; + fullName: string; + name: string; + htmlUrl: string; + defaultBranch: string | null; + private: boolean; +}; + +export type ProviderBranch = { + name: string; + commitSha: string; +}; + +export type ProviderCodeFile = { + path: string; + size: number; + checksum: string; + htmlUrl: string; + rawUrl?: string; + content: string; +}; +``` + +Client methods: + +- `listRepositories(installationOrAccount)` +- `listBranches(repository)` +- `loadRepositorySnapshot(repository, branch)` returning commit SHA + supported code files. +- `verifyWebhook(request)` returning normalized event fields. +- `createOrRefreshInstallationToken(...)` for GitHub. + +GitHub implementation requirements: + +- Generate a GitHub App JWT with RS256 using Node/Bun crypto and the stored private key. +- Create installation tokens with `POST /app/installations/{installation_id}/access_tokens` and request at most `contents: read` plus metadata. Do not request write permissions. +- List repositories with `GET /installation/repositories` using the installation token. +- List branches with `GET /repos/{owner}/{repo}/branches`. +- Resolve the selected branch to an immutable commit SHA before loading files. +- List tree recursively via provider API; filter with existing `isSupportedCodePath` and skipped generated directories from `repository-url.ts`. +- Fetch contents via provider API using the installation token so private repositories work. Do not use unauthenticated `git clone`. +- Enforce existing repository code limits: max files, max total bytes, max per-file bytes. + +GitLab implementation requirements: + +- Normalize `baseUrl` and use `/api/v4`. +- Use OAuth token with the minimal workable scopes. Prefer `read_api read_repository`; if project webhook creation requires `api`, make that explicit in the connector UI and plan tests. +- List projects for the connected user/account. +- List branches and repository tree/files through GitLab API. +- Verify GitLab push webhooks with `X-Gitlab-Token` against the encrypted connector webhook secret. + +**Verify**: provider unit tests with mocked `fetch` for GitHub token creation, repo listing, branch listing, content loading, and webhook verification. + +### Step 4: Add system-admin connector creation routes and UI + +Add an authenticated connector route module under `apps/api/src/routes/connectors.ts` and mount it after `authMiddleware`. + +System-admin endpoints: + +- `GET /connectors` — list active connectors visible to the current user; system admins see full config metadata but never secrets. +- `POST /connectors/github/manifest/start` — system admin only. Creates a short-lived signed state and returns a GitHub App manifest URL. +- `GET /connectors/github/manifest/callback?code&state` — system admin only. Exchanges the manifest `code` for app credentials and stores a connector. +- `POST /connectors/gitlab` — system admin only. Stores manually-created GitLab application config and webhook secret. +- `PATCH /connectors/:id` — system admin only, enable/disable/rename/rotate secrets. + +GitHub App manifest defaults: + +- App name: include KIWI instance name/host to avoid collisions. +- Homepage URL: configured frontend/base URL. +- Webhook URL: `${API_URL}/connectors/webhooks/github` or `${API_URL}/connectors/:id/webhooks/github` if known after creation. If connector ID is not known before creation, use provider-level route and resolve by app ID in the payload. +- Callback/setup URLs: + - Manifest callback: `/connectors/github/callback`. + - Installation setup URL: `/connectors/github/setup`. +- Repository permissions: `Contents: read`, `Metadata: read`. +- Events: `push`, and optionally `installation` / `installation_repositories` to keep repository access state fresh. + +Frontend: + +- Add `/connectors` page for listing connectors. +- Add `/connectors/github/new` page for system admins. It should show what permissions/events will be requested and a "Create GitHub App" button that calls the manifest-start endpoint then redirects to GitHub. +- Add `/connectors/gitlab/new` page for system admins with base URL, application ID, client secret, and webhook secret fields. +- Reuse the settings admin guard pattern from `apps/frontend/app/(app)/settings/page.tsx` for server-side system-admin protection. +- Add a system-admin settings link/section only if it does not duplicate the top-level `/connectors` page. + +**Verify**: API tests prove non-admins get 403 for creation/patch routes and system admins can create listable connectors without secrets in responses. Frontend tests cover visibility/guard helpers if added. + +### Step 5: Add graph-manager installation/connect flow + +Endpoints: + +- `GET /connectors/:id/connect?organizationId=...&teamId=...` — checks graph-management rights for the requested owner scope, creates signed state, redirects to provider installation/OAuth flow. +- GitHub callback/setup route: records `installation_id`, provider account, repository selection, and owner scope into `connector_installations`. +- GitLab OAuth callback: exchanges code for token, stores encrypted token under `connector_installations` for the owner scope. +- `GET /connectors/:id/installations` — list installations/accounts available to the current user for owner scopes they can manage. + +Authorization rules: + +- Organization scope: require organization admin. +- Team scope: require `requireTeamGraphCreateAccess`; this allows organization admin, team admin, and team moderator, matching graph creation. +- User/private graph scope is out of scope for connectors unless product explicitly asks; private provider apps create ownership ambiguity. + +GitHub redirect target: + +- Use the GitHub App installation URL for the connector app, e.g. `https://github.com/apps//installations/new?state=` after the connector stores the app slug/name. +- On callback, verify state and ensure the KIWI user still has graph-management rights for the requested scope. + +**Verify**: route tests cover org admin, team admin, team moderator allowed; team member denied; stale/invalid state denied; connector disabled denied. + +### Step 6: Add repository and branch selection endpoints + +Endpoints: + +- `GET /connectors/:id/repositories?installationId=...` — list provider repositories visible through that installation/account. +- `GET /connectors/:id/repositories/:providerRepositoryId/branches?installationId=...` — list branches. + +Requirements: + +- Check the current user can manage the installation owner scope before listing. +- Return stable provider repository IDs, full names, default branch, private flag, and HTML URL. +- Never return provider access tokens. +- Cache repository/branch lists only if cache invalidation is clear; otherwise fetch live and paginate. +- Handle provider pagination deterministically. + +Frontend: + +- Add a repository picker page under `/connectors/:id/connect` after installation completes. +- Let the user choose owner scope, installation/account, repository, and branch. +- Default branch should be preselected. + +**Verify**: mocked API tests for pagination, denied installation access, empty repositories, and branches. Frontend component tests for default branch preselection and disabled submit without repo/branch. + +### Step 7: Create graphs from selected repository branches + +Add an endpoint: + +```http +POST /connectors/:id/repository-graphs +{ + "installationId": "...", + "providerRepositoryId": "...", + "branch": "main", + "name": "optional display name", + "description": "optional", + "teamId": "optional" +} +``` + +Behavior: + +1. Authorize the requested owner scope using the same rules as graph creation. +2. Resolve repository and branch through the provider API. +3. Insert a graph row with `state = 'updating'`, organization/team owner fields matching existing graph creation semantics. +4. Insert `repository_graph_bindings` with the selected branch and latest branch commit SHA as `last_seen_commit_sha`. +5. Enqueue a new worker workflow, e.g. `syncRepositoryGraphSpec`, with `{ bindingId, reason: 'initial', commitSha }`. +6. Return graph, binding, and workflow run ID. + +Do not create all file rows synchronously in the API route. The provider API may be slow and large; the worker should load the snapshot and commit file rows/process runs. + +**Verify**: API tests prove graph row + binding transactionality, owner authorization, and enqueue failure rollback/failed state behavior. + +### Step 8: Add repository sync worker workflow + +Add `syncRepositoryGraphSpec` and implementation in `apps/worker/workflows/sync-repository-graph.ts`. Register it in `apps/worker/worker.ts`. + +Workflow input: + +```ts +{ + bindingId: string; + reason: "initial" | "webhook" | "manual"; + commitSha?: string; + deliveryId?: string; +} +``` + +Workflow behavior: + +1. Load binding, connector installation, connector credentials, and graph. +2. If binding/webhook disabled or graph missing, exit without side effects. +3. Resolve selected branch to commit SHA unless input already supplies one from a verified webhook. +4. If `lastSyncedCommitSha === commitSha`, mark webhook event duplicate/synced and exit. +5. Load repository snapshot through provider API, not `git clone`. +6. Convert each supported code file into an external `files` row: + - `storageKind: 'external'` + - `externalProvider: provider` + - `externalUrl`: provider HTML URL or raw API URL, whichever plan 006's proxy/content-source layer supports after extension. + - metadata: provider, repository full name/id, branch, commit SHA, path, html URL, and any API raw URL needed for worker reads. + - checksum: provider blob SHA/content hash. +7. Use or extend `commitGraphFileUploads` so it can create file rows + process run for these external files without S3 cleanup. +8. Mark superseded file rows for this binding as deleted only after new rows/process run commit. +9. Run `processFilesSpec` with `code: { kind: 'repository' }` or direct child workflows according to plan 005. +10. Update binding `lastSeenCommitSha`, `syncStatus`, and eventually `lastSyncedCommitSha` after processing succeeds. + +Integration with plan 007: + +- On successful full snapshot processing, old code sources for the same binding/repository/branch must get `validUntil` and stop appearing in graph tools. +- If processing fails terminally, keep previous sources current and set binding `syncStatus = 'failed'`. + +**Verify**: worker tests with mocked provider client prove no `git` process is spawned, file rows are external, older binding files are marked deleted after commit, duplicate SHA exits, and failure leaves previous binding state intact. + +### Step 9: Extend external file content/proxy handling for private providers + +Plan 006 introduced external GitHub files. Private connector repositories need credentialed reads. + +Update content-source handling: + +- Internal S3 files keep current behavior. +- Public external GitHub URL files from URL imports can keep allowlisted raw fetch behavior. +- Connector-backed external files must read through provider API using `repository_graph_bindings` + connector installation credentials, not unauthenticated raw URLs. + +Add metadata or DB columns needed to locate the binding from a file row. Preferred explicit column: + +- `files.repository_binding_id text references repository_graph_bindings(id) on delete set null` + +If adding this column, include it in the same migration or a new migration and update file insert/select helpers. + +Proxy behavior: + +- For private files, do not redirect to provider raw URLs that may leak or fail. +- Either redirect to provider HTML URL for humans or proxy raw content through KIWI after graph access checks. +- If proxying bytes, preserve size limits and content type; do not stream arbitrary provider responses without host/API validation. + +**Verify**: API proxy tests for public URL-import external file, private connector-backed external file, missing binding, disabled connector, and unauthorized graph access. + +### Step 10: Add webhook endpoint before auth middleware + +In `apps/api/src/server.ts`, mount a public webhook route before `.use(authMiddleware)`, e.g.: + +```ts +.use(connectorWebhookRoute) +.use(authMiddleware) +.use(connectorRoute) +``` + +Webhook route requirements: + +- Route: `POST /connectors/webhooks/github` and `POST /connectors/webhooks/gitlab`, or connector-specific paths if the provider can include connector ID safely. +- Read raw body for signature verification before JSON business logic. +- GitHub: verify `X-Hub-Signature-256` HMAC SHA-256 using connector webhook secret. Use `timingSafeEqual`. +- GitHub: use `X-GitHub-Event` and `X-GitHub-Delivery` for event/dedupe. +- GitLab: verify `X-Gitlab-Token` against connector webhook secret. Use `timingSafeEqual` for token bytes. +- Store a row in `connector_webhook_events` before enqueueing. Duplicate delivery returns 202 without enqueue. +- Ignore non-push events with status `ignored`. +- For push events, normalize branch from `refs/heads/` and commit from `after`; ignore branch deletes where commit is all zeroes. +- Find active bindings for `(provider repository id/full name, branch)` and enqueue `syncRepositoryGraphSpec` once per binding. +- Return 202 quickly; do not process repository contents in the API request. + +**Verify**: webhook route tests cover valid signature enqueue, invalid signature 401/403 with no DB writes, duplicate delivery no duplicate enqueue, wrong branch ignored, branch delete ignored, and multiple graph bindings enqueue separately. + +### Step 11: Add manual resync and status endpoints + +Endpoints: + +- `POST /repository-graph-bindings/:id/sync` — graph manager only; enqueue sync for current branch head. +- `GET /repository-graph-bindings/:id` — graph viewer can see sync status, provider, repo full name, branch, last seen/synced commit. +- Include binding summary in graph detail response if graph is connector-backed. + +This gives operators a recovery path when provider webhook delivery fails. + +**Verify**: API tests for allowed manager sync, viewer status, member denied sync, disabled binding denied. + +### Step 12: Update contracts and frontend API client + +Update `packages/contracts/src/routes.ts` with connector records/responses: + +- Connector list/create/update responses. +- Installation list response. +- Repository list response. +- Branch list response. +- Repository graph create response. +- Binding status response. + +Update `apps/frontend/lib/api` with typed functions: + +- `fetchConnectors` +- `startGitHubConnectorManifest` +- `createGitLabConnector` +- `fetchConnectorInstallations` +- `fetchConnectorRepositories` +- `fetchConnectorBranches` +- `createRepositoryGraph` +- `syncRepositoryGraphBinding` + +Keep API errors using existing `ApiError`/`unwrapApiResponse` patterns. + +**Verify**: frontend API client unit tests with mocked `fetch` for success and error paths. + +### Step 13: Add frontend pages + +Add pages under `apps/frontend/app/(app)/connectors`: + +- `/connectors` — connector list. System admins see create/manage actions; graph managers see connect/use actions for active connectors. +- `/connectors/github/new` — system-admin GitHub App manifest starter, with permission/event explanation. +- `/connectors/gitlab/new` — system-admin manual GitLab application config. +- `/connectors/[connectorId]/connect` — owner scope selection, provider installation/OAuth status, repository picker, branch picker, create graph button. +- Optional `/connectors/[connectorId]/repositories/new-graph` if the connect page becomes too large. + +UI requirements: + +- Reuse existing shadcn components from `components/ui`. +- Use existing `useAuth`, query hooks, and API client patterns. +- Never render secrets after submit. +- Show provider-specific permission copy: + - GitHub: Contents read, Metadata read, Push webhook. + - GitLab: read repository/API scopes needed, Push webhook token. +- After graph creation, navigate to the new project route using existing group/project routing patterns. + +**Verify**: component tests for guard/visibility, disabled submit states, default branch preselection, and successful navigation callback if existing test utilities support it. + +### Step 14: Update route/server tests and workflow registration tests + +Add/extend tests so the new public webhook route remains before auth middleware. A regression test should prove a valid webhook without a session reaches the webhook handler while invalid signatures are rejected. + +Add a worker registration test or static import test if the repo has a pattern for workflow registration; otherwise the workspace test/build will catch missing exports. + +**Verify**: `bun test apps/api/src/routes/__tests__ apps/worker` → exit 0. + +### Step 15: Run repo checks + +**Verify**: `bun run test` → exit 0. + +**Verify**: `bun run lint` → exit 0; no new errors. + +## Test plan + +Minimum focused tests before full checks: + +1. DB schema/migration tests for connector tables, binding table, webhook event ledger, and optional `files.repository_binding_id`. +2. Credential encryption tests: no plaintext secrets in selected API responses. +3. GitHub provider client tests with mocked `fetch`: + - app JWT/token request, + - installation repositories, + - branches, + - snapshot load with limits, + - push webhook signature verification. +4. GitLab provider client tests with mocked `fetch`: + - OAuth token use/refresh if implemented, + - project/branch/tree/raw file calls, + - webhook token verification. +5. Connector route authorization tests: + - system admin can create connector, + - non-admin cannot, + - org admin/team admin/team moderator can connect/install/use connector, + - team member cannot. +6. Repository graph creation tests: + - creates graph + binding + sync workflow atomically, + - enqueue failure leaves a recoverable failed graph/binding state or rolls back cleanly. +7. Webhook tests: + - valid push to selected branch enqueues sync, + - wrong branch ignored, + - duplicate delivery ignored, + - invalid signature rejected before payload trust. +8. Worker sync tests: + - no git clone, + - external code file rows created, + - private content read through provider API, + - duplicate commit exits, + - failed sync does not invalidate current sources. +9. Frontend tests for connector create/connect/repo-select forms. + +## Done criteria + +- [ ] System admins can create at least GitHub connectors from `/connectors/github/new` with prefilled app permissions/events and no secret exposure after creation. +- [ ] System admins can configure GitLab connector metadata or GitLab is explicitly disabled with a clear STOP/report if safe implementation is blocked. +- [ ] Org admins, team admins, and team moderators can connect a connector installation/account for scopes they manage. +- [ ] Team members and ordinary org members cannot connect/use repository connectors for graph creation. +- [ ] Users can list provider repos and branches through connector endpoints. +- [ ] Users can create a graph from selected repo + branch. +- [ ] Repository sync loads code through provider APIs, not `git clone`, and supports private GitHub repositories. +- [ ] Provider push webhooks enqueue sync only for matching active branch bindings. +- [ ] Webhook delivery handling is signature-verified and idempotent. +- [ ] Plan 007 source invalidation is used so rebuilds keep graph answers on the latest branch state. +- [ ] Manual resync endpoint exists for recovery. +- [ ] `bun test packages/db/src/__tests__/migration-compat.test.ts` exits 0. +- [ ] API, worker, frontend focused tests exit 0. +- [ ] `bun run test` exits 0. +- [ ] `bun run lint` exits 0. +- [ ] `plans/README.md` status row updated. + +## STOP conditions + +Stop and report if: + +- Plan 007 is not implemented; webhook rebuilds without source invalidation will accumulate stale function evidence. +- GitHub App manifest flow cannot produce/store the private key and webhook secret for this instance; do not ask sysadmins to paste secrets into random logs or responses. +- GitLab requires broad write/admin scopes to create webhooks and the product owner has not accepted that scope. +- Elysia cannot expose raw request bodies for signature verification on the webhook route. Do not verify signatures against re-serialized JSON. +- Private repository content cannot be fetched through provider APIs within existing size/rate limits. +- Provider rate limits require queueing/backoff beyond OpenWorkflow's current retry policy; report before shipping a loop that can hammer provider APIs. +- Any test fixture includes a real provider token/private key/webhook secret. Use synthetic values only. + +## Maintenance notes + +Keep URL imports and connectors separate. URL imports are convenient public one-offs; connectors are authenticated, owner-scoped, branch-bound sync sources. Future providers should implement the provider interface and webhook verifier, not special-case graph routes. History features should query invalidated sources from plan 007 explicitly; current graph tools must remain latest-only. diff --git a/plans/009-incremental-connector-repository-updates.md b/plans/009-incremental-connector-repository-updates.md new file mode 100644 index 00000000..0e2c6307 --- /dev/null +++ b/plans/009-incremental-connector-repository-updates.md @@ -0,0 +1,348 @@ +# Plan 009: Incremental connector repository updates for changed files only + +> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. +> +> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/connectors/src apps/worker/workflows/sync-repository-graph.ts apps/worker/lib/code-manifest.ts apps/worker/lib/code-repository-finalizer.ts apps/api/src/routes/connector-webhooks.ts apps/api/src/lib/graph-file-proxy.ts apps/worker/lib/file-content-source.ts packages/graph/src/code/metadata.ts` +> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. + +## Status + +- **Priority**: P1 +- **Effort**: L +- **Risk**: HIGH +- **Depends on**: `007-version-code-sources-with-valid-until`, `008-provider-connectors-for-repository-graphs` +- **Category**: performance / correctness / repository sync +- **Planned at**: commit `1dea5eb77`, 2026-06-14 + +## Why this matters + +Connector-backed repository graphs now stay fresh via manual sync and push webhooks, but every update still rebuilds the entire supported-code snapshot for the bound branch. That means: + +1. A one-file change on `main` re-reads every supported code file from the provider API. +2. KIWI creates new file rows for the whole repository instead of just the touched paths. +3. The code workflow reprocesses unchanged files, wasting worker time and provider rate limit budget. +4. Large repositories pay full-branch rebuild cost on every push even when only a handful of files changed. + +The product requirement is narrower and stricter: the initial connector import may stay full-snapshot, but **subsequent branch updates must only process changed supported code files, not silently reprocess the whole repository again**. + +## Current state + +Relevant files: + +- `packages/connectors/src/types.ts` — provider client interface; it can list repos/branches, load a full snapshot, and read one file, but it cannot compare two revisions. +- `packages/connectors/src/github.ts` and `packages/connectors/src/gitlab.ts` — current provider clients load whole supported-code snapshots and fetch per-file content. +- `apps/worker/workflows/sync-repository-graph.ts` — connector sync loads a full branch snapshot, inserts a row per file, marks all prior binding rows deleted, and processes every inserted file. +- `apps/worker/lib/code-manifest.ts` — repository code manifests are still grouped by `repositoryUrl + commitSha`, so changed-file processing assumes one full-snapshot commit scope. +- `apps/worker/lib/code-repository-finalizer.ts` — current invalidation helper is repository-wide for the latest import batch, not path-targeted. +- `apps/api/src/routes/connector-webhooks.ts` — push webhooks already enqueue one sync per bound branch and commit. +- `plans/008-provider-connectors-for-repository-graphs.md` — incremental file-level updates were explicitly left out of scope when connectors landed. + +Current provider client contract: + +```ts +// packages/connectors/src/types.ts:81-86 +export type ProviderRepositoryClient = { + readonly provider: ConnectorProvider; + listRepositories(): Promise; + listBranches(repository: ProviderRepository): Promise; + loadRepositorySnapshot(repository: ProviderRepository, branch: string, commitSha?: string): Promise; + readFile(repository: ProviderRepository, path: string, commitSha: string): Promise; +}; +``` + +Current connector sync rebuilds the whole binding snapshot: + +```ts +// apps/worker/workflows/sync-repository-graph.ts:190-235 +const insertedFiles = await tx.insert(filesTable).values(fileRows(row, snapshot, commitSha)).onConflictDoNothing().returning({ id: filesTable.id }); +... +await tx + .update(filesTable) + .set({ deleted: true }) + .where(and(eq(filesTable.repositoryBindingId, row.binding.id), notInArray(filesTable.id, insertedFiles.map((file) => file.id)))); +... +await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: created.fileIds, + processRunId: created.processRunId, + code: { kind: "repository" }, +}); +``` + +Current code-manifest scope is commit-wide: + +```ts +// apps/worker/lib/code-manifest.ts:45-65,102-104 +const selectedRepositoryKeys = new Set( + selectedRows + .map((row) => parseCodeFileMetadata(row.metadata)) + .filter((metadata) => metadata !== null) + .map(repositoryManifestScopeKey) +); +... +function repositoryManifestScopeKey(metadata: Pick): string { + return `${metadata.repositoryUrl}\0${metadata.commitSha}`; +} +``` + +Current repository-source invalidation is repo-wide for older batches: + +```ts +// apps/worker/lib/code-repository-finalizer.ts:64-68,96-103 +const candidateRows = await tx + .select({ id: filesTable.id, metadata: filesTable.metadata }) + .from(filesTable) + .where(and(eq(filesTable.graphId, options.graphId), eq(filesTable.type, "code"))); +... +await tx.execute(sql` + UPDATE sources source + SET valid_until = NOW() + FROM text_units text_unit + WHERE source.text_unit_id = text_unit.id + AND text_unit.file_id = ANY(${textArray(targets.olderFileIds)}) + AND ${currentSourceSql("source")} +`); +``` + +Current webhook route already passes one binding + commit into the sync workflow: + +```ts +// apps/api/src/routes/connector-webhooks.ts:180-191 +if (status === "enqueued" && normalized.commitSha) { + for (const binding of bindings) { + await db + .update(repositoryGraphBindingsTable) + .set({ lastSeenCommitSha: normalized.commitSha, syncStatus: "pending", syncErrorCode: null }) + .where(eq(repositoryGraphBindingsTable.id, binding.id)); + await ow.runWorkflow(syncRepositoryGraphSpec, { + bindingId: binding.id, + reason: "webhook", + commitSha: normalized.commitSha, + deliveryId, + }); + } +} +``` + +Current connector plan explicitly deferred this work: + +```md + +- Fine-grained per-file incremental graph updates; rebuild the selected branch snapshot and rely on plan 007 source invalidation. +``` + +Provider docs observed during planning: + +- GitHub supports `GET /repos/{owner}/{repo}/compare/{base...head}` and returns changed files between two refs/commits, plus raw file reads at `GET /repos/{owner}/{repo}/contents/{path}?ref=`. +- GitLab supports `GET /projects/:id/repository/compare?from=&to=` and returns `diffs[]` with `new_path`, `old_path`, `new_file`, `renamed_file`, `deleted_file`, and `compare_timeout`, plus raw file reads at `GET /projects/:id/repository/files/:path/raw?ref=`. + +## Design decision + +Keep the connector feature set and DB model from plan 008, but change update semantics from **branch snapshot replacement** to **binding-scoped incremental cutover**. + +Core rules: + +- **Initial connector graph creation stays full-snapshot.** The repository has no prior binding state to diff against. +- **Later manual syncs and push-webhook syncs are incremental.** Only changed supported code paths create new file rows and enter the code workflow. +- **Unchanged active file rows stay active.** Do not mint duplicate rows just to refresh `commitSha` metadata when the file content and path did not change. +- **Current binding state is the set of non-deleted code files for one `repositoryBindingId`, not “every file from one commit”.** +- **Repository code manifests for connector-backed files must be binding-scoped, not `repositoryUrl + commitSha` scoped.** Changed files still need unchanged siblings present for import resolution. +- **Removed/replaced paths invalidate only their own current sources.** Do not invalidate the whole binding when one file changes. +- **Normal document uploads and public URL imports remain unchanged.** This plan is connector-only. +- **No silent full-sync fallback for a routine update path.** If a provider compare cannot produce a safe delta, stop/report instead of quietly reprocessing the whole branch. + +This deliberately keeps exact provenance for unchanged files: a source citation can still point at the last commit where that file content changed. If product later wants branch-head links for unchanged files too, that is a separate feature and must not reintroduce full-file churn here. + +## Commands you will need + +| Purpose | Command | Expected on success | +|---|---|---| +| Connector provider tests | `bun test packages/connectors/src/__tests__/credentials.test.ts packages/connectors/src/__tests__/github.test.ts packages/connectors/src/__tests__/gitlab.test.ts` | exit 0 | +| Worker manifest + sync tests | `bun test apps/worker/lib/__tests__/code-manifest.test.ts apps/worker/workflows/process-file.test.ts apps/worker/workflows/sync-repository-graph.test.ts` | exit 0 | +| API regression tests | `bun test apps/api/src/lib/__tests__/graph-file-proxy.test.ts apps/api/src/lib/__tests__/source-reference.test.ts apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` | exit 0 | +| Workspace tests | `bun run test` | exit 0 | +| Lint | `bun run lint` | exit 0; no new errors | + +## Scope + +**In scope**: + +- Extend `@kiwi/connectors` with provider-neutral compare/change APIs. +- GitHub and GitLab provider implementations for changed-path detection between two commits. +- Incremental connector sync in `sync-repository-graph.ts`. +- Binding-scoped repository manifests for connector-backed code files. +- Targeted file deletion/source invalidation for removed or replaced paths. +- Tests for changed-only sync, delete/rename handling, and no-op syncs when no supported code changed. + +**Out of scope**: + +- Changing the normal document pipeline. +- Changing public URL repository imports. +- Rewriting unchanged file rows solely to update commit metadata. +- Pull request preview graphs. +- Automatic repair for provider compare timeouts or missing compare ancestry. +- Provider support beyond GitHub and GitLab. + +## Git workflow + +- Branch name suggestion: `advisor/009-incremental-connector-updates`. +- Commit message style: `feat(connectors): sync only changed repository files`. +- Do not push or open a PR unless instructed. + +## Steps + +### Step 1: Extend provider APIs with compare/change support + +Add a provider-neutral change model in `packages/connectors/src/types.ts`, for example: + +- `ProviderRepositoryChange` +- `ProviderRepositoryDelta` +- statuses covering `added`, `modified`, `deleted`, and `renamed` +- both `oldPath` and `newPath` where rename/delete semantics need them + +Then add a new method to `ProviderRepositoryClient`, for example: + +```ts +compareRepository(repository, fromCommitSha, toCommitSha): Promise +``` + +Implementation requirements: + +- **GitHub**: call the compare endpoint for `base...head`, normalize changed-file records, and reject responses that do not safely describe changed paths. +- **GitLab**: call the compare endpoint, reject `compare_timeout === true`, and normalize `diffs[]` into the same provider-neutral shape. +- Preserve existing `readFile()` methods; incremental sync should reuse them to fetch only the touched file contents at the target commit. +- Keep path filtering consistent with current supported-code rules (`isSupportedCodePath`, skipped path segments, file-size limits). + +Add/extend tests in `packages/connectors/src/__tests__/github.test.ts` and `packages/connectors/src/__tests__/gitlab.test.ts` for: + +- modified supported file +- added supported file +- deleted supported file +- rename from supported → supported +- rename from supported → unsupported +- compare-timeout / malformed-response rejection + +### Step 2: Teach code manifests about binding-scoped current snapshots + +`prepareCodeManifest()` currently groups repository context by `repositoryUrl + commitSha`. That is incompatible with changed-only updates because unchanged active files will often keep older commit metadata. + +Change the manifest scope logic so connector-backed repository files use a **binding-wide active snapshot** instead: + +- if `repositoryBindingId` exists, scope by `repositoryBindingId` +- otherwise keep the existing public URL import behavior (`repositoryUrl + commitSha`) + +Implementation notes: + +- Keep normal document files and non-repository code files unchanged. +- Continue reading content through `readFileContentSource()`. +- Do not broaden the manifest beyond the selected binding; one repository binding should never leak another repository’s files into import resolution. + +Add worker tests covering: + +- incremental connector reprocessing where selected file IDs are at a newer commit but unchanged sibling files remain on older rows +- public URL repository imports still using the old commit-scoped manifest behavior + +### Step 3: Compute incremental connector deltas instead of full snapshots + +Update `apps/worker/workflows/sync-repository-graph.ts`. + +Required behavior: + +1. If `lastSyncedCommitSha` is missing, keep the current full-snapshot bootstrap behavior. +2. If `lastSyncedCommitSha === targetCommitSha`, keep the existing fast no-op. +3. Otherwise: + - call the provider compare API + - derive the set of affected supported code paths + - load the current active binding rows keyed by path + - build three sets: + - **new/updated paths** → need new file rows + code processing + - **removed/replaced old paths** → need targeted invalidation after successful cutover + - **unchanged paths** → keep current active rows untouched + +Path rules: + +- `added` / `modified` / rename target with a supported path → fetch target file content and insert a new external file row for that path. +- `deleted` / rename source from a supported path → keep the old active row until success, then mark it deleted and invalidate its current sources. +- rename supported → supported is both a delete for the old path and an add for the new path. +- changes that only touch unsupported paths must not create a process run. + +Do **not** call `loadRepositorySnapshot()` for incremental updates. The whole point of this plan is to stop loading every file when one file changed. + +### Step 4: Cut over only the touched file rows + +Replace the current “insert whole snapshot, mark everything else deleted” transaction with a targeted cutover: + +- insert only new/updated file rows +- create a process run only when there are changed supported files to process +- do not mark old rows deleted until the changed-file workflows succeed +- if the changed-file workflow batch fails, leave the previous active rows intact + +Use `repositoryBindingId` + parsed metadata path to identify the active row being replaced or removed. + +Important edge cases: + +- If compare says the branch changed but none of the changed paths are supported code, mark the binding synced and advance `lastSeenCommitSha` / `lastSyncedCommitSha` without a process run. +- If the same path appears twice in the delta after provider normalization, treat that as a bug and STOP rather than guessing. +- If a path was already deleted from the active binding state, ignore duplicate delete signals. + +### Step 5: Make source invalidation path-targeted + +`invalidateSupersededRepositorySources()` currently derives “older file IDs” by repository URL across the latest import batch. That is too broad for incremental sync. + +Refactor it so the caller can pass explicit file IDs to retire, or add a new helper dedicated to binding/path cutover. + +Required behavior after a successful incremental batch: + +- mark only removed/replaced active file rows `deleted = true` +- set `sources.valid_until = NOW()` only for current sources attached to those retired file IDs +- keep unchanged file rows and their current sources active +- regenerate descriptions only for affected entities/relationships returned by the targeted invalidation helper + +Keep the existing full-batch repository invalidation path for public URL imports if it is still used there. Do not accidentally couple connector-only incremental semantics back into the URL-import path. + +### Step 6: Verification and regression coverage + +Add or update focused tests for the behaviors above. + +Minimum coverage: + +- `packages/connectors/src/__tests__/github.test.ts` — compare normalization +- `packages/connectors/src/__tests__/gitlab.test.ts` — compare normalization and timeout handling +- `apps/worker/lib/__tests__/code-manifest.test.ts` — binding-scoped manifests across mixed commit rows +- `apps/worker/workflows/sync-repository-graph.test.ts` — changed-only file insertion, delete/rename handling, and no-op supported-code delta handling +- `apps/worker/workflows/process-file.test.ts` — repository batch finalization still requires all child workflows to succeed +- `apps/api/src/lib/__tests__/graph-file-proxy.test.ts` and/or `source-reference.test.ts` — unchanged active rows with older commit metadata still resolve content correctly + +Then run the verification commands from the table above. + +## STOP conditions + +Stop and report instead of improvising if any of these occur: + +1. Provider compare responses cannot safely enumerate changed paths for one of the supported providers. +2. A live repository binding can contain multiple non-deleted rows for the same path, and no existing invariant guarantees which one is current. +3. Binding-scoped manifests break public URL repository imports instead of staying connector-only. +4. The targeted invalidation change would require broad source/citation semantics beyond connector-backed repository files. +5. A safe incremental path for GitHub and GitLab diverges so much that the shared provider contract becomes misleading. + +## Acceptance checklist + +Do not mark this plan done until all are true: + +- Subsequent connector syncs no longer load full branch snapshots for routine updates. +- A one-file change on a bound branch processes only that file plus any delete/rename counterpart, not every active file in the binding. +- Changes outside supported code paths do not create a process run. +- Unchanged active files remain available to the changed-file code manifest. +- Removed/replaced paths invalidate only their own current sources. +- Normal document uploads and public URL repository imports still behave the same as before. +- Tests and `bun run lint` pass with no new warnings/errors. + +## Notes for the follow-up executor + +The hard part is not the provider compare call; it is the cutover invariant: + +- changed files must see unchanged siblings in the manifest +- failed incremental batches must leave the previous active snapshot intact +- unchanged files must not churn just because branch HEAD moved + +Favor small helper modules over growing `sync-repository-graph.ts` into a second monolith. Keep provider compare logic in `packages/connectors`, binding snapshot logic in worker helpers, and public URL import behavior separate from connector-specific incremental semantics. diff --git a/plans/README.md b/plans/README.md new file mode 100644 index 00000000..391c7712 --- /dev/null +++ b/plans/README.md @@ -0,0 +1,74 @@ +# Implementation Plans + +Generated by the improve skill on 2026-06-13 for `branch` mode. Current branch is `main`, but the working tree has unstaged/untracked changes; the initial audit scoped to those changed files and direct callers only. Plan 006 was added afterward from the product requirement that repository code files can be external instead of copied to S3. Plan 007 was added afterward from the product requirement that re-imported code repositories represent the latest snapshot while retaining invalidated historical sources. Plan 008 was added afterward from the product requirement for provider connectors, private repository access, and webhook-driven branch updates. Plan 009 was added afterward from the follow-up requirement that connector branch updates process only changed supported code files instead of rebuilding whole repository snapshots. + +Each executor: read the plan fully before starting, honor its STOP conditions, run every verification command from the plan, and update your row when done. + +## Execution order & status + +| Plan | Title | Priority | Effort | Depends on | Status | +|---|---|---|---|---|---| +| 001 | Align the relationship migration snapshot with the schema | P1 | S | — | DONE | +| 002 | Return no path for missing same-entity path requests | P1 | S | — | DONE | +| 003 | Validate repository import models before upload and enqueue | P1 | S | — | DONE | +| 004 | Sanitize repository URL loader errors returned to clients | P2 | S | — | DONE | +| 005 | Route direct code uploads through the code workflow | P2 | M | — | DONE | +| 006 | Support external GitHub code files without copying them to S3 | P1 | L | 003, 004, ideally 005 | DONE | +| 007 | Version code sources with `validUntil` so graph tools use only the latest repository snapshot | P1 | L | 005, 006 | DONE | +| 008 | Provider connectors for private repository graphs and webhook-driven updates | P1 | XL | 006, 007 | DONE | +| 009 | Incremental connector repository updates for changed files only | P1 | L | 007, 008 | DONE | + +Status values: TODO | IN PROGRESS | DONE | BLOCKED (with one-line reason) | REJECTED (with one-line rationale — finding fixed independently or approach abandoned) + +## Dependency notes + +- Plans 001-004 are independent. +- Plan 005 is independent of the first four, but it touches broader upload/worker routing and should run after the smaller correctness fixes if execution capacity is limited. +- Plan 006 depends on the repository import safety fixes in plans 003 and 004. It should ideally run after plan 005 so external code rows enter the dedicated code workflow consistently. +- Plan 007 depends on plans 005 and 006 because it assumes code files have a dedicated workflow and repository code file rows can be external/latest-snapshot records. It should run before webhook-based repository updates. +- Plan 008 depends on plan 006 for external code file storage and plan 007 for latest-snapshot source invalidation. It should run before replacing URL imports with connector-backed private repository sync. +- Plan 009 depends on plans 007 and 008 because it narrows connector sync from whole-snapshot rebuilds to changed-file-only cutovers using `validUntil` and binding-scoped manifests. It should run after connector sync is stable. +## Verification baseline observed during planning + +- `bun run test` exited 0; all 10 Turbo tasks successful. +- `bun run lint` exited 0 with one pre-existing frontend warning: `apps/frontend/components/theme/ThemePresetScript.tsx` uses `next/script` `beforeInteractive` outside `pages/_document.js`. +- `bun run build` was not run because it writes build artifacts; use plan-specific read-only checks unless the operator explicitly allows build artifacts. + +## Vetted branch findings + +| # | Finding | Tag | Category | Impact | Effort | Risk | Evidence | Plan | +|---|---|---|---|---|---|---|---|---| +| 1 | Migration snapshot omits new `relationships.kind` and `relationships.directed` columns | introduced | migration | Future Drizzle migrations can be generated from stale schema state | S | LOW | `packages/db/src/tables/graph.ts:248-249`; `migrations/20260613184716_sturdy_triton/migration.sql:1-2`; `migrations/20260613184716_sturdy_triton/snapshot.json:3008-3021` | `plans/001-align-migration-snapshot.md` | +| 2 | Same-entity path requests return a fake `Unknown` path when the entity is absent | introduced | bug | Invalid/deleted/cross-graph IDs can be presented to the model as existing graph entities | S | LOW | `packages/ai/src/tools/relationship.ts:449-463` | `plans/002-return-none-for-missing-same-entity-path.md` | +| 3 | Repository URL imports skip pre-upload model validation | introduced | bug | Missing model configuration is discovered after S3/DB/workflow side effects instead of before them | S | LOW | `apps/api/src/routes/graph.ts:632-646`; `apps/api/src/routes/graph.ts:847-858` | `plans/003-validate-repository-import-models.md` | +| 4 | Repository URL loader returns raw git stderr to clients | introduced | security | Client responses disclose operational git/provider details and depend on unstable tool output | S | LOW | `apps/api/src/lib/repository-url.ts:194-197`; `apps/api/src/routes/graph.ts:109-116` | `plans/004-sanitize-repository-url-errors.md` | +| 5 | Direct and archive-expanded code uploads do not reach the code workflow | introduced | bug | Standalone source uploads are processed as generic text; mixed batches need per-file workflow routing | M | MED | `packages/graph/src/file-type.ts:1-21`; `packages/graph/src/file-type.ts:52-187`; `apps/api/src/lib/graph-upload-file-type.ts:23-31`; `apps/worker/workflows/process-file.ts:47-56` | `plans/005-route-code-uploads-through-code-workflow.md` | +| 6 | Repository import limits are enforced after full clones and only per repository | introduced | security/perf | One authenticated request can force multiple large public clones before code byte/file limits reject anything | M | MED | `apps/api/src/routes/graph.ts:582-596`; `apps/api/src/lib/repository-url.ts:89-121` | not planned by default | +| 7 | Code symbol resolution keys every in-file definition by simple name | introduced | bug | Same-named functions/methods can produce false `CALLS` edges and polluted code graph answers | M | MED | `packages/graph/src/code/repository.ts:51-65`; `packages/graph/src/code/repository.ts:236-262`; `packages/graph/src/code/__tests__/repository.test.ts:20-77` | not planned by default | +| 8 | Batch process runs complete when only some child file workflows fail | pre-existing in touched files | bug | Mixed success/failure batches can record process-run success despite failed files | S | MED | `apps/worker/workflows/process-file.ts:105-143`; `apps/worker/workflows/process-file.ts:495-500`; `apps/worker/workflows/process-code-file.ts:221-226` | not planned by default | +| 9 | Relationship endpoints are not constrained to the same graph as the relationship | pre-existing in touched files | security | Malformed writes can make graph-scoped tools disclose entity names/descriptions from another graph | M | MED | `packages/db/src/tables/graph.ts:239-247`; `packages/ai/src/tools/relationship.ts:289-324`; `packages/ai/src/tools/relationship.ts:365-405` | not planned by default | +| 10 | Code-manifest preparation serially reads same-repository files before fan-out | introduced | performance | Large repository batches delay child processing and hold more content in memory | M | MED | `apps/worker/workflows/process-file.ts:95-103`; `apps/worker/lib/code-manifest.ts:46-86` | not planned by default | +| 11 | Source chunk validator accepts new code-location fields without validating their types | introduced | correctness | Malformed stored chunks can pass validation and leak invalid location metadata into source references | S | LOW | `packages/contracts/src/source.ts:21-27`; `packages/contracts/src/source.ts:93-122`; `apps/api/src/lib/source-reference-record.ts:107-110` | not planned by default | + +## Direction / product plans + +| Plan | Direction | Evidence | Trade-off | +|---|---|---|---| +| `plans/006-support-external-github-code-files.md` | Make repository code files external for GitHub imports instead of copying every source file to S3. | `files.file_key` is currently S3-oriented and non-null (`packages/db/src/tables/graph.ts:136-148`); repository imports upload every code source via `putGraphFile` before DB insert (`apps/api/src/routes/graph.ts:652-667`); code processing reads source through S3 (`apps/worker/workflows/process-code-file.ts:79-93`). | Saves storage and makes source links canonical, but requires first-class file origin columns, external fetch allowlisting, and proxy/worker changes. | +| `plans/007-version-code-sources-with-valid-until.md` | Re-importing the same repository should supersede old code evidence with `sources.validUntil` while retaining historical source rows. | `sources` currently has `active` but no validity window (`packages/db/src/tables/graph.ts:291-314`); graph save appends source rows (`apps/worker/lib/save-graph.ts:118-143`); description regeneration reads all linked sources (`apps/worker/lib/regenerate-descriptions.ts:148-163`); source tools filter only `active` (`packages/ai/src/tools/source.ts:205-310`). | Keeps current answers fresh and preserves history, but requires schema migration, repository snapshot cutover semantics, source invalidation, description deactivation, and tool/citation filtering. | +| `plans/008-provider-connectors-for-repository-graphs.md` | Add system-admin GitHub/GitLab connectors, graph-manager repository installation/selection, private repo graph creation, and push-webhook rebuilds for selected branches. | Graph managers already map to org admin/team admin/team moderator (`apps/api/src/lib/team-access.ts:140-155`); URL imports still load repositories from public URLs (`apps/api/src/lib/repository-url.ts:161-169`); graph routes can enqueue repository processing (`apps/api/src/routes/graph.ts:807-812`); webhooks need a public verified route before auth middleware (`apps/api/src/server.ts:53-61`). | Unlocks private repos and automatic branch freshness, but adds provider auth, encrypted credentials, webhook verification/idempotency, repository binding state, and provider API rate-limit handling. | +| `plans/009-incremental-connector-repository-updates.md` | Update connector-backed repository graphs incrementally so later branch syncs process only changed supported code files instead of rebuilding whole snapshots. | Connector sync currently loads a full provider snapshot and processes every inserted file (`apps/worker/workflows/sync-repository-graph.ts:190-235`); repository manifests are still scoped by `repositoryUrl + commitSha` (`apps/worker/lib/code-manifest.ts:45-65,102-104`); plan 008 explicitly left fine-grained incremental updates out of scope (`plans/008-provider-connectors-for-repository-graphs.md:219-226`). | Cuts worker/runtime churn and provider API load, but requires provider compare APIs, binding-scoped manifests, and targeted file/source invalidation so unchanged files stay current without full reprocessing. | +## Findings considered and rejected + +- Archive traversal in changed upload flow: not retained. Existing archive tests cover unsafe paths and option-looking names; no new high-confidence traversal issue was found in the changed files. +- Directed/undirected relationship dedupe key normalization: not retained. `packages/graph/src/relationship-key.ts` and `packages/graph/src/dedupe.ts` consistently include kind, direction, and normalized endpoints for undirected edges. +- Worker Docker tree-sitter rebuild: not retained. The Dockerfile copies graph package sources and rebuilds the native binding; no branch-specific failure was confirmed. +- Raw repository URL credentials in clone commands: not retained. `normalizeRepositoryUrl` rejects username/password and restricts hosts before cloning. + +## Not audited + +- Full frontend behavior outside files touched by the working tree. +- Production Compose/Caddy behavior beyond changed Dockerfiles. +- Dependency advisories/audit output. +- Runtime e2e repository import against real GitHub/GitLab/Bitbucket repositories. +- Build output, because build commands write artifacts. From 93a907e006315166d6652a750a923c16a5719f27 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 10:42:44 +0200 Subject: [PATCH 02/23] fix: ignore plan files --- .gitignore | 3 + plans/001-align-migration-snapshot.md | 153 ---- ...eturn-none-for-missing-same-entity-path.md | 152 ---- .../003-validate-repository-import-models.md | 171 ---- plans/004-sanitize-repository-url-errors.md | 178 ----- ...oute-code-uploads-through-code-workflow.md | 226 ------ .../006-support-external-github-code-files.md | 356 --------- ...7-version-code-sources-with-valid-until.md | 431 ----------- ...ovider-connectors-for-repository-graphs.md | 731 ------------------ ...ncremental-connector-repository-updates.md | 348 --------- plans/README.md | 74 -- 11 files changed, 3 insertions(+), 2820 deletions(-) delete mode 100644 plans/001-align-migration-snapshot.md delete mode 100644 plans/002-return-none-for-missing-same-entity-path.md delete mode 100644 plans/003-validate-repository-import-models.md delete mode 100644 plans/004-sanitize-repository-url-errors.md delete mode 100644 plans/005-route-code-uploads-through-code-workflow.md delete mode 100644 plans/006-support-external-github-code-files.md delete mode 100644 plans/007-version-code-sources-with-valid-until.md delete mode 100644 plans/008-provider-connectors-for-repository-graphs.md delete mode 100644 plans/009-incremental-connector-repository-updates.md delete mode 100644 plans/README.md diff --git a/.gitignore b/.gitignore index d8d32ccd..02090efe 100644 --- a/.gitignore +++ b/.gitignore @@ -91,3 +91,6 @@ backend/server.exe # turbo .turbo + +# AI +./plans diff --git a/plans/001-align-migration-snapshot.md b/plans/001-align-migration-snapshot.md deleted file mode 100644 index e1601d37..00000000 --- a/plans/001-align-migration-snapshot.md +++ /dev/null @@ -1,153 +0,0 @@ -# Plan 001: Align the relationship migration snapshot with the schema - -> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. -> -> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/db/src/tables/graph.ts migrations/20260613184716_sturdy_triton packages/db/src/__tests__/migration-compat.test.ts` -> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. - -## Status - -- **Priority**: P1 -- **Effort**: S -- **Risk**: LOW -- **Depends on**: none -- **Category**: migration -- **Planned at**: commit `1dea5eb77`, 2026-06-13 - -## Why this matters - -The branch adds `relationships.kind` and `relationships.directed` in both the Drizzle schema and the SQL migration, but the migration snapshot does not contain those columns. Drizzle uses snapshots as migration-generation state. If this lands stale, the next generated migration can diff from the wrong schema state and re-emit already-applied relationship column changes. - -## Current state - -Relevant files: - -- `packages/db/src/tables/graph.ts` — source Drizzle schema for `relationships`. -- `migrations/20260613184716_sturdy_triton/migration.sql` — SQL migration for the branch. -- `migrations/20260613184716_sturdy_triton/snapshot.json` — generated Drizzle snapshot that is currently stale. -- `packages/db/src/__tests__/migration-compat.test.ts` — migration compatibility tests. - -Current schema excerpt: - -```ts -// packages/db/src/tables/graph.ts:245-250 -graphId: text("graph_id") - .notNull() - .references(() => graphTable.id, { onDelete: "cascade" }), -kind: text("kind").notNull().default("RELATED"), -directed: boolean("directed").notNull().default(false), -rank: doublePrecision("rank").notNull().default(0), -``` - -Current migration excerpt: - -```sql --- migrations/20260613184716_sturdy_triton/migration.sql:1-2 -ALTER TABLE "relationships" ADD COLUMN IF NOT EXISTS "kind" text DEFAULT 'RELATED' NOT NULL; -ALTER TABLE "relationships" ADD COLUMN IF NOT EXISTS "directed" boolean DEFAULT false NOT NULL; -``` - -Current stale snapshot excerpt: - -```json -// migrations/20260613184716_sturdy_triton/snapshot.json:3008-3021 -"name": "graph_id", -"entityType": "columns", -"schema": "public", -"table": "relationships" -}, -{ -"type": "double precision", -"notNull": true, -"default": "0", -"name": "rank", -"table": "relationships" -``` - -Repo conventions: - -- Use Bun from the repo root. -- Do not run `bun run db:migrate`. -- Do not hand-create a new migration. This plan fixes the existing branch migration metadata. -- Root verification commands available: `bun run test`, `bun run lint`. `bun run test` passed at planning time; `bun run lint` exited 0 with one existing frontend warning in `apps/frontend/components/theme/ThemePresetScript.tsx`. - -## Commands you will need - -| Purpose | Command | Expected on success | -|---|---|---| -| Tests | `bun run test` | exit 0; all workspace tests pass | -| Lint | `bun run lint` | exit 0; existing warning may remain, no new errors | - -## Scope - -**In scope**: - -- `migrations/20260613184716_sturdy_triton/snapshot.json` -- `packages/db/src/__tests__/migration-compat.test.ts` - -**Out of scope**: - -- Creating another migration directory. -- Running `bun run db:migrate`. -- Changing relationship application code. -- Editing older migration snapshots. - -## Git workflow - -- Branch name suggestion: `advisor/001-align-migration-snapshot`. -- Commit message style from repo history: Conventional Commits, e.g. `fix(db): align relationship migration snapshot`. -- Do not push or open a PR unless instructed. - -## Steps - -### Step 1: Add the missing snapshot columns - -Update `migrations/20260613184716_sturdy_triton/snapshot.json` so the `relationships` table has both columns between `graph_id` and `rank`, matching schema/migration semantics: - -- `kind`: type `text`, `notNull: true`, default `'RELATED'` in the snapshot's existing default representation for string defaults. -- `directed`: type `boolean`, `notNull: true`, default `false` in the snapshot's existing default representation for boolean defaults. - -Prefer regenerating the snapshot with the repo's Drizzle workflow if that produces only this snapshot correction. If regeneration wants to create another migration or broad unrelated changes, stop and hand-edit only these two column entries to match the snapshot format around nearby columns. - -**Verify**: `bun run test --filter @kiwi/db` → exit 0; DB package tests pass. - -### Step 2: Add a compatibility assertion for the snapshot - -In `packages/db/src/__tests__/migration-compat.test.ts`, extend the existing migration compatibility coverage for `20260613184716_sturdy_triton` so it checks that `snapshot.json` contains `relationships.kind` and `relationships.directed`. Keep this as a metadata regression test; do not assert fragile line numbers. - -**Verify**: `bun run test --filter @kiwi/db` → exit 0; the new assertion fails before Step 1 and passes after Step 1. - -### Step 3: Run repo checks - -Run the standard read-only checks from the repo root. - -**Verify**: `bun run test` → exit 0; all tests pass. - -**Verify**: `bun run lint` → exit 0; no new errors. - -## Test plan - -- Extend `packages/db/src/__tests__/migration-compat.test.ts`. -- Cover both new relationship columns in `snapshot.json`. -- Do not add a migration execution test; this is a snapshot/schema parity regression. - -## Done criteria - -- [ ] `migrations/20260613184716_sturdy_triton/snapshot.json` contains `relationships.kind` and `relationships.directed` with defaults/nullability matching `graph.ts` and `migration.sql`. -- [ ] `bun run test --filter @kiwi/db` exits 0. -- [ ] `bun run test` exits 0. -- [ ] `bun run lint` exits 0. -- [ ] No files outside the in-scope list are modified. -- [ ] `plans/README.md` status row updated. - -## STOP conditions - -Stop and report if: - -- Regenerating the snapshot creates a new migration directory or changes unrelated snapshots. -- The live schema/migration excerpts no longer match the current-state snippets above. -- The snapshot format for defaults is unclear after comparing nearby string/boolean defaults. - -## Maintenance notes - -Reviewers should check this before any follow-up Drizzle migration is generated. A stale snapshot is cheap to fix now and expensive to unwind after another migration is generated on top of it. diff --git a/plans/002-return-none-for-missing-same-entity-path.md b/plans/002-return-none-for-missing-same-entity-path.md deleted file mode 100644 index 5752ff65..00000000 --- a/plans/002-return-none-for-missing-same-entity-path.md +++ /dev/null @@ -1,152 +0,0 @@ -# Plan 002: Return no path for missing same-entity path requests - -> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. -> -> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/ai/src/tools/relationship.ts packages/ai/src/__tests__/relationship-tools.test.ts` -> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. - -## Status - -- **Priority**: P1 -- **Effort**: S -- **Risk**: LOW -- **Depends on**: none -- **Category**: bug -- **Planned at**: commit `1dea5eb77`, 2026-06-13 - -## Why this matters - -`get_path_between_entities` special-cases identical source and target IDs. It correctly queries the current graph for that entity, but when the entity is absent it still returns a one-line path with the caller-supplied ID and `Unknown` fields. That lets an invalid, deleted, or cross-graph ID look like a valid zero-hop path to the model. - -## Current state - -Relevant files: - -- `packages/ai/src/tools/relationship.ts` — relationship graph tools. -- `packages/ai/src/__tests__/relationship-tools.test.ts` — current mocked DB tests for path/neighbour behavior. - -Current behavior: - -```ts -// packages/ai/src/tools/relationship.ts:449-463 -if (sourceEntityId === targetEntityId) { - const [entity] = await db - .select({ - id: entityTable.id, - name: entityTable.name, - type: entityTable.type, - }) - .from(entityTable) - .where(and(eq(entityTable.graphId, graphId), eq(entityTable.id, sourceEntityId))) - .limit(1); - - return [ - "## Path", - `- ${entity?.id ?? sourceEntityId}, ${entity?.name ?? "Unknown"}, ${entity?.type ?? "Unknown"}`, - ].join("\n"); -} -``` - -Existing test style: - -```ts -// packages/ai/src/__tests__/relationship-tools.test.ts:44-52 -const { getNeighboursTool, getPathBetweenTool } = await import("../tools/relationship"); - -async function executeTool(tool: { execute?: (input: unknown) => Promise }, input: unknown) { - if (!tool.execute) { - throw new Error("Tool has no execute function"); - } - - return tool.execute(input); -} -``` - -Repo conventions: - -- Tests use `bun:test`, module mocks, and result text assertions. -- Keep tool output stable and compact; existing no-path wording is `## Path\n- none found within 5 hops`. -- Root verification commands available: `bun run test`, `bun run lint`. `bun run test` passed at planning time; `bun run lint` exited 0 with one existing frontend warning. - -## Commands you will need - -| Purpose | Command | Expected on success | -|---|---|---| -| Targeted tests | `bun test packages/ai/src/__tests__/relationship-tools.test.ts` | exit 0; relationship tool tests pass | -| Workspace tests | `bun run test` | exit 0; all workspace tests pass | -| Lint | `bun run lint` | exit 0; no new errors | - -## Scope - -**In scope**: - -- `packages/ai/src/tools/relationship.ts` -- `packages/ai/src/__tests__/relationship-tools.test.ts` - -**Out of scope**: - -- Changing BFS path depth or direction semantics. -- Changing relationship/neighbour output formats outside this missing-entity branch. -- Adding DB constraints; that is a separate pre-existing finding. - -## Git workflow - -- Branch name suggestion: `advisor/002-same-entity-path-not-found`. -- Commit message style: `fix(ai): return no path for missing entity`. -- Do not push or open a PR unless instructed. - -## Steps - -### Step 1: Add the failing test - -Add a test in `packages/ai/src/__tests__/relationship-tools.test.ts` for `sourceEntityId === targetEntityId` where the entity lookup returns no rows: - -- Push one empty select result into `selectResults`. -- Execute `getPathBetweenTool("graph-1")` with identical IDs. -- Assert the output does not contain the requested ID as a path line. -- Assert it returns a no-path response. Prefer reusing the existing wording `## Path\n- none found within 5 hops` unless you first introduce a small helper for no-path text. - -**Verify**: `bun test packages/ai/src/__tests__/relationship-tools.test.ts` → the new test fails before Step 2. - -### Step 2: Return no path when the graph-scoped entity is absent - -In `packages/ai/src/tools/relationship.ts`, change only the same-entity branch: - -- Keep the existing graph-scoped entity query. -- If `entity` is undefined, return the same no-path wording used by the normal not-found branch. -- If `entity` exists, keep the existing zero-hop output. - -**Verify**: `bun test packages/ai/src/__tests__/relationship-tools.test.ts` → exit 0; new and existing relationship tests pass. - -### Step 3: Run repo checks - -**Verify**: `bun run test` → exit 0. - -**Verify**: `bun run lint` → exit 0; no new errors. - -## Test plan - -- New unit test in `packages/ai/src/__tests__/relationship-tools.test.ts` for missing same-ID entity. -- Existing directed/undirected path tests must still pass. - -## Done criteria - -- [ ] Missing same-ID entity returns no path, not `Unknown` entity output. -- [ ] Existing same-ID entity still returns a one-line zero-hop path. -- [ ] `bun test packages/ai/src/__tests__/relationship-tools.test.ts` exits 0. -- [ ] `bun run test` exits 0. -- [ ] `bun run lint` exits 0. -- [ ] No files outside the in-scope list are modified. -- [ ] `plans/README.md` status row updated. - -## STOP conditions - -Stop and report if: - -- The relationship tool test harness no longer uses `selectResults` or `executeTool` as shown above. -- The tool output contract for no-path responses has changed elsewhere. -- Fixing this requires changing graph access or authorization code. - -## Maintenance notes - -This is defense-in-depth for model-facing tool output. Future graph tools should never turn missing graph-scoped DB rows into `Unknown` entities unless the caller explicitly requested a best-effort external ID echo. diff --git a/plans/003-validate-repository-import-models.md b/plans/003-validate-repository-import-models.md deleted file mode 100644 index ee953455..00000000 --- a/plans/003-validate-repository-import-models.md +++ /dev/null @@ -1,171 +0,0 @@ -# Plan 003: Validate repository import models before upload and enqueue - -> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. -> -> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- apps/api/src/routes/graph.ts apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts apps/api/src/lib/graph-upload-file-type.ts` -> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. - -## Status - -- **Priority**: P1 -- **Effort**: S -- **Risk**: LOW -- **Depends on**: none -- **Category**: bug -- **Planned at**: commit `1dea5eb77`, 2026-06-13 - -## Why this matters - -The regular file-upload route validates required processing models before writing files to S3 or inserting DB rows. The new repository URL route skips that validation and can create code file rows plus enqueue a workflow that later fails in the worker if required graph processing models are unavailable. Repository imports should fail before side effects, just like normal uploads. - -## Current state - -Relevant files: - -- `apps/api/src/routes/graph.ts` — repository URL route and regular file upload route. -- `apps/api/src/lib/graph-upload-file-type.ts` — existing upload model assertion helper. -- `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` — route tests with mocked uploads/workflows. - -Repository route currently proceeds directly to uploads: - -```ts -// apps/api/src/routes/graph.ts:632-646 -if (repositorySources.length === 0) { - return status(200, { ... }); -} - -const uploadedFiles: UploadedFile[] = []; -try { - for (const source of repositorySources) { - const fileId = ulid(); -``` - -Regular upload path validates first: - -```ts -// apps/api/src/routes/graph.ts:847-858 -const uploadModelResult = await Result.tryPromise(async () => { - await assertConfiguredUploadModels({ - organizationId: await getGraphOwnerModelOrganizationId({ - ownerMode: "graph", - graphId: existingGraph.id, - }), - files: supportedUpload.files, - secret: env.AUTH_SECRET, - }); -}); -if (uploadModelResult.isErr()) { - return mapGraphError(status, uploadModelResult.error); -} -``` - -Existing repository route test expectations: - -```ts -// apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts:398-424 -expect(response.status).toBe(200); -expect(insertedFileValues.map((file) => file.type)).toEqual(["code", "code"]); -expect(workflowInputs).toEqual([ - { - graphId: "graph-1", - fileIds: body.data.addedFiles.map((file: { id: string }) => file.id), - processRunId: "process-run-1", - code: { kind: "repository" }, - }, -]); -``` - -Repo conventions: - -- API routes use `Result.tryPromise` and `mapGraphError` for expected async errors. -- Tests use `bun:test` and module-level arrays to assert no side effects. -- Root verification commands available: `bun run test`, `bun run lint`. `bun run test` passed at planning time; `bun run lint` exited 0 with one existing frontend warning. - -## Commands you will need - -| Purpose | Command | Expected on success | -|---|---|---| -| Targeted API tests | `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` | exit 0; graph route upload tests pass | -| Workspace tests | `bun run test` | exit 0; all workspace tests pass | -| Lint | `bun run lint` | exit 0; no new errors | - -## Scope - -**In scope**: - -- `apps/api/src/routes/graph.ts` -- `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` -- `apps/api/src/lib/graph-upload-file-type.ts` only if a tiny shared type/helper is needed - -**Out of scope**: - -- Worker model resolution. -- Repository cloning limits. -- Changing normal file upload behavior. -- Creating compatibility shims for old repository import behavior. - -## Git workflow - -- Branch name suggestion: `advisor/003-validate-repository-import-models`. -- Commit message style: `fix(api): validate repository import models`. -- Do not push or open a PR unless instructed. - -## Steps - -### Step 1: Add a failing route test for validation-before-side-effects - -In `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts`, extend the mocks so `assertConfiguredUploadModels` can be forced to fail for the repository URL route. Add a test that: - -- Sets the model assertion mock to reject with an error that `mapGraphError` can convert consistently. -- Calls `POST /graphs/graph-1/urls` with a valid repository URL. -- Asserts the response is an error. -- Asserts `uploadedFiles`, `insertedFileValues`, and `workflowInputs` are still empty. - -Use the existing arrays at lines 4-21 as the side-effect oracle. - -**Verify**: `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` → the new test fails before Step 2. - -### Step 2: Validate repository code imports before upload - -In `apps/api/src/routes/graph.ts`, after `repositorySources.length > 0` and before the `uploadedFiles` loop: - -- Build a minimal list of supported upload descriptors for the repository sources with `type: "code"`. -- Call `assertConfiguredUploadModels` with `organizationId` from `getGraphOwnerModelOrganizationId({ ownerMode: "graph", graphId: existingGraph.id })` and `secret: env.AUTH_SECRET`. -- Wrap with `Result.tryPromise` and return `mapGraphError(status, error)` on failure, matching the regular upload path. - -If `assertConfiguredUploadModels` intentionally only checks audio/video today, still add the call. It keeps repository imports aligned when code/text model requirements are added. - -**Verify**: `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` → exit 0. - -### Step 3: Run repo checks - -**Verify**: `bun run test` → exit 0. - -**Verify**: `bun run lint` → exit 0; no new errors. - -## Test plan - -- New route test for repository URL model-validation failure before upload/DB/workflow side effects. -- Existing repository URL success, duplicate, no-enqueue, and limit tests must continue passing. - -## Done criteria - -- [ ] Repository URL imports call the same model validation path before S3 upload and DB insert. -- [ ] Validation failure leaves `uploadedFiles`, `insertedFileValues`, and `workflowInputs` empty in the route test. -- [ ] `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` exits 0. -- [ ] `bun run test` exits 0. -- [ ] `bun run lint` exits 0. -- [ ] No files outside the in-scope list are modified. -- [ ] `plans/README.md` status row updated. - -## STOP conditions - -Stop and report if: - -- `assertConfiguredUploadModels` has been removed or changed to require browser `File` instances specifically. -- `getGraphOwnerModelOrganizationId` no longer accepts graph-owned uploads. -- The test harness cannot observe upload/DB/workflow side effects without broad rewrites. - -## Maintenance notes - -This keeps all graph file ingestion paths consistent. If code imports later require a text model explicitly, this route will already have the right pre-side-effect validation point. diff --git a/plans/004-sanitize-repository-url-errors.md b/plans/004-sanitize-repository-url-errors.md deleted file mode 100644 index 15e3210f..00000000 --- a/plans/004-sanitize-repository-url-errors.md +++ /dev/null @@ -1,178 +0,0 @@ -# Plan 004: Sanitize repository URL loader errors returned to clients - -> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. -> -> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- apps/api/src/lib/repository-url.ts apps/api/src/routes/graph.ts apps/api/src/lib/__tests__/repository-url.test.ts apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` -> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. - -## Status - -- **Priority**: P2 -- **Effort**: S -- **Risk**: LOW -- **Depends on**: none -- **Category**: security -- **Planned at**: commit `1dea5eb77`, 2026-06-13 - -## Why this matters - -The repository URL loader rejects failed git commands with raw stderr. The route then returns `error.message` verbatim to the client. That leaks implementation-specific git/provider output and makes the API contract depend on git wording instead of stable KIWI error codes/messages. - -## Current state - -Relevant files: - -- `apps/api/src/lib/repository-url.ts` — URL normalization, git clone, file loading. -- `apps/api/src/routes/graph.ts` — maps repository loader errors to HTTP responses. -- `apps/api/src/lib/__tests__/repository-url.test.ts` — URL helper unit tests. -- `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` — route tests for repository URL error mapping. - -Raw git stderr source: - -```ts -// apps/api/src/lib/repository-url.ts:194-197 -finish(() => { - const stderrText = Buffer.concat(stderr).toString("utf8").trim(); - reject(new Error(stderrText || "Repository git command failed")); -}); -``` - -Verbatim HTTP response mapping: - -```ts -// apps/api/src/routes/graph.ts:109-116 -function repositoryUrlErrorResponse(statusFn: (code: number, body: unknown) => unknown, error: unknown) { - const message = error instanceof Error ? error.message : "Unsupported repository URL"; - - if (message.includes("too many") || message.includes("too much")) { - return statusFn(413, errorResponse(message, API_ERROR_CODES.UPLOAD_LIMIT_EXCEEDED)); - } - - return statusFn(400, errorResponse(message, API_ERROR_CODES.UNSUPPORTED_FILE_TYPE)); -} -``` - -Existing tests cover limits but not generic git failures: - -```ts -// apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts:478-493 -test("maps repository loader limits to upload limit responses", async () => { - repositoryLoadMode = "limit-error"; - ... - expect(response.status).toBe(413); - expect(body.code).toBe("UPLOAD_LIMIT_EXCEEDED"); - expect(uploadedFiles).toEqual([]); - expect(workflowInputs).toEqual([]); -}); -``` - -Repo conventions: - -- API error responses use `errorResponse(message, API_ERROR_CODES.X)`. -- Keep raw operational details in server logs, not response bodies. -- Never reproduce secret values in tests or fixtures. -- Root verification commands available: `bun run test`, `bun run lint`. `bun run test` passed at planning time; `bun run lint` exited 0 with one existing frontend warning. - -## Commands you will need - -| Purpose | Command | Expected on success | -|---|---|---| -| Repository helper tests | `bun test apps/api/src/lib/__tests__/repository-url.test.ts` | exit 0 | -| Route tests | `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` | exit 0 | -| Workspace tests | `bun run test` | exit 0 | -| Lint | `bun run lint` | exit 0; no new errors | - -## Scope - -**In scope**: - -- `apps/api/src/lib/repository-url.ts` -- `apps/api/src/routes/graph.ts` -- `apps/api/src/lib/__tests__/repository-url.test.ts` -- `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` - -**Out of scope**: - -- Changing allowed hosts or clone strategy. -- Adding aggregate repository import budgets. -- Logging secret values or raw credentials. -- Changing 413 behavior for known file/byte/count limits. - -## Git workflow - -- Branch name suggestion: `advisor/004-sanitize-repository-url-errors`. -- Commit message style: `fix(api): sanitize repository import errors`. -- Do not push or open a PR unless instructed. - -## Steps - -### Step 1: Introduce typed repository loader errors - -In `apps/api/src/lib/repository-url.ts`, add a small exported error class or discriminated helper so callers can distinguish: - -- validation errors with safe messages (`Repository URL must use HTTPS`, unsupported host, credentials, non-root URL), -- limit errors with safe messages (`too many`, `too much`), -- git/load failures with a generic safe client message. - -Do not expose raw stderr via `.message` for git command failures. If retaining stderr for logs, store it in a non-enumerable field or `cause`, and do not serialize it into responses. - -**Verify**: `bun test apps/api/src/lib/__tests__/repository-url.test.ts` → exit 0. - -### Step 2: Map only safe messages to responses - -Update `repositoryUrlErrorResponse` in `apps/api/src/routes/graph.ts`: - -- Return 413 only for typed limit errors. -- Return 400 and `UNSUPPORTED_FILE_TYPE` for invalid/unsupported repository inputs with safe validation messages. -- Return 400 and a generic message such as `Repository could not be loaded` for git/provider/load failures. -- If raw stderr is logged, log server-side only and keep values out of tests/plans. - -**Verify**: `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` → existing limit test still passes. - -### Step 3: Add route coverage for generic git failure sanitization - -Extend `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` with a repository loader mode that throws an error representing raw git stderr. Assert: - -- Response status is 400. -- Response code is `UNSUPPORTED_FILE_TYPE`. -- Response message is the generic sanitized text. -- The raw stderr string does not appear in the response body. -- `uploadedFiles` and `workflowInputs` stay empty. - -**Verify**: `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` → exit 0. - -### Step 4: Run repo checks - -**Verify**: `bun run test` → exit 0. - -**Verify**: `bun run lint` → exit 0; no new errors. - -## Test plan - -- Unit tests for repository URL validation should keep safe validation messages. -- Route test for generic git failure must prove raw stderr is not returned. -- Existing limit-error route test must continue returning 413. - -## Done criteria - -- [ ] Raw git stderr cannot flow into `error.message` returned by `repositoryUrlErrorResponse`. -- [ ] Limit errors still return `UPLOAD_LIMIT_EXCEEDED` with safe limit text. -- [ ] Generic clone/load failures return a stable sanitized client message. -- [ ] `bun test apps/api/src/lib/__tests__/repository-url.test.ts` exits 0. -- [ ] `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` exits 0. -- [ ] `bun run test` exits 0. -- [ ] `bun run lint` exits 0. -- [ ] No files outside the in-scope list are modified. -- [ ] `plans/README.md` status row updated. - -## STOP conditions - -Stop and report if: - -- Existing clients/tests intentionally depend on exact raw git stderr in the HTTP response. -- Typed error handling requires changing shared API error contracts outside the in-scope files. -- You encounter credentials in logs or tests; do not copy them, report the location only. - -## Maintenance notes - -Keep provider and git diagnostics observable in server logs, but keep client responses stable. This makes future clone strategy changes possible without breaking API consumers. diff --git a/plans/005-route-code-uploads-through-code-workflow.md b/plans/005-route-code-uploads-through-code-workflow.md deleted file mode 100644 index b935b029..00000000 --- a/plans/005-route-code-uploads-through-code-workflow.md +++ /dev/null @@ -1,226 +0,0 @@ -# Plan 005: Route direct code uploads through the code workflow - -> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. -> -> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/graph/src/file-type.ts apps/api/src/lib/graph-upload-file-type.ts apps/api/src/routes/graph.ts apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts apps/worker/workflows/process-file.ts apps/worker/workflows/process-files-spec.ts apps/worker/workflows/process-code-file.ts packages/graph/src/code/file-path.ts packages/graph/src/__tests__` -> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. - -## Status - -- **Priority**: P2 -- **Effort**: M -- **Risk**: MED -- **Depends on**: none -- **Category**: bug -- **Planned at**: commit `1dea5eb77`, 2026-06-13 - -## Why this matters - -The branch adds a `code` graph file type and a dedicated `process-code-file` workflow, but ordinary uploads and archive-expanded source files still infer `.ts/.tsx/.js/.jsx/.mts/.cts` as `text`. Even if inference is fixed, the parent workflow currently routes all children through the code workflow only when the whole batch has `input.code`. Users uploading source files directly would miss code-specific graph extraction, while mixed batches risk being routed all-code or all-generic. - -## Current state - -Relevant files: - -- `packages/graph/src/file-type.ts` — declares `code` but does not infer code extensions. -- `packages/graph/src/code/file-path.ts` — existing supported-code path predicate used by repository URL imports. -- `apps/api/src/lib/graph-upload-file-type.ts` — direct upload type inference. -- `apps/api/src/routes/graph.ts` — file and repository URL routes enqueue `processFilesSpec`. -- `apps/worker/workflows/process-file.ts` — parent workflow chooses child workflow. -- `apps/worker/workflows/process-code-file.ts` — code-file workflow already handles rows without repository metadata using fallbacks. - -Current file type state: - -```ts -// packages/graph/src/file-type.ts:1-21 -export const GRAPH_FILE_TYPES = [ - "pdf", - ... - "toml", - "code", - "text", -] as const; -``` - -```ts -// packages/graph/src/file-type.ts:52-187 -export function inferGraphFileType(file: Pick): GraphFileType { - ... - if (mime === "application/toml" || mime === "text/toml" || ext === "toml") { - return "toml"; - } - - return "text"; -} -``` - -Direct upload caller: - -```ts -// apps/api/src/lib/graph-upload-file-type.ts:23-31 -export function inferSupportedUploadedFiles(files: FileWithChecksum[]): UploadFileTypeCheck { - const typedFiles: SupportedFileWithChecksum[] = []; - - for (const fileWithChecksum of files) { - const type = inferGraphFileType(fileWithChecksum.file); - typedFiles.push({ ...fileWithChecksum, type }); - } - - return { ok: true, files: typedFiles }; -} -``` - -Current workflow routing is batch-wide: - -```ts -// apps/worker/workflows/process-file.ts:47-56 -function fileProcessingWorkflow(input: { graphId: string; code?: { kind: "repository" } }, fileId: string, codeManifestKey?: string) { - return input.code - ? { - spec: processCodeFile.spec, - input: { - graphId: input.graphId, - fileId, - ...(codeManifestKey ? { codeManifestKey } : {}), - }, - } -``` - -Code workflow fallback for non-repository metadata: - -```ts -// apps/worker/workflows/process-code-file.ts:34-40 -return { - fileId: file.id, - repositoryUrl: metadata?.repositoryUrl ?? `graph:${file.graphId}`, - repositoryName: metadata?.repositoryName ?? "code", - commitSha: metadata?.commitSha ?? "unknown", - path: metadata?.path ?? file.name, - content, -}; -``` - -Repo conventions: - -- Keep changes minimal and local. -- Use existing `isSupportedCodePath` rather than duplicating extension lists. -- API route tests assert workflow inputs using arrays in `graph-archive-upload-route.test.ts`. -- Root verification commands available: `bun run test`, `bun run lint`. `bun run test` passed at planning time; `bun run lint` exited 0 with one existing frontend warning. - -## Commands you will need - -| Purpose | Command | Expected on success | -|---|---|---| -| Graph tests | `bun test packages/graph/src` | exit 0 | -| API route tests | `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` | exit 0 | -| Worker tests | `bun test apps/worker` | exit 0 | -| Workspace tests | `bun run test` | exit 0 | -| Lint | `bun run lint` | exit 0; no new errors | - -## Scope - -**In scope**: - -- `packages/graph/src/file-type.ts` -- `packages/graph/src/code/file-path.ts` only if exporting/reusing a helper requires a small adjustment -- `apps/api/src/lib/graph-upload-file-type.ts` -- `apps/api/src/routes/graph.ts` -- `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` -- `apps/worker/workflows/process-file.ts` -- `apps/worker/workflows/process-files-spec.ts` -- `apps/worker/workflows/process-code-file.ts` only if its input schema needs a small metadata/manfiest adjustment -- Existing tests under `packages/graph/src/__tests__` or `packages/graph/src/code/__tests__` - -**Out of scope**: - -- Adding support for languages beyond the existing `isSupportedCodePath` set. -- Changing repository URL cloning/loading behavior. -- Changing code graph AST extraction semantics. -- Rewriting process run status handling. - -## Git workflow - -- Branch name suggestion: `advisor/005-route-code-uploads-through-code-workflow`. -- Commit message style: `fix(graph): route code uploads through code workflow`. -- Do not push or open a PR unless instructed. - -## Steps - -### Step 1: Classify supported source files as `code` - -Update `inferGraphFileType` so paths accepted by `isSupportedCodePath` return `code` before falling back to `text`. Keep structured data formats (`json`, `csv`, `xml`, `yaml`, `toml`) classified as their existing types, not code. - -Add tests for at least: - -- `src/index.ts` → `code` -- `src/component.tsx` → `code` -- `src/script.js` → `code` -- `README.md` → `text` or existing behavior -- `data.json` remains `json` - -**Verify**: `bun test packages/graph/src` → exit 0; file type tests pass. - -### Step 2: Route child workflows by actual file type, not a batch-wide flag - -Change `apps/worker/workflows/process-file.ts` so `processFiles` can process mixed batches safely: - -- Query the selected file IDs and their `type` once near the start of the parent workflow, or inside `fileProcessingWorkflow` via a small precomputed map. -- Use `processCodeFile` only for files whose DB `type` is `code`. -- Use `processFile` for all other file types. -- Keep `codeManifestKey` optional and pass it only to code-file child workflows. -- Preserve repository URL behavior: repository imports still prepare a shared manifest and code files still receive it. - -Avoid a batch-wide `input.code` switch for child selection. Keeping the `code` field as a hint for manifest preparation is acceptable; removing it is acceptable only if all API callers and tests are migrated cleanly. - -**Verify**: `bun test apps/worker` → exit 0. - -### Step 3: Enqueue direct code uploads without breaking mixed file batches - -Update `apps/api/src/routes/graph.ts` only if needed after Step 2: - -- Direct file uploads should continue enqueueing `processFilesSpec` with all added file IDs. -- Repository URL uploads should continue passing enough information to prepare a repository manifest. -- Mixed upload batches containing one `.ts` file and one non-code file must not route the non-code file through `processCodeFile`. - -Add or extend route tests in `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` for direct or archive-expanded code files. Assert code files are inserted with `type: "code"`. If the test harness can observe child workflow selection only at parent input level, cover the worker selection in worker tests instead. - -**Verify**: `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` → exit 0. - -### Step 4: Run repo checks - -**Verify**: `bun run test` → exit 0. - -**Verify**: `bun run lint` → exit 0; no new errors. - -## Test plan - -- Graph/file-type test for supported code extensions and structured-format precedence. -- Worker test for mixed batch child workflow selection: code file uses `processCodeFile`, text/PDF/etc. file uses `processFile`. -- API route test that direct or archive-expanded `.ts` files are stored as `code`. -- Existing repository URL tests must still assert `code: { kind: "repository" }` or the new equivalent manifest hint. - -## Done criteria - -- [ ] Supported TypeScript/JavaScript source paths infer `GraphFileType` `code`. -- [ ] Direct and archive-expanded code uploads are inserted with `type: "code"`. -- [ ] Mixed batches route code files to `processCodeFile` and non-code files to `processFile`. -- [ ] Repository URL imports still build repository metadata and shared manifests. -- [ ] `bun test packages/graph/src` exits 0. -- [ ] `bun test apps/worker` exits 0. -- [ ] `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` exits 0. -- [ ] `bun run test` exits 0. -- [ ] `bun run lint` exits 0. -- [ ] No files outside the in-scope list are modified. -- [ ] `plans/README.md` status row updated. - -## STOP conditions - -Stop and report if: - -- `isSupportedCodePath` accepts extensions that should remain structured document types in product behavior. -- OpenWorkflow cannot dispatch child workflows dynamically based on a precomputed type map. -- Fixing mixed-batch routing requires changing process-run schema or adding new persistent status values. - -## Maintenance notes - -The important invariant after this plan: file type decides child workflow, repository metadata only improves cross-file code graph quality. Do not reintroduce a batch-wide switch that assumes every file in one upload batch has the same processing path. diff --git a/plans/006-support-external-github-code-files.md b/plans/006-support-external-github-code-files.md deleted file mode 100644 index 625eb7ed..00000000 --- a/plans/006-support-external-github-code-files.md +++ /dev/null @@ -1,356 +0,0 @@ -# Plan 006: Support external GitHub code files without copying them to S3 - -> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. -> -> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/db/src/tables/graph.ts apps/api/src/routes/graph.ts apps/api/src/lib/repository-url.ts apps/api/src/lib/graph-file-proxy.ts apps/worker/workflows/process-code-file.ts apps/worker/lib/code-manifest.ts packages/graph/src/code/metadata.ts packages/files/src` -> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. - -## Status - -- **Priority**: P1 -- **Effort**: L -- **Risk**: MED -- **Depends on**: `003-validate-repository-import-models`, `004-sanitize-repository-url-errors`, ideally `005-route-code-uploads-through-code-workflow` -- **Category**: feature / storage architecture -- **Planned at**: commit `1dea5eb77`, 2026-06-13 - -## Why this matters - -Repository code imports currently create one `files` row per source file and upload every selected source file into S3. For GitHub repositories this is unnecessary: code content has a stable external origin when addressed by commit SHA and path. The product needs files that can be external so code imports can link to GitHub directly, avoid S3 duplication, and still feed the worker/code graph pipeline. - -The target invariant: **internal uploaded files use S3; external GitHub code files use immutable GitHub commit+path references and are fetched only when processing/proxying requires bytes.** - -## Current state - -Relevant files: - -- `packages/db/src/tables/graph.ts` — `files` table assumes every file has a non-null S3 key. -- `apps/api/src/routes/graph.ts` — repository URL route uploads each repository source to S3 before inserting DB rows. -- `apps/api/src/lib/repository-url.ts` — clones repository, enumerates supported files, returns content. -- `packages/graph/src/code/metadata.ts` — code metadata only stores repository URL/name/commit/path. -- `apps/worker/workflows/process-code-file.ts` — code workflow reads code content from S3 using `fileData.key`. -- `apps/worker/lib/code-manifest.ts` — repository manifest preparation reads every matching code file from S3. -- `apps/api/src/lib/graph-file-proxy.ts` — file proxy streams only S3 objects. - -Current `files` schema excerpt: - -```ts -// packages/db/src/tables/graph.ts:136-148 -name: text("name").notNull(), -size: integer("file_size").notNull(), -type: text("file_type").notNull(), -mimeType: text("mime_type").notNull(), -key: text("file_key").notNull(), -checksum: text("checksum"), -deleted: boolean("deleted").default(false), -status: text("status", { enum: FILE_PROCESS_STATUS_VALUES }).notNull().default("processing"), -processStep: text("process_step", { enum: FILE_PROCESS_STEP_VALUES }).notNull().default("pending"), -processErrorCode: text("process_error_code").$type(), -tokenCount: integer("token_count").notNull().default(0), -metadata: text("metadata"), -``` - -Current repository URL route copies to S3: - -```ts -// apps/api/src/routes/graph.ts:652-667 -const upload = await putGraphFile(existingGraph.id, fileId, source.file, env.S3_BUCKET); -uploadedFiles.push({ - graphId: existingGraph.id, - fileId, - name: source.name, - size: source.size, - type: "code", - mimeType: "text/plain", - key: upload.key, - checksum: source.checksum, - metadata: serializeCodeFileMetadata({ - repositoryUrl: source.repository.url, - repositoryName: source.repository.name, - commitSha: source.repository.commitSha, - path: source.path, - }), -}); -``` - -Current code workflow reads S3: - -```ts -// apps/worker/workflows/process-code-file.ts:79-93 -const paths = getGraphFileArtifactPaths({ - graphId: input.graphId, - fileId: input.fileId, - fileKey: fileData.key, -}); -... -const source = await getFile(fileData.key, env.S3_BUCKET, "text"); -if (!source) { - throw new Error("File content not found"); -} -``` - -Current code metadata: - -```ts -// packages/graph/src/code/metadata.ts:3-30 -export type CodeFileMetadata = Omit; -... -return { - repositoryUrl: parsed.repositoryUrl, - repositoryName: parsed.repositoryName, - commitSha: parsed.commitSha, - path: parsed.path, -}; -``` - -Repo conventions: - -- Run commands from the repo root. -- Do not run `bun run db:migrate`. -- Do not hand-create migrations first; for custom/manual migrations run `bun run db:generate --custom` before editing. -- Use `better-result` / `Result.tryPromise` in routes when mapping expected async errors. -- Keep changes minimal and local; no compatibility shims unless explicitly required. -- Root verification commands available: `bun run test`, `bun run lint`. At planning time, `bun run test` passed; `bun run lint` exited 0 with one existing frontend warning. - -## Design decision - -Use explicit file storage origin columns instead of overloading `file_key` or only `metadata`. - -Add these concepts: - -- `files.storage_kind`: enum-like text, `"internal" | "external"`, default `"internal"`, not null. -- `files.external_url`: nullable text. Canonical immutable raw-content URL for external files. -- `files.external_provider`: nullable text, initially only `"github"`. -- Keep `files.file_key` non-null for now, but for external GitHub code set it to a deterministic synthetic key: `external:github:/@:`. This preserves existing key-based DB lookups, dedupe UI references, and unique indexes while making it clear the key is not an S3 key. - -External GitHub code metadata must include enough to rebuild safe URLs without trusting arbitrary user input: - -```ts -type CodeFileMetadata = { - repositoryUrl: string; - repositoryName: string; - commitSha: string; - path: string; - external?: { - provider: "github"; - rawUrl: string; // https://raw.githubusercontent.com//// - htmlUrl: string; // https://github.com///blob// - }; -}; -``` - -Security invariant: external URLs are generated by server-side normalization from allowed GitHub repository URL + commit SHA + supported code path. Never accept arbitrary external URLs from request bodies in this plan. - -## Commands you will need - -| Purpose | Command | Expected on success | -|---|---|---| -| Generate migration scaffold | `bun run db:generate --custom` | creates a new migration scaffold to edit | -| DB tests | `bun test packages/db/src/__tests__/migration-compat.test.ts` | exit 0 | -| API repository tests | `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts apps/api/src/lib/__tests__/repository-url.test.ts` | exit 0 | -| Worker code tests | `bun test apps/worker/lib/__tests__/code-manifest.test.ts packages/graph/src/code/__tests__/repository.test.ts` | exit 0 | -| Workspace tests | `bun run test` | exit 0 | -| Lint | `bun run lint` | exit 0; no new errors | - -## Scope - -**In scope**: - -- `packages/db/src/tables/graph.ts` -- New migration under `migrations/` created by `bun run db:generate --custom` -- `packages/db/src/__tests__/migration-compat.test.ts` -- `packages/graph/src/code/metadata.ts` -- `apps/api/src/lib/repository-url.ts` -- `apps/api/src/routes/graph.ts` -- `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` -- `apps/api/src/lib/__tests__/repository-url.test.ts` -- `apps/api/src/lib/graph-file-proxy.ts` -- `apps/worker/workflows/process-code-file.ts` -- `apps/worker/lib/code-manifest.ts` -- `apps/worker/lib/__tests__/code-manifest.test.ts` -- A small shared helper module if needed for external file access, preferably near existing file/code helpers. - -**Out of scope**: - -- Accepting arbitrary external URLs from users. -- Supporting private GitHub repositories or GitHub API tokens. -- Supporting GitLab/Bitbucket external raw links. Keep existing non-GitHub behavior internal/S3 unless product explicitly expands scope. -- Externalizing PDFs/images/audio/video/documents. -- Removing S3 support for normal uploads. -- Solving repository clone budget limits; plan 006 can coexist with later partial-clone/no-clone work. - -## Git workflow - -- Branch name suggestion: `advisor/006-external-github-code-files`. -- Commit message style: `feat(files): support external github code files`. -- Do not push or open a PR unless instructed. - -## Steps - -### Step 1: Add file storage origin columns - -Run `bun run db:generate --custom` from the repo root. Edit the generated migration to add: - -- `storage_kind text NOT NULL DEFAULT 'internal'` on `files`. -- `external_url text` on `files`. -- `external_provider text` on `files`. -- A check constraint requiring internal files to have no external URL/provider and external files to have both. Suggested shape: - - internal: `storage_kind = 'internal' AND external_url IS NULL AND external_provider IS NULL` - - external: `storage_kind = 'external' AND external_url IS NOT NULL AND external_provider IS NOT NULL` -- A provider check limiting `external_provider` to `github` when not null. - -Update `packages/db/src/tables/graph.ts` with matching columns and checks. Keep `key` non-null. - -Add migration compatibility tests in `packages/db/src/__tests__/migration-compat.test.ts` for the new columns and checks. - -**Verify**: `bun test packages/db/src/__tests__/migration-compat.test.ts` → exit 0. - -### Step 2: Extend code metadata and GitHub URL helpers - -In `packages/graph/src/code/metadata.ts`, extend `CodeFileMetadata` parsing/serialization to preserve optional external GitHub link metadata. Validate all external fields strictly: - -- `provider === "github"` -- `rawUrl` is HTTPS, host `raw.githubusercontent.com` -- `htmlUrl` is HTTPS, host `github.com` - -In `apps/api/src/lib/repository-url.ts`, add helpers that only emit external links for GitHub repository URLs: - -- Parse normalized repo URL `https://github.com//.git`. -- Build immutable raw URL: `https://raw.githubusercontent.com////`. -- Build immutable HTML URL: `https://github.com///blob//`. -- Build synthetic key: `external:github:/@:`. - -Use `encodeURIComponent` only where URL path construction requires it; do not double-encode slashes in repository file paths. Add unit tests for paths with spaces and nested directories. - -**Verify**: `bun test apps/api/src/lib/__tests__/repository-url.test.ts` → exit 0. - -### Step 3: Stop uploading GitHub repository code files to S3 - -In `apps/api/src/routes/graph.ts`, change only the repository URL add path: - -- For GitHub repository sources, skip `putGraphFile`. -- Insert `files` rows with: - - `type: "code"` - - `mimeType: "text/plain"` - - `key: synthetic external key` - - `storageKind: "external"` - - `externalProvider: "github"` - - `externalUrl: raw GitHub URL` - - `checksum: source.checksum` if content was available during repository enumeration - - `metadata` including `external.rawUrl` and `external.htmlUrl` -- For non-GitHub providers, keep existing S3 upload behavior unless Step 2 chose to reject externalization for them explicitly. -- Ensure cleanup code does not call `deleteFile` for external synthetic keys. - -Update `apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts`: - -- Existing repository URL test should expect zero S3 uploads for GitHub imports. -- Inserted file values should include `storageKind: "external"`, external provider/url, synthetic key, and code metadata with GitHub links. -- Workflow input should still enqueue `processFilesSpec` with `code: { kind: "repository" }` unless plan 005 already changed this contract. -- Duplicate checksum behavior should still skip already-present code files. - -**Verify**: `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` → exit 0. - -### Step 4: Add a safe external content reader for workers - -Add a small helper used by `process-code-file` and `code-manifest`: - -```ts -type FileContentSource = - | { kind: "internal"; key: string } - | { kind: "external"; provider: "github"; url: string }; -``` - -Behavior: - -- Internal source: use existing `getFile(key, env.S3_BUCKET, "text")`. -- External GitHub source: fetch `external_url` with `fetch` only after validating HTTPS host `raw.githubusercontent.com`. -- Enforce a hard response size limit consistent with repository code limits. Do not read unbounded responses. -- Require `2xx` status and text content; classify failures as file processing failures. -- Do not follow redirects to non-allowlisted hosts. If the runtime follows redirects automatically, set redirect handling explicitly and validate the final URL if the Fetch API exposes it. - -Use this helper in: - -- `apps/worker/workflows/process-code-file.ts` instead of direct `getFile(fileData.key, ...)`. -- `apps/worker/lib/code-manifest.ts` instead of direct `getFile(row.key, ...)`. - -Add tests for: - -- Internal S3 path still works. -- External GitHub URL is fetched and returned. -- Non-GitHub external URL is rejected before network fetch. -- Oversized external response is rejected. - -**Verify**: `bun test apps/worker/lib/__tests__/code-manifest.test.ts` → exit 0. - -### Step 5: Make file proxy external-aware - -Update `apps/api/src/lib/graph-file-proxy.ts` so external GitHub files do not call S3 metadata/stream APIs: - -- Load `storageKind`, `externalProvider`, `externalUrl`, `size`, `mimeType`, `name` with the file row. -- For `storageKind === "internal"`, keep existing behavior. -- For external GitHub code files, return either: - - a `302`/`307` redirect to the immutable GitHub HTML URL from metadata for browser viewing, or - - a proxied raw response fetched from `raw.githubusercontent.com` if current UI expects same-origin bytes. - -Prefer redirecting to the GitHub HTML URL for user-facing file open/download behavior. If the existing route requires byte-range preview, use proxied raw bytes for `Range` requests and keep the same headers. - -Add focused tests for external proxy behavior. Do not expose arbitrary external URLs. - -**Verify**: run the relevant API test file containing graph-file-proxy tests; if no file exists, add a focused test next to existing API lib tests and run it with `bun test `. - -### Step 6: Preserve source/citation behavior - -Check source references that include `file_key` still work with synthetic external keys: - -- `apps/api/src/lib/source-reference-record.ts` -- `apps/api/src/lib/source-reference.ts` -- `apps/api/src/routes/graph-files.ts` - -If UI/API output needs an external URL, add it explicitly to the response shape rather than overloading `file_key`. Keep existing `file_key` for backward-compatible identification inside this branch's clean cutover. - -**Verify**: run source-reference/graph-files tests if touched. - -### Step 7: Run repo checks - -**Verify**: `bun run test` → exit 0. - -**Verify**: `bun run lint` → exit 0; no new errors. - -## Test plan - -- DB migration compatibility test for `storage_kind`, `external_url`, `external_provider`, and checks. -- Repository URL helper tests for GitHub raw/html/synthetic-key generation. -- Route test proving GitHub repository code imports insert external file rows without S3 uploads. -- Worker tests for external GitHub content fetch and allowlist rejection. -- Proxy test for external GitHub file behavior. -- Existing repository URL duplicate/checksum tests must still pass. - -## Done criteria - -- [ ] `files` table supports explicit internal/external origin with constraints. -- [ ] GitHub repository URL imports create external code file rows and do not upload source contents to S3. -- [ ] External rows use immutable commit SHA raw/html GitHub links generated server-side. -- [ ] Worker code processing can read external GitHub code content safely. -- [ ] Code manifest preparation can include external GitHub files without S3 reads. -- [ ] File proxy/source reference behavior is defined for external files. -- [ ] Non-GitHub repository imports keep existing behavior or are explicitly rejected with a safe message; no silent partial externalization. -- [ ] `bun test packages/db/src/__tests__/migration-compat.test.ts` exits 0. -- [ ] `bun test apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts apps/api/src/lib/__tests__/repository-url.test.ts` exits 0. -- [ ] Worker/API proxy focused tests exit 0. -- [ ] `bun run test` exits 0. -- [ ] `bun run lint` exits 0. -- [ ] `plans/README.md` status row updated. - -## STOP conditions - -Stop and report if: - -- The product needs private GitHub repositories in the first version. This plan intentionally supports public immutable GitHub links only. -- The DB migration generator wants broad unrelated schema changes. -- Existing UI requires same-origin byte streaming for code files and cannot tolerate redirects to GitHub HTML URLs. -- External content fetch cannot enforce host allowlisting and response-size limits in the current runtime. -- Non-GitHub repository support must remain no-S3 in the same release; that expands provider-specific URL generation and needs a separate design decision. - -## Maintenance notes - -Do not hide external state only in `metadata`. Storage origin is a first-class file invariant because cleanup, proxying, worker reads, and dedupe all branch on it. Future providers should add provider-specific URL builders and allowlists; they should not accept arbitrary URLs from clients. diff --git a/plans/007-version-code-sources-with-valid-until.md b/plans/007-version-code-sources-with-valid-until.md deleted file mode 100644 index fb737f82..00000000 --- a/plans/007-version-code-sources-with-valid-until.md +++ /dev/null @@ -1,431 +0,0 @@ -# Plan 007: Version code sources with `validUntil` so graph tools use only the latest repository snapshot - -> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. -> -> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/db/src/tables/graph.ts apps/api/src/routes/graph.ts apps/api/src/lib/graph-route.ts apps/worker/lib/save-graph.ts apps/worker/lib/regenerate-descriptions.ts apps/worker/workflows/process-file.ts apps/worker/workflows/process-code-file.ts packages/ai/src/tools/source.ts packages/ai/src/tools/entity.ts apps/api/src/lib/source-reference.ts apps/api/src/lib/chat.ts apps/api/src/lib/team-chat.ts apps/api/src/lib/graph-file-proxy.ts packages/graph/src/code/identity.ts packages/graph/src/code/repository.ts` -> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. - -## Status - -- **Priority**: P1 -- **Effort**: L -- **Risk**: HIGH -- **Depends on**: `005-route-code-uploads-through-code-workflow`, `006-support-external-github-code-files` -- **Category**: correctness / source history -- **Planned at**: commit `1dea5eb77`, 2026-06-13 - -## Why this matters - -Repository code imports are snapshots. Re-uploading the same repository at a newer commit should make the graph represent the latest code, not append new snippets to the old function evidence forever. - -Today source rows are append-only. When a function changes, the canonical deduped entity receives both old and new sources; description regeneration reads all sources for the entity, so the function description can mix stale and latest code. When a function is removed, the old entity can remain active because nothing invalidates its sources. - -The target invariant: **code entities and relationships use only current valid sources; historical sources stay in the database with `validUntil` for future change-history features.** - -## Current state - -Relevant files: - -- `packages/db/src/tables/graph.ts` — `sources` has `active` but no validity window. -- `apps/worker/lib/save-graph.ts` — inserts new sources and dedupes entities/relationships, but does not invalidate old sources. -- `apps/worker/lib/regenerate-descriptions.ts` — regenerates descriptions from every linked source, not only current sources. -- `packages/ai/src/tools/source.ts` — graph source tools filter `sources.active = true` but not source validity. -- `packages/ai/src/tools/entity.ts` — file-scoped entity listing checks for any linked source, not only valid sources. -- `apps/api/src/lib/source-reference.ts`, `apps/api/src/lib/chat.ts`, `apps/api/src/lib/team-chat.ts` — source ID lookup paths do not reject invalid sources. -- `apps/api/src/routes/graph.ts` and `apps/api/src/lib/graph-route.ts` — repository imports insert files by checksum and do not model a repository update/supersession. - -Current source schema: - -```ts -// packages/db/src/tables/graph.ts:291-314 -export const sourcesTable = pgTable.withRLS( - "sources", - { - id: text("id") - .primaryKey() - .$default(() => ulid()), - entityId: text("entity_id").references(() => entityTable.id, { onDelete: "cascade" }), - relationshipId: text("relationship_id").references(() => relationshipTable.id, { onDelete: "cascade" }), - textUnitId: text("text_unit_id") - .notNull() - .references(() => textUnitTable.id, { onDelete: "cascade" }), - active: boolean("active").notNull().default(false), - description: text("description").notNull(), - sourceChunkIds: json("source_chunk_ids") - .$type() - .notNull() - .default(sql`'[]'::json`), - embedding: vector("embedding", { dimensions: 4096 }).notNull(), - searchTsv: tsvector("search_tsv").generatedAlwaysAs(() => weightedTsvectorGenerated(["description"])), - createdAt: timestamp("created_at", { withTimezone: true, mode: "date" }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, mode: "date" }) - .defaultNow() - .$onUpdate(() => sql`NOW()`), - }, -``` - -Current save behavior: - -```ts -// apps/worker/lib/save-graph.ts:118-143 -const sourceRows = [ - ...graph.entities.flatMap((entity) => - entity.sources.map((source) => ({ - id: source.id, - entityId: entity.id, - relationshipId: null, - textUnitId: source.unitId, - active: false, - description: source.description, - sourceChunkIds: source.sourceChunkIds ?? [], - embedding: EMPTY_VECTOR_SQL, - })) - ), - ...graph.relationships.flatMap((relationship) => - relationship.sources.map((source) => ({ - id: source.id, - entityId: null, - relationshipId: relationship.id, - textUnitId: source.unitId, - active: false, - description: source.description, - sourceChunkIds: source.sourceChunkIds ?? [], - embedding: EMPTY_VECTOR_SQL, - })) - ), -]; -``` - -```ts -// apps/worker/lib/save-graph.ts:199-200 -for (const chunk of chunkItems(rows.sourceRows)) { - await tx.insert(sourcesTable).values(chunk).onConflictDoNothing(); -} -``` - -Current description regeneration uses stale sources too: - -```ts -// apps/worker/lib/regenerate-descriptions.ts:148-163 -const sources = await db - .select({ - id: sourcesTable.id, - entityId: sourcesTable.entityId, - description: sourcesTable.description, - }) - .from(sourcesTable) - .where( - and( - inArray( - sourcesTable.entityId, - entities.map((entity) => entity.id) - ), - isNotNull(sourcesTable.entityId) - ) - ); -``` - -Current source tool validity is only `active`: - -```ts -// packages/ai/src/tools/source.ts:205-207 -if (terms.length === 0) { - const clauses = [eq(sourcesTable.active, true), eq(filesTable.graphId, graphId)]; -``` - -```ts -// packages/ai/src/tools/source.ts:306-310 -from sources source -inner join text_units text_unit on text_unit.id = source.text_unit_id -inner join files file on file.id = text_unit.file_id -where source.active = true - and file.graph_id = ${graphId} -``` - -Current code identity makes dedupe necessary across commits: - -```ts -// packages/graph/src/code/identity.ts:8-13 -export function fileEntityName(file: Pick): string { - return `${file.repositoryUrl}:${file.path}`; -} - -export function fileEntityId(repositoryUrl: string, commitSha: string, filePath: string): string { - return stableId("code_file", repositoryUrl, commitSha, filePath); -} -``` - -The entity name excludes commit, but the generated ID includes commit. `save-graph.ts` dedupes entities by graph/type/normalized name and rewrites new sources to the canonical active entity, so changed functions append new sources to the old canonical entity unless old sources are invalidated. - -Repo conventions: - -- Run commands from the repo root. -- Do not run `bun run db:migrate`. -- Do not hand-create migrations first; for custom/manual migrations run `bun run db:generate --custom` before editing. -- Use Drizzle schema + migration compatibility tests for schema changes. -- Keep changes local; prefer shared helpers for repeated validity predicates. -- Root verification commands available: `bun run test`, `bun run lint`. At planning time, `bun run test` passed; `bun run lint` exited 0 with one existing frontend warning. - -## Design decision - -Add `sources.valid_until` as the history boundary. Do not delete old code sources. - -Definition: - -- A **current source** is `sources.active = true AND sources.valid_until IS NULL`. -- A **pending current source** is `sources.active = false AND sources.valid_until IS NULL` and exists only between graph save and description/embedding activation. -- An **historical source** has `sources.valid_until IS NOT NULL`. Historical rows keep their text unit, chunk IDs, description, embedding, and active flag for future history queries, but current graph tools ignore them. - -This plan keeps `entities` and `relationships` as the current graph surface: - -- If a subject has one or more current sources, regenerate its description from current sources only and keep it active. -- If a code subject has no current sources after a repository update, deactivate it so graph tools stop returning removed functions/edges. - -## Commands you will need - -| Purpose | Command | Expected on success | -|---|---|---| -| Generate migration scaffold | `bun run db:generate --custom` | creates a new migration scaffold to edit | -| DB migration tests | `bun test packages/db/src/__tests__/migration-compat.test.ts` | exit 0 | -| Worker focused tests | `bun test apps/worker/lib apps/worker/workflows` | exit 0 | -| AI tool tests | `bun test packages/ai/src/__tests__` | exit 0 | -| API source tests | `bun test apps/api/src/lib/__tests__ apps/api/src/routes/__tests__` | exit 0 | -| Workspace tests | `bun run test` | exit 0 | -| Lint | `bun run lint` | exit 0; no new errors | - -## Scope - -**In scope**: - -- `packages/db/src/tables/graph.ts` -- New migration under `migrations/` created by `bun run db:generate --custom` -- `packages/db/src/__tests__/migration-compat.test.ts` -- `apps/worker/lib/save-graph.ts` -- `apps/worker/lib/regenerate-descriptions.ts` -- `apps/worker/workflows/process-file.ts` -- `apps/worker/workflows/process-code-file.ts` -- `apps/api/src/routes/graph.ts` -- `apps/api/src/lib/graph-route.ts` -- `packages/ai/src/tools/source.ts` -- `packages/ai/src/tools/entity.ts` -- `packages/ai/src/tools/correction.ts` if corrections can target stale source IDs -- `apps/api/src/lib/source-reference.ts` -- `apps/api/src/lib/chat.ts` -- `apps/api/src/lib/team-chat.ts` -- Tests for the touched modules - -**Out of scope**: - -- Building the webhook integration. -- Building a user-facing history UI. -- Arbitrary non-code document source versioning. -- Changing source IDs already emitted in old chat messages beyond making invalid IDs no longer resolve as current citations. -- Rewriting code entity identity to remove commit SHA from IDs. Dedupe by name remains the current bridge across commits. - -## Git workflow - -- Branch name suggestion: `advisor/007-version-code-sources-valid-until`. -- Commit message style: `feat(graph): version code sources with validUntil`. -- Do not push or open a PR unless instructed. - -## Steps - -### Step 1: Add `sources.valid_until` - -Run `bun run db:generate --custom` from the repo root. Edit the generated migration to add: - -```sql -ALTER TABLE "sources" ADD COLUMN "valid_until" timestamp with time zone; -``` - -Update `packages/db/src/tables/graph.ts`: - -- Add `validUntil: timestamp("valid_until", { withTimezone: true, mode: "date" })` to `sourcesTable`. -- Keep existing `active` semantics; do not replace it with `validUntil`. -- Add indexes for current source lookups. Suggested indexes: - - `sources_entity_current_id_idx` on `(entity_id, active, id)` where `valid_until IS NULL`. - - `sources_relationship_current_id_idx` on `(relationship_id, active, id)` where `valid_until IS NULL`. - - `sources_current_id_idx` on `(active, id)` where `valid_until IS NULL`. - -Extend `packages/db/src/__tests__/migration-compat.test.ts` so the migration and snapshot include `sources.valid_until` and the current-source indexes. - -**Verify**: `bun test packages/db/src/__tests__/migration-compat.test.ts` → exit 0. - -### Step 2: Centralize the current-source predicate - -Add a tiny helper where it can be imported without cycles. Preferred options: - -- `packages/db/src/source-validity.ts`, exporting Drizzle predicates/helpers, or -- local helpers in each package if the DB package should avoid query helpers. - -The helper must express: - -```ts -source.active = true AND source.validUntil IS NULL -``` - -For raw SQL in `packages/ai/src/tools/source.ts`, use the exact SQL predicate: - -```sql -source.active = true AND source.valid_until IS NULL -``` - -Do not use only `active` in any current source query after this plan. - -**Verify**: no command yet; this step is covered by later tests. - -### Step 3: Make repository imports represent a latest snapshot - -Update the repository URL add flow so uploading the same repository is an update, not an append-only import. - -In `apps/api/src/routes/graph.ts` / `apps/api/src/lib/graph-route.ts`: - -- Detect repository scope by normalized `repository.url`. -- For repository imports, do not skip files solely because their checksum already exists from a previous commit of the same repository. Latest snapshots need rows for unchanged files too, especially because external GitHub URLs include the latest commit SHA. -- Create a new file row for every supported file in the latest repository snapshot. -- Keep current duplicate protection for repeated URLs within the same request. -- Mark older non-deleted code file rows for the same graph + repository URL as `deleted = true` only after the new file rows and process run have been committed successfully. Do not call the delete-file workflow; old text units and sources must remain for history. -- If a DB transaction fails, do not mark old files deleted and clean up only internal S3 uploads. - -Implementation hint: repository URL is currently only inside `files.metadata`. If querying JSON text is too brittle, add a small repository import manifest table or include a stable repository scope field in code file metadata parsing before building the SQL. Keep this plan scoped to repository code imports only. - -**Verify**: route tests should prove a second import of `https://github.com/acme/widgets` creates new file rows for the latest commit and marks previous repository file rows deleted without deleting their text units/sources. - -### Step 4: Invalidate superseded sources when a code subject is refreshed - -Update `apps/worker/lib/save-graph.ts` after inserts and dedupe have rewritten new sources to canonical entity/relationship IDs: - -- Capture inserted source IDs from `rows.sourceRows`. -- Query those inserted source rows after dedupe to get their canonical `entityId` / `relationshipId` and the file metadata of their text unit. -- For inserted sources whose file is `type = 'code'`, set `valid_until = NOW()` on older current code sources for the same canonical `entityId` or `relationshipId`, excluding the newly inserted source IDs. -- Scope invalidation to code sources only by joining `sources -> text_units -> files` and requiring `files.file_type = 'code'`. -- Do not invalidate sources from PDFs/docs/manual suggestions that happen to mention the same entity. - -This handles the main case: same function or relationship appears in a changed file. The new source becomes the only current source for that function/edge; old snippets remain historical. - -**Verify**: add a worker/lib test that saves two code graphs for the same function with different snippets. After the second save + description activation, the first source has `validUntil` set and the second source remains current. - -### Step 5: Invalidate removed functions/edges at repository update finalization - -A function removed from the latest repository snapshot has no new source, so Step 4 will not touch it. Add a repository update finalizer in the parent code batch flow (`apps/worker/workflows/process-file.ts`) after all child code workflows for a repository snapshot complete successfully: - -- Identify repository URL and commit SHA for the current batch from code file metadata. -- Identify the latest file IDs in this process run. -- Find current code sources for the same graph + repository URL whose text unit belongs to older file rows not in the latest file ID set. -- Set `valid_until = NOW()` for those sources. -- Collect affected entity and relationship IDs. -- Trigger `updateDescriptionsSpec` for affected IDs so removed functions/edges are deactivated or descriptions rebuilt from remaining current sources. - -If any child code workflow fails terminally, do not run the repository finalizer. Keep old sources current until a complete latest snapshot is available. - -**Verify**: add a process-files workflow test or lower-level finalizer test covering a repository update where `old.ts#removedFunction` has no counterpart in the new file set; after finalization its source is invalid and the entity is inactive or absent from source tools. - -### Step 6: Regenerate descriptions from current sources only - -Update `apps/worker/lib/regenerate-descriptions.ts`: - -- Select only `sources.validUntil IS NULL` for entity and relationship source lists. -- `updateSourceEmbeddingsBatch` should only activate source IDs that still have `validUntil IS NULL`. -- If an entity/relationship has zero current sources: - - Set `active = false`. - - Clear or retain description consistently. Prefer clearing to `""` and setting embedding to `EMPTY_VECTOR_SQL` if the schema requires a non-null vector. - - Do not call the LLM for it. -- If it has current sources, regenerate from those current sources only and set active true. - -**Verify**: add tests for: - -- Entity description after update includes only the new source description. -- Entity with only invalidated sources becomes inactive. -- Relationship equivalent behavior. - -### Step 7: Filter graph tools and citation lookups to current sources - -Update current-source consumers: - -- `packages/ai/src/tools/source.ts` - - `get_entity_sources`, `get_relationship_sources`, semantic source search, and `get_source_file_metadata` must require `source.valid_until IS NULL` plus `source.active = true`. - - Add `file.deleted = false` where appropriate so superseded repository file rows do not leak through current tools. -- `packages/ai/src/tools/entity.ts` - - File-scoped `list_entities` subquery must require current sources, not any historical source. -- `packages/ai/src/tools/correction.ts` - - Reject correction suggestions for historical sources. -- `apps/api/src/lib/source-reference.ts` - - `loadSourceReference` and batch loading must reject historical sources for normal citation resolution. -- `apps/api/src/lib/chat.ts` and `apps/api/src/lib/team-chat.ts` - - Source ID lookups used by chat citation normalization must reject historical sources. - -Do not add history-query behavior in this plan. Historical sources should be retained but invisible to current tools. - -**Verify**: add/extend tests proving an invalidated source ID is not returned by source tools and does not resolve as a current citation. - -### Step 8: Keep history rows intact - -Confirm invalidation does not delete: - -- `sources` -- `text_units` -- old `files` rows for code snapshots - -Older file rows may be marked `deleted = true` for current file-list behavior, but text/source rows stay available for future history. Do not call `deleteProjectFile` for superseded repository snapshots. - -**Verify**: in tests, after repository update, old source/text unit rows still exist with `validUntil` set. - -### Step 9: Run repo checks - -**Verify**: `bun run test` → exit 0. - -**Verify**: `bun run lint` → exit 0; no new errors. - -## Test plan - -Add focused tests before broad checks: - -1. DB migration compatibility: - - `sources.valid_until` exists in migration and snapshot. - - Current-source indexes exist. -2. Save graph source invalidation: - - Same code function, changed snippet → old source `validUntil` set, new source current. - - Same relationship, changed snippet → old source `validUntil` set, new source current. -3. Repository update finalization: - - Removed function's only source is invalidated when a full newer snapshot succeeds. - - Finalizer does not run when any child workflow fails. -4. Description regeneration: - - Descriptions use only current sources. - - Entity/relationship with no current sources becomes inactive. -5. Tool filtering: - - `get_entity_sources` and `get_relationship_sources` never return invalidated sources. - - `get_source_file_metadata` ignores invalidated source IDs. - - File-scoped entity listing ignores invalidated sources. -6. API citation/source reference: - - Invalidated source IDs do not resolve as current citations. - -## Done criteria - -- [ ] `sources.valid_until` exists in Drizzle schema, migration SQL, and snapshot. -- [ ] Current-source queries require `active = true AND valid_until IS NULL`. -- [ ] Re-uploading a repository creates a latest snapshot rather than deduping away unchanged files by checksum alone. -- [ ] Changed functions invalidate previous code sources for the canonical function entity and store new current sources. -- [ ] Removed functions/edges from a successful latest repository snapshot stop appearing in graph tools. -- [ ] Entity/relationship descriptions are regenerated from current sources only. -- [ ] Historical source/text rows remain in the database for future history features. -- [ ] Current graph source tools and citation resolvers ignore invalidated sources. -- [ ] `bun test packages/db/src/__tests__/migration-compat.test.ts` exits 0. -- [ ] Worker focused tests exit 0. -- [ ] AI/API source focused tests exit 0. -- [ ] `bun run test` exits 0. -- [ ] `bun run lint` exits 0. -- [ ] `plans/README.md` status row updated. - -## STOP conditions - -Stop and report if: - -- Source history must preserve old GitHub commit URLs per source but the current schema only stores repository metadata on `files`. That may require source-level metadata before this plan is safe. -- Repository update processing cannot reliably know when a full snapshot has succeeded. Do not invalidate removed functions on partial success. -- Drizzle migration generation produces broad unrelated schema changes. -- Existing UI/API contracts require old source IDs from previous chat messages to remain resolvable as current citations. -- Any implementation would delete old sources/text units instead of marking `validUntil`. - -## Maintenance notes - -`active` means “embedded/generated and eligible if otherwise current.” `validUntil` means “superseded by a newer code snapshot.” Future history features should query `validUntil IS NOT NULL` explicitly and should not overload current source tools with historical data. diff --git a/plans/008-provider-connectors-for-repository-graphs.md b/plans/008-provider-connectors-for-repository-graphs.md deleted file mode 100644 index 278d2900..00000000 --- a/plans/008-provider-connectors-for-repository-graphs.md +++ /dev/null @@ -1,731 +0,0 @@ -# Plan 008: Provider connectors for private repository graphs and webhook-driven updates - -> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. -> -> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/db/src/tables apps/api/src/server.ts apps/api/src/routes apps/api/src/lib apps/worker/workflows apps/worker/lib packages/contracts/src apps/frontend/app apps/frontend/components apps/frontend/lib` -> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. - -## Status - -- **Priority**: P1 -- **Effort**: XL -- **Risk**: HIGH -- **Depends on**: `006-support-external-github-code-files`, `007-version-code-sources-with-valid-until` -- **Category**: feature / integrations / repository sync -- **Planned at**: commit `1dea5eb77`, 2026-06-13 - -## Why this matters - -The URL import path still starts from a public HTTPS repository URL and historically used `git clone`. That cannot cover private repositories cleanly and cannot keep a selected branch up to date. The product needs first-class provider connectors: - -1. A system admin creates an instance-level connector app for GitHub/GitLab. -2. A graph manager (organization admin, team admin, or team moderator) connects that app to repositories they control. -3. KIWI lists accessible repositories and branches through the provider API. -4. The user creates a graph from a selected repo + branch. -5. Provider push webhooks enqueue rebuilds when that branch moves, so the graph stays current. - -Plan 007 supplies the source invalidation model (`validUntil`) needed to replace old function evidence with latest-branch evidence instead of accumulating stale snippets. - -## Current state - -Relevant files: - -- `packages/auth/src/permissions.ts` — graph management permission vocabulary. -- `apps/api/src/lib/team-access.ts` — organization admins, team admins, and team moderators can create/manage team graphs. -- `apps/api/src/lib/graph-access.ts` — graph create/file-manage authorization helpers. -- `apps/api/src/routes/graph.ts` — graph creation and URL-based repository import. -- `apps/api/src/lib/repository-url.ts` — public URL repo loader, external GitHub URL helper, still provider-agnostic by URL. -- `apps/api/src/server.ts` — all current routes are mounted after `authMiddleware`; webhooks need a verified unauthenticated route before auth. -- `apps/worker/workflows/process-files-spec.ts` — parent workflow already accepts `code: { kind: "repository" }`. -- `apps/worker/worker.ts` — workflow registration point. -- `packages/db/src/tables/graph.ts` and `packages/db/src/tables/auth.ts` — graph/team/user schema. -- `packages/ai/src/models.ts` — existing AES-GCM-style encrypted credential storage pattern to reuse for connector secrets. -- `apps/frontend/app/(app)/settings/page.tsx` and `apps/frontend/components/settings/sections.tsx` — system-admin UI guard pattern. -- `apps/frontend/lib/api/client.ts` and `apps/frontend/lib/api/projects.ts` — frontend API client conventions. - -Current permission model: - -```ts -// packages/auth/src/permissions.ts:16-18 -group: ["create", "update", "delete", "view:all", "view", "add:user", "remove:user", "list:user"], -graph: ["view", "create", "update", "delete", "add:file", "delete:file", "list:file"], -chat: ["create"], -``` - -Current team graph manager rule: - -```ts -// apps/api/src/lib/team-access.ts:140-147 -export async function requireTeamGraphCreateAccess(user: AuthUser, teamId: string) { - const access = await requireTeamAccess(user, teamId); - if (access.organizationAdmin || access.role === "admin" || access.role === "moderator") { - return access; - } - - throw new Error(API_ERROR_CODES.FORBIDDEN); -} -``` - -Current graph creation owner resolution: - -```ts -// apps/api/src/routes/graph.ts:267-289 -const ownerResult = await Result.tryPromise(async () => { - if (body.teamId) { - const access = await assertCanCreateTeamGraph(user, body.teamId); - return { - ownerMode: "team" as const, - organizationId: access.team.organizationId, - teamId: body.teamId, - }; - } - - if (body.graphId) { - await assertCanCreateUnderParentGraph(user, body.graphId); - return { - ownerMode: "graph" as const, - graphId: body.graphId, - }; - } - - const access = await assertCanCreateTopLevelGraph(user); - return { - ownerMode: "organization" as const, - organizationId: access.organizationId, - }; -}); -``` - -Current URL repository add path enqueues code processing: - -```ts -// apps/api/src/routes/graph.ts:807-812 -const handle = await ow.runWorkflow(processFilesSpec, { - graphId: existingGraph.id, - fileIds: result.addedFiles.map((file) => file.id), - processRunId: result.processRunId, - code: { kind: "repository" }, -}); -``` - -Current repository loader still starts with a clone: - -```ts -// apps/api/src/lib/repository-url.ts:161-169 -export async function loadRepositoryFromUrl(input: string): Promise { - const repository = normalizeRepositoryUrl(input); - const tempDir = await mkdtemp(path.join(tmpdir(), "kiwi-repository-")); - const repoPath = path.join(tempDir, "repo"); - - try { - await runGit(["clone", "--depth", "1", "--", repository.url, repoPath], tempDir); - const commitSha = (await runGit(["rev-parse", "HEAD"], repoPath)).trim(); -``` - -Current route mount order means webhook routes must be special-cased: - -```ts -// apps/api/src/server.ts:53-61 -.use(mcpRoute) -.use(authMiddleware) -.use(authRoute) -.use(chatRoute) -... -.use(graphRoute) -``` - -Existing encryption pattern: - -```ts -// packages/ai/src/models.ts:162-174 -export function encryptModelCredentials(credentials: ModelCredentials, secret: string): string { - const iv = randomBytes(IV_BYTE_LENGTH); - const cipher = createCipheriv(ENCRYPTION_ALGORITHM, deriveEncryptionKey(secret), iv, { - authTagLength: AUTH_TAG_BYTE_LENGTH, - }); - ... -} - -export function decryptModelCredentials(value: string, secret: string): ModelCredentials { -``` - -Provider docs observed during planning: - -- GitHub Apps create installation access tokens with `POST /app/installations/{installation_id}/access_tokens`; tokens expire and can be scoped to repositories and permissions. `contents: read` is the critical permission for reading repository files. -- GitHub Apps list accessible repositories with `GET /installation/repositories` using an installation token. -- GitHub push webhooks include delivery IDs and push ref/after commit data; verify `X-Hub-Signature-256` with the app webhook secret. -- GitLab push hooks include `event_name: "push"`, `ref: "refs/heads/"`, `after`, project path/id, and headers such as `X-Gitlab-Event`, `X-Gitlab-Webhook-UUID`, and `X-Gitlab-Token`. - -Repo conventions: - -- Run commands from the repo root. -- Do not run `bun run db:migrate`. -- Do not hand-create migrations first; for custom/manual migrations run `bun run db:generate --custom` before editing. -- Use `Result.tryPromise` in API routes for expected async errors. -- Keep comments rare; prefer small helper modules over duplicated provider logic. -- Root verification commands available: `bun run test`, `bun run lint`. At planning time, `bun run test` passed; `bun run lint` exited 0 with one existing frontend warning. - -## Design decision - -Build a provider connector layer rather than extending the public URL import path. - -Core concepts: - -- **Connector**: instance-level provider app config created by a system admin. Example: one GitHub App for this KIWI instance. -- **Connector installation/account**: a provider authorization connected by a graph manager to a KIWI owner scope. GitHub installation ID; GitLab OAuth account/project-token context. -- **Repository binding**: one KIWI graph tracks one provider repository + one branch. This binding drives manual sync and webhook sync. -- **Webhook event ledger**: stores delivery IDs and enqueue results so retries are idempotent. - -Provider support for this plan: - -- GitHub: full end-to-end implementation with GitHub App manifest/create flow, installation flow, repo/branch listing, content fetch, and push webhook sync. -- GitLab: implement the same DB/API/provider interface and manual sysadmin connector config, plus OAuth/project listing and webhook verification if feasible. If GitLab project webhook creation needs a broader `api` scope or self-managed-instance behavior diverges, finish GitHub and leave GitLab behind a `disabled` connector state with explicit STOP/report rather than inventing an unsafe token flow. - -Security invariants: - -- Connector secrets, private keys, OAuth client secrets, webhook secrets, and refresh tokens are encrypted at rest with the existing `AUTH_SECRET`-derived pattern. Never log them. -- Webhook endpoints are unauthenticated but must verify provider signatures/tokens before reading business fields or enqueueing work. -- Users can only create repository graphs in scopes where they can already manage graphs: organization admin, team admin, team moderator. -- Do not accept arbitrary raw URLs from users. Repository content comes from provider API clients using stored connector authorization. -- Webhook processing is idempotent by provider delivery ID and by `(repositoryBindingId, commitSha)`. - -## Commands you will need - -| Purpose | Command | Expected on success | -|---|---|---| -| Generate migration scaffold | `bun run db:generate --custom` | creates a new migration scaffold to edit | -| DB migration tests | `bun test packages/db/src/__tests__/migration-compat.test.ts` | exit 0 | -| API connector tests | `bun test apps/api/src/routes/__tests__ apps/api/src/lib/__tests__` | exit 0 | -| Worker connector tests | `bun test apps/worker/lib apps/worker/workflows` | exit 0 | -| Frontend tests | `bun test apps/frontend` | exit 0 | -| Workspace tests | `bun run test` | exit 0 | -| Lint | `bun run lint` | exit 0; no new errors | - -## Scope - -**In scope**: - -- New connector DB tables and migration. -- Shared provider client/helpers, preferably a new `packages/connectors` workspace package if both API and worker need provider logic. -- New API routes under `/connectors` plus public webhook routes under `/connectors/:id/webhooks/:provider` mounted before auth middleware. -- GitHub App manifest setup UI at `/connectors/github/new` for system admins. -- Connector installation/connect UI at `/connectors/:id/connect` for graph managers. -- Repository/branch list endpoints. -- Graph creation from repository + branch. -- Worker workflow to sync a connector repository snapshot and enqueue/process code files without `git clone`. -- Push webhook handling that enqueues updates for matching branch bindings. -- Contract/frontend types and minimal UI to select connector, repo, branch, and owner scope. - -**Out of scope**: - -- Pull request preview graphs. -- User-facing history UI for old function versions. -- Write access to repositories. -- GitHub Checks/Statuses. -- Fine-grained per-file incremental graph updates; rebuild the selected branch snapshot and rely on plan 007 source invalidation. -- Supporting arbitrary provider URLs or Bitbucket. -- Moving existing URL import users to connectors automatically. - -## Git workflow - -- Branch name suggestion: `advisor/008-provider-connectors-repository-sync`. -- Commit message style: `feat(connectors): add repository graph sync`. -- Do not push or open a PR unless instructed. - -## Steps - -### Step 1: Add connector schema - -Run `bun run db:generate --custom` from the repo root. Add Drizzle tables in `packages/db/src/tables/connectors.ts` or the repo's preferred table module. Export them from the package index if needed. - -Suggested tables: - -#### `connectors` - -- `id text primary key` -- `provider text not null` enum-like: `github | gitlab` -- `name text not null` -- `slug text not null unique` -- `status text not null default 'active'` enum-like: `draft | active | disabled` -- `app_id text` — GitHub App ID or GitLab application ID. -- `client_id text` -- `encrypted_credentials text not null` — provider secrets; shape validated in code. -- `webhook_secret_encrypted text not null` -- `created_by_user_id text references user(id) on delete set null` -- timestamps - -#### `connector_installations` - -- `id text primary key` -- `connector_id text not null references connectors(id) on delete cascade` -- `provider text not null` -- `provider_installation_id text not null` — GitHub installation ID; GitLab account/project auth identifier. -- `provider_account_login text not null` -- `provider_account_type text` — `user | organization | group`. -- `organization_id text references organization(id) on delete cascade` -- `team_id text references team(id) on delete cascade` -- `installed_by_user_id text references user(id) on delete set null` -- `encrypted_credentials text` — GitLab OAuth refresh/access token if needed; null for GitHub App installations. -- `repository_selection text` — `all | selected | unknown`. -- `status text not null default 'active'` -- timestamps -- unique `(connector_id, provider_installation_id, organization_id, team_id)` -- check: either organization scope or team scope is present, not both unless team requires organization. For teams, require organization too if that matches existing graph team ownership. - -#### `repository_graph_bindings` - -- `id text primary key` -- `graph_id text not null references graphs(id) on delete cascade unique` -- `connector_installation_id text not null references connector_installations(id) on delete restrict` -- `provider text not null` -- `provider_repository_id text not null` -- `repository_full_name text not null` — `owner/repo` or GitLab `namespace/project`. -- `repository_html_url text not null` -- `branch text not null` -- `last_seen_commit_sha text` -- `last_synced_commit_sha text` -- `sync_status text not null default 'pending'` — `pending | syncing | synced | failed`. -- `sync_error_code text` -- `webhook_enabled boolean not null default true` -- timestamps -- unique `(connector_installation_id, provider_repository_id, branch)` if one branch graph per installation is desired; otherwise unique only on `graph_id` and let multiple graphs track the same branch. - -#### `connector_webhook_events` - -- `id text primary key` -- `connector_id text not null references connectors(id) on delete cascade` -- `provider text not null` -- `delivery_id text not null` -- `event_name text not null` -- `provider_repository_id text` -- `branch text` -- `commit_sha text` -- `status text not null` — `ignored | enqueued | duplicate | failed`. -- `error_code text` -- `created_at timestamp not null default now()` -- unique `(connector_id, provider, delivery_id)` - -Add migration compatibility tests for table/column/check/index presence. - -**Verify**: `bun test packages/db/src/__tests__/migration-compat.test.ts` → exit 0. - -### Step 2: Add encrypted connector credential helpers - -Create a small helper module, ideally in a new shared package if both API and worker consume it: - -- Reuse the `packages/ai/src/models.ts` encryption approach: HKDF from `AUTH_SECRET`, random IV, auth tag, versioned string. -- Supported secret shapes: - - GitHub connector credentials: `{ appId: string; privateKeyPem: string; clientId?: string; clientSecret?: string }` - - GitHub webhook secret: store separately or inside credentials, but keep a dedicated accessor for verification. - - GitLab connector credentials: `{ baseUrl: string; clientId: string; clientSecret: string }` - - GitLab installation credentials: `{ accessToken: string; refreshToken?: string; expiresAt?: string }` -- Validate decrypted shape before returning it. - -Do not export raw decrypted credentials beyond provider client factories. - -**Verify**: add unit tests for encrypt/decrypt, invalid version, invalid shape, and wrong secret failure. Run the new test file. - -### Step 3: Add provider client interfaces - -Create provider-neutral interfaces. Suggested package: `packages/connectors/src`. - -Types: - -```ts -export type ConnectorProvider = "github" | "gitlab"; - -export type ProviderRepository = { - provider: ConnectorProvider; - id: string; - fullName: string; - name: string; - htmlUrl: string; - defaultBranch: string | null; - private: boolean; -}; - -export type ProviderBranch = { - name: string; - commitSha: string; -}; - -export type ProviderCodeFile = { - path: string; - size: number; - checksum: string; - htmlUrl: string; - rawUrl?: string; - content: string; -}; -``` - -Client methods: - -- `listRepositories(installationOrAccount)` -- `listBranches(repository)` -- `loadRepositorySnapshot(repository, branch)` returning commit SHA + supported code files. -- `verifyWebhook(request)` returning normalized event fields. -- `createOrRefreshInstallationToken(...)` for GitHub. - -GitHub implementation requirements: - -- Generate a GitHub App JWT with RS256 using Node/Bun crypto and the stored private key. -- Create installation tokens with `POST /app/installations/{installation_id}/access_tokens` and request at most `contents: read` plus metadata. Do not request write permissions. -- List repositories with `GET /installation/repositories` using the installation token. -- List branches with `GET /repos/{owner}/{repo}/branches`. -- Resolve the selected branch to an immutable commit SHA before loading files. -- List tree recursively via provider API; filter with existing `isSupportedCodePath` and skipped generated directories from `repository-url.ts`. -- Fetch contents via provider API using the installation token so private repositories work. Do not use unauthenticated `git clone`. -- Enforce existing repository code limits: max files, max total bytes, max per-file bytes. - -GitLab implementation requirements: - -- Normalize `baseUrl` and use `/api/v4`. -- Use OAuth token with the minimal workable scopes. Prefer `read_api read_repository`; if project webhook creation requires `api`, make that explicit in the connector UI and plan tests. -- List projects for the connected user/account. -- List branches and repository tree/files through GitLab API. -- Verify GitLab push webhooks with `X-Gitlab-Token` against the encrypted connector webhook secret. - -**Verify**: provider unit tests with mocked `fetch` for GitHub token creation, repo listing, branch listing, content loading, and webhook verification. - -### Step 4: Add system-admin connector creation routes and UI - -Add an authenticated connector route module under `apps/api/src/routes/connectors.ts` and mount it after `authMiddleware`. - -System-admin endpoints: - -- `GET /connectors` — list active connectors visible to the current user; system admins see full config metadata but never secrets. -- `POST /connectors/github/manifest/start` — system admin only. Creates a short-lived signed state and returns a GitHub App manifest URL. -- `GET /connectors/github/manifest/callback?code&state` — system admin only. Exchanges the manifest `code` for app credentials and stores a connector. -- `POST /connectors/gitlab` — system admin only. Stores manually-created GitLab application config and webhook secret. -- `PATCH /connectors/:id` — system admin only, enable/disable/rename/rotate secrets. - -GitHub App manifest defaults: - -- App name: include KIWI instance name/host to avoid collisions. -- Homepage URL: configured frontend/base URL. -- Webhook URL: `${API_URL}/connectors/webhooks/github` or `${API_URL}/connectors/:id/webhooks/github` if known after creation. If connector ID is not known before creation, use provider-level route and resolve by app ID in the payload. -- Callback/setup URLs: - - Manifest callback: `/connectors/github/callback`. - - Installation setup URL: `/connectors/github/setup`. -- Repository permissions: `Contents: read`, `Metadata: read`. -- Events: `push`, and optionally `installation` / `installation_repositories` to keep repository access state fresh. - -Frontend: - -- Add `/connectors` page for listing connectors. -- Add `/connectors/github/new` page for system admins. It should show what permissions/events will be requested and a "Create GitHub App" button that calls the manifest-start endpoint then redirects to GitHub. -- Add `/connectors/gitlab/new` page for system admins with base URL, application ID, client secret, and webhook secret fields. -- Reuse the settings admin guard pattern from `apps/frontend/app/(app)/settings/page.tsx` for server-side system-admin protection. -- Add a system-admin settings link/section only if it does not duplicate the top-level `/connectors` page. - -**Verify**: API tests prove non-admins get 403 for creation/patch routes and system admins can create listable connectors without secrets in responses. Frontend tests cover visibility/guard helpers if added. - -### Step 5: Add graph-manager installation/connect flow - -Endpoints: - -- `GET /connectors/:id/connect?organizationId=...&teamId=...` — checks graph-management rights for the requested owner scope, creates signed state, redirects to provider installation/OAuth flow. -- GitHub callback/setup route: records `installation_id`, provider account, repository selection, and owner scope into `connector_installations`. -- GitLab OAuth callback: exchanges code for token, stores encrypted token under `connector_installations` for the owner scope. -- `GET /connectors/:id/installations` — list installations/accounts available to the current user for owner scopes they can manage. - -Authorization rules: - -- Organization scope: require organization admin. -- Team scope: require `requireTeamGraphCreateAccess`; this allows organization admin, team admin, and team moderator, matching graph creation. -- User/private graph scope is out of scope for connectors unless product explicitly asks; private provider apps create ownership ambiguity. - -GitHub redirect target: - -- Use the GitHub App installation URL for the connector app, e.g. `https://github.com/apps//installations/new?state=` after the connector stores the app slug/name. -- On callback, verify state and ensure the KIWI user still has graph-management rights for the requested scope. - -**Verify**: route tests cover org admin, team admin, team moderator allowed; team member denied; stale/invalid state denied; connector disabled denied. - -### Step 6: Add repository and branch selection endpoints - -Endpoints: - -- `GET /connectors/:id/repositories?installationId=...` — list provider repositories visible through that installation/account. -- `GET /connectors/:id/repositories/:providerRepositoryId/branches?installationId=...` — list branches. - -Requirements: - -- Check the current user can manage the installation owner scope before listing. -- Return stable provider repository IDs, full names, default branch, private flag, and HTML URL. -- Never return provider access tokens. -- Cache repository/branch lists only if cache invalidation is clear; otherwise fetch live and paginate. -- Handle provider pagination deterministically. - -Frontend: - -- Add a repository picker page under `/connectors/:id/connect` after installation completes. -- Let the user choose owner scope, installation/account, repository, and branch. -- Default branch should be preselected. - -**Verify**: mocked API tests for pagination, denied installation access, empty repositories, and branches. Frontend component tests for default branch preselection and disabled submit without repo/branch. - -### Step 7: Create graphs from selected repository branches - -Add an endpoint: - -```http -POST /connectors/:id/repository-graphs -{ - "installationId": "...", - "providerRepositoryId": "...", - "branch": "main", - "name": "optional display name", - "description": "optional", - "teamId": "optional" -} -``` - -Behavior: - -1. Authorize the requested owner scope using the same rules as graph creation. -2. Resolve repository and branch through the provider API. -3. Insert a graph row with `state = 'updating'`, organization/team owner fields matching existing graph creation semantics. -4. Insert `repository_graph_bindings` with the selected branch and latest branch commit SHA as `last_seen_commit_sha`. -5. Enqueue a new worker workflow, e.g. `syncRepositoryGraphSpec`, with `{ bindingId, reason: 'initial', commitSha }`. -6. Return graph, binding, and workflow run ID. - -Do not create all file rows synchronously in the API route. The provider API may be slow and large; the worker should load the snapshot and commit file rows/process runs. - -**Verify**: API tests prove graph row + binding transactionality, owner authorization, and enqueue failure rollback/failed state behavior. - -### Step 8: Add repository sync worker workflow - -Add `syncRepositoryGraphSpec` and implementation in `apps/worker/workflows/sync-repository-graph.ts`. Register it in `apps/worker/worker.ts`. - -Workflow input: - -```ts -{ - bindingId: string; - reason: "initial" | "webhook" | "manual"; - commitSha?: string; - deliveryId?: string; -} -``` - -Workflow behavior: - -1. Load binding, connector installation, connector credentials, and graph. -2. If binding/webhook disabled or graph missing, exit without side effects. -3. Resolve selected branch to commit SHA unless input already supplies one from a verified webhook. -4. If `lastSyncedCommitSha === commitSha`, mark webhook event duplicate/synced and exit. -5. Load repository snapshot through provider API, not `git clone`. -6. Convert each supported code file into an external `files` row: - - `storageKind: 'external'` - - `externalProvider: provider` - - `externalUrl`: provider HTML URL or raw API URL, whichever plan 006's proxy/content-source layer supports after extension. - - metadata: provider, repository full name/id, branch, commit SHA, path, html URL, and any API raw URL needed for worker reads. - - checksum: provider blob SHA/content hash. -7. Use or extend `commitGraphFileUploads` so it can create file rows + process run for these external files without S3 cleanup. -8. Mark superseded file rows for this binding as deleted only after new rows/process run commit. -9. Run `processFilesSpec` with `code: { kind: 'repository' }` or direct child workflows according to plan 005. -10. Update binding `lastSeenCommitSha`, `syncStatus`, and eventually `lastSyncedCommitSha` after processing succeeds. - -Integration with plan 007: - -- On successful full snapshot processing, old code sources for the same binding/repository/branch must get `validUntil` and stop appearing in graph tools. -- If processing fails terminally, keep previous sources current and set binding `syncStatus = 'failed'`. - -**Verify**: worker tests with mocked provider client prove no `git` process is spawned, file rows are external, older binding files are marked deleted after commit, duplicate SHA exits, and failure leaves previous binding state intact. - -### Step 9: Extend external file content/proxy handling for private providers - -Plan 006 introduced external GitHub files. Private connector repositories need credentialed reads. - -Update content-source handling: - -- Internal S3 files keep current behavior. -- Public external GitHub URL files from URL imports can keep allowlisted raw fetch behavior. -- Connector-backed external files must read through provider API using `repository_graph_bindings` + connector installation credentials, not unauthenticated raw URLs. - -Add metadata or DB columns needed to locate the binding from a file row. Preferred explicit column: - -- `files.repository_binding_id text references repository_graph_bindings(id) on delete set null` - -If adding this column, include it in the same migration or a new migration and update file insert/select helpers. - -Proxy behavior: - -- For private files, do not redirect to provider raw URLs that may leak or fail. -- Either redirect to provider HTML URL for humans or proxy raw content through KIWI after graph access checks. -- If proxying bytes, preserve size limits and content type; do not stream arbitrary provider responses without host/API validation. - -**Verify**: API proxy tests for public URL-import external file, private connector-backed external file, missing binding, disabled connector, and unauthorized graph access. - -### Step 10: Add webhook endpoint before auth middleware - -In `apps/api/src/server.ts`, mount a public webhook route before `.use(authMiddleware)`, e.g.: - -```ts -.use(connectorWebhookRoute) -.use(authMiddleware) -.use(connectorRoute) -``` - -Webhook route requirements: - -- Route: `POST /connectors/webhooks/github` and `POST /connectors/webhooks/gitlab`, or connector-specific paths if the provider can include connector ID safely. -- Read raw body for signature verification before JSON business logic. -- GitHub: verify `X-Hub-Signature-256` HMAC SHA-256 using connector webhook secret. Use `timingSafeEqual`. -- GitHub: use `X-GitHub-Event` and `X-GitHub-Delivery` for event/dedupe. -- GitLab: verify `X-Gitlab-Token` against connector webhook secret. Use `timingSafeEqual` for token bytes. -- Store a row in `connector_webhook_events` before enqueueing. Duplicate delivery returns 202 without enqueue. -- Ignore non-push events with status `ignored`. -- For push events, normalize branch from `refs/heads/` and commit from `after`; ignore branch deletes where commit is all zeroes. -- Find active bindings for `(provider repository id/full name, branch)` and enqueue `syncRepositoryGraphSpec` once per binding. -- Return 202 quickly; do not process repository contents in the API request. - -**Verify**: webhook route tests cover valid signature enqueue, invalid signature 401/403 with no DB writes, duplicate delivery no duplicate enqueue, wrong branch ignored, branch delete ignored, and multiple graph bindings enqueue separately. - -### Step 11: Add manual resync and status endpoints - -Endpoints: - -- `POST /repository-graph-bindings/:id/sync` — graph manager only; enqueue sync for current branch head. -- `GET /repository-graph-bindings/:id` — graph viewer can see sync status, provider, repo full name, branch, last seen/synced commit. -- Include binding summary in graph detail response if graph is connector-backed. - -This gives operators a recovery path when provider webhook delivery fails. - -**Verify**: API tests for allowed manager sync, viewer status, member denied sync, disabled binding denied. - -### Step 12: Update contracts and frontend API client - -Update `packages/contracts/src/routes.ts` with connector records/responses: - -- Connector list/create/update responses. -- Installation list response. -- Repository list response. -- Branch list response. -- Repository graph create response. -- Binding status response. - -Update `apps/frontend/lib/api` with typed functions: - -- `fetchConnectors` -- `startGitHubConnectorManifest` -- `createGitLabConnector` -- `fetchConnectorInstallations` -- `fetchConnectorRepositories` -- `fetchConnectorBranches` -- `createRepositoryGraph` -- `syncRepositoryGraphBinding` - -Keep API errors using existing `ApiError`/`unwrapApiResponse` patterns. - -**Verify**: frontend API client unit tests with mocked `fetch` for success and error paths. - -### Step 13: Add frontend pages - -Add pages under `apps/frontend/app/(app)/connectors`: - -- `/connectors` — connector list. System admins see create/manage actions; graph managers see connect/use actions for active connectors. -- `/connectors/github/new` — system-admin GitHub App manifest starter, with permission/event explanation. -- `/connectors/gitlab/new` — system-admin manual GitLab application config. -- `/connectors/[connectorId]/connect` — owner scope selection, provider installation/OAuth status, repository picker, branch picker, create graph button. -- Optional `/connectors/[connectorId]/repositories/new-graph` if the connect page becomes too large. - -UI requirements: - -- Reuse existing shadcn components from `components/ui`. -- Use existing `useAuth`, query hooks, and API client patterns. -- Never render secrets after submit. -- Show provider-specific permission copy: - - GitHub: Contents read, Metadata read, Push webhook. - - GitLab: read repository/API scopes needed, Push webhook token. -- After graph creation, navigate to the new project route using existing group/project routing patterns. - -**Verify**: component tests for guard/visibility, disabled submit states, default branch preselection, and successful navigation callback if existing test utilities support it. - -### Step 14: Update route/server tests and workflow registration tests - -Add/extend tests so the new public webhook route remains before auth middleware. A regression test should prove a valid webhook without a session reaches the webhook handler while invalid signatures are rejected. - -Add a worker registration test or static import test if the repo has a pattern for workflow registration; otherwise the workspace test/build will catch missing exports. - -**Verify**: `bun test apps/api/src/routes/__tests__ apps/worker` → exit 0. - -### Step 15: Run repo checks - -**Verify**: `bun run test` → exit 0. - -**Verify**: `bun run lint` → exit 0; no new errors. - -## Test plan - -Minimum focused tests before full checks: - -1. DB schema/migration tests for connector tables, binding table, webhook event ledger, and optional `files.repository_binding_id`. -2. Credential encryption tests: no plaintext secrets in selected API responses. -3. GitHub provider client tests with mocked `fetch`: - - app JWT/token request, - - installation repositories, - - branches, - - snapshot load with limits, - - push webhook signature verification. -4. GitLab provider client tests with mocked `fetch`: - - OAuth token use/refresh if implemented, - - project/branch/tree/raw file calls, - - webhook token verification. -5. Connector route authorization tests: - - system admin can create connector, - - non-admin cannot, - - org admin/team admin/team moderator can connect/install/use connector, - - team member cannot. -6. Repository graph creation tests: - - creates graph + binding + sync workflow atomically, - - enqueue failure leaves a recoverable failed graph/binding state or rolls back cleanly. -7. Webhook tests: - - valid push to selected branch enqueues sync, - - wrong branch ignored, - - duplicate delivery ignored, - - invalid signature rejected before payload trust. -8. Worker sync tests: - - no git clone, - - external code file rows created, - - private content read through provider API, - - duplicate commit exits, - - failed sync does not invalidate current sources. -9. Frontend tests for connector create/connect/repo-select forms. - -## Done criteria - -- [ ] System admins can create at least GitHub connectors from `/connectors/github/new` with prefilled app permissions/events and no secret exposure after creation. -- [ ] System admins can configure GitLab connector metadata or GitLab is explicitly disabled with a clear STOP/report if safe implementation is blocked. -- [ ] Org admins, team admins, and team moderators can connect a connector installation/account for scopes they manage. -- [ ] Team members and ordinary org members cannot connect/use repository connectors for graph creation. -- [ ] Users can list provider repos and branches through connector endpoints. -- [ ] Users can create a graph from selected repo + branch. -- [ ] Repository sync loads code through provider APIs, not `git clone`, and supports private GitHub repositories. -- [ ] Provider push webhooks enqueue sync only for matching active branch bindings. -- [ ] Webhook delivery handling is signature-verified and idempotent. -- [ ] Plan 007 source invalidation is used so rebuilds keep graph answers on the latest branch state. -- [ ] Manual resync endpoint exists for recovery. -- [ ] `bun test packages/db/src/__tests__/migration-compat.test.ts` exits 0. -- [ ] API, worker, frontend focused tests exit 0. -- [ ] `bun run test` exits 0. -- [ ] `bun run lint` exits 0. -- [ ] `plans/README.md` status row updated. - -## STOP conditions - -Stop and report if: - -- Plan 007 is not implemented; webhook rebuilds without source invalidation will accumulate stale function evidence. -- GitHub App manifest flow cannot produce/store the private key and webhook secret for this instance; do not ask sysadmins to paste secrets into random logs or responses. -- GitLab requires broad write/admin scopes to create webhooks and the product owner has not accepted that scope. -- Elysia cannot expose raw request bodies for signature verification on the webhook route. Do not verify signatures against re-serialized JSON. -- Private repository content cannot be fetched through provider APIs within existing size/rate limits. -- Provider rate limits require queueing/backoff beyond OpenWorkflow's current retry policy; report before shipping a loop that can hammer provider APIs. -- Any test fixture includes a real provider token/private key/webhook secret. Use synthetic values only. - -## Maintenance notes - -Keep URL imports and connectors separate. URL imports are convenient public one-offs; connectors are authenticated, owner-scoped, branch-bound sync sources. Future providers should implement the provider interface and webhook verifier, not special-case graph routes. History features should query invalidated sources from plan 007 explicitly; current graph tools must remain latest-only. diff --git a/plans/009-incremental-connector-repository-updates.md b/plans/009-incremental-connector-repository-updates.md deleted file mode 100644 index 0e2c6307..00000000 --- a/plans/009-incremental-connector-repository-updates.md +++ /dev/null @@ -1,348 +0,0 @@ -# Plan 009: Incremental connector repository updates for changed files only - -> **Executor instructions**: Follow this plan step by step. Run every verification command and confirm the expected result before moving to the next step. If anything in the "STOP conditions" section occurs, stop and report — do not improvise. When done, update the status row for this plan in `plans/README.md`. -> -> **Drift check (run first)**: `git diff --stat 1dea5eb77..HEAD -- packages/connectors/src apps/worker/workflows/sync-repository-graph.ts apps/worker/lib/code-manifest.ts apps/worker/lib/code-repository-finalizer.ts apps/api/src/routes/connector-webhooks.ts apps/api/src/lib/graph-file-proxy.ts apps/worker/lib/file-content-source.ts packages/graph/src/code/metadata.ts` -> If any in-scope file changed since this plan was written, compare the "Current state" excerpts against the live code before proceeding; on a mismatch, treat it as a STOP condition. - -## Status - -- **Priority**: P1 -- **Effort**: L -- **Risk**: HIGH -- **Depends on**: `007-version-code-sources-with-valid-until`, `008-provider-connectors-for-repository-graphs` -- **Category**: performance / correctness / repository sync -- **Planned at**: commit `1dea5eb77`, 2026-06-14 - -## Why this matters - -Connector-backed repository graphs now stay fresh via manual sync and push webhooks, but every update still rebuilds the entire supported-code snapshot for the bound branch. That means: - -1. A one-file change on `main` re-reads every supported code file from the provider API. -2. KIWI creates new file rows for the whole repository instead of just the touched paths. -3. The code workflow reprocesses unchanged files, wasting worker time and provider rate limit budget. -4. Large repositories pay full-branch rebuild cost on every push even when only a handful of files changed. - -The product requirement is narrower and stricter: the initial connector import may stay full-snapshot, but **subsequent branch updates must only process changed supported code files, not silently reprocess the whole repository again**. - -## Current state - -Relevant files: - -- `packages/connectors/src/types.ts` — provider client interface; it can list repos/branches, load a full snapshot, and read one file, but it cannot compare two revisions. -- `packages/connectors/src/github.ts` and `packages/connectors/src/gitlab.ts` — current provider clients load whole supported-code snapshots and fetch per-file content. -- `apps/worker/workflows/sync-repository-graph.ts` — connector sync loads a full branch snapshot, inserts a row per file, marks all prior binding rows deleted, and processes every inserted file. -- `apps/worker/lib/code-manifest.ts` — repository code manifests are still grouped by `repositoryUrl + commitSha`, so changed-file processing assumes one full-snapshot commit scope. -- `apps/worker/lib/code-repository-finalizer.ts` — current invalidation helper is repository-wide for the latest import batch, not path-targeted. -- `apps/api/src/routes/connector-webhooks.ts` — push webhooks already enqueue one sync per bound branch and commit. -- `plans/008-provider-connectors-for-repository-graphs.md` — incremental file-level updates were explicitly left out of scope when connectors landed. - -Current provider client contract: - -```ts -// packages/connectors/src/types.ts:81-86 -export type ProviderRepositoryClient = { - readonly provider: ConnectorProvider; - listRepositories(): Promise; - listBranches(repository: ProviderRepository): Promise; - loadRepositorySnapshot(repository: ProviderRepository, branch: string, commitSha?: string): Promise; - readFile(repository: ProviderRepository, path: string, commitSha: string): Promise; -}; -``` - -Current connector sync rebuilds the whole binding snapshot: - -```ts -// apps/worker/workflows/sync-repository-graph.ts:190-235 -const insertedFiles = await tx.insert(filesTable).values(fileRows(row, snapshot, commitSha)).onConflictDoNothing().returning({ id: filesTable.id }); -... -await tx - .update(filesTable) - .set({ deleted: true }) - .where(and(eq(filesTable.repositoryBindingId, row.binding.id), notInArray(filesTable.id, insertedFiles.map((file) => file.id)))); -... -await step.runWorkflow(processFilesSpec, { - graphId: row.binding.graphId, - fileIds: created.fileIds, - processRunId: created.processRunId, - code: { kind: "repository" }, -}); -``` - -Current code-manifest scope is commit-wide: - -```ts -// apps/worker/lib/code-manifest.ts:45-65,102-104 -const selectedRepositoryKeys = new Set( - selectedRows - .map((row) => parseCodeFileMetadata(row.metadata)) - .filter((metadata) => metadata !== null) - .map(repositoryManifestScopeKey) -); -... -function repositoryManifestScopeKey(metadata: Pick): string { - return `${metadata.repositoryUrl}\0${metadata.commitSha}`; -} -``` - -Current repository-source invalidation is repo-wide for older batches: - -```ts -// apps/worker/lib/code-repository-finalizer.ts:64-68,96-103 -const candidateRows = await tx - .select({ id: filesTable.id, metadata: filesTable.metadata }) - .from(filesTable) - .where(and(eq(filesTable.graphId, options.graphId), eq(filesTable.type, "code"))); -... -await tx.execute(sql` - UPDATE sources source - SET valid_until = NOW() - FROM text_units text_unit - WHERE source.text_unit_id = text_unit.id - AND text_unit.file_id = ANY(${textArray(targets.olderFileIds)}) - AND ${currentSourceSql("source")} -`); -``` - -Current webhook route already passes one binding + commit into the sync workflow: - -```ts -// apps/api/src/routes/connector-webhooks.ts:180-191 -if (status === "enqueued" && normalized.commitSha) { - for (const binding of bindings) { - await db - .update(repositoryGraphBindingsTable) - .set({ lastSeenCommitSha: normalized.commitSha, syncStatus: "pending", syncErrorCode: null }) - .where(eq(repositoryGraphBindingsTable.id, binding.id)); - await ow.runWorkflow(syncRepositoryGraphSpec, { - bindingId: binding.id, - reason: "webhook", - commitSha: normalized.commitSha, - deliveryId, - }); - } -} -``` - -Current connector plan explicitly deferred this work: - -```md - -- Fine-grained per-file incremental graph updates; rebuild the selected branch snapshot and rely on plan 007 source invalidation. -``` - -Provider docs observed during planning: - -- GitHub supports `GET /repos/{owner}/{repo}/compare/{base...head}` and returns changed files between two refs/commits, plus raw file reads at `GET /repos/{owner}/{repo}/contents/{path}?ref=`. -- GitLab supports `GET /projects/:id/repository/compare?from=&to=` and returns `diffs[]` with `new_path`, `old_path`, `new_file`, `renamed_file`, `deleted_file`, and `compare_timeout`, plus raw file reads at `GET /projects/:id/repository/files/:path/raw?ref=`. - -## Design decision - -Keep the connector feature set and DB model from plan 008, but change update semantics from **branch snapshot replacement** to **binding-scoped incremental cutover**. - -Core rules: - -- **Initial connector graph creation stays full-snapshot.** The repository has no prior binding state to diff against. -- **Later manual syncs and push-webhook syncs are incremental.** Only changed supported code paths create new file rows and enter the code workflow. -- **Unchanged active file rows stay active.** Do not mint duplicate rows just to refresh `commitSha` metadata when the file content and path did not change. -- **Current binding state is the set of non-deleted code files for one `repositoryBindingId`, not “every file from one commit”.** -- **Repository code manifests for connector-backed files must be binding-scoped, not `repositoryUrl + commitSha` scoped.** Changed files still need unchanged siblings present for import resolution. -- **Removed/replaced paths invalidate only their own current sources.** Do not invalidate the whole binding when one file changes. -- **Normal document uploads and public URL imports remain unchanged.** This plan is connector-only. -- **No silent full-sync fallback for a routine update path.** If a provider compare cannot produce a safe delta, stop/report instead of quietly reprocessing the whole branch. - -This deliberately keeps exact provenance for unchanged files: a source citation can still point at the last commit where that file content changed. If product later wants branch-head links for unchanged files too, that is a separate feature and must not reintroduce full-file churn here. - -## Commands you will need - -| Purpose | Command | Expected on success | -|---|---|---| -| Connector provider tests | `bun test packages/connectors/src/__tests__/credentials.test.ts packages/connectors/src/__tests__/github.test.ts packages/connectors/src/__tests__/gitlab.test.ts` | exit 0 | -| Worker manifest + sync tests | `bun test apps/worker/lib/__tests__/code-manifest.test.ts apps/worker/workflows/process-file.test.ts apps/worker/workflows/sync-repository-graph.test.ts` | exit 0 | -| API regression tests | `bun test apps/api/src/lib/__tests__/graph-file-proxy.test.ts apps/api/src/lib/__tests__/source-reference.test.ts apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts` | exit 0 | -| Workspace tests | `bun run test` | exit 0 | -| Lint | `bun run lint` | exit 0; no new errors | - -## Scope - -**In scope**: - -- Extend `@kiwi/connectors` with provider-neutral compare/change APIs. -- GitHub and GitLab provider implementations for changed-path detection between two commits. -- Incremental connector sync in `sync-repository-graph.ts`. -- Binding-scoped repository manifests for connector-backed code files. -- Targeted file deletion/source invalidation for removed or replaced paths. -- Tests for changed-only sync, delete/rename handling, and no-op syncs when no supported code changed. - -**Out of scope**: - -- Changing the normal document pipeline. -- Changing public URL repository imports. -- Rewriting unchanged file rows solely to update commit metadata. -- Pull request preview graphs. -- Automatic repair for provider compare timeouts or missing compare ancestry. -- Provider support beyond GitHub and GitLab. - -## Git workflow - -- Branch name suggestion: `advisor/009-incremental-connector-updates`. -- Commit message style: `feat(connectors): sync only changed repository files`. -- Do not push or open a PR unless instructed. - -## Steps - -### Step 1: Extend provider APIs with compare/change support - -Add a provider-neutral change model in `packages/connectors/src/types.ts`, for example: - -- `ProviderRepositoryChange` -- `ProviderRepositoryDelta` -- statuses covering `added`, `modified`, `deleted`, and `renamed` -- both `oldPath` and `newPath` where rename/delete semantics need them - -Then add a new method to `ProviderRepositoryClient`, for example: - -```ts -compareRepository(repository, fromCommitSha, toCommitSha): Promise -``` - -Implementation requirements: - -- **GitHub**: call the compare endpoint for `base...head`, normalize changed-file records, and reject responses that do not safely describe changed paths. -- **GitLab**: call the compare endpoint, reject `compare_timeout === true`, and normalize `diffs[]` into the same provider-neutral shape. -- Preserve existing `readFile()` methods; incremental sync should reuse them to fetch only the touched file contents at the target commit. -- Keep path filtering consistent with current supported-code rules (`isSupportedCodePath`, skipped path segments, file-size limits). - -Add/extend tests in `packages/connectors/src/__tests__/github.test.ts` and `packages/connectors/src/__tests__/gitlab.test.ts` for: - -- modified supported file -- added supported file -- deleted supported file -- rename from supported → supported -- rename from supported → unsupported -- compare-timeout / malformed-response rejection - -### Step 2: Teach code manifests about binding-scoped current snapshots - -`prepareCodeManifest()` currently groups repository context by `repositoryUrl + commitSha`. That is incompatible with changed-only updates because unchanged active files will often keep older commit metadata. - -Change the manifest scope logic so connector-backed repository files use a **binding-wide active snapshot** instead: - -- if `repositoryBindingId` exists, scope by `repositoryBindingId` -- otherwise keep the existing public URL import behavior (`repositoryUrl + commitSha`) - -Implementation notes: - -- Keep normal document files and non-repository code files unchanged. -- Continue reading content through `readFileContentSource()`. -- Do not broaden the manifest beyond the selected binding; one repository binding should never leak another repository’s files into import resolution. - -Add worker tests covering: - -- incremental connector reprocessing where selected file IDs are at a newer commit but unchanged sibling files remain on older rows -- public URL repository imports still using the old commit-scoped manifest behavior - -### Step 3: Compute incremental connector deltas instead of full snapshots - -Update `apps/worker/workflows/sync-repository-graph.ts`. - -Required behavior: - -1. If `lastSyncedCommitSha` is missing, keep the current full-snapshot bootstrap behavior. -2. If `lastSyncedCommitSha === targetCommitSha`, keep the existing fast no-op. -3. Otherwise: - - call the provider compare API - - derive the set of affected supported code paths - - load the current active binding rows keyed by path - - build three sets: - - **new/updated paths** → need new file rows + code processing - - **removed/replaced old paths** → need targeted invalidation after successful cutover - - **unchanged paths** → keep current active rows untouched - -Path rules: - -- `added` / `modified` / rename target with a supported path → fetch target file content and insert a new external file row for that path. -- `deleted` / rename source from a supported path → keep the old active row until success, then mark it deleted and invalidate its current sources. -- rename supported → supported is both a delete for the old path and an add for the new path. -- changes that only touch unsupported paths must not create a process run. - -Do **not** call `loadRepositorySnapshot()` for incremental updates. The whole point of this plan is to stop loading every file when one file changed. - -### Step 4: Cut over only the touched file rows - -Replace the current “insert whole snapshot, mark everything else deleted” transaction with a targeted cutover: - -- insert only new/updated file rows -- create a process run only when there are changed supported files to process -- do not mark old rows deleted until the changed-file workflows succeed -- if the changed-file workflow batch fails, leave the previous active rows intact - -Use `repositoryBindingId` + parsed metadata path to identify the active row being replaced or removed. - -Important edge cases: - -- If compare says the branch changed but none of the changed paths are supported code, mark the binding synced and advance `lastSeenCommitSha` / `lastSyncedCommitSha` without a process run. -- If the same path appears twice in the delta after provider normalization, treat that as a bug and STOP rather than guessing. -- If a path was already deleted from the active binding state, ignore duplicate delete signals. - -### Step 5: Make source invalidation path-targeted - -`invalidateSupersededRepositorySources()` currently derives “older file IDs” by repository URL across the latest import batch. That is too broad for incremental sync. - -Refactor it so the caller can pass explicit file IDs to retire, or add a new helper dedicated to binding/path cutover. - -Required behavior after a successful incremental batch: - -- mark only removed/replaced active file rows `deleted = true` -- set `sources.valid_until = NOW()` only for current sources attached to those retired file IDs -- keep unchanged file rows and their current sources active -- regenerate descriptions only for affected entities/relationships returned by the targeted invalidation helper - -Keep the existing full-batch repository invalidation path for public URL imports if it is still used there. Do not accidentally couple connector-only incremental semantics back into the URL-import path. - -### Step 6: Verification and regression coverage - -Add or update focused tests for the behaviors above. - -Minimum coverage: - -- `packages/connectors/src/__tests__/github.test.ts` — compare normalization -- `packages/connectors/src/__tests__/gitlab.test.ts` — compare normalization and timeout handling -- `apps/worker/lib/__tests__/code-manifest.test.ts` — binding-scoped manifests across mixed commit rows -- `apps/worker/workflows/sync-repository-graph.test.ts` — changed-only file insertion, delete/rename handling, and no-op supported-code delta handling -- `apps/worker/workflows/process-file.test.ts` — repository batch finalization still requires all child workflows to succeed -- `apps/api/src/lib/__tests__/graph-file-proxy.test.ts` and/or `source-reference.test.ts` — unchanged active rows with older commit metadata still resolve content correctly - -Then run the verification commands from the table above. - -## STOP conditions - -Stop and report instead of improvising if any of these occur: - -1. Provider compare responses cannot safely enumerate changed paths for one of the supported providers. -2. A live repository binding can contain multiple non-deleted rows for the same path, and no existing invariant guarantees which one is current. -3. Binding-scoped manifests break public URL repository imports instead of staying connector-only. -4. The targeted invalidation change would require broad source/citation semantics beyond connector-backed repository files. -5. A safe incremental path for GitHub and GitLab diverges so much that the shared provider contract becomes misleading. - -## Acceptance checklist - -Do not mark this plan done until all are true: - -- Subsequent connector syncs no longer load full branch snapshots for routine updates. -- A one-file change on a bound branch processes only that file plus any delete/rename counterpart, not every active file in the binding. -- Changes outside supported code paths do not create a process run. -- Unchanged active files remain available to the changed-file code manifest. -- Removed/replaced paths invalidate only their own current sources. -- Normal document uploads and public URL repository imports still behave the same as before. -- Tests and `bun run lint` pass with no new warnings/errors. - -## Notes for the follow-up executor - -The hard part is not the provider compare call; it is the cutover invariant: - -- changed files must see unchanged siblings in the manifest -- failed incremental batches must leave the previous active snapshot intact -- unchanged files must not churn just because branch HEAD moved - -Favor small helper modules over growing `sync-repository-graph.ts` into a second monolith. Keep provider compare logic in `packages/connectors`, binding snapshot logic in worker helpers, and public URL import behavior separate from connector-specific incremental semantics. diff --git a/plans/README.md b/plans/README.md deleted file mode 100644 index 391c7712..00000000 --- a/plans/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Implementation Plans - -Generated by the improve skill on 2026-06-13 for `branch` mode. Current branch is `main`, but the working tree has unstaged/untracked changes; the initial audit scoped to those changed files and direct callers only. Plan 006 was added afterward from the product requirement that repository code files can be external instead of copied to S3. Plan 007 was added afterward from the product requirement that re-imported code repositories represent the latest snapshot while retaining invalidated historical sources. Plan 008 was added afterward from the product requirement for provider connectors, private repository access, and webhook-driven branch updates. Plan 009 was added afterward from the follow-up requirement that connector branch updates process only changed supported code files instead of rebuilding whole repository snapshots. - -Each executor: read the plan fully before starting, honor its STOP conditions, run every verification command from the plan, and update your row when done. - -## Execution order & status - -| Plan | Title | Priority | Effort | Depends on | Status | -|---|---|---|---|---|---| -| 001 | Align the relationship migration snapshot with the schema | P1 | S | — | DONE | -| 002 | Return no path for missing same-entity path requests | P1 | S | — | DONE | -| 003 | Validate repository import models before upload and enqueue | P1 | S | — | DONE | -| 004 | Sanitize repository URL loader errors returned to clients | P2 | S | — | DONE | -| 005 | Route direct code uploads through the code workflow | P2 | M | — | DONE | -| 006 | Support external GitHub code files without copying them to S3 | P1 | L | 003, 004, ideally 005 | DONE | -| 007 | Version code sources with `validUntil` so graph tools use only the latest repository snapshot | P1 | L | 005, 006 | DONE | -| 008 | Provider connectors for private repository graphs and webhook-driven updates | P1 | XL | 006, 007 | DONE | -| 009 | Incremental connector repository updates for changed files only | P1 | L | 007, 008 | DONE | - -Status values: TODO | IN PROGRESS | DONE | BLOCKED (with one-line reason) | REJECTED (with one-line rationale — finding fixed independently or approach abandoned) - -## Dependency notes - -- Plans 001-004 are independent. -- Plan 005 is independent of the first four, but it touches broader upload/worker routing and should run after the smaller correctness fixes if execution capacity is limited. -- Plan 006 depends on the repository import safety fixes in plans 003 and 004. It should ideally run after plan 005 so external code rows enter the dedicated code workflow consistently. -- Plan 007 depends on plans 005 and 006 because it assumes code files have a dedicated workflow and repository code file rows can be external/latest-snapshot records. It should run before webhook-based repository updates. -- Plan 008 depends on plan 006 for external code file storage and plan 007 for latest-snapshot source invalidation. It should run before replacing URL imports with connector-backed private repository sync. -- Plan 009 depends on plans 007 and 008 because it narrows connector sync from whole-snapshot rebuilds to changed-file-only cutovers using `validUntil` and binding-scoped manifests. It should run after connector sync is stable. -## Verification baseline observed during planning - -- `bun run test` exited 0; all 10 Turbo tasks successful. -- `bun run lint` exited 0 with one pre-existing frontend warning: `apps/frontend/components/theme/ThemePresetScript.tsx` uses `next/script` `beforeInteractive` outside `pages/_document.js`. -- `bun run build` was not run because it writes build artifacts; use plan-specific read-only checks unless the operator explicitly allows build artifacts. - -## Vetted branch findings - -| # | Finding | Tag | Category | Impact | Effort | Risk | Evidence | Plan | -|---|---|---|---|---|---|---|---|---| -| 1 | Migration snapshot omits new `relationships.kind` and `relationships.directed` columns | introduced | migration | Future Drizzle migrations can be generated from stale schema state | S | LOW | `packages/db/src/tables/graph.ts:248-249`; `migrations/20260613184716_sturdy_triton/migration.sql:1-2`; `migrations/20260613184716_sturdy_triton/snapshot.json:3008-3021` | `plans/001-align-migration-snapshot.md` | -| 2 | Same-entity path requests return a fake `Unknown` path when the entity is absent | introduced | bug | Invalid/deleted/cross-graph IDs can be presented to the model as existing graph entities | S | LOW | `packages/ai/src/tools/relationship.ts:449-463` | `plans/002-return-none-for-missing-same-entity-path.md` | -| 3 | Repository URL imports skip pre-upload model validation | introduced | bug | Missing model configuration is discovered after S3/DB/workflow side effects instead of before them | S | LOW | `apps/api/src/routes/graph.ts:632-646`; `apps/api/src/routes/graph.ts:847-858` | `plans/003-validate-repository-import-models.md` | -| 4 | Repository URL loader returns raw git stderr to clients | introduced | security | Client responses disclose operational git/provider details and depend on unstable tool output | S | LOW | `apps/api/src/lib/repository-url.ts:194-197`; `apps/api/src/routes/graph.ts:109-116` | `plans/004-sanitize-repository-url-errors.md` | -| 5 | Direct and archive-expanded code uploads do not reach the code workflow | introduced | bug | Standalone source uploads are processed as generic text; mixed batches need per-file workflow routing | M | MED | `packages/graph/src/file-type.ts:1-21`; `packages/graph/src/file-type.ts:52-187`; `apps/api/src/lib/graph-upload-file-type.ts:23-31`; `apps/worker/workflows/process-file.ts:47-56` | `plans/005-route-code-uploads-through-code-workflow.md` | -| 6 | Repository import limits are enforced after full clones and only per repository | introduced | security/perf | One authenticated request can force multiple large public clones before code byte/file limits reject anything | M | MED | `apps/api/src/routes/graph.ts:582-596`; `apps/api/src/lib/repository-url.ts:89-121` | not planned by default | -| 7 | Code symbol resolution keys every in-file definition by simple name | introduced | bug | Same-named functions/methods can produce false `CALLS` edges and polluted code graph answers | M | MED | `packages/graph/src/code/repository.ts:51-65`; `packages/graph/src/code/repository.ts:236-262`; `packages/graph/src/code/__tests__/repository.test.ts:20-77` | not planned by default | -| 8 | Batch process runs complete when only some child file workflows fail | pre-existing in touched files | bug | Mixed success/failure batches can record process-run success despite failed files | S | MED | `apps/worker/workflows/process-file.ts:105-143`; `apps/worker/workflows/process-file.ts:495-500`; `apps/worker/workflows/process-code-file.ts:221-226` | not planned by default | -| 9 | Relationship endpoints are not constrained to the same graph as the relationship | pre-existing in touched files | security | Malformed writes can make graph-scoped tools disclose entity names/descriptions from another graph | M | MED | `packages/db/src/tables/graph.ts:239-247`; `packages/ai/src/tools/relationship.ts:289-324`; `packages/ai/src/tools/relationship.ts:365-405` | not planned by default | -| 10 | Code-manifest preparation serially reads same-repository files before fan-out | introduced | performance | Large repository batches delay child processing and hold more content in memory | M | MED | `apps/worker/workflows/process-file.ts:95-103`; `apps/worker/lib/code-manifest.ts:46-86` | not planned by default | -| 11 | Source chunk validator accepts new code-location fields without validating their types | introduced | correctness | Malformed stored chunks can pass validation and leak invalid location metadata into source references | S | LOW | `packages/contracts/src/source.ts:21-27`; `packages/contracts/src/source.ts:93-122`; `apps/api/src/lib/source-reference-record.ts:107-110` | not planned by default | - -## Direction / product plans - -| Plan | Direction | Evidence | Trade-off | -|---|---|---|---| -| `plans/006-support-external-github-code-files.md` | Make repository code files external for GitHub imports instead of copying every source file to S3. | `files.file_key` is currently S3-oriented and non-null (`packages/db/src/tables/graph.ts:136-148`); repository imports upload every code source via `putGraphFile` before DB insert (`apps/api/src/routes/graph.ts:652-667`); code processing reads source through S3 (`apps/worker/workflows/process-code-file.ts:79-93`). | Saves storage and makes source links canonical, but requires first-class file origin columns, external fetch allowlisting, and proxy/worker changes. | -| `plans/007-version-code-sources-with-valid-until.md` | Re-importing the same repository should supersede old code evidence with `sources.validUntil` while retaining historical source rows. | `sources` currently has `active` but no validity window (`packages/db/src/tables/graph.ts:291-314`); graph save appends source rows (`apps/worker/lib/save-graph.ts:118-143`); description regeneration reads all linked sources (`apps/worker/lib/regenerate-descriptions.ts:148-163`); source tools filter only `active` (`packages/ai/src/tools/source.ts:205-310`). | Keeps current answers fresh and preserves history, but requires schema migration, repository snapshot cutover semantics, source invalidation, description deactivation, and tool/citation filtering. | -| `plans/008-provider-connectors-for-repository-graphs.md` | Add system-admin GitHub/GitLab connectors, graph-manager repository installation/selection, private repo graph creation, and push-webhook rebuilds for selected branches. | Graph managers already map to org admin/team admin/team moderator (`apps/api/src/lib/team-access.ts:140-155`); URL imports still load repositories from public URLs (`apps/api/src/lib/repository-url.ts:161-169`); graph routes can enqueue repository processing (`apps/api/src/routes/graph.ts:807-812`); webhooks need a public verified route before auth middleware (`apps/api/src/server.ts:53-61`). | Unlocks private repos and automatic branch freshness, but adds provider auth, encrypted credentials, webhook verification/idempotency, repository binding state, and provider API rate-limit handling. | -| `plans/009-incremental-connector-repository-updates.md` | Update connector-backed repository graphs incrementally so later branch syncs process only changed supported code files instead of rebuilding whole snapshots. | Connector sync currently loads a full provider snapshot and processes every inserted file (`apps/worker/workflows/sync-repository-graph.ts:190-235`); repository manifests are still scoped by `repositoryUrl + commitSha` (`apps/worker/lib/code-manifest.ts:45-65,102-104`); plan 008 explicitly left fine-grained incremental updates out of scope (`plans/008-provider-connectors-for-repository-graphs.md:219-226`). | Cuts worker/runtime churn and provider API load, but requires provider compare APIs, binding-scoped manifests, and targeted file/source invalidation so unchanged files stay current without full reprocessing. | -## Findings considered and rejected - -- Archive traversal in changed upload flow: not retained. Existing archive tests cover unsafe paths and option-looking names; no new high-confidence traversal issue was found in the changed files. -- Directed/undirected relationship dedupe key normalization: not retained. `packages/graph/src/relationship-key.ts` and `packages/graph/src/dedupe.ts` consistently include kind, direction, and normalized endpoints for undirected edges. -- Worker Docker tree-sitter rebuild: not retained. The Dockerfile copies graph package sources and rebuilds the native binding; no branch-specific failure was confirmed. -- Raw repository URL credentials in clone commands: not retained. `normalizeRepositoryUrl` rejects username/password and restricts hosts before cloning. - -## Not audited - -- Full frontend behavior outside files touched by the working tree. -- Production Compose/Caddy behavior beyond changed Dockerfiles. -- Dependency advisories/audit output. -- Runtime e2e repository import against real GitHub/GitLab/Bitbucket repositories. -- Build output, because build commands write artifacts. From 10f44845911757a5d15c6b53efdf68db065adf71 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 10:51:47 +0200 Subject: [PATCH 03/23] fix: repair code graph ci --- apps/worker/lib/code-repository-finalizer.ts | 5 +++-- apps/worker/workflows/sync-repository-graph.test.ts | 10 +++++----- bun.lock | 3 +++ package.json | 5 ++++- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/apps/worker/lib/code-repository-finalizer.ts b/apps/worker/lib/code-repository-finalizer.ts index 56265c73..f91a7dbe 100644 --- a/apps/worker/lib/code-repository-finalizer.ts +++ b/apps/worker/lib/code-repository-finalizer.ts @@ -61,7 +61,8 @@ export async function invalidateSupersededRepositorySources(options: { }); } - if (!options.latestFileIds || options.latestFileIds.length === 0) { + const latestFileIds = options.latestFileIds; + if (!latestFileIds || latestFileIds.length === 0) { return { entityIds: [], relationshipIds: [] }; } @@ -69,7 +70,7 @@ export async function invalidateSupersededRepositorySources(options: { const latestRows = await tx .select({ id: filesTable.id, metadata: filesTable.metadata }) .from(filesTable) - .where(and(eq(filesTable.graphId, options.graphId), inArray(filesTable.id, options.latestFileIds))); + .where(and(eq(filesTable.graphId, options.graphId), inArray(filesTable.id, latestFileIds))); const candidateRows = await tx .select({ id: filesTable.id, metadata: filesTable.metadata }) .from(filesTable) diff --git a/apps/worker/workflows/sync-repository-graph.test.ts b/apps/worker/workflows/sync-repository-graph.test.ts index 36591651..7f96fb37 100644 --- a/apps/worker/workflows/sync-repository-graph.test.ts +++ b/apps/worker/workflows/sync-repository-graph.test.ts @@ -190,22 +190,22 @@ async function runWorkflow(input: { bindingId: string; reason: "manual" | "webho return syncRepositoryGraph.fn({ input, step: { - run: async (_config: { name: string }, fn: () => Promise | unknown) => fn(), - runWorkflow: async (spec: { name: string }, workflowInput?: Record) => { + run: async (_config: { name: string }, fn: () => unknown) => fn(), + runWorkflow: async (spec: { name: string }, workflowInput?: unknown) => { if (spec.name === "process-files") { - processWorkflowInputs.push(workflowInput ?? {}); + processWorkflowInputs.push((workflowInput ?? {}) as Record); if (processFilesError) { throw processFilesError; } return undefined; } if (spec.name === "delete-file") { - deleteWorkflowInputs.push(workflowInput ?? {}); + deleteWorkflowInputs.push((workflowInput ?? {}) as Record); return undefined; } throw new Error(`Unexpected workflow ${spec.name}`); }, - }, + } as never, version: null, run: { id: "workflow-run-1", diff --git a/bun.lock b/bun.lock index 411bca06..cfee9309 100644 --- a/bun.lock +++ b/bun.lock @@ -295,6 +295,9 @@ }, }, }, + "trustedDependencies": [ + "tree-sitter", + ], "patchedDependencies": { "openworkflow@0.9.0": "patches/openworkflow@0.9.0.patch", }, diff --git a/package.json b/package.json index 0d1cb2d4..eba652a2 100644 --- a/package.json +++ b/package.json @@ -65,5 +65,8 @@ }, "patchedDependencies": { "openworkflow@0.9.0": "patches/openworkflow@0.9.0.patch" - } + }, + "trustedDependencies": [ + "tree-sitter" + ] } From 9eb081712e1568dc67ff6b2dd87ee3a58beffa2a Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 11:09:15 +0200 Subject: [PATCH 04/23] fix: prepare tree-sitter native bindings in ci --- .github/workflows/ci.yml | 16 ++++++++++++++++ apps/worker/Dockerfile | 9 +++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ece29b8e..5e27a076 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,14 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile + env: + CXXFLAGS: -std=c++20 + + - name: Patch tree-sitter Bun native binding + run: | + mkdir -p node_modules/.bun/tree-sitter@0.25.0/node_modules/tree-sitter/prebuilds/linux-x64 + cp node_modules/.bun/tree-sitter@0.25.0/node_modules/tree-sitter/build/Release/tree_sitter_runtime_binding.node node_modules/.bun/tree-sitter@0.25.0/node_modules/tree-sitter/prebuilds/linux-x64/tree-sitter.node + cp node_modules/.bun/@tree-sitter-grammars+tree-sitter-zig@1.1.2+7aa2583e580bc282/node_modules/@tree-sitter-grammars/tree-sitter-zig/prebuilds/linux-x64/@tree-sitter-grammars+tree-sitter-zig.node node_modules/.bun/@tree-sitter-grammars+tree-sitter-zig@1.1.2+7aa2583e580bc282/node_modules/@tree-sitter-grammars/tree-sitter-zig/prebuilds/linux-x64/tree-sitter-zig.node - name: Prepare environment run: cp .env.sample .env @@ -103,6 +111,14 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile + env: + CXXFLAGS: -std=c++20 + + - name: Patch tree-sitter Bun native binding + run: | + mkdir -p node_modules/.bun/tree-sitter@0.25.0/node_modules/tree-sitter/prebuilds/linux-x64 + cp node_modules/.bun/tree-sitter@0.25.0/node_modules/tree-sitter/build/Release/tree_sitter_runtime_binding.node node_modules/.bun/tree-sitter@0.25.0/node_modules/tree-sitter/prebuilds/linux-x64/tree-sitter.node + cp node_modules/.bun/@tree-sitter-grammars+tree-sitter-zig@1.1.2+7aa2583e580bc282/node_modules/@tree-sitter-grammars/tree-sitter-zig/prebuilds/linux-x64/@tree-sitter-grammars+tree-sitter-zig.node node_modules/.bun/@tree-sitter-grammars+tree-sitter-zig@1.1.2+7aa2583e580bc282/node_modules/@tree-sitter-grammars/tree-sitter-zig/prebuilds/linux-x64/tree-sitter-zig.node - name: Prepare environment run: cp .env.sample .env diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index a4e2e056..eb0ee011 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -18,6 +18,7 @@ COPY apps/api/package.json apps/api/package.json COPY apps/frontend/package.json apps/frontend/package.json COPY apps/worker/package.json apps/worker/package.json COPY packages/ai/package.json packages/ai/package.json +COPY packages/connectors/package.json packages/connectors/package.json COPY packages/auth/package.json packages/auth/package.json COPY packages/contracts/package.json packages/contracts/package.json COPY packages/db/package.json packages/db/package.json @@ -25,13 +26,15 @@ COPY packages/files/package.json packages/files/package.json COPY packages/graph/package.json packages/graph/package.json COPY packages/logger/package.json packages/logger/package.json -RUN bun install --frozen-lockfile +RUN CXXFLAGS="-std=c++20" bun install --frozen-lockfile RUN arch="$(node -p 'process.arch')" \ && cd packages/graph/node_modules/tree-sitter \ && CXXFLAGS="-std=c++20" node-gyp rebuild \ && mkdir -p "prebuilds/linux-${arch}" \ - && cp build/Release/tree_sitter_runtime_binding.node "prebuilds/linux-${arch}/tree-sitter.node" + && cp build/Release/tree_sitter_runtime_binding.node "prebuilds/linux-${arch}/tree-sitter.node" \ + && cd ../@tree-sitter-grammars/tree-sitter-zig \ + && cp "prebuilds/linux-${arch}/@tree-sitter-grammars+tree-sitter-zig.node" "prebuilds/linux-${arch}/tree-sitter-zig.node" COPY apps/worker/index.ts apps/worker/index.ts COPY apps/worker/env.ts apps/worker/env.ts @@ -42,6 +45,8 @@ COPY apps/worker/lib apps/worker/lib COPY apps/worker/workflows apps/worker/workflows COPY packages/ai/src packages/ai/src COPY packages/ai/tsconfig.json packages/ai/tsconfig.json +COPY packages/connectors/src packages/connectors/src +COPY packages/connectors/tsconfig.json packages/connectors/tsconfig.json COPY packages/contracts/src packages/contracts/src COPY packages/contracts/tsconfig.json packages/contracts/tsconfig.json COPY packages/db/src packages/db/src From 958459440ec800487f50d0b4c4a536ec81d83331 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 11:43:39 +0200 Subject: [PATCH 05/23] fix: address greptile feedback --- .../lib/__tests__/repository-url-git.test.ts | 44 + apps/api/src/lib/repository-url.ts | 23 +- .../__tests__/connector-webhooks.test.ts | 163 + apps/api/src/routes/connector-webhooks.ts | 175 +- .../workflows/sync-repository-graph.test.ts | 139 +- .../worker/workflows/sync-repository-graph.ts | 190 +- .../20260614105716_tricky_plazm/snapshot.json | 107 +- .../migration.sql | 2 + .../snapshot.json | 8288 +++++++++++++++++ .../connectors/src/__tests__/github.test.ts | 39 +- .../connectors/src/__tests__/gitlab.test.ts | 15 +- packages/connectors/src/github.ts | 74 +- packages/connectors/src/gitlab.ts | 56 +- packages/connectors/src/types.ts | 17 +- packages/db/src/tables/connectors.ts | 46 +- packages/db/src/tables/graph.ts | 2 +- 16 files changed, 9186 insertions(+), 194 deletions(-) create mode 100644 apps/api/src/lib/__tests__/repository-url-git.test.ts create mode 100644 apps/api/src/routes/__tests__/connector-webhooks.test.ts create mode 100644 migrations/20260615094125_fluffy_gladiator/migration.sql create mode 100644 migrations/20260615094125_fluffy_gladiator/snapshot.json diff --git a/apps/api/src/lib/__tests__/repository-url-git.test.ts b/apps/api/src/lib/__tests__/repository-url-git.test.ts new file mode 100644 index 00000000..14e6f7b0 --- /dev/null +++ b/apps/api/src/lib/__tests__/repository-url-git.test.ts @@ -0,0 +1,44 @@ +import { EventEmitter } from "node:events"; +import { beforeEach, describe, expect, mock, test } from "bun:test"; + +const MAX_GIT_OUTPUT_BYTES = 1 * 1024 * 1024; +let spawnCall = 0; + +mock.module("node:child_process", () => ({ + spawn: () => { + spawnCall += 1; + const child = Object.assign(new EventEmitter(), { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + kill: () => undefined, + }); + + queueMicrotask(() => { + if (spawnCall === 2) { + child.stdout.emit("data", Buffer.from("commit-sha\n")); + } + if (spawnCall === 3) { + child.stdout.emit("data", Buffer.alloc(MAX_GIT_OUTPUT_BYTES + 1)); + } + child.emit("close", spawnCall === 3 ? null : 0); + }); + + return child; + }, +})); + +// Dynamic import is required so Bun module mocks replace child_process before evaluation. +const { loadRepositoryFromUrl } = await import("../repository-url"); + +describe("repository URL git loading", () => { + beforeEach(() => { + spawnCall = 0; + }); + + test("rejects truncated git stdout as a repository limit", async () => { + await expect(loadRepositoryFromUrl("https://github.com/acme/app")).rejects.toMatchObject({ + name: "RepositoryUrlError", + kind: "limit", + }); + }); +}); diff --git a/apps/api/src/lib/repository-url.ts b/apps/api/src/lib/repository-url.ts index 5043cf21..a39143a1 100644 --- a/apps/api/src/lib/repository-url.ts +++ b/apps/api/src/lib/repository-url.ts @@ -233,6 +233,7 @@ function runGit(args: string[], cwd: string): Promise { const stderr: Buffer[] = []; let stdoutBytes = 0; let stderrBytes = 0; + let stdoutLimitExceeded = false; let settled = false; const finish = (callback: () => void) => { @@ -258,9 +259,12 @@ function runGit(args: string[], cwd: string): Promise { child.stdout.on("data", (chunk: Buffer) => { stdoutBytes += chunk.byteLength; - if (stdoutBytes <= MAX_GIT_OUTPUT_BYTES) { - stdout.push(chunk); + if (stdoutBytes > MAX_GIT_OUTPUT_BYTES) { + stdoutLimitExceeded = true; + child.kill("SIGKILL"); + return; } + stdout.push(chunk); }); child.stderr.on("data", (chunk: Buffer) => { stderrBytes += chunk.byteLength; @@ -269,9 +273,22 @@ function runGit(args: string[], cwd: string): Promise { } }); child.on("error", (error) => - finish(() => reject(new RepositoryUrlError("load", "Repository could not be loaded", { cause: error }))) + finish(() => + reject( + stdoutLimitExceeded + ? new RepositoryUrlError("limit", "Repository git output exceeded the supported size") + : new RepositoryUrlError("load", "Repository could not be loaded", { cause: error }) + ) + ) ); child.on("close", (code) => { + if (stdoutLimitExceeded) { + finish(() => + reject(new RepositoryUrlError("limit", "Repository git output exceeded the supported size")) + ); + return; + } + if (code === 0) { finish(() => resolve(Buffer.concat(stdout).toString("utf8"))); return; diff --git a/apps/api/src/routes/__tests__/connector-webhooks.test.ts b/apps/api/src/routes/__tests__/connector-webhooks.test.ts new file mode 100644 index 00000000..8b6c4c1d --- /dev/null +++ b/apps/api/src/routes/__tests__/connector-webhooks.test.ts @@ -0,0 +1,163 @@ +import { createHmac } from "node:crypto"; +import { beforeEach, describe, expect, mock, test } from "bun:test"; + +type SelectResult = { kind: "where"; value: unknown[] } | { kind: "limit"; value: unknown[] }; + +const connector = { + id: "connector-1", + provider: "github", + status: "active", + webhookSecretEncrypted: "encrypted-secret", +}; + +const bindingOne = { id: "binding-1" }; +const bindingTwo = { id: "binding-2" }; +const ledger = { id: "event-1", status: "enqueued" }; +const failedLedger = { id: "event-1", status: "failed" }; + +const selectResults: SelectResult[] = []; +const insertResults: unknown[][] = []; +const updates: Array> = []; +const joins: unknown[] = []; +const workflowInputs: Array> = []; +let workflowFailureIndex: number | null = null; + +function selectQuery() { + const result = selectResults.shift(); + if (!result) { + throw new Error("Unexpected select call"); + } + + const chain = { + from: () => chain, + innerJoin: (...args: unknown[]) => { + joins.push(args); + return chain; + }, + where: () => (result.kind === "where" ? result.value : chain), + limit: () => result.value, + }; + return chain; +} + +mock.module("@kiwi/db", () => ({ + db: { + select: () => selectQuery(), + insert: () => ({ + values: () => ({ + onConflictDoNothing: () => ({ + returning: () => insertResults.shift() ?? [], + }), + }), + }), + update: () => ({ + set: (values: Record) => ({ + where: async () => { + updates.push(values); + return undefined; + }, + }), + }), + }, +})); + +mock.module("../../lib/connectors", () => ({ + decryptSecret: () => "secret", +})); + +mock.module("../../openworkflow", () => ({ + ow: { + runWorkflow: async (_spec: unknown, input: Record) => { + workflowInputs.push(input); + if (workflowFailureIndex === workflowInputs.length) { + throw new Error("enqueue failed"); + } + return { workflowRun: { id: `run-${workflowInputs.length}` } }; + }, + }, +})); + +function signedGitHubRequest(body: string) { + const signature = `sha256=${createHmac("sha256", "secret").update(body).digest("hex")}`; + return new Request("http://localhost/connectors/webhooks/github", { + method: "POST", + headers: { + "x-hub-signature-256": signature, + "x-github-event": "push", + "x-github-delivery": "delivery-1", + }, + body, + }); +} + +function pushBody() { + return JSON.stringify({ + ref: "refs/heads/main", + after: "commit-new", + repository: { id: 7, full_name: "acme/app" }, + }); +} + +// Dynamic import is required so Bun module mocks are installed before the route module is evaluated. +const { connectorWebhookRoute } = await import("../connector-webhooks"); + +describe("connector webhook route", () => { + beforeEach(() => { + selectResults.length = 0; + insertResults.length = 0; + updates.length = 0; + joins.length = 0; + workflowInputs.length = 0; + workflowFailureIndex = null; + }); + + test("returns a structured 400 for signed invalid JSON", async () => { + selectResults.push({ kind: "where", value: [connector] }); + + const response = await connectorWebhookRoute.handle(signedGitHubRequest("not-json")); + + expect(response.status).toBe(400); + await expect(response.json()).resolves.toMatchObject({ + status: "error", + code: "INVALID_CHAT_REQUEST", + }); + expect(insertResults).toEqual([]); + }); + + test("marks failed workflow enqueues so delivery retries can recover", async () => { + selectResults.push( + { kind: "where", value: [connector] }, + { kind: "where", value: [{ binding: bindingOne }, { binding: bindingTwo }] } + ); + insertResults.push([ledger]); + workflowFailureIndex = 2; + + const response = await connectorWebhookRoute.handle(signedGitHubRequest(pushBody())); + + expect(response.status).toBe(500); + expect(workflowInputs.map((input) => input.bindingId)).toEqual(["binding-1", "binding-2"]); + expect(updates).toContainEqual({ lastSeenCommitSha: "commit-new", syncStatus: "pending", syncErrorCode: null }); + expect(updates).toContainEqual({ syncStatus: "failed", syncErrorCode: "enqueue_failed" }); + expect(updates).toContainEqual({ status: "failed", errorCode: "enqueue_failed" }); + expect(joins).toHaveLength(1); + }); + + test("re-enqueues failed duplicate webhook deliveries", async () => { + selectResults.push( + { kind: "where", value: [connector] }, + { kind: "where", value: [{ binding: bindingOne }, { binding: bindingTwo }] }, + { kind: "limit", value: [failedLedger] } + ); + insertResults.push([]); + + const response = await connectorWebhookRoute.handle(signedGitHubRequest(pushBody())); + + expect(response.status).toBe(202); + await expect(response.json()).resolves.toMatchObject({ + status: "success", + data: { status: "enqueued", enqueued: 2 }, + }); + expect(workflowInputs.map((input) => input.bindingId)).toEqual(["binding-1", "binding-2"]); + expect(updates).toContainEqual({ status: "enqueued", errorCode: null }); + }); +}); diff --git a/apps/api/src/routes/connector-webhooks.ts b/apps/api/src/routes/connector-webhooks.ts index f517f7c4..0b840dbb 100644 --- a/apps/api/src/routes/connector-webhooks.ts +++ b/apps/api/src/routes/connector-webhooks.ts @@ -2,6 +2,7 @@ import { createHmac, timingSafeEqual } from "node:crypto"; import { db } from "@kiwi/db"; import { connectorWebhookEventsTable, + connectorInstallationsTable, connectorsTable, repositoryGraphBindingsTable, type ConnectorProvider, @@ -12,6 +13,7 @@ import Elysia from "elysia"; import { decryptSecret } from "../lib/connectors"; import { ow } from "../openworkflow"; import { API_ERROR_CODES, errorResponse, successResponse } from "../types"; +import type { ApiErrorCode } from "../types"; type NormalizedPush = { eventName: string; @@ -30,6 +32,19 @@ function equalSecret(left: string, right: string) { const rightBuffer = Buffer.from(right); return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer); } +function jsonError(message: string, code: ApiErrorCode, status: number) { + return new Response(JSON.stringify(errorResponse(message, code)), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +function jsonSuccess(data: TData, status: number) { + return new Response(JSON.stringify(successResponse(data)), { + status, + headers: { "Content-Type": "application/json" }, + }); +} function verifyGitHubSignature(rawBody: string, signatureHeader: string | null, secret: string) { if (!signatureHeader?.startsWith("sha256=")) { @@ -75,6 +90,77 @@ function normalizeGitLab(payload: Record, eventName: string, de deleted: !commitSha || ZERO_SHA.test(commitSha), }; } +function parseWebhookPayload(rawBody: string): Record | null { + try { + const payload = JSON.parse(rawBody); + return payload && typeof payload === "object" && !Array.isArray(payload) + ? (payload as Record) + : null; + } catch { + return null; + } +} + +async function loadWebhookEvent(connectorId: string, provider: ConnectorProvider, deliveryId: string) { + const [event] = await db + .select() + .from(connectorWebhookEventsTable) + .where( + and( + eq(connectorWebhookEventsTable.connectorId, connectorId), + eq(connectorWebhookEventsTable.provider, provider), + eq(connectorWebhookEventsTable.deliveryId, deliveryId) + ) + ) + .limit(1); + return event ?? null; +} + +async function markWebhookEvent(id: string, values: { status: "enqueued" | "failed"; errorCode: string | null }) { + await db.update(connectorWebhookEventsTable).set(values).where(eq(connectorWebhookEventsTable.id, id)); +} + +async function enqueueBindingWorkflows( + bindings: Array, + commitSha: string, + deliveryId: string +) { + const enqueuedBindingIds = new Set(); + try { + for (const binding of bindings) { + await ow.runWorkflow(syncRepositoryGraphSpec, { + bindingId: binding.id, + reason: "webhook", + commitSha, + deliveryId, + }); + enqueuedBindingIds.add(binding.id); + } + } catch (error) { + await Promise.all( + bindings.map((binding) => + db + .update(repositoryGraphBindingsTable) + .set( + enqueuedBindingIds.has(binding.id) + ? { lastSeenCommitSha: commitSha, syncStatus: "pending", syncErrorCode: null } + : { syncStatus: "failed", syncErrorCode: "enqueue_failed" } + ) + .where(eq(repositoryGraphBindingsTable.id, binding.id)) + ) + ); + throw error; + } + + await Promise.all( + bindings.map((binding) => + db + .update(repositoryGraphBindingsTable) + .set({ lastSeenCommitSha: commitSha, syncStatus: "pending", syncErrorCode: null }) + .where(eq(repositoryGraphBindingsTable.id, binding.id)) + ) + ); +} async function findVerifiedConnector(provider: ConnectorProvider, request: Request, rawBody: string) { const connectors = await db @@ -100,13 +186,13 @@ async function handleWebhook(provider: ConnectorProvider, request: Request) { const rawBody = await request.text(); const connector = await findVerifiedConnector(provider, request, rawBody); if (!connector) { - return new Response(JSON.stringify(errorResponse("Invalid webhook signature", API_ERROR_CODES.FORBIDDEN)), { - status: 403, - headers: { "Content-Type": "application/json" }, - }); + return jsonError("Invalid webhook signature", API_ERROR_CODES.FORBIDDEN, 403); } - const payload = JSON.parse(rawBody) as Record; + const payload = parseWebhookPayload(rawBody); + if (!payload) { + return jsonError("Invalid webhook payload", API_ERROR_CODES.INVALID_CHAT_REQUEST, 400); + } const eventName = provider === "github" ? request.headers.get("x-github-event") || "unknown" @@ -114,9 +200,14 @@ async function handleWebhook(provider: ConnectorProvider, request: Request) { const deliveryId = provider === "github" ? request.headers.get("x-github-delivery") || "missing" - : request.headers.get("x-gitlab-webhook-uuid") || `${String(payload.event_name ?? "event")}:${String(payload.after ?? Date.now())}`; - const normalized = provider === "github" ? normalizeGitHub(payload, eventName, deliveryId) : normalizeGitLab(payload, eventName, deliveryId); - const isPush = provider === "github" ? eventName === "push" : eventName === "Push Hook" || normalized.eventName === "push"; + : request.headers.get("x-gitlab-webhook-uuid") || + `${String(payload.event_name ?? "event")}:${String(payload.after ?? Date.now())}`; + const normalized = + provider === "github" + ? normalizeGitHub(payload, eventName, deliveryId) + : normalizeGitLab(payload, eventName, deliveryId); + const isPush = + provider === "github" ? eventName === "push" : eventName === "Push Hook" || normalized.eventName === "push"; let bindings: Array = []; if ( @@ -135,17 +226,23 @@ async function handleWebhook(provider: ConnectorProvider, request: Request) { : normalized.providerRepositoryId ? eq(repositoryGraphBindingsTable.providerRepositoryId, normalized.providerRepositoryId) : eq(repositoryGraphBindingsTable.repositoryFullName, normalized.repositoryFullName!); - bindings = await db - .select() + const bindingRows = await db + .select({ binding: repositoryGraphBindingsTable }) .from(repositoryGraphBindingsTable) + .innerJoin( + connectorInstallationsTable, + eq(repositoryGraphBindingsTable.connectorInstallationId, connectorInstallationsTable.id) + ) .where( and( + eq(connectorInstallationsTable.connectorId, connector.id), eq(repositoryGraphBindingsTable.provider, provider), eq(repositoryGraphBindingsTable.branch, normalized.branch), eq(repositoryGraphBindingsTable.webhookEnabled, true), repositoryWhere ) ); + bindings = bindingRows.map((row) => row.binding); } const status = !isPush || normalized.deleted || bindings.length === 0 ? "ignored" : "enqueued"; @@ -170,40 +267,42 @@ async function handleWebhook(provider: ConnectorProvider, request: Request) { }) .returning(); - if (!ledger) { - return new Response(JSON.stringify(successResponse({ status: "duplicate" })), { - status: 202, - headers: { "Content-Type": "application/json" }, - }); + let activeLedger = ledger ?? null; + if (!activeLedger) { + const existingLedger = await loadWebhookEvent(connector.id, provider, deliveryId); + if (existingLedger?.status !== "failed" || status !== "enqueued" || !normalized.commitSha) { + return jsonSuccess({ status: "duplicate" }, 202); + } + activeLedger = existingLedger; } if (status === "enqueued" && normalized.commitSha) { - for (const binding of bindings) { - await db - .update(repositoryGraphBindingsTable) - .set({ lastSeenCommitSha: normalized.commitSha, syncStatus: "pending", syncErrorCode: null }) - .where(eq(repositoryGraphBindingsTable.id, binding.id)); - await ow.runWorkflow(syncRepositoryGraphSpec, { - bindingId: binding.id, - reason: "webhook", - commitSha: normalized.commitSha, - deliveryId, - }); + try { + await enqueueBindingWorkflows(bindings, normalized.commitSha, deliveryId); + if (activeLedger.status === "failed") { + await markWebhookEvent(activeLedger.id, { status: "enqueued", errorCode: null }); + } + } catch (error) { + await markWebhookEvent(activeLedger.id, { status: "failed", errorCode: "enqueue_failed" }); + throw error; } } - return new Response(JSON.stringify(successResponse({ status, enqueued: status === "enqueued" ? bindings.length : 0 })), { - status: 202, - headers: { "Content-Type": "application/json" }, - }); + return jsonSuccess({ status, enqueued: status === "enqueued" ? bindings.length : 0 }, 202); } -export const connectorWebhookRoute = new Elysia().post("/connectors/webhooks/:provider", async ({ params, request }) => { - if (params.provider !== "github" && params.provider !== "gitlab") { - return new Response(JSON.stringify(errorResponse("Unsupported provider", API_ERROR_CODES.INVALID_CHAT_REQUEST)), { - status: 404, - headers: { "Content-Type": "application/json" }, - }); +export const connectorWebhookRoute = new Elysia().post( + "/connectors/webhooks/:provider", + async ({ params, request }) => { + if (params.provider !== "github" && params.provider !== "gitlab") { + return new Response( + JSON.stringify(errorResponse("Unsupported provider", API_ERROR_CODES.INVALID_CHAT_REQUEST)), + { + status: 404, + headers: { "Content-Type": "application/json" }, + } + ); + } + return handleWebhook(params.provider, request); } - return handleWebhook(params.provider, request); -}); +); diff --git a/apps/worker/workflows/sync-repository-graph.test.ts b/apps/worker/workflows/sync-repository-graph.test.ts index 7f96fb37..4dcfbe2b 100644 --- a/apps/worker/workflows/sync-repository-graph.test.ts +++ b/apps/worker/workflows/sync-repository-graph.test.ts @@ -1,9 +1,7 @@ import { beforeEach, describe, expect, mock, test } from "bun:test"; -import type { ProviderRepositoryChange } from "@kiwi/connectors"; +import type { ProviderCodeFile, ProviderRepositoryChange } from "@kiwi/connectors"; -type SelectResult = - | { kind: "limit"; value: unknown[] } - | { kind: "where"; value: unknown[] }; +type SelectResult = { kind: "limit"; value: unknown[] } | { kind: "where"; value: unknown[] }; const insertedFileValues: Array> = []; const processWorkflowInputs: Array> = []; @@ -13,8 +11,13 @@ const compareCalls: Array<{ fromCommitSha: string; toCommitSha: string }> = []; const readFileCalls: Array<{ path: string; commitSha: string }> = []; let selectResults: SelectResult[] = []; +let txSelectResults: unknown[][] = []; +let insertedFileRowsOverride: Array<{ id: string; key: string }> | null = null; +let processRunInsertCount = 0; let processFilesError: Error | null = null; let compareChanges: ProviderRepositoryChange[] = []; +let compareIsIncremental = true; +let snapshotFiles: ProviderCodeFile[] = []; let readFileContents: Record = {}; let loadSnapshotCalls = 0; @@ -33,7 +36,23 @@ function createSelectQuery() { return chain; } +function createTxSelectQuery() { + const result = txSelectResults.shift(); + if (!result) { + throw new Error("Unexpected transaction select call"); + } + + const chain = { + from: () => chain, + innerJoin: () => chain, + where: () => result, + limit: () => result, + }; + return chain; +} + const transactionDb = { + select: () => createTxSelectQuery(), update: () => ({ set: (values: Record) => ({ where: async () => { @@ -46,7 +65,10 @@ const transactionDb = { values: (values: unknown) => { if ( Array.isArray(values) && - values.every((value) => typeof value === "object" && value !== null && "processRunId" in value && "fileId" in value) + values.every( + (value) => + typeof value === "object" && value !== null && "processRunId" in value && "fileId" in value + ) ) { return undefined; } @@ -58,10 +80,19 @@ const transactionDb = { throw new Error("Expected file rows"); } insertedFileValues.push(...(values as Array>)); - return (values as Array<{ id: string }>).map((value) => ({ id: value.id })); + return ( + insertedFileRowsOverride ?? + (values as Array<{ id: string; key: string }>).map((value) => ({ + id: value.id, + key: value.key, + })) + ); }, }), - returning: () => [{ id: "process-run-1" }], + returning: () => { + processRunInsertCount += 1; + return [{ id: "process-run-1" }]; + }, }; }, }), @@ -84,7 +115,10 @@ mock.module("@kiwi/db", () => ({ mock.module("@kiwi/connectors", () => ({ ConnectorProviderError: class ConnectorProviderError extends Error { - constructor(public readonly kind: string, message: string) { + constructor( + public readonly kind: string, + message: string + ) { super(message); this.name = "ConnectorProviderError"; } @@ -109,12 +143,12 @@ mock.module("@kiwi/connectors", () => ({ }, branch: { name: "main", commitSha: "commit-new" }, commitSha: "commit-new", - files: [], + files: snapshotFiles, }; }, compareRepository: async (_repository: unknown, fromCommitSha: string, toCommitSha: string) => { compareCalls.push({ fromCommitSha, toCommitSha }); - return { fromCommitSha, toCommitSha, changes: compareChanges }; + return { fromCommitSha, toCommitSha, isIncremental: compareIsIncremental, changes: compareChanges }; }, readFile: async (_repository: unknown, path: string, commitSha: string) => { readFileCalls.push({ path, commitSha }); @@ -233,8 +267,13 @@ describe("syncRepositoryGraph", () => { selectResults = []; processFilesError = null; compareChanges = []; + compareIsIncremental = true; + snapshotFiles = []; readFileContents = {}; loadSnapshotCalls = 0; + txSelectResults = []; + insertedFileRowsOverride = null; + processRunInsertCount = 0; }); test("processes only changed supported connector files", async () => { @@ -269,6 +308,38 @@ describe("syncRepositoryGraph", () => { expect(String(insertedFileValues[0]?.checksum)).toStartWith("commit-new:src/index.ts:"); }); + test("reuses existing repository files and process run when file insert conflicts", async () => { + compareChanges = [{ status: "modified", newPath: "src/index.ts" }]; + readFileContents = { + "src/index.ts": "export const next = shared;\n", + }; + selectResults = [ + { kind: "limit", value: [bindingRow("commit-old")] }, + { + kind: "where", + value: [activeFile("old-file", "commit-old", "src/index.ts", 25)], + }, + ]; + insertedFileRowsOverride = []; + txSelectResults = [ + [{ id: "existing-file", key: "connector:binding-1:commit-new:src/index.ts" }], + [{ id: "existing-process-run" }], + ]; + + const result = await runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" }); + + expect(result).toMatchObject({ commitSha: "commit-new", fileCount: 1 }); + expect(processRunInsertCount).toBe(0); + expect(processWorkflowInputs).toEqual([ + { + graphId: "graph-1", + fileIds: ["existing-file"], + processRunId: "existing-process-run", + code: { kind: "repository", retiredFileIds: ["old-file"] }, + }, + ]); + }); + test("finalizes delete-only connector deltas without inserting new files", async () => { compareChanges = [{ status: "deleted", oldPath: "src/removed.ts" }]; selectResults = [ @@ -319,6 +390,42 @@ describe("syncRepositoryGraph", () => { }); }); + test("falls back to a full snapshot when compare is not incremental", async () => { + compareIsIncremental = false; + snapshotFiles = [ + { + path: "src/reset.ts", + size: 24, + checksum: "reset-sha", + htmlUrl: "https://github.com/acme/widgets/blob/commit-new/src/reset.ts", + rawUrl: "https://raw.githubusercontent.com/acme/widgets/commit-new/src/reset.ts", + content: "export const reset = true;\n", + }, + ]; + selectResults = [ + { kind: "limit", value: [bindingRow("commit-old")] }, + { + kind: "where", + value: [ + activeFile("old-file", "commit-old", "src/index.ts", 25), + activeFile("shared-file", "commit-old", "src/shared.ts", 22), + ], + }, + ]; + + const result = await runWorkflow({ bindingId: "binding-1", reason: "webhook", commitSha: "commit-new" }); + + expect(result).toMatchObject({ commitSha: "commit-new", fileCount: 1 }); + expect(loadSnapshotCalls).toBe(1); + expect(readFileCalls).toEqual([]); + expect(insertedFileValues.map((row) => row.name)).toEqual(["src/reset.ts"]); + expect(processWorkflowInputs[0]).toMatchObject({ + graphId: "graph-1", + processRunId: "process-run-1", + code: { kind: "repository", retiredFileIds: ["old-file", "shared-file"] }, + }); + }); + test("rolls back inserted files when incremental processing fails", async () => { compareChanges = [{ status: "modified", newPath: "src/index.ts" }]; readFileContents = { @@ -336,9 +443,9 @@ describe("syncRepositoryGraph", () => { }, ]; - await expect(runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" })).rejects.toThrow( - "child failure" - ); + await expect( + runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" }) + ).rejects.toThrow("child failure"); expect(deleteWorkflowInputs).toEqual([{ graphId: "graph-1", fileId: insertedFileValues[0]?.id }]); }); @@ -352,9 +459,9 @@ describe("syncRepositoryGraph", () => { { kind: "where", value: [activeFile("old-file", "commit-old", "src/index.ts", 25)] }, ]; - await expect(runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" })).rejects.toThrow( - "Provider delta contained duplicate path src/index.ts" - ); + await expect( + runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" }) + ).rejects.toThrow("Provider delta contained duplicate path src/index.ts"); expect(processWorkflowInputs).toEqual([]); }); }); diff --git a/apps/worker/workflows/sync-repository-graph.ts b/apps/worker/workflows/sync-repository-graph.ts index 24292dda..370199c8 100644 --- a/apps/worker/workflows/sync-repository-graph.ts +++ b/apps/worker/workflows/sync-repository-graph.ts @@ -29,7 +29,7 @@ import { } from "@kiwi/db/tables/connectors"; import { filesTable, graphTable, processRunFilesTable, processRunsTable } from "@kiwi/db/tables/graph"; import { serializeCodeFileMetadata } from "@kiwi/graph/code/metadata"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { defineWorkflow } from "openworkflow"; import { parseCodeFileMetadata } from "../lib/code-file-metadata"; import { env } from "../env"; @@ -155,7 +155,9 @@ async function resolveTargetCommitSha( return inputCommitSha; } - const branch = (await client.listBranches(repositoryFromBinding(row))).find((candidate) => candidate.name === row.binding.branch); + const branch = (await client.listBranches(repositoryFromBinding(row))).find( + (candidate) => candidate.name === row.binding.branch + ); if (!branch) { throw new ConnectorProviderError("not-found", "Repository branch was not found"); } @@ -163,8 +165,16 @@ async function resolveTargetCommitSha( return branch.commitSha; } -async function loadSnapshot(row: BindingGraphRow, client: ProviderRepositoryClient, commitSha: string): Promise { - return client.loadRepositorySnapshot(repositoryFromBinding(row), row.binding.branch, commitSha) as Promise; +async function loadSnapshot( + row: BindingGraphRow, + client: ProviderRepositoryClient, + commitSha: string +): Promise { + return client.loadRepositorySnapshot( + repositoryFromBinding(row), + row.binding.branch, + commitSha + ) as Promise; } async function loadActiveBindingFiles(bindingId: string): Promise> { @@ -175,7 +185,13 @@ async function loadActiveBindingFiles(bindingId: string): Promise(); for (const row of rows) { @@ -279,7 +295,8 @@ function assertBindingSnapshotLimits( newFiles: ProviderCodeFile[] ) { const retiredIds = new Set(retiredFileIds); - const totalFileCount = [...activeFiles.values()].filter((file) => !retiredIds.has(file.id)).length + newFiles.length; + const totalFileCount = + [...activeFiles.values()].filter((file) => !retiredIds.has(file.id)).length + newFiles.length; if (totalFileCount > MAX_REPOSITORY_CODE_FILES) { throw new ConnectorProviderError("limit", "Repository contains too many supported code files"); } @@ -319,6 +336,23 @@ function fileRows(row: BindingGraphRow, files: ProviderCodeFile[], commitSha: st })); } +type InsertedFileRow = { + id: string; + key: string; +}; + +function orderedFilesByKey(rows: ReturnType, files: InsertedFileRow[]): InsertedFileRow[] { + const filesByKey = new Map(files.map((file) => [file.key, file])); + const orderedFiles: InsertedFileRow[] = []; + for (const row of rows) { + const file = filesByKey.get(row.key); + if (file) { + orderedFiles.push(file); + } + } + return orderedFiles; +} + async function insertRepositoryFiles( row: BindingGraphRow, files: ProviderCodeFile[], @@ -330,32 +364,73 @@ async function insertRepositoryFiles( .set({ syncStatus: "syncing", lastSeenCommitSha: commitSha, syncErrorCode: null }) .where(eq(repositoryGraphBindingsTable.id, row.binding.id)); + const rows = fileRows(row, files, commitSha); const insertedFiles = await tx .insert(filesTable) - .values(fileRows(row, files, commitSha)) + .values(rows) .onConflictDoNothing() - .returning({ id: filesTable.id }); - if (insertedFiles.length !== files.length) { + .returning({ id: filesTable.id, key: filesTable.key }); + let committedFiles = orderedFilesByKey(rows, insertedFiles); + if (committedFiles.length !== files.length) { + const existingFiles = await tx + .select({ id: filesTable.id, key: filesTable.key }) + .from(filesTable) + .where( + and( + eq(filesTable.graphId, row.binding.graphId), + eq(filesTable.deleted, false), + inArray( + filesTable.key, + rows.map((file) => file.key) + ) + ) + ); + committedFiles = orderedFilesByKey(rows, existingFiles); + } + if (committedFiles.length !== files.length) { throw new Error("Failed to insert all repository files"); } - const [processRun] = await tx - .insert(processRunsTable) - .values({ graphId: row.binding.graphId, status: "pending" }) - .returning({ id: processRunsTable.id }); - if (!processRun) { - throw new Error("Failed to create process run"); + let processRunId: string | null = null; + if (insertedFiles.length === 0) { + const existingProcessRuns = await tx + .select({ id: processRunFilesTable.processRunId }) + .from(processRunFilesTable) + .innerJoin(processRunsTable, eq(processRunFilesTable.processRunId, processRunsTable.id)) + .where( + and( + eq(processRunsTable.graphId, row.binding.graphId), + inArray( + processRunFilesTable.fileId, + committedFiles.map((file) => file.id) + ) + ) + ); + processRunId = existingProcessRuns[0]?.id ?? null; + } + + if (!processRunId) { + const [processRun] = await tx + .insert(processRunsTable) + .values({ graphId: row.binding.graphId, status: "pending" }) + .returning({ id: processRunsTable.id }); + if (!processRun) { + throw new Error("Failed to create process run"); + } + const newProcessRunId = processRun.id; + processRunId = newProcessRunId; + + await tx.insert(processRunFilesTable).values( + committedFiles.map((file) => ({ + processRunId: newProcessRunId, + fileId: file.id, + })) + ); } - await tx.insert(processRunFilesTable).values( - insertedFiles.map((file) => ({ - processRunId: processRun.id, - fileId: file.id, - })) - ); return { - fileIds: insertedFiles.map((file) => file.id), - processRunId: processRun.id, + fileIds: committedFiles.map((file) => file.id), + processRunId, }; }); } @@ -364,19 +439,34 @@ async function markWebhookDuplicate(provider: typeof connectorsTable.$inferSelec await db .update(connectorWebhookEventsTable) .set({ status: "duplicate" }) - .where(and(eq(connectorWebhookEventsTable.provider, provider), eq(connectorWebhookEventsTable.deliveryId, deliveryId))); + .where( + and( + eq(connectorWebhookEventsTable.provider, provider), + eq(connectorWebhookEventsTable.deliveryId, deliveryId) + ) + ); } async function markBindingSynced(bindingId: string, commitSha: string) { await db .update(repositoryGraphBindingsTable) - .set({ syncStatus: "synced", lastSeenCommitSha: commitSha, lastSyncedCommitSha: commitSha, syncErrorCode: null }) + .set({ + syncStatus: "synced", + lastSeenCommitSha: commitSha, + lastSyncedCommitSha: commitSha, + syncErrorCode: null, + }) .where(eq(repositoryGraphBindingsTable.id, bindingId)); } export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async ({ input, step }) => { const row = await step.run({ name: "load-binding" }, async () => loadBindingGraph(input.bindingId)); - if (!row || row.connector.status !== "active" || row.installation.status !== "active" || !row.binding.webhookEnabled) { + if ( + !row || + row.connector.status !== "active" || + row.installation.status !== "active" || + !row.binding.webhookEnabled + ) { return { skipped: true }; } @@ -399,7 +489,9 @@ export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async loadSnapshot(row, context.client, commitSha) ); if (snapshot.files.length === 0) { - await step.run({ name: "mark-empty-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); + await step.run({ name: "mark-empty-binding-synced" }, async () => + markBindingSynced(row.binding.id, commitSha) + ); return { commitSha, fileCount: 0 }; } @@ -435,6 +527,50 @@ export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async const delta = await step.run({ name: "compare-provider-commits" }, async () => context.client.compareRepository(repositoryFromBinding(row), row.binding.lastSyncedCommitSha!, commitSha) ); + if (!delta.isIncremental) { + const snapshot = await step.run({ name: "load-provider-snapshot" }, async () => + loadSnapshot(row, context.client, commitSha) + ); + const retiredFileIds = [...activeFiles.values()].map((file) => file.id); + assertBindingSnapshotLimits(activeFiles, retiredFileIds, snapshot.files); + + if (snapshot.files.length > 0) { + const created = await step.run({ name: "commit-external-files" }, async () => + insertRepositoryFiles(row, snapshot.files, commitSha) + ); + try { + await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: created.fileIds, + processRunId: created.processRunId, + code: { kind: "repository", retiredFileIds }, + }); + } catch (error) { + await Promise.all( + created.fileIds.map((fileId) => + step.runWorkflow(deleteFileSpec, { + graphId: row.binding.graphId, + fileId, + }) + ) + ); + throw error; + } + + await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); + return { commitSha, fileCount: created.fileIds.length }; + } + + if (retiredFileIds.length > 0) { + await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: [], + code: { kind: "repository", retiredFileIds }, + }); + } + await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); + return { commitSha, fileCount: 0 }; + } const plan = planIncrementalChanges(activeFiles, delta.changes); if (plan.newPaths.length === 0 && plan.retiredFileIds.length === 0) { await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); diff --git a/migrations/20260614105716_tricky_plazm/snapshot.json b/migrations/20260614105716_tricky_plazm/snapshot.json index cbcc90c7..ddf59921 100644 --- a/migrations/20260614105716_tricky_plazm/snapshot.json +++ b/migrations/20260614105716_tricky_plazm/snapshot.json @@ -7681,65 +7681,115 @@ "table": "files" }, { + "value": "\"provider\" in ('github', 'gitlab')", "name": "connectors_provider_check", "entityType": "checks", "schema": "public", "table": "connectors" }, { + "value": "\"status\" in ('draft', 'active', 'disabled')", "name": "connectors_status_check", "entityType": "checks", "schema": "public", "table": "connectors" }, { + "value": "\"provider\" in ('github', 'gitlab')", "name": "connector_installations_provider_check", "entityType": "checks", "schema": "public", "table": "connector_installations" }, { + "value": "\"status\" in ('active', 'disabled')", "name": "connector_installations_status_check", "entityType": "checks", "schema": "public", "table": "connector_installations" }, { + "value": "(\"organization_id\" is not null and \"team_id\" is null) or (\"organization_id\" is not null and \"team_id\" is not null)", "name": "connector_installations_owner_scope_check", "entityType": "checks", "schema": "public", "table": "connector_installations" }, { + "value": "\"provider\" in ('github', 'gitlab')", "name": "repository_graph_bindings_provider_check", "entityType": "checks", "schema": "public", "table": "repository_graph_bindings" }, { + "value": "\"sync_status\" in ('pending', 'syncing', 'synced', 'failed')", "name": "repository_graph_bindings_sync_status_check", "entityType": "checks", "schema": "public", "table": "repository_graph_bindings" }, { + "value": "\"provider\" in ('github', 'gitlab')", "name": "connector_webhook_events_provider_check", "entityType": "checks", "schema": "public", "table": "connector_webhook_events" }, { + "value": "\"status\" in ('ignored', 'enqueued', 'duplicate', 'failed')", "name": "connector_webhook_events_status_check", "entityType": "checks", "schema": "public", "table": "connector_webhook_events" }, { + "value": "\"external_provider\" IS NULL OR \"external_provider\" in ('github', 'gitlab')", "name": "files_external_provider_check", "entityType": "checks", "schema": "public", "table": "files" }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "connector_installations_pkey", + "schema": "public", + "table": "connector_installations", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "connector_webhook_events_pkey", + "schema": "public", + "table": "connector_webhook_events", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "connectors_pkey", + "schema": "public", + "table": "connectors", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "repository_graph_bindings_pkey", + "schema": "public", + "table": "repository_graph_bindings", + "entityType": "pks" + }, { "nameExplicit": true, "columns": [ @@ -7792,42 +7842,14 @@ { "nameExplicit": true, "columns": [ - { - "value": "connector_id", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - }, - { - "value": "provider_installation_id", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - }, - { - "value": "organization_id", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - }, - { - "value": "team_id", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } + "connector_id", + "provider_installation_id", + "organization_id", + "team_id" ], - "isUnique": true, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, + "nullsNotDistinct": false, "name": "connector_installations_provider_scope_unique", - "entityType": "indexes", + "entityType": "uniques", "schema": "public", "table": "connector_installations" }, @@ -8260,23 +8282,6 @@ "entityType": "fks", "schema": "public", "table": "connector_webhook_events" - }, - { - "nameExplicit": true, - "columns": [ - "repository_binding_id" - ], - "schemaTo": "public", - "tableTo": "repository_graph_bindings", - "columnsTo": [ - "id" - ], - "onUpdate": "NO ACTION", - "onDelete": "SET NULL", - "name": "files_repository_binding_id_repository_graph_bindings_id_fk", - "entityType": "fks", - "schema": "public", - "table": "files" } ], "renames": [] diff --git a/migrations/20260615094125_fluffy_gladiator/migration.sql b/migrations/20260615094125_fluffy_gladiator/migration.sql new file mode 100644 index 00000000..543d512f --- /dev/null +++ b/migrations/20260615094125_fluffy_gladiator/migration.sql @@ -0,0 +1,2 @@ +DROP INDEX "files_graph_active_key_idx";--> statement-breakpoint +CREATE UNIQUE INDEX "files_graph_active_key_idx" ON "files" ("graph_id","file_key") WHERE "deleted" = false; \ No newline at end of file diff --git a/migrations/20260615094125_fluffy_gladiator/snapshot.json b/migrations/20260615094125_fluffy_gladiator/snapshot.json new file mode 100644 index 00000000..f516722d --- /dev/null +++ b/migrations/20260615094125_fluffy_gladiator/snapshot.json @@ -0,0 +1,8288 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "ab5d8cc3-076f-4287-ad54-18a6f3f26fc8", + "prevIds": [ + "9b6b85c6-0014-4a15-bb55-fd27e7e3b3d2" + ], + "ddl": [ + { + "isRlsEnabled": true, + "name": "account", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "apikey", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "invitation", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "member", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "organization_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "organization", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "session", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "team_member_roles", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "teamMember", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "team_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "team", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "user_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "user", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "verification", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "chats", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "messages", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "connector_installations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "connector_webhook_events", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "connectors", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "repository_graph_bindings", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "file_type_configs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "entities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "files", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graph_prompts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graphs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graph_updates", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "process_run_files", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "process_runs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "process_stats", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "relationships", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "sources", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "text_units", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "models", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": true, + "name": "graph_suggestions", + "entityType": "tables", + "schema": "public" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "accountId", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "providerId", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "accessToken", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refreshToken", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "accessTokenExpiresAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refreshTokenExpiresAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "scope", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "idToken", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "password", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "account" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "config_id", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "start", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prefix", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "key", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reference_id", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refill_interval", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "refill_amount", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_refill_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "enabled", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rate_limit_enabled", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rate_limit_time_window", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "rate_limit_max", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "request_count", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remaining", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_request", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "timestamp(6) with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "permissions", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata", + "entityType": "columns", + "schema": "public", + "table": "apikey" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organizationId", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "email", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "teamId", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expiresAt", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "inviterId", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "invitation" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organizationId", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'member'", + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "systemRoleProvisioned", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "member" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "organization_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "logo", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "organization" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expiresAt", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ipAddress", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userAgent", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "impersonatedBy", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activeOrganizationId", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activeTeamId", + "entityType": "columns", + "schema": "public", + "table": "session" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_member_id", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'member'", + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "team_member_roles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "teamId", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "userId", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "teamMember" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "team_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organizationId", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "team" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "user_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "email", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "emailVerified", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "image", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banned", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banReason", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banExpires", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "user" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "identifier", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "value", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expiresAt", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "createdAt", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updatedAt", + "entityType": "columns", + "schema": "public", + "table": "verification" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'graph'", + "generated": null, + "identity": null, + "name": "scope", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "project_id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "pinned_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "archived_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "chats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chat_id", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "parts", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tokens_per_second", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "time_to_first_token", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "input_tokens", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "output_tokens", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "total_tokens", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "messages" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connector_id", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_installation_id", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_account_login", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_account_type", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "installed_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "encrypted_credentials", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'unknown'", + "generated": null, + "identity": null, + "name": "repository_selection", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'active'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "connector_installations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connector_id", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "delivery_id", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "event_name", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_repository_id", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "branch", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "commit_sha", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error_code", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'active'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_id", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_slug", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "client_id", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "encrypted_credentials", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "webhook_secret_encrypted", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "connectors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "connector_installation_id", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_repository_id", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repository_full_name", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repository_html_url", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "branch", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_seen_commit_sha", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_synced_commit_sha", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "sync_status", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "sync_error_code", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "webhook_enabled", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_type", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "loader", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunker", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunk_size", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "document_mode", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "file_type_configs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "active", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "vector(4096)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('simple', coalesce(name, '')), 'A') || setweight(to_tsvector('simple', coalesce(description, '')), 'B')", + "type": "stored" + }, + "identity": null, + "name": "search_tsv", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "entities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_size", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_type", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "mime_type", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_key", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'internal'", + "generated": null, + "identity": null, + "name": "storage_kind", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_url", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "external_provider", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "repository_binding_id", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "checksum", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "deleted", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'processing'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "process_step", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "process_error_code", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "token_count", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "metadata", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "loader", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunker", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chunk_size", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "document_mode", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "prompt", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graph_prompts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "team_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'ready'", + "generated": null, + "identity": null, + "name": "state", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "hidden", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graphs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "update_type", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "json", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "update_message", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graph_updates" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "process_run_id", + "entityType": "columns", + "schema": "public", + "table": "process_run_files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_id", + "entityType": "columns", + "schema": "public", + "table": "process_run_files" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "process_run_files" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "completed_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "process_runs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "total_time", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "files", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "file_sizes", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'unknown'", + "generated": null, + "identity": null, + "name": "file_type", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "token_count", + "entityType": "columns", + "schema": "public", + "table": "process_stats" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "active", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "target_id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'RELATED'", + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "directed", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "rank", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "vector(4096)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('simple', coalesce(description, '')), 'A')", + "type": "stored" + }, + "identity": null, + "name": "search_tsv", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relationships" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "entity_id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "relationship_id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "text_unit_id", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "active", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "valid_until", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "json", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "source_chunk_ids", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "vector(4096)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embedding", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('simple', coalesce(description, '')), 'A')", + "type": "stored" + }, + "identity": null, + "name": "search_tsv", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "sources" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "file_id", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "text", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "start_page", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "end_page", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "json", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'[]'", + "generated": null, + "identity": null, + "name": "chunks", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "text_units" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "organization_id", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "model_id", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "display_name", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "adapter", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "provider_model", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "250000", + "generated": null, + "identity": null, + "name": "context_window", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "encrypted_credentials", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "is_default", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "models" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "graph_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'pending'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "entity_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "reference", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "suggestion", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "suggested_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "chat_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "message_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_by_user_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_source_id", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "applied_at", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "providerId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "account_user_provider_idx", + "entityType": "indexes", + "schema": "public", + "table": "account" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "email", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_email_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "role", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_role_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "teamId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "invitation_team_idx", + "entityType": "indexes", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "member_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "member_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "member_organization_user_unique", + "entityType": "indexes", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "organization_prompts_organization_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "organization_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "slug", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "organization_slug_idx", + "entityType": "indexes", + "schema": "public", + "table": "organization" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "teamId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_member_team_idx", + "entityType": "indexes", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_member_user_idx", + "entityType": "indexes", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "teamId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "userId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_member_team_user_unique", + "entityType": "indexes", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_prompts_team_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "team_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organizationId", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "team_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "team_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "user_prompts_user_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "user_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "project_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_project_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_team_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "project_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "(\"pinned_at\" is null)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"archived_at\" IS NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_project_archived_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "(\"pinned_at\" is null)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "updated_at", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": false, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"archived_at\" IS NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_user_chats_user_team_archived_updated_at", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "title", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "chats_title_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "chat_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_chat_messages_chat_id_id", + "entityType": "indexes", + "schema": "public", + "table": "messages" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "chat_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "role", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_chat_messages_chat_role_status_id", + "entityType": "indexes", + "schema": "public", + "table": "messages" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_installations_connector_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_installations_organization_idx", + "entityType": "indexes", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_installations_team_idx", + "entityType": "indexes", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "delivery_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_webhook_events_delivery_unique", + "entityType": "indexes", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider_repository_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "branch", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_webhook_events_binding_lookup_idx", + "entityType": "indexes", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "slug", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connectors_slug_unique", + "entityType": "indexes", + "schema": "public", + "table": "connectors" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "connectors_provider_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "connectors" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "repository_graph_bindings_graph_unique", + "entityType": "indexes", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector_installation_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider_repository_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "branch", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "repository_graph_bindings_repository_branch_unique", + "entityType": "indexes", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "provider", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider_repository_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "branch", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "repository_graph_bindings_provider_repo_branch_idx", + "entityType": "indexes", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector_installation_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "sync_status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "repository_graph_bindings_installation_status_idx", + "entityType": "indexes", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "file_type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "file_type_configs_organization_file_type_unique", + "entityType": "indexes", + "schema": "public", + "table": "file_type_configs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "entities_graph_active_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "entities_graph_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "entities_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "embedding", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "vector_cosine_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "diskann", + "concurrently": false, + "name": "entities_embedding_diskann_idx", + "entityType": "indexes", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "checksum", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"deleted\" = false AND \"checksum\" IS NOT NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_checksum_active_unique", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "files_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_active_created_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "file_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_graph_active_key_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "repository_binding_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"deleted\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "files_repository_binding_active_idx", + "entityType": "indexes", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_prompts_graph_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_prompts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_organization_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_team_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "user_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_user_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_graph_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "graphs_name_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"graph_id\" IS NULL AND \"team_id\" IS NULL AND \"hidden\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_visible_root_organization_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"graph_id\" IS NULL AND \"hidden\" = false", + "with": "", + "method": "btree", + "concurrently": false, + "name": "graphs_visible_root_team_name_idx", + "entityType": "indexes", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "file_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "process_run_files_file_idx", + "entityType": "indexes", + "schema": "public", + "table": "process_run_files" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "process_runs_graph_status_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "process_runs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "source_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_source_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "target_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "relationships_graph_active_target_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "description", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "relationships_description_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "embedding", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "vector_cosine_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "diskann", + "concurrently": false, + "name": "relationships_embedding_diskann_idx", + "entityType": "indexes", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "entity_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_entity_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "relationship_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "active", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_relationship_active_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"active\" = true AND \"valid_until\" IS NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_current_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "entity_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"active\" = true AND \"valid_until\" IS NULL AND \"entity_id\" IS NOT NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_entity_current_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "relationship_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": "\"active\" = true AND \"valid_until\" IS NULL AND \"relationship_id\" IS NOT NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_relationship_current_id_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "text_unit_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "sources_text_unit_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "description", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "gin_trgm_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "sources_description_trgm_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "embedding", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": { + "name": "vector_cosine_ops", + "default": false + } + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "diskann", + "concurrently": false, + "name": "sources_embedding_diskann_idx", + "entityType": "indexes", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "file_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "text_units_file_idx", + "entityType": "indexes", + "schema": "public", + "table": "text_units" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "model_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "models_organization_model_id_unique", + "entityType": "indexes", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"is_default\" = true", + "with": "", + "method": "btree", + "concurrently": false, + "name": "models_organization_type_default_unique", + "entityType": "indexes", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "models_organization_type_idx", + "entityType": "indexes", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "graph_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_suggestions_graph_status_created_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_suggestions_source_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "entity_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "graph_suggestions_entity_idx", + "entityType": "indexes", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "account_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "account" + }, + { + "nameExplicit": false, + "columns": [ + "organizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "invitation_organizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": false, + "columns": [ + "teamId" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "invitation_teamId_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": false, + "columns": [ + "inviterId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "invitation_inviterId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "invitation" + }, + { + "nameExplicit": false, + "columns": [ + "organizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "member_organizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "member_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "member" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "organization_prompts_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "organization_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "session_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "impersonatedBy" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "session_impersonatedBy_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "activeOrganizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "session_activeOrganizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "activeTeamId" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "session_activeTeamId_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "session" + }, + { + "nameExplicit": false, + "columns": [ + "team_member_id" + ], + "schemaTo": "public", + "tableTo": "teamMember", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "team_member_roles_team_member_id_teamMember_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "team_member_roles" + }, + { + "nameExplicit": false, + "columns": [ + "teamId" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "teamMember_teamId_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": false, + "columns": [ + "userId" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "teamMember_userId_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "teamMember" + }, + { + "nameExplicit": false, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "team_prompts_team_id_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "team_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "organizationId" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "team_organizationId_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": false, + "columns": [ + "user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "user_prompts_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "user_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "chats_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": false, + "columns": [ + "project_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "chats_project_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": false, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "chats_team_id_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "chats" + }, + { + "nameExplicit": false, + "columns": [ + "chat_id" + ], + "schemaTo": "public", + "tableTo": "chats", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "messages_chat_id_chats_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "messages" + }, + { + "nameExplicit": true, + "columns": [ + "connector_id" + ], + "schemaTo": "public", + "tableTo": "connectors", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "connector_installations_connector_id_connectors_id_fk", + "entityType": "fks", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "connector_installations_organization_id_organization_id_fk", + "entityType": "fks", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "connector_installations_team_id_team_id_fk", + "entityType": "fks", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + "installed_by_user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "connector_installations_installed_by_user_id_user_id_fk", + "entityType": "fks", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + "connector_id" + ], + "schemaTo": "public", + "tableTo": "connectors", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "connector_webhook_events_connector_id_connectors_id_fk", + "entityType": "fks", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "nameExplicit": true, + "columns": [ + "created_by_user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "connectors_created_by_user_id_user_id_fk", + "entityType": "fks", + "schema": "public", + "table": "connectors" + }, + { + "nameExplicit": true, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "repository_graph_bindings_graph_id_graphs_id_fk", + "entityType": "fks", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "nameExplicit": true, + "columns": [ + "connector_installation_id" + ], + "schemaTo": "public", + "tableTo": "connector_installations", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "RESTRICT", + "name": "repository_graph_bindings_connector_installation_id_fk", + "entityType": "fks", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "file_type_configs_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "file_type_configs" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "entities_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "entities" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "files_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "files" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_prompts_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_prompts" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "team_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_team_id_team_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": true, + "columns": [ + "team_id", + "organization_id" + ], + "schemaTo": "public", + "tableTo": "team", + "columnsTo": [ + "id", + "organizationId" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graphs_team_organization_fkey", + "entityType": "fks", + "schema": "public", + "table": "graphs" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_updates_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_updates" + }, + { + "nameExplicit": false, + "columns": [ + "process_run_id" + ], + "schemaTo": "public", + "tableTo": "process_runs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "process_run_files_process_run_id_process_runs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "process_run_files" + }, + { + "nameExplicit": false, + "columns": [ + "file_id" + ], + "schemaTo": "public", + "tableTo": "files", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "process_run_files_file_id_files_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "process_run_files" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "process_runs_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "process_runs" + }, + { + "nameExplicit": false, + "columns": [ + "source_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "relationships_source_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": false, + "columns": [ + "target_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "relationships_target_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "relationships_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "relationships" + }, + { + "nameExplicit": false, + "columns": [ + "entity_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "sources_entity_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": false, + "columns": [ + "relationship_id" + ], + "schemaTo": "public", + "tableTo": "relationships", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "sources_relationship_id_relationships_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": false, + "columns": [ + "text_unit_id" + ], + "schemaTo": "public", + "tableTo": "text_units", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "sources_text_unit_id_text_units_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "sources" + }, + { + "nameExplicit": false, + "columns": [ + "file_id" + ], + "schemaTo": "public", + "tableTo": "files", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "text_units_file_id_files_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "text_units" + }, + { + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "models_organization_id_organization_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "models" + }, + { + "nameExplicit": false, + "columns": [ + "graph_id" + ], + "schemaTo": "public", + "tableTo": "graphs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_graph_id_graphs_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "source_id" + ], + "schemaTo": "public", + "tableTo": "sources", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_source_id_sources_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "entity_id" + ], + "schemaTo": "public", + "tableTo": "entities", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_entity_id_entities_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "suggested_by_user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "graph_suggestions_suggested_by_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "chat_id" + ], + "schemaTo": "public", + "tableTo": "chats", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_chat_id_chats_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "message_id" + ], + "schemaTo": "public", + "tableTo": "messages", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_message_id_messages_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "applied_by_user_id" + ], + "schemaTo": "public", + "tableTo": "user", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_applied_by_user_id_user_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "nameExplicit": false, + "columns": [ + "applied_source_id" + ], + "schemaTo": "public", + "tableTo": "sources", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "graph_suggestions_applied_source_id_sources_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "graph_suggestions" + }, + { + "columns": [ + "process_run_id", + "file_id" + ], + "nameExplicit": true, + "name": "process_run_files_pk", + "entityType": "pks", + "schema": "public", + "table": "process_run_files" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "account_pkey", + "schema": "public", + "table": "account", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "apikey_pkey", + "schema": "public", + "table": "apikey", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "invitation_pkey", + "schema": "public", + "table": "invitation", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "member_pkey", + "schema": "public", + "table": "member", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "organization_prompts_pkey", + "schema": "public", + "table": "organization_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "organization_pkey", + "schema": "public", + "table": "organization", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "session_pkey", + "schema": "public", + "table": "session", + "entityType": "pks" + }, + { + "columns": [ + "team_member_id" + ], + "nameExplicit": false, + "name": "team_member_roles_pkey", + "schema": "public", + "table": "team_member_roles", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "teamMember_pkey", + "schema": "public", + "table": "teamMember", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "team_prompts_pkey", + "schema": "public", + "table": "team_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "team_pkey", + "schema": "public", + "table": "team", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "user_prompts_pkey", + "schema": "public", + "table": "user_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "user_pkey", + "schema": "public", + "table": "user", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "verification_pkey", + "schema": "public", + "table": "verification", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "chats_pkey", + "schema": "public", + "table": "chats", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "messages_pkey", + "schema": "public", + "table": "messages", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "connector_installations_pkey", + "schema": "public", + "table": "connector_installations", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "connector_webhook_events_pkey", + "schema": "public", + "table": "connector_webhook_events", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "connectors_pkey", + "schema": "public", + "table": "connectors", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "repository_graph_bindings_pkey", + "schema": "public", + "table": "repository_graph_bindings", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "file_type_configs_pkey", + "schema": "public", + "table": "file_type_configs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "entities_pkey", + "schema": "public", + "table": "entities", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "files_pkey", + "schema": "public", + "table": "files", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graph_prompts_pkey", + "schema": "public", + "table": "graph_prompts", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graphs_pkey", + "schema": "public", + "table": "graphs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graph_updates_pkey", + "schema": "public", + "table": "graph_updates", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "process_runs_pkey", + "schema": "public", + "table": "process_runs", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "process_stats_pkey", + "schema": "public", + "table": "process_stats", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "relationships_pkey", + "schema": "public", + "table": "relationships", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "sources_pkey", + "schema": "public", + "table": "sources", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "text_units_pkey", + "schema": "public", + "table": "text_units", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "models_pkey", + "schema": "public", + "table": "models", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "graph_suggestions_pkey", + "schema": "public", + "table": "graph_suggestions", + "entityType": "pks" + }, + { + "nameExplicit": true, + "columns": [ + "id", + "organizationId" + ], + "nullsNotDistinct": false, + "name": "team_id_organization_unique", + "entityType": "uniques", + "schema": "public", + "table": "team" + }, + { + "nameExplicit": true, + "columns": [ + "connector_id", + "provider_installation_id", + "organization_id", + "team_id" + ], + "nullsNotDistinct": false, + "name": "connector_installations_provider_scope_unique", + "entityType": "uniques", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": false, + "columns": [ + "id" + ], + "nullsNotDistinct": false, + "name": "apikey_id_key", + "schema": "public", + "table": "apikey", + "entityType": "uniques" + }, + { + "nameExplicit": false, + "columns": [ + "slug" + ], + "nullsNotDistinct": false, + "name": "organization_slug_key", + "schema": "public", + "table": "organization", + "entityType": "uniques" + }, + { + "nameExplicit": false, + "columns": [ + "token" + ], + "nullsNotDistinct": false, + "name": "session_token_key", + "schema": "public", + "table": "session", + "entityType": "uniques" + }, + { + "nameExplicit": false, + "columns": [ + "email" + ], + "nullsNotDistinct": false, + "name": "user_email_key", + "schema": "public", + "table": "user", + "entityType": "uniques" + }, + { + "value": "\n (\n \"scope\" = 'graph'\n AND \"project_id\" IS NOT NULL\n AND \"team_id\" IS NULL\n )\n OR\n (\n \"scope\" = 'team'\n AND \"project_id\" IS NULL\n AND \"team_id\" IS NOT NULL\n )\n ", + "name": "chats_scope_target_check", + "entityType": "checks", + "schema": "public", + "table": "chats" + }, + { + "value": "jsonb_typeof(\"parts\") = 'array'", + "name": "chat_messages_parts_array_check", + "entityType": "checks", + "schema": "public", + "table": "messages" + }, + { + "value": "\"provider\" in ('github', 'gitlab')", + "name": "connector_installations_provider_check", + "entityType": "checks", + "schema": "public", + "table": "connector_installations" + }, + { + "value": "\"status\" in ('active', 'disabled')", + "name": "connector_installations_status_check", + "entityType": "checks", + "schema": "public", + "table": "connector_installations" + }, + { + "value": "(\"organization_id\" is not null and \"team_id\" is null) or (\"organization_id\" is not null and \"team_id\" is not null)", + "name": "connector_installations_owner_scope_check", + "entityType": "checks", + "schema": "public", + "table": "connector_installations" + }, + { + "value": "\"provider\" in ('github', 'gitlab')", + "name": "connector_webhook_events_provider_check", + "entityType": "checks", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "value": "\"status\" in ('ignored', 'enqueued', 'duplicate', 'failed')", + "name": "connector_webhook_events_status_check", + "entityType": "checks", + "schema": "public", + "table": "connector_webhook_events" + }, + { + "value": "\"provider\" in ('github', 'gitlab')", + "name": "connectors_provider_check", + "entityType": "checks", + "schema": "public", + "table": "connectors" + }, + { + "value": "\"status\" in ('draft', 'active', 'disabled')", + "name": "connectors_status_check", + "entityType": "checks", + "schema": "public", + "table": "connectors" + }, + { + "value": "\"provider\" in ('github', 'gitlab')", + "name": "repository_graph_bindings_provider_check", + "entityType": "checks", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "value": "\"sync_status\" in ('pending', 'syncing', 'synced', 'failed')", + "name": "repository_graph_bindings_sync_status_check", + "entityType": "checks", + "schema": "public", + "table": "repository_graph_bindings" + }, + { + "value": "\n (\n \"storage_kind\" = 'internal'\n AND \"external_url\" IS NULL\n AND \"external_provider\" IS NULL\n )\n OR (\n \"storage_kind\" = 'external'\n AND \"external_url\" IS NOT NULL\n AND \"external_provider\" IS NOT NULL\n )\n ", + "name": "files_storage_origin_check", + "entityType": "checks", + "schema": "public", + "table": "files" + }, + { + "value": "\"external_provider\" IS NULL OR \"external_provider\" in ('github', 'gitlab')", + "name": "files_external_provider_check", + "entityType": "checks", + "schema": "public", + "table": "files" + }, + { + "value": "(((\"organization_id\" IS NOT NULL)::int + (\"user_id\" IS NOT NULL)::int + (\"graph_id\" IS NOT NULL)::int) = 1)", + "name": "graphs_single_owner_check", + "entityType": "checks", + "schema": "public", + "table": "graphs" + }, + { + "value": "\"team_id\" IS NULL OR \"organization_id\" IS NOT NULL", + "name": "graphs_team_requires_organization_check", + "entityType": "checks", + "schema": "public", + "table": "graphs" + }, + { + "value": "((\"start_page\" IS NULL AND \"end_page\" IS NULL) OR (\"start_page\" IS NOT NULL AND \"end_page\" IS NOT NULL AND \"start_page\" >= 1 AND \"end_page\" >= \"start_page\"))", + "name": "text_units_page_span_check", + "entityType": "checks", + "schema": "public", + "table": "text_units" + }, + { + "value": "\n (\n \"kind\" = 'source_correction'\n AND \"source_id\" IS NOT NULL\n AND \"entity_id\" IS NULL\n )\n OR\n (\n \"kind\" = 'entity_addition'\n AND \"source_id\" IS NULL\n AND \"entity_id\" IS NOT NULL\n )\n ", + "name": "graph_suggestions_target_check", + "entityType": "checks", + "schema": "public", + "table": "graph_suggestions" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/packages/connectors/src/__tests__/github.test.ts b/packages/connectors/src/__tests__/github.test.ts index 895deb56..f62a6c5a 100644 --- a/packages/connectors/src/__tests__/github.test.ts +++ b/packages/connectors/src/__tests__/github.test.ts @@ -67,10 +67,16 @@ describe("GitHub connector", () => { } return jsonResponse([{ name: "main", commit: { sha: "commit-sha" } }]); }; - const client = createGitHubClient({ installationToken: "token", apiBaseUrl: "https://github.test", fetch: fetchImpl }); + const client = createGitHubClient({ + installationToken: "token", + apiBaseUrl: "https://github.test", + fetch: fetchImpl, + }); await expect(client.listRepositories()).resolves.toEqual([GITHUB_REPOSITORY]); - await expect(client.listBranches(GITHUB_REPOSITORY)).resolves.toEqual([{ name: "main", commitSha: "commit-sha" }]); + await expect(client.listBranches(GITHUB_REPOSITORY)).resolves.toEqual([ + { name: "main", commitSha: "commit-sha" }, + ]); expect(urls).toContain("https://github.test/installation/repositories?per_page=100&page=1"); expect(urls).toContain("https://github.test/repos/acme/app/branches?per_page=100&page=1"); }); @@ -93,7 +99,10 @@ describe("GitHub connector", () => { }); } if (url.includes("/git/blobs/blob-sha")) { - return jsonResponse({ encoding: "base64", content: Buffer.from("export const ok = 1;", "utf8").toString("base64") }); + return jsonResponse({ + encoding: "base64", + content: Buffer.from("export const ok = 1;", "utf8").toString("base64"), + }); } throw new Error(`unexpected URL ${url}`); }; @@ -148,6 +157,7 @@ describe("GitHub connector", () => { await expect(client.compareRepository(GITHUB_REPOSITORY, "commit-old", "commit-new")).resolves.toEqual({ fromCommitSha: "commit-old", toCommitSha: "commit-new", + isIncremental: true, changes: [ { status: "modified", newPath: "src/modified.ts" }, { status: "added", newPath: "src/added.ts" }, @@ -158,16 +168,19 @@ describe("GitHub connector", () => { }); }); - test("rejects GitHub compare responses that are not safe to apply incrementally", async () => { + test("marks non-forward GitHub compares as requiring a snapshot sync", async () => { const client = createGitHubClient({ installationToken: "token", apiBaseUrl: "https://github.test", - fetch: async () => jsonResponse({ status: "diverged", files: [] }), + fetch: async () => jsonResponse({ status: "diverged" }), }); - await expect(client.compareRepository(GITHUB_REPOSITORY, "commit-old", "commit-new")).rejects.toThrow( - "GitHub compare response is invalid" - ); + await expect(client.compareRepository(GITHUB_REPOSITORY, "commit-old", "commit-new")).resolves.toEqual({ + fromCommitSha: "commit-old", + toCommitSha: "commit-new", + isIncremental: false, + changes: [], + }); }); test("verifies webhooks and normalizes push events", () => { @@ -175,7 +188,9 @@ describe("GitHub connector", () => { const signature = `sha256=${createHmac("sha256", "secret").update(body).digest("hex")}`; expect(verifyGitHubWebhookSignature({ body, webhookSecret: "secret", signatureHeader: signature })).toBe(true); - expect(verifyGitHubWebhookSignature({ body, webhookSecret: "secret", signatureHeader: "sha256=bad" })).toBe(false); + expect(verifyGitHubWebhookSignature({ body, webhookSecret: "secret", signatureHeader: "sha256=bad" })).toBe( + false + ); expect( normalizeGitHubWebhookEvent({ eventName: "push", @@ -201,5 +216,9 @@ describe("GitHub connector", () => { }); function jsonResponse(value: unknown, init?: ResponseInit): Response { - return new Response(JSON.stringify(value), { status: 200, ...init, headers: { "content-type": "application/json", ...init?.headers } }); + return new Response(JSON.stringify(value), { + status: 200, + ...init, + headers: { "content-type": "application/json", ...init?.headers }, + }); } diff --git a/packages/connectors/src/__tests__/gitlab.test.ts b/packages/connectors/src/__tests__/gitlab.test.ts index cb30ecbe..ff3a9e41 100644 --- a/packages/connectors/src/__tests__/gitlab.test.ts +++ b/packages/connectors/src/__tests__/gitlab.test.ts @@ -22,7 +22,9 @@ describe("GitLab connector", () => { test("normalizes base URLs", () => { expect(normalizeGitLabBaseUrl("https://gitlab.example.com///")).toBe("https://gitlab.example.com"); expect(normalizeGitLabBaseUrl("https://gitlab.example.com/root/")).toBe("https://gitlab.example.com/root"); - expect(() => normalizeGitLabBaseUrl("ftp://gitlab.example.com")).toThrow("GitLab base URL must use HTTP or HTTPS"); + expect(() => normalizeGitLabBaseUrl("ftp://gitlab.example.com")).toThrow( + "GitLab base URL must use HTTP or HTTPS" + ); }); test("lists projects and branches", async () => { @@ -47,7 +49,9 @@ describe("GitLab connector", () => { const client = createGitLabClient({ baseUrl: "https://gitlab.test", accessToken: "token", fetch: fetchImpl }); await expect(client.listRepositories()).resolves.toEqual([GITLAB_REPOSITORY]); - await expect(client.listBranches(GITLAB_REPOSITORY)).resolves.toEqual([{ name: "main", commitSha: "commit-sha" }]); + await expect(client.listBranches(GITLAB_REPOSITORY)).resolves.toEqual([ + { name: "main", commitSha: "commit-sha" }, + ]); expect(urls).toContain("https://gitlab.test/api/v4/projects?membership=true&per_page=100&page=1"); expect(urls).toContain("https://gitlab.test/api/v4/projects/7/repository/branches?per_page=100&page=1"); }); @@ -161,6 +165,7 @@ describe("GitLab connector", () => { await expect(client.compareRepository(GITLAB_REPOSITORY, "commit-old", "commit-new")).resolves.toEqual({ fromCommitSha: "commit-old", toCommitSha: "commit-new", + isIncremental: true, changes: [ { status: "modified", newPath: "src/modified.ts" }, { status: "added", newPath: "src/added.ts" }, @@ -225,5 +230,9 @@ describe("GitLab connector", () => { }); function jsonResponse(value: unknown, init?: ResponseInit): Response { - return new Response(JSON.stringify(value), { status: 200, ...init, headers: { "content-type": "application/json", ...init?.headers } }); + return new Response(JSON.stringify(value), { + status: 200, + ...init, + headers: { "content-type": "application/json", ...init?.headers }, + }); } diff --git a/packages/connectors/src/github.ts b/packages/connectors/src/github.ts index adba8735..555a6a5d 100644 --- a/packages/connectors/src/github.ts +++ b/packages/connectors/src/github.ts @@ -138,7 +138,9 @@ export async function listGitHubInstallationRepositories(options: GitHubClientOp } } -export async function listGitHubBranches(options: GitHubClientOptions & { repository: ProviderRepository }): Promise { +export async function listGitHubBranches( + options: GitHubClientOptions & { repository: ProviderRepository } +): Promise { const [owner, repo] = splitFullName(options.repository.fullName); const branches: ProviderBranch[] = []; for (let page = 1; ; page += 1) { @@ -152,7 +154,12 @@ export async function listGitHubBranches(options: GitHubClientOptions & { reposi throw new ConnectorProviderError("provider", "GitHub branches response is invalid"); } for (const branch of json) { - if (isObject(branch) && typeof branch.name === "string" && isObject(branch.commit) && typeof branch.commit.sha === "string") { + if ( + isObject(branch) && + typeof branch.name === "string" && + isObject(branch.commit) && + typeof branch.commit.sha === "string" + ) { branches.push({ name: branch.name, commitSha: branch.commit.sha }); } } @@ -256,11 +263,20 @@ export async function compareGitHubRepository( options.installationToken, options.fetch ); - if ( - !isObject(json) || - (json.status !== "ahead" && json.status !== "identical") || - !Array.isArray(json.files) - ) { + if (!isObject(json) || !isGitHubCompareStatus(json.status)) { + throw new ConnectorProviderError("provider", "GitHub compare response is invalid"); + } + + if (json.status === "behind" || json.status === "diverged") { + return { + fromCommitSha: options.fromCommitSha, + toCommitSha: options.toCommitSha, + isIncremental: false, + changes: [], + }; + } + + if (!Array.isArray(json.files)) { throw new ConnectorProviderError("provider", "GitHub compare response is invalid"); } @@ -310,15 +326,18 @@ export async function compareGitHubRepository( return { fromCommitSha: options.fromCommitSha, toCommitSha: options.toCommitSha, + isIncremental: true, changes, }; } -export async function readGitHubRepositoryFile(options: GitHubClientOptions & { - repository: ProviderRepository; - path: string; - commitSha: string; -}): Promise { +export async function readGitHubRepositoryFile( + options: GitHubClientOptions & { + repository: ProviderRepository; + path: string; + commitSha: string; + } +): Promise { const [owner, repo] = splitFullName(options.repository.fullName); const apiBaseUrl = normalizeApiBaseUrl(options.apiBaseUrl ?? GITHUB_API_BASE_URL); const response = await (options.fetch ?? fetch)( @@ -373,11 +392,17 @@ export function normalizeGitHubWebhookEvent(options: { provider: "github", deliveryId: options.deliveryId, eventName: options.eventName, - repositoryId: repository && (typeof repository.id === "string" || typeof repository.id === "number") ? String(repository.id) : null, + repositoryId: + repository && (typeof repository.id === "string" || typeof repository.id === "number") + ? String(repository.id) + : null, repositoryFullName: repository && typeof repository.full_name === "string" ? repository.full_name : null, branch: branchNameFromGitRef(payload.ref), commitSha: typeof payload.after === "string" ? payload.after : null, - installationId: installation && (typeof installation.id === "string" || typeof installation.id === "number") ? String(installation.id) : undefined, + installationId: + installation && (typeof installation.id === "string" || typeof installation.id === "number") + ? String(installation.id) + : undefined, raw: options.payload, }; } @@ -386,7 +411,10 @@ async function getGitHubJson(url: string, token: string, fetchImpl: FetchLike | const response = await (fetchImpl ?? fetch)(url, { headers: githubHeaders(token) }); const json = await readJson(response); if (!response.ok) { - throw new ConnectorProviderError(response.status === 404 ? "not-found" : "provider", "GitHub API request failed"); + throw new ConnectorProviderError( + response.status === 404 ? "not-found" : "provider", + "GitHub API request failed" + ); } return json; } @@ -401,7 +429,12 @@ function githubHeaders(token: string): Record { } function mapGitHubRepository(value: unknown): ProviderRepository { - if (!isObject(value) || typeof value.full_name !== "string" || typeof value.name !== "string" || (typeof value.id !== "string" && typeof value.id !== "number")) { + if ( + !isObject(value) || + typeof value.full_name !== "string" || + typeof value.name !== "string" || + (typeof value.id !== "string" && typeof value.id !== "number") + ) { throw new ConnectorProviderError("provider", "GitHub repository response is invalid"); } return { @@ -434,7 +467,10 @@ async function readJson(response: Response): Promise { function shouldLoadCodePath(filePath: string): boolean { const normalized = filePath.replaceAll("\\", "/"); - return isSupportedCodePath(normalized) && normalized.split("/").every((segment) => SKIPPED_PATH_SEGMENTS[segment] !== true); + return ( + isSupportedCodePath(normalized) && + normalized.split("/").every((segment) => SKIPPED_PATH_SEGMENTS[segment] !== true) + ); } function normalizeApiBaseUrl(value: string): string { @@ -454,6 +490,10 @@ function isGitHubBlob(value: unknown): value is { path: string; sha: string; siz typeof value.size === "number" ); } +function isGitHubCompareStatus(value: unknown): value is "ahead" | "behind" | "diverged" | "identical" { + return value === "ahead" || value === "behind" || value === "diverged" || value === "identical"; +} + function isGitHubCompareFile(value: unknown): value is { filename: string; status: "added" | "changed" | "copied" | "modified" | "removed" | "renamed" | "unchanged"; diff --git a/packages/connectors/src/gitlab.ts b/packages/connectors/src/gitlab.ts index 4f9c6e1a..4faff126 100644 --- a/packages/connectors/src/gitlab.ts +++ b/packages/connectors/src/gitlab.ts @@ -89,7 +89,9 @@ export async function listGitLabProjects(options: GitLabClientOptions): Promise< } } -export async function listGitLabBranches(options: GitLabClientOptions & { repository: ProviderRepository }): Promise { +export async function listGitLabBranches( + options: GitLabClientOptions & { repository: ProviderRepository } +): Promise { const branches: ProviderBranch[] = []; for (let page = 1; ; page += 1) { const json = await getGitLabJson( @@ -101,7 +103,12 @@ export async function listGitLabBranches(options: GitLabClientOptions & { reposi throw new ConnectorProviderError("provider", "GitLab branches response is invalid"); } for (const branch of json) { - if (isObject(branch) && typeof branch.name === "string" && isObject(branch.commit) && typeof branch.commit.id === "string") { + if ( + isObject(branch) && + typeof branch.name === "string" && + isObject(branch.commit) && + typeof branch.commit.id === "string" + ) { branches.push({ name: branch.name, commitSha: branch.commit.id }); } } @@ -215,15 +222,18 @@ export async function compareGitLabRepository( return { fromCommitSha: options.fromCommitSha, toCommitSha: options.toCommitSha, + isIncremental: true, changes, }; } -export async function readGitLabRepositoryFile(options: GitLabClientOptions & { - repository: ProviderRepository; - path: string; - commitSha: string; -}): Promise { +export async function readGitLabRepositoryFile( + options: GitLabClientOptions & { + repository: ProviderRepository; + path: string; + commitSha: string; + } +): Promise { const rawUrl = `${gitLabApiBase(options.baseUrl)}/projects/${encodeURIComponent(options.repository.id)}/repository/files/${encodeURIComponent(options.path)}/raw?ref=${encodeURIComponent(options.commitSha)}`; const content = await getGitLabText(rawUrl, options.accessToken, options.fetch); if (Buffer.byteLength(content, "utf8") > MAX_REPOSITORY_CODE_FILE_BYTES) { @@ -250,8 +260,10 @@ export function normalizeGitLabWebhookEvent(options: { provider: "gitlab", deliveryId: options.deliveryId, eventName: options.eventName, - repositoryId: project && (typeof project.id === "string" || typeof project.id === "number") ? String(project.id) : null, - repositoryFullName: project && typeof project.path_with_namespace === "string" ? project.path_with_namespace : null, + repositoryId: + project && (typeof project.id === "string" || typeof project.id === "number") ? String(project.id) : null, + repositoryFullName: + project && typeof project.path_with_namespace === "string" ? project.path_with_namespace : null, branch: branchNameFromGitRef(payload.ref), commitSha: typeof payload.after === "string" ? payload.after : null, raw: options.payload, @@ -277,11 +289,16 @@ async function listGitLabTree(options: SnapshotOptions, ref: string): Promise { - const response = await (fetchImpl ?? fetch)(url, { headers: { authorization: `Bearer ${token}`, accept: "application/json" } }); + const response = await (fetchImpl ?? fetch)(url, { + headers: { authorization: `Bearer ${token}`, accept: "application/json" }, + }); const text = await response.text(); const json = text.length === 0 ? null : JSON.parse(text); if (!response.ok) { - throw new ConnectorProviderError(response.status === 404 ? "not-found" : "provider", "GitLab API request failed"); + throw new ConnectorProviderError( + response.status === 404 ? "not-found" : "provider", + "GitLab API request failed" + ); } return json; } @@ -294,13 +311,21 @@ async function getGitLabText(url: string, token: string, fetchImpl: FetchLike | } const text = await response.text(); if (!response.ok) { - throw new ConnectorProviderError(response.status === 404 ? "not-found" : "provider", "GitLab raw file request failed"); + throw new ConnectorProviderError( + response.status === 404 ? "not-found" : "provider", + "GitLab raw file request failed" + ); } return text; } function mapGitLabProject(value: unknown): ProviderRepository { - if (!isObject(value) || typeof value.path_with_namespace !== "string" || typeof value.name !== "string" || (typeof value.id !== "string" && typeof value.id !== "number")) { + if ( + !isObject(value) || + typeof value.path_with_namespace !== "string" || + typeof value.name !== "string" || + (typeof value.id !== "string" && typeof value.id !== "number") + ) { throw new ConnectorProviderError("provider", "GitLab project response is invalid"); } return { @@ -320,7 +345,10 @@ function gitLabApiBase(baseUrl: string): string { function shouldLoadCodePath(filePath: string): boolean { const normalized = filePath.replaceAll("\\", "/"); - return isSupportedCodePath(normalized) && normalized.split("/").every((segment) => SKIPPED_PATH_SEGMENTS[segment] !== true); + return ( + isSupportedCodePath(normalized) && + normalized.split("/").every((segment) => SKIPPED_PATH_SEGMENTS[segment] !== true) + ); } function isGitLabBlob(value: unknown): value is { id: string; path: string } { diff --git a/packages/connectors/src/types.ts b/packages/connectors/src/types.ts index 7a7f0a8c..c1cae810 100644 --- a/packages/connectors/src/types.ts +++ b/packages/connectors/src/types.ts @@ -48,14 +48,13 @@ export type ProviderRepositoryChange = export type ProviderRepositoryDelta = { fromCommitSha: string; toCommitSha: string; + isIncremental: boolean; changes: ProviderRepositoryChange[]; }; export type ConnectorCredentials = GitHubConnectorCredentials | GitLabConnectorCredentials; -export type ConnectorInstallationCredentials = - | GitHubInstallationCredentials - | GitLabInstallationCredentials; +export type ConnectorInstallationCredentials = GitHubInstallationCredentials | GitLabInstallationCredentials; export type GitHubConnectorCredentials = { provider: "github"; @@ -102,8 +101,16 @@ export type ProviderRepositoryClient = { readonly provider: ConnectorProvider; listRepositories(): Promise; listBranches(repository: ProviderRepository): Promise; - loadRepositorySnapshot(repository: ProviderRepository, branch: string, commitSha?: string): Promise; - compareRepository(repository: ProviderRepository, fromCommitSha: string, toCommitSha: string): Promise; + loadRepositorySnapshot( + repository: ProviderRepository, + branch: string, + commitSha?: string + ): Promise; + compareRepository( + repository: ProviderRepository, + fromCommitSha: string, + toCommitSha: string + ): Promise; readFile(repository: ProviderRepository, path: string, commitSha: string): Promise; }; diff --git a/packages/db/src/tables/connectors.ts b/packages/db/src/tables/connectors.ts index fc9f4f01..bcb52756 100644 --- a/packages/db/src/tables/connectors.ts +++ b/packages/db/src/tables/connectors.ts @@ -34,7 +34,10 @@ export const connectorsTable = pgTable.withRLS( clientId: text("client_id"), encryptedCredentials: text("encrypted_credentials").notNull(), webhookSecretEncrypted: text("webhook_secret_encrypted").notNull(), - createdByUserId: text("created_by_user_id").references(() => userTable.id, { onDelete: "set null" }), + createdByUserId: text("created_by_user_id").references(() => userTable.id, { + name: "connectors_created_by_user_id_user_id_fk", + onDelete: "set null", + }), createdAt: timestamp("created_at", { withTimezone: true, mode: "date" }).defaultNow(), updatedAt: timestamp("updated_at", { withTimezone: true, mode: "date" }) .defaultNow() @@ -56,14 +59,26 @@ export const connectorInstallationsTable = pgTable.withRLS( .$default(() => ulid()), connectorId: text("connector_id") .notNull() - .references(() => connectorsTable.id, { onDelete: "cascade" }), + .references(() => connectorsTable.id, { + name: "connector_installations_connector_id_connectors_id_fk", + onDelete: "cascade", + }), provider: text("provider", { enum: CONNECTOR_PROVIDER_VALUES }).notNull(), providerInstallationId: text("provider_installation_id").notNull(), providerAccountLogin: text("provider_account_login").notNull(), providerAccountType: text("provider_account_type"), - organizationId: text("organization_id").references(() => organizationTable.id, { onDelete: "cascade" }), - teamId: text("team_id").references(() => teamTable.id, { onDelete: "cascade" }), - installedByUserId: text("installed_by_user_id").references(() => userTable.id, { onDelete: "set null" }), + organizationId: text("organization_id").references(() => organizationTable.id, { + name: "connector_installations_organization_id_organization_id_fk", + onDelete: "cascade", + }), + teamId: text("team_id").references(() => teamTable.id, { + name: "connector_installations_team_id_team_id_fk", + onDelete: "cascade", + }), + installedByUserId: text("installed_by_user_id").references(() => userTable.id, { + name: "connector_installations_installed_by_user_id_user_id_fk", + onDelete: "set null", + }), encryptedCredentials: text("encrypted_credentials"), repositorySelection: text("repository_selection").notNull().default("unknown"), status: text("status", { enum: CONNECTOR_INSTALLATION_STATUS_VALUES }).notNull().default("active"), @@ -99,10 +114,16 @@ export const repositoryGraphBindingsTable = pgTable.withRLS( .$default(() => ulid()), graphId: text("graph_id") .notNull() - .references(() => graphTable.id, { onDelete: "cascade" }), + .references(() => graphTable.id, { + name: "repository_graph_bindings_graph_id_graphs_id_fk", + onDelete: "cascade", + }), connectorInstallationId: text("connector_installation_id") .notNull() - .references(() => connectorInstallationsTable.id, { onDelete: "restrict" }), + .references(() => connectorInstallationsTable.id, { + name: "repository_graph_bindings_connector_installation_id_fk", + onDelete: "restrict", + }), provider: text("provider", { enum: CONNECTOR_PROVIDER_VALUES }).notNull(), providerRepositoryId: text("provider_repository_id").notNull(), repositoryFullName: text("repository_full_name").notNull(), @@ -147,7 +168,10 @@ export const connectorWebhookEventsTable = pgTable.withRLS( .$default(() => ulid()), connectorId: text("connector_id") .notNull() - .references(() => connectorsTable.id, { onDelete: "cascade" }), + .references(() => connectorsTable.id, { + name: "connector_webhook_events_connector_id_connectors_id_fk", + onDelete: "cascade", + }), provider: text("provider", { enum: CONNECTOR_PROVIDER_VALUES }).notNull(), deliveryId: text("delivery_id").notNull(), eventName: text("event_name").notNull(), @@ -160,7 +184,11 @@ export const connectorWebhookEventsTable = pgTable.withRLS( }, (table) => [ uniqueIndex("connector_webhook_events_delivery_unique").on(table.connectorId, table.provider, table.deliveryId), - index("connector_webhook_events_binding_lookup_idx").on(table.provider, table.providerRepositoryId, table.branch), + index("connector_webhook_events_binding_lookup_idx").on( + table.provider, + table.providerRepositoryId, + table.branch + ), check("connector_webhook_events_provider_check", sql`${table.provider} in ('github', 'gitlab')`), check( "connector_webhook_events_status_check", diff --git a/packages/db/src/tables/graph.ts b/packages/db/src/tables/graph.ts index dc9b9f4d..e3dcd3c7 100644 --- a/packages/db/src/tables/graph.ts +++ b/packages/db/src/tables/graph.ts @@ -169,7 +169,7 @@ export const filesTable = pgTable.withRLS( index("files_graph_active_id_idx") .on(table.graphId, table.id) .where(sql`${table.deleted} = false`), - index("files_graph_active_key_idx") + uniqueIndex("files_graph_active_key_idx") .on(table.graphId, table.key) .where(sql`${table.deleted} = false`), check( From a281c4f57196929a8f05e6154db8530204857e9f Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 11:49:53 +0200 Subject: [PATCH 06/23] fix: test --- .../20260614105716_tricky_plazm/snapshot.json | 18 ++++++++++++++++++ .../snapshot.json | 17 +++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/migrations/20260614105716_tricky_plazm/snapshot.json b/migrations/20260614105716_tricky_plazm/snapshot.json index ddf59921..86417c20 100644 --- a/migrations/20260614105716_tricky_plazm/snapshot.json +++ b/migrations/20260614105716_tricky_plazm/snapshot.json @@ -8283,6 +8283,24 @@ "schema": "public", "table": "connector_webhook_events" } + , + { + "nameExplicit": true, + "columns": [ + "repository_binding_id" + ], + "schemaTo": "public", + "tableTo": "repository_graph_bindings", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "files_repository_binding_id_repository_graph_bindings_id_fk", + "entityType": "fks", + "schema": "public", + "table": "files" + } ], "renames": [] } diff --git a/migrations/20260615094125_fluffy_gladiator/snapshot.json b/migrations/20260615094125_fluffy_gladiator/snapshot.json index f516722d..00334254 100644 --- a/migrations/20260615094125_fluffy_gladiator/snapshot.json +++ b/migrations/20260615094125_fluffy_gladiator/snapshot.json @@ -7207,6 +7207,23 @@ "schema": "public", "table": "connector_webhook_events" }, + { + "nameExplicit": true, + "columns": [ + "repository_binding_id" + ], + "schemaTo": "public", + "tableTo": "repository_graph_bindings", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "files_repository_binding_id_repository_graph_bindings_id_fk", + "entityType": "fks", + "schema": "public", + "table": "files" + }, { "nameExplicit": true, "columns": [ From 38e91c0c2764ad27ea622a88895bb88c8caa3e1d Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 12:13:07 +0200 Subject: [PATCH 07/23] Mark graph binding failed on terminal workflow errors - mark bindings failed with `sync_failed` when terminal processing aborts - add coverage for failed child workflow cleanup path --- .../workflows/sync-repository-graph.test.ts | 26 ++ .../worker/workflows/sync-repository-graph.ts | 273 ++++++++++-------- 2 files changed, 173 insertions(+), 126 deletions(-) diff --git a/apps/worker/workflows/sync-repository-graph.test.ts b/apps/worker/workflows/sync-repository-graph.test.ts index 4dcfbe2b..3c55dd51 100644 --- a/apps/worker/workflows/sync-repository-graph.test.ts +++ b/apps/worker/workflows/sync-repository-graph.test.ts @@ -449,6 +449,32 @@ describe("syncRepositoryGraph", () => { expect(deleteWorkflowInputs).toEqual([{ graphId: "graph-1", fileId: insertedFileValues[0]?.id }]); }); + test("marks binding failed when terminal processing fails", async () => { + compareChanges = [{ status: "modified", newPath: "src/index.ts" }]; + readFileContents = { + "src/index.ts": "export const next = shared;\n", + }; + processFilesError = new Error("child failure"); + selectResults = [ + { kind: "limit", value: [bindingRow("commit-old")] }, + { + kind: "where", + value: [ + activeFile("old-file", "commit-old", "src/index.ts", 25), + activeFile("shared-file", "commit-old", "src/shared.ts", 22), + ], + }, + ]; + + await expect( + runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" }) + ).rejects.toThrow("child failure"); + expect(bindingUpdates).toContainEqual({ + syncStatus: "failed", + syncErrorCode: "sync_failed", + }); + }); + test("rejects duplicate normalized paths from provider deltas", async () => { compareChanges = [ { status: "modified", newPath: "src/index.ts" }, diff --git a/apps/worker/workflows/sync-repository-graph.ts b/apps/worker/workflows/sync-repository-graph.ts index 370199c8..c1375fad 100644 --- a/apps/worker/workflows/sync-repository-graph.ts +++ b/apps/worker/workflows/sync-repository-graph.ts @@ -69,6 +69,8 @@ type InsertedRepositoryFiles = { processRunId: string; }; +const NO_RETRY = { maximumAttempts: 1 } as const; + async function loadBindingGraph(bindingId: string): Promise { const [row] = await db .select({ @@ -459,82 +461,50 @@ async function markBindingSynced(bindingId: string, commitSha: string) { .where(eq(repositoryGraphBindingsTable.id, bindingId)); } -export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async ({ input, step }) => { - const row = await step.run({ name: "load-binding" }, async () => loadBindingGraph(input.bindingId)); - if ( - !row || - row.connector.status !== "active" || - row.installation.status !== "active" || - !row.binding.webhookEnabled - ) { - return { skipped: true }; - } - - const context = await step.run({ name: "create-provider-client" }, async () => createProviderClient(row)); - const commitSha = await step.run({ name: "resolve-target-commit" }, async () => - resolveTargetCommitSha(row, context.client, input.commitSha) - ); +async function markBindingFailed(bindingId: string) { + await db + .update(repositoryGraphBindingsTable) + .set({ syncStatus: "failed", syncErrorCode: "sync_failed" }) + .where(eq(repositoryGraphBindingsTable.id, bindingId)); +} - if (row.binding.lastSyncedCommitSha === commitSha) { - if (input.deliveryId) { - await step.run({ name: "mark-webhook-duplicate" }, async () => - markWebhookDuplicate(row.connector.provider, input.deliveryId!) - ); +export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async ({ input, step, run }) => { + try { + const row = await step.run({ name: "load-binding" }, async () => loadBindingGraph(input.bindingId)); + if ( + !row || + row.connector.status !== "active" || + row.installation.status !== "active" || + !row.binding.webhookEnabled + ) { + return { skipped: true }; } - return { skipped: true, commitSha }; - } - if (!row.binding.lastSyncedCommitSha) { - const snapshot = await step.run({ name: "load-provider-snapshot" }, async () => - loadSnapshot(row, context.client, commitSha) + const context = await step.run({ name: "create-provider-client" }, async () => createProviderClient(row)); + const commitSha = await step.run({ name: "resolve-target-commit" }, async () => + resolveTargetCommitSha(row, context.client, input.commitSha) ); - if (snapshot.files.length === 0) { - await step.run({ name: "mark-empty-binding-synced" }, async () => - markBindingSynced(row.binding.id, commitSha) - ); - return { commitSha, fileCount: 0 }; - } - const created = await step.run({ name: "commit-external-files" }, async () => - insertRepositoryFiles(row, snapshot.files, commitSha) - ); - try { - await step.runWorkflow(processFilesSpec, { - graphId: row.binding.graphId, - fileIds: created.fileIds, - processRunId: created.processRunId, - code: { kind: "repository", retiredFileIds: [] }, - }); - } catch (error) { - await Promise.all( - created.fileIds.map((fileId) => - step.runWorkflow(deleteFileSpec, { - graphId: row.binding.graphId, - fileId, - }) - ) - ); - throw error; + if (row.binding.lastSyncedCommitSha === commitSha) { + if (input.deliveryId) { + await step.run({ name: "mark-webhook-duplicate" }, async () => + markWebhookDuplicate(row.connector.provider, input.deliveryId!) + ); + } + return { skipped: true, commitSha }; } - await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); - return { commitSha, fileCount: created.fileIds.length }; - } - - const activeFiles = await step.run({ name: "load-active-binding-files" }, async () => - loadActiveBindingFiles(row.binding.id) - ); - const delta = await step.run({ name: "compare-provider-commits" }, async () => - context.client.compareRepository(repositoryFromBinding(row), row.binding.lastSyncedCommitSha!, commitSha) - ); - if (!delta.isIncremental) { - const snapshot = await step.run({ name: "load-provider-snapshot" }, async () => - loadSnapshot(row, context.client, commitSha) - ); - const retiredFileIds = [...activeFiles.values()].map((file) => file.id); - assertBindingSnapshotLimits(activeFiles, retiredFileIds, snapshot.files); + if (!row.binding.lastSyncedCommitSha) { + const snapshot = await step.run({ name: "load-provider-snapshot" }, async () => + loadSnapshot(row, context.client, commitSha) + ); + if (snapshot.files.length === 0) { + await step.run({ name: "mark-empty-binding-synced" }, async () => + markBindingSynced(row.binding.id, commitSha) + ); + return { commitSha, fileCount: 0 }; + } - if (snapshot.files.length > 0) { const created = await step.run({ name: "commit-external-files" }, async () => insertRepositoryFiles(row, snapshot.files, commitSha) ); @@ -543,7 +513,7 @@ export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async graphId: row.binding.graphId, fileIds: created.fileIds, processRunId: created.processRunId, - code: { kind: "repository", retiredFileIds }, + code: { kind: "repository", retiredFileIds: [] }, }); } catch (error) { await Promise.all( @@ -561,71 +531,122 @@ export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async return { commitSha, fileCount: created.fileIds.length }; } - if (retiredFileIds.length > 0) { - await step.runWorkflow(processFilesSpec, { - graphId: row.binding.graphId, - fileIds: [], - code: { kind: "repository", retiredFileIds }, - }); + const activeFiles = await step.run({ name: "load-active-binding-files" }, async () => + loadActiveBindingFiles(row.binding.id) + ); + const delta = await step.run({ name: "compare-provider-commits" }, async () => + context.client.compareRepository(repositoryFromBinding(row), row.binding.lastSyncedCommitSha!, commitSha) + ); + if (!delta.isIncremental) { + const snapshot = await step.run({ name: "load-provider-snapshot" }, async () => + loadSnapshot(row, context.client, commitSha) + ); + const retiredFileIds = [...activeFiles.values()].map((file) => file.id); + assertBindingSnapshotLimits(activeFiles, retiredFileIds, snapshot.files); + + if (snapshot.files.length > 0) { + const created = await step.run({ name: "commit-external-files" }, async () => + insertRepositoryFiles(row, snapshot.files, commitSha) + ); + try { + await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: created.fileIds, + processRunId: created.processRunId, + code: { kind: "repository", retiredFileIds }, + }); + } catch (error) { + await Promise.all( + created.fileIds.map((fileId) => + step.runWorkflow(deleteFileSpec, { + graphId: row.binding.graphId, + fileId, + }) + ) + ); + throw error; + } + + await step.run({ name: "mark-binding-synced" }, async () => + markBindingSynced(row.binding.id, commitSha) + ); + return { commitSha, fileCount: created.fileIds.length }; + } + + if (retiredFileIds.length > 0) { + await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: [], + code: { kind: "repository", retiredFileIds }, + }); + } + await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); + return { commitSha, fileCount: 0 }; + } + const plan = planIncrementalChanges(activeFiles, delta.changes); + if (plan.newPaths.length === 0 && plan.retiredFileIds.length === 0) { + await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); + return { commitSha, fileCount: 0 }; } - await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); - return { commitSha, fileCount: 0 }; - } - const plan = planIncrementalChanges(activeFiles, delta.changes); - if (plan.newPaths.length === 0 && plan.retiredFileIds.length === 0) { - await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); - return { commitSha, fileCount: 0 }; - } - const changedFiles = - plan.newPaths.length > 0 - ? await step.run({ name: "load-changed-files" }, async () => - Promise.all( - plan.newPaths.map(async (path) => - buildConnectorFile( - row, - context, - commitSha, - path, - await context.client.readFile(repositoryFromBinding(row), path, commitSha) + const changedFiles = + plan.newPaths.length > 0 + ? await step.run({ name: "load-changed-files" }, async () => + Promise.all( + plan.newPaths.map(async (path) => + buildConnectorFile( + row, + context, + commitSha, + path, + await context.client.readFile(repositoryFromBinding(row), path, commitSha) + ) ) ) ) - ) - : []; - assertBindingSnapshotLimits(activeFiles, plan.retiredFileIds, changedFiles); + : []; + assertBindingSnapshotLimits(activeFiles, plan.retiredFileIds, changedFiles); - if (changedFiles.length > 0) { - const created = await step.run({ name: "commit-external-files" }, async () => - insertRepositoryFiles(row, changedFiles, commitSha) - ); - try { - await step.runWorkflow(processFilesSpec, { - graphId: row.binding.graphId, - fileIds: created.fileIds, - processRunId: created.processRunId, - code: { kind: "repository", retiredFileIds: plan.retiredFileIds }, - }); - } catch (error) { - await Promise.all( - created.fileIds.map((fileId) => - step.runWorkflow(deleteFileSpec, { - graphId: row.binding.graphId, - fileId, - }) - ) + if (changedFiles.length > 0) { + const created = await step.run({ name: "commit-external-files" }, async () => + insertRepositoryFiles(row, changedFiles, commitSha) ); - throw error; + try { + await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: created.fileIds, + processRunId: created.processRunId, + code: { kind: "repository", retiredFileIds: plan.retiredFileIds }, + }); + } catch (error) { + await Promise.all( + created.fileIds.map((fileId) => + step.runWorkflow(deleteFileSpec, { + graphId: row.binding.graphId, + fileId, + }) + ) + ); + throw error; + } + await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); + return { commitSha, fileCount: created.fileIds.length }; } + + await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: [], + code: { kind: "repository", retiredFileIds: plan.retiredFileIds }, + }); await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); - return { commitSha, fileCount: created.fileIds.length }; - } + return { commitSha, fileCount: 0 }; + } catch (error) { + if (run.retryTerminal) { + await step.run({ name: "mark-binding-failed", retryPolicy: NO_RETRY }, async () => + markBindingFailed(input.bindingId) + ); + } - await step.runWorkflow(processFilesSpec, { - graphId: row.binding.graphId, - fileIds: [], - code: { kind: "repository", retiredFileIds: plan.retiredFileIds }, - }); - await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); - return { commitSha, fileCount: 0 }; + throw error; + } }); From 4c8cbfb61865598ddae4bca42f9e40ce2f42e902 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 12:23:33 +0200 Subject: [PATCH 08/23] Fallback to full snapshot on GitLab compare timeout - Return an empty non-incremental snapshot when GitLab compare times out - Keep invalid compare payloads rejected --- packages/connectors/src/__tests__/gitlab.test.ts | 11 +++++++---- packages/connectors/src/gitlab.ts | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/connectors/src/__tests__/gitlab.test.ts b/packages/connectors/src/__tests__/gitlab.test.ts index ff3a9e41..2fe0a2a6 100644 --- a/packages/connectors/src/__tests__/gitlab.test.ts +++ b/packages/connectors/src/__tests__/gitlab.test.ts @@ -176,16 +176,19 @@ describe("GitLab connector", () => { }); }); - test("rejects GitLab compare timeouts", async () => { + test("falls back to full snapshot when GitLab compare times out", async () => { const client = createGitLabClient({ baseUrl: "https://gitlab.test", accessToken: "token", fetch: async () => jsonResponse({ compare_timeout: true, diffs: [] }), }); - await expect(client.compareRepository(GITLAB_REPOSITORY, "commit-old", "commit-new")).rejects.toThrow( - "GitLab compare response is invalid" - ); + await expect(client.compareRepository(GITLAB_REPOSITORY, "commit-old", "commit-new")).resolves.toEqual({ + fromCommitSha: "commit-old", + toCommitSha: "commit-new", + isIncremental: false, + changes: [], + }); }); test("rejects malformed GitLab compare responses", async () => { diff --git a/packages/connectors/src/gitlab.ts b/packages/connectors/src/gitlab.ts index 4faff126..0239149c 100644 --- a/packages/connectors/src/gitlab.ts +++ b/packages/connectors/src/gitlab.ts @@ -177,7 +177,20 @@ export async function compareGitLabRepository( options.accessToken, options.fetch ); - if (!isObject(json) || json.compare_timeout === true || !Array.isArray(json.diffs)) { + if (!isObject(json)) { + throw new ConnectorProviderError("provider", "GitLab compare response is invalid"); + } + + if (json.compare_timeout === true) { + return { + fromCommitSha: options.fromCommitSha, + toCommitSha: options.toCommitSha, + isIncremental: false, + changes: [], + }; + } + + if (!Array.isArray(json.diffs)) { throw new ConnectorProviderError("provider", "GitLab compare response is invalid"); } From 9de8328f1ae43e989bdfa64c743d596cfa8bd507 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 12:31:53 +0200 Subject: [PATCH 09/23] Remove connector UI from backend PR --- .../connectors/[connectorId]/connect/page.tsx | 10 - .../(app)/connectors/github/callback/page.tsx | 13 - .../github/install/callback/page.tsx | 24 - .../app/(app)/connectors/github/new/page.tsx | 14 - .../app/(app)/connectors/gitlab/new/page.tsx | 14 - apps/frontend/app/(app)/connectors/page.tsx | 5 - .../components/connectors/ConnectorPages.tsx | 807 ------------------ 7 files changed, 887 deletions(-) delete mode 100644 apps/frontend/app/(app)/connectors/[connectorId]/connect/page.tsx delete mode 100644 apps/frontend/app/(app)/connectors/github/callback/page.tsx delete mode 100644 apps/frontend/app/(app)/connectors/github/install/callback/page.tsx delete mode 100644 apps/frontend/app/(app)/connectors/github/new/page.tsx delete mode 100644 apps/frontend/app/(app)/connectors/gitlab/new/page.tsx delete mode 100644 apps/frontend/app/(app)/connectors/page.tsx delete mode 100644 apps/frontend/components/connectors/ConnectorPages.tsx diff --git a/apps/frontend/app/(app)/connectors/[connectorId]/connect/page.tsx b/apps/frontend/app/(app)/connectors/[connectorId]/connect/page.tsx deleted file mode 100644 index 579f91d7..00000000 --- a/apps/frontend/app/(app)/connectors/[connectorId]/connect/page.tsx +++ /dev/null @@ -1,10 +0,0 @@ -import { ConnectorConnectPage } from "@/components/connectors/ConnectorPages"; - -type Props = { - params: Promise<{ connectorId: string }>; -}; - -export default async function ConnectConnectorPage({ params }: Props) { - const { connectorId } = await params; - return ; -} diff --git a/apps/frontend/app/(app)/connectors/github/callback/page.tsx b/apps/frontend/app/(app)/connectors/github/callback/page.tsx deleted file mode 100644 index 89f43af5..00000000 --- a/apps/frontend/app/(app)/connectors/github/callback/page.tsx +++ /dev/null @@ -1,13 +0,0 @@ -import { GitHubConnectorCallbackPage } from "@/components/connectors/ConnectorPages"; - -export default async function GitHubConnectorCallback({ - searchParams, -}: { - searchParams: Promise<{ code?: string | string[]; state?: string | string[] }>; -}) { - const params = await searchParams; - const code = typeof params.code === "string" ? params.code : ""; - const state = typeof params.state === "string" ? params.state : ""; - - return ; -} diff --git a/apps/frontend/app/(app)/connectors/github/install/callback/page.tsx b/apps/frontend/app/(app)/connectors/github/install/callback/page.tsx deleted file mode 100644 index af84647d..00000000 --- a/apps/frontend/app/(app)/connectors/github/install/callback/page.tsx +++ /dev/null @@ -1,24 +0,0 @@ -import { GitHubConnectorInstallCallbackPage } from "@/components/connectors/ConnectorPages"; - -export default async function GitHubConnectorInstallCallback({ - searchParams, -}: { - searchParams: Promise<{ - installation_id?: string | string[]; - setup_action?: string | string[]; - state?: string | string[]; - }>; -}) { - const params = await searchParams; - const installationId = typeof params.installation_id === "string" ? params.installation_id : ""; - const setupAction = typeof params.setup_action === "string" ? params.setup_action : ""; - const state = typeof params.state === "string" ? params.state : ""; - - return ( - - ); -} diff --git a/apps/frontend/app/(app)/connectors/github/new/page.tsx b/apps/frontend/app/(app)/connectors/github/new/page.tsx deleted file mode 100644 index be7821ac..00000000 --- a/apps/frontend/app/(app)/connectors/github/new/page.tsx +++ /dev/null @@ -1,14 +0,0 @@ -import { hasRole } from "@kiwi/auth/permissions"; -import { forbidden } from "next/navigation"; - -import { GitHubConnectorNewPage } from "@/components/connectors/ConnectorPages"; -import { getServerSession } from "@/lib/auth/get-server-session"; - -export default async function NewGitHubConnectorPage() { - const session = await getServerSession(); - if (!hasRole((session?.user as { role?: string })?.role ?? null, "admin")) { - forbidden(); - } - - return ; -} diff --git a/apps/frontend/app/(app)/connectors/gitlab/new/page.tsx b/apps/frontend/app/(app)/connectors/gitlab/new/page.tsx deleted file mode 100644 index dc3e1bb8..00000000 --- a/apps/frontend/app/(app)/connectors/gitlab/new/page.tsx +++ /dev/null @@ -1,14 +0,0 @@ -import { hasRole } from "@kiwi/auth/permissions"; -import { forbidden } from "next/navigation"; - -import { GitLabConnectorNewPage } from "@/components/connectors/ConnectorPages"; -import { getServerSession } from "@/lib/auth/get-server-session"; - -export default async function NewGitLabConnectorPage() { - const session = await getServerSession(); - if (!hasRole((session?.user as { role?: string })?.role ?? null, "admin")) { - forbidden(); - } - - return ; -} diff --git a/apps/frontend/app/(app)/connectors/page.tsx b/apps/frontend/app/(app)/connectors/page.tsx deleted file mode 100644 index 4722281b..00000000 --- a/apps/frontend/app/(app)/connectors/page.tsx +++ /dev/null @@ -1,5 +0,0 @@ -import { ConnectorListPage } from "@/components/connectors/ConnectorPages"; - -export default function ConnectorsPage() { - return ; -} diff --git a/apps/frontend/components/connectors/ConnectorPages.tsx b/apps/frontend/components/connectors/ConnectorPages.tsx deleted file mode 100644 index d5595b12..00000000 --- a/apps/frontend/components/connectors/ConnectorPages.tsx +++ /dev/null @@ -1,807 +0,0 @@ -"use client"; - -import { - completeGitHubConnectorInstallation, - completeGitHubConnectorManifest, - createGitLabConnector, - createRepositoryGraph, - fetchConnectorBranches, - fetchConnectorInstallations, - fetchConnectorRepositories, - fetchConnectors, - startConnectorConnect, - startGitHubConnectorManifest, - type ConnectorBranchRecord, - type ConnectorInstallationRecord, - type ConnectorRepositoryRecord, -} from "@/lib/api"; -import { ORGANIZATION_GROUP_ID } from "@/lib/api/projects"; -import { useGroupsWithProjects } from "@/hooks/use-data"; -import { useApiClient } from "@/providers/ApiClientProvider"; -import { useAuth } from "@/providers/AuthProvider"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { ExternalLink, Loader2, Plug, RefreshCw } from "lucide-react"; -import Link from "next/link"; -import { useRouter } from "next/navigation"; -import { useEffect, useMemo, useState, type ReactNode } from "react"; -import { toast } from "sonner"; - -const connectorQueryKeys = { - connectors: ["connectors"] as const, - installations: (connectorId: string) => ["connectors", connectorId, "installations"] as const, - repositories: (connectorId: string, installationId: string) => - ["connectors", connectorId, "installations", installationId, "repositories"] as const, - branches: (connectorId: string, installationId: string, repositoryId: string) => - ["connectors", connectorId, "installations", installationId, "repositories", repositoryId, "branches"] as const, -}; - -const EMPTY_INSTALLATIONS: ConnectorInstallationRecord[] = []; -const EMPTY_REPOSITORIES: ConnectorRepositoryRecord[] = []; -const EMPTY_BRANCHES: ConnectorBranchRecord[] = []; - - -export function ConnectorListPage() { - const apiClient = useApiClient(); - const { isAdmin, isSystemAdmin } = useAuth(); - const { data: groups = [] } = useGroupsWithProjects(); - const canManageGraphs = - isAdmin || groups.some((group) => group.scope === "organization" || group.role === "admin" || group.role === "moderator"); - const { data: connectors = [], isLoading, error } = useQuery({ - queryKey: connectorQueryKeys.connectors, - queryFn: () => fetchConnectors(apiClient), - }); - - return ( -
-
-
-
-

Repository connectors

-

- Connect provider apps to create private repository graphs and keep them synced. -

-
- {isSystemAdmin ? ( -
- - -
- ) : null} -
- - {error ? ( - - - Unable to load connectors - Please try again later. - - - ) : null} - - {isLoading ? ( - - - Loading connectors… - - - ) : null} - - {!isLoading && connectors.length === 0 ? ( - - - No connectors configured - - {isSystemAdmin - ? "Create a GitHub or GitLab connector before managers can add repository graphs." - : "Ask a system administrator to configure a repository connector."} - - - - ) : null} - -
- {connectors.map((connector) => ( - - - - {connector.name} - - {connector.provider === "github" ? "GitHub" : "GitLab"} connector - - -
- - Slug: {connector.slug} - - - {connector.status} - -
-
- {canManageGraphs && connector.status === "active" ? ( - - ) : null} - {isSystemAdmin ? ( - - ) : null} -
- - - ))} -
-
-
- ); -} - -export function GitHubConnectorNewPage() { - const apiClient = useApiClient(); - const [name, setName] = useState("KIWI GitHub Connector"); - const manifestMutation = useMutation({ - mutationFn: () => startGitHubConnectorManifest(apiClient, { name: name.trim() }), - onSuccess: ({ manifestUrl }) => { - window.location.assign(manifestUrl); - }, - onError: () => toast.error("Unable to start GitHub manifest flow."), - }); - const disabled = manifestMutation.isPending || name.trim().length === 0; - - return ( -
-
-

Create GitHub connector

-

- Start a GitHub App manifest flow for this KIWI instance. GitHub will return the app credentials to KIWI. -

-
- - - Required permissions - The generated app requests only read access and push events. - - -
    -
  • Contents: read, so KIWI can ingest repository files.
    • -
  • Metadata: read, so KIWI can list repositories and branches.
    • -
  • Push webhook, so KIWI can enqueue a sync when the selected branch changes.
    • -
- - - - - Manifest starter - Name the connector before continuing to GitHub. - - -
{ - event.preventDefault(); - if (!disabled) manifestMutation.mutate(); - }} - > -
- - setName(event.target.value)} - autoComplete="off" - /> -
- -
- - -
- ); -} - -type GitHubConnectorCallbackPageProps = { - code: string; - state: string; -}; - -export function GitHubConnectorCallbackPage({ code, state }: GitHubConnectorCallbackPageProps) { - const apiClient = useApiClient(); - const router = useRouter(); - const [error, setError] = useState(null); - - useEffect(() => { - if (!code || !state) { - setError("Missing GitHub manifest callback parameters."); - return; - } - - let cancelled = false; - void completeGitHubConnectorManifest(apiClient, { code, state }) - .then((connector) => { - if (cancelled) return; - router.replace(`/connectors/${connector.id}/connect`); - }) - .catch(() => { - if (cancelled) return; - setError("Unable to finish GitHub connector creation."); - toast.error("Unable to finish GitHub connector creation."); - }); - - return () => { - cancelled = true; - }; - }, [apiClient, code, router, state]); - - return ( - - - Finishing GitHub connector setup - - {error ?? "KIWI is exchanging the GitHub manifest callback and preparing the connector."} - - - {!error ? ( - - Redirecting… - - ) : null} - - ); -} - -type GitHubConnectorInstallCallbackPageProps = { - installationId: string; - setupAction: string; - state: string; -}; - -export function GitHubConnectorInstallCallbackPage({ - installationId, - setupAction, - state, -}: GitHubConnectorInstallCallbackPageProps) { - const apiClient = useApiClient(); - const router = useRouter(); - const [error, setError] = useState(null); - - useEffect(() => { - if (!installationId || !state) { - setError("Missing GitHub installation callback parameters."); - return; - } - - let cancelled = false; - void completeGitHubConnectorInstallation(apiClient, { - installation_id: installationId, - setup_action: setupAction || undefined, - state, - }) - .then((installation) => { - if (cancelled) return; - router.replace(`/connectors/${installation.connectorId}/connect`); - }) - .catch(() => { - if (cancelled) return; - setError("Unable to finish the GitHub installation flow."); - toast.error("Unable to finish the GitHub installation flow."); - }); - - return () => { - cancelled = true; - }; - }, [apiClient, installationId, router, setupAction, state]); - - return ( - - - Finishing GitHub installation - - {error ?? "KIWI is storing the GitHub installation before returning you to connector setup."} - - - {!error ? ( - - Redirecting… - - ) : null} - - ); -} - -export function GitLabConnectorNewPage() { - const apiClient = useApiClient(); - const queryClient = useQueryClient(); - const [name, setName] = useState("KIWI GitLab Connector"); - const [baseUrl, setBaseUrl] = useState("https://gitlab.com"); - const [clientId, setClientId] = useState(""); - const [clientSecret, setClientSecret] = useState(""); - const [webhookSecret, setWebhookSecret] = useState(""); - - const createMutation = useMutation({ - mutationFn: () => - createGitLabConnector(apiClient, { - name: name.trim(), - baseUrl: baseUrl.trim(), - clientId: clientId.trim(), - clientSecret, - webhookSecret, - }), - onSuccess: () => { - setClientSecret(""); - setWebhookSecret(""); - queryClient.invalidateQueries({ queryKey: connectorQueryKeys.connectors }); - toast.success("GitLab connector saved in disabled state until OAuth install flow is available."); - window.location.assign("/connectors"); - }, - onError: () => toast.error("Unable to save GitLab connector."), - }); - const disabled = - createMutation.isPending || - name.trim().length === 0 || - baseUrl.trim().length === 0 || - clientId.trim().length === 0 || - clientSecret.length === 0 || - webhookSecret.length === 0; - - return ( -
-
-

Create GitLab connector

-

- Register a GitLab application manually, then store its credentials encrypted in KIWI. -

-
- - - Required GitLab access - Use the narrowest application scopes that allow repository reads and push hooks. - - -
    -
  • Read repository access for file ingestion.
    • -
  • API access only when needed to list projects, branches, or configure push webhooks.
    • -
  • A push webhook token that KIWI can verify without exposing it back to the browser.
    • -
- - - - - Connector settings - Secrets are submitted once and cleared from the form after saving. - - -
{ - event.preventDefault(); - if (!disabled) createMutation.mutate(); - }} - > -
-
- - setName(event.target.value)} /> -
-
- - setBaseUrl(event.target.value)} - placeholder="https://gitlab.com" - /> -
-
- - setClientId(event.target.value)} - autoComplete="off" - /> -
-
- - setClientSecret(event.target.value)} - type="password" - autoComplete="new-password" - /> -
-
- - setWebhookSecret(event.target.value)} - type="password" - autoComplete="new-password" - /> -
-
- -
- - -
- ); -} - -type ConnectorConnectPageProps = { - connectorId: string; -}; - -export function ConnectorConnectPage({ connectorId }: ConnectorConnectPageProps) { - const apiClient = useApiClient(); - const router = useRouter(); - const { isAdmin } = useAuth(); - const { data: groups = [], isLoading: groupsLoading } = useGroupsWithProjects(); - const { data: connectors = [] } = useQuery({ - queryKey: connectorQueryKeys.connectors, - queryFn: () => fetchConnectors(apiClient), - }); - const connector = connectors.find((item) => item.id === connectorId) ?? null; - const manageableOwners = useMemo(() => { - const owners = groups.filter( - (group) => group.scope === "organization" || group.role === "admin" || group.role === "moderator" - ); - if (isAdmin && !owners.some((group) => group.scope === "organization")) { - return [ - { - id: ORGANIZATION_GROUP_ID, - name: "Organization", - role: "admin" as const, - scope: "organization" as const, - projects: [], - }, - ...owners, - ]; - } - return owners; - }, [groups, isAdmin]); - const canManageGraphs = manageableOwners.length > 0; - const connectorActive = connector?.status === "active"; - const [ownerValue, setOwnerValue] = useState(""); - const [installationId, setInstallationId] = useState(""); - const [repositoryId, setRepositoryId] = useState(""); - const [branchName, setBranchName] = useState(""); - const [graphName, setGraphName] = useState(""); - - useEffect(() => { - if (!ownerValue && manageableOwners.length > 0) setOwnerValue(manageableOwners[0].id); - }, [manageableOwners, ownerValue]); - - const installationsQuery = useQuery({ - queryKey: connectorQueryKeys.installations(connectorId), - queryFn: () => fetchConnectorInstallations(apiClient, connectorId), - enabled: canManageGraphs && connectorActive, - }); - const installations = installationsQuery.data ?? EMPTY_INSTALLATIONS; - - useEffect(() => { - if (!installationId && installations.length > 0) setInstallationId(installations[0].id); - }, [installationId, installations]); - - const repositoriesQuery = useQuery({ - queryKey: connectorQueryKeys.repositories(connectorId, installationId), - queryFn: () => fetchConnectorRepositories(apiClient, connectorId, installationId), - enabled: canManageGraphs && connectorActive && installationId.length > 0, - }); - const repositories = repositoriesQuery.data ?? EMPTY_REPOSITORIES; - const selectedRepository = repositories.find((repository) => repository.id === repositoryId) ?? null; - - useEffect(() => { - if (repositories.length === 0) { - if (repositoryId) setRepositoryId(""); - return; - } - if (!repositories.some((repository) => repository.id === repositoryId)) { - const defaultRepository = repositories.find((repository) => repository.defaultBranch) ?? repositories[0]; - setRepositoryId(defaultRepository.id); - } - }, [repositories, repositoryId]); - - const branchesQuery = useQuery({ - queryKey: connectorQueryKeys.branches(connectorId, installationId, repositoryId), - queryFn: () => fetchConnectorBranches(apiClient, connectorId, installationId, repositoryId), - enabled: canManageGraphs && connectorActive && installationId.length > 0 && repositoryId.length > 0, - }); - const branches = branchesQuery.data ?? EMPTY_BRANCHES; - - useEffect(() => { - if (branches.length === 0) { - if (branchName) setBranchName(""); - return; - } - if (!branches.some((branch) => branch.name === branchName)) { - const defaultBranch = selectedRepository?.defaultBranch; - setBranchName(branches.find((branch) => branch.name === defaultBranch)?.name ?? branches[0].name); - } - }, [branches, branchName, selectedRepository?.defaultBranch]); - - useEffect(() => { - if (!graphName && selectedRepository) setGraphName(selectedRepository.name); - }, [graphName, selectedRepository]); - - - const connectMutation = useMutation({ - mutationFn: () => - startConnectorConnect(apiClient, connectorId, { - ...(ownerValue === ORGANIZATION_GROUP_ID ? {} : { teamId: ownerValue }), - }), - onSuccess: ({ redirectUrl }) => { - window.location.assign(redirectUrl); - }, - onError: () => toast.error("Unable to open the provider installation flow."), - }); - const createMutation = useMutation({ - mutationFn: async () => { - if (!selectedRepository) throw new Error("Repository is required"); - const owner = - ownerValue === ORGANIZATION_GROUP_ID - ? { kind: "organization" as const } - : { kind: "team" as const, teamId: ownerValue }; - return createRepositoryGraph(apiClient, connectorId, { - connectorInstallationId: installationId, - repositoryId: selectedRepository.id, - repositoryFullName: selectedRepository.fullName, - repositoryHtmlUrl: selectedRepository.htmlUrl, - branch: branchName, - name: graphName.trim() || selectedRepository.name, - owner, - }); - }, - onSuccess: ({ graph }) => { - const routeGroupId = ownerValue === ORGANIZATION_GROUP_ID ? ORGANIZATION_GROUP_ID : ownerValue; - router.push(`/${routeGroupId}/${graph.graphId ?? graph.id}`); - }, - onError: () => toast.error("Unable to create repository graph."), - }); - - const disabled = - createMutation.isPending || - !canManageGraphs || - !connectorActive || - ownerValue.length === 0 || - installationId.length === 0 || - repositoryId.length === 0 || - branchName.length === 0 || - !selectedRepository; - const installDisabled = - connectMutation.isPending || !connectorActive || ownerValue.length === 0 || connector?.provider !== "github"; - - if (!canManageGraphs && groupsLoading) { - return ( - - - Loading connector access… - - - ); - } - - if (!canManageGraphs) { - return ( - - - Repository connector access denied - You need manager access to connect repositories. - - - ); - } - - if (connector && !connectorActive) { - return ( - - - {connector.name} is disabled - - {connector.provider === "gitlab" - ? "GitLab connectors are saved in a disabled state until OAuth installation support is available." - : "Enable this connector before connecting repositories."} - - - - ); - } - - return ( -
-
-

Connect repository

-

- {connector ? `${connector.provider === "github" ? "GitHub" : "GitLab"} · ${connector.name}` : "Choose an installation, repository, and branch."} -

-
- - - - Repository graph - Create a graph from a provider repository without exposing provider credentials. - - - {connector?.provider === "github" ? ( -
-
-

Need a GitHub installation?

-

- Install or refresh the GitHub App for the selected owner before choosing a repository. -

-
- -
- ) : null} - -
{ - event.preventDefault(); - if (!disabled) createMutation.mutate(); - }} - > -
- - {manageableOwners.map((group) => ( - - {group.name} ({group.scope}) - - ))} - - - { - setInstallationId(value); - setRepositoryId(""); - setBranchName(""); - }} - disabled={installations.length === 0} - > - {installations.map((installation) => ( - - {installation.providerAccountLogin} ({installation.repositorySelection}) - - ))} - - - { - setRepositoryId(value); - setBranchName(""); - const repository = repositories.find((item) => item.id === value); - if (repository) setGraphName(repository.name); - }} - disabled={repositories.length === 0 || repositoriesQuery.isLoading} - > - {repositories.map((repository) => ( - - {repository.fullName} - {repository.private ? " · private" : ""} - - ))} - - - - {branches.map((branch) => ( - - {branch.name} - {branch.name === selectedRepository?.defaultBranch ? " · default" : ""} - - ))} - -
- -
- - setGraphName(event.target.value)} - placeholder={selectedRepository?.name ?? "Repository graph"} - /> -
- - - - - -
- -
- ); -} - - -function SelectField({ - label, - value, - onValueChange, - disabled, - children, -}: { - label: string; - value: string; - onValueChange: (value: string) => void; - disabled?: boolean; - children: ReactNode; -}) { - return ( -
- - -
- ); -} - -function ConnectorSelectionState({ - repositories, - branches, - repositoriesLoading, - branchesLoading, -}: { - repositories: ConnectorRepositoryRecord[]; - branches: ConnectorBranchRecord[]; - repositoriesLoading: boolean; - branchesLoading: boolean; -}) { - if (repositoriesLoading || branchesLoading) { - return

Loading provider metadata…

; - } - if (repositories.length === 0) { - return

No repositories are available for this installation.

; - } - if (branches.length === 0) { - return

No branches are available for this repository.

; - } - return null; -} From a62a6d8cb1bc1c56e25171295151b7ba2ec75611 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 12:50:58 +0200 Subject: [PATCH 10/23] Harden repository loading against symlink escapes - Resolve repo and file paths before reading code files - Skip entries whose real paths escape the clone root - Add regression coverage for symlinked repository files --- .../lib/__tests__/repository-url-git.test.ts | 45 ++++++++++++++++++- apps/api/src/lib/repository-url.ts | 20 ++++++--- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/apps/api/src/lib/__tests__/repository-url-git.test.ts b/apps/api/src/lib/__tests__/repository-url-git.test.ts index 14e6f7b0..376fe106 100644 --- a/apps/api/src/lib/__tests__/repository-url-git.test.ts +++ b/apps/api/src/lib/__tests__/repository-url-git.test.ts @@ -2,6 +2,7 @@ import { EventEmitter } from "node:events"; import { beforeEach, describe, expect, mock, test } from "bun:test"; const MAX_GIT_OUTPUT_BYTES = 1 * 1024 * 1024; +let scenario: "limit" | "symlink" = "limit"; let spawnCall = 0; mock.module("node:child_process", () => ({ @@ -18,21 +19,48 @@ mock.module("node:child_process", () => ({ child.stdout.emit("data", Buffer.from("commit-sha\n")); } if (spawnCall === 3) { - child.stdout.emit("data", Buffer.alloc(MAX_GIT_OUTPUT_BYTES + 1)); + child.stdout.emit( + "data", + Buffer.from( + scenario === "symlink" ? "config.ts\0src/index.ts\0" : Buffer.alloc(MAX_GIT_OUTPUT_BYTES + 1) + ) + ); } - child.emit("close", spawnCall === 3 ? null : 0); + child.emit("close", spawnCall === 3 && scenario !== "symlink" ? null : 0); }); return child; }, })); +mock.module("node:fs/promises", () => ({ + mkdtemp: async () => "/tmp/kiwi-repository-test", + readFile: async (filePath: string) => { + if (filePath.endsWith("config.ts")) { + return "server-secret"; + } + return "export const safe = true;\n"; + }, + realpath: async (filePath: string) => { + if (filePath.endsWith("config.ts")) { + return "/etc/passwd"; + } + return filePath; + }, + rm: async () => undefined, + stat: async (filePath: string) => ({ + isFile: () => true, + size: filePath.endsWith("config.ts") ? "server-secret".length : "export const safe = true;\n".length, + }), +})); + // Dynamic import is required so Bun module mocks replace child_process before evaluation. const { loadRepositoryFromUrl } = await import("../repository-url"); describe("repository URL git loading", () => { beforeEach(() => { spawnCall = 0; + scenario = "limit"; }); test("rejects truncated git stdout as a repository limit", async () => { @@ -41,4 +69,17 @@ describe("repository URL git loading", () => { kind: "limit", }); }); + + test("skips repository files whose real path escapes the clone root", async () => { + scenario = "symlink"; + + await expect(loadRepositoryFromUrl("https://github.com/acme/app")).resolves.toMatchObject({ + files: [ + { + path: "src/index.ts", + content: "export const safe = true;\n", + }, + ], + }); + }); }); diff --git a/apps/api/src/lib/repository-url.ts b/apps/api/src/lib/repository-url.ts index a39143a1..eef60008 100644 --- a/apps/api/src/lib/repository-url.ts +++ b/apps/api/src/lib/repository-url.ts @@ -1,5 +1,5 @@ import { spawn } from "node:child_process"; -import { mkdtemp, readFile, rm, stat } from "node:fs/promises"; +import { mkdtemp, readFile, realpath, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { isSupportedCodePath } from "@kiwi/graph/code/file-path"; @@ -168,7 +168,7 @@ export async function loadRepositoryFromUrl(input: string): Promise MAX_REPOSITORY_CODE_FILE_BYTES) { continue; } @@ -194,7 +199,7 @@ export async function loadRepositoryFromUrl(input: string): Promise 0 && !relative.startsWith("..") && !path.isAbsolute(relative); +} + function shouldLoadCodePath(filePath: string): boolean { const normalized = filePath.replaceAll("\\", "/"); if (!isSupportedCodePath(normalized)) { From 6b047a3f8eff232dc18370a79aa448e2ca50ed0b Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 13:50:00 +0200 Subject: [PATCH 11/23] Store GitHub installation account metadata - Fetch installation account details during GitHub connector setup - Split connector installation uniqueness by org and team scope - Add coverage for the new account lookup and upsert behavior --- apps/api/src/lib/connectors.ts | 37 ++++- .../src/routes/__tests__/connectors.test.ts | 143 ++++++++++++++++++ apps/api/src/routes/connectors.ts | 83 +++++++--- .../20260614105716_tricky_plazm/migration.sql | 5 +- .../20260614105716_tricky_plazm/snapshot.json | 80 ++++++++-- .../snapshot.json | 93 ++++++++++-- .../connectors/src/__tests__/github.test.ts | 29 ++++ packages/connectors/src/github.ts | 42 +++++ packages/connectors/src/types.ts | 6 + .../db/src/__tests__/migration-compat.test.ts | 43 ++++-- packages/db/src/tables/connectors.ts | 14 +- 11 files changed, 506 insertions(+), 69 deletions(-) create mode 100644 apps/api/src/routes/__tests__/connectors.test.ts diff --git a/apps/api/src/lib/connectors.ts b/apps/api/src/lib/connectors.ts index bce95657..4c3cadf6 100644 --- a/apps/api/src/lib/connectors.ts +++ b/apps/api/src/lib/connectors.ts @@ -7,11 +7,13 @@ import { decryptConnectorSecret, encryptConnectorCredentials, encryptConnectorSecret, + getGitHubInstallationAccount, type ConnectorProvider, type ConnectorSecretPayload, type GitHubConnectorCredentials, type GitLabConnectorCredentials, type GitLabInstallationCredentials, + type ProviderInstallationAccount, type ProviderBranch, type ProviderRepository, type ProviderRepositoryClient, @@ -112,7 +114,11 @@ export function signConnectorState(state: Omit): st return `${STATE_VERSION}.${payload}.${signature}`; } -export function verifyConnectorState(value: string, purpose: ConnectorState["purpose"], userId?: string): ConnectorState | null { +export function verifyConnectorState( + value: string, + purpose: ConnectorState["purpose"], + userId?: string +): ConnectorState | null { const [version, payload, signature] = value.split("."); if (version !== STATE_VERSION || !payload || !signature) { return null; @@ -184,7 +190,10 @@ export async function exchangeGitHubManifestCode(code: string) { }; } -export async function createProviderClient(connector: ConnectorRow, installation: InstallationRow): Promise { +export async function createProviderClient( + connector: ConnectorRow, + installation: InstallationRow +): Promise { const credentials = decryptConnectorCredentials(connector.encryptedCredentials, env.AUTH_SECRET); if (connector.provider === "github") { if (!isGitHubConnectorCredentials(credentials)) { @@ -212,7 +221,25 @@ export async function createProviderClient(connector: ConnectorRow, installation }); } -export async function listProviderRepositories(connector: ConnectorRow, installation: InstallationRow): Promise { +export async function getGitHubConnectorInstallationAccount( + connector: ConnectorRow, + installationId: string +): Promise { + const credentials = decryptConnectorCredentials(connector.encryptedCredentials, env.AUTH_SECRET); + if (connector.provider !== "github" || !isGitHubConnectorCredentials(credentials)) { + throw new Error("Invalid connector credentials"); + } + + return getGitHubInstallationAccount({ + credentials, + installationId, + }); +} + +export async function listProviderRepositories( + connector: ConnectorRow, + installation: InstallationRow +): Promise { return (await createProviderClient(connector, installation)).listRepositories(); } @@ -223,7 +250,9 @@ export async function listProviderBranches( ): Promise { const client = await createProviderClient(connector, installation); const repositories = await client.listRepositories(); - const repository = repositories.find((candidate) => candidate.id === repositoryId || candidate.fullName === repositoryId); + const repository = repositories.find( + (candidate) => candidate.id === repositoryId || candidate.fullName === repositoryId + ); if (!repository) { throw new Error("Repository not found"); } diff --git a/apps/api/src/routes/__tests__/connectors.test.ts b/apps/api/src/routes/__tests__/connectors.test.ts new file mode 100644 index 00000000..d50ef900 --- /dev/null +++ b/apps/api/src/routes/__tests__/connectors.test.ts @@ -0,0 +1,143 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import { Elysia } from "elysia"; + +const authUser = { + id: "user-1", + activeOrganizationId: "org-1", + activeTeamId: null, + isSystemAdmin: false, +}; + +const connector = { + id: "connector-1", + provider: "github", + status: "active", + encryptedCredentials: "encrypted", +}; + +const insertValues: Array> = []; +const conflictConfigs: Array> = []; +const installationAccountCalls: string[] = []; + +mock.module("../../middleware/auth", () => ({ + authMiddleware: new Elysia().derive({ as: "scoped" }, () => ({ + user: authUser, + })), +})); + +mock.module("../../lib/graph-access", () => ({ + assertCanCreateTeamGraph: async () => ({ team: { id: "team-1", organizationId: "org-1" } }), + assertCanCreateTopLevelGraph: async () => ({ organizationId: "org-1" }), +})); + +mock.module("../../lib/connector-access", () => ({ + assertCanSyncBinding: async () => ({ binding: { id: "binding-1" } }), + assertCanUseInstallation: async () => ({ id: "installation-1", provider: "github" }), + assertCanViewBinding: async () => ({ binding: { id: "binding-1" } }), + requireActiveConnector: async () => connector, +})); + +mock.module("../../lib/connectors", () => ({ + createManifestUrl: () => "https://github.com/settings/apps/new", + encryptCredentials: () => "encrypted", + encryptSecret: () => "encrypted-secret", + exchangeGitHubManifestCode: async () => { + throw new Error("manifest exchange not expected"); + }, + getGitHubConnectorInstallationAccount: async (_connector: unknown, installationId: string) => { + installationAccountCalls.push(installationId); + return { + login: "acme", + type: "organization", + repositorySelection: "selected", + }; + }, + listProviderBranches: async () => [], + listProviderRepositories: async () => [], + signConnectorState: () => "state", + toPublicConnector: (row: unknown) => row, + toPublicInstallation: (row: unknown) => row, + verifyConnectorState: () => ({ + purpose: "github-installation", + userId: "user-1", + connectorId: "connector-1", + organizationId: "org-1", + createdAt: Date.now(), + }), +})); + +mock.module("../../openworkflow", () => ({ + ow: { + runWorkflow: async () => ({ workflowRun: { id: "run-1" } }), + }, +})); + +mock.module("@kiwi/db", () => ({ + db: { + insert: () => ({ + values: (values: Record) => { + insertValues.push(values); + return { + onConflictDoUpdate: (config: Record) => { + conflictConfigs.push(config); + return { + returning: () => [{ id: "installation-1", ...values, status: "active" }], + }; + }, + returning: () => [{ id: "row-1", ...values }], + }; + }, + }), + update: () => ({ + set: () => ({ + where: () => ({ + returning: () => [], + }), + }), + }), + select: () => ({ + from: () => ({ + orderBy: () => [], + }), + }), + }, +})); + +// Dynamic import is required so module mocks are installed before the route module is evaluated. +const { connectorRoute } = await import("../connectors"); + +describe("connector route", () => { + beforeEach(() => { + insertValues.length = 0; + conflictConfigs.length = 0; + installationAccountCalls.length = 0; + }); + + test("stores GitHub installation account metadata and targets org-scope upserts", async () => { + const response = await connectorRoute.handle( + new Request("http://localhost/connectors/github/install/callback?state=state&installation_id=99") + ); + + expect(response.status).toBe(200); + expect(installationAccountCalls).toEqual(["99"]); + expect(insertValues[0]).toMatchObject({ + connectorId: "connector-1", + provider: "github", + providerInstallationId: "99", + providerAccountLogin: "acme", + providerAccountType: "organization", + organizationId: "org-1", + teamId: null, + repositorySelection: "selected", + }); + expect(conflictConfigs[0]?.target).toHaveLength(3); + expect(conflictConfigs[0]).toHaveProperty("targetWhere"); + expect(conflictConfigs[0]?.set).toMatchObject({ + providerAccountLogin: "acme", + providerAccountType: "organization", + repositorySelection: "selected", + status: "active", + installedByUserId: "user-1", + }); + }); +}); diff --git a/apps/api/src/routes/connectors.ts b/apps/api/src/routes/connectors.ts index d1225457..a8f1a52a 100644 --- a/apps/api/src/routes/connectors.ts +++ b/apps/api/src/routes/connectors.ts @@ -9,7 +9,7 @@ import { import { graphTable } from "@kiwi/db/tables/graph"; import { syncRepositoryGraphSpec } from "@kiwi/worker/sync-repository-graph-spec"; import { Result } from "better-result"; -import { and, asc, eq } from "drizzle-orm"; +import { and, asc, eq, sql } from "drizzle-orm"; import Elysia from "elysia"; import z from "zod"; import { assertCanCreateTeamGraph, assertCanCreateTopLevelGraph } from "../lib/graph-access"; @@ -24,6 +24,7 @@ import { encryptCredentials, encryptSecret, exchangeGitHubManifestCode, + getGitHubConnectorInstallationAccount, listProviderBranches, listProviderRepositories, signConnectorState, @@ -99,7 +100,10 @@ function mapConnectorError(status: RouteStatus, error: unknown) { if (error.message === API_ERROR_CODES.GRAPH_NOT_FOUND) { return status(404, errorResponse("Not found", API_ERROR_CODES.GRAPH_NOT_FOUND)); } - return status(400, errorResponse(error.message || "Invalid connector request", API_ERROR_CODES.INVALID_CHAT_REQUEST)); + return status( + 400, + errorResponse(error.message || "Invalid connector request", API_ERROR_CODES.INVALID_CHAT_REQUEST) + ); } async function runConnectorAction(options: { @@ -155,7 +159,9 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) status, action: async (currentUser) => { const rows = await db.select().from(connectorsTable).orderBy(asc(connectorsTable.name)); - return rows.filter((row) => currentUser.isSystemAdmin || row.status === "active").map(toPublicConnector); + return rows + .filter((row) => currentUser.isSystemAdmin || row.status === "active") + .map(toPublicConnector); }, }) ) @@ -256,7 +262,9 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) .set({ ...(body.name ? { name: body.name } : {}), ...(body.status ? { status: body.status } : {}), - ...(body.webhookSecret ? { webhookSecretEncrypted: encryptSecret(body.webhookSecret) } : {}), + ...(body.webhookSecret + ? { webhookSecretEncrypted: encryptSecret(body.webhookSecret) } + : {}), }) .where(eq(connectorsTable.id, params.id)) .returning(); @@ -318,27 +326,48 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) } else { await assertCanCreateTopLevelGraph(currentUser); } + const account = await getGitHubConnectorInstallationAccount(connector, query.installation_id); + const ownerOrganizationId = state.organizationId ?? currentUser.activeOrganizationId; + const conflictTarget = state.teamId + ? { + target: [ + connectorInstallationsTable.connectorId, + connectorInstallationsTable.providerInstallationId, + connectorInstallationsTable.organizationId, + connectorInstallationsTable.teamId, + ], + targetWhere: sql`${connectorInstallationsTable.teamId} is not null`, + } + : { + target: [ + connectorInstallationsTable.connectorId, + connectorInstallationsTable.providerInstallationId, + connectorInstallationsTable.organizationId, + ], + targetWhere: sql`${connectorInstallationsTable.teamId} is null`, + }; const [installation] = await db .insert(connectorInstallationsTable) .values({ connectorId: connector.id, provider: "github", providerInstallationId: query.installation_id, - providerAccountLogin: query.installation_id, - providerAccountType: "organization", - organizationId: state.organizationId ?? currentUser.activeOrganizationId, + providerAccountLogin: account.login, + providerAccountType: account.type, + organizationId: ownerOrganizationId, teamId: state.teamId ?? null, installedByUserId: currentUser.id, - repositorySelection: query.setup_action ?? "unknown", + repositorySelection: account.repositorySelection, }) .onConflictDoUpdate({ - target: [ - connectorInstallationsTable.connectorId, - connectorInstallationsTable.providerInstallationId, - connectorInstallationsTable.organizationId, - connectorInstallationsTable.teamId, - ], - set: { status: "active", installedByUserId: currentUser.id }, + ...conflictTarget, + set: { + providerAccountLogin: account.login, + providerAccountType: account.type, + repositorySelection: account.repositorySelection, + status: "active", + installedByUserId: currentUser.id, + }, }) .returning(); return toPublicInstallation(installation); @@ -381,7 +410,10 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) status, action: async (currentUser) => { const installation = await assertCanUseInstallation(currentUser, query.installationId); - const connector = await requireActiveConnector(params.id, installation.provider as ConnectorProvider); + const connector = await requireActiveConnector( + params.id, + installation.provider as ConnectorProvider + ); const repositories = await listProviderRepositories(connector, installation); return repositories.map((repository) => ({ id: repository.id, @@ -404,7 +436,10 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) status, action: async (currentUser) => { const installation = await assertCanUseInstallation(currentUser, query.installationId); - const connector = await requireActiveConnector(params.id, installation.provider as ConnectorProvider); + const connector = await requireActiveConnector( + params.id, + installation.provider as ConnectorProvider + ); const branches = await listProviderBranches(connector, installation, params.repositoryId); return branches.map((branch) => ({ name: branch.name, commitSha: branch.commitSha })); }, @@ -431,9 +466,14 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) await assertCanCreateTopLevelGraph(currentUser); } - const connector = await requireActiveConnector(params.id, installation.provider as ConnectorProvider); + const connector = await requireActiveConnector( + params.id, + installation.provider as ConnectorProvider + ); const repositories = await listProviderRepositories(connector, installation); - const repository = repositories.find((candidate) => repositoryMatches(candidate, body.repositoryId)); + const repository = repositories.find((candidate) => + repositoryMatches(candidate, body.repositoryId) + ); if (!repository) { throw new Error(API_ERROR_CODES.GRAPH_NOT_FOUND); } @@ -524,7 +564,10 @@ export const repositoryGraphBindingRoute = new Elysia({ prefix: "/repository-gra .set({ syncStatus: "pending", syncErrorCode: null }) .where(eq(repositoryGraphBindingsTable.id, binding.id)) .returning(); - const handle = await ow.runWorkflow(syncRepositoryGraphSpec, { bindingId: binding.id, reason: "manual" }); + const handle = await ow.runWorkflow(syncRepositoryGraphSpec, { + bindingId: binding.id, + reason: "manual", + }); return { binding: toBindingResponse(updatedBinding ?? binding), workflowRunId: handle.workflowRun.id, diff --git a/migrations/20260614105716_tricky_plazm/migration.sql b/migrations/20260614105716_tricky_plazm/migration.sql index d1a67328..5e39488b 100644 --- a/migrations/20260614105716_tricky_plazm/migration.sql +++ b/migrations/20260614105716_tricky_plazm/migration.sql @@ -51,7 +51,6 @@ CREATE TABLE IF NOT EXISTS "connector_installations" ( CONSTRAINT "connector_installations_organization_id_organization_id_fk" FOREIGN KEY ("organization_id") REFERENCES "public"."organization"("id") ON DELETE cascade, CONSTRAINT "connector_installations_team_id_team_id_fk" FOREIGN KEY ("team_id") REFERENCES "public"."team"("id") ON DELETE cascade, CONSTRAINT "connector_installations_installed_by_user_id_user_id_fk" FOREIGN KEY ("installed_by_user_id") REFERENCES "public"."user"("id") ON DELETE set null, - CONSTRAINT "connector_installations_provider_scope_unique" UNIQUE("connector_id", "provider_installation_id", "organization_id", "team_id"), CONSTRAINT "connector_installations_provider_check" CHECK ("connector_installations"."provider" in ('github', 'gitlab')), CONSTRAINT "connector_installations_status_check" CHECK ("connector_installations"."status" in ('active', 'disabled')), CONSTRAINT "connector_installations_owner_scope_check" CHECK (("organization_id" is not null and "team_id" is null) or ("organization_id" is not null and "team_id" is not null)) @@ -103,6 +102,8 @@ ALTER TABLE "files" DROP CONSTRAINT IF EXISTS "files_repository_binding_id_repos ALTER TABLE "files" ADD CONSTRAINT "files_repository_binding_id_repository_graph_bindings_id_fk" FOREIGN KEY ("repository_binding_id") REFERENCES "public"."repository_graph_bindings"("id") ON DELETE set null; CREATE INDEX IF NOT EXISTS "connectors_provider_status_idx" ON "connectors" ("provider", "status"); +CREATE UNIQUE INDEX IF NOT EXISTS "connector_installations_org_scope_unique" ON "connector_installations" ("connector_id", "provider_installation_id", "organization_id") WHERE "team_id" IS NULL; +CREATE UNIQUE INDEX IF NOT EXISTS "connector_installations_team_scope_unique" ON "connector_installations" ("connector_id", "provider_installation_id", "organization_id", "team_id") WHERE "team_id" IS NOT NULL; CREATE INDEX IF NOT EXISTS "connector_installations_connector_status_idx" ON "connector_installations" ("connector_id", "status"); CREATE INDEX IF NOT EXISTS "connector_installations_organization_idx" ON "connector_installations" ("organization_id"); CREATE INDEX IF NOT EXISTS "connector_installations_team_idx" ON "connector_installations" ("team_id"); @@ -112,4 +113,4 @@ CREATE INDEX IF NOT EXISTS "repository_graph_bindings_provider_repo_branch_idx" CREATE INDEX IF NOT EXISTS "repository_graph_bindings_installation_status_idx" ON "repository_graph_bindings" ("connector_installation_id", "sync_status"); CREATE UNIQUE INDEX IF NOT EXISTS "connector_webhook_events_delivery_unique" ON "connector_webhook_events" ("connector_id", "provider", "delivery_id"); CREATE INDEX IF NOT EXISTS "connector_webhook_events_binding_lookup_idx" ON "connector_webhook_events" ("provider", "provider_repository_id", "branch"); -CREATE INDEX IF NOT EXISTS "files_repository_binding_active_idx" ON "files" ("repository_binding_id", "created_at", "id") WHERE "deleted" = false; \ No newline at end of file +CREATE INDEX IF NOT EXISTS "files_repository_binding_active_idx" ON "files" ("repository_binding_id", "created_at", "id") WHERE "deleted" = false; diff --git a/migrations/20260614105716_tricky_plazm/snapshot.json b/migrations/20260614105716_tricky_plazm/snapshot.json index 86417c20..7fdfb6e6 100644 --- a/migrations/20260614105716_tricky_plazm/snapshot.json +++ b/migrations/20260614105716_tricky_plazm/snapshot.json @@ -7842,14 +7842,77 @@ { "nameExplicit": true, "columns": [ - "connector_id", - "provider_installation_id", - "organization_id", - "team_id" + { + "value": "connector_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider_installation_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } ], - "nullsNotDistinct": false, - "name": "connector_installations_provider_scope_unique", - "entityType": "uniques", + "isUnique": true, + "where": "\"team_id\" IS NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_installations_org_scope_unique", + "entityType": "indexes", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider_installation_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"team_id\" IS NOT NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_installations_team_scope_unique", + "entityType": "indexes", "schema": "public", "table": "connector_installations" }, @@ -8282,8 +8345,7 @@ "entityType": "fks", "schema": "public", "table": "connector_webhook_events" - } - , + }, { "nameExplicit": true, "columns": [ diff --git a/migrations/20260615094125_fluffy_gladiator/snapshot.json b/migrations/20260615094125_fluffy_gladiator/snapshot.json index 00334254..7aac8d98 100644 --- a/migrations/20260615094125_fluffy_gladiator/snapshot.json +++ b/migrations/20260615094125_fluffy_gladiator/snapshot.json @@ -5229,6 +5229,83 @@ "schema": "public", "table": "messages" }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider_installation_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"team_id\" IS NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_installations_org_scope_unique", + "entityType": "indexes", + "schema": "public", + "table": "connector_installations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "connector_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider_installation_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "team_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": "\"team_id\" IS NOT NULL", + "with": "", + "method": "btree", + "concurrently": false, + "name": "connector_installations_team_scope_unique", + "entityType": "indexes", + "schema": "public", + "table": "connector_installations" + }, { "nameExplicit": true, "columns": [ @@ -8123,20 +8200,6 @@ "schema": "public", "table": "team" }, - { - "nameExplicit": true, - "columns": [ - "connector_id", - "provider_installation_id", - "organization_id", - "team_id" - ], - "nullsNotDistinct": false, - "name": "connector_installations_provider_scope_unique", - "entityType": "uniques", - "schema": "public", - "table": "connector_installations" - }, { "nameExplicit": false, "columns": [ @@ -8302,4 +8365,4 @@ } ], "renames": [] -} \ No newline at end of file +} diff --git a/packages/connectors/src/__tests__/github.test.ts b/packages/connectors/src/__tests__/github.test.ts index f62a6c5a..9ac56a86 100644 --- a/packages/connectors/src/__tests__/github.test.ts +++ b/packages/connectors/src/__tests__/github.test.ts @@ -4,6 +4,7 @@ import { createGitHubAppJwt, createGitHubClient, createGitHubInstallationToken, + getGitHubInstallationAccount, loadGitHubRepositorySnapshot, normalizeGitHubWebhookEvent, verifyGitHubWebhookSignature, @@ -47,6 +48,34 @@ describe("GitHub connector", () => { expect(calls[0]?.body).toBe(JSON.stringify({ permissions: { contents: "read", metadata: "read" } })); }); + test("loads GitHub installation account metadata", async () => { + const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); + const privateKeyPem = privateKey.export({ type: "pkcs8", format: "pem" }).toString(); + const calls: string[] = []; + const fetchImpl: FetchLike = async (input) => { + calls.push(String(input)); + return jsonResponse({ + account: { login: "acme", type: "Organization" }, + repository_selection: "selected", + }); + }; + + await expect( + getGitHubInstallationAccount({ + credentials: { provider: "github", appId: "42", privateKeyPem }, + installationId: "99", + apiBaseUrl: "https://github.test", + fetch: fetchImpl, + now: new Date("2026-01-01T00:00:00Z"), + }) + ).resolves.toEqual({ + login: "acme", + type: "organization", + repositorySelection: "selected", + }); + expect(calls).toEqual(["https://github.test/app/installations/99"]); + }); + test("lists repositories and branches with pagination", async () => { const urls: string[] = []; const fetchImpl: FetchLike = async (input) => { diff --git a/packages/connectors/src/github.ts b/packages/connectors/src/github.ts index 555a6a5d..cf25bc12 100644 --- a/packages/connectors/src/github.ts +++ b/packages/connectors/src/github.ts @@ -6,6 +6,7 @@ import type { NormalizedWebhookEvent, ProviderBranch, ProviderCodeFile, + ProviderInstallationAccount, ProviderRepository, ProviderRepositoryClient, ProviderRepositoryDelta, @@ -96,6 +97,37 @@ export async function createGitHubInstallationToken(options: InstallationTokenOp return { token: json.token, expiresAt: json.expires_at }; } +export async function getGitHubInstallationAccount( + options: InstallationTokenOptions +): Promise { + const apiBaseUrl = normalizeApiBaseUrl(options.apiBaseUrl ?? GITHUB_API_BASE_URL); + const response = await (options.fetch ?? fetch)( + `${apiBaseUrl}/app/installations/${encodeURIComponent(options.installationId)}`, + { + headers: githubHeaders( + createGitHubAppJwt({ + appId: options.credentials.appId, + privateKeyPem: options.credentials.privateKeyPem, + now: options.now, + }) + ), + } + ); + const json = await readJson(response); + if (!response.ok || !isObject(json) || !isObject(json.account) || typeof json.account.login !== "string") { + throw new ConnectorProviderError("provider", "GitHub installation response is invalid"); + } + + return { + login: json.account.login, + type: gitHubAccountType(json.account.type), + repositorySelection: + json.repository_selection === "all" || json.repository_selection === "selected" + ? json.repository_selection + : "unknown", + }; +} + export function createGitHubClient(options: GitHubClientOptions): ProviderRepositoryClient { return { provider: "github", @@ -460,6 +492,16 @@ function encodePathSegments(value: string): string { return value.split("/").map(encodeURIComponent).join("/"); } +function gitHubAccountType(value: unknown): ProviderInstallationAccount["type"] { + if (value === "User") { + return "user"; + } + if (value === "Organization") { + return "organization"; + } + return null; +} + async function readJson(response: Response): Promise { const text = await response.text(); return text.length === 0 ? null : JSON.parse(text); diff --git a/packages/connectors/src/types.ts b/packages/connectors/src/types.ts index c1cae810..5d4a5501 100644 --- a/packages/connectors/src/types.ts +++ b/packages/connectors/src/types.ts @@ -15,6 +15,12 @@ export type ProviderBranch = { commitSha: string; }; +export type ProviderInstallationAccount = { + login: string; + type: "user" | "organization" | "group" | null; + repositorySelection: "all" | "selected" | "unknown"; +}; + export type ProviderCodeFile = { path: string; size: number; diff --git a/packages/db/src/__tests__/migration-compat.test.ts b/packages/db/src/__tests__/migration-compat.test.ts index 6ca801ea..50568308 100644 --- a/packages/db/src/__tests__/migration-compat.test.ts +++ b/packages/db/src/__tests__/migration-compat.test.ts @@ -16,10 +16,7 @@ const codeGraphMigration = new URL( "../../../../migrations/20260613184716_sturdy_triton/migration.sql", import.meta.url ); -const codeGraphSnapshot = new URL( - "../../../../migrations/20260613184716_sturdy_triton/snapshot.json", - import.meta.url -); +const codeGraphSnapshot = new URL("../../../../migrations/20260613184716_sturdy_triton/snapshot.json", import.meta.url); const externalFileStorageMigration = new URL( "../../../../migrations/20260613201908_mature_emma_frost/migration.sql", import.meta.url @@ -37,10 +34,7 @@ const sourceValiditySnapshot = new URL( import.meta.url ); const saveGraphModule = new URL("../../../../apps/worker/lib/save-graph.ts", import.meta.url); -const regenerateDescriptionsModule = new URL( - "../../../../apps/worker/lib/regenerate-descriptions.ts", - import.meta.url -); +const regenerateDescriptionsModule = new URL("../../../../apps/worker/lib/regenerate-descriptions.ts", import.meta.url); const sourceReferenceModule = new URL("../../../../apps/api/src/lib/source-reference.ts", import.meta.url); describe("source reference migration compatibility", () => { @@ -168,8 +162,12 @@ describe("source validity migration compatibility", () => { const snapshot = (await Bun.file(sourceValiditySnapshot).json()) as { ddl: Array>; }; - const sourceColumns = snapshot.ddl.filter((entry) => entry.entityType === "columns" && entry.table === "sources"); - const sourceIndexes = snapshot.ddl.filter((entry) => entry.entityType === "indexes" && entry.table === "sources"); + const sourceColumns = snapshot.ddl.filter( + (entry) => entry.entityType === "columns" && entry.table === "sources" + ); + const sourceIndexes = snapshot.ddl.filter( + (entry) => entry.entityType === "indexes" && entry.table === "sources" + ); const connectorTables = snapshot.ddl.filter((entry) => entry.entityType === "tables"); const connectorColumns = snapshot.ddl.filter((entry) => entry.entityType === "columns"); const connectorIndexes = snapshot.ddl.filter((entry) => entry.entityType === "indexes"); @@ -205,13 +203,29 @@ describe("source validity migration compatibility", () => { expect.objectContaining({ table: "repository_graph_bindings", name: "sync_status", default: "'pending'" }) ); expect(connectorIndexes).toContainEqual(expect.objectContaining({ name: "connectors_provider_status_idx" })); + expect(connectorIndexes).toContainEqual( + expect.objectContaining({ + name: "connector_installations_org_scope_unique", + isUnique: true, + where: '"team_id" IS NULL', + }) + ); + expect(connectorIndexes).toContainEqual( + expect.objectContaining({ + name: "connector_installations_team_scope_unique", + isUnique: true, + where: '"team_id" IS NOT NULL', + }) + ); expect(connectorIndexes).toContainEqual( expect.objectContaining({ name: "repository_graph_bindings_graph_unique", isUnique: true }) ); expect(connectorIndexes).toContainEqual( expect.objectContaining({ name: "connector_webhook_events_delivery_unique", isUnique: true }) ); - expect(connectorIndexes).toContainEqual(expect.objectContaining({ name: "files_repository_binding_active_idx" })); + expect(connectorIndexes).toContainEqual( + expect.objectContaining({ name: "files_repository_binding_active_idx" }) + ); expect(connectorChecks).toContainEqual(expect.objectContaining({ name: "connectors_provider_check" })); expect(connectorChecks).toContainEqual( expect.objectContaining({ name: "repository_graph_bindings_sync_status_check" }) @@ -234,6 +248,11 @@ describe("source validity migration compatibility", () => { expect(sql).toContain('CREATE TABLE IF NOT EXISTS "connector_webhook_events"'); expect(sql).toContain('ADD COLUMN IF NOT EXISTS "repository_binding_id" text'); expect(sql).toContain('ADD CONSTRAINT "files_repository_binding_id_repository_graph_bindings_id_fk"'); + expect(sql).not.toContain('CONSTRAINT "connector_installations_provider_scope_unique"'); + expect(sql).toContain('CREATE UNIQUE INDEX IF NOT EXISTS "connector_installations_org_scope_unique"'); + expect(sql).toContain('WHERE "team_id" IS NULL'); + expect(sql).toContain('CREATE UNIQUE INDEX IF NOT EXISTS "connector_installations_team_scope_unique"'); + expect(sql).toContain('WHERE "team_id" IS NOT NULL'); expect(sql).toContain('CREATE UNIQUE INDEX IF NOT EXISTS "connector_webhook_events_delivery_unique"'); expect(sql).toContain("\"external_provider\" IS NULL OR \"external_provider\" in ('github', 'gitlab')"); }); @@ -242,7 +261,7 @@ describe("source validity migration compatibility", () => { expect(source).toContain("SET valid_until = NOW()"); expect(source).toContain("old_file.file_type = 'code'"); - expect(source).toContain("currentSourceSql(\"old_source\")"); + expect(source).toContain('currentSourceSql("old_source")'); expect(source).not.toMatch(/DELETE\s+FROM\s+sources\s+source/i); }); diff --git a/packages/db/src/tables/connectors.ts b/packages/db/src/tables/connectors.ts index bcb52756..2673deef 100644 --- a/packages/db/src/tables/connectors.ts +++ b/packages/db/src/tables/connectors.ts @@ -1,5 +1,5 @@ import { sql } from "drizzle-orm"; -import { boolean, check, index, pgTable, text, timestamp, unique, uniqueIndex } from "drizzle-orm/pg-core"; +import { boolean, check, index, pgTable, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core"; import { ulid } from "ulid"; import { organizationTable, teamTable, userTable } from "./auth"; import { graphTable } from "./graph"; @@ -88,12 +88,12 @@ export const connectorInstallationsTable = pgTable.withRLS( .$onUpdate(() => sql`NOW()`), }, (table) => [ - unique("connector_installations_provider_scope_unique").on( - table.connectorId, - table.providerInstallationId, - table.organizationId, - table.teamId - ), + uniqueIndex("connector_installations_org_scope_unique") + .on(table.connectorId, table.providerInstallationId, table.organizationId) + .where(sql`${table.teamId} is null`), + uniqueIndex("connector_installations_team_scope_unique") + .on(table.connectorId, table.providerInstallationId, table.organizationId, table.teamId) + .where(sql`${table.teamId} is not null`), index("connector_installations_connector_status_idx").on(table.connectorId, table.status), index("connector_installations_organization_idx").on(table.organizationId), index("connector_installations_team_idx").on(table.teamId), From abee5b8a80f90e3b54226ccae34dcc4ed078a28f Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 14:16:59 +0200 Subject: [PATCH 12/23] Exclude completed process runs from file conflict lookup - Avoid reusing finished process runs when a file insert hits a conflict - Add regression coverage for completed run filtering --- .../workflows/sync-repository-graph.test.ts | 54 ++++++++++++++++++- .../worker/workflows/sync-repository-graph.ts | 3 +- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/apps/worker/workflows/sync-repository-graph.test.ts b/apps/worker/workflows/sync-repository-graph.test.ts index 3c55dd51..eec3866c 100644 --- a/apps/worker/workflows/sync-repository-graph.test.ts +++ b/apps/worker/workflows/sync-repository-graph.test.ts @@ -9,6 +9,7 @@ const deleteWorkflowInputs: Array> = []; const bindingUpdates: Array> = []; const compareCalls: Array<{ fromCommitSha: string; toCommitSha: string }> = []; const readFileCalls: Array<{ path: string; commitSha: string }> = []; +const txWhereConditions: unknown[] = []; let selectResults: SelectResult[] = []; let txSelectResults: unknown[][] = []; @@ -45,12 +46,32 @@ function createTxSelectQuery() { const chain = { from: () => chain, innerJoin: () => chain, - where: () => result, + where: (condition: unknown) => { + txWhereConditions.push(condition); + return result; + }, limit: () => result, }; return chain; } +function queryContainsParamValue(value: unknown, expected: unknown, seen = new WeakSet()): boolean { + if (Array.isArray(value)) { + return value.some((item) => queryContainsParamValue(item, expected, seen)); + } + if (typeof value === "object" && value !== null) { + if (seen.has(value)) { + return false; + } + seen.add(value); + if (value.constructor.name === "Param" && "value" in value) { + return value.value === expected; + } + return Object.values(value).some((item) => queryContainsParamValue(item, expected, seen)); + } + return false; +} + const transactionDb = { select: () => createTxSelectQuery(), update: () => ({ @@ -264,6 +285,7 @@ describe("syncRepositoryGraph", () => { bindingUpdates.length = 0; compareCalls.length = 0; readFileCalls.length = 0; + txWhereConditions.length = 0; selectResults = []; processFilesError = null; compareChanges = []; @@ -340,6 +362,36 @@ describe("syncRepositoryGraph", () => { ]); }); + test("does not reuse completed process runs when file insert conflicts", async () => { + compareChanges = [{ status: "modified", newPath: "src/index.ts" }]; + readFileContents = { + "src/index.ts": "export const next = shared;\n", + }; + selectResults = [ + { kind: "limit", value: [bindingRow("commit-old")] }, + { + kind: "where", + value: [activeFile("old-file", "commit-old", "src/index.ts", 25)], + }, + ]; + insertedFileRowsOverride = []; + txSelectResults = [[{ id: "existing-file", key: "connector:binding-1:commit-new:src/index.ts" }], []]; + + const result = await runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" }); + + expect(result).toMatchObject({ commitSha: "commit-new", fileCount: 1 }); + expect(processRunInsertCount).toBe(1); + expect(queryContainsParamValue(txWhereConditions[1], "completed")).toBe(true); + expect(processWorkflowInputs).toEqual([ + { + graphId: "graph-1", + fileIds: ["existing-file"], + processRunId: "process-run-1", + code: { kind: "repository", retiredFileIds: ["old-file"] }, + }, + ]); + }); + test("finalizes delete-only connector deltas without inserting new files", async () => { compareChanges = [{ status: "deleted", oldPath: "src/removed.ts" }]; selectResults = [ diff --git a/apps/worker/workflows/sync-repository-graph.ts b/apps/worker/workflows/sync-repository-graph.ts index c1375fad..5bca4327 100644 --- a/apps/worker/workflows/sync-repository-graph.ts +++ b/apps/worker/workflows/sync-repository-graph.ts @@ -29,7 +29,7 @@ import { } from "@kiwi/db/tables/connectors"; import { filesTable, graphTable, processRunFilesTable, processRunsTable } from "@kiwi/db/tables/graph"; import { serializeCodeFileMetadata } from "@kiwi/graph/code/metadata"; -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray, ne } from "drizzle-orm"; import { defineWorkflow } from "openworkflow"; import { parseCodeFileMetadata } from "../lib/code-file-metadata"; import { env } from "../env"; @@ -402,6 +402,7 @@ async function insertRepositoryFiles( .where( and( eq(processRunsTable.graphId, row.binding.graphId), + ne(processRunsTable.status, "completed"), inArray( processRunFilesTable.fileId, committedFiles.map((file) => file.id) From 2e84a4d848cc9ee0fd468117bb525cb2db93b376 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 14:26:41 +0200 Subject: [PATCH 13/23] Handle missing symlink targets and truncated GitHub trees - Skip repository files when realpath resolution fails - Reject truncated GitHub tree responses with a limit error - Add tests for both edge cases --- .../lib/__tests__/repository-url-git.test.ts | 24 ++++++++++++++--- apps/api/src/lib/repository-url.ts | 7 ++++- .../connectors/src/__tests__/github.test.ts | 26 +++++++++++++++++++ packages/connectors/src/github.ts | 3 +++ 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/apps/api/src/lib/__tests__/repository-url-git.test.ts b/apps/api/src/lib/__tests__/repository-url-git.test.ts index 376fe106..5790bd4e 100644 --- a/apps/api/src/lib/__tests__/repository-url-git.test.ts +++ b/apps/api/src/lib/__tests__/repository-url-git.test.ts @@ -2,7 +2,7 @@ import { EventEmitter } from "node:events"; import { beforeEach, describe, expect, mock, test } from "bun:test"; const MAX_GIT_OUTPUT_BYTES = 1 * 1024 * 1024; -let scenario: "limit" | "symlink" = "limit"; +let scenario: "broken-symlink" | "limit" | "symlink" = "limit"; let spawnCall = 0; mock.module("node:child_process", () => ({ @@ -22,11 +22,13 @@ mock.module("node:child_process", () => ({ child.stdout.emit( "data", Buffer.from( - scenario === "symlink" ? "config.ts\0src/index.ts\0" : Buffer.alloc(MAX_GIT_OUTPUT_BYTES + 1) + scenario === "symlink" || scenario === "broken-symlink" + ? "config.ts\0src/index.ts\0" + : Buffer.alloc(MAX_GIT_OUTPUT_BYTES + 1) ) ); } - child.emit("close", spawnCall === 3 && scenario !== "symlink" ? null : 0); + child.emit("close", spawnCall === 3 && scenario === "limit" ? null : 0); }); return child; @@ -42,6 +44,9 @@ mock.module("node:fs/promises", () => ({ return "export const safe = true;\n"; }, realpath: async (filePath: string) => { + if (scenario === "broken-symlink" && filePath.endsWith("config.ts")) { + throw Object.assign(new Error("ENOENT: no such file or directory"), { code: "ENOENT" }); + } if (filePath.endsWith("config.ts")) { return "/etc/passwd"; } @@ -82,4 +87,17 @@ describe("repository URL git loading", () => { ], }); }); + + test("skips repository files whose real path cannot be resolved", async () => { + scenario = "broken-symlink"; + + await expect(loadRepositoryFromUrl("https://github.com/acme/app")).resolves.toMatchObject({ + files: [ + { + path: "src/index.ts", + content: "export const safe = true;\n", + }, + ], + }); + }); }); diff --git a/apps/api/src/lib/repository-url.ts b/apps/api/src/lib/repository-url.ts index eef60008..f55776ef 100644 --- a/apps/api/src/lib/repository-url.ts +++ b/apps/api/src/lib/repository-url.ts @@ -181,7 +181,12 @@ export async function loadRepositoryFromUrl(input: string): Promise { expect(calls.some((url) => url.includes("readme-sha"))).toBe(false); }); + test("rejects truncated GitHub repository trees", async () => { + const fetchImpl: FetchLike = async (input) => { + const url = String(input); + if (url.endsWith("/branches/main")) { + return jsonResponse({ commit: { sha: "commit-sha", commit: { tree: { sha: "tree-sha" } } } }); + } + if (url.includes("/git/trees/tree-sha")) { + return jsonResponse({ truncated: true, tree: [] }); + } + throw new Error(`unexpected URL ${url}`); + }; + + await expect( + loadGitHubRepositorySnapshot({ + installationToken: "token", + apiBaseUrl: "https://github.test", + fetch: fetchImpl, + repository: GITHUB_REPOSITORY, + branch: "main", + }) + ).rejects.toMatchObject({ + name: "ConnectorProviderError", + kind: "limit", + }); + }); + test("normalizes incremental compare changes for supported code paths", async () => { const client = createGitHubClient({ installationToken: "token", diff --git a/packages/connectors/src/github.ts b/packages/connectors/src/github.ts index cf25bc12..befd40f3 100644 --- a/packages/connectors/src/github.ts +++ b/packages/connectors/src/github.ts @@ -242,6 +242,9 @@ export async function loadGitHubRepositorySnapshot(options: SnapshotOptions): Pr if (!isObject(treeJson) || !Array.isArray(treeJson.tree)) { throw new ConnectorProviderError("provider", "GitHub tree response is invalid"); } + if (treeJson.truncated === true) { + throw new ConnectorProviderError("limit", "GitHub repository tree is too large to load completely"); + } const files: ProviderCodeFile[] = []; let totalBytes = 0; From f49146693db652c890fa2f4472ed47ddb88ed834 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 14:37:00 +0200 Subject: [PATCH 14/23] Handle repository graph sync failures and GitLab errors - Mark manual repository graph syncs failed when workflow enqueue throws - Preserve typed provider errors for non-JSON GitLab API failures --- .../src/routes/__tests__/connectors.test.ts | 38 +++++++++++++++---- apps/api/src/routes/connectors.ts | 24 ++++++++---- .../connectors/src/__tests__/gitlab.test.ts | 13 +++++++ packages/connectors/src/gitlab.ts | 3 +- 4 files changed, 61 insertions(+), 17 deletions(-) diff --git a/apps/api/src/routes/__tests__/connectors.test.ts b/apps/api/src/routes/__tests__/connectors.test.ts index d50ef900..5d3070ce 100644 --- a/apps/api/src/routes/__tests__/connectors.test.ts +++ b/apps/api/src/routes/__tests__/connectors.test.ts @@ -18,6 +18,8 @@ const connector = { const insertValues: Array> = []; const conflictConfigs: Array> = []; const installationAccountCalls: string[] = []; +const updateValues: Array> = []; +let workflowError: Error | null = null; mock.module("../../middleware/auth", () => ({ authMiddleware: new Elysia().derive({ as: "scoped" }, () => ({ @@ -68,7 +70,12 @@ mock.module("../../lib/connectors", () => ({ mock.module("../../openworkflow", () => ({ ow: { - runWorkflow: async () => ({ workflowRun: { id: "run-1" } }), + runWorkflow: async () => { + if (workflowError) { + throw workflowError; + } + return { workflowRun: { id: "run-1" } }; + }, }, })); @@ -89,11 +96,14 @@ mock.module("@kiwi/db", () => ({ }, }), update: () => ({ - set: () => ({ - where: () => ({ - returning: () => [], - }), - }), + set: (values: Record) => { + updateValues.push(values); + return { + where: () => ({ + returning: () => [], + }), + }; + }, }), select: () => ({ from: () => ({ @@ -104,13 +114,15 @@ mock.module("@kiwi/db", () => ({ })); // Dynamic import is required so module mocks are installed before the route module is evaluated. -const { connectorRoute } = await import("../connectors"); +const { connectorRoute, repositoryGraphBindingRoute } = await import("../connectors"); describe("connector route", () => { beforeEach(() => { insertValues.length = 0; conflictConfigs.length = 0; installationAccountCalls.length = 0; + updateValues.length = 0; + workflowError = null; }); test("stores GitHub installation account metadata and targets org-scope upserts", async () => { @@ -140,4 +152,16 @@ describe("connector route", () => { installedByUserId: "user-1", }); }); + + test("marks manual repository graph sync failed when workflow enqueue fails", async () => { + workflowError = new Error("enqueue failed"); + + const response = await repositoryGraphBindingRoute.handle( + new Request("http://localhost/repository-graph-bindings/binding-1/sync", { method: "POST" }) + ); + + expect(response.status).toBe(400); + expect(updateValues).toContainEqual({ syncStatus: "pending", syncErrorCode: null }); + expect(updateValues).toContainEqual({ syncStatus: "failed", syncErrorCode: "enqueue_failed" }); + }); }); diff --git a/apps/api/src/routes/connectors.ts b/apps/api/src/routes/connectors.ts index a8f1a52a..a6d55f00 100644 --- a/apps/api/src/routes/connectors.ts +++ b/apps/api/src/routes/connectors.ts @@ -564,14 +564,22 @@ export const repositoryGraphBindingRoute = new Elysia({ prefix: "/repository-gra .set({ syncStatus: "pending", syncErrorCode: null }) .where(eq(repositoryGraphBindingsTable.id, binding.id)) .returning(); - const handle = await ow.runWorkflow(syncRepositoryGraphSpec, { - bindingId: binding.id, - reason: "manual", - }); - return { - binding: toBindingResponse(updatedBinding ?? binding), - workflowRunId: handle.workflowRun.id, - }; + try { + const handle = await ow.runWorkflow(syncRepositoryGraphSpec, { + bindingId: binding.id, + reason: "manual", + }); + return { + binding: toBindingResponse(updatedBinding ?? binding), + workflowRunId: handle.workflowRun.id, + }; + } catch (error) { + await db + .update(repositoryGraphBindingsTable) + .set({ syncStatus: "failed", syncErrorCode: "enqueue_failed" }) + .where(eq(repositoryGraphBindingsTable.id, binding.id)); + throw error; + } }, }) ); diff --git a/packages/connectors/src/__tests__/gitlab.test.ts b/packages/connectors/src/__tests__/gitlab.test.ts index 2fe0a2a6..c4795d52 100644 --- a/packages/connectors/src/__tests__/gitlab.test.ts +++ b/packages/connectors/src/__tests__/gitlab.test.ts @@ -56,6 +56,19 @@ describe("GitLab connector", () => { expect(urls).toContain("https://gitlab.test/api/v4/projects/7/repository/branches?per_page=100&page=1"); }); + test("throws typed provider errors for non-json GitLab error responses", async () => { + const client = createGitLabClient({ + baseUrl: "https://gitlab.test", + accessToken: "token", + fetch: async () => new Response("upstream unavailable", { status: 500 }), + }); + + await expect(client.listBranches(GITLAB_REPOSITORY)).rejects.toMatchObject({ + name: "ConnectorProviderError", + kind: "provider", + }); + }); + test("loads supported repository snapshots through tree and file APIs", async () => { const calls: string[] = []; const fetchImpl: FetchLike = async (input) => { diff --git a/packages/connectors/src/gitlab.ts b/packages/connectors/src/gitlab.ts index 0239149c..f22fd3dc 100644 --- a/packages/connectors/src/gitlab.ts +++ b/packages/connectors/src/gitlab.ts @@ -306,14 +306,13 @@ async function getGitLabJson(url: string, token: string, fetchImpl: FetchLike | headers: { authorization: `Bearer ${token}`, accept: "application/json" }, }); const text = await response.text(); - const json = text.length === 0 ? null : JSON.parse(text); if (!response.ok) { throw new ConnectorProviderError( response.status === 404 ? "not-found" : "provider", "GitLab API request failed" ); } - return json; + return text.length === 0 ? null : JSON.parse(text); } async function getGitLabText(url: string, token: string, fetchImpl: FetchLike | undefined): Promise { From ff5b9d421854eda1dd0deb3f1c757b51cd6f2984 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 14:53:17 +0200 Subject: [PATCH 15/23] Enforce connector ownership for installation actions - Reject repository, branch, and graph requests when an installation belongs to another connector - Normalize connector error messages before mapping them to API responses - Add tests for the new 403 checks --- .../src/routes/__tests__/connectors.test.ts | 166 ++++++++++++++---- apps/api/src/routes/connectors.ts | 21 ++- 2 files changed, 144 insertions(+), 43 deletions(-) diff --git a/apps/api/src/routes/__tests__/connectors.test.ts b/apps/api/src/routes/__tests__/connectors.test.ts index 5d3070ce..02c6e880 100644 --- a/apps/api/src/routes/__tests__/connectors.test.ts +++ b/apps/api/src/routes/__tests__/connectors.test.ts @@ -18,9 +18,36 @@ const connector = { const insertValues: Array> = []; const conflictConfigs: Array> = []; const installationAccountCalls: string[] = []; +const listBranchesCalls: Array> = []; +const listRepositoriesCalls: Array> = []; const updateValues: Array> = []; +let installationConnectorId = "connector-1"; let workflowError: Error | null = null; +type MockDb = { + insert: () => { + values: (values: Record) => { + onConflictDoUpdate: (config: Record) => { + returning: () => Array>; + }; + returning: () => Array>; + }; + }; + update: () => { + set: (values: Record) => { + where: () => { + returning: () => never[]; + }; + }; + }; + select: () => { + from: () => { + orderBy: () => never[]; + }; + }; + transaction: (callback: (tx: MockDb) => unknown) => Promise; +}; + mock.module("../../middleware/auth", () => ({ authMiddleware: new Elysia().derive({ as: "scoped" }, () => ({ user: authUser, @@ -34,9 +61,15 @@ mock.module("../../lib/graph-access", () => ({ mock.module("../../lib/connector-access", () => ({ assertCanSyncBinding: async () => ({ binding: { id: "binding-1" } }), - assertCanUseInstallation: async () => ({ id: "installation-1", provider: "github" }), + assertCanUseInstallation: async () => ({ + id: "installation-1", + connectorId: installationConnectorId, + provider: "github", + organizationId: "org-1", + teamId: null, + }), assertCanViewBinding: async () => ({ binding: { id: "binding-1" } }), - requireActiveConnector: async () => connector, + requireActiveConnector: async (id: string) => ({ ...connector, id }), })); mock.module("../../lib/connectors", () => ({ @@ -54,8 +87,24 @@ mock.module("../../lib/connectors", () => ({ repositorySelection: "selected", }; }, - listProviderBranches: async () => [], - listProviderRepositories: async () => [], + listProviderBranches: async (_connector: unknown, _installation: unknown, repositoryId: string) => { + listBranchesCalls.push({ repositoryId }); + return [{ name: "main", commitSha: "commit-sha" }]; + }, + listProviderRepositories: async (connector: { id: string }, installation: { id: string }) => { + listRepositoriesCalls.push({ connectorId: connector.id, installationId: installation.id }); + return [ + { + id: "repo-1", + provider: "github", + fullName: "acme/app", + name: "app", + htmlUrl: "https://github.com/acme/app", + defaultBranch: "main", + private: true, + }, + ]; + }, signConnectorState: () => "state", toPublicConnector: (row: unknown) => row, toPublicInstallation: (row: unknown) => row, @@ -79,39 +128,40 @@ mock.module("../../openworkflow", () => ({ }, })); -mock.module("@kiwi/db", () => ({ - db: { - insert: () => ({ - values: (values: Record) => { - insertValues.push(values); - return { - onConflictDoUpdate: (config: Record) => { - conflictConfigs.push(config); - return { - returning: () => [{ id: "installation-1", ...values, status: "active" }], - }; - }, - returning: () => [{ id: "row-1", ...values }], - }; - }, - }), - update: () => ({ - set: (values: Record) => { - updateValues.push(values); - return { - where: () => ({ - returning: () => [], - }), - }; - }, - }), - select: () => ({ - from: () => ({ - orderBy: () => [], - }), +const mockDb: MockDb = { + insert: () => ({ + values: (values: Record) => { + insertValues.push(values); + return { + onConflictDoUpdate: (config: Record) => { + conflictConfigs.push(config); + return { + returning: () => [{ id: "installation-1", ...values, status: "active" }], + }; + }, + returning: () => [{ id: "row-1", ...values }], + }; + }, + }), + update: () => ({ + set: (values: Record) => { + updateValues.push(values); + return { + where: () => ({ + returning: () => [], + }), + }; + }, + }), + select: () => ({ + from: () => ({ + orderBy: () => [], }), - }, -})); + }), + transaction: async (callback: (tx: typeof mockDb) => unknown) => callback(mockDb), +}; + +mock.module("@kiwi/db", () => ({ db: mockDb })); // Dynamic import is required so module mocks are installed before the route module is evaluated. const { connectorRoute, repositoryGraphBindingRoute } = await import("../connectors"); @@ -121,7 +171,10 @@ describe("connector route", () => { insertValues.length = 0; conflictConfigs.length = 0; installationAccountCalls.length = 0; + listBranchesCalls.length = 0; + listRepositoriesCalls.length = 0; updateValues.length = 0; + installationConnectorId = "connector-1"; workflowError = null; }); @@ -164,4 +217,45 @@ describe("connector route", () => { expect(updateValues).toContainEqual({ syncStatus: "pending", syncErrorCode: null }); expect(updateValues).toContainEqual({ syncStatus: "failed", syncErrorCode: "enqueue_failed" }); }); + + test("rejects repository listing when installation belongs to another connector", async () => { + const response = await connectorRoute.handle( + new Request("http://localhost/connectors/connector-2/repositories?installationId=installation-1") + ); + + expect(response.status).toBe(403); + expect(listRepositoriesCalls).toEqual([]); + }); + + test("rejects branch listing when installation belongs to another connector", async () => { + const response = await connectorRoute.handle( + new Request( + "http://localhost/connectors/connector-2/repositories/repo-1/branches?installationId=installation-1" + ) + ); + + expect(response.status).toBe(403); + expect(listBranchesCalls).toEqual([]); + }); + + test("rejects repository graph creation when installation belongs to another connector", async () => { + const response = await connectorRoute.handle( + new Request("http://localhost/connectors/connector-2/repository-graphs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + connectorInstallationId: "installation-1", + repositoryId: "repo-1", + repositoryFullName: "acme/app", + repositoryHtmlUrl: "https://github.com/acme/app", + branch: "main", + name: "App", + owner: { kind: "organization" }, + }), + }) + ); + + expect(response.status).toBe(403); + expect(listRepositoriesCalls).toEqual([]); + }); }); diff --git a/apps/api/src/routes/connectors.ts b/apps/api/src/routes/connectors.ts index a6d55f00..f9b1d7ca 100644 --- a/apps/api/src/routes/connectors.ts +++ b/apps/api/src/routes/connectors.ts @@ -91,19 +91,17 @@ function mapConnectorError(status: RouteStatus, error: unknown) { if (!(error instanceof Error)) { return status(500, errorResponse("Internal server error", API_ERROR_CODES.INTERNAL_SERVER_ERROR)); } - if (error.message === API_ERROR_CODES.UNAUTHORIZED) { + const message = error.message.replace(/^Unhandled exception: /u, ""); + if (message === API_ERROR_CODES.UNAUTHORIZED) { return status(401, errorResponse("Unauthorized", API_ERROR_CODES.UNAUTHORIZED)); } - if (error.message === API_ERROR_CODES.FORBIDDEN) { + if (message === API_ERROR_CODES.FORBIDDEN) { return status(403, errorResponse("Forbidden", API_ERROR_CODES.FORBIDDEN)); } - if (error.message === API_ERROR_CODES.GRAPH_NOT_FOUND) { + if (message === API_ERROR_CODES.GRAPH_NOT_FOUND) { return status(404, errorResponse("Not found", API_ERROR_CODES.GRAPH_NOT_FOUND)); } - return status( - 400, - errorResponse(error.message || "Invalid connector request", API_ERROR_CODES.INVALID_CHAT_REQUEST) - ); + return status(400, errorResponse(message || "Invalid connector request", API_ERROR_CODES.INVALID_CHAT_REQUEST)); } async function runConnectorAction(options: { @@ -131,6 +129,12 @@ function repositoryMatches(repository: { id: string; fullName: string }, request return repository.id === requestedId || repository.fullName === requestedId; } +function assertInstallationBelongsToConnector(installation: { connectorId: string }, connectorId: string) { + if (installation.connectorId !== connectorId) { + throw new Error(API_ERROR_CODES.FORBIDDEN); + } +} + function toBindingResponse(binding: typeof repositoryGraphBindingsTable.$inferSelect) { return { id: binding.id, @@ -410,6 +414,7 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) status, action: async (currentUser) => { const installation = await assertCanUseInstallation(currentUser, query.installationId); + assertInstallationBelongsToConnector(installation, params.id); const connector = await requireActiveConnector( params.id, installation.provider as ConnectorProvider @@ -436,6 +441,7 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) status, action: async (currentUser) => { const installation = await assertCanUseInstallation(currentUser, query.installationId); + assertInstallationBelongsToConnector(installation, params.id); const connector = await requireActiveConnector( params.id, installation.provider as ConnectorProvider @@ -454,6 +460,7 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) status, action: async (currentUser) => { const installation = await assertCanUseInstallation(currentUser, body.connectorInstallationId); + assertInstallationBelongsToConnector(installation, params.id); if (body.owner.kind === "team") { if (installation.teamId !== body.owner.teamId) { throw new Error(API_ERROR_CODES.FORBIDDEN); From 7fac1c0c0296ed3c22440bb2db618ae3823989e8 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 15:15:37 +0200 Subject: [PATCH 16/23] Add direct repository lookup to connector clients - Add `getRepository` to GitHub and GitLab clients - Use direct repository fetches for branch and graph sync flows - Move credential helpers into the credentials module and update tests --- apps/api/src/lib/__tests__/connectors.test.ts | 90 +++++++++++++++++++ apps/api/src/lib/connectors.ts | 20 ++--- .../workflows/sync-repository-graph.test.ts | 10 ++- .../worker/workflows/sync-repository-graph.ts | 70 ++++++++------- packages/connectors/src/github.ts | 14 +++ packages/connectors/src/gitlab.ts | 16 ++++ packages/connectors/src/types.ts | 1 + 7 files changed, 176 insertions(+), 45 deletions(-) create mode 100644 apps/api/src/lib/__tests__/connectors.test.ts diff --git a/apps/api/src/lib/__tests__/connectors.test.ts b/apps/api/src/lib/__tests__/connectors.test.ts new file mode 100644 index 00000000..f46a3fbc --- /dev/null +++ b/apps/api/src/lib/__tests__/connectors.test.ts @@ -0,0 +1,90 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; + +type TestRepository = { + provider: "github"; + id: string; + fullName: string; + name: string; + htmlUrl: string; + defaultBranch: string; + private: boolean; +}; + +const branchRepositoryCalls: TestRepository[] = []; +let listRepositoriesCalls = 0; + +mock.module("@kiwi/connectors", () => ({ + createGitHubClient: () => ({ + provider: "github", + getRepository: async (repositoryId: string) => ({ + provider: "github", + id: "1", + fullName: repositoryId, + name: "app", + htmlUrl: `https://github.com/${repositoryId}`, + defaultBranch: "main", + private: true, + }), + listRepositories: async () => { + listRepositoriesCalls += 1; + return []; + }, + listBranches: async (repository: TestRepository) => { + branchRepositoryCalls.push(repository); + return [{ name: "main", commitSha: "commit-sha" }]; + }, + }), + createGitHubInstallationToken: async () => ({ token: "installation-token", expiresAt: "2026-01-01T01:00:00Z" }), + createGitLabClient: () => { + throw new Error("GitLab client was not expected"); + }, + getGitHubInstallationAccount: async () => ({ + login: "acme", + type: "organization", + repositorySelection: "selected", + }), +})); + +mock.module("@kiwi/connectors/credentials", () => ({ + decryptConnectorCredentials: () => ({ provider: "github", appId: "app-1", privateKeyPem: "pem" }), + decryptConnectorSecret: () => "secret", + encryptConnectorCredentials: () => "encrypted", + encryptConnectorSecret: () => "encrypted-secret", +})); + +mock.module("../env", () => ({ + env: { + AUTH_SECRET: "test-secret", + API_URL: "http://localhost:4321", + TRUSTED_ORIGINS: "http://localhost:3000", + }, +})); + +const { listProviderBranches } = await import("../connectors"); + +describe("connector library helpers", () => { + beforeEach(() => { + branchRepositoryCalls.length = 0; + listRepositoriesCalls = 0; + }); + + test("loads branches through direct repository lookup", async () => { + await expect( + listProviderBranches( + { + id: "connector-1", + provider: "github", + encryptedCredentials: "encrypted", + } as never, + { + id: "installation-1", + providerInstallationId: "99", + } as never, + "acme/app" + ) + ).resolves.toEqual([{ name: "main", commitSha: "commit-sha" }]); + + expect(listRepositoriesCalls).toBe(0); + expect(branchRepositoryCalls[0]).toMatchObject({ fullName: "acme/app" }); + }); +}); diff --git a/apps/api/src/lib/connectors.ts b/apps/api/src/lib/connectors.ts index 4c3cadf6..96c32eeb 100644 --- a/apps/api/src/lib/connectors.ts +++ b/apps/api/src/lib/connectors.ts @@ -3,13 +3,8 @@ import { createGitHubClient, createGitHubInstallationToken, createGitLabClient, - decryptConnectorCredentials, - decryptConnectorSecret, - encryptConnectorCredentials, - encryptConnectorSecret, getGitHubInstallationAccount, type ConnectorProvider, - type ConnectorSecretPayload, type GitHubConnectorCredentials, type GitLabConnectorCredentials, type GitLabInstallationCredentials, @@ -18,6 +13,13 @@ import { type ProviderRepository, type ProviderRepositoryClient, } from "@kiwi/connectors"; +import { + decryptConnectorCredentials, + decryptConnectorSecret, + encryptConnectorCredentials, + encryptConnectorSecret, + type ConnectorSecretPayload, +} from "@kiwi/connectors/credentials"; import type { connectorsTable, connectorInstallationsTable } from "@kiwi/db/tables/connectors"; import { env } from "../env"; @@ -249,12 +251,6 @@ export async function listProviderBranches( repositoryId: string ): Promise { const client = await createProviderClient(connector, installation); - const repositories = await client.listRepositories(); - const repository = repositories.find( - (candidate) => candidate.id === repositoryId || candidate.fullName === repositoryId - ); - if (!repository) { - throw new Error("Repository not found"); - } + const repository = await client.getRepository(repositoryId); return client.listBranches(repository); } diff --git a/apps/worker/workflows/sync-repository-graph.test.ts b/apps/worker/workflows/sync-repository-graph.test.ts index eec3866c..0595bc0c 100644 --- a/apps/worker/workflows/sync-repository-graph.test.ts +++ b/apps/worker/workflows/sync-repository-graph.test.ts @@ -184,10 +184,13 @@ mock.module("@kiwi/connectors", () => ({ createGitLabClient: () => { throw new Error("GitLab client was not expected"); }, - decryptConnectorCredentials: () => ({ provider: "github", appId: "app-1", privateKeyPem: "pem" }), normalizeGitLabBaseUrl: (value: string) => value.replace(/\/+$/, ""), })); +mock.module("@kiwi/connectors/credentials", () => ({ + decryptConnectorCredentials: () => ({ provider: "github", appId: "app-1", privateKeyPem: "pem" }), +})); + mock.module("../env", () => ({ env: { AUTH_SECRET: "test-secret", @@ -245,7 +248,10 @@ async function runWorkflow(input: { bindingId: string; reason: "manual" | "webho return syncRepositoryGraph.fn({ input, step: { - run: async (_config: { name: string }, fn: () => unknown) => fn(), + run: async (_config: { name: string }, fn: () => unknown) => { + const result = await fn(); + return result === undefined ? undefined : JSON.parse(JSON.stringify(result)); + }, runWorkflow: async (spec: { name: string }, workflowInput?: unknown) => { if (spec.name === "process-files") { processWorkflowInputs.push((workflowInput ?? {}) as Record); diff --git a/apps/worker/workflows/sync-repository-graph.ts b/apps/worker/workflows/sync-repository-graph.ts index 5bca4327..af4f7faa 100644 --- a/apps/worker/workflows/sync-repository-graph.ts +++ b/apps/worker/workflows/sync-repository-graph.ts @@ -6,11 +6,9 @@ import { createGitHubClient, createGitHubInstallationToken, createGitLabClient, - decryptConnectorCredentials, normalizeGitLabBaseUrl, } from "@kiwi/connectors"; import type { - ConnectorSecretPayload, GitHubConnectorCredentials, GitLabConnectorCredentials, GitLabInstallationCredentials, @@ -20,6 +18,7 @@ import type { ProviderRepositoryClient, ProviderRepositorySnapshot, } from "@kiwi/connectors"; +import { decryptConnectorCredentials, type ConnectorSecretPayload } from "@kiwi/connectors/credentials"; import { db } from "@kiwi/db"; import { connectorInstallationsTable, @@ -148,15 +147,12 @@ async function createProviderClient(row: BindingGraphRow): Promise { +async function resolveTargetCommitSha(row: BindingGraphRow, inputCommitSha?: string): Promise { if (inputCommitSha) { return inputCommitSha; } + const { client } = await createProviderClient(row); const branch = (await client.listBranches(repositoryFromBinding(row))).find( (candidate) => candidate.name === row.binding.branch ); @@ -167,11 +163,8 @@ async function resolveTargetCommitSha( return branch.commitSha; } -async function loadSnapshot( - row: BindingGraphRow, - client: ProviderRepositoryClient, - commitSha: string -): Promise { +async function loadSnapshot(row: BindingGraphRow, commitSha: string): Promise { + const { client } = await createProviderClient(row); return client.loadRepositorySnapshot( repositoryFromBinding(row), row.binding.branch, @@ -179,7 +172,28 @@ async function loadSnapshot( ) as Promise; } -async function loadActiveBindingFiles(bindingId: string): Promise> { +async function compareProviderCommits(row: BindingGraphRow, fromCommitSha: string, toCommitSha: string) { + const { client } = await createProviderClient(row); + return client.compareRepository(repositoryFromBinding(row), fromCommitSha, toCommitSha); +} + +async function loadChangedFiles(row: BindingGraphRow, commitSha: string, paths: string[]): Promise { + const context = await createProviderClient(row); + const repository = repositoryFromBinding(row); + return Promise.all( + paths.map(async (path) => + buildConnectorFile( + row, + context, + commitSha, + path, + await context.client.readFile(repository, path, commitSha) + ) + ) + ); +} + +async function loadActiveBindingFiles(bindingId: string): Promise { const rows = await db .select({ id: filesTable.id, @@ -211,7 +225,11 @@ async function loadActiveBindingFiles(bindingId: string): Promise { + return new Map(files.map((file) => [file.path, file])); } function planIncrementalChanges( @@ -481,9 +499,8 @@ export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async return { skipped: true }; } - const context = await step.run({ name: "create-provider-client" }, async () => createProviderClient(row)); const commitSha = await step.run({ name: "resolve-target-commit" }, async () => - resolveTargetCommitSha(row, context.client, input.commitSha) + resolveTargetCommitSha(row, input.commitSha) ); if (row.binding.lastSyncedCommitSha === commitSha) { @@ -497,7 +514,7 @@ export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async if (!row.binding.lastSyncedCommitSha) { const snapshot = await step.run({ name: "load-provider-snapshot" }, async () => - loadSnapshot(row, context.client, commitSha) + loadSnapshot(row, commitSha) ); if (snapshot.files.length === 0) { await step.run({ name: "mark-empty-binding-synced" }, async () => @@ -532,15 +549,16 @@ export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async return { commitSha, fileCount: created.fileIds.length }; } - const activeFiles = await step.run({ name: "load-active-binding-files" }, async () => + const activeFileRows = await step.run({ name: "load-active-binding-files" }, async () => loadActiveBindingFiles(row.binding.id) ); + const activeFiles = activeFilesByPath(activeFileRows); const delta = await step.run({ name: "compare-provider-commits" }, async () => - context.client.compareRepository(repositoryFromBinding(row), row.binding.lastSyncedCommitSha!, commitSha) + compareProviderCommits(row, row.binding.lastSyncedCommitSha!, commitSha) ); if (!delta.isIncremental) { const snapshot = await step.run({ name: "load-provider-snapshot" }, async () => - loadSnapshot(row, context.client, commitSha) + loadSnapshot(row, commitSha) ); const retiredFileIds = [...activeFiles.values()].map((file) => file.id); assertBindingSnapshotLimits(activeFiles, retiredFileIds, snapshot.files); @@ -593,17 +611,7 @@ export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async const changedFiles = plan.newPaths.length > 0 ? await step.run({ name: "load-changed-files" }, async () => - Promise.all( - plan.newPaths.map(async (path) => - buildConnectorFile( - row, - context, - commitSha, - path, - await context.client.readFile(repositoryFromBinding(row), path, commitSha) - ) - ) - ) + loadChangedFiles(row, commitSha, plan.newPaths) ) : []; assertBindingSnapshotLimits(activeFiles, plan.retiredFileIds, changedFiles); diff --git a/packages/connectors/src/github.ts b/packages/connectors/src/github.ts index befd40f3..c365a65b 100644 --- a/packages/connectors/src/github.ts +++ b/packages/connectors/src/github.ts @@ -131,6 +131,9 @@ export async function getGitHubInstallationAccount( export function createGitHubClient(options: GitHubClientOptions): ProviderRepositoryClient { return { provider: "github", + async getRepository(repositoryId) { + return getGitHubRepository(options, repositoryId); + }, async listRepositories() { return listGitHubInstallationRepositories(options); }, @@ -149,6 +152,17 @@ export function createGitHubClient(options: GitHubClientOptions): ProviderReposi }; } +export async function getGitHubRepository( + options: GitHubClientOptions, + repositoryId: string +): Promise { + const apiBaseUrl = normalizeApiBaseUrl(options.apiBaseUrl ?? GITHUB_API_BASE_URL); + const path = repositoryId.includes("/") + ? `/repos/${encodePathSegments(repositoryId)}` + : `/repositories/${encodeURIComponent(repositoryId)}`; + return mapGitHubRepository(await getGitHubJson(`${apiBaseUrl}${path}`, options.installationToken, options.fetch)); +} + export async function listGitHubInstallationRepositories(options: GitHubClientOptions): Promise { const repositories: ProviderRepository[] = []; for (let page = 1; ; page += 1) { diff --git a/packages/connectors/src/gitlab.ts b/packages/connectors/src/gitlab.ts index f22fd3dc..baa48e02 100644 --- a/packages/connectors/src/gitlab.ts +++ b/packages/connectors/src/gitlab.ts @@ -51,6 +51,9 @@ export function normalizeGitLabBaseUrl(value: string): string { export function createGitLabClient(options: GitLabClientOptions): ProviderRepositoryClient { return { provider: "gitlab", + async getRepository(repositoryId) { + return getGitLabProject(options, repositoryId); + }, async listRepositories() { return listGitLabProjects(options); }, @@ -69,6 +72,19 @@ export function createGitLabClient(options: GitLabClientOptions): ProviderReposi }; } +export async function getGitLabProject( + options: GitLabClientOptions, + repositoryId: string +): Promise { + return mapGitLabProject( + await getGitLabJson( + `${gitLabApiBase(options.baseUrl)}/projects/${encodeURIComponent(repositoryId)}`, + options.accessToken, + options.fetch + ) + ); +} + export async function listGitLabProjects(options: GitLabClientOptions): Promise { const repositories: ProviderRepository[] = []; for (let page = 1; ; page += 1) { diff --git a/packages/connectors/src/types.ts b/packages/connectors/src/types.ts index 5d4a5501..bfef98f8 100644 --- a/packages/connectors/src/types.ts +++ b/packages/connectors/src/types.ts @@ -105,6 +105,7 @@ export type NormalizedWebhookEvent = { export type ProviderRepositoryClient = { readonly provider: ConnectorProvider; + getRepository(repositoryId: string): Promise; listRepositories(): Promise; listBranches(repository: ProviderRepository): Promise; loadRepositorySnapshot( From 9a9da32898215efa44c11a39a2efc9d989a99ee2 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 15:20:41 +0200 Subject: [PATCH 17/23] Fix worker connector credential import --- apps/worker/lib/file-content-source.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/worker/lib/file-content-source.ts b/apps/worker/lib/file-content-source.ts index 0bc04714..ee153e8c 100644 --- a/apps/worker/lib/file-content-source.ts +++ b/apps/worker/lib/file-content-source.ts @@ -2,13 +2,12 @@ import { createGitHubClient, createGitHubInstallationToken, createGitLabClient, - decryptConnectorCredentials, - type ConnectorSecretPayload, type GitHubConnectorCredentials, type GitLabConnectorCredentials, type GitLabInstallationCredentials, type ProviderRepository, } from "@kiwi/connectors"; +import { decryptConnectorCredentials, type ConnectorSecretPayload } from "@kiwi/connectors/credentials"; import { db } from "@kiwi/db"; import { connectorInstallationsTable, connectorsTable, repositoryGraphBindingsTable } from "@kiwi/db/tables/connectors"; import { getFile } from "@kiwi/files"; From 7620aa3190082388cffee7dbb8c818ab7e605bda Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 15:36:10 +0200 Subject: [PATCH 18/23] Fix external file redirects and provider read concurrency --- .../lib/__tests__/graph-file-proxy.test.ts | 4 +- apps/api/src/lib/graph-file-proxy.ts | 2 +- .../workflows/sync-repository-graph.test.ts | 57 +++++++++++++++++++ .../worker/workflows/sync-repository-graph.ts | 30 ++++++---- 4 files changed, 79 insertions(+), 14 deletions(-) diff --git a/apps/api/src/lib/__tests__/graph-file-proxy.test.ts b/apps/api/src/lib/__tests__/graph-file-proxy.test.ts index 1bc778e3..665f01a3 100644 --- a/apps/api/src/lib/__tests__/graph-file-proxy.test.ts +++ b/apps/api/src/lib/__tests__/graph-file-proxy.test.ts @@ -90,7 +90,7 @@ describe("graph file proxy", () => { } }); - test("redirects external GitHub files to immutable HTML URLs without S3 calls", async () => { + test("redirects external GitHub files to immutable raw URLs without S3 calls", async () => { selectedFile = { key: "external:github:acme/widgets@commit-1:src/index.ts", name: "widgets/src/index.ts", @@ -126,7 +126,7 @@ describe("graph file proxy", () => { if (result.status === "ok") { expect(result.response.status).toBe(307); expect(result.response.headers.get("location")).toBe( - "https://github.com/acme/widgets/blob/commit-1/src/index.ts" + "https://raw.githubusercontent.com/acme/widgets/commit-1/src/index.ts" ); } }); diff --git a/apps/api/src/lib/graph-file-proxy.ts b/apps/api/src/lib/graph-file-proxy.ts index 1a5902fa..3c7ea614 100644 --- a/apps/api/src/lib/graph-file-proxy.ts +++ b/apps/api/src/lib/graph-file-proxy.ts @@ -119,7 +119,7 @@ export async function getGraphFileProxyResponse(options: { status: 307, headers: { "Cache-Control": "private, no-cache", - Location: metadata.external.htmlUrl, + Location: file.externalUrl, "X-Content-Type-Options": "nosniff", }, }), diff --git a/apps/worker/workflows/sync-repository-graph.test.ts b/apps/worker/workflows/sync-repository-graph.test.ts index 0595bc0c..fef1857e 100644 --- a/apps/worker/workflows/sync-repository-graph.test.ts +++ b/apps/worker/workflows/sync-repository-graph.test.ts @@ -10,6 +10,7 @@ const bindingUpdates: Array> = []; const compareCalls: Array<{ fromCommitSha: string; toCommitSha: string }> = []; const readFileCalls: Array<{ path: string; commitSha: string }> = []; const txWhereConditions: unknown[] = []; +const pendingReadResolutions: Array<() => void> = []; let selectResults: SelectResult[] = []; let txSelectResults: unknown[][] = []; @@ -21,6 +22,9 @@ let compareIsIncremental = true; let snapshotFiles: ProviderCodeFile[] = []; let readFileContents: Record = {}; let loadSnapshotCalls = 0; +let activeReadFileCalls = 0; +let holdReadFiles = false; +let maxReadFileConcurrency = 0; function createSelectQuery() { const result = selectResults.shift(); @@ -173,6 +177,14 @@ mock.module("@kiwi/connectors", () => ({ }, readFile: async (_repository: unknown, path: string, commitSha: string) => { readFileCalls.push({ path, commitSha }); + activeReadFileCalls += 1; + maxReadFileConcurrency = Math.max(maxReadFileConcurrency, activeReadFileCalls); + if (holdReadFiles) { + await new Promise((resolve) => { + pendingReadResolutions.push(resolve); + }); + } + activeReadFileCalls -= 1; const content = readFileContents[path]; if (!content) { throw new Error(`Missing mocked content for ${path}`); @@ -292,6 +304,7 @@ describe("syncRepositoryGraph", () => { compareCalls.length = 0; readFileCalls.length = 0; txWhereConditions.length = 0; + pendingReadResolutions.length = 0; selectResults = []; processFilesError = null; compareChanges = []; @@ -299,6 +312,9 @@ describe("syncRepositoryGraph", () => { snapshotFiles = []; readFileContents = {}; loadSnapshotCalls = 0; + activeReadFileCalls = 0; + holdReadFiles = false; + maxReadFileConcurrency = 0; txSelectResults = []; insertedFileRowsOverride = null; processRunInsertCount = 0; @@ -336,6 +352,47 @@ describe("syncRepositoryGraph", () => { expect(String(insertedFileValues[0]?.checksum)).toStartWith("commit-new:src/index.ts:"); }); + test("limits concurrent provider reads for incremental changed files", async () => { + const changedPaths = Array.from({ length: 10 }, (_, index) => `src/file-${index}.ts`); + compareChanges = changedPaths.map((newPath) => ({ status: "modified", newPath })); + readFileContents = Object.fromEntries(changedPaths.map((path) => [path, `export const value = "${path}";\n`])); + holdReadFiles = true; + selectResults = [ + { kind: "limit", value: [bindingRow("commit-old")] }, + { + kind: "where", + value: changedPaths.map((path, index) => activeFile(`old-file-${index}`, "commit-old", path, 25)), + }, + ]; + + let workflowDone = false; + const resultPromise = runWorkflow({ + bindingId: "binding-1", + reason: "manual", + commitSha: "commit-new", + }).finally(() => { + workflowDone = true; + }); + + while (pendingReadResolutions.length < 4) { + await new Promise((resolve) => setTimeout(resolve, 0)); + } + expect(maxReadFileConcurrency).toBeLessThanOrEqual(4); + + while (!workflowDone) { + while (pendingReadResolutions.length === 0 && !workflowDone) { + await new Promise((resolve) => setTimeout(resolve, 0)); + } + pendingReadResolutions.splice(0).forEach((resolve) => resolve()); + await new Promise((resolve) => setTimeout(resolve, 0)); + } + + const result = await resultPromise; + + expect(result).toMatchObject({ commitSha: "commit-new", fileCount: 10 }); + expect(maxReadFileConcurrency).toBeLessThanOrEqual(4); + }); + test("reuses existing repository files and process run when file insert conflicts", async () => { compareChanges = [{ status: "modified", newPath: "src/index.ts" }]; readFileContents = { diff --git a/apps/worker/workflows/sync-repository-graph.ts b/apps/worker/workflows/sync-repository-graph.ts index af4f7faa..52386cb4 100644 --- a/apps/worker/workflows/sync-repository-graph.ts +++ b/apps/worker/workflows/sync-repository-graph.ts @@ -69,6 +69,7 @@ type InsertedRepositoryFiles = { }; const NO_RETRY = { maximumAttempts: 1 } as const; +const PROVIDER_FILE_READ_CONCURRENCY = 4; async function loadBindingGraph(bindingId: string): Promise { const [row] = await db @@ -180,17 +181,24 @@ async function compareProviderCommits(row: BindingGraphRow, fromCommitSha: strin async function loadChangedFiles(row: BindingGraphRow, commitSha: string, paths: string[]): Promise { const context = await createProviderClient(row); const repository = repositoryFromBinding(row); - return Promise.all( - paths.map(async (path) => - buildConnectorFile( - row, - context, - commitSha, - path, - await context.client.readFile(repository, path, commitSha) - ) - ) - ); + const files: ProviderCodeFile[] = []; + for (let index = 0; index < paths.length; index += PROVIDER_FILE_READ_CONCURRENCY) { + const batch = paths.slice(index, index + PROVIDER_FILE_READ_CONCURRENCY); + files.push( + ...(await Promise.all( + batch.map(async (path) => + buildConnectorFile( + row, + context, + commitSha, + path, + await context.client.readFile(repository, path, commitSha) + ) + ) + )) + ); + } + return files; } async function loadActiveBindingFiles(bindingId: string): Promise { From 05f56c828c1d9afaf770e5fbbf8c56d7ce15dda1 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 15:50:02 +0200 Subject: [PATCH 19/23] Fix repository URL refresh supersession --- .../graph-archive-upload-route.test.ts | 114 +++++++++++++----- apps/api/src/routes/graph.ts | 9 +- 2 files changed, 91 insertions(+), 32 deletions(-) diff --git a/apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts b/apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts index b18bca20..70eb97aa 100644 --- a/apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts +++ b/apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts @@ -7,7 +7,7 @@ const workflowInputs: Array<{ graphId: string; fileIds: string[]; processRunId: string; - code?: { kind: "repository" }; + code?: { kind: "repository"; retiredFileIds?: string[] }; }> = []; const loadedUrls: string[] = []; const insertedFileValues: Array<{ @@ -22,6 +22,7 @@ const insertedFileValues: Array<{ checksum?: string; }> = []; const existingChecksumRows: Array<{ checksum: string }> = []; +const supersededFileIds: string[] = []; let repositoryLoadMode: "success" | "limit-error" | "git-error" = "success"; class RepositoryUrlError extends Error { @@ -154,9 +155,10 @@ const transactionDb = { }), }), update: () => ({ - set: () => ({ + set: (values: Record) => ({ where: () => ({ - returning: () => [existingGraph], + returning: () => + values.deleted === true ? supersededFileIds.map((id) => ({ id })) : [existingGraph], }), }), }), @@ -267,15 +269,20 @@ mock.module("../../lib/repository-url", () => ({ repositoryUrl: string; commitSha: string; path: string; - }) => - repositoryUrl === "https://github.com/acme/widgets.git" - ? { - provider: "github", - rawUrl: `https://raw.githubusercontent.com/acme/widgets/${commitSha}/${path}`, - htmlUrl: `https://github.com/acme/widgets/blob/${commitSha}/${path}`, - key: `external:github:acme/widgets@${commitSha}:${path}`, - } - : null, + }) => { + const match = repositoryUrl.match(/^https:\/\/github\.com\/acme\/([^/]+)\.git$/); + if (!match) { + return null; + } + + const repositoryName = match[1]; + return { + provider: "github", + rawUrl: `https://raw.githubusercontent.com/acme/${repositoryName}/${commitSha}/${path}`, + htmlUrl: `https://github.com/acme/${repositoryName}/blob/${commitSha}/${path}`, + key: `external:github:acme/${repositoryName}@${commitSha}:${path}`, + }; + }, loadRepositoryFromUrl: async (url: string) => { loadedUrls.push(url); if (repositoryLoadMode === "limit-error") { @@ -287,22 +294,32 @@ mock.module("../../lib/repository-url", () => ({ }); } + const repositoryName = url.includes("tools") ? "tools" : "widgets"; return { - url: "https://github.com/acme/widgets.git", - name: "widgets", + url: `https://github.com/acme/${repositoryName}.git`, + name: repositoryName, commitSha: "commit-1", - files: [ - { - path: "src/index.ts", - content: "import { helper } from './helper';\nexport function main() { return helper(); }\n", - size: 75, - }, - { - path: "src/helper.ts", - content: "export function helper() { return 1; }\n", - size: 38, - }, - ], + files: + repositoryName === "tools" + ? [ + { + path: "src/index.ts", + content: "import { helper } from './helper';\nexport function main() { return helper(); }\n", + size: 75, + }, + ] + : [ + { + path: "src/index.ts", + content: "import { helper } from './helper';\nexport function main() { return helper(); }\n", + size: 75, + }, + { + path: "src/helper.ts", + content: "export function helper() { return 1; }\n", + size: 38, + }, + ], }; }, })); @@ -365,6 +382,7 @@ describe("graph route archive uploads", () => { loadedUrls.length = 0; insertedFileValues.length = 0; existingChecksumRows.length = 0; + supersededFileIds.length = 0; archiveExpansionMode = "success"; repositoryLoadMode = "success"; uploadModelMode = "success"; @@ -532,11 +550,51 @@ describe("graph route archive uploads", () => { graphId: "graph-1", fileIds: body.data.addedFiles.map((file: { id: string }) => file.id), processRunId: "process-run-1", - code: { kind: "repository" }, + code: { kind: "repository", retiredFileIds: [] }, }, ]); }); + test("passes superseded repository file ids to the processing workflow", async () => { + supersededFileIds.push("old-file-1", "old-file-2"); + + const response = await app().handle( + new Request("http://localhost/graphs/graph-1/urls", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ urls: ["https://github.com/acme/widgets"] }), + }) + ); + + expect(response.status).toBe(200); + expect(workflowInputs).toHaveLength(1); + expect(workflowInputs[0]?.code).toEqual({ + kind: "repository", + retiredFileIds: ["old-file-1", "old-file-2"], + }); + }); + + test("keeps identical repository snapshot content once per repository", async () => { + const response = await app().handle( + new Request("http://localhost/graphs/graph-1/urls", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + urls: ["https://github.com/acme/widgets", "https://github.com/acme/tools"], + }), + }) + ); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.status).toBe("success"); + expect(insertedFileValues.map((file) => file.name)).toEqual([ + "widgets/src/index.ts", + "widgets/src/helper.ts", + "tools/src/index.ts", + ]); + }); + test("validates repository URL models before upload or workflow side effects", async () => { uploadModelMode = "missing"; @@ -580,7 +638,7 @@ describe("graph route archive uploads", () => { "widgets/src/helper.ts", ]); expect(uploadedFiles).toEqual([]); - expect(workflowInputs[0]?.code).toEqual({ kind: "repository" }); + expect(workflowInputs[0]?.code).toEqual({ kind: "repository", retiredFileIds: [] }); }); test("still enqueues repository workflow when every URL file matches an older checksum", async () => { diff --git a/apps/api/src/routes/graph.ts b/apps/api/src/routes/graph.ts index 863eeec4..0e3eee9a 100644 --- a/apps/api/src/routes/graph.ts +++ b/apps/api/src/routes/graph.ts @@ -627,17 +627,18 @@ export const graphRoute = new Elysia({ prefix: "/graphs" }) } const repositories = repositoriesResult.value; - const seenSnapshotChecksums = new Set(); + const seenSnapshotFiles = new Set(); const repositorySources: RepositoryUploadSource[] = []; for (const repository of repositories) { for (const file of repository.files) { const checksum = await contentChecksum(file.content); - if (seenSnapshotChecksums.has(checksum)) { + const snapshotFileKey = `${repository.url}:${checksum}`; + if (seenSnapshotFiles.has(snapshotFileKey)) { continue; } - seenSnapshotChecksums.add(checksum); + seenSnapshotFiles.add(snapshotFileKey); repositorySources.push({ repository, file, checksum }); } } @@ -800,7 +801,7 @@ export const graphRoute = new Elysia({ prefix: "/graphs" }) graphId: existingGraph.id, fileIds: result.addedFiles.map((file) => file.id), processRunId: result.processRunId, - code: { kind: "repository" }, + code: { kind: "repository", retiredFileIds: result.supersededFileIds }, }); return status(200, { From 06229418f086daa8e8574bdf525b5454be68e804 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Mon, 15 Jun 2026 16:13:44 +0200 Subject: [PATCH 20/23] Enforce owner checks for connector installs and graph sync - require org admin or matching team access when signing and consuming connector state - avoid reprocessing completed graph runs and reuse exact matching runs when possible - add coverage for cross-org install callbacks and process run reuse --- .../src/routes/__tests__/connectors.test.ts | 94 +++++- apps/api/src/routes/connectors.ts | 42 ++- .../workflows/sync-repository-graph.test.ts | 48 ++- .../worker/workflows/sync-repository-graph.ts | 290 ++++++++++++------ 4 files changed, 358 insertions(+), 116 deletions(-) diff --git a/apps/api/src/routes/__tests__/connectors.test.ts b/apps/api/src/routes/__tests__/connectors.test.ts index 02c6e880..e3c63fdb 100644 --- a/apps/api/src/routes/__tests__/connectors.test.ts +++ b/apps/api/src/routes/__tests__/connectors.test.ts @@ -13,6 +13,7 @@ const connector = { provider: "github", status: "active", encryptedCredentials: "encrypted", + appSlug: "kiwi-app", }; const insertValues: Array> = []; @@ -21,7 +22,17 @@ const installationAccountCalls: string[] = []; const listBranchesCalls: Array> = []; const listRepositoriesCalls: Array> = []; const updateValues: Array> = []; +const signedStates: Array> = []; +const organizationAdminChecks: Array = []; let installationConnectorId = "connector-1"; +let teamAccessOrganizationId = "org-1"; +let verifiedState: Record = { + purpose: "github-installation", + userId: "user-1", + connectorId: "connector-1", + organizationId: "org-1", + createdAt: Date.now(), +}; let workflowError: Error | null = null; type MockDb = { @@ -55,8 +66,14 @@ mock.module("../../middleware/auth", () => ({ })); mock.module("../../lib/graph-access", () => ({ - assertCanCreateTeamGraph: async () => ({ team: { id: "team-1", organizationId: "org-1" } }), - assertCanCreateTopLevelGraph: async () => ({ organizationId: "org-1" }), + assertCanCreateTeamGraph: async () => ({ team: { id: "team-1", organizationId: teamAccessOrganizationId } }), +})); + +mock.module("../../lib/team-access", () => ({ + requireOrganizationAdmin: async (_user: unknown, organizationId?: string) => { + organizationAdminChecks.push(organizationId); + return { organizationId: organizationId ?? "org-1" }; + }, })); mock.module("../../lib/connector-access", () => ({ @@ -105,16 +122,13 @@ mock.module("../../lib/connectors", () => ({ }, ]; }, - signConnectorState: () => "state", + signConnectorState: (state: Record) => { + signedStates.push(state); + return "state"; + }, toPublicConnector: (row: unknown) => row, toPublicInstallation: (row: unknown) => row, - verifyConnectorState: () => ({ - purpose: "github-installation", - userId: "user-1", - connectorId: "connector-1", - organizationId: "org-1", - createdAt: Date.now(), - }), + verifyConnectorState: () => verifiedState, })); mock.module("../../openworkflow", () => ({ @@ -174,7 +188,17 @@ describe("connector route", () => { listBranchesCalls.length = 0; listRepositoriesCalls.length = 0; updateValues.length = 0; + signedStates.length = 0; + organizationAdminChecks.length = 0; installationConnectorId = "connector-1"; + teamAccessOrganizationId = "org-1"; + verifiedState = { + purpose: "github-installation", + userId: "user-1", + connectorId: "connector-1", + organizationId: "org-1", + createdAt: Date.now(), + }; workflowError = null; }); @@ -206,6 +230,56 @@ describe("connector route", () => { }); }); + test("connect state uses requested organization after admin check", async () => { + const response = await connectorRoute.handle( + new Request("http://localhost/connectors/connector-1/connect?organizationId=org-2") + ); + + expect(response.status).toBe(200); + expect(organizationAdminChecks).toEqual(["org-2"]); + expect(signedStates[0]).toMatchObject({ + purpose: "github-installation", + userId: "user-1", + connectorId: "connector-1", + organizationId: "org-2", + }); + expect(signedStates[0]).not.toHaveProperty("teamId"); + }); + + test("connect state derives organization from team access", async () => { + const response = await connectorRoute.handle( + new Request("http://localhost/connectors/connector-1/connect?teamId=team-1&organizationId=org-2") + ); + + expect(response.status).toBe(200); + expect(organizationAdminChecks).toEqual([]); + expect(signedStates[0]).toMatchObject({ + purpose: "github-installation", + userId: "user-1", + connectorId: "connector-1", + organizationId: "org-1", + teamId: "team-1", + }); + }); + + test("rejects install callback when team state crosses organization", async () => { + verifiedState = { + purpose: "github-installation", + userId: "user-1", + connectorId: "connector-1", + organizationId: "org-2", + teamId: "team-1", + createdAt: Date.now(), + }; + + const response = await connectorRoute.handle( + new Request("http://localhost/connectors/github/install/callback?state=state&installation_id=99") + ); + + expect(response.status).toBe(403); + expect(insertValues).toEqual([]); + }); + test("marks manual repository graph sync failed when workflow enqueue fails", async () => { workflowError = new Error("enqueue failed"); diff --git a/apps/api/src/routes/connectors.ts b/apps/api/src/routes/connectors.ts index f9b1d7ca..61b2518b 100644 --- a/apps/api/src/routes/connectors.ts +++ b/apps/api/src/routes/connectors.ts @@ -12,13 +12,14 @@ import { Result } from "better-result"; import { and, asc, eq, sql } from "drizzle-orm"; import Elysia from "elysia"; import z from "zod"; -import { assertCanCreateTeamGraph, assertCanCreateTopLevelGraph } from "../lib/graph-access"; +import { assertCanCreateTeamGraph } from "../lib/graph-access"; import { assertCanSyncBinding, assertCanUseInstallation, assertCanViewBinding, requireActiveConnector, } from "../lib/connector-access"; +import { requireOrganizationAdmin } from "../lib/team-access"; import { createManifestUrl, encryptCredentials, @@ -288,17 +289,20 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) status, action: async (currentUser) => { const connector = await requireActiveConnector(params.id); + let owner: { organizationId: string; teamId?: string }; if (query.teamId) { - await assertCanCreateTeamGraph(currentUser, query.teamId); + const access = await assertCanCreateTeamGraph(currentUser, query.teamId); + owner = { organizationId: access.team.organizationId, teamId: query.teamId }; } else { - await assertCanCreateTopLevelGraph(currentUser); + const membership = await requireOrganizationAdmin(currentUser, query.organizationId); + owner = { organizationId: membership.organizationId }; } const state = signConnectorState({ purpose: connector.provider === "github" ? "github-installation" : "gitlab-oauth", userId: currentUser.id, connectorId: connector.id, - organizationId: query.organizationId ?? currentUser.activeOrganizationId ?? undefined, - teamId: query.teamId, + organizationId: owner.organizationId, + ...(owner.teamId ? { teamId: owner.teamId } : {}), }); if (connector.provider === "github") { if (!connector.appSlug) { @@ -325,14 +329,20 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) throw new Error(API_ERROR_CODES.FORBIDDEN); } const connector = await requireActiveConnector(state.connectorId, "github"); + let ownerOrganizationId: string; + let ownerTeamId: string | null = null; if (state.teamId) { - await assertCanCreateTeamGraph(currentUser, state.teamId); + const access = await assertCanCreateTeamGraph(currentUser, state.teamId); + ownerOrganizationId = access.team.organizationId; + ownerTeamId = state.teamId; + if (state.organizationId && state.organizationId !== ownerOrganizationId) { + throw new Error(API_ERROR_CODES.FORBIDDEN); + } } else { - await assertCanCreateTopLevelGraph(currentUser); + const membership = await requireOrganizationAdmin(currentUser, state.organizationId); + ownerOrganizationId = membership.organizationId; } - const account = await getGitHubConnectorInstallationAccount(connector, query.installation_id); - const ownerOrganizationId = state.organizationId ?? currentUser.activeOrganizationId; - const conflictTarget = state.teamId + const conflictTarget = ownerTeamId ? { target: [ connectorInstallationsTable.connectorId, @@ -350,6 +360,7 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) ], targetWhere: sql`${connectorInstallationsTable.teamId} is null`, }; + const account = await getGitHubConnectorInstallationAccount(connector, query.installation_id); const [installation] = await db .insert(connectorInstallationsTable) .values({ @@ -359,7 +370,7 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) providerAccountLogin: account.login, providerAccountType: account.type, organizationId: ownerOrganizationId, - teamId: state.teamId ?? null, + teamId: ownerTeamId, installedByUserId: currentUser.id, repositorySelection: account.repositorySelection, }) @@ -465,12 +476,15 @@ export const connectorRoute = new Elysia({ prefix: "/connectors" }) if (installation.teamId !== body.owner.teamId) { throw new Error(API_ERROR_CODES.FORBIDDEN); } - await assertCanCreateTeamGraph(currentUser, body.owner.teamId); + const access = await assertCanCreateTeamGraph(currentUser, body.owner.teamId); + if (installation.organizationId !== access.team.organizationId) { + throw new Error(API_ERROR_CODES.FORBIDDEN); + } } else { - if (installation.teamId !== null) { + if (installation.teamId !== null || !installation.organizationId) { throw new Error(API_ERROR_CODES.FORBIDDEN); } - await assertCanCreateTopLevelGraph(currentUser); + await requireOrganizationAdmin(currentUser, installation.organizationId); } const connector = await requireActiveConnector( diff --git a/apps/worker/workflows/sync-repository-graph.test.ts b/apps/worker/workflows/sync-repository-graph.test.ts index fef1857e..e4bae293 100644 --- a/apps/worker/workflows/sync-repository-graph.test.ts +++ b/apps/worker/workflows/sync-repository-graph.test.ts @@ -408,7 +408,8 @@ describe("syncRepositoryGraph", () => { insertedFileRowsOverride = []; txSelectResults = [ [{ id: "existing-file", key: "connector:binding-1:commit-new:src/index.ts" }], - [{ id: "existing-process-run" }], + [{ id: "existing-process-run", status: "pending", fileId: "existing-file" }], + [{ processRunId: "existing-process-run", fileId: "existing-file" }], ]; const result = await runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" }); @@ -425,7 +426,7 @@ describe("syncRepositoryGraph", () => { ]); }); - test("does not reuse completed process runs when file insert conflicts", async () => { + test("skips processing when file insert conflicts with a completed process run", async () => { compareChanges = [{ status: "modified", newPath: "src/index.ts" }]; readFileContents = { "src/index.ts": "export const next = shared;\n", @@ -438,13 +439,52 @@ describe("syncRepositoryGraph", () => { }, ]; insertedFileRowsOverride = []; - txSelectResults = [[{ id: "existing-file", key: "connector:binding-1:commit-new:src/index.ts" }], []]; + txSelectResults = [ + [{ id: "existing-file", key: "connector:binding-1:commit-new:src/index.ts" }], + [{ id: "completed-process-run", status: "completed", fileId: "existing-file" }], + [{ processRunId: "completed-process-run", fileId: "existing-file" }], + ]; const result = await runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" }); expect(result).toMatchObject({ commitSha: "commit-new", fileCount: 1 }); - expect(processRunInsertCount).toBe(1); + expect(processRunInsertCount).toBe(0); expect(queryContainsParamValue(txWhereConditions[1], "completed")).toBe(true); + expect(processWorkflowInputs).toEqual([]); + expect(bindingUpdates).toContainEqual({ + syncStatus: "synced", + lastSeenCommitSha: "commit-new", + lastSyncedCommitSha: "commit-new", + syncErrorCode: null, + }); + }); + + test("creates a process run when matching run has extra files", async () => { + compareChanges = [{ status: "modified", newPath: "src/index.ts" }]; + readFileContents = { + "src/index.ts": "export const next = shared;\n", + }; + selectResults = [ + { kind: "limit", value: [bindingRow("commit-old")] }, + { + kind: "where", + value: [activeFile("old-file", "commit-old", "src/index.ts", 25)], + }, + ]; + insertedFileRowsOverride = []; + txSelectResults = [ + [{ id: "existing-file", key: "connector:binding-1:commit-new:src/index.ts" }], + [{ id: "larger-process-run", status: "completed", fileId: "existing-file" }], + [ + { processRunId: "larger-process-run", fileId: "existing-file" }, + { processRunId: "larger-process-run", fileId: "other-file" }, + ], + ]; + + const result = await runWorkflow({ bindingId: "binding-1", reason: "manual", commitSha: "commit-new" }); + + expect(result).toMatchObject({ commitSha: "commit-new", fileCount: 1 }); + expect(processRunInsertCount).toBe(1); expect(processWorkflowInputs).toEqual([ { graphId: "graph-1", diff --git a/apps/worker/workflows/sync-repository-graph.ts b/apps/worker/workflows/sync-repository-graph.ts index 52386cb4..4cf864cb 100644 --- a/apps/worker/workflows/sync-repository-graph.ts +++ b/apps/worker/workflows/sync-repository-graph.ts @@ -26,9 +26,15 @@ import { connectorWebhookEventsTable, repositoryGraphBindingsTable, } from "@kiwi/db/tables/connectors"; -import { filesTable, graphTable, processRunFilesTable, processRunsTable } from "@kiwi/db/tables/graph"; +import { + filesTable, + graphTable, + processRunFilesTable, + processRunsTable, + type ProcessRunStatus, +} from "@kiwi/db/tables/graph"; import { serializeCodeFileMetadata } from "@kiwi/graph/code/metadata"; -import { and, eq, inArray, ne } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { defineWorkflow } from "openworkflow"; import { parseCodeFileMetadata } from "../lib/code-file-metadata"; import { env } from "../env"; @@ -63,9 +69,12 @@ type IncrementalSyncPlan = { retiredFileIds: string[]; }; +type ReusableProcessRunStatus = Exclude; + type InsertedRepositoryFiles = { fileIds: string[]; processRunId: string; + processRunStatus: ReusableProcessRunStatus; }; const NO_RETRY = { maximumAttempts: 1 } as const; @@ -381,6 +390,115 @@ function orderedFilesByKey(rows: ReturnType, files: InsertedFil return orderedFiles; } +type ReusableProcessRun = { + id: string; + status: ReusableProcessRunStatus; +}; + +function isReusableProcessRunStatus(status: ProcessRunStatus): status is ReusableProcessRunStatus { + return status !== "failed"; +} + +function processRunStatusPriority(status: ReusableProcessRunStatus) { + switch (status) { + case "completed": + return 0; + case "started": + return 1; + case "pending": + return 2; + } +} + +async function findReusableProcessRun( + tx: Parameters[0]>[0], + graphId: string, + fileIds: string[] +): Promise { + if (fileIds.length === 0) { + return null; + } + + const matchingRunRows = await tx + .select({ + id: processRunFilesTable.processRunId, + status: processRunsTable.status, + fileId: processRunFilesTable.fileId, + }) + .from(processRunFilesTable) + .innerJoin(processRunsTable, eq(processRunFilesTable.processRunId, processRunsTable.id)) + .where( + and( + eq(processRunsTable.graphId, graphId), + inArray(processRunsTable.status, ["pending", "started", "completed"]), + inArray(processRunFilesTable.fileId, fileIds) + ) + ); + + const expectedFileIds = new Set(fileIds); + const candidates = new Map }>(); + for (const row of matchingRunRows) { + if (!isReusableProcessRunStatus(row.status)) { + continue; + } + const candidate = candidates.get(row.id); + if (candidate) { + candidate.matchedFileIds.add(row.fileId); + } else { + candidates.set(row.id, { + id: row.id, + status: row.status, + matchedFileIds: new Set([row.fileId]), + }); + } + } + + const completeCandidateIds = [...candidates.values()] + .filter((candidate) => candidate.matchedFileIds.size === expectedFileIds.size) + .map((candidate) => candidate.id); + if (completeCandidateIds.length === 0) { + return null; + } + + const runFileRows = await tx + .select({ processRunId: processRunFilesTable.processRunId, fileId: processRunFilesTable.fileId }) + .from(processRunFilesTable) + .where(inArray(processRunFilesTable.processRunId, completeCandidateIds)); + const fileIdsByRun = new Map>(); + for (const row of runFileRows) { + const runFileIds = fileIdsByRun.get(row.processRunId); + if (runFileIds) { + runFileIds.add(row.fileId); + } else { + fileIdsByRun.set(row.processRunId, new Set([row.fileId])); + } + } + + const exactCandidates: ReusableProcessRun[] = []; + for (const candidateId of completeCandidateIds) { + const runFileIds = fileIdsByRun.get(candidateId); + if (!runFileIds || runFileIds.size !== expectedFileIds.size) { + continue; + } + let exact = true; + for (const fileId of runFileIds) { + if (!expectedFileIds.has(fileId)) { + exact = false; + break; + } + } + if (exact) { + const candidate = candidates.get(candidateId); + if (candidate) { + exactCandidates.push({ id: candidate.id, status: candidate.status }); + } + } + } + + exactCandidates.sort((left, right) => processRunStatusPriority(left.status) - processRunStatusPriority(right.status)); + return exactCandidates[0] ?? null; +} + async function insertRepositoryFiles( row: BindingGraphRow, files: ProviderCodeFile[], @@ -419,47 +537,37 @@ async function insertRepositoryFiles( throw new Error("Failed to insert all repository files"); } - let processRunId: string | null = null; + const fileIds = committedFiles.map((file) => file.id); if (insertedFiles.length === 0) { - const existingProcessRuns = await tx - .select({ id: processRunFilesTable.processRunId }) - .from(processRunFilesTable) - .innerJoin(processRunsTable, eq(processRunFilesTable.processRunId, processRunsTable.id)) - .where( - and( - eq(processRunsTable.graphId, row.binding.graphId), - ne(processRunsTable.status, "completed"), - inArray( - processRunFilesTable.fileId, - committedFiles.map((file) => file.id) - ) - ) - ); - processRunId = existingProcessRuns[0]?.id ?? null; + const reusableProcessRun = await findReusableProcessRun(tx, row.binding.graphId, fileIds); + if (reusableProcessRun) { + return { + fileIds, + processRunId: reusableProcessRun.id, + processRunStatus: reusableProcessRun.status, + }; + } } - if (!processRunId) { - const [processRun] = await tx - .insert(processRunsTable) - .values({ graphId: row.binding.graphId, status: "pending" }) - .returning({ id: processRunsTable.id }); - if (!processRun) { - throw new Error("Failed to create process run"); - } - const newProcessRunId = processRun.id; - processRunId = newProcessRunId; - - await tx.insert(processRunFilesTable).values( - committedFiles.map((file) => ({ - processRunId: newProcessRunId, - fileId: file.id, - })) - ); + const [processRun] = await tx + .insert(processRunsTable) + .values({ graphId: row.binding.graphId, status: "pending" }) + .returning({ id: processRunsTable.id }); + if (!processRun) { + throw new Error("Failed to create process run"); } + await tx.insert(processRunFilesTable).values( + fileIds.map((fileId) => ({ + processRunId: processRun.id, + fileId, + })) + ); + return { - fileIds: committedFiles.map((file) => file.id), - processRunId, + fileIds, + processRunId: processRun.id, + processRunStatus: "pending", }; }); } @@ -534,23 +642,25 @@ export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async const created = await step.run({ name: "commit-external-files" }, async () => insertRepositoryFiles(row, snapshot.files, commitSha) ); - try { - await step.runWorkflow(processFilesSpec, { - graphId: row.binding.graphId, - fileIds: created.fileIds, - processRunId: created.processRunId, - code: { kind: "repository", retiredFileIds: [] }, - }); - } catch (error) { - await Promise.all( - created.fileIds.map((fileId) => - step.runWorkflow(deleteFileSpec, { - graphId: row.binding.graphId, - fileId, - }) - ) - ); - throw error; + if (created.processRunStatus !== "completed") { + try { + await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: created.fileIds, + processRunId: created.processRunId, + code: { kind: "repository", retiredFileIds: [] }, + }); + } catch (error) { + await Promise.all( + created.fileIds.map((fileId) => + step.runWorkflow(deleteFileSpec, { + graphId: row.binding.graphId, + fileId, + }) + ) + ); + throw error; + } } await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); @@ -575,23 +685,25 @@ export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async const created = await step.run({ name: "commit-external-files" }, async () => insertRepositoryFiles(row, snapshot.files, commitSha) ); - try { - await step.runWorkflow(processFilesSpec, { - graphId: row.binding.graphId, - fileIds: created.fileIds, - processRunId: created.processRunId, - code: { kind: "repository", retiredFileIds }, - }); - } catch (error) { - await Promise.all( - created.fileIds.map((fileId) => - step.runWorkflow(deleteFileSpec, { - graphId: row.binding.graphId, - fileId, - }) - ) - ); - throw error; + if (created.processRunStatus !== "completed") { + try { + await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: created.fileIds, + processRunId: created.processRunId, + code: { kind: "repository", retiredFileIds }, + }); + } catch (error) { + await Promise.all( + created.fileIds.map((fileId) => + step.runWorkflow(deleteFileSpec, { + graphId: row.binding.graphId, + fileId, + }) + ) + ); + throw error; + } } await step.run({ name: "mark-binding-synced" }, async () => @@ -628,23 +740,25 @@ export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async const created = await step.run({ name: "commit-external-files" }, async () => insertRepositoryFiles(row, changedFiles, commitSha) ); - try { - await step.runWorkflow(processFilesSpec, { - graphId: row.binding.graphId, - fileIds: created.fileIds, - processRunId: created.processRunId, - code: { kind: "repository", retiredFileIds: plan.retiredFileIds }, - }); - } catch (error) { - await Promise.all( - created.fileIds.map((fileId) => - step.runWorkflow(deleteFileSpec, { - graphId: row.binding.graphId, - fileId, - }) - ) - ); - throw error; + if (created.processRunStatus !== "completed") { + try { + await step.runWorkflow(processFilesSpec, { + graphId: row.binding.graphId, + fileIds: created.fileIds, + processRunId: created.processRunId, + code: { kind: "repository", retiredFileIds: plan.retiredFileIds }, + }); + } catch (error) { + await Promise.all( + created.fileIds.map((fileId) => + step.runWorkflow(deleteFileSpec, { + graphId: row.binding.graphId, + fileId, + }) + ) + ); + throw error; + } } await step.run({ name: "mark-binding-synced" }, async () => markBindingSynced(row.binding.id, commitSha)); return { commitSha, fileCount: created.fileIds.length }; From 86fb0ca7fb74e0d4dfcdad9ece426aa33fb87bb2 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Fri, 19 Jun 2026 12:14:23 +0200 Subject: [PATCH 21/23] fix(frontend): add code file type metadata --- .../SystemConfigurationSection.test.tsx | 21 +++++++++++++++---- .../sections/SystemConfigurationSection.tsx | 6 +++++- apps/frontend/messages/de.json | 4 +++- apps/frontend/messages/en.json | 4 +++- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/apps/frontend/components/settings/sections/SystemConfigurationSection.test.tsx b/apps/frontend/components/settings/sections/SystemConfigurationSection.test.tsx index c6f46912..b9d7eaed 100644 --- a/apps/frontend/components/settings/sections/SystemConfigurationSection.test.tsx +++ b/apps/frontend/components/settings/sections/SystemConfigurationSection.test.tsx @@ -56,6 +56,15 @@ const records: FileTypeConfigRecord[] = [ chunk_size_editable: true, document_mode_editable: false, }, + { + file_type: "code", + loader: "text", + chunker: "semantic", + chunk_size: 2000, + document_mode: null, + chunk_size_editable: true, + document_mode_editable: false, + }, ]; const fetchFileTypeConfigsMock = vi.mocked(fetchFileTypeConfigs); @@ -76,11 +85,15 @@ describe("SystemConfigurationSection", () => { renderSection(); expect(await screen.findByText("Dateiverarbeitung")).toBeInTheDocument(); - expect(await screen.findByText("3 Dateitypen geladen")).toBeInTheDocument(); + expect(await screen.findByText("4 Dateitypen geladen")).toBeInTheDocument(); expect(screen.getByText("Dokumente")).toBeInTheDocument(); expect(screen.getByText("Medien")).toBeInTheDocument(); - expect(screen.getByText("pdf")).toBeInTheDocument(); - expect(screen.getByText("semantic")).toBeInTheDocument(); + const pdfRow = screen.getByText("PDF").closest("[data-file-type='pdf']"); + expect(pdfRow).not.toBeNull(); + expect(within(pdfRow!).getByText("pdf")).toBeInTheDocument(); + expect(within(pdfRow!).getByText("semantic")).toBeInTheDocument(); + expect(screen.getByText("Code")).toBeInTheDocument(); + expect(screen.getByText(".js, .ts, .tsx, .rs, .zig, .c, .h")).toBeInTheDocument(); }); test("keeps save disabled until an editable field changes", async () => { @@ -190,7 +203,7 @@ describe("SystemConfigurationSection", () => { await user.click(screen.getByRole("button", { name: "Erneut versuchen" })); - expect(await screen.findByText("3 Dateitypen geladen")).toBeInTheDocument(); + expect(await screen.findByText("4 Dateitypen geladen")).toBeInTheDocument(); expect(fetchFileTypeConfigsMock).toHaveBeenCalledTimes(2); }); diff --git a/apps/frontend/components/settings/sections/SystemConfigurationSection.tsx b/apps/frontend/components/settings/sections/SystemConfigurationSection.tsx index a2d7ee08..7f2fc139 100644 --- a/apps/frontend/components/settings/sections/SystemConfigurationSection.tsx +++ b/apps/frontend/components/settings/sections/SystemConfigurationSection.tsx @@ -103,6 +103,10 @@ const FILE_TYPE_META: Record; diff --git a/apps/frontend/messages/de.json b/apps/frontend/messages/de.json index df392efa..5298b52f 100644 --- a/apps/frontend/messages/de.json +++ b/apps/frontend/messages/de.json @@ -470,7 +470,7 @@ "settings.systemConfig.fileProcessing.group.media": "Medien", "settings.systemConfig.fileProcessing.group.structured": "Strukturierte Daten", "settings.systemConfig.fileProcessing.group.communication": "Kommunikation & Web", - "settings.systemConfig.fileProcessing.group.text": "Text", + "settings.systemConfig.fileProcessing.group.text": "Text & Code", "settings.systemConfig.fileProcessing.fileType.pdf": "PDF", "settings.systemConfig.fileProcessing.fileType.doc": "DOC/DOCX", "settings.systemConfig.fileProcessing.fileType.sheet": "Tabellen", @@ -489,6 +489,7 @@ "settings.systemConfig.fileProcessing.fileType.xml": "XML", "settings.systemConfig.fileProcessing.fileType.yaml": "YAML", "settings.systemConfig.fileProcessing.fileType.toml": "TOML", + "settings.systemConfig.fileProcessing.fileType.code": "Code", "settings.systemConfig.fileProcessing.fileType.text": "Text", "settings.systemConfig.fileProcessing.extensions.pdf": ".pdf", "settings.systemConfig.fileProcessing.extensions.doc": ".doc, .docx", @@ -508,6 +509,7 @@ "settings.systemConfig.fileProcessing.extensions.xml": ".xml, .xsd", "settings.systemConfig.fileProcessing.extensions.yaml": ".yaml, .yml", "settings.systemConfig.fileProcessing.extensions.toml": ".toml", + "settings.systemConfig.fileProcessing.extensions.code": ".js, .ts, .tsx, .rs, .zig, .c, .h", "settings.systemConfig.fileProcessing.extensions.text": ".txt und Fallback", "settings.systemConfig.fileProcessing.notApplicable": "Nicht anwendbar", "settings.systemConfig.fileProcessing.chunkSize.aria": "Chunk-Größe für {type}", diff --git a/apps/frontend/messages/en.json b/apps/frontend/messages/en.json index ac0942ad..845d18ed 100644 --- a/apps/frontend/messages/en.json +++ b/apps/frontend/messages/en.json @@ -470,7 +470,7 @@ "settings.systemConfig.fileProcessing.group.media": "Media", "settings.systemConfig.fileProcessing.group.structured": "Structured data", "settings.systemConfig.fileProcessing.group.communication": "Communication & web", - "settings.systemConfig.fileProcessing.group.text": "Text", + "settings.systemConfig.fileProcessing.group.text": "Text & code", "settings.systemConfig.fileProcessing.fileType.pdf": "PDF", "settings.systemConfig.fileProcessing.fileType.doc": "DOC/DOCX", "settings.systemConfig.fileProcessing.fileType.sheet": "Spreadsheets", @@ -489,6 +489,7 @@ "settings.systemConfig.fileProcessing.fileType.xml": "XML", "settings.systemConfig.fileProcessing.fileType.yaml": "YAML", "settings.systemConfig.fileProcessing.fileType.toml": "TOML", + "settings.systemConfig.fileProcessing.fileType.code": "Code", "settings.systemConfig.fileProcessing.fileType.text": "Text", "settings.systemConfig.fileProcessing.extensions.pdf": ".pdf", "settings.systemConfig.fileProcessing.extensions.doc": ".doc, .docx", @@ -508,6 +509,7 @@ "settings.systemConfig.fileProcessing.extensions.xml": ".xml, .xsd", "settings.systemConfig.fileProcessing.extensions.yaml": ".yaml, .yml", "settings.systemConfig.fileProcessing.extensions.toml": ".toml", + "settings.systemConfig.fileProcessing.extensions.code": ".js, .ts, .tsx, .rs, .zig, .c, .h", "settings.systemConfig.fileProcessing.extensions.text": ".txt and fallback", "settings.systemConfig.fileProcessing.notApplicable": "Not applicable", "settings.systemConfig.fileProcessing.chunkSize.aria": "Chunk size for {type}", From e2e1fdf151f5987a082eaa3c8dd56c00a20928d8 Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Fri, 19 Jun 2026 12:30:23 +0200 Subject: [PATCH 22/23] fix(api): avoid repository invalidation on code retry --- .../graph-archive-upload-route.test.ts | 57 +++++++++++++++++-- apps/api/src/routes/graph.ts | 7 +-- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts b/apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts index 70eb97aa..ed51ad5b 100644 --- a/apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts +++ b/apps/api/src/routes/__tests__/graph-archive-upload-route.test.ts @@ -22,6 +22,13 @@ const insertedFileValues: Array<{ checksum?: string; }> = []; const existingChecksumRows: Array<{ checksum: string }> = []; +const retryFileRows: Array<{ + id: string; + type: string; + status: string; + processStep: string; + processErrorCode: string | null; +}> = []; const supersededFileIds: string[] = []; let repositoryLoadMode: "success" | "limit-error" | "git-error" = "success"; @@ -113,6 +120,16 @@ function insertReturning(values: unknown) { return undefined; } +function selectableRows(rows: TRows[], limitedRows: TLimitedRows[]) { + return { + limit: async (count: number) => limitedRows.slice(0, count), + then: ( + onfulfilled?: ((value: TRows[]) => TResult1 | PromiseLike) | null, + onrejected?: ((reason: unknown) => TResult2 | PromiseLike) | null + ) => Promise.resolve(rows).then(onfulfilled, onrejected), + }; +} + const db = { insert: () => ({ values: (values: unknown) => ({ @@ -121,7 +138,7 @@ const db = { }), select: () => ({ from: () => ({ - where: async () => existingChecksumRows, + where: () => selectableRows(existingChecksumRows, retryFileRows), }), }), transaction: async (callback: (tx: typeof transactionDb) => Promise) => callback(transactionDb), @@ -157,8 +174,7 @@ const transactionDb = { update: () => ({ set: (values: Record) => ({ where: () => ({ - returning: () => - values.deleted === true ? supersededFileIds.map((id) => ({ id })) : [existingGraph], + returning: () => (values.deleted === true ? supersededFileIds.map((id) => ({ id })) : [existingGraph]), }), }), }), @@ -304,14 +320,16 @@ mock.module("../../lib/repository-url", () => ({ ? [ { path: "src/index.ts", - content: "import { helper } from './helper';\nexport function main() { return helper(); }\n", + content: + "import { helper } from './helper';\nexport function main() { return helper(); }\n", size: 75, }, ] : [ { path: "src/index.ts", - content: "import { helper } from './helper';\nexport function main() { return helper(); }\n", + content: + "import { helper } from './helper';\nexport function main() { return helper(); }\n", size: 75, }, { @@ -383,6 +401,7 @@ describe("graph route archive uploads", () => { insertedFileValues.length = 0; existingChecksumRows.length = 0; supersededFileIds.length = 0; + retryFileRows.length = 0; archiveExpansionMode = "success"; repositoryLoadMode = "success"; uploadModelMode = "success"; @@ -671,6 +690,34 @@ describe("graph route archive uploads", () => { expect(workflowInputs).toHaveLength(1); }); + test("retries code files without retiring repository siblings", async () => { + retryFileRows.push({ + id: "file-code", + type: "code", + status: "failed", + processStep: "failed", + processErrorCode: "loader_failed", + }); + + const response = await app().handle( + new Request("http://localhost/graphs/graph-1/files/file-code/retry", { + method: "POST", + }) + ); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.status).toBe("success"); + expect(workflowInputs).toEqual([ + { + graphId: "graph-1", + fileIds: ["file-code"], + processRunId: "process-run-1", + code: { kind: "repository", retiredFileIds: [] }, + }, + ]); + }); + test("maps repository loader limits to upload limit responses", async () => { repositoryLoadMode = "limit-error"; diff --git a/apps/api/src/routes/graph.ts b/apps/api/src/routes/graph.ts index 0e3eee9a..0fdeeb1f 100644 --- a/apps/api/src/routes/graph.ts +++ b/apps/api/src/routes/graph.ts @@ -127,10 +127,7 @@ function getRepositoryUrlError(error: unknown): RepositoryUrlError | undefined { function repositoryUrlErrorResponse(statusFn: (code: number, body: unknown) => unknown, error: unknown) { const repositoryError = getRepositoryUrlError(error); if (!repositoryError) { - return statusFn( - 400, - errorResponse("Repository could not be loaded", API_ERROR_CODES.UNSUPPORTED_FILE_TYPE) - ); + return statusFn(400, errorResponse("Repository could not be loaded", API_ERROR_CODES.UNSUPPORTED_FILE_TYPE)); } if (repositoryError.kind === "limit") { @@ -1157,7 +1154,7 @@ export const graphRoute = new Elysia({ prefix: "/graphs" }) graphId: existingGraph.id, fileIds: [file.id], processRunId: retry.runId, - ...(file.type === "code" ? { code: { kind: "repository" as const } } : {}), + ...(file.type === "code" ? { code: { kind: "repository" as const, retiredFileIds: [] } } : {}), }); return status(200, { From 15d2f48222c0a711817b764b773d9e1cfb4f849d Mon Sep 17 00:00:00 2001 From: Malte Eiting Date: Fri, 19 Jun 2026 12:48:36 +0200 Subject: [PATCH 23/23] fix(worker): scope duplicate webhook events --- .../workflows/sync-repository-graph.test.ts | 29 +++++++++++++++++-- .../worker/workflows/sync-repository-graph.ts | 13 +++++++-- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/apps/worker/workflows/sync-repository-graph.test.ts b/apps/worker/workflows/sync-repository-graph.test.ts index e4bae293..0e571f7c 100644 --- a/apps/worker/workflows/sync-repository-graph.test.ts +++ b/apps/worker/workflows/sync-repository-graph.test.ts @@ -10,6 +10,7 @@ const bindingUpdates: Array> = []; const compareCalls: Array<{ fromCommitSha: string; toCommitSha: string }> = []; const readFileCalls: Array<{ path: string; commitSha: string }> = []; const txWhereConditions: unknown[] = []; +const updateWhereConditions: unknown[] = []; const pendingReadResolutions: Array<() => void> = []; let selectResults: SelectResult[] = []; @@ -128,8 +129,9 @@ mock.module("@kiwi/db", () => ({ select: () => createSelectQuery(), update: () => ({ set: (values: Record) => ({ - where: async () => { + where: async (condition: unknown) => { bindingUpdates.push(values); + updateWhereConditions.push(condition); return undefined; }, }), @@ -256,7 +258,12 @@ function activeFile(id: string, commitSha: string, path: string, size: number) { }; } -async function runWorkflow(input: { bindingId: string; reason: "manual" | "webhook" | "initial"; commitSha?: string }) { +async function runWorkflow(input: { + bindingId: string; + reason: "manual" | "webhook" | "initial"; + commitSha?: string; + deliveryId?: string; +}) { return syncRepositoryGraph.fn({ input, step: { @@ -304,6 +311,7 @@ describe("syncRepositoryGraph", () => { compareCalls.length = 0; readFileCalls.length = 0; txWhereConditions.length = 0; + updateWhereConditions.length = 0; pendingReadResolutions.length = 0; selectResults = []; processFilesError = null; @@ -545,6 +553,23 @@ describe("syncRepositoryGraph", () => { }); }); + test("scopes duplicate webhook markers to the connector", async () => { + selectResults = [{ kind: "limit", value: [bindingRow("commit-new")] }]; + + const result = await runWorkflow({ + bindingId: "binding-1", + reason: "webhook", + commitSha: "commit-new", + deliveryId: "delivery-1", + }); + + expect(result).toEqual({ skipped: true, commitSha: "commit-new" }); + expect(bindingUpdates).toContainEqual({ status: "duplicate" }); + expect(queryContainsParamValue(updateWhereConditions.at(-1), "connector-1")).toBe(true); + expect(queryContainsParamValue(updateWhereConditions.at(-1), "github")).toBe(true); + expect(queryContainsParamValue(updateWhereConditions.at(-1), "delivery-1")).toBe(true); + }); + test("falls back to a full snapshot when compare is not incremental", async () => { compareIsIncremental = false; snapshotFiles = [ diff --git a/apps/worker/workflows/sync-repository-graph.ts b/apps/worker/workflows/sync-repository-graph.ts index 4cf864cb..42c0d657 100644 --- a/apps/worker/workflows/sync-repository-graph.ts +++ b/apps/worker/workflows/sync-repository-graph.ts @@ -495,7 +495,9 @@ async function findReusableProcessRun( } } - exactCandidates.sort((left, right) => processRunStatusPriority(left.status) - processRunStatusPriority(right.status)); + exactCandidates.sort( + (left, right) => processRunStatusPriority(left.status) - processRunStatusPriority(right.status) + ); return exactCandidates[0] ?? null; } @@ -572,12 +574,17 @@ async function insertRepositoryFiles( }); } -async function markWebhookDuplicate(provider: typeof connectorsTable.$inferSelect.provider, deliveryId: string) { +async function markWebhookDuplicate( + connectorId: string, + provider: typeof connectorsTable.$inferSelect.provider, + deliveryId: string +) { await db .update(connectorWebhookEventsTable) .set({ status: "duplicate" }) .where( and( + eq(connectorWebhookEventsTable.connectorId, connectorId), eq(connectorWebhookEventsTable.provider, provider), eq(connectorWebhookEventsTable.deliveryId, deliveryId) ) @@ -622,7 +629,7 @@ export const syncRepositoryGraph = defineWorkflow(syncRepositoryGraphSpec, async if (row.binding.lastSyncedCommitSha === commitSha) { if (input.deliveryId) { await step.run({ name: "mark-webhook-duplicate" }, async () => - markWebhookDuplicate(row.connector.provider, input.deliveryId!) + markWebhookDuplicate(row.connector.id, row.connector.provider, input.deliveryId!) ); } return { skipped: true, commitSha };