From 8005eccd0eec96a81b29d07f62d0123bc1a4fd89 Mon Sep 17 00:00:00 2001 From: R-sh Date: Fri, 10 Jul 2026 18:15:09 -0400 Subject: [PATCH] Fix domain surface consistency across landing page and registry --- src/lib/catalog.ts | 68 +++++++++++ src/lib/curated.ts | 73 +++++++++++ src/lib/discovered-catalog.ts | 51 ++++++++ src/lib/discovered-domains.ts | 55 +++++++++ src/lib/domain-discovery.test.ts | 57 +++++++++ src/lib/domain-discovery.ts | 173 +++++++++++++++++++++++++++ src/lib/domain-surface-types.test.ts | 36 ++++++ src/lib/domain-surface-types.ts | 40 +++++++ src/pages/[domain].astro | 14 ++- src/pages/disc/[domain].json.ts | 14 ++- src/pages/sitemap-surfaces.xml.ts | 14 ++- 11 files changed, 589 insertions(+), 6 deletions(-) create mode 100644 src/lib/curated.ts create mode 100644 src/lib/discovered-catalog.ts create mode 100644 src/lib/discovered-domains.ts create mode 100644 src/lib/domain-discovery.test.ts create mode 100644 src/lib/domain-discovery.ts create mode 100644 src/lib/domain-surface-types.test.ts create mode 100644 src/lib/domain-surface-types.ts diff --git a/src/lib/catalog.ts b/src/lib/catalog.ts index c21d6864..f9a01b93 100644 --- a/src/lib/catalog.ts +++ b/src/lib/catalog.ts @@ -6,6 +6,10 @@ * uses (no ad-hoc grouping in templates). Built over the enriched index. */ import { index } from "./data.ts"; +import { allCuratedProviders, curatedProviderForDomain } from "./curated.ts"; +import { readDomainCatalogTree } from "./discovered-catalog.ts"; +import { discoveredDomainsByTarget } from "./discovered-domains.ts"; +import { mergeDomainSurfaceCounts } from "./domain-surface-types.ts"; import { faviconUrl, isJunkDomain } from "./favicon.ts"; import { searchIndex } from "./search-index.ts"; import type { Kind } from "./types.ts"; @@ -22,6 +26,8 @@ export interface DomainSummary { const KIND_ORDER: Kind[] = ["mcp", "openapi", "graphql", "cli"]; +const discoveredByDomain = discoveredDomainsByTarget(readDomainCatalogTree().domains); + const DOMAINS: DomainSummary[] = (() => { const map = new Map(); for (const r of index) { @@ -55,6 +61,68 @@ const DOMAINS: DomainSummary[] = (() => { description: entry.description, }); } + for (const [domain, group] of map) { + const discovered = discoveredByDomain.get(domain); + const curated = curatedProviderForDomain(domain); + group.formats = mergeDomainSurfaceCounts(group.formats, { + catalog: Object.entries(group.formats) + .filter(([, count]) => (count ?? 0) > 0) + .map(([kind]) => kind as Kind), + discovered: discovered?.surfaces.map((surface) => surface.type), + curated: curated?.interfaces.map((iface) => iface.format), + }); + if (!group.description) { + group.description = + discovered?.description?.replace(/\s+/g, " ").slice(0, 110) ?? + curated?.description?.replace(/\s+/g, " ").slice(0, 110) ?? + ""; + } + } + for (const [domain, discovered] of discoveredByDomain) { + if (isJunkDomain(domain)) continue; + if (map.has(domain)) continue; + const curated = curatedProviderForDomain(domain); + map.set(domain, { + domain, + icon: faviconUrl(domain), + total: 0, + formats: mergeDomainSurfaceCounts({}, { + discovered: discovered.surfaces.map((surface) => surface.type), + curated: curated?.interfaces.map((iface) => iface.format), + }), + popularity: 0, + devtool: false, + description: + discovered.description?.replace(/\s+/g, " ").slice(0, 110) ?? + curated?.description?.replace(/\s+/g, " ").slice(0, 110) ?? + "", + }); + } + for (const domain of [...map.keys()]) { + const curated = curatedProviderForDomain(domain); + if (!curated) continue; + const group = map.get(domain)!; + group.formats = mergeDomainSurfaceCounts(group.formats, { + catalog: Object.entries(group.formats) + .filter(([, count]) => (count ?? 0) > 0) + .map(([kind]) => kind as Kind), + curated: curated.interfaces.map((iface) => iface.format), + }); + } + for (const curated of allCuratedProviders()) { + const domain = curated.domain; + if (isJunkDomain(domain)) continue; + if (map.has(domain)) continue; + map.set(domain, { + domain, + icon: faviconUrl(domain), + total: 0, + formats: mergeDomainSurfaceCounts({}, { curated: curated.interfaces.map((iface) => iface.format) }), + popularity: 0, + devtool: false, + description: curated.description.replace(/\s+/g, " ").slice(0, 110), + }); + } // Dev tools first — the audience's daily drivers — then popularity. return [...map.values()].sort( (a, b) => Number(b.devtool) - Number(a.devtool) || b.popularity - a.popularity || b.total - a.total || a.domain.localeCompare(b.domain), diff --git a/src/lib/curated.ts b/src/lib/curated.ts new file mode 100644 index 00000000..7607c694 --- /dev/null +++ b/src/lib/curated.ts @@ -0,0 +1,73 @@ +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { getDomain as tldGetDomain } from "tldts"; +import { canonicalDomain } from "./domain-aliases.ts"; +import type { Kind } from "./types.ts"; + +export interface CuratedInterface { + format: Kind; + name: string; + endpoint?: string; + specUrl?: string; + auth: "oauth" | "api_key" | "token" | "none" | "mixed"; + authHeader?: string; + install?: string; + docs?: string; + note?: string; + origin: "vendor" | "community"; + maintainer?: string; + repo?: string; +} + +export interface CuratedProvider { + slug: string; + name: string; + tagline: string; + description: string; + domain: string; + icon?: string; + categories: string[]; + interfaces: CuratedInterface[]; + links?: { homepage?: string; docs?: string }; +} + +const CURATED_DIR = join(process.cwd(), "curated"); +const getDomain = (value: string) => tldGetDomain(value, { allowPrivateDomains: true }); + +const providers: CuratedProvider[] = (() => { + if (!existsSync(CURATED_DIR)) return []; + return readdirSync(CURATED_DIR) + .filter((file) => file.endsWith(".json")) + .map((file) => JSON.parse(readFileSync(join(CURATED_DIR, file), "utf8")) as CuratedProvider); +})(); + +function targetDomains(provider: CuratedProvider): string[] { + const targets = new Set(); + targets.add(canonicalDomain(provider.domain)); + const urls = [ + provider.links?.homepage, + provider.links?.docs, + ...provider.interfaces.flatMap((iface) => [iface.endpoint, iface.specUrl, iface.docs, iface.repo]), + ]; + for (const url of urls) { + if (!url) continue; + const domain = getDomain(url); + if (domain) targets.add(canonicalDomain(domain)); + } + return [...targets]; +} + +const byDomain = new Map(); +for (const provider of providers) { + for (const domain of targetDomains(provider)) { + if (domain && !byDomain.has(domain)) byDomain.set(domain, provider); + } +} + +export function allCuratedProviders(): CuratedProvider[] { + return providers; +} + +export function curatedProviderForDomain(domain: string): CuratedProvider | null { + return byDomain.get(canonicalDomain(domain)) ?? null; +} diff --git a/src/lib/discovered-catalog.ts b/src/lib/discovered-catalog.ts new file mode 100644 index 00000000..cf0c44fc --- /dev/null +++ b/src/lib/discovered-catalog.ts @@ -0,0 +1,51 @@ +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { canonicalDomain } from "./domain-aliases.ts"; + +export interface CatalogPackage { + registryType: string; + identifier: string; + runtimeHint?: string; +} + +export interface CatalogSurface { + slug: string; + name: string; + type: "http" | "graphql" | "mcp" | "cli"; + url?: string; + spec?: string; + command?: string; + packages?: CatalogPackage[]; + authStatus: "none" | "required" | "unknown"; +} + +export interface CatalogDomain { + domain: string; + description?: string; + summary: string; + discoveredAt?: string; + surfaces: CatalogSurface[]; +} + +export interface Catalog { + domains: CatalogDomain[]; +} + +const DOMAINS_DIR = join(process.cwd(), "domains"); + +function domainFilePath(root: string, domain: string): string { + return join(root, domain, "integrations.json"); +} + +export function readDomainCatalogTree(root = DOMAINS_DIR): Catalog { + if (!existsSync(root)) return { domains: [] }; + + const domains = readdirSync(root, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => domainFilePath(root, entry.name)) + .filter((path) => existsSync(path) && statSync(path).isFile()) + .map((path) => JSON.parse(readFileSync(path, "utf8")) as CatalogDomain) + .sort((a, b) => canonicalDomain(a.domain).localeCompare(canonicalDomain(b.domain)) || a.domain.localeCompare(b.domain)); + + return { domains }; +} diff --git a/src/lib/discovered-domains.ts b/src/lib/discovered-domains.ts new file mode 100644 index 00000000..4388c7c0 --- /dev/null +++ b/src/lib/discovered-domains.ts @@ -0,0 +1,55 @@ +import { getDomain as tldGetDomain } from "tldts"; +import { canonicalDomain } from "./domain-aliases.ts"; +import type { CatalogDomain } from "./discovered-catalog.ts"; + +const getDomain = (value: string) => tldGetDomain(value, { allowPrivateDomains: true }); + +export function discoveredTargetDomains(entry: CatalogDomain): string[] { + const targets = new Set(); + targets.add(canonicalDomain(entry.domain)); + for (const surface of entry.surfaces) { + for (const value of [surface.url, surface.spec]) { + if (!value) continue; + const domain = getDomain(value); + if (domain) targets.add(canonicalDomain(domain)); + } + } + return [...targets]; +} + +function surfaceMatchCount(entry: CatalogDomain, target: string): number { + let count = 0; + for (const surface of entry.surfaces) { + for (const value of [surface.url, surface.spec]) { + if (!value) continue; + const domain = getDomain(value); + if (domain && canonicalDomain(domain) === target) count++; + } + } + return count; +} + +function outranks(target: string, next: CatalogDomain, current: CatalogDomain): boolean { + const nextExact = canonicalDomain(next.domain) === target; + const currentExact = canonicalDomain(current.domain) === target; + if (nextExact !== currentExact) return nextExact; + + const nextMatches = surfaceMatchCount(next, target); + const currentMatches = surfaceMatchCount(current, target); + if (nextMatches !== currentMatches) return nextMatches > currentMatches; + + if (next.surfaces.length !== current.surfaces.length) return next.surfaces.length > current.surfaces.length; + + return next.domain.length < current.domain.length; +} + +export function discoveredDomainsByTarget(entries: readonly CatalogDomain[]): Map { + const map = new Map(); + for (const entry of entries) { + for (const domain of discoveredTargetDomains(entry)) { + const current = map.get(domain); + if (!current || outranks(domain, entry, current)) map.set(domain, entry); + } + } + return map; +} diff --git a/src/lib/domain-discovery.test.ts b/src/lib/domain-discovery.test.ts new file mode 100644 index 00000000..356b6b0f --- /dev/null +++ b/src/lib/domain-discovery.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, test } from "bun:test"; +import { buildDomainDiscovery } from "./domain-discovery.ts"; +import type { Integration } from "./types.ts"; + +describe("buildDomainDiscovery", () => { + test("merges discovered CLI surfaces into the baseline domain discovery", () => { + const records: Integration[] = [ + { + id: "mcp/notion", + kind: "mcp", + slug: "notion", + name: "Notion MCP", + description: "", + categories: [], + feeds: ["openai"], + mcp: { remoteUrl: "https://mcp.notion.com/mcp" }, + raw: {}, + }, + { + id: "openapi/notion", + kind: "openapi", + slug: "notion-api", + name: "Notion API", + description: "", + categories: [], + feeds: ["override"], + openapi: { + provider: "notion.so", + version: "1", + specUrl: "https://developers.notion.com/openapi.json", + docsUrl: "https://developers.notion.com/reference", + openapiVer: "3.1.0", + }, + raw: {}, + }, + ]; + + const discovered = { + domain: "notion.so", + summary: "Notion exposes APIs, MCP, and a CLI.", + surfaces: [ + { + slug: "notion-cli", + name: "Notion CLI", + type: "cli" as const, + command: "ntn", + authStatus: "required" as const, + }, + ], + }; + + const doc = buildDomainDiscovery("notion.so", records, discovered, null); + + expect(doc.surfaces.map((surface) => surface.type)).toEqual(["cli", "mcp", "http"]); + expect(doc.surfaces.find((surface) => surface.type === "cli")?.command).toBe("ntn"); + }); +}); diff --git a/src/lib/domain-discovery.ts b/src/lib/domain-discovery.ts new file mode 100644 index 00000000..80414fd3 --- /dev/null +++ b/src/lib/domain-discovery.ts @@ -0,0 +1,173 @@ +import type { CatalogDomain } from "./discovered-catalog.ts"; +import { catalogDiscovery, recordToSurface } from "./catalog-to-discovery.ts"; +import type { CuratedProvider } from "./curated.ts"; +import type { DiscoverData } from "./surface-sections.ts"; +import type { Surface } from "./surface-view.ts"; +import type { Integration } from "./types.ts"; + +const DISCOVERED_BASIS = { via: "discovered" as const, evidence: [] as string[] }; +const REGISTRY_BASIS = { via: "detected" as const, signal: "registry" }; + +function slugify(value: string): string { + return ( + value + .normalize("NFKD") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-|-$/g, "") || "surface" + ); +} + +function assignSlug(name: string, existing: { slug: string }[]): string { + const base = slugify(name); + const taken = new Set(existing.map((item) => item.slug)); + if (!taken.has(base)) return base; + let i = 2; + while (taken.has(`${base}-${i}`)) i++; + return `${base}-${i}`; +} + +function authStatusToSurface(status: "none" | "required" | "unknown") { + return status === "none" + ? { status: "none" as const, basis: DISCOVERED_BASIS } + : status === "required" + ? { status: "required" as const, entries: [] } + : { status: "unknown" as const }; +} + +function discoveredSurfaceToSurface(surface: CatalogDomain["surfaces"][number]): Surface { + if (surface.type === "cli") { + return { + slug: surface.slug, + name: surface.name, + type: "cli", + command: surface.command, + packages: surface.packages, + basis: DISCOVERED_BASIS, + auth: authStatusToSurface(surface.authStatus), + }; + } + if (surface.type === "mcp") { + return { + slug: surface.slug, + name: surface.name, + type: "mcp", + url: surface.url, + basis: DISCOVERED_BASIS, + auth: authStatusToSurface(surface.authStatus), + }; + } + if (surface.type === "graphql") { + return { + slug: surface.slug, + name: surface.name, + type: "graphql", + url: surface.url ?? "", + basis: DISCOVERED_BASIS, + auth: authStatusToSurface(surface.authStatus), + }; + } + return { + slug: surface.slug, + name: surface.name, + type: "http", + spec: surface.spec, + url: surface.url, + basis: DISCOVERED_BASIS, + auth: authStatusToSurface(surface.authStatus), + }; +} + +function curatedInterfaceToSurface(provider: CuratedProvider, iface: CuratedProvider["interfaces"][number], slugs: { slug: string }[]): Surface { + const slug = assignSlug(iface.name || `${provider.name} ${iface.format}`, slugs); + slugs.push({ slug }); + const auth = iface.auth === "none" ? { status: "none" as const } : { status: "unknown" as const }; + if (iface.format === "cli") { + return { + slug, + name: iface.name, + type: "cli", + command: iface.install?.split(/\s+/).find(Boolean), + docs: iface.docs, + notes: iface.install ?? iface.note, + basis: REGISTRY_BASIS, + auth, + }; + } + if (iface.format === "mcp") { + return { + slug, + name: iface.name, + type: "mcp", + url: iface.endpoint, + docs: iface.docs, + basis: REGISTRY_BASIS, + auth, + }; + } + if (iface.format === "graphql") { + return { + slug, + name: iface.name, + type: "graphql", + url: iface.endpoint ?? "", + docs: iface.docs, + basis: REGISTRY_BASIS, + auth, + }; + } + return { + slug, + name: iface.name, + type: "http", + url: iface.endpoint, + spec: iface.specUrl, + docs: iface.docs, + basis: REGISTRY_BASIS, + auth, + }; +} + +function surfaceKey(surface: Pick & Partial): string { + if (surface.type === "cli") return `cli:${surface.command ?? surface.name.toLowerCase()}`; + if (surface.type === "mcp") return `mcp:${surface.url ?? surface.name.toLowerCase()}`; + if (surface.type === "graphql") return `graphql:${surface.url ?? surface.name.toLowerCase()}`; + return `http:${surface.spec ?? surface.url ?? surface.name.toLowerCase()}`; +} + +export function buildDomainDiscovery( + domain: string, + records: Integration[], + discovered: CatalogDomain | null = null, + curated: CuratedProvider | null = null, +): DiscoverData { + const baseline: DiscoverData = catalogDiscovery(domain, records); + const surfaces: Surface[] = []; + const seen = new Set(); + + const push = (surface: Surface) => { + const key = surfaceKey(surface); + if (seen.has(key)) return; + seen.add(key); + surfaces.push(surface); + }; + + for (const surface of discovered?.surfaces ?? []) push(discoveredSurfaceToSurface(surface)); + + const curatedSlugs = surfaces.map((surface) => ({ slug: surface.slug })); + for (const iface of curated?.interfaces ?? []) push(curatedInterfaceToSurface(curated, iface, curatedSlugs)); + + for (const record of records) { + const surface = recordToSurface(record); + if (!surface) continue; + push({ ...surface, slug: baseline.surfaces.find((item) => item.name === surface.name && item.type === surface.type)?.slug ?? assignSlug(surface.name, surfaces) }); + } + + return { + ...baseline, + description: discovered?.description ?? curated?.description, + summary: discovered?.summary ?? baseline.summary, + discoveredAt: discovered?.discoveredAt, + surfaces, + }; +} diff --git a/src/lib/domain-surface-types.test.ts b/src/lib/domain-surface-types.test.ts new file mode 100644 index 00000000..e5829300 --- /dev/null +++ b/src/lib/domain-surface-types.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, test } from "bun:test"; +import { mergeDomainSurfaceTypes } from "./domain-surface-types.ts"; + +describe("mergeDomainSurfaceTypes", () => { + test("includes CLI for Notion when any underlying source exposes it", () => { + expect( + mergeDomainSurfaceTypes({ + curated: ["mcp", "openapi"], + discovered: ["http", "mcp"], + catalog: ["mcp", "openapi", "cli"], + }), + ).toEqual(["mcp", "openapi", "cli"]); + }); + + test("deduplicates repeated surface types while preserving project order", () => { + expect( + mergeDomainSurfaceTypes({ + curated: ["graphql", "mcp"], + discovered: ["http", "graphql", "cli"], + catalog: ["cli", "mcp", "openapi"], + }), + ).toEqual(["mcp", "openapi", "graphql", "cli"]); + }); + + test("keeps curated-only surfaces visible", () => { + expect(mergeDomainSurfaceTypes({ curated: ["graphql"] })).toEqual(["graphql"]); + }); + + test("keeps raw-catalog-only surfaces visible", () => { + expect(mergeDomainSurfaceTypes({ catalog: ["cli"] })).toEqual(["cli"]); + }); + + test("handles domains with no surfaces safely", () => { + expect(mergeDomainSurfaceTypes({})).toEqual([]); + }); +}); diff --git a/src/lib/domain-surface-types.ts b/src/lib/domain-surface-types.ts new file mode 100644 index 00000000..d60fc183 --- /dev/null +++ b/src/lib/domain-surface-types.ts @@ -0,0 +1,40 @@ +import { KIND_ORDER } from "./domain-labels.ts"; +import type { Kind } from "./types.ts"; + +type SurfaceType = Kind | "http" | "rest" | "openapi" | "mcp" | "graphql" | "cli" | null | undefined; + +export interface DomainSurfaceTypeSources { + curated?: Iterable; + discovered?: Iterable; + catalog?: Iterable; +} + +export interface DomainSurfaceCounts extends Partial> {} + +export function normalizeSurfaceType(type: SurfaceType): Kind | null { + if (type === "http" || type === "rest" || type === "openapi") return "openapi"; + if (type === "mcp" || type === "graphql" || type === "cli") return type; + return null; +} + +export function mergeDomainSurfaceTypes(sources: DomainSurfaceTypeSources): Kind[] { + const kinds = new Set(); + for (const group of [sources.curated, sources.discovered, sources.catalog]) { + for (const type of group ?? []) { + const kind = normalizeSurfaceType(type); + if (kind) kinds.add(kind); + } + } + return KIND_ORDER.filter((kind) => kinds.has(kind)); +} + +export function mergeDomainSurfaceCounts( + counts: DomainSurfaceCounts, + sources: DomainSurfaceTypeSources, +): DomainSurfaceCounts { + const merged: DomainSurfaceCounts = { ...counts }; + for (const kind of mergeDomainSurfaceTypes(sources)) { + merged[kind] = Math.max(merged[kind] ?? 0, 1); + } + return merged; +} diff --git a/src/pages/[domain].astro b/src/pages/[domain].astro index 2d5564de..3787b4d7 100644 --- a/src/pages/[domain].astro +++ b/src/pages/[domain].astro @@ -4,9 +4,12 @@ // intercepts the request and SSRs the same DomainPage with the map baked in // (src/pages/ssr/[domain].astro) instead of serving this asset. import DomainPage from "~/components/DomainPage.astro"; +import { curatedProviderForDomain } from "~/lib/curated.ts"; import { all, domainById } from "~/lib/data.ts"; +import { readDomainCatalogTree } from "~/lib/discovered-catalog.ts"; +import { discoveredDomainsByTarget } from "~/lib/discovered-domains.ts"; +import { buildDomainDiscovery } from "~/lib/domain-discovery.ts"; import { allDomains } from "~/lib/catalog.ts"; -import { catalogDiscovery } from "~/lib/catalog-to-discovery.ts"; // Group every enriched record by its registrable domain (the same key the // homepage groups by: `domain`, else the slug). One page per domain. @@ -18,7 +21,14 @@ export function getStaticPaths() { } const { domain } = Astro.props; -const baseline = catalogDiscovery(domain, all.filter((r) => (domainById.get(r.id) || r.slug) === domain)); +const records = all.filter((r) => (domainById.get(r.id) || r.slug) === domain); +const discoveredByDomain = discoveredDomainsByTarget(readDomainCatalogTree().domains); +const baseline = buildDomainDiscovery( + domain, + records, + discoveredByDomain.get(domain) ?? null, + curatedProviderForDomain(domain), +); const domainSummary = allDomains().find((row) => row.domain === domain); const zeroSurfaceSummary = domainSummary?.total === 0 diff --git a/src/pages/disc/[domain].json.ts b/src/pages/disc/[domain].json.ts index 307a5fd6..4df193b5 100644 --- a/src/pages/disc/[domain].json.ts +++ b/src/pages/disc/[domain].json.ts @@ -7,11 +7,16 @@ * produce baseline surfaces here. Emitted at build via getStaticPaths. */ import type { APIRoute } from "astro"; +import { curatedProviderForDomain } from "~/lib/curated.ts"; import { all, domainById } from "~/lib/data.ts"; +import { readDomainCatalogTree } from "~/lib/discovered-catalog.ts"; +import { discoveredDomainsByTarget } from "~/lib/discovered-domains.ts"; +import { buildDomainDiscovery } from "~/lib/domain-discovery.ts"; import { allDomains } from "~/lib/catalog.ts"; -import { baselineDiscoveryGroups, catalogDiscovery } from "~/lib/catalog-to-discovery.ts"; +import { baselineDiscoveryGroups } from "~/lib/catalog-to-discovery.ts"; const groups = baselineDiscoveryGroups(all, (r) => domainById.get(r.id) || r.slug); +const discoveredByDomain = discoveredDomainsByTarget(readDomainCatalogTree().domains); export function getStaticPaths() { return allDomains().map(({ domain }) => ({ params: { domain } })); @@ -19,7 +24,12 @@ export function getStaticPaths() { export const GET: APIRoute = ({ params }) => { const domain = params.domain ?? ""; - const body = catalogDiscovery(domain, groups.get(domain) ?? []); + const body = buildDomainDiscovery( + domain, + groups.get(domain) ?? [], + discoveredByDomain.get(domain) ?? null, + curatedProviderForDomain(domain), + ); return new Response(JSON.stringify(body), { headers: { "content-type": "application/json; charset=utf-8" }, }); diff --git a/src/pages/sitemap-surfaces.xml.ts b/src/pages/sitemap-surfaces.xml.ts index 6521f5f1..4a3acbf7 100644 --- a/src/pages/sitemap-surfaces.xml.ts +++ b/src/pages/sitemap-surfaces.xml.ts @@ -1,6 +1,10 @@ import type { APIRoute } from "astro"; +import { curatedProviderForDomain } from "~/lib/curated.ts"; import { all, domainById } from "~/lib/data.ts"; -import { baselineDiscoveryGroups, catalogDiscovery } from "~/lib/catalog-to-discovery.ts"; +import { readDomainCatalogTree } from "~/lib/discovered-catalog.ts"; +import { discoveredDomainsByTarget } from "~/lib/discovered-domains.ts"; +import { buildDomainDiscovery } from "~/lib/domain-discovery.ts"; +import { baselineDiscoveryGroups } from "~/lib/catalog-to-discovery.ts"; export const prerender = true; @@ -12,11 +16,17 @@ function escapeXml(s: string): string { } const groups = baselineDiscoveryGroups(all, (r) => domainById.get(r.id) || r.slug); +const discoveredByDomain = discoveredDomainsByTarget(readDomainCatalogTree().domains); export const GET: APIRoute = () => { const urls: string[] = []; for (const [domain, records] of groups) { - const doc = catalogDiscovery(domain, records); + const doc = buildDomainDiscovery( + domain, + records, + discoveredByDomain.get(domain) ?? null, + curatedProviderForDomain(domain), + ); for (const surface of doc.surfaces) { if (RESERVED_SURFACE_SLUGS.has(surface.slug.toLowerCase())) continue; const loc = new URL(`/${encodeURIComponent(domain)}/${encodeURIComponent(surface.slug)}/`, SITE).href;