diff --git a/.cursor/plans/data_table_changelogs_ui_cc951009.plan.md b/.cursor/plans/data_table_changelogs_ui_cc951009.plan.md new file mode 100644 index 000000000..5a4390210 --- /dev/null +++ b/.cursor/plans/data_table_changelogs_ui_cc951009.plan.md @@ -0,0 +1,133 @@ +--- +name: Data Table Changelogs UI +overview: Wire overlay data table changelog entries into the Tables tab history, replace generic publish-modal placeholders with proper field-group renderers, and roll data-table events into the Summarized Changes view under their parent layers. +todos: + - id: api-data-table-changelogs + content: Add table_of_contents_items_data_table_change_logs() to current.sql + GraphQL query + codegen + status: completed + - id: field-group-components + content: Create 5 DataTable* field group list items + shared summary helpers; register in fieldGroups/index.ts + status: completed + - id: tables-tab-history + content: Add DataTablesChangeLogList to DataTablesEditor + refetch after data-table mutations + status: completed + - id: publish-all-changes + content: Extend titleForChangeLog() for overlay_data_table entries in PublishTableOfContentsModal + status: completed + - id: publish-summarized + content: Merge data-table logs into publishChangelogSummary + dataTables badge + PublishBadgeDetailContent popover + status: completed + - id: lint-verify + content: Run client lint and manually verify Tables tab + publish modal flows + status: completed +isProject: false +--- + +# Data Table Changelog UI + +## Problem + +Data table events are recorded correctly in the DB (`entity_type = 'overlay_data_table'`, field groups `data_table:*`, summaries with `{name, version}`, `meta.table_of_contents_item_id`) via [`000429.sql`](packages/api/migrations/committed/000429.sql), but the client never queries or renders them properly: + +```mermaid +flowchart LR + subgraph recorded [Recorded in DB] + DT["overlay_data_table changelogs"] + end + subgraph missing [Currently missing] + TablesTab["Tables tab History"] + Summary["Summarized Changes badges"] + end + subgraph broken [Broken UI] + AllChanges["All Changes → GenericFieldGroupListItem"] + end + DT --> AllChanges + DT -.-> TablesTab + DT -.-> Summary +``` + +- **Tables tab:** no history query at all ([`DataTablesEditor.tsx`](packages/client/src/admin/data/overlayDataTables/DataTablesEditor.tsx)) +- **All Changes:** falls through to [`GenericFieldGroupListItem.tsx`](packages/client/src/admin/changelogs/fieldGroups/GenericFieldGroupListItem.tsx) → `"updated DATA_TABLE_CREATED"` +- **Summarized Changes:** [`publishChangelogSummary.ts`](packages/client/src/admin/data/publishChangelogSummary.ts) filters `entityType === "table_of_contents_items"` only, so data-table logs never attach to layer rows + +## 1. API: expose data-table changelogs for a layer + +Add a PostGraphile computed column on `table_of_contents_items` in [`packages/api/migrations/current.sql`](packages/api/migrations/current.sql): + +```sql +create or replace function table_of_contents_items_data_table_change_logs(item table_of_contents_items) + returns setof change_logs + language sql stable security definer as $$ + select * from change_logs + where entity_type = 'overlay_data_table' + and net_zero_changes = false + and (meta->>'table_of_contents_item_id')::int = item.id + order by last_at desc; + $$; +-- grant + @simpleCollections only comment (match 000414.sql pattern) +``` + +Extend [`LayerSettingsChangeLog`](packages/client/src/queries/DraftTableOfContents.graphql) **or** add a dedicated query `DataTableChangeLog($id, $first)` on `tableOfContentsItem.dataTableChangeLogs(first: $first) { ...ChangeLogDetails }`. Prefer a **separate query** so Settings-tab history stays unchanged per your preference. + +Run `graphql:codegen`. + +## 2. Tables tab: dedicated History section + +Create [`DataTablesChangeLogList.tsx`](packages/client/src/admin/changelogs/DataTablesChangeLogList.tsx) modeled on [`LayerSettingsChangeLogList.tsx`](packages/client/src/admin/changelogs/LayerSettingsChangeLogList.tsx): + +- Query `dataTableChangeLogs` for the TOC item id +- Paginate with same page sizes / “View full history” pattern +- Render via existing [`ChangeLogListItem`](packages/client/src/admin/changelogs/ChangeLogListItem.tsx) (no `itemTitle` needed — user is already on that layer) +- Return `null` when empty (before any tables exist) + +Mount at the bottom of [`DataTablesEditor.tsx`](packages/client/src/admin/data/overlayDataTables/DataTablesEditor.tsx), below `RelatedDataTables`, when `enableDataTables && dataTableJoinColumn`. + +Add refetch helper (e.g. `dataTableChangeLogRefetchQueries(tocItemId)`) and wire into data-table mutations in [`RelatedDataTables.tsx`](packages/client/src/admin/data/overlayDataTables/RelatedDataTables.tsx) and upload completion refetch paths so History updates after upload/rename/delete/replace/rollback. + +## 3. Field-group renderers (All Changes + Tables tab) + +Add shared helpers in [`fieldGroups/dataTableSummary.ts`](packages/client/src/admin/changelogs/fieldGroups/dataTableSummary.ts): + +- `tocItemIdFromMeta(meta)` — parse `table_of_contents_item_id` +- `tableLabel(from/to summary, meta)` — `"name (vN)"` from `{name, version}` + +Add five list-item components following existing patterns: + +| Component | Pattern | Example copy | +| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | +| `DataTableCreatedFieldGroupListItem` | [`FolderCreatedFieldGroupListItem`](packages/client/src/admin/changelogs/fieldGroups/FolderCreatedFieldGroupListItem.tsx) | uploaded data table **swath-short (v1)** | +| `DataTableReplacedFieldGroupListItem` | [`LayerUploadedFieldGroupListItem`](packages/client/src/admin/changelogs/fieldGroups/LayerUploadedFieldGroupListItem.tsx) replacement branch | replaced **swath-short** v1 → v2 | +| `DataTableRenamedFieldGroupListItem` | [`LayerTitleFieldGroupListItem`](packages/client/src/admin/changelogs/fieldGroups/LayerTitleFieldGroupListItem.tsx) | renamed **old** → **new** (v2) | +| `DataTableDeletedFieldGroupListItem` | [`LayerDeletedFieldGroupListItem`](packages/client/src/admin/changelogs/fieldGroups/LayerDeletedFieldGroupListItem.tsx) | deleted **swath-short (v1)** | +| `DataTableRollbackFieldGroupListItem` | new (CounterClockwise/Undo icon) | rolled back **swath-short** to v1 (removed v2) | + +Register all five in [`fieldGroups/index.ts`](packages/client/src/admin/changelogs/fieldGroups/index.ts) for `ChangeLogFieldGroup.DataTable*`. + +**Publish modal context:** extend `titleForChangeLog()` in [`PublishTableOfContentsModal.tsx`](packages/client/src/admin/data/PublishTableOfContentsModal.tsx) to resolve parent layer title when `entityType === "overlay_data_table"` via `meta.table_of_contents_item_id` + existing `itemTitleById` map (shows layer name above the entry in All Changes, matching TOC entries). + +## 4. Summarized Changes: roll up under parent layers + +Update [`publishChangelogSummary.ts`](packages/client/src/admin/data/publishChangelogSummary.ts): + +1. After building `byEntity` from TOC logs, **merge** `overlay_data_table` logs whose `meta.table_of_contents_item_id` matches a draft TOC item +2. Layers with **only** data-table changes since last publish should appear in **Updated** (not dropped) +3. Add badge key `"dataTables"` to `PublishBadgeKey` + `PUBLISH_BADGE_ORDER` (after `"downloads"` or near `"source"`) +4. Map all five `DATA_TABLE_*` field groups → `"dataTables"` in `FIELD_GROUP_TO_BADGE` +5. Include merged logs in `changeCount`, `lastChangeAt`, and `editors` aggregation + +Add badge label in [`PublishSummarizedChangesPanel.tsx`](packages/client/src/admin/data/PublishSummarizedChangesPanel.tsx) (`t("data tables")`). + +Add popover body in [`PublishBadgeDetailContent.tsx`](packages/client/src/admin/data/publishBadgeDetails/PublishBadgeDetailContent.tsx) — list each event with human-readable action + table name/version (reuse `dataTableSummary` helpers). No modal deep-link needed initially. + +## 5. Verification + +- **Tables tab:** upload, rename, replace, delete, rollback → entries appear in Tables History with correct icons/copy +- **Publish → All Changes:** same entries styled properly; parent layer title shown +- **Publish → Summarized Changes:** affected layers show a **data tables** badge; popover lists individual table events; change counts include table operations +- **Settings tab:** unchanged (no data-table entries mixed in) +- Run `npm run lint` in `packages/client` + +## Out of scope (noted) + +- Changelog entries for `enableDataTables` / `dataTableJoinColumn` toggles (not recorded in DB today) +- Fix for replacement upload changelogs missing `editor_id` when completed by background worker (`complete_overlay_data_table_upload` uses `session.user_id` only for replace path in `000429.sql`) — separate bug if you notice missing replace entries diff --git a/AGENTS.md b/AGENTS.md index 1cb28b5c2..95bbf7c35 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,7 +76,9 @@ export const MyComponent: React.FC = ({ title }) => { ## React Client -- verifying your work -- Whenever making changes to the client, run `npm run lint` from `packages/client` to ensure there are no linter errors that will break the build. +- Whenever making changes to the client, verify that your work will compile. +- Identify the **Client devserver** terminal session in the IDE. If available, that will show any compiler or linter errors when the live-reload cycle runs. This will usually yield any errors faster than running the linter directly. +- Alternatively, run `npm run lint` from `packages/client` to ensure there are no linter errors that will break the build. ## Report Widgets diff --git a/packages/api/.env.template b/packages/api/.env.template index 99847bd0b..41f5a511e 100644 --- a/packages/api/.env.template +++ b/packages/api/.env.template @@ -26,3 +26,5 @@ POEDITOR_API_TOKEN="op://SeaSketch/dev/POEditor/APIToken" POEDITOR_PROJECT="op://SeaSketch/dev/POEditor/Project" REDIS_PASSWORD="op://SeaSketch/dev/Redis/Password" CLIENT_DOMAIN="op://SeaSketch/dev/ClientDomain" +# Local data table upload processing (run `npm run dev` in packages/data-tables-handler) +DATA_TABLES_LAMBDA_DEV_HANDLER=http://localhost:3006 diff --git a/packages/api/generated-schema-clean.gql b/packages/api/generated-schema-clean.gql index 2ee31f0c6..7e9b42b35 100644 --- a/packages/api/generated-schema-clean.gql +++ b/packages/api/generated-schema-clean.gql @@ -1101,6 +1101,12 @@ input ChangeLogCondition { } enum ChangeLogFieldGroup { + DATA_TABLE_CREATED + DATA_TABLE_DELETED + DATA_TABLE_RENAMED + DATA_TABLE_REPLACED + DATA_TABLE_ROLLBACK + DATA_TABLE_VISUALIZATION_SETTINGS_UPDATED FOLDER_ACL FOLDER_CREATED FOLDER_DELETED @@ -1718,6 +1724,7 @@ input CreateDataUploadInput { clientMutationId: String contentType: String filename: String + processingOptions: JSON projectId: Int replaceTableOfContentsItemId: Int } @@ -2153,6 +2160,7 @@ input CreateMapBookmarkInput { payload verbatim. May be used to track mutations by the client. """ clientMutationId: String + dataTableStates: JSON isPublic: Boolean layerNames: JSON mapDimensions: [Int] @@ -2258,6 +2266,46 @@ type CreateOptionalBasemapLayerPayload { query: Query } +"""All input for the `createOverlayDataTableUpload` mutation.""" +input CreateOverlayDataTableUploadInput { + """ + An arbitrary string value with no semantic meaning. Will be included in the + payload verbatim. May be used to track mutations by the client. + """ + clientMutationId: String + contentType: String + filename: String + processingOptions: JSON + replaceOverlayDataTableId: Int + tocItemId: Int +} + +"""The output of our `createOverlayDataTableUpload` mutation.""" +type CreateOverlayDataTableUploadPayload { + """ + The exact same `clientMutationId` that was provided in the mutation input, + unchanged and unused. May be used by a client to track mutations. + """ + clientMutationId: String + overlayDataTableUpload: OverlayDataTableUpload + + """An edge for our `OverlayDataTableUpload`. May be used by Relay 1.""" + overlayDataTableUploadEdge( + """The method to use when ordering `OverlayDataTableUpload`.""" + orderBy: [OverlayDataTableUploadsOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTableUploadsEdge + + """ + Reads a single `ProjectBackgroundJob` that is related to this `OverlayDataTableUpload`. + """ + projectBackgroundJob: ProjectBackgroundJob + + """ + Our root query field type. Allows us to run any query from our mutation payload. + """ + query: Query +} + input CreateProjectGeographyClippingLayerInput { """ If provided, features used for clipping will be filtered based on this @@ -4193,6 +4241,7 @@ type DataUploadOutput implements Node { } enum DataUploadOutputType { + CSV FLAT_GEOBUF GEO_JSON GEO_TIFF @@ -4237,6 +4286,13 @@ type DataUploadTask implements Node { """Use to upload source data to s3. Must be an admin.""" presignedUploadUrl: String + """ + Format-specific processing instructions supplied by the client at upload time + (e.g. column mapping and CRS for delimited text uploads). Consumed by the + spatial-uploads-handler. + """ + processingOptions: JSON + """ Reads a single `ProjectBackgroundJob` that is related to this `DataUploadTask`. """ @@ -6397,6 +6453,11 @@ type FailDataUploadPayload { } type FeatureFlags { + """ + When true, project admins see the Data Tables tab in layer editors + and related map UI. Controlled from SeaSketch developer settings. + """ + dataTables: Boolean iNaturalistLayers: Boolean } @@ -8810,6 +8871,11 @@ type MapBookmark { """ clientGeneratedThumbnail: String createdAt: Datetime! + + """ + JSON map of TOC stableId -> { stableId, column?, op?, filters? } for activated overlay data tables. + """ + dataTableStates: JSON! id: UUID! imageId: String isPublic: Boolean! @@ -9333,6 +9399,12 @@ type Mutation { """ input: CreateOptionalBasemapLayerInput! ): CreateOptionalBasemapLayerPayload + createOverlayDataTableUpload( + """ + The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. + """ + input: CreateOverlayDataTableUploadInput! + ): CreateOverlayDataTableUploadPayload createPost(message: JSON!, topicId: Int!): Post! """ @@ -10328,6 +10400,12 @@ type Mutation { """ input: RemoveValidChildSketchClassInput! ): RemoveValidChildSketchClassPayload + renameOverlayDataTable( + """ + The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. + """ + input: RenameOverlayDataTableInput! + ): RenameOverlayDataTablePayload renameReportTab( """ The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. @@ -10408,6 +10486,12 @@ type Mutation { """ input: RevokeApiKeyInput! ): RevokeApiKeyPayload + rollbackOverlayDataTableVersion( + """ + The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. + """ + input: RollbackOverlayDataTableVersionInput! + ): RollbackOverlayDataTableVersionPayload rollbackToArchivedSource( """ The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. @@ -10501,6 +10585,12 @@ type Mutation { """ input: SetForumOrderInput! ): SetForumOrderPayload + setOverlayDataTableVisualizationSettings( + """ + The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. + """ + input: SetOverlayDataTableVisualizationSettingsInput! + ): SetOverlayDataTableVisualizationSettingsPayload """ Admins can use this function to hide the contents of a message. Message will @@ -10570,6 +10660,12 @@ type Mutation { """ input: ShareSpriteInput! ): ShareSpritePayload + softDeleteOverlayDataTable( + """ + The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. + """ + input: SoftDeleteOverlayDataTableInput! + ): SoftDeleteOverlayDataTablePayload """ Superusers only. "Deletes" a sprite but keeps it in the DB in case layers are already referencing it. @@ -10586,6 +10682,12 @@ type Mutation { """ input: SubmitDataUploadInput! ): SubmitDataUploadPayload + submitOverlayDataTableUpload( + """ + The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. + """ + input: SubmitOverlayDataTableUploadInput! + ): SubmitOverlayDataTableUploadPayload """ Toggle admin access for the given project and user. User must have already @@ -11688,6 +11790,185 @@ type OutstandingSurveyInvites { token: String! } +type OverlayDataTable implements Node { + columnStatsRemote: String! + columnStatsUrl: String + createdAt: Datetime + createdBy: Int! + deletedAt: Datetime + id: Int! + joinColumn: String! + name: String! + + """ + A globally unique identifier. Can be used in various places throughout the system to identify this single value. + """ + nodeId: ID! + overlayJoinColumn: String! + parquetRemote: String! + parquetUrl: String + + """Reads a single `Project` that is related to this `OverlayDataTable`.""" + project: Project + projectId: Int! + queryUrl: String + replacedById: Int + + """ + Columns that must appear as filters when this table is displayed on the map. + End users can change the filter values but cannot remove these filters. + """ + requiredFilterColumns: [String] + rowCount: Int! + + """ + Stable logical identity for a data table across version replace and TOC + publish. Draft and published copies share the same UUID. + """ + stableId: UUID! + tableOfContentsItemId: Int! + updatedAt: Datetime + version: Int! + + """ + Columns that may/should be used for creating thematic maps. For example `count` or `density` + """ + visualizationColumns: [String] + + """ + Operations that may/should be used for creating thematic maps. For example `mean` or `max` + """ + visualizationOps: [String] +} + +""" +A condition to be used against `OverlayDataTable` object types. All fields are +tested for equality and combined with a logical ‘and.’ +""" +input OverlayDataTableCondition { + """Checks for equality with the object’s `id` field.""" + id: Int + + """Checks for equality with the object’s `projectId` field.""" + projectId: Int +} + +"""A connection to a list of `OverlayDataTable` values.""" +type OverlayDataTablesConnection { + """ + A list of edges which contains the `OverlayDataTable` and cursor to aid in pagination. + """ + edges: [OverlayDataTablesEdge!]! + + """A list of `OverlayDataTable` objects.""" + nodes: [OverlayDataTable!]! + + """Information to aid in pagination.""" + pageInfo: PageInfo! + + """ + The count of *all* `OverlayDataTable` you could get from the connection. + """ + totalCount: Int! +} + +"""A `OverlayDataTable` edge in the connection.""" +type OverlayDataTablesEdge { + """A cursor for use in pagination.""" + cursor: Cursor + + """The `OverlayDataTable` at the end of the edge.""" + node: OverlayDataTable! +} + +"""Methods to use when ordering `OverlayDataTable`.""" +enum OverlayDataTablesOrderBy { + ID_ASC + ID_DESC + NATURAL + PRIMARY_KEY_ASC + PRIMARY_KEY_DESC + PROJECT_ID_ASC + PROJECT_ID_DESC +} + +type OverlayDataTableUpload implements Node { + contentType: String! + createdAt: Datetime + errorDetails: JSON + filename: String! + id: UUID! + + """ + A globally unique identifier. Can be used in various places throughout the system to identify this single value. + """ + nodeId: ID! + overlayGeostats: JSON! + overlayJoinColumn: String + presignedUploadUrl: String + processingOptions: JSON! + + """ + Reads a single `ProjectBackgroundJob` that is related to this `OverlayDataTableUpload`. + """ + projectBackgroundJob: ProjectBackgroundJob + projectBackgroundJobId: UUID! + replaceOverlayDataTableId: Int + tableOfContentsItemId: Int! + updatedAt: Datetime +} + +""" +A condition to be used against `OverlayDataTableUpload` object types. All fields +are tested for equality and combined with a logical ‘and.’ +""" +input OverlayDataTableUploadCondition { + """Checks for equality with the object’s `id` field.""" + id: UUID + + """Checks for equality with the object’s `projectBackgroundJobId` field.""" + projectBackgroundJobId: UUID +} + +"""A connection to a list of `OverlayDataTableUpload` values.""" +type OverlayDataTableUploadsConnection { + """ + A list of edges which contains the `OverlayDataTableUpload` and cursor to aid in pagination. + """ + edges: [OverlayDataTableUploadsEdge!]! + + """A list of `OverlayDataTableUpload` objects.""" + nodes: [OverlayDataTableUpload!]! + + """Information to aid in pagination.""" + pageInfo: PageInfo! + + """ + The count of *all* `OverlayDataTableUpload` you could get from the connection. + """ + totalCount: Int! +} + +"""A `OverlayDataTableUpload` edge in the connection.""" +type OverlayDataTableUploadsEdge { + """A cursor for use in pagination.""" + cursor: Cursor + + """The `OverlayDataTableUpload` at the end of the edge.""" + node: OverlayDataTableUpload! +} + +"""Methods to use when ordering `OverlayDataTableUpload`.""" +enum OverlayDataTableUploadsOrderBy { + ID_ASC + ID_DESC + NATURAL + PRIMARY_KEY_ASC + PRIMARY_KEY_DESC + PROJECT_BACKGROUND_JOB_ID_ASC + PROJECT_BACKGROUND_JOB_ID_DESC +} + """Information about pagination in a connection.""" type PageInfo { """When paginating forwards, the cursor to continue.""" @@ -12279,6 +12560,15 @@ type Project implements Node { hideForums: Boolean! hideOverlays: Boolean! hideSketches: Boolean! + + """ + Content-addressed tileset UUIDs whose hosted tiles/uploads requests + must include a map access token. Equals hosted UUIDs from published + TOC items (and draft TOC items for project admins) that are not in the + published ACL document's public list. Empty for projects with no + protected overlays. + """ + hostedTileUuidsRequiringAuth: [String!]! id: Int! importedArcgisServices: [String] @@ -12383,6 +12673,13 @@ type Project implements Node { displayed at 48x48 pixels and must be a public url. """ logoUrl: String + + """ + Short-lived (90m) JWT for authorized overlay tile requests on + hosted tiles/uploads URLs with access_token (and optional ns). Returns null if the user is not signed in. + Claims include role (admin|user) and project group ids. + """ + mapAccessToken: String mapboxPublicKey: String mapboxSecretKey: String @@ -12482,6 +12779,35 @@ type Project implements Node { orderBy: [OfflineTileSettingsOrderBy!] ): [OfflineTileSetting!]! + """Reads and enables pagination through a set of `OverlayDataTable`.""" + overlayDataTablesConnection( + """Read all values in the set after (below) this cursor.""" + after: Cursor + + """Read all values in the set before (above) this cursor.""" + before: Cursor + + """ + A condition to be used in determining which values should be returned by the collection. + """ + condition: OverlayDataTableCondition + + """Only read the first `n` values of the set.""" + first: Int + + """Only read the last `n` values of the set.""" + last: Int + + """ + Skip the first `n` values from our `after` cursor, an alternative to cursor + based pagination. May not be used with `last`. + """ + offset: Int + + """The method to use when ordering `OverlayDataTable`.""" + orderBy: [OverlayDataTablesOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTablesConnection! + """ Count of all users who have opted into participating in the project, sharing their profile with project administrators. """ @@ -12835,6 +13161,42 @@ type ProjectBackgroundJob implements Node { A globally unique identifier. Can be used in various places throughout the system to identify this single value. """ nodeId: ID! + + """ + Reads a single `OverlayDataTableUpload` that is related to this `ProjectBackgroundJob`. + """ + overlayDataTableUpload: OverlayDataTableUpload + + """ + Reads and enables pagination through a set of `OverlayDataTableUpload`. + """ + overlayDataTableUploadsConnection( + """Read all values in the set after (below) this cursor.""" + after: Cursor + + """Read all values in the set before (above) this cursor.""" + before: Cursor + + """ + A condition to be used in determining which values should be returned by the collection. + """ + condition: OverlayDataTableUploadCondition + + """Only read the first `n` values of the set.""" + first: Int + + """Only read the last `n` values of the set.""" + last: Int + + """ + Skip the first `n` values from our `after` cursor, an alternative to cursor + based pagination. May not be used with `last`. + """ + offset: Int + + """The method to use when ordering `OverlayDataTableUpload`.""" + orderBy: [OverlayDataTableUploadsOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTableUploadsConnection! @deprecated(reason: "Please use overlayDataTableUpload instead") progress: BigFloat progressMessage: String! @@ -12892,6 +13254,7 @@ type ProjectBackgroundJobSubscriptionPayload { enum ProjectBackgroundJobType { ARCGIS_IMPORT CONSOLIDATE_DATA_SOURCES + DATA_TABLE_UPLOAD DATA_UPLOAD REPLACEMENT_UPLOAD } @@ -14167,6 +14530,89 @@ type Query implements Node { """ nodeId: ID! ): OptionalBasemapLayer + overlayDataTable(id: Int!): OverlayDataTable + + """Reads a single `OverlayDataTable` using its globally unique `ID`.""" + overlayDataTableByNodeId( + """ + The globally unique `ID` to be used in selecting a single `OverlayDataTable`. + """ + nodeId: ID! + ): OverlayDataTable + overlayDataTableLinkedTocIsDraft(pid: Int, tocItemId: Int): Boolean + overlayDataTableParquetPublicUrl(pRemote: String): String + + """Reads and enables pagination through a set of `OverlayDataTable`.""" + overlayDataTablesConnection( + """Read all values in the set after (below) this cursor.""" + after: Cursor + + """Read all values in the set before (above) this cursor.""" + before: Cursor + + """ + A condition to be used in determining which values should be returned by the collection. + """ + condition: OverlayDataTableCondition + + """Only read the first `n` values of the set.""" + first: Int + + """Only read the last `n` values of the set.""" + last: Int + + """ + Skip the first `n` values from our `after` cursor, an alternative to cursor + based pagination. May not be used with `last`. + """ + offset: Int + + """The method to use when ordering `OverlayDataTable`.""" + orderBy: [OverlayDataTablesOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTablesConnection + overlayDataTableUpload(id: UUID!): OverlayDataTableUpload + + """ + Reads a single `OverlayDataTableUpload` using its globally unique `ID`. + """ + overlayDataTableUploadByNodeId( + """ + The globally unique `ID` to be used in selecting a single `OverlayDataTableUpload`. + """ + nodeId: ID! + ): OverlayDataTableUpload + overlayDataTableUploadByProjectBackgroundJobId(projectBackgroundJobId: UUID!): OverlayDataTableUpload + + """ + Reads and enables pagination through a set of `OverlayDataTableUpload`. + """ + overlayDataTableUploadsConnection( + """Read all values in the set after (below) this cursor.""" + after: Cursor + + """Read all values in the set before (above) this cursor.""" + before: Cursor + + """ + A condition to be used in determining which values should be returned by the collection. + """ + condition: OverlayDataTableUploadCondition + + """Only read the first `n` values of the set.""" + first: Int + + """Only read the last `n` values of the set.""" + last: Int + + """ + Skip the first `n` values from our `after` cursor, an alternative to cursor + based pagination. May not be used with `last`. + """ + offset: Int + + """The method to use when ordering `OverlayDataTableUpload`.""" + orderBy: [OverlayDataTableUploadsOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTableUploadsConnection post(id: Int!): Post """Reads a single `Post` using its globally unique `ID`.""" @@ -14916,6 +15362,41 @@ type RemoveValidChildSketchClassPayload { query: Query } +"""All input for the `renameOverlayDataTable` mutation.""" +input RenameOverlayDataTableInput { + """ + An arbitrary string value with no semantic meaning. Will be included in the + payload verbatim. May be used to track mutations by the client. + """ + clientMutationId: String + newName: String + tableId: Int +} + +"""The output of our `renameOverlayDataTable` mutation.""" +type RenameOverlayDataTablePayload { + """ + The exact same `clientMutationId` that was provided in the mutation input, + unchanged and unused. May be used by a client to track mutations. + """ + clientMutationId: String + overlayDataTable: OverlayDataTable + + """An edge for our `OverlayDataTable`. May be used by Relay 1.""" + overlayDataTableEdge( + """The method to use when ordering `OverlayDataTable`.""" + orderBy: [OverlayDataTablesOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTablesEdge + + """Reads a single `Project` that is related to this `OverlayDataTable`.""" + project: Project + + """ + Our root query field type. Allows us to run any query from our mutation payload. + """ + query: Query +} + """All input for the `renameReportTab` mutation.""" input RenameReportTabInput { alternateLanguageSettings: JSON @@ -15558,6 +16039,40 @@ type RevokeApiKeyPayload { query: Query } +"""All input for the `rollbackOverlayDataTableVersion` mutation.""" +input RollbackOverlayDataTableVersionInput { + """ + An arbitrary string value with no semantic meaning. Will be included in the + payload verbatim. May be used to track mutations by the client. + """ + clientMutationId: String + tableId: Int +} + +"""The output of our `rollbackOverlayDataTableVersion` mutation.""" +type RollbackOverlayDataTableVersionPayload { + """ + The exact same `clientMutationId` that was provided in the mutation input, + unchanged and unused. May be used by a client to track mutations. + """ + clientMutationId: String + overlayDataTable: OverlayDataTable + + """An edge for our `OverlayDataTable`. May be used by Relay 1.""" + overlayDataTableEdge( + """The method to use when ordering `OverlayDataTable`.""" + orderBy: [OverlayDataTablesOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTablesEdge + + """Reads a single `Project` that is related to this `OverlayDataTable`.""" + project: Project + + """ + Our root query field type. Allows us to run any query from our mutation payload. + """ + query: Query +} + """All input for the `rollbackToArchivedSource` mutation.""" input RollbackToArchivedSourceInput { """ @@ -15797,6 +16312,43 @@ type SetForumOrderPayload { query: Query } +"""All input for the `setOverlayDataTableVisualizationSettings` mutation.""" +input SetOverlayDataTableVisualizationSettingsInput { + """ + An arbitrary string value with no semantic meaning. Will be included in the + payload verbatim. May be used to track mutations by the client. + """ + clientMutationId: String + requiredFilterColumns: [String] + tableId: Int + visualizationColumns: [String] + visualizationOps: [String] +} + +"""The output of our `setOverlayDataTableVisualizationSettings` mutation.""" +type SetOverlayDataTableVisualizationSettingsPayload { + """ + The exact same `clientMutationId` that was provided in the mutation input, + unchanged and unused. May be used by a client to track mutations. + """ + clientMutationId: String + overlayDataTable: OverlayDataTable + + """An edge for our `OverlayDataTable`. May be used by Relay 1.""" + overlayDataTableEdge( + """The method to use when ordering `OverlayDataTable`.""" + orderBy: [OverlayDataTablesOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTablesEdge + + """Reads a single `Project` that is related to this `OverlayDataTable`.""" + project: Project + + """ + Our root query field type. Allows us to run any query from our mutation payload. + """ + query: Query +} + """All input for the `setPostHiddenByModerator` mutation.""" input SetPostHiddenByModeratorInput { """ @@ -16560,6 +17112,40 @@ enum SketchGeometryType { POLYGON } +"""All input for the `softDeleteOverlayDataTable` mutation.""" +input SoftDeleteOverlayDataTableInput { + """ + An arbitrary string value with no semantic meaning. Will be included in the + payload verbatim. May be used to track mutations by the client. + """ + clientMutationId: String + tableId: Int +} + +"""The output of our `softDeleteOverlayDataTable` mutation.""" +type SoftDeleteOverlayDataTablePayload { + """ + The exact same `clientMutationId` that was provided in the mutation input, + unchanged and unused. May be used by a client to track mutations. + """ + clientMutationId: String + overlayDataTable: OverlayDataTable + + """An edge for our `OverlayDataTable`. May be used by Relay 1.""" + overlayDataTableEdge( + """The method to use when ordering `OverlayDataTable`.""" + orderBy: [OverlayDataTablesOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTablesEdge + + """Reads a single `Project` that is related to this `OverlayDataTable`.""" + project: Project + + """ + Our root query field type. Allows us to run any query from our mutation payload. + """ + query: Query +} + """All input for the `softDeleteSprite` mutation.""" input SoftDeleteSpriteInput { """ @@ -16821,6 +17407,36 @@ type SubmitDataUploadPayload { query: Query } +"""All input for the `submitOverlayDataTableUpload` mutation.""" +input SubmitOverlayDataTableUploadInput { + """ + An arbitrary string value with no semantic meaning. Will be included in the + payload verbatim. May be used to track mutations by the client. + """ + clientMutationId: String + jobId: UUID +} + +"""The output of our `submitOverlayDataTableUpload` mutation.""" +type SubmitOverlayDataTableUploadPayload { + """ + The exact same `clientMutationId` that was provided in the mutation input, + unchanged and unused. May be used by a client to track mutations. + """ + clientMutationId: String + + """ + Reads a single `Project` that is related to this `ProjectBackgroundJob`. + """ + project: Project + projectBackgroundJob: ProjectBackgroundJob + + """ + Our root query field type. Allows us to run any query from our mutation payload. + """ + query: Query +} + """ The root subscription type: contains realtime events you can subscribe to with the `subscription` operation. """ @@ -17518,6 +18134,20 @@ type TableOfContentsItem implements Node { dataLayerId: Int dataSourceType: DataSourceTypes + """Reads and enables pagination through a set of `ChangeLog`.""" + dataTableChangeLogs( + """Only read the first `n` values of the set.""" + first: Int + + """Skip the first `n` values.""" + offset: Int + ): [ChangeLog!] + + """ + Overlay attribute name used as the canonical feature ID for linked data tables. Required when enable_data_tables is true. + """ + dataTableJoinColumn: String + """Reads and enables pagination through a set of `DownloadOption`.""" downloadOptions( """Only read the first `n` values of the set.""" @@ -17526,6 +18156,11 @@ type TableOfContentsItem implements Node { """Skip the first `n` values.""" offset: Int ): [DownloadOption!] + + """ + When true, admins can attach CSV data tables linked to this layer by a canonical join column. + """ + enableDataTables: Boolean! enableDownload: Boolean! ftsAr: String ftsDa: String @@ -17593,6 +18228,15 @@ type TableOfContentsItem implements Node { """ nodeId: ID! + """Reads and enables pagination through a set of `OverlayDataTable`.""" + overlayDataTables( + """Only read the first `n` values of the set.""" + first: Int + + """Skip the first `n` values.""" + offset: Int + ): [OverlayDataTable!] + """ stable_id of the parent folder, if any. This property cannot be changed directly. To rearrange items into folders, use the @@ -17813,6 +18457,16 @@ input TableOfContentsItemPatch { If is_folder=false, a DataLayers visibility will be controlled by this item """ dataLayerId: Int + + """ + Overlay attribute name used as the canonical feature ID for linked data tables. Required when enable_data_tables is true. + """ + dataTableJoinColumn: String + + """ + When true, admins can attach CSV data tables linked to this layer by a canonical join column. + """ + enableDataTables: Boolean enableDownload: Boolean geoprocessingReferenceId: String hideChildren: Boolean diff --git a/packages/api/generated-schema.gql b/packages/api/generated-schema.gql index f01a9689b..3ee9c8765 100644 --- a/packages/api/generated-schema.gql +++ b/packages/api/generated-schema.gql @@ -1101,6 +1101,12 @@ input ChangeLogCondition { } enum ChangeLogFieldGroup { + DATA_TABLE_CREATED + DATA_TABLE_DELETED + DATA_TABLE_RENAMED + DATA_TABLE_REPLACED + DATA_TABLE_ROLLBACK + DATA_TABLE_VISUALIZATION_SETTINGS_UPDATED FOLDER_ACL FOLDER_CREATED FOLDER_DELETED @@ -1715,6 +1721,7 @@ input CreateDataUploadInput { clientMutationId: String contentType: String filename: String + processingOptions: JSON projectId: Int replaceTableOfContentsItemId: Int } @@ -2150,6 +2157,7 @@ input CreateMapBookmarkInput { payload verbatim. May be used to track mutations by the client. """ clientMutationId: String + dataTableStates: JSON isPublic: Boolean layerNames: JSON mapDimensions: [Int] @@ -2255,34 +2263,39 @@ type CreateOptionalBasemapLayerPayload { query: Query } -"""All input for the create `OriginalSourceId` mutation.""" -input CreateOriginalSourceIdInput { +"""All input for the `createOverlayDataTableUpload` mutation.""" +input CreateOverlayDataTableUploadInput { """ An arbitrary string value with no semantic meaning. Will be included in the payload verbatim. May be used to track mutations by the client. """ clientMutationId: String - - """The `OriginalSourceId` to be created by this mutation.""" - originalSourceId: OriginalSourceIdInput! + contentType: String + filename: String + processingOptions: JSON + replaceOverlayDataTableId: Int + tocItemId: Int } -"""The output of our create `OriginalSourceId` mutation.""" -type CreateOriginalSourceIdPayload { +"""The output of our `createOverlayDataTableUpload` mutation.""" +type CreateOverlayDataTableUploadPayload { """ The exact same `clientMutationId` that was provided in the mutation input, unchanged and unused. May be used by a client to track mutations. """ clientMutationId: String + overlayDataTableUpload: OverlayDataTableUpload - """The `OriginalSourceId` that was created by this mutation.""" - originalSourceId: OriginalSourceId + """An edge for our `OverlayDataTableUpload`. May be used by Relay 1.""" + overlayDataTableUploadEdge( + """The method to use when ordering `OverlayDataTableUpload`.""" + orderBy: [OverlayDataTableUploadsOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTableUploadsEdge - """An edge for our `OriginalSourceId`. May be used by Relay 1.""" - originalSourceIdEdge( - """The method to use when ordering `OriginalSourceId`.""" - orderBy: [OriginalSourceIdsOrderBy!] = [NATURAL] - ): OriginalSourceIdsEdge + """ + Reads a single `ProjectBackgroundJob` that is related to this `OverlayDataTableUpload`. + """ + projectBackgroundJob: ProjectBackgroundJob """ Our root query field type. Allows us to run any query from our mutation payload. @@ -2483,41 +2496,6 @@ input CreateProjectWithGeographiesInput { slug: String! } -"""All input for the create `PublishedTocItemId` mutation.""" -input CreatePublishedTocItemIdInput { - """ - An arbitrary string value with no semantic meaning. Will be included in the - payload verbatim. May be used to track mutations by the client. - """ - clientMutationId: String - - """The `PublishedTocItemId` to be created by this mutation.""" - publishedTocItemId: PublishedTocItemIdInput! -} - -"""The output of our create `PublishedTocItemId` mutation.""" -type CreatePublishedTocItemIdPayload { - """ - The exact same `clientMutationId` that was provided in the mutation input, - unchanged and unused. May be used by a client to track mutations. - """ - clientMutationId: String - - """The `PublishedTocItemId` that was created by this mutation.""" - publishedTocItemId: PublishedTocItemId - - """An edge for our `PublishedTocItemId`. May be used by Relay 1.""" - publishedTocItemIdEdge( - """The method to use when ordering `PublishedTocItemId`.""" - orderBy: [PublishedTocItemIdsOrderBy!] = [NATURAL] - ): PublishedTocItemIdsEdge - - """ - Our root query field type. Allows us to run any query from our mutation payload. - """ - query: Query -} - """All input for the `createRemoteGeojsonSource` mutation.""" input CreateRemoteGeojsonSourceInput { bounds: [BigFloat] @@ -4257,6 +4235,7 @@ type DataUploadOutput implements Node { } enum DataUploadOutputType { + CSV FLAT_GEOBUF GEO_JSON GEO_TIFF @@ -4301,6 +4280,13 @@ type DataUploadTask implements Node { """Use to upload source data to s3. Must be an admin.""" presignedUploadUrl: String + """ + Format-specific processing instructions supplied by the client at upload time + (e.g. column mapping and CRS for delimited text uploads). Consumed by the + spatial-uploads-handler. + """ + processingOptions: JSON + """ Reads a single `ProjectBackgroundJob` that is related to this `DataUploadTask`. """ @@ -6458,6 +6444,11 @@ type FailDataUploadPayload { } type FeatureFlags { + """ + When true, project admins see the Data Tables tab in layer editors + and related map UI. Controlled from SeaSketch developer settings. + """ + dataTables: Boolean iNaturalistLayers: Boolean } @@ -8105,31 +8096,6 @@ type GetChildFoldersRecursivePayload { query: Query } -"""All input for the `getPublishedCardIdFromDraft` mutation.""" -input GetPublishedCardIdFromDraftInput { - """ - An arbitrary string value with no semantic meaning. Will be included in the - payload verbatim. May be used to track mutations by the client. - """ - clientMutationId: String - draftReportCardId: Int -} - -"""The output of our `getPublishedCardIdFromDraft` mutation.""" -type GetPublishedCardIdFromDraftPayload { - """ - The exact same `clientMutationId` that was provided in the mutation input, - unchanged and unused. May be used by a client to track mutations. - """ - clientMutationId: String - integer: Int - - """ - Our root query field type. Allows us to run any query from our mutation payload. - """ - query: Query -} - type GoogleMapsTileApiSession implements Node { expiresAt: Datetime! id: Int! @@ -8896,6 +8862,11 @@ type MapBookmark { """ clientGeneratedThumbnail: String createdAt: Datetime! + + """ + JSON map of TOC stableId -> { stableId, column?, op?, filters? } for activated overlay data tables. + """ + dataTableStates: JSON! id: UUID! imageId: String isPublic: Boolean! @@ -9419,14 +9390,12 @@ type Mutation { """ input: CreateOptionalBasemapLayerInput! ): CreateOptionalBasemapLayerPayload - - """Creates a single `OriginalSourceId`.""" - createOriginalSourceId( + createOverlayDataTableUpload( """ The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. """ - input: CreateOriginalSourceIdInput! - ): CreateOriginalSourceIdPayload + input: CreateOverlayDataTableUploadInput! + ): CreateOverlayDataTableUploadPayload createPost(message: JSON!, topicId: Int!): Post! """ @@ -9487,14 +9456,6 @@ type Mutation { """ input: CreateProjectWithGeographiesInput! ): CreateProjectPayload - - """Creates a single `PublishedTocItemId`.""" - createPublishedTocItemId( - """ - The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. - """ - input: CreatePublishedTocItemIdInput! - ): CreatePublishedTocItemIdPayload createRemoteGeojsonSource( """ The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. @@ -10235,12 +10196,6 @@ type Mutation { width: Int! ): Sprite getPresignedPMTilesUploadUrl(bytes: BigInt!, filename: String!): PresignedUrl! - getPublishedCardIdFromDraft( - """ - The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. - """ - input: GetPublishedCardIdFromDraftInput! - ): GetPublishedCardIdFromDraftPayload """ Give a user admin access to a project. User must have already joined the project and shared their user profile. @@ -10436,6 +10391,12 @@ type Mutation { """ input: RemoveValidChildSketchClassInput! ): RemoveValidChildSketchClassPayload + renameOverlayDataTable( + """ + The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. + """ + input: RenameOverlayDataTableInput! + ): RenameOverlayDataTablePayload renameReportTab( """ The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. @@ -10516,6 +10477,12 @@ type Mutation { """ input: RevokeApiKeyInput! ): RevokeApiKeyPayload + rollbackOverlayDataTableVersion( + """ + The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. + """ + input: RollbackOverlayDataTableVersionInput! + ): RollbackOverlayDataTableVersionPayload rollbackToArchivedSource( """ The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. @@ -10609,6 +10576,12 @@ type Mutation { """ input: SetForumOrderInput! ): SetForumOrderPayload + setOverlayDataTableVisualizationSettings( + """ + The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. + """ + input: SetOverlayDataTableVisualizationSettingsInput! + ): SetOverlayDataTableVisualizationSettingsPayload """ Admins can use this function to hide the contents of a message. Message will @@ -10678,6 +10651,12 @@ type Mutation { """ input: ShareSpriteInput! ): ShareSpritePayload + softDeleteOverlayDataTable( + """ + The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. + """ + input: SoftDeleteOverlayDataTableInput! + ): SoftDeleteOverlayDataTablePayload """ Superusers only. "Deletes" a sprite but keeps it in the DB in case layers are already referencing it. @@ -10694,6 +10673,12 @@ type Mutation { """ input: SubmitDataUploadInput! ): SubmitDataUploadPayload + submitOverlayDataTableUpload( + """ + The exclusive input argument for this mutation. An object type, make sure to see documentation for this object’s fields. + """ + input: SubmitOverlayDataTableUploadInput! + ): SubmitOverlayDataTableUploadPayload """ Toggle admin access for the given project and user. User must have already @@ -11790,52 +11775,189 @@ enum OptionalBasemapLayersOrderBy { PRIMARY_KEY_DESC } -type OriginalSourceId { - dataSourceId: Int +type OutstandingSurveyInvites { + projectId: Int! + surveyId: Int! + token: String! } -"""An input for mutations affecting `OriginalSourceId`""" -input OriginalSourceIdInput { - dataSourceId: Int +type OverlayDataTable implements Node { + columnStatsRemote: String! + columnStatsUrl: String + createdAt: Datetime + createdBy: Int! + deletedAt: Datetime + id: Int! + joinColumn: String! + name: String! + + """ + A globally unique identifier. Can be used in various places throughout the system to identify this single value. + """ + nodeId: ID! + overlayJoinColumn: String! + parquetRemote: String! + parquetUrl: String + + """Reads a single `Project` that is related to this `OverlayDataTable`.""" + project: Project + projectId: Int! + queryUrl: String + replacedById: Int + + """ + Columns that must appear as filters when this table is displayed on the map. + End users can change the filter values but cannot remove these filters. + """ + requiredFilterColumns: [String] + rowCount: Int! + + """ + Stable logical identity for a data table across version replace and TOC + publish. Draft and published copies share the same UUID. + """ + stableId: UUID! + tableOfContentsItemId: Int! + updatedAt: Datetime + version: Int! + + """ + Columns that may/should be used for creating thematic maps. For example `count` or `density` + """ + visualizationColumns: [String] + + """ + Operations that may/should be used for creating thematic maps. For example `mean` or `max` + """ + visualizationOps: [String] } -"""A connection to a list of `OriginalSourceId` values.""" -type OriginalSourceIdsConnection { +""" +A condition to be used against `OverlayDataTable` object types. All fields are +tested for equality and combined with a logical ‘and.’ +""" +input OverlayDataTableCondition { + """Checks for equality with the object’s `id` field.""" + id: Int + + """Checks for equality with the object’s `projectId` field.""" + projectId: Int +} + +"""A connection to a list of `OverlayDataTable` values.""" +type OverlayDataTablesConnection { """ - A list of edges which contains the `OriginalSourceId` and cursor to aid in pagination. + A list of edges which contains the `OverlayDataTable` and cursor to aid in pagination. """ - edges: [OriginalSourceIdsEdge!]! + edges: [OverlayDataTablesEdge!]! - """A list of `OriginalSourceId` objects.""" - nodes: [OriginalSourceId!]! + """A list of `OverlayDataTable` objects.""" + nodes: [OverlayDataTable!]! """Information to aid in pagination.""" pageInfo: PageInfo! """ - The count of *all* `OriginalSourceId` you could get from the connection. + The count of *all* `OverlayDataTable` you could get from the connection. """ totalCount: Int! } -"""A `OriginalSourceId` edge in the connection.""" -type OriginalSourceIdsEdge { +"""A `OverlayDataTable` edge in the connection.""" +type OverlayDataTablesEdge { """A cursor for use in pagination.""" cursor: Cursor - """The `OriginalSourceId` at the end of the edge.""" - node: OriginalSourceId! + """The `OverlayDataTable` at the end of the edge.""" + node: OverlayDataTable! } -"""Methods to use when ordering `OriginalSourceId`.""" -enum OriginalSourceIdsOrderBy { +"""Methods to use when ordering `OverlayDataTable`.""" +enum OverlayDataTablesOrderBy { + ID_ASC + ID_DESC NATURAL + PRIMARY_KEY_ASC + PRIMARY_KEY_DESC + PROJECT_ID_ASC + PROJECT_ID_DESC } -type OutstandingSurveyInvites { - projectId: Int! - surveyId: Int! - token: String! +type OverlayDataTableUpload implements Node { + contentType: String! + createdAt: Datetime + errorDetails: JSON + filename: String! + id: UUID! + + """ + A globally unique identifier. Can be used in various places throughout the system to identify this single value. + """ + nodeId: ID! + overlayGeostats: JSON! + overlayJoinColumn: String + presignedUploadUrl: String + processingOptions: JSON! + + """ + Reads a single `ProjectBackgroundJob` that is related to this `OverlayDataTableUpload`. + """ + projectBackgroundJob: ProjectBackgroundJob + projectBackgroundJobId: UUID! + replaceOverlayDataTableId: Int + tableOfContentsItemId: Int! + updatedAt: Datetime +} + +""" +A condition to be used against `OverlayDataTableUpload` object types. All fields +are tested for equality and combined with a logical ‘and.’ +""" +input OverlayDataTableUploadCondition { + """Checks for equality with the object’s `id` field.""" + id: UUID + + """Checks for equality with the object’s `projectBackgroundJobId` field.""" + projectBackgroundJobId: UUID +} + +"""A connection to a list of `OverlayDataTableUpload` values.""" +type OverlayDataTableUploadsConnection { + """ + A list of edges which contains the `OverlayDataTableUpload` and cursor to aid in pagination. + """ + edges: [OverlayDataTableUploadsEdge!]! + + """A list of `OverlayDataTableUpload` objects.""" + nodes: [OverlayDataTableUpload!]! + + """Information to aid in pagination.""" + pageInfo: PageInfo! + + """ + The count of *all* `OverlayDataTableUpload` you could get from the connection. + """ + totalCount: Int! +} + +"""A `OverlayDataTableUpload` edge in the connection.""" +type OverlayDataTableUploadsEdge { + """A cursor for use in pagination.""" + cursor: Cursor + + """The `OverlayDataTableUpload` at the end of the edge.""" + node: OverlayDataTableUpload! +} + +"""Methods to use when ordering `OverlayDataTableUpload`.""" +enum OverlayDataTableUploadsOrderBy { + ID_ASC + ID_DESC + NATURAL + PRIMARY_KEY_ASC + PRIMARY_KEY_DESC + PROJECT_BACKGROUND_JOB_ID_ASC + PROJECT_BACKGROUND_JOB_ID_DESC } """Information about pagination in a connection.""" @@ -12429,11 +12551,13 @@ type Project implements Node { hideForums: Boolean! hideOverlays: Boolean! hideSketches: Boolean! + """ - Content-addressed tileset UUIDs whose /v2 requests must include a map - access token. Equals hosted UUIDs from published TOC items (and draft - TOC items for project admins) that are not in the published ACL - document's public list. Empty for projects with no protected overlays. + Content-addressed tileset UUIDs whose hosted tiles/uploads requests + must include a map access token. Equals hosted UUIDs from published + TOC items (and draft TOC items for project admins) that are not in the + published ACL document's public list. Empty for projects with no + protected overlays. """ hostedTileUuidsRequiringAuth: [String!]! id: Int! @@ -12540,9 +12664,10 @@ type Project implements Node { displayed at 48x48 pixels and must be a public url. """ logoUrl: String + """ Short-lived (90m) JWT for authorized overlay tile requests on - tiles.seasketch.org/v2/{ns}/.... Returns null if the user is not signed in. + hosted tiles/uploads URLs with access_token (and optional ns). Returns null if the user is not signed in. Claims include role (admin|user) and project group ids. """ mapAccessToken: String @@ -12645,6 +12770,35 @@ type Project implements Node { orderBy: [OfflineTileSettingsOrderBy!] ): [OfflineTileSetting!]! + """Reads and enables pagination through a set of `OverlayDataTable`.""" + overlayDataTablesConnection( + """Read all values in the set after (below) this cursor.""" + after: Cursor + + """Read all values in the set before (above) this cursor.""" + before: Cursor + + """ + A condition to be used in determining which values should be returned by the collection. + """ + condition: OverlayDataTableCondition + + """Only read the first `n` values of the set.""" + first: Int + + """Only read the last `n` values of the set.""" + last: Int + + """ + Skip the first `n` values from our `after` cursor, an alternative to cursor + based pagination. May not be used with `last`. + """ + offset: Int + + """The method to use when ordering `OverlayDataTable`.""" + orderBy: [OverlayDataTablesOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTablesConnection! + """ Count of all users who have opted into participating in the project, sharing their profile with project administrators. """ @@ -12998,6 +13152,42 @@ type ProjectBackgroundJob implements Node { A globally unique identifier. Can be used in various places throughout the system to identify this single value. """ nodeId: ID! + + """ + Reads a single `OverlayDataTableUpload` that is related to this `ProjectBackgroundJob`. + """ + overlayDataTableUpload: OverlayDataTableUpload + + """ + Reads and enables pagination through a set of `OverlayDataTableUpload`. + """ + overlayDataTableUploadsConnection( + """Read all values in the set after (below) this cursor.""" + after: Cursor + + """Read all values in the set before (above) this cursor.""" + before: Cursor + + """ + A condition to be used in determining which values should be returned by the collection. + """ + condition: OverlayDataTableUploadCondition + + """Only read the first `n` values of the set.""" + first: Int + + """Only read the last `n` values of the set.""" + last: Int + + """ + Skip the first `n` values from our `after` cursor, an alternative to cursor + based pagination. May not be used with `last`. + """ + offset: Int + + """The method to use when ordering `OverlayDataTableUpload`.""" + orderBy: [OverlayDataTableUploadsOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTableUploadsConnection! @deprecated(reason: "Please use overlayDataTableUpload instead") progress: BigFloat progressMessage: String! @@ -13055,6 +13245,7 @@ type ProjectBackgroundJobSubscriptionPayload { enum ProjectBackgroundJobType { ARCGIS_IMPORT CONSOLIDATE_DATA_SOURCES + DATA_TABLE_UPLOAD DATA_UPLOAD REPLACEMENT_UPLOAD } @@ -13536,48 +13727,6 @@ type PublicProjectDetail { supportEmail: String } -type PublishedTocItemId { - id: Int -} - -"""An input for mutations affecting `PublishedTocItemId`""" -input PublishedTocItemIdInput { - id: Int -} - -"""A connection to a list of `PublishedTocItemId` values.""" -type PublishedTocItemIdsConnection { - """ - A list of edges which contains the `PublishedTocItemId` and cursor to aid in pagination. - """ - edges: [PublishedTocItemIdsEdge!]! - - """A list of `PublishedTocItemId` objects.""" - nodes: [PublishedTocItemId!]! - - """Information to aid in pagination.""" - pageInfo: PageInfo! - - """ - The count of *all* `PublishedTocItemId` you could get from the connection. - """ - totalCount: Int! -} - -"""A `PublishedTocItemId` edge in the connection.""" -type PublishedTocItemIdsEdge { - """A cursor for use in pagination.""" - cursor: Cursor - - """The `PublishedTocItemId` at the end of the edge.""" - node: PublishedTocItemId! -} - -"""Methods to use when ordering `PublishedTocItemId`.""" -enum PublishedTocItemIdsOrderBy { - NATURAL -} - """All input for the `publishReport` mutation.""" input PublishReportInput { """ @@ -14372,15 +14521,74 @@ type Query implements Node { """ nodeId: ID! ): OptionalBasemapLayer + overlayDataTable(id: Int!): OverlayDataTable + + """Reads a single `OverlayDataTable` using its globally unique `ID`.""" + overlayDataTableByNodeId( + """ + The globally unique `ID` to be used in selecting a single `OverlayDataTable`. + """ + nodeId: ID! + ): OverlayDataTable + overlayDataTableLinkedTocIsDraft(pid: Int, tocItemId: Int): Boolean + overlayDataTableParquetPublicUrl(pRemote: String): String + + """Reads and enables pagination through a set of `OverlayDataTable`.""" + overlayDataTablesConnection( + """Read all values in the set after (below) this cursor.""" + after: Cursor + + """Read all values in the set before (above) this cursor.""" + before: Cursor + + """ + A condition to be used in determining which values should be returned by the collection. + """ + condition: OverlayDataTableCondition + + """Only read the first `n` values of the set.""" + first: Int + + """Only read the last `n` values of the set.""" + last: Int + + """ + Skip the first `n` values from our `after` cursor, an alternative to cursor + based pagination. May not be used with `last`. + """ + offset: Int + + """The method to use when ordering `OverlayDataTable`.""" + orderBy: [OverlayDataTablesOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTablesConnection + overlayDataTableUpload(id: UUID!): OverlayDataTableUpload + + """ + Reads a single `OverlayDataTableUpload` using its globally unique `ID`. + """ + overlayDataTableUploadByNodeId( + """ + The globally unique `ID` to be used in selecting a single `OverlayDataTableUpload`. + """ + nodeId: ID! + ): OverlayDataTableUpload + overlayDataTableUploadByProjectBackgroundJobId(projectBackgroundJobId: UUID!): OverlayDataTableUpload - """Reads and enables pagination through a set of `OriginalSourceId`.""" - originalSourceIdsConnection( + """ + Reads and enables pagination through a set of `OverlayDataTableUpload`. + """ + overlayDataTableUploadsConnection( """Read all values in the set after (below) this cursor.""" after: Cursor """Read all values in the set before (above) this cursor.""" before: Cursor + """ + A condition to be used in determining which values should be returned by the collection. + """ + condition: OverlayDataTableUploadCondition + """Only read the first `n` values of the set.""" first: Int @@ -14393,9 +14601,9 @@ type Query implements Node { """ offset: Int - """The method to use when ordering `OriginalSourceId`.""" - orderBy: [OriginalSourceIdsOrderBy!] = [NATURAL] - ): OriginalSourceIdsConnection + """The method to use when ordering `OverlayDataTableUpload`.""" + orderBy: [OverlayDataTableUploadsOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTableUploadsConnection post(id: Int!): Post """Reads a single `Post` using its globally unique `ID`.""" @@ -14581,30 +14789,6 @@ type Query implements Node { offset: Int ): [Sprite!] - """Reads and enables pagination through a set of `PublishedTocItemId`.""" - publishedTocItemIdsConnection( - """Read all values in the set after (below) this cursor.""" - after: Cursor - - """Read all values in the set before (above) this cursor.""" - before: Cursor - - """Only read the first `n` values of the set.""" - first: Int - - """Only read the last `n` values of the set.""" - last: Int - - """ - Skip the first `n` values from our `after` cursor, an alternative to cursor - based pagination. May not be used with `last`. - """ - offset: Int - - """The method to use when ordering `PublishedTocItemId`.""" - orderBy: [PublishedTocItemIdsOrderBy!] = [NATURAL] - ): PublishedTocItemIdsConnection - """ Exposes the root query type nested one level down. This is helpful for Relay 1 which can only query top level fields if they are in a particular form. @@ -15169,6 +15353,41 @@ type RemoveValidChildSketchClassPayload { query: Query } +"""All input for the `renameOverlayDataTable` mutation.""" +input RenameOverlayDataTableInput { + """ + An arbitrary string value with no semantic meaning. Will be included in the + payload verbatim. May be used to track mutations by the client. + """ + clientMutationId: String + newName: String + tableId: Int +} + +"""The output of our `renameOverlayDataTable` mutation.""" +type RenameOverlayDataTablePayload { + """ + The exact same `clientMutationId` that was provided in the mutation input, + unchanged and unused. May be used by a client to track mutations. + """ + clientMutationId: String + overlayDataTable: OverlayDataTable + + """An edge for our `OverlayDataTable`. May be used by Relay 1.""" + overlayDataTableEdge( + """The method to use when ordering `OverlayDataTable`.""" + orderBy: [OverlayDataTablesOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTablesEdge + + """Reads a single `Project` that is related to this `OverlayDataTable`.""" + project: Project + + """ + Our root query field type. Allows us to run any query from our mutation payload. + """ + query: Query +} + """All input for the `renameReportTab` mutation.""" input RenameReportTabInput { alternateLanguageSettings: JSON @@ -15774,6 +15993,40 @@ type RevokeApiKeyPayload { query: Query } +"""All input for the `rollbackOverlayDataTableVersion` mutation.""" +input RollbackOverlayDataTableVersionInput { + """ + An arbitrary string value with no semantic meaning. Will be included in the + payload verbatim. May be used to track mutations by the client. + """ + clientMutationId: String + tableId: Int +} + +"""The output of our `rollbackOverlayDataTableVersion` mutation.""" +type RollbackOverlayDataTableVersionPayload { + """ + The exact same `clientMutationId` that was provided in the mutation input, + unchanged and unused. May be used by a client to track mutations. + """ + clientMutationId: String + overlayDataTable: OverlayDataTable + + """An edge for our `OverlayDataTable`. May be used by Relay 1.""" + overlayDataTableEdge( + """The method to use when ordering `OverlayDataTable`.""" + orderBy: [OverlayDataTablesOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTablesEdge + + """Reads a single `Project` that is related to this `OverlayDataTable`.""" + project: Project + + """ + Our root query field type. Allows us to run any query from our mutation payload. + """ + query: Query +} + """All input for the `rollbackToArchivedSource` mutation.""" input RollbackToArchivedSourceInput { """ @@ -16013,6 +16266,43 @@ type SetForumOrderPayload { query: Query } +"""All input for the `setOverlayDataTableVisualizationSettings` mutation.""" +input SetOverlayDataTableVisualizationSettingsInput { + """ + An arbitrary string value with no semantic meaning. Will be included in the + payload verbatim. May be used to track mutations by the client. + """ + clientMutationId: String + requiredFilterColumns: [String] + tableId: Int + visualizationColumns: [String] + visualizationOps: [String] +} + +"""The output of our `setOverlayDataTableVisualizationSettings` mutation.""" +type SetOverlayDataTableVisualizationSettingsPayload { + """ + The exact same `clientMutationId` that was provided in the mutation input, + unchanged and unused. May be used by a client to track mutations. + """ + clientMutationId: String + overlayDataTable: OverlayDataTable + + """An edge for our `OverlayDataTable`. May be used by Relay 1.""" + overlayDataTableEdge( + """The method to use when ordering `OverlayDataTable`.""" + orderBy: [OverlayDataTablesOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTablesEdge + + """Reads a single `Project` that is related to this `OverlayDataTable`.""" + project: Project + + """ + Our root query field type. Allows us to run any query from our mutation payload. + """ + query: Query +} + """All input for the `setPostHiddenByModerator` mutation.""" input SetPostHiddenByModeratorInput { """ @@ -16776,6 +17066,40 @@ enum SketchGeometryType { POLYGON } +"""All input for the `softDeleteOverlayDataTable` mutation.""" +input SoftDeleteOverlayDataTableInput { + """ + An arbitrary string value with no semantic meaning. Will be included in the + payload verbatim. May be used to track mutations by the client. + """ + clientMutationId: String + tableId: Int +} + +"""The output of our `softDeleteOverlayDataTable` mutation.""" +type SoftDeleteOverlayDataTablePayload { + """ + The exact same `clientMutationId` that was provided in the mutation input, + unchanged and unused. May be used by a client to track mutations. + """ + clientMutationId: String + overlayDataTable: OverlayDataTable + + """An edge for our `OverlayDataTable`. May be used by Relay 1.""" + overlayDataTableEdge( + """The method to use when ordering `OverlayDataTable`.""" + orderBy: [OverlayDataTablesOrderBy!] = [PRIMARY_KEY_ASC] + ): OverlayDataTablesEdge + + """Reads a single `Project` that is related to this `OverlayDataTable`.""" + project: Project + + """ + Our root query field type. Allows us to run any query from our mutation payload. + """ + query: Query +} + """All input for the `softDeleteSprite` mutation.""" input SoftDeleteSpriteInput { """ @@ -17037,6 +17361,36 @@ type SubmitDataUploadPayload { query: Query } +"""All input for the `submitOverlayDataTableUpload` mutation.""" +input SubmitOverlayDataTableUploadInput { + """ + An arbitrary string value with no semantic meaning. Will be included in the + payload verbatim. May be used to track mutations by the client. + """ + clientMutationId: String + jobId: UUID +} + +"""The output of our `submitOverlayDataTableUpload` mutation.""" +type SubmitOverlayDataTableUploadPayload { + """ + The exact same `clientMutationId` that was provided in the mutation input, + unchanged and unused. May be used by a client to track mutations. + """ + clientMutationId: String + + """ + Reads a single `Project` that is related to this `ProjectBackgroundJob`. + """ + project: Project + projectBackgroundJob: ProjectBackgroundJob + + """ + Our root query field type. Allows us to run any query from our mutation payload. + """ + query: Query +} + """ The root subscription type: contains realtime events you can subscribe to with the `subscription` operation. """ @@ -17734,6 +18088,20 @@ type TableOfContentsItem implements Node { dataLayerId: Int dataSourceType: DataSourceTypes + """Reads and enables pagination through a set of `ChangeLog`.""" + dataTableChangeLogs( + """Only read the first `n` values of the set.""" + first: Int + + """Skip the first `n` values.""" + offset: Int + ): [ChangeLog!] + + """ + Overlay attribute name used as the canonical feature ID for linked data tables. Required when enable_data_tables is true. + """ + dataTableJoinColumn: String + """Reads and enables pagination through a set of `DownloadOption`.""" downloadOptions( """Only read the first `n` values of the set.""" @@ -17742,6 +18110,11 @@ type TableOfContentsItem implements Node { """Skip the first `n` values.""" offset: Int ): [DownloadOption!] + + """ + When true, admins can attach CSV data tables linked to this layer by a canonical join column. + """ + enableDataTables: Boolean! enableDownload: Boolean! ftsAr: String ftsDa: String @@ -17809,6 +18182,15 @@ type TableOfContentsItem implements Node { """ nodeId: ID! + """Reads and enables pagination through a set of `OverlayDataTable`.""" + overlayDataTables( + """Only read the first `n` values of the set.""" + first: Int + + """Skip the first `n` values.""" + offset: Int + ): [OverlayDataTable!] + """ stable_id of the parent folder, if any. This property cannot be changed directly. To rearrange items into folders, use the @@ -18029,6 +18411,16 @@ input TableOfContentsItemPatch { If is_folder=false, a DataLayers visibility will be controlled by this item """ dataLayerId: Int + + """ + Overlay attribute name used as the canonical feature ID for linked data tables. Required when enable_data_tables is true. + """ + dataTableJoinColumn: String + + """ + When true, admins can attach CSV data tables linked to this layer by a canonical join column. + """ + enableDataTables: Boolean enableDownload: Boolean geoprocessingReferenceId: String hideChildren: Boolean diff --git a/packages/api/migrations/committed/000429.sql b/packages/api/migrations/committed/000429.sql new file mode 100644 index 000000000..d1d7a8868 --- /dev/null +++ b/packages/api/migrations/committed/000429.sql @@ -0,0 +1,844 @@ +--! Previous: sha1:09532ee08a6abd672c0fbed235e5b2c3d6e8fdcf +--! Hash: sha1:6bb7e77902519cb8af23ff3e71d98c944520112a + +-- Enter migration here +alter type data_upload_output_type add value if not exists 'CSV'; + +alter table data_upload_tasks add column if not exists processing_options jsonb; + +comment on column data_upload_tasks.processing_options is 'Format-specific processing instructions supplied by the client at upload time (e.g. column mapping and CRS for delimited text uploads). Consumed by the spatial-uploads-handler.'; + +drop function if exists create_data_upload(text, integer, text, integer); +CREATE OR REPLACE FUNCTION public.create_data_upload(filename text, project_id integer, content_type text, replace_table_of_contents_item_id integer, processing_options jsonb default null) RETURNS public.data_upload_tasks + LANGUAGE plpgsql SECURITY DEFINER + AS $$ + declare + upload data_upload_tasks; + used bigint; + quota bigint; + job project_background_jobs; + begin + if session_is_admin(project_id) then + select projects_data_hosting_quota_used(projects.*), projects_data_hosting_quota(projects.*) into used, quota from projects where id = project_id; + if replace_table_of_contents_item_id is not null and (select exists( + select + data_upload_tasks.id + from + data_upload_tasks + inner join + project_background_jobs + on + data_upload_tasks.project_background_job_id = project_background_jobs.id + where + data_upload_tasks.replace_table_of_contents_item_id is not null and + project_background_jobs.state in ('queued', 'running') and + data_upload_tasks.replace_table_of_contents_item_id = create_data_upload.replace_table_of_contents_item_id + )) then + raise exception 'There is already an active upload task for this layer'; + end if; + if quota - used > 0 then + insert into project_background_jobs ( + project_id, + title, + user_id, + type, + timeout_at + ) values ( + project_id, + ( + case when replace_table_of_contents_item_id is not null then 'Replacement upload ' else '' end + ) || filename, + nullif(current_setting('session.user_id', TRUE), '')::integer, + 'data_upload', + timezone('utc'::text, now()) + interval '15 minutes' + ) + returning * into job; + insert into data_upload_tasks( + filename, + content_type, + project_background_job_id, + replace_table_of_contents_item_id, + processing_options + ) values ( + create_data_upload.filename, + create_data_upload.content_type, + job.id, + create_data_upload.replace_table_of_contents_item_id, + create_data_upload.processing_options + ) returning * into upload; + return upload; + else + raise exception 'data hosting quota exceeded'; + end if; + else + raise exception 'permission denied'; + end if; + end; + $$; + +grant execute on function create_data_upload to seasketch_user; + +drop table if exists overlay_data_table_uploads cascade; +drop table if exists overlay_data_tables cascade; + +create table if not exists overlay_data_tables ( + id integer generated always as identity primary key, + table_of_contents_item_id integer not null references table_of_contents_items(id) on delete cascade, + project_id integer not null references projects(id) on delete cascade, + name text not null, + join_column text not null, + overlay_join_column text not null, + row_count integer not null, + created_at timestamptz default now(), + updated_at timestamptz default now(), + created_by integer not null references users(id), + deleted_at timestamptz, + replaced_by_id integer references overlay_data_tables(id), + version integer not null default 1, + parquet_remote text not null, + column_stats_remote text not null +); + + +create table if not exists overlay_data_table_uploads ( + id uuid primary key default uuid_generate_v4(), + project_background_job_id uuid not null references project_background_jobs(id) on delete cascade, + table_of_contents_item_id integer not null references table_of_contents_items(id) on delete cascade, + filename text not null, + content_type text not null, + processing_options jsonb not null default '{}', + overlay_geostats jsonb not null, + overlay_join_column text, + replace_overlay_data_table_id integer references overlay_data_tables(id), + error_details jsonb, + created_at timestamptz default now(), + updated_at timestamptz default now(), + constraint overlay_data_table_uploads_unique_project_background_job unique (project_background_job_id) +); + +-- Indexes and constraints +create unique index if not exists overlay_data_tables_active_name_per_toc + on overlay_data_tables (table_of_contents_item_id, name) + where deleted_at is null; + +create index if not exists overlay_data_tables_toc_active_idx + on overlay_data_tables (table_of_contents_item_id) + where deleted_at is null; + +create index if not exists overlay_data_tables_project_idx + on overlay_data_tables (project_id); + +do $$ begin + alter table overlay_data_tables + add constraint overlay_data_tables_version_positive check (version > 0); +exception + when duplicate_object then null; +end $$; + +alter type project_background_job_type add value if not exists 'data_table_upload'; + +-- Changelog field groups +do $$ begin + if not exists ( + select 1 from pg_enum e + inner join pg_type t on t.oid = e.enumtypid + where t.typname = 'change_log_field_group' and e.enumlabel = 'data_table:created' + ) then + alter type change_log_field_group add value 'data_table:created'; + end if; +end $$; + +do $$ begin + if not exists ( + select 1 from pg_enum e + inner join pg_type t on t.oid = e.enumtypid + where t.typname = 'change_log_field_group' and e.enumlabel = 'data_table:deleted' + ) then + alter type change_log_field_group add value 'data_table:deleted'; + end if; +end $$; + +do $$ begin + if not exists ( + select 1 from pg_enum e + inner join pg_type t on t.oid = e.enumtypid + where t.typname = 'change_log_field_group' and e.enumlabel = 'data_table:renamed' + ) then + alter type change_log_field_group add value 'data_table:renamed'; + end if; +end $$; + +do $$ begin + if not exists ( + select 1 from pg_enum e + inner join pg_type t on t.oid = e.enumtypid + where t.typname = 'change_log_field_group' and e.enumlabel = 'data_table:replaced' + ) then + alter type change_log_field_group add value 'data_table:replaced'; + end if; +end $$; + +do $$ begin + if not exists ( + select 1 from pg_enum e + inner join pg_type t on t.oid = e.enumtypid + where t.typname = 'change_log_field_group' and e.enumlabel = 'data_table:rollback' + ) then + alter type change_log_field_group add value 'data_table:rollback'; + end if; +end $$; + +comment on table overlay_data_tables is '@omit delete'; +comment on table overlay_data_table_uploads is '@omit delete'; + +alter table overlay_data_tables enable row level security; +alter table overlay_data_table_uploads enable row level security; + +grant select on overlay_data_tables to anon; +grant select on overlay_data_tables to seasketch_user; +grant select on overlay_data_table_uploads to seasketch_user; + +create or replace function public.overlay_data_table_linked_toc_is_draft( + toc_item_id integer, + pid integer +) returns boolean +language sql +stable +security definer +as $$ + select exists ( + select 1 + from table_of_contents_items + where id = toc_item_id + and project_id = pid + and is_draft = true + ); +$$; + +grant execute on function public.overlay_data_table_linked_toc_is_draft(integer, integer) to seasketch_user; + +drop policy if exists overlay_data_tables_select on overlay_data_tables; +create policy overlay_data_tables_select on overlay_data_tables + for select using ( + session_is_admin(project_id) + or ( + exists ( + select 1 from table_of_contents_items toc + where toc.id = overlay_data_tables.table_of_contents_item_id + and toc.is_draft = false + and _session_on_toc_item_acl(toc.path) + ) + ) + ); + +drop policy if exists overlay_data_tables_insert on overlay_data_tables; +create policy overlay_data_tables_insert on overlay_data_tables + for insert with check ( + session_is_admin(project_id) + and overlay_data_table_linked_toc_is_draft( + table_of_contents_item_id, + project_id + ) + ); + +drop policy if exists overlay_data_tables_update on overlay_data_tables; +create policy overlay_data_tables_update on overlay_data_tables + for update using ( + session_is_admin(project_id) + and overlay_data_table_linked_toc_is_draft( + table_of_contents_item_id, + project_id + ) + ); + +drop policy if exists overlay_data_tables_delete on overlay_data_tables; +create policy overlay_data_tables_delete on overlay_data_tables + for delete using ( + session_is_admin(project_id) + and overlay_data_table_linked_toc_is_draft( + table_of_contents_item_id, + project_id + ) + ); + +drop policy if exists overlay_data_table_uploads_select on overlay_data_table_uploads; +create policy overlay_data_table_uploads_select on overlay_data_table_uploads + for select using ( + session_is_admin(( + select project_id from table_of_contents_items where id = overlay_data_table_uploads.table_of_contents_item_id + )) + ); + +drop policy if exists overlay_data_table_uploads_insert on overlay_data_table_uploads; +create policy overlay_data_table_uploads_insert on overlay_data_table_uploads + for insert with check ( + session_is_admin(( + select project_id from table_of_contents_items where id = overlay_data_table_uploads.table_of_contents_item_id + )) + ); + +create or replace function overlay_data_tables_draft_toc_has_changes() +returns trigger +language plpgsql +security definer +as $$ +declare + pid int; + toc_is_draft boolean; +begin + if tg_op = 'DELETE' then + select project_id, table_of_contents_items.is_draft into pid, toc_is_draft + from table_of_contents_items where id = old.table_of_contents_item_id; + else + select project_id, table_of_contents_items.is_draft into pid, toc_is_draft + from table_of_contents_items where id = new.table_of_contents_item_id; + end if; + if toc_is_draft then + update projects set draft_table_of_contents_has_changes = true where id = pid; + end if; + if tg_op = 'DELETE' then + return old; + end if; + return new; +end; +$$; + +drop trigger if exists overlay_data_tables_draft_toc_has_changes on overlay_data_tables; +create trigger overlay_data_tables_draft_toc_has_changes + after insert or update or delete on overlay_data_tables + for each row execute function overlay_data_tables_draft_toc_has_changes(); + +create or replace function trg_publish_overlay_data_tables_for_toc_item() +returns trigger +language plpgsql +security definer +as $$ +declare + draft_toc_id int; +begin + if new.is_draft = false and new.is_folder = false then + select id into draft_toc_id + from table_of_contents_items + where stable_id = new.stable_id + and project_id = new.project_id + and table_of_contents_items.is_draft = true + limit 1; + if draft_toc_id is not null then + insert into overlay_data_tables ( + table_of_contents_item_id, + project_id, + name, + join_column, + overlay_join_column, + row_count, + created_by, + version, + parquet_remote, + column_stats_remote + ) + select + new.id, + odt.project_id, + odt.name, + odt.join_column, + odt.overlay_join_column, + odt.row_count, + odt.created_by, + odt.version, + odt.parquet_remote, + odt.column_stats_remote + from overlay_data_tables odt + where odt.table_of_contents_item_id = draft_toc_id + and odt.deleted_at is null; + end if; + end if; + return new; +end; +$$; + +drop trigger if exists publish_overlay_data_tables_for_toc_item on table_of_contents_items; +create trigger publish_overlay_data_tables_for_toc_item + after insert on table_of_contents_items + for each row execute function trg_publish_overlay_data_tables_for_toc_item(); + +create or replace function table_of_contents_items_overlay_data_tables(item public.table_of_contents_items) +returns setof overlay_data_tables +language sql +stable +security definer +as $$ + select odt.* + from overlay_data_tables odt + where odt.table_of_contents_item_id = item.id + and odt.deleted_at is null + and ( + session_is_admin(item.project_id) + or (item.is_draft = false and _session_on_toc_item_acl(item.path)) + ); +$$; + +comment on function table_of_contents_items_overlay_data_tables(public.table_of_contents_items) is '@simpleCollections only'; + +grant execute on function table_of_contents_items_overlay_data_tables(public.table_of_contents_items) to seasketch_user; + +create or replace function public.table_of_contents_items_project_background_jobs(item public.table_of_contents_items) +returns setof public.project_background_jobs +language sql +stable +security definer +as $$ + select * + from project_background_jobs + where session_is_admin(item.project_id) + and ( + id = ( + select project_background_job_id + from esri_feature_layer_conversion_tasks + where table_of_contents_item_id = item.id + limit 1 + ) + or id in ( + select odtu.project_background_job_id + from overlay_data_table_uploads odtu + where odtu.table_of_contents_item_id = item.id + ) + ); +$$; + +create or replace function create_overlay_data_table_upload( + toc_item_id integer, + filename text, + content_type text, + processing_options jsonb default '{}'::jsonb, + replace_overlay_data_table_id integer default null +) returns overlay_data_table_uploads +language plpgsql +security definer +as $$ +declare + upload overlay_data_table_uploads; + job project_background_jobs; + pid int; + geostats jsonb; +begin + select project_id into pid + from table_of_contents_items + where id = toc_item_id and is_draft = true and is_folder = false; + if pid is null then + raise exception 'Draft layer table of contents item not found'; + end if; + if not session_is_admin(pid) then + raise exception 'permission denied'; + end if; + + select ds.geostats into geostats + from table_of_contents_items toc + inner join data_layers dl on dl.id = toc.data_layer_id + inner join data_sources ds on ds.id = dl.data_source_id + where toc.id = toc_item_id; + if geostats is null then + raise exception 'Overlay layer has no geostats'; + end if; + + if replace_overlay_data_table_id is not null then + if not exists ( + select 1 from overlay_data_tables + where id = replace_overlay_data_table_id + and table_of_contents_item_id = toc_item_id + and deleted_at is null + ) then + raise exception 'Replace target data table not found or not active'; + end if; + end if; + + if exists ( + select 1 + from overlay_data_table_uploads odtu + inner join project_background_jobs pbj on pbj.id = odtu.project_background_job_id + where odtu.table_of_contents_item_id = toc_item_id + and odtu.filename = create_overlay_data_table_upload.filename + and pbj.state in ('queued', 'running') + ) then + raise exception 'There is already an active data table upload for this file'; + end if; + + insert into project_background_jobs ( + project_id, + title, + user_id, + type, + timeout_at + ) values ( + pid, + (case when replace_overlay_data_table_id is not null then 'Replacement data table ' else 'Data table ' end) || filename, + nullif(current_setting('session.user_id', true), '')::integer, + 'data_table_upload', + timezone('utc', now()) + interval '15 minutes' + ) returning * into job; + + insert into overlay_data_table_uploads ( + project_background_job_id, + table_of_contents_item_id, + filename, + content_type, + processing_options, + overlay_geostats, + overlay_join_column, + replace_overlay_data_table_id + ) values ( + job.id, + toc_item_id, + create_overlay_data_table_upload.filename, + content_type, + coalesce(processing_options, '{}'::jsonb), + geostats, + processing_options->>'overlayJoinColumn', + replace_overlay_data_table_id + ) returning * into upload; + + return upload; +end; +$$; + +grant execute on function create_overlay_data_table_upload(integer, text, text, jsonb, integer) to seasketch_user; + +create or replace function submit_overlay_data_table_upload(job_id uuid) +returns project_background_jobs +language plpgsql +security definer +as $$ +declare + job project_background_jobs; + pid int; +begin + select project_id into pid + from project_background_jobs + where id = submit_overlay_data_table_upload.job_id; + if not session_is_admin(pid) then + raise exception 'permission denied'; + end if; + if not exists ( + select 1 from overlay_data_table_uploads where project_background_job_id = job_id + ) then + raise exception 'Data table upload not found for job'; + end if; + update project_background_jobs + set state = 'running', progress_message = 'uploaded', started_at = now() + where id = job_id + returning * into job; + perform graphile_worker.add_job( + 'processDataTableUpload', + json_build_object('jobId', job.id), + max_attempts := 1 + ); + return job; +end; +$$; + +grant execute on function submit_overlay_data_table_upload(uuid) to seasketch_user; + +create or replace function rename_overlay_data_table(table_id integer, new_name text) +returns overlay_data_tables +language plpgsql +security definer +as $$ +declare + row overlay_data_tables; + old_name text; + editor_id int; +begin + select * into row from overlay_data_tables where id = table_id and deleted_at is null; + if row is null then + raise exception 'Active data table not found'; + end if; + if not session_is_admin(row.project_id) then + raise exception 'permission denied'; + end if; + if not exists ( + select 1 from table_of_contents_items + where id = row.table_of_contents_item_id and is_draft = true + ) then + raise exception 'Can only rename data tables on draft layers'; + end if; + old_name := row.name; + update overlay_data_tables + set name = new_name, updated_at = now() + where id = table_id + returning * into row; + + editor_id := nullif(current_setting('session.user_id', true), '')::int; + if editor_id is not null then + perform record_changelog( + row.project_id, + editor_id, + 'overlay_data_table', + row.id, + 'data_table:renamed'::change_log_field_group, + jsonb_build_object('name', old_name), + jsonb_build_object('name', new_name), + null, null, + jsonb_build_object('table_of_contents_item_id', row.table_of_contents_item_id, 'version', row.version) + ); + end if; + return row; +end; +$$; + +grant execute on function rename_overlay_data_table(integer, text) to seasketch_user; + +create or replace function soft_delete_overlay_data_table(table_id integer) +returns overlay_data_tables +language plpgsql +security definer +as $$ +declare + row overlay_data_tables; + editor_id int; +begin + select * into row from overlay_data_tables where id = table_id and deleted_at is null; + if row is null then + raise exception 'Active data table not found'; + end if; + if not session_is_admin(row.project_id) then + raise exception 'permission denied'; + end if; + if not exists ( + select 1 from table_of_contents_items + where id = row.table_of_contents_item_id and is_draft = true + ) then + raise exception 'Can only delete data tables on draft layers'; + end if; + + update overlay_data_tables + set deleted_at = now(), updated_at = now() + where id = table_id + returning * into row; + + editor_id := nullif(current_setting('session.user_id', true), '')::int; + if editor_id is not null then + perform record_changelog( + row.project_id, + editor_id, + 'overlay_data_table', + row.id, + 'data_table:deleted'::change_log_field_group, + jsonb_build_object('name', row.name, 'version', row.version), + '{}'::jsonb, + null, null, + jsonb_build_object('table_of_contents_item_id', row.table_of_contents_item_id) + ); + end if; + return row; +end; +$$; + +grant execute on function soft_delete_overlay_data_table(integer) to seasketch_user; + +create or replace function rollback_overlay_data_table_version(table_id integer) +returns overlay_data_tables +language plpgsql +security definer +as $$ +declare + target overlay_data_tables; + successor overlay_data_tables; + editor_id int; +begin + select * into target from overlay_data_tables where id = table_id; + if target is null then + raise exception 'Data table not found'; + end if; + if target.deleted_at is null then + select * into target + from overlay_data_tables + where replaced_by_id = table_id + and deleted_at is not null + order by version desc + limit 1; + if target is null then + raise exception 'No previous version found to rollback to'; + end if; + end if; + if not session_is_admin(target.project_id) then + raise exception 'permission denied'; + end if; + if not exists ( + select 1 from table_of_contents_items + where id = target.table_of_contents_item_id and is_draft = true + ) then + raise exception 'Can only rollback data tables on draft layers'; + end if; + + select * into successor + from overlay_data_tables + where id = target.replaced_by_id + and deleted_at is null + limit 1; + if successor is null then + raise exception 'No successor version found to rollback from'; + end if; + + delete from overlay_data_tables where id = successor.id; + + update overlay_data_tables + set deleted_at = null, replaced_by_id = null, updated_at = now() + where id = target.id + returning * into target; + + editor_id := nullif(current_setting('session.user_id', true), '')::int; + if editor_id is not null then + perform record_changelog( + target.project_id, + editor_id, + 'overlay_data_table', + target.id, + 'data_table:rollback'::change_log_field_group, + jsonb_build_object('removed_version', successor.version), + jsonb_build_object('name', target.name, 'version', target.version), + null, null, + jsonb_build_object( + 'table_of_contents_item_id', target.table_of_contents_item_id, + 'removed_id', successor.id + ) + ); + end if; + return target; +end; +$$; + +grant execute on function rollback_overlay_data_table_version(integer) to seasketch_user; + +create or replace function complete_overlay_data_table_upload( + job_id uuid, + p_name text, + p_join_column text, + p_overlay_join_column text, + p_row_count integer, + p_parquet_remote text, + p_column_stats_remote text +) returns overlay_data_tables +language plpgsql +security definer +as $$ +declare + upload overlay_data_table_uploads; + job project_background_jobs; + new_row overlay_data_tables; + old_row overlay_data_tables; + editor_id int; + new_version int := 1; +begin + select * into upload + from overlay_data_table_uploads + where project_background_job_id = job_id; + if upload is null then + raise exception 'Upload not found for job'; + end if; + + select * into job from project_background_jobs where id = job_id; + + if upload.replace_overlay_data_table_id is not null then + select * into old_row + from overlay_data_tables + where id = upload.replace_overlay_data_table_id + and deleted_at is null; + if old_row is null then + raise exception 'Replace target no longer active'; + end if; + new_version := old_row.version + 1; + update overlay_data_tables + set deleted_at = now(), updated_at = now() + where id = old_row.id; + end if; + + insert into overlay_data_tables ( + table_of_contents_item_id, + project_id, + name, + join_column, + overlay_join_column, + row_count, + created_by, + version, + parquet_remote, + column_stats_remote + ) values ( + upload.table_of_contents_item_id, + job.project_id, + p_name, + p_join_column, + p_overlay_join_column, + p_row_count, + coalesce(job.user_id, nullif(current_setting('session.user_id', true), '')::integer), + new_version, + p_parquet_remote, + p_column_stats_remote + ) returning * into new_row; + + if upload.replace_overlay_data_table_id is not null then + update overlay_data_tables + set replaced_by_id = new_row.id, updated_at = now() + where id = old_row.id; + + editor_id := nullif(current_setting('session.user_id', true), '')::int; + if editor_id is not null then + perform record_changelog( + new_row.project_id, + editor_id, + 'overlay_data_table', + new_row.id, + 'data_table:replaced'::change_log_field_group, + jsonb_build_object('name', old_row.name, 'version', old_row.version, 'id', old_row.id), + jsonb_build_object('name', new_row.name, 'version', new_row.version, 'id', new_row.id), + null, null, + jsonb_build_object('table_of_contents_item_id', new_row.table_of_contents_item_id) + ); + end if; + else + editor_id := coalesce(job.user_id, nullif(current_setting('session.user_id', true), '')::int); + if editor_id is not null then + perform record_changelog( + new_row.project_id, + editor_id, + 'overlay_data_table', + new_row.id, + 'data_table:created'::change_log_field_group, + '{}'::jsonb, + jsonb_build_object('name', new_row.name, 'version', new_row.version), + null, null, + jsonb_build_object('table_of_contents_item_id', new_row.table_of_contents_item_id) + ); + end if; + end if; + + update project_background_jobs + set state = 'complete', progress = 1, progress_message = 'complete', error_message = null + where id = job_id; + + return new_row; +end; +$$; + +revoke all on function complete_overlay_data_table_upload(uuid, text, text, text, integer, text, text) from public; + +create or replace function fail_overlay_data_table_upload( + job_id uuid, + error_message text, + error_details jsonb default null +) returns void +language plpgsql +security definer +as $$ +begin + update overlay_data_table_uploads + set error_details = coalesce(fail_overlay_data_table_upload.error_details, error_details), + updated_at = now() + where project_background_job_id = job_id; + + update project_background_jobs + set state = 'failed', progress_message = 'failed', error_message = fail_overlay_data_table_upload.error_message + where id = job_id; +end; +$$; + +revoke all on function fail_overlay_data_table_upload(uuid, text, jsonb) from public; diff --git a/packages/api/migrations/committed/000430.sql b/packages/api/migrations/committed/000430.sql new file mode 100644 index 000000000..fcedcd36e --- /dev/null +++ b/packages/api/migrations/committed/000430.sql @@ -0,0 +1,518 @@ +--! Previous: sha1:6bb7e77902519cb8af23ff3e71d98c944520112a +--! Hash: sha1:e9e5727083a7f5c1a647218152adf0d50a29ff92 + +-- Enter migration here + +alter table table_of_contents_items + add column if not exists enable_data_tables boolean not null default false, + add column if not exists data_table_join_column text; + +alter table table_of_contents_items + drop constraint if exists table_of_contents_items_data_tables_join_column_required; + +alter table table_of_contents_items + add constraint table_of_contents_items_data_tables_join_column_required + check ( + not enable_data_tables + or ( + data_table_join_column is not null + and length(trim(data_table_join_column)) > 0 + ) + ); + +grant select (enable_data_tables, data_table_join_column) on table_of_contents_items to anon; +grant update (enable_data_tables, data_table_join_column) on table_of_contents_items to seasketch_user; + +comment on column table_of_contents_items.enable_data_tables is + 'When true, admins can attach CSV data tables linked to this layer by a canonical join column.'; +comment on column table_of_contents_items.data_table_join_column is + 'Overlay attribute name used as the canonical feature ID for linked data tables. Required when enable_data_tables is true.'; + +create or replace function trg_copy_data_table_settings_from_draft_toc() +returns trigger +language plpgsql +as $$ +begin + if new.is_draft = false and not new.is_folder then + select + draft.enable_data_tables, + draft.data_table_join_column + into + new.enable_data_tables, + new.data_table_join_column + from table_of_contents_items draft + where + draft.project_id = new.project_id + and draft.stable_id = new.stable_id + and draft.is_draft = true; + end if; + return new; +end; +$$; + +drop trigger if exists copy_data_table_settings_from_draft_toc on table_of_contents_items; +create trigger copy_data_table_settings_from_draft_toc + before insert on table_of_contents_items + for each row execute function trg_copy_data_table_settings_from_draft_toc(); + +drop index if exists overlay_data_tables_active_name_per_toc; + +create or replace function create_overlay_data_table_upload( + toc_item_id integer, + filename text, + content_type text, + processing_options jsonb default '{}'::jsonb, + replace_overlay_data_table_id integer default null +) returns overlay_data_table_uploads +language plpgsql +security definer +as $$ +declare + upload overlay_data_table_uploads; + job project_background_jobs; + pid int; + geostats jsonb; + join_col text; + enabled boolean; +begin + select project_id, enable_data_tables, data_table_join_column + into pid, enabled, join_col + from table_of_contents_items + where id = toc_item_id and is_draft = true and is_folder = false; + if pid is null then + raise exception 'Draft layer table of contents item not found'; + end if; + if not session_is_admin(pid) then + raise exception 'permission denied'; + end if; + if not coalesce(enabled, false) then + raise exception 'Data tables are not enabled for this layer'; + end if; + if join_col is null or length(trim(join_col)) = 0 then + raise exception 'Data table join column is not configured for this layer'; + end if; + + select ds.geostats into geostats + from table_of_contents_items toc + inner join data_layers dl on dl.id = toc.data_layer_id + inner join data_sources ds on ds.id = dl.data_source_id + where toc.id = toc_item_id; + if geostats is null then + raise exception 'Overlay layer has no geostats'; + end if; + + if replace_overlay_data_table_id is not null then + if not exists ( + select 1 from overlay_data_tables + where id = replace_overlay_data_table_id + and table_of_contents_item_id = toc_item_id + and deleted_at is null + ) then + raise exception 'Replace target data table not found or not active'; + end if; + end if; + + if exists ( + select 1 + from overlay_data_table_uploads odtu + inner join project_background_jobs pbj on pbj.id = odtu.project_background_job_id + where odtu.table_of_contents_item_id = toc_item_id + and odtu.filename = create_overlay_data_table_upload.filename + and pbj.state in ('queued', 'running') + ) then + raise exception 'There is already an active data table upload for this file'; + end if; + + insert into project_background_jobs ( + project_id, + title, + user_id, + type, + timeout_at + ) values ( + pid, + (case when replace_overlay_data_table_id is not null then 'Replacement data table ' else 'Data table ' end) || filename, + nullif(current_setting('session.user_id', true), '')::integer, + 'data_table_upload', + timezone('utc', now()) + interval '15 minutes' + ) returning * into job; + + insert into overlay_data_table_uploads ( + project_background_job_id, + table_of_contents_item_id, + filename, + content_type, + processing_options, + overlay_geostats, + overlay_join_column, + replace_overlay_data_table_id + ) values ( + job.id, + toc_item_id, + create_overlay_data_table_upload.filename, + content_type, + coalesce(processing_options, '{}'::jsonb), + geostats, + join_col, + replace_overlay_data_table_id + ) returning * into upload; + + return upload; +end; +$$; + +grant execute on function create_overlay_data_table_upload(integer, text, text, jsonb, integer) to seasketch_user; + +create or replace function fail_overlay_data_table_upload( + job_id uuid, + error_message text, + error_details jsonb default null +) returns void +language plpgsql +security definer +as $$ +begin + update overlay_data_table_uploads odtu + set error_details = coalesce(fail_overlay_data_table_upload.error_details, odtu.error_details), + updated_at = now() + where odtu.project_background_job_id = job_id; + + -- Only fail jobs still in flight; never clobber a completed job. + update project_background_jobs + set + state = 'failed', + progress_message = case + when fail_overlay_data_table_upload.error_message = 'Timed out' then 'timeout' + else 'failed' + end, + error_message = fail_overlay_data_table_upload.error_message + where id = job_id + and state in ('queued', 'running'); +end; +$$; + +revoke all on function fail_overlay_data_table_upload(uuid, text, jsonb) from public; + +create or replace function submit_overlay_data_table_upload(job_id uuid) +returns project_background_jobs +language plpgsql +security definer +as $$ +declare + job project_background_jobs; + pid int; +begin + select project_id into pid + from project_background_jobs + where id = submit_overlay_data_table_upload.job_id; + if not session_is_admin(pid) then + raise exception 'permission denied'; + end if; + if not exists ( + select 1 from overlay_data_table_uploads where project_background_job_id = job_id + ) then + raise exception 'Data table upload not found for job'; + end if; + update project_background_jobs + set + state = 'running', + progress_message = 'uploaded', + started_at = now(), + timeout_at = timezone('utc', now()) + interval '60 seconds' + where id = job_id + returning * into job; + perform graphile_worker.add_job( + 'processDataTableUpload', + json_build_object('jobId', job.id), + max_attempts := 1 + ); + return job; +end; +$$; + +grant execute on function submit_overlay_data_table_upload(uuid) to seasketch_user; + +create or replace function table_of_contents_items_data_table_change_logs(item table_of_contents_items) + returns setof change_logs + language sql + security definer + stable + as $$ + select * from change_logs + where entity_type = 'overlay_data_table' + and net_zero_changes = false + and (meta->>'table_of_contents_item_id')::int = item.id + order by last_at desc; + $$; + +grant execute on function table_of_contents_items_data_table_change_logs(item table_of_contents_items) to seasketch_user; + +comment on function table_of_contents_items_data_table_change_logs(item table_of_contents_items) is '@simpleCollections only'; + +create or replace function overlay_data_table_parquet_public_url(p_remote text) +returns text +language plpgsql +stable +as $$ +declare + base text; + key text; +begin + if p_remote is null or length(trim(p_remote)) = 0 then + return null; + end if; + + -- Prefer an optional session override (tests); otherwise the public uploads host. + base := coalesce( + nullif(trim(current_setting('seasketch.uploads_base_url', true)), ''), + 'https://uploads.seasketch.org' + ); + + key := regexp_replace(p_remote, '^r2://[^/]+/', ''); + if key is null or length(trim(key)) = 0 then + return null; + end if; + + return rtrim(base, '/') || '/' || key; +end; +$$; + +grant execute on function overlay_data_table_parquet_public_url(text) to seasketch_user; + +-- Replacement uploads complete in the background worker without a session. +-- Use job.user_id for the changelog editor, same as the initial upload path. +create or replace function complete_overlay_data_table_upload( + job_id uuid, + p_name text, + p_join_column text, + p_overlay_join_column text, + p_row_count integer, + p_parquet_remote text, + p_column_stats_remote text +) returns overlay_data_tables +language plpgsql +security definer +as $$ +declare + upload overlay_data_table_uploads; + job project_background_jobs; + new_row overlay_data_tables; + old_row overlay_data_tables; + editor_id int; + new_version int := 1; +begin + select * into upload + from overlay_data_table_uploads + where project_background_job_id = job_id; + if upload is null then + raise exception 'Upload not found for job'; + end if; + + select * into job from project_background_jobs where id = job_id; + -- Don't resurrect a job that already timed out or was cancelled. + if job.state not in ('queued', 'running') then + raise exception 'Job is no longer active (state: %)', job.state; + end if; + + if upload.replace_overlay_data_table_id is not null then + select * into old_row + from overlay_data_tables + where id = upload.replace_overlay_data_table_id + and deleted_at is null; + if old_row is null then + raise exception 'Replace target no longer active'; + end if; + new_version := old_row.version + 1; + update overlay_data_tables + set deleted_at = now(), updated_at = now() + where id = old_row.id; + end if; + + insert into overlay_data_tables ( + table_of_contents_item_id, + project_id, + name, + join_column, + overlay_join_column, + row_count, + created_by, + version, + parquet_remote, + column_stats_remote, + visualization_columns, + visualization_ops, + required_filter_columns + ) values ( + upload.table_of_contents_item_id, + job.project_id, + p_name, + p_join_column, + p_overlay_join_column, + p_row_count, + coalesce(job.user_id, nullif(current_setting('session.user_id', true), '')::integer), + new_version, + p_parquet_remote, + p_column_stats_remote, + coalesce(old_row.visualization_columns, '{}'), + coalesce(old_row.visualization_ops, '{mean}'), + coalesce(old_row.required_filter_columns, '{}') + ) returning * into new_row; + + if upload.replace_overlay_data_table_id is not null then + update overlay_data_tables + set replaced_by_id = new_row.id, updated_at = now() + where id = old_row.id; + + editor_id := coalesce(job.user_id, nullif(current_setting('session.user_id', true), '')::int); + if editor_id is not null then + perform record_changelog( + new_row.project_id, + editor_id, + 'overlay_data_table', + new_row.id, + 'data_table:replaced'::change_log_field_group, + jsonb_build_object( + 'name', old_row.name, + 'version', old_row.version, + 'id', old_row.id, + 'parquet_url', overlay_data_table_parquet_public_url(old_row.parquet_remote) + ), + jsonb_build_object('name', new_row.name, 'version', new_row.version, 'id', new_row.id), + null, null, + jsonb_build_object('table_of_contents_item_id', new_row.table_of_contents_item_id) + ); + end if; + else + editor_id := coalesce(job.user_id, nullif(current_setting('session.user_id', true), '')::int); + if editor_id is not null then + perform record_changelog( + new_row.project_id, + editor_id, + 'overlay_data_table', + new_row.id, + 'data_table:created'::change_log_field_group, + '{}'::jsonb, + jsonb_build_object('name', new_row.name, 'version', new_row.version), + null, null, + jsonb_build_object('table_of_contents_item_id', new_row.table_of_contents_item_id) + ); + end if; + end if; + + update project_background_jobs + set state = 'complete', progress = 1, progress_message = 'complete', error_message = null + where id = job_id; + + return new_row; +end; +$$; + +revoke all on function complete_overlay_data_table_upload(uuid, text, text, text, integer, text, text) from public; + +alter table overlay_data_tables add column if not exists visualization_columns text[] default '{}'; +comment on column overlay_data_tables.visualization_columns is + 'Columns that may/should be used for creating thematic maps. For example `count` or `density`'; + +alter table overlay_data_tables add column if not exists visualization_ops text[] default '{mean}'; +comment on column overlay_data_tables.visualization_ops is + 'Operations that may/should be used for creating thematic maps. For example `mean` or `max`'; + +alter table overlay_data_tables add column if not exists required_filter_columns text[] default '{}'; +comment on column overlay_data_tables.required_filter_columns is + 'Columns that must appear as filters when this table is displayed on the map. End users can change the filter values but cannot remove these filters.'; + +do $$ begin + if not exists ( + select 1 from pg_enum e + inner join pg_type t on t.oid = e.enumtypid + where t.typname = 'change_log_field_group' and e.enumlabel = 'data_table:visualization_settings_updated' + ) then + alter type change_log_field_group add value 'data_table:visualization_settings_updated'; + end if; +end $$; + +-- Signature change (added required_filter_columns); replace rather than overload. +drop function if exists set_overlay_data_table_visualization_settings(integer, text[], text[]); + +create or replace function set_overlay_data_table_visualization_settings( + table_id integer, + visualization_columns text[], + visualization_ops text[], + required_filter_columns text[] default '{}' +) +returns overlay_data_tables +language plpgsql +security definer +as $$ +declare + row overlay_data_tables; + old_columns text[]; + old_ops text[]; + old_required text[]; + editor_id int; + invalid_ops text[]; +begin + select * into row from overlay_data_tables where id = table_id and deleted_at is null; + if row is null then + raise exception 'Active data table not found'; + end if; + if not session_is_admin(row.project_id) then + raise exception 'permission denied'; + end if; + if not exists ( + select 1 from table_of_contents_items + where id = row.table_of_contents_item_id and is_draft = true + ) then + raise exception 'Can only update visualization settings on draft layers'; + end if; + + select array_agg(op) into invalid_ops + from unnest(coalesce(set_overlay_data_table_visualization_settings.visualization_ops, '{}')) op + where op not in ('count', 'sum', 'mean', 'min', 'max', 'median'); + if invalid_ops is not null and array_length(invalid_ops, 1) > 0 then + raise exception 'Invalid visualization op(s): %', array_to_string(invalid_ops, ', '); + end if; + + old_columns := row.visualization_columns; + old_ops := row.visualization_ops; + old_required := row.required_filter_columns; + + update overlay_data_tables + set visualization_columns = coalesce(set_overlay_data_table_visualization_settings.visualization_columns, '{}'), + visualization_ops = coalesce(set_overlay_data_table_visualization_settings.visualization_ops, '{}'), + required_filter_columns = coalesce(set_overlay_data_table_visualization_settings.required_filter_columns, '{}'), + updated_at = now() + where id = table_id + returning * into row; + + editor_id := nullif(current_setting('session.user_id', true), '')::int; + if editor_id is not null then + perform record_changelog( + row.project_id, + editor_id, + 'overlay_data_table', + row.id, + 'data_table:visualization_settings_updated'::change_log_field_group, + jsonb_build_object( + 'name', row.name, + 'version', row.version, + 'visualizationColumns', old_columns, + 'visualizationOps', old_ops, + 'requiredFilterColumns', old_required + ), + jsonb_build_object( + 'name', row.name, + 'version', row.version, + 'visualizationColumns', row.visualization_columns, + 'visualizationOps', row.visualization_ops, + 'requiredFilterColumns', row.required_filter_columns + ), + null, null, + jsonb_build_object('table_of_contents_item_id', row.table_of_contents_item_id, 'version', row.version) + ); + end if; + return row; +end; +$$; + +grant execute on function set_overlay_data_table_visualization_settings(integer, text[], text[], text[]) to seasketch_user; diff --git a/packages/api/migrations/committed/000431.sql b/packages/api/migrations/committed/000431.sql new file mode 100644 index 000000000..275864184 --- /dev/null +++ b/packages/api/migrations/committed/000431.sql @@ -0,0 +1,448 @@ +--! Previous: sha1:e9e5727083a7f5c1a647218152adf0d50a29ff92 +--! Hash: sha1:bb8c6bd2247a8005d6dbb3ccae1f3c0e85f54760 + +-- Enter migration here + +-- Move overlay data table publishing out of per-row TOC insert triggers and into +-- publish_table_of_contents. Large projects have 1000+ layers; a set-based copy +-- after TOC rows exist is much cheaper than N trigger firings. + +drop trigger if exists publish_overlay_data_tables_for_toc_item on table_of_contents_items; +drop function if exists trg_publish_overlay_data_tables_for_toc_item(); + +drop trigger if exists copy_data_table_settings_from_draft_toc on table_of_contents_items; +drop function if exists trg_copy_data_table_settings_from_draft_toc(); + +create or replace function public.publish_table_of_contents("projectId" integer) +returns setof public.table_of_contents_items +language plpgsql +security definer +as $$ +declare + v_editor int; + v_layer_count int; + lid int; + item table_of_contents_items; + source_id int; + copied_source_id int; + acl_type access_control_list_type; + acl_id int; + orig_acl_id int; + new_toc_id int; + new_interactivity_settings_id int; +begin + -- check permissions + if session_is_admin("projectId") = false then + raise 'Permission denied. Must be a project admin'; + end if; + + -- delete existing published table of contents items, layers, sources, and interactivity settings + -- (published overlay_data_tables cascade-delete with their TOC rows) + delete from + interactivity_settings + where + id in ( + select + data_layers.interactivity_settings_id + from + data_layers + inner join + table_of_contents_items + on + data_layers.id = table_of_contents_items.data_layer_id + where + table_of_contents_items.project_id = "projectId" and + is_draft = false + ); + + delete from data_sources where data_sources.id in ( + select + data_source_id + from + data_layers + inner join + table_of_contents_items + on + data_layers.id = table_of_contents_items.data_layer_id + where + table_of_contents_items.project_id = "projectId" and + is_draft = false + ); + delete from data_layers where id in ( + select + data_layer_id + from + table_of_contents_items + where + project_id = "projectId" and + is_draft = false + ); + delete from + table_of_contents_items + where + project_id = "projectId" and + is_draft = false; + + -- one-by-one, copy related layers and link table of contents items + for item in + select + * + from + table_of_contents_items + where + is_draft = true and + project_id = "projectId" + loop + if item.is_folder = false then + -- copy interactivity settings first + insert into interactivity_settings ( + type, + short_template, + long_template, + cursor, + title + ) select + type, + short_template, + long_template, + cursor, + title + from + interactivity_settings + where + interactivity_settings.id = ( + select interactivity_settings_id from data_layers where data_layers.id = item.data_layer_id + ) + returning + id + into + new_interactivity_settings_id; + + insert into data_layers ( + project_id, + data_source_id, + source_layer, + sublayer, + sublayer_type, + render_under, + mapbox_gl_styles, + interactivity_settings_id, + z_index + ) + select "projectId", + data_source_id, + source_layer, + sublayer, + sublayer_type, + render_under, + mapbox_gl_styles, + new_interactivity_settings_id, + z_index + from + data_layers + where + id = item.data_layer_id + returning id into lid; + else + lid = item.data_layer_id; + end if; + -- TODO: this will have to be modified with the addition of any columns + insert into table_of_contents_items ( + is_draft, + project_id, + path, + stable_id, + parent_stable_id, + title, + is_folder, + show_radio_children, + is_click_off_only, + metadata, + bounds, + data_layer_id, + sort_index, + hide_children, + geoprocessing_reference_id, + translated_props, + enable_download, + enable_data_tables, + data_table_join_column + ) values ( + false, + "projectId", + item.path, + item.stable_id, + item.parent_stable_id, + item.title, + item.is_folder, + item.show_radio_children, + item.is_click_off_only, + item.metadata, + item.bounds, + lid, + item.sort_index, + item.hide_children, + item.geoprocessing_reference_id, + item.translated_props, + item.enable_download, + item.enable_data_tables, + item.data_table_join_column + ) returning id into new_toc_id; + select + type, id into acl_type, orig_acl_id + from + access_control_lists + where + table_of_contents_item_id = ( + select + id + from + table_of_contents_items + where is_draft = true and stable_id = item.stable_id + ); + -- copy access control list settings + if acl_type != 'public' then + update + access_control_lists + set type = acl_type + where table_of_contents_item_id = new_toc_id + returning id into acl_id; + if acl_type = 'group' then + insert into + access_control_list_groups ( + access_control_list_id, + group_id + ) + select + acl_id, + group_id + from + access_control_list_groups + where + access_control_list_id = orig_acl_id; + end if; + end if; + end loop; + + -- Copy active draft overlay data tables onto newly published TOC items in one + -- statement (matched by stable_id). Soft-deleted draft history is skipped. + insert into overlay_data_tables ( + table_of_contents_item_id, + project_id, + name, + join_column, + overlay_join_column, + row_count, + created_by, + version, + parquet_remote, + column_stats_remote, + visualization_columns, + visualization_ops, + required_filter_columns + ) + select + published_toc.id, + odt.project_id, + odt.name, + odt.join_column, + odt.overlay_join_column, + odt.row_count, + odt.created_by, + odt.version, + odt.parquet_remote, + odt.column_stats_remote, + odt.visualization_columns, + odt.visualization_ops, + odt.required_filter_columns + from overlay_data_tables odt + inner join table_of_contents_items draft_toc + on draft_toc.id = odt.table_of_contents_item_id + inner join table_of_contents_items published_toc + on published_toc.project_id = draft_toc.project_id + and published_toc.stable_id = draft_toc.stable_id + and published_toc.is_draft = false + where draft_toc.project_id = "projectId" + and draft_toc.is_draft = true + and draft_toc.is_folder = false + and odt.deleted_at is null; + + -- one-by-one, copy related sources and update foreign keys of layers + for source_id in + select distinct(data_source_id) from data_layers where id in ( + select + data_layer_id + from + table_of_contents_items + where + is_draft = false and + project_id = "projectId" and + is_folder = false + ) + loop + -- TODO: This function will have to be updated whenever the schema + -- changes since these columns are hard coded... no way around it. + insert into data_sources ( + project_id, + type, + attribution, + bounds, + maxzoom, + minzoom, + url, + scheme, + tiles, + tile_size, + encoding, + buffer, + cluster, + cluster_max_zoom, + cluster_properties, + cluster_radius, + generate_id, + line_metrics, + promote_id, + tolerance, + coordinates, + urls, + query_parameters, + use_device_pixel_ratio, + import_type, + original_source_url, + enhanced_security, + byte_length, + supports_dynamic_layers, + uploaded_source_filename, + uploaded_source_layername, + normalized_source_object_key, + normalized_source_bytes, + geostats, + upload_task_id, + translated_props, + arcgis_fetch_strategy, + created_by + ) + select + "projectId", + type, + attribution, + bounds, + maxzoom, + minzoom, + url, + scheme, + tiles, + tile_size, + encoding, + buffer, + cluster, + cluster_max_zoom, + cluster_properties, + cluster_radius, + generate_id, + line_metrics, + promote_id, + tolerance, + coordinates, + urls, + query_parameters, + use_device_pixel_ratio, + import_type, + original_source_url, + enhanced_security, + byte_length, + supports_dynamic_layers, + uploaded_source_filename, + uploaded_source_layername, + normalized_source_object_key, + normalized_source_bytes, + geostats, + upload_task_id, + translated_props, + arcgis_fetch_strategy, + created_by + from + data_sources + where + id = source_id + returning id into copied_source_id; + -- copy data_upload_outputs + insert into data_upload_outputs ( + data_source_id, + project_id, + type, + created_at, + url, + remote, + is_original, + size, + filename, + original_filename, + source_processing_job_key, + epsg + ) select + copied_source_id, + project_id, + type, + created_at, + url, + remote, + is_original, + size, + filename, + original_filename, + source_processing_job_key, + epsg + from + data_upload_outputs + where + data_source_id = source_id; + -- update data_layers that should now reference the copy + update + data_layers + set data_source_id = copied_source_id + where + data_source_id = source_id and + data_layers.id in (( + select distinct(data_layer_id) from table_of_contents_items where is_draft = false and + project_id = "projectId" and + is_folder = false + )); + end loop; + update + projects + set + draft_table_of_contents_has_changes = false, + table_of_contents_last_published = now() + where + id = "projectId"; + + v_editor := nullif(current_setting('session.user_id', true), '')::int; + if v_editor is not null then + select count(*)::int into v_layer_count + from table_of_contents_items + where project_id = "projectId" + and is_draft = true + and is_folder = false; + + perform record_changelog( + "projectId", + v_editor, + 'projects', + "projectId", + 'layers:published'::change_log_field_group, + '{}'::jsonb, + jsonb_build_object('layer_count', v_layer_count), + null, + null, + null + ); + end if; + -- return items + return query select * from table_of_contents_items + where project_id = "projectId" and is_draft = false; +end; +$$; + +comment on function public.publish_table_of_contents(integer) is + 'Copies draft TOC to published; sets project flags. Records change_logs (layers:published) with draft layer_count when session.user_id is set.'; diff --git a/packages/api/migrations/committed/000432.sql b/packages/api/migrations/committed/000432.sql new file mode 100644 index 000000000..de90a9c71 --- /dev/null +++ b/packages/api/migrations/committed/000432.sql @@ -0,0 +1,678 @@ +--! Previous: sha1:bb8c6bd2247a8005d6dbb3ccae1f3c0e85f54760 +--! Hash: sha1:9d7cf102f1004e681cf9a6a0461123ee670222b2 + +-- Enter migration here + +-- ============================================================================= +-- overlay_data_tables.stable_id: logical identity across replace + publish +-- ============================================================================= + +-- Public/anonymous viewers need this computed collection on published TOC items +-- (table select is already granted to anon; execute was seasketch_user-only). +grant execute on function table_of_contents_items_overlay_data_tables(public.table_of_contents_items) to anon; + +alter table overlay_data_tables + add column if not exists stable_id uuid; + +update overlay_data_tables +set stable_id = uuid_generate_v4() +where stable_id is null; + +alter table overlay_data_tables + alter column stable_id set default uuid_generate_v4(), + alter column stable_id set not null; + +comment on column overlay_data_tables.stable_id is + 'Stable logical identity for a data table across version replace and TOC publish. Draft and published copies share the same UUID.'; + +create unique index if not exists overlay_data_tables_active_stable_id_per_toc + on overlay_data_tables (table_of_contents_item_id, stable_id) + where deleted_at is null; + +-- Preserve stable_id on replace; new uploads get the column default. +create or replace function complete_overlay_data_table_upload( + job_id uuid, + p_name text, + p_join_column text, + p_overlay_join_column text, + p_row_count integer, + p_parquet_remote text, + p_column_stats_remote text +) returns overlay_data_tables +language plpgsql +security definer +as $$ +declare + upload overlay_data_table_uploads; + job project_background_jobs; + new_row overlay_data_tables; + old_row overlay_data_tables; + editor_id int; + new_version int := 1; +begin + select * into upload + from overlay_data_table_uploads + where project_background_job_id = job_id; + if upload is null then + raise exception 'Upload not found for job'; + end if; + + select * into job from project_background_jobs where id = job_id; + -- Don't resurrect a job that already timed out or was cancelled. + if job.state not in ('queued', 'running') then + raise exception 'Job is no longer active (state: %)', job.state; + end if; + + if upload.replace_overlay_data_table_id is not null then + select * into old_row + from overlay_data_tables + where id = upload.replace_overlay_data_table_id + and deleted_at is null; + if old_row is null then + raise exception 'Replace target no longer active'; + end if; + new_version := old_row.version + 1; + update overlay_data_tables + set deleted_at = now(), updated_at = now() + where id = old_row.id; + end if; + + insert into overlay_data_tables ( + table_of_contents_item_id, + project_id, + name, + join_column, + overlay_join_column, + row_count, + created_by, + version, + parquet_remote, + column_stats_remote, + visualization_columns, + visualization_ops, + required_filter_columns, + stable_id + ) values ( + upload.table_of_contents_item_id, + job.project_id, + p_name, + p_join_column, + p_overlay_join_column, + p_row_count, + coalesce(job.user_id, nullif(current_setting('session.user_id', true), '')::integer), + new_version, + p_parquet_remote, + p_column_stats_remote, + coalesce(old_row.visualization_columns, '{}'), + coalesce(old_row.visualization_ops, '{mean}'), + coalesce(old_row.required_filter_columns, '{}'), + coalesce(old_row.stable_id, uuid_generate_v4()) + ) returning * into new_row; + + if upload.replace_overlay_data_table_id is not null then + update overlay_data_tables + set replaced_by_id = new_row.id, updated_at = now() + where id = old_row.id; + + editor_id := coalesce(job.user_id, nullif(current_setting('session.user_id', true), '')::int); + if editor_id is not null then + perform record_changelog( + new_row.project_id, + editor_id, + 'overlay_data_table', + new_row.id, + 'data_table:replaced'::change_log_field_group, + jsonb_build_object( + 'name', old_row.name, + 'version', old_row.version, + 'id', old_row.id, + 'parquet_url', overlay_data_table_parquet_public_url(old_row.parquet_remote) + ), + jsonb_build_object('name', new_row.name, 'version', new_row.version, 'id', new_row.id), + null, null, + jsonb_build_object('table_of_contents_item_id', new_row.table_of_contents_item_id) + ); + end if; + else + editor_id := coalesce(job.user_id, nullif(current_setting('session.user_id', true), '')::int); + if editor_id is not null then + perform record_changelog( + new_row.project_id, + editor_id, + 'overlay_data_table', + new_row.id, + 'data_table:created'::change_log_field_group, + '{}'::jsonb, + jsonb_build_object('name', new_row.name, 'version', new_row.version), + null, null, + jsonb_build_object('table_of_contents_item_id', new_row.table_of_contents_item_id) + ); + end if; + end if; + + update project_background_jobs + set state = 'complete', progress = 1, progress_message = 'complete', error_message = null + where id = job_id; + + return new_row; +end; +$$; + +-- Copy stable_id when publishing draft overlay data tables. +create or replace function public.publish_table_of_contents("projectId" integer) +returns setof public.table_of_contents_items +language plpgsql +security definer +as $$ +declare + v_editor int; + v_layer_count int; + lid int; + item table_of_contents_items; + source_id int; + copied_source_id int; + acl_type access_control_list_type; + acl_id int; + orig_acl_id int; + new_toc_id int; + new_interactivity_settings_id int; +begin + -- check permissions + if session_is_admin("projectId") = false then + raise 'Permission denied. Must be a project admin'; + end if; + + -- delete existing published table of contents items, layers, sources, and interactivity settings + -- (published overlay_data_tables cascade-delete with their TOC rows) + delete from + interactivity_settings + where + id in ( + select + data_layers.interactivity_settings_id + from + data_layers + inner join + table_of_contents_items + on + data_layers.id = table_of_contents_items.data_layer_id + where + table_of_contents_items.project_id = "projectId" and + is_draft = false + ); + + delete from data_sources where data_sources.id in ( + select + data_source_id + from + data_layers + inner join + table_of_contents_items + on + data_layers.id = table_of_contents_items.data_layer_id + where + table_of_contents_items.project_id = "projectId" and + is_draft = false + ); + delete from data_layers where id in ( + select + data_layer_id + from + table_of_contents_items + where + project_id = "projectId" and + is_draft = false + ); + delete from + table_of_contents_items + where + project_id = "projectId" and + is_draft = false; + + -- one-by-one, copy related layers and link table of contents items + for item in + select + * + from + table_of_contents_items + where + is_draft = true and + project_id = "projectId" + loop + if item.is_folder = false then + -- copy interactivity settings first + insert into interactivity_settings ( + type, + short_template, + long_template, + cursor, + title + ) select + type, + short_template, + long_template, + cursor, + title + from + interactivity_settings + where + interactivity_settings.id = ( + select interactivity_settings_id from data_layers where data_layers.id = item.data_layer_id + ) + returning + id + into + new_interactivity_settings_id; + + insert into data_layers ( + project_id, + data_source_id, + source_layer, + sublayer, + sublayer_type, + render_under, + mapbox_gl_styles, + interactivity_settings_id, + z_index + ) + select "projectId", + data_source_id, + source_layer, + sublayer, + sublayer_type, + render_under, + mapbox_gl_styles, + new_interactivity_settings_id, + z_index + from + data_layers + where + id = item.data_layer_id + returning id into lid; + else + lid = item.data_layer_id; + end if; + -- TODO: this will have to be modified with the addition of any columns + insert into table_of_contents_items ( + is_draft, + project_id, + path, + stable_id, + parent_stable_id, + title, + is_folder, + show_radio_children, + is_click_off_only, + metadata, + bounds, + data_layer_id, + sort_index, + hide_children, + geoprocessing_reference_id, + translated_props, + enable_download, + enable_data_tables, + data_table_join_column + ) values ( + false, + "projectId", + item.path, + item.stable_id, + item.parent_stable_id, + item.title, + item.is_folder, + item.show_radio_children, + item.is_click_off_only, + item.metadata, + item.bounds, + lid, + item.sort_index, + item.hide_children, + item.geoprocessing_reference_id, + item.translated_props, + item.enable_download, + item.enable_data_tables, + item.data_table_join_column + ) returning id into new_toc_id; + select + type, id into acl_type, orig_acl_id + from + access_control_lists + where + table_of_contents_item_id = ( + select + id + from + table_of_contents_items + where is_draft = true and stable_id = item.stable_id + ); + -- copy access control list settings + if acl_type != 'public' then + update + access_control_lists + set type = acl_type + where table_of_contents_item_id = new_toc_id + returning id into acl_id; + if acl_type = 'group' then + insert into + access_control_list_groups ( + access_control_list_id, + group_id + ) + select + acl_id, + group_id + from + access_control_list_groups + where + access_control_list_id = orig_acl_id; + end if; + end if; + end loop; + + -- Copy active draft overlay data tables onto newly published TOC items in one + -- statement (matched by TOC stable_id). Soft-deleted draft history is skipped. + -- Data table stable_id is preserved so bookmarks/prefs survive publish. + insert into overlay_data_tables ( + table_of_contents_item_id, + project_id, + name, + join_column, + overlay_join_column, + row_count, + created_by, + version, + parquet_remote, + column_stats_remote, + visualization_columns, + visualization_ops, + required_filter_columns, + stable_id + ) + select + published_toc.id, + odt.project_id, + odt.name, + odt.join_column, + odt.overlay_join_column, + odt.row_count, + odt.created_by, + odt.version, + odt.parquet_remote, + odt.column_stats_remote, + odt.visualization_columns, + odt.visualization_ops, + odt.required_filter_columns, + odt.stable_id + from overlay_data_tables odt + inner join table_of_contents_items draft_toc + on draft_toc.id = odt.table_of_contents_item_id + inner join table_of_contents_items published_toc + on published_toc.project_id = draft_toc.project_id + and published_toc.stable_id = draft_toc.stable_id + and published_toc.is_draft = false + where draft_toc.project_id = "projectId" + and draft_toc.is_draft = true + and draft_toc.is_folder = false + and odt.deleted_at is null; + + -- one-by-one, copy related sources and update foreign keys of layers + for source_id in + select distinct(data_source_id) from data_layers where id in ( + select + data_layer_id + from + table_of_contents_items + where + is_draft = false and + project_id = "projectId" and + is_folder = false + ) + loop + -- TODO: This function will have to be updated whenever the schema + -- changes since these columns are hard coded... no way around it. + insert into data_sources ( + project_id, + type, + attribution, + bounds, + maxzoom, + minzoom, + url, + scheme, + tiles, + tile_size, + encoding, + buffer, + cluster, + cluster_max_zoom, + cluster_properties, + cluster_radius, + generate_id, + line_metrics, + promote_id, + tolerance, + coordinates, + urls, + query_parameters, + use_device_pixel_ratio, + import_type, + original_source_url, + enhanced_security, + byte_length, + supports_dynamic_layers, + uploaded_source_filename, + uploaded_source_layername, + normalized_source_object_key, + normalized_source_bytes, + geostats, + upload_task_id, + translated_props, + arcgis_fetch_strategy, + created_by + ) + select + "projectId", + type, + attribution, + bounds, + maxzoom, + minzoom, + url, + scheme, + tiles, + tile_size, + encoding, + buffer, + cluster, + cluster_max_zoom, + cluster_properties, + cluster_radius, + generate_id, + line_metrics, + promote_id, + tolerance, + coordinates, + urls, + query_parameters, + use_device_pixel_ratio, + import_type, + original_source_url, + enhanced_security, + byte_length, + supports_dynamic_layers, + uploaded_source_filename, + uploaded_source_layername, + normalized_source_object_key, + normalized_source_bytes, + geostats, + upload_task_id, + translated_props, + arcgis_fetch_strategy, + created_by + from + data_sources + where + id = source_id + returning id into copied_source_id; + -- copy data_upload_outputs + insert into data_upload_outputs ( + data_source_id, + project_id, + type, + created_at, + url, + remote, + is_original, + size, + filename, + original_filename, + source_processing_job_key, + epsg + ) select + copied_source_id, + project_id, + type, + created_at, + url, + remote, + is_original, + size, + filename, + original_filename, + source_processing_job_key, + epsg + from + data_upload_outputs + where + data_source_id = source_id; + -- update data_layers that should now reference the copy + update + data_layers + set data_source_id = copied_source_id + where + data_source_id = source_id and + data_layers.id in (( + select distinct(data_layer_id) from table_of_contents_items where is_draft = false and + project_id = "projectId" and + is_folder = false + )); + end loop; + update + projects + set + draft_table_of_contents_has_changes = false, + table_of_contents_last_published = now() + where + id = "projectId"; + + v_editor := nullif(current_setting('session.user_id', true), '')::int; + if v_editor is not null then + select count(*)::int into v_layer_count + from table_of_contents_items + where project_id = "projectId" + and is_draft = true + and is_folder = false; + + perform record_changelog( + "projectId", + v_editor, + 'projects', + "projectId", + 'layers:published'::change_log_field_group, + '{}'::jsonb, + jsonb_build_object('layer_count', v_layer_count), + null, + null, + null + ); + end if; + -- return items + return query select * from table_of_contents_items + where project_id = "projectId" and is_draft = false; +end; +$$; + +-- ============================================================================= +-- map_bookmarks.data_table_states +-- ============================================================================= + +alter table map_bookmarks + add column if not exists data_table_states jsonb not null default '{}'::jsonb; + +comment on column map_bookmarks.data_table_states is + 'JSON map of TOC stableId -> { stableId, column?, op?, filters? } for activated overlay data tables.'; + +grant select(data_table_states) on table map_bookmarks to anon; + +drop function if exists create_map_bookmark(text, boolean, jsonb, text[], integer, jsonb, jsonb, integer[], integer[], jsonb, text, jsonb, jsonb, text); + +create or replace function public.create_map_bookmark( + slug text, + "isPublic" boolean, + style jsonb, + "visibleDataLayers" text[], + "selectedBasemap" integer, + "basemapOptionalLayerStates" jsonb, + "cameraOptions" jsonb, + "mapDimensions" integer[], + "visibleSketches" integer[], + "sidebarState" jsonb, + "basemapName" text, + "layerNames" jsonb, + "sketchNames" jsonb, + "clientGeneratedThumbnail" text, + "dataTableStates" jsonb default '{}'::jsonb +) returns public.map_bookmarks +language plpgsql +security definer +as $$ +declare + bookmark map_bookmarks; + pid int; +begin + select id into pid from projects where projects.slug = create_map_bookmark.slug; + if session_has_project_access(pid) then + insert into map_bookmarks ( + project_id, + user_id, + is_public, + style, + visible_data_layers, + selected_basemap, + basemap_optional_layer_states, + camera_options, + map_dimensions, + visible_sketches, + sidebar_state, + basemap_name, + layer_names, + sketch_names, + client_generated_thumbnail, + data_table_states + ) values ( + pid, + nullif(current_setting('session.user_id', TRUE), '')::int, + "isPublic", + create_map_bookmark.style, + "visibleDataLayers", + "selectedBasemap", + "basemapOptionalLayerStates", + "cameraOptions", + "mapDimensions", + "visibleSketches", + "sidebarState", + "basemapName", + "layerNames", + "sketchNames", + "clientGeneratedThumbnail", + coalesce("dataTableStates", '{}'::jsonb) + ) returning * into bookmark; + return bookmark; + else + raise exception 'Permission denied'; + end if; +end; +$$; + +grant execute on function create_map_bookmark to seasketch_user; diff --git a/packages/api/process-env.d.ts b/packages/api/process-env.d.ts index 299fd9926..796a36d23 100644 --- a/packages/api/process-env.d.ts +++ b/packages/api/process-env.d.ts @@ -32,6 +32,8 @@ declare namespace NodeJS { NORMALIZED_SPATIAL_UPLOADS_BUCKET: string; SPATIAL_UPLOADS_LAMBDA_DEV_HANDLER?: string; SPATIAL_UPLOADS_LAMBDA_ARN?: string; + DATA_TABLES_LAMBDA_DEV_HANDLER?: string; + DATA_TABLES_HANDLER_LAMBDA_ARN?: string; /** Geostats PII classifier — optional warm ping on createDataUpload */ GEOSTATS_PII_CLASSIFIER_ARN?: string; R2_ACCESS_KEY_ID: string; diff --git a/packages/api/schema.sql b/packages/api/schema.sql index 681e18dee..5ce7f2d50 100644 --- a/packages/api/schema.sql +++ b/packages/api/schema.sql @@ -251,7 +251,13 @@ CREATE TYPE public.change_log_field_group AS ENUM ( 'resolvable_layer_comments:created', 'resolvable_layer_comments:responded', 'resolvable_layer_comments:resolved', - 'resolvable_layer_comments:reopened' + 'resolvable_layer_comments:reopened', + 'data_table:created', + 'data_table:deleted', + 'data_table:renamed', + 'data_table:replaced', + 'data_table:rollback', + 'data_table:visualization_settings_updated' ); @@ -307,7 +313,8 @@ CREATE TYPE public.data_upload_output_type AS ENUM ( 'XMLMetadata', 'NetCDF', 'ReportingFlatgeobufV1', - 'ReportingCOG' + 'ReportingCOG', + 'CSV' ); @@ -789,7 +796,8 @@ CREATE TYPE public.project_background_job_type AS ENUM ( 'data_upload', 'arcgis_import', 'consolidate_data_sources', - 'replacement_upload' + 'replacement_upload', + 'data_table_upload' ); @@ -984,10 +992,9 @@ CREATE TYPE public.spatial_metric_type AS ENUM ( 'presence_table', 'contextualized_mean', 'overlay_area', - 'column_stats', 'column_values', - 'raster_stats', - 'distance_to_shore' + 'distance_to_shore', + 'raster_stats' ); @@ -1672,8 +1679,8 @@ CREATE TABLE public.sketch_classes ( filter_api_version integer DEFAULT 1 NOT NULL, filter_api_server_location text, is_geography_clipping_enabled boolean DEFAULT false NOT NULL, - report_id integer, draft_report_id integer, + report_id integer, preview_new_reports boolean DEFAULT false NOT NULL, CONSTRAINT sketch_classes_geoprocessing_client_url_check CHECK ((geoprocessing_client_url ~* 'https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,255}\.[a-z]{2,9}\y([-a-zA-Z0-9@:%_\+.~#?&//=]*)$'::text)), CONSTRAINT sketch_classes_geoprocessing_project_url_check CHECK ((geoprocessing_project_url ~* 'https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,255}\.[a-z]{2,9}\y([-a-zA-Z0-9@:%_\+.~#?&//=]*)$'::text)), @@ -3738,6 +3745,7 @@ $$; CREATE FUNCTION public.generate_export_id(id integer, export_id text, body jsonb) RETURNS text LANGUAGE plpgsql IMMUTABLE + SET search_path TO 'public', 'pg_catalog' AS $$ declare collected_text text; @@ -3760,6 +3768,7 @@ CREATE FUNCTION public.generate_export_id(id integer, export_id text, body jsonb CREATE FUNCTION public.generate_label(id integer, body jsonb) RETURNS text LANGUAGE plpgsql IMMUTABLE + SET search_path TO 'public', 'pg_catalog' AS $$ declare collected_text text; @@ -4976,7 +4985,8 @@ CREATE TABLE public.map_bookmarks ( basemap_name text, layer_names jsonb, sketch_names jsonb, - client_generated_thumbnail text + client_generated_thumbnail text, + data_table_states jsonb DEFAULT '{}'::jsonb NOT NULL ); @@ -4994,6 +5004,13 @@ COMMENT ON TABLE public.map_bookmarks IS '@omit create'; COMMENT ON COLUMN public.map_bookmarks.client_generated_thumbnail IS 'Generated by clients. Should not be used if authorative thumbnail (image_id) is available.'; +-- +-- Name: COLUMN map_bookmarks.data_table_states; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.map_bookmarks.data_table_states IS 'JSON map of TOC stableId -> { stableId, column?, op?, filters? } for activated overlay data tables.'; + + -- -- Name: bookmark_by_id(uuid); Type: FUNCTION; Schema: public; Owner: - -- @@ -5023,83 +5040,6 @@ CREATE FUNCTION public.bookmark_data(id text) RETURNS jsonb COMMENT ON FUNCTION public.bookmark_data(id text) IS '@omit'; --- --- Name: build_layer_interactivity_acl_payload(integer, integer, integer, text); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.build_layer_interactivity_acl_payload(p_acl_id integer, p_exclude_group_id integer DEFAULT NULL::integer, p_include_group_id integer DEFAULT NULL::integer, p_type_override text DEFAULT NULL::text, OUT acl_summary jsonb, OUT acl_blob jsonb) RETURNS record - LANGUAGE plpgsql SECURITY DEFINER - SET search_path TO 'public', 'pg_temp' - AS $$ -declare - v_acl_type access_control_list_type; - v_project_id int; - v_eff text; -begin - select acl.type, acl.project_id - into v_acl_type, v_project_id - from access_control_lists acl - where acl.id = p_acl_id; - - if not found then - acl_summary := null; - acl_blob := null; - return; - end if; - - v_eff := coalesce(p_type_override, v_acl_type::text); - - if v_eff in ('public', 'admins_only') then - acl_summary := jsonb_build_object('type', v_eff, 'groups', '[]'::jsonb); - acl_blob := jsonb_build_object('type', v_eff, 'groups', '[]'::jsonb); - return; - end if; - - with members as ( - select distinct s.id, s.name - from ( - select pg.id, pg.name - from access_control_list_groups aclg - join project_groups pg on pg.id = aclg.group_id and pg.project_id = v_project_id - where aclg.access_control_list_id = p_acl_id - and (p_exclude_group_id is null or aclg.group_id is distinct from p_exclude_group_id) - union - select pg.id, pg.name - from project_groups pg - where p_include_group_id is not null - and pg.id = p_include_group_id - and pg.project_id = v_project_id - ) s - ) - select - jsonb_build_object( - 'type', 'group', - 'groups', coalesce( - (select to_jsonb(array_agg(m.name order by m.name)) from members m), - '[]'::jsonb - ) - ), - jsonb_build_object( - 'type', 'group', - 'groups', coalesce( - (select jsonb_agg(jsonb_build_object('id', m.id, 'name', m.name) order by m.id) from members m), - '[]'::jsonb - ) - ) - into acl_summary, acl_blob; - - return; -end; -$$; - - --- --- Name: FUNCTION build_layer_interactivity_acl_payload(p_acl_id integer, p_exclude_group_id integer, p_include_group_id integer, p_type_override text, OUT acl_summary jsonb, OUT acl_blob jsonb); Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON FUNCTION public.build_layer_interactivity_acl_payload(p_acl_id integer, p_exclude_group_id integer, p_include_group_id integer, p_type_override text, OUT acl_summary jsonb, OUT acl_blob jsonb) IS 'Builds AclSummary JSON (type + group names) and blob JSON (type + {id,name}[]) for one ACL; exclude/include group ids simulate membership before INSERT/DELETE.'; - - -- -- Name: bump_parent_collection_updated_at(integer, integer); Type: FUNCTION; Schema: public; Owner: - -- @@ -5901,6 +5841,193 @@ CREATE FUNCTION public.collection_as_geojson(id integer) RETURNS jsonb COMMENT ON FUNCTION public.collection_as_geojson(id integer) IS '@omit'; +-- +-- Name: overlay_data_tables; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.overlay_data_tables ( + id integer NOT NULL, + table_of_contents_item_id integer NOT NULL, + project_id integer NOT NULL, + name text NOT NULL, + join_column text NOT NULL, + overlay_join_column text NOT NULL, + row_count integer NOT NULL, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now(), + created_by integer NOT NULL, + deleted_at timestamp with time zone, + replaced_by_id integer, + version integer DEFAULT 1 NOT NULL, + parquet_remote text NOT NULL, + column_stats_remote text NOT NULL, + visualization_columns text[] DEFAULT '{}'::text[], + visualization_ops text[] DEFAULT '{mean}'::text[], + required_filter_columns text[] DEFAULT '{}'::text[], + stable_id uuid DEFAULT public.uuid_generate_v4() NOT NULL, + CONSTRAINT overlay_data_tables_version_positive CHECK ((version > 0)) +); + + +-- +-- Name: TABLE overlay_data_tables; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.overlay_data_tables IS '@omit delete'; + + +-- +-- Name: COLUMN overlay_data_tables.visualization_columns; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.overlay_data_tables.visualization_columns IS 'Columns that may/should be used for creating thematic maps. For example `count` or `density`'; + + +-- +-- Name: COLUMN overlay_data_tables.visualization_ops; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.overlay_data_tables.visualization_ops IS 'Operations that may/should be used for creating thematic maps. For example `mean` or `max`'; + + +-- +-- Name: COLUMN overlay_data_tables.required_filter_columns; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.overlay_data_tables.required_filter_columns IS 'Columns that must appear as filters when this table is displayed on the map. End users can change the filter values but cannot remove these filters.'; + + +-- +-- Name: COLUMN overlay_data_tables.stable_id; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.overlay_data_tables.stable_id IS 'Stable logical identity for a data table across version replace and TOC publish. Draft and published copies share the same UUID.'; + + +-- +-- Name: complete_overlay_data_table_upload(uuid, text, text, text, integer, text, text); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.complete_overlay_data_table_upload(job_id uuid, p_name text, p_join_column text, p_overlay_join_column text, p_row_count integer, p_parquet_remote text, p_column_stats_remote text) RETURNS public.overlay_data_tables + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +declare + upload overlay_data_table_uploads; + job project_background_jobs; + new_row overlay_data_tables; + old_row overlay_data_tables; + editor_id int; + new_version int := 1; +begin + select * into upload + from overlay_data_table_uploads + where project_background_job_id = job_id; + if upload is null then + raise exception 'Upload not found for job'; + end if; + + select * into job from project_background_jobs where id = job_id; + -- Don't resurrect a job that already timed out or was cancelled. + if job.state not in ('queued', 'running') then + raise exception 'Job is no longer active (state: %)', job.state; + end if; + + if upload.replace_overlay_data_table_id is not null then + select * into old_row + from overlay_data_tables + where id = upload.replace_overlay_data_table_id + and deleted_at is null; + if old_row is null then + raise exception 'Replace target no longer active'; + end if; + new_version := old_row.version + 1; + update overlay_data_tables + set deleted_at = now(), updated_at = now() + where id = old_row.id; + end if; + + insert into overlay_data_tables ( + table_of_contents_item_id, + project_id, + name, + join_column, + overlay_join_column, + row_count, + created_by, + version, + parquet_remote, + column_stats_remote, + visualization_columns, + visualization_ops, + required_filter_columns, + stable_id + ) values ( + upload.table_of_contents_item_id, + job.project_id, + p_name, + p_join_column, + p_overlay_join_column, + p_row_count, + coalesce(job.user_id, nullif(current_setting('session.user_id', true), '')::integer), + new_version, + p_parquet_remote, + p_column_stats_remote, + coalesce(old_row.visualization_columns, '{}'), + coalesce(old_row.visualization_ops, '{mean}'), + coalesce(old_row.required_filter_columns, '{}'), + coalesce(old_row.stable_id, uuid_generate_v4()) + ) returning * into new_row; + + if upload.replace_overlay_data_table_id is not null then + update overlay_data_tables + set replaced_by_id = new_row.id, updated_at = now() + where id = old_row.id; + + editor_id := coalesce(job.user_id, nullif(current_setting('session.user_id', true), '')::int); + if editor_id is not null then + perform record_changelog( + new_row.project_id, + editor_id, + 'overlay_data_table', + new_row.id, + 'data_table:replaced'::change_log_field_group, + jsonb_build_object( + 'name', old_row.name, + 'version', old_row.version, + 'id', old_row.id, + 'parquet_url', overlay_data_table_parquet_public_url(old_row.parquet_remote) + ), + jsonb_build_object('name', new_row.name, 'version', new_row.version, 'id', new_row.id), + null, null, + jsonb_build_object('table_of_contents_item_id', new_row.table_of_contents_item_id) + ); + end if; + else + editor_id := coalesce(job.user_id, nullif(current_setting('session.user_id', true), '')::int); + if editor_id is not null then + perform record_changelog( + new_row.project_id, + editor_id, + 'overlay_data_table', + new_row.id, + 'data_table:created'::change_log_field_group, + '{}'::jsonb, + jsonb_build_object('name', new_row.name, 'version', new_row.version), + null, null, + jsonb_build_object('table_of_contents_item_id', new_row.table_of_contents_item_id) + ); + end if; + end if; + + update project_background_jobs + set state = 'complete', progress = 1, progress_message = 'complete', error_message = null + where id = job_id; + + return new_row; +end; +$$; + + -- -- Name: compute_project_geography_hash(integer); Type: FUNCTION; Schema: public; Owner: - -- @@ -6296,6 +6423,7 @@ COMMENT ON FUNCTION public.copy_appearance(form_element_id integer, copy_from_id CREATE FUNCTION public.toc_to_tsvector(lang text, title text, metadata jsonb, translated_props jsonb) RETURNS tsvector LANGUAGE plpgsql IMMUTABLE + SET search_path TO 'public', 'pg_catalog' AS $$ DECLARE title_translated_prop_is_filled_in boolean; @@ -6398,6 +6526,9 @@ CREATE TABLE public.table_of_contents_items ( data_library_template_id text, copied_from_data_library_template_id text, has_metadata boolean GENERATED ALWAYS AS ((metadata IS NOT NULL)) STORED, + enable_data_tables boolean DEFAULT false NOT NULL, + data_table_join_column text, + CONSTRAINT table_of_contents_items_data_tables_join_column_required CHECK (((NOT enable_data_tables) OR ((data_table_join_column IS NOT NULL) AND (length(btrim(data_table_join_column)) > 0)))), CONSTRAINT table_of_contents_items_metadata_check CHECK (((metadata IS NULL) OR (char_length((metadata)::text) < 100000))), CONSTRAINT titlechk CHECK ((char_length(title) > 0)) ); @@ -6532,6 +6663,20 @@ COMMENT ON COLUMN public.table_of_contents_items.original_source_upload_availabl COMMENT ON COLUMN public.table_of_contents_items.data_library_template_id IS '@omit'; +-- +-- Name: COLUMN table_of_contents_items.enable_data_tables; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.table_of_contents_items.enable_data_tables IS 'When true, admins can attach CSV data tables linked to this layer by a canonical join column.'; + + +-- +-- Name: COLUMN table_of_contents_items.data_table_join_column; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.table_of_contents_items.data_table_join_column IS 'Overlay attribute name used as the canonical feature ID for linked data tables. Required when enable_data_tables is true.'; + + -- -- Name: copy_data_library_template_item(text, text); Type: FUNCTION; Schema: public; Owner: - -- @@ -6567,6 +6712,7 @@ CREATE FUNCTION public.copy_data_library_template_item(template_id text, project CREATE FUNCTION public.create_bbox(geom public.geometry, sketch_id integer) RETURNS real[] LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER + SET search_path TO 'public', 'pg_catalog' AS $$ declare child_ids int[]; @@ -7406,6 +7552,7 @@ $$; CREATE FUNCTION public.create_bbox(geom public.geometry) RETURNS real[] LANGUAGE sql IMMUTABLE SECURITY DEFINER + SET search_path TO 'public', 'pg_catalog' AS $$ select array[st_xmin(geom)::real, st_ymin(geom)::real, st_xmax(geom)::real, st_ymax(geom)::real]; $$; @@ -7573,7 +7720,8 @@ CREATE TABLE public.data_upload_tasks ( project_background_job_id uuid NOT NULL, changelog text, replace_table_of_contents_item_id integer, - data_library_metadata jsonb + data_library_metadata jsonb, + processing_options jsonb ); @@ -7592,10 +7740,17 @@ COMMENT ON COLUMN public.data_upload_tasks.content_type IS 'Content-Type of the -- --- Name: create_data_upload(text, integer, text, integer); Type: FUNCTION; Schema: public; Owner: - +-- Name: COLUMN data_upload_tasks.processing_options; Type: COMMENT; Schema: public; Owner: - -- -CREATE FUNCTION public.create_data_upload(filename text, project_id integer, content_type text, replace_table_of_contents_item_id integer) RETURNS public.data_upload_tasks +COMMENT ON COLUMN public.data_upload_tasks.processing_options IS 'Format-specific processing instructions supplied by the client at upload time (e.g. column mapping and CRS for delimited text uploads). Consumed by the spatial-uploads-handler.'; + + +-- +-- Name: create_data_upload(text, integer, text, integer, jsonb); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.create_data_upload(filename text, project_id integer, content_type text, replace_table_of_contents_item_id integer, processing_options jsonb DEFAULT NULL::jsonb) RETURNS public.data_upload_tasks LANGUAGE plpgsql SECURITY DEFINER AS $$ declare @@ -7643,12 +7798,14 @@ CREATE FUNCTION public.create_data_upload(filename text, project_id integer, con filename, content_type, project_background_job_id, - replace_table_of_contents_item_id + replace_table_of_contents_item_id, + processing_options ) values ( create_data_upload.filename, create_data_upload.content_type, job.id, - create_data_upload.replace_table_of_contents_item_id + create_data_upload.replace_table_of_contents_item_id, + create_data_upload.processing_options ) returning * into upload; return upload; else @@ -8061,57 +8218,59 @@ $$; -- --- Name: create_map_bookmark(text, boolean, jsonb, text[], integer, jsonb, jsonb, integer[], integer[], jsonb, text, jsonb, jsonb, text); Type: FUNCTION; Schema: public; Owner: - +-- Name: create_map_bookmark(text, boolean, jsonb, text[], integer, jsonb, jsonb, integer[], integer[], jsonb, text, jsonb, jsonb, text, jsonb); Type: FUNCTION; Schema: public; Owner: - -- -CREATE FUNCTION public.create_map_bookmark(slug text, "isPublic" boolean, style jsonb, "visibleDataLayers" text[], "selectedBasemap" integer, "basemapOptionalLayerStates" jsonb, "cameraOptions" jsonb, "mapDimensions" integer[], "visibleSketches" integer[], "sidebarState" jsonb, "basemapName" text, "layerNames" jsonb, "sketchNames" jsonb, "clientGeneratedThumbnail" text) RETURNS public.map_bookmarks +CREATE FUNCTION public.create_map_bookmark(slug text, "isPublic" boolean, style jsonb, "visibleDataLayers" text[], "selectedBasemap" integer, "basemapOptionalLayerStates" jsonb, "cameraOptions" jsonb, "mapDimensions" integer[], "visibleSketches" integer[], "sidebarState" jsonb, "basemapName" text, "layerNames" jsonb, "sketchNames" jsonb, "clientGeneratedThumbnail" text, "dataTableStates" jsonb DEFAULT '{}'::jsonb) RETURNS public.map_bookmarks LANGUAGE plpgsql SECURITY DEFINER AS $$ - declare - bookmark map_bookmarks; - pid int; - begin - select id into pid from projects where projects.slug = create_map_bookmark.slug; - if session_has_project_access(pid) then - insert into map_bookmarks ( - project_id, - user_id, - is_public, - style, - visible_data_layers, - selected_basemap, - basemap_optional_layer_states, - camera_options, - map_dimensions, - visible_sketches, - sidebar_state, - basemap_name, - layer_names, - sketch_names, - client_generated_thumbnail - ) values ( - pid, - nullif(current_setting('session.user_id', TRUE), '')::int, - "isPublic", - create_map_bookmark.style, - "visibleDataLayers", - "selectedBasemap", - "basemapOptionalLayerStates", - "cameraOptions", - "mapDimensions", - "visibleSketches", - "sidebarState", - "basemapName", - "layerNames", - "sketchNames", - "clientGeneratedThumbnail" - ) returning * into bookmark; - return bookmark; - else - raise exception 'Permission denied'; - end if; - end; - $$; +declare + bookmark map_bookmarks; + pid int; +begin + select id into pid from projects where projects.slug = create_map_bookmark.slug; + if session_has_project_access(pid) then + insert into map_bookmarks ( + project_id, + user_id, + is_public, + style, + visible_data_layers, + selected_basemap, + basemap_optional_layer_states, + camera_options, + map_dimensions, + visible_sketches, + sidebar_state, + basemap_name, + layer_names, + sketch_names, + client_generated_thumbnail, + data_table_states + ) values ( + pid, + nullif(current_setting('session.user_id', TRUE), '')::int, + "isPublic", + create_map_bookmark.style, + "visibleDataLayers", + "selectedBasemap", + "basemapOptionalLayerStates", + "cameraOptions", + "mapDimensions", + "visibleSketches", + "sidebarState", + "basemapName", + "layerNames", + "sketchNames", + "clientGeneratedThumbnail", + coalesce("dataTableStates", '{}'::jsonb) + ) returning * into bookmark; + return bookmark; + else + raise exception 'Permission denied'; + end if; +end; +$$; -- @@ -8225,6 +8384,135 @@ $$; COMMENT ON FUNCTION public.create_metadata_xml_output(data_source_id integer, url text, remote text, size bigint, filename text, metadata_type text) IS '@omit'; +-- +-- Name: overlay_data_table_uploads; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.overlay_data_table_uploads ( + id uuid DEFAULT public.uuid_generate_v4() NOT NULL, + project_background_job_id uuid NOT NULL, + table_of_contents_item_id integer NOT NULL, + filename text NOT NULL, + content_type text NOT NULL, + processing_options jsonb DEFAULT '{}'::jsonb NOT NULL, + overlay_geostats jsonb NOT NULL, + overlay_join_column text, + replace_overlay_data_table_id integer, + error_details jsonb, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now() +); + + +-- +-- Name: TABLE overlay_data_table_uploads; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.overlay_data_table_uploads IS '@omit delete'; + + +-- +-- Name: create_overlay_data_table_upload(integer, text, text, jsonb, integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.create_overlay_data_table_upload(toc_item_id integer, filename text, content_type text, processing_options jsonb DEFAULT '{}'::jsonb, replace_overlay_data_table_id integer DEFAULT NULL::integer) RETURNS public.overlay_data_table_uploads + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +declare + upload overlay_data_table_uploads; + job project_background_jobs; + pid int; + geostats jsonb; + join_col text; + enabled boolean; +begin + select project_id, enable_data_tables, data_table_join_column + into pid, enabled, join_col + from table_of_contents_items + where id = toc_item_id and is_draft = true and is_folder = false; + if pid is null then + raise exception 'Draft layer table of contents item not found'; + end if; + if not session_is_admin(pid) then + raise exception 'permission denied'; + end if; + if not coalesce(enabled, false) then + raise exception 'Data tables are not enabled for this layer'; + end if; + if join_col is null or length(trim(join_col)) = 0 then + raise exception 'Data table join column is not configured for this layer'; + end if; + + select ds.geostats into geostats + from table_of_contents_items toc + inner join data_layers dl on dl.id = toc.data_layer_id + inner join data_sources ds on ds.id = dl.data_source_id + where toc.id = toc_item_id; + if geostats is null then + raise exception 'Overlay layer has no geostats'; + end if; + + if replace_overlay_data_table_id is not null then + if not exists ( + select 1 from overlay_data_tables + where id = replace_overlay_data_table_id + and table_of_contents_item_id = toc_item_id + and deleted_at is null + ) then + raise exception 'Replace target data table not found or not active'; + end if; + end if; + + if exists ( + select 1 + from overlay_data_table_uploads odtu + inner join project_background_jobs pbj on pbj.id = odtu.project_background_job_id + where odtu.table_of_contents_item_id = toc_item_id + and odtu.filename = create_overlay_data_table_upload.filename + and pbj.state in ('queued', 'running') + ) then + raise exception 'There is already an active data table upload for this file'; + end if; + + insert into project_background_jobs ( + project_id, + title, + user_id, + type, + timeout_at + ) values ( + pid, + (case when replace_overlay_data_table_id is not null then 'Replacement data table ' else 'Data table ' end) || filename, + nullif(current_setting('session.user_id', true), '')::integer, + 'data_table_upload', + timezone('utc', now()) + interval '15 minutes' + ) returning * into job; + + insert into overlay_data_table_uploads ( + project_background_job_id, + table_of_contents_item_id, + filename, + content_type, + processing_options, + overlay_geostats, + overlay_join_column, + replace_overlay_data_table_id + ) values ( + job.id, + toc_item_id, + create_overlay_data_table_upload.filename, + content_type, + coalesce(processing_options, '{}'::jsonb), + geostats, + join_col, + replace_overlay_data_table_id + ) returning * into upload; + + return upload; +end; +$$; + + -- -- Name: extract_post_bookmark_attachments(jsonb); Type: FUNCTION; Schema: public; Owner: - -- @@ -11055,39 +11343,6 @@ END; $$; --- --- Name: enforce_resolvable_layer_comment_parent(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.enforce_resolvable_layer_comment_parent() RETURNS trigger - LANGUAGE plpgsql - AS $$ -declare - v_parent record; -begin - if new.parent_comment_id is null then - return new; - end if; - - select id, table_of_contents_item_id, project_id - into v_parent - from resolvable_layer_comment - where id = new.parent_comment_id; - - if v_parent.id is null then - raise exception 'Parent comment not found'; - end if; - - if v_parent.table_of_contents_item_id is distinct from new.table_of_contents_item_id - or v_parent.project_id is distinct from new.project_id then - raise exception 'Parent comment must belong to the same layer'; - end if; - - return new; -end; -$$; - - -- -- Name: enqueue_metric_calculations_for_sketch(integer); Type: FUNCTION; Schema: public; Owner: - -- @@ -11343,6 +11598,34 @@ CREATE FUNCTION public.fail_data_upload(id uuid, msg text) RETURNS public.data_u $$; +-- +-- Name: fail_overlay_data_table_upload(uuid, text, jsonb); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.fail_overlay_data_table_upload(job_id uuid, error_message text, error_details jsonb DEFAULT NULL::jsonb) RETURNS void + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +begin + update overlay_data_table_uploads odtu + set error_details = coalesce(fail_overlay_data_table_upload.error_details, odtu.error_details), + updated_at = now() + where odtu.project_background_job_id = job_id; + + -- Only fail jobs still in flight; never clobber a completed job. + update project_background_jobs + set + state = 'failed', + progress_message = case + when fail_overlay_data_table_upload.error_message = 'Timed out' then 'timeout' + else 'failed' + end, + error_message = fail_overlay_data_table_upload.error_message + where id = job_id + and state in ('queued', 'running'); +end; +$$; + + -- -- Name: filter_state_to_search_string(jsonb, integer); Type: FUNCTION; Schema: public; Owner: - -- @@ -11878,15 +12161,8 @@ $$; CREATE FUNCTION public.geography_clipping_layers_object_key(g public.geography_clipping_layers) RETURNS text LANGUAGE sql STABLE SECURITY DEFINER AS $$ - select - regexp_replace( - regexp_replace(remote, '^[^:]+://[^/]+/', ''), -- Remove protocol and host - '^/', '' -- Remove leading slash if present - ) - from data_upload_outputs - where type = 'FlatGeobuf' - and data_source_id = ( - select data_source_id from data_layers where id = g.data_layer_id + select remote from data_upload_outputs where type = 'FlatGeobuf' and data_source_id = ( + select data_source_id from data_layers where id = g.data_layer_id ); $$; @@ -12813,57 +13089,6 @@ CREATE FUNCTION public.get_public_jwk(id uuid) RETURNS text COMMENT ON FUNCTION public.get_public_jwk(id uuid) IS '@omit'; --- --- Name: get_published_card_id_from_draft(integer); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.get_published_card_id_from_draft(draft_report_card_id integer) RETURNS integer - LANGUAGE plpgsql SECURITY DEFINER - AS $$ -DECLARE - draft_tab_id integer; - draft_tab_position integer; - draft_card_position integer; - sketch_class_id integer; - published_report_id integer; - published_tab_id integer; - published_card_id integer; -BEGIN - -- Gather draft card/tab positions and sketch_class - SELECT rt.id, rt.position, rc.position, r.sketch_class_id - INTO draft_tab_id, draft_tab_position, draft_card_position, sketch_class_id - FROM public.report_cards rc - JOIN public.report_tabs rt ON rt.id = rc.report_tab_id - JOIN public.reports r ON r.id = rt.report_id - WHERE rc.id = draft_report_card_id; - - -- Determine the published report id - SELECT sc.report_id - INTO published_report_id - FROM public.sketch_classes sc - WHERE sc.id = sketch_class_id; - - IF published_report_id IS NULL THEN - RETURN NULL; - END IF; - - -- Match the tab by position in the published report - SELECT id - INTO published_tab_id - FROM public.report_tabs - WHERE report_id = published_report_id AND position = draft_tab_position; - - -- Match the card by position within the matched tab - SELECT id - INTO published_card_id - FROM public.report_cards - WHERE report_tab_id = published_tab_id AND position = draft_card_position; - - RETURN published_card_id; -END -$$; - - -- -- Name: get_referenced_stable_ids_for_report(integer); Type: FUNCTION; Schema: public; Owner: - -- @@ -14750,6 +14975,83 @@ $$; COMMENT ON FUNCTION public.overlapping_fragments_for_collection(input_collection_id integer, input_envelopes public.geometry[], edited_sketch_id integer) IS '@omit'; +-- +-- Name: overlay_data_table_linked_toc_is_draft(integer, integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.overlay_data_table_linked_toc_is_draft(toc_item_id integer, pid integer) RETURNS boolean + LANGUAGE sql STABLE SECURITY DEFINER + AS $$ + select exists ( + select 1 + from table_of_contents_items + where id = toc_item_id + and project_id = pid + and is_draft = true + ); +$$; + + +-- +-- Name: overlay_data_table_parquet_public_url(text); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.overlay_data_table_parquet_public_url(p_remote text) RETURNS text + LANGUAGE plpgsql STABLE + AS $$ +declare + base text; + key text; +begin + if p_remote is null or length(trim(p_remote)) = 0 then + return null; + end if; + + -- Prefer an optional session override (tests); otherwise the public uploads host. + base := coalesce( + nullif(trim(current_setting('seasketch.uploads_base_url', true)), ''), + 'https://uploads.seasketch.org' + ); + + key := regexp_replace(p_remote, '^r2://[^/]+/', ''); + if key is null or length(trim(key)) = 0 then + return null; + end if; + + return rtrim(base, '/') || '/' || key; +end; +$$; + + +-- +-- Name: overlay_data_tables_draft_toc_has_changes(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.overlay_data_tables_draft_toc_has_changes() RETURNS trigger + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +declare + pid int; + toc_is_draft boolean; +begin + if tg_op = 'DELETE' then + select project_id, table_of_contents_items.is_draft into pid, toc_is_draft + from table_of_contents_items where id = old.table_of_contents_item_id; + else + select project_id, table_of_contents_items.is_draft into pid, toc_is_draft + from table_of_contents_items where id = new.table_of_contents_item_id; + end if; + if toc_is_draft then + update projects set draft_table_of_contents_has_changes = true where id = pid; + end if; + if tg_op = 'DELETE' then + return old; + end if; + return new; +end; +$$; + + -- -- Name: posts_author_profile(public.posts); Type: FUNCTION; Schema: public; Owner: - -- @@ -15591,21 +15893,21 @@ CREATE FUNCTION public.projects_admin_count(p public.projects) RETURNS integer CREATE FUNCTION public.projects_admins(p public.projects) RETURNS SETOF public.users LANGUAGE sql STABLE AS $$ - select - users.* - from - project_participants - inner join - users - on - project_participants.user_id = users.id - where - project_participants.is_admin = true and - ( - project_participants.approved = true or - exists(select 1 from projects where id = project_participants.project_id and projects.access_control = 'public') - ) - ; + select + users.* + from + project_participants + inner join + users + on + project_participants.user_id = users.id + where + project_participants.is_admin = true and + ( + project_participants.approved = true or + exists(select 1 from projects where id = project_participants.project_id and projects.access_control = 'public') + ) + ; $$; @@ -17023,68 +17325,19 @@ CREATE TABLE public.project_visitor_metrics ( -- CREATE FUNCTION public.projects_visitor_metrics(p public.projects, period public.activity_stats_period) RETURNS SETOF public.project_visitor_metrics - LANGUAGE plpgsql STABLE SECURITY DEFINER + LANGUAGE sql STABLE SECURITY DEFINER AS $$ -DECLARE - lookback interval; -BEGIN - IF NOT session_is_admin(p.id) THEN - RETURN; - END IF; - - IF period IN ('6-months', '1-year') THEN - lookback := CASE period - WHEN '6-months' THEN '6 months'::interval - ELSE '1 year'::interval - END; - - RETURN QUERY SELECT - p.id, - '30 days'::interval, - now()::timestamptz, - date_part('month', timezone('UTC', now()))::int, - (SELECT coalesce(jsonb_agg(jsonb_build_object('label', s.label, 'count', s.total) ORDER BY s.total DESC), '[]'::jsonb) - FROM (SELECT elem->>'label' as label, SUM((elem->>'count')::int) as total - FROM public.project_visitor_metrics pvm, jsonb_array_elements(pvm.top_referrers) elem - WHERE pvm.interval = '30 days'::interval AND pvm.project_id = p.id AND pvm.timestamp >= now() - lookback - GROUP BY elem->>'label' ORDER BY total DESC LIMIT 15) s), - (SELECT coalesce(jsonb_agg(jsonb_build_object('label', s.label, 'count', s.total) ORDER BY s.total DESC), '[]'::jsonb) - FROM (SELECT elem->>'label' as label, SUM((elem->>'count')::int) as total - FROM public.project_visitor_metrics pvm, jsonb_array_elements(pvm.top_operating_systems) elem - WHERE pvm.interval = '30 days'::interval AND pvm.project_id = p.id AND pvm.timestamp >= now() - lookback - GROUP BY elem->>'label' ORDER BY total DESC LIMIT 15) s), - (SELECT coalesce(jsonb_agg(jsonb_build_object('label', s.label, 'count', s.total) ORDER BY s.total DESC), '[]'::jsonb) - FROM (SELECT elem->>'label' as label, SUM((elem->>'count')::int) as total - FROM public.project_visitor_metrics pvm, jsonb_array_elements(pvm.top_browsers) elem - WHERE pvm.interval = '30 days'::interval AND pvm.project_id = p.id AND pvm.timestamp >= now() - lookback - GROUP BY elem->>'label' ORDER BY total DESC LIMIT 15) s), - (SELECT coalesce(jsonb_agg(jsonb_build_object('label', s.label, 'count', s.total) ORDER BY s.total DESC), '[]'::jsonb) - FROM (SELECT elem->>'label' as label, SUM((elem->>'count')::int) as total - FROM public.project_visitor_metrics pvm, jsonb_array_elements(pvm.top_device_types) elem - WHERE pvm.interval = '30 days'::interval AND pvm.project_id = p.id AND pvm.timestamp >= now() - lookback - GROUP BY elem->>'label' ORDER BY total DESC LIMIT 15) s), - (SELECT coalesce(jsonb_agg(jsonb_build_object('label', s.label, 'count', s.total) ORDER BY s.total DESC), '[]'::jsonb) - FROM (SELECT elem->>'label' as label, SUM((elem->>'count')::int) as total - FROM public.project_visitor_metrics pvm, jsonb_array_elements(pvm.top_countries) elem - WHERE pvm.interval = '30 days'::interval AND pvm.project_id = p.id AND pvm.timestamp >= now() - lookback - GROUP BY elem->>'label' ORDER BY total DESC LIMIT 15) s); - ELSE - RETURN QUERY - SELECT * FROM project_visitor_metrics - WHERE session_is_admin(p.id) - AND project_id = p.id - AND interval = ( - CASE period - WHEN '24hrs' THEN '24 hours'::interval - WHEN '7-days' THEN '7 days'::interval - WHEN '30-days' THEN '30 days'::interval - ELSE '1 day'::interval - END - ) - ORDER BY timestamp DESC LIMIT 1; - END IF; -END; -$$; + select * from project_visitor_metrics where session_is_admin(p.id) and + project_id = p.id and + interval = ( + case period + when '24hrs' then '24 hours'::interval + when '7-days' then '7 days'::interval + when '30-days' then '30 days'::interval + else '1 day'::interval + end + ) order by timestamp desc limit 1; + $$; -- @@ -17527,384 +17780,434 @@ $$; CREATE FUNCTION public.publish_table_of_contents("projectId" integer) RETURNS SETOF public.table_of_contents_items LANGUAGE plpgsql SECURITY DEFINER AS $$ - declare - v_editor int; - v_layer_count int; - lid int; - item table_of_contents_items; - source_id int; - copied_source_id int; - acl_type access_control_list_type; - acl_id int; - orig_acl_id int; - new_toc_id int; - new_interactivity_settings_id int; - new_source_id integer; - ref record; - begin - -- check permissions - if session_is_admin("projectId") = false then - raise 'Permission denied. Must be a project admin'; - end if; - - -- delete existing published table of contents items, layers, sources, and interactivity settings - delete from - interactivity_settings +declare + v_editor int; + v_layer_count int; + lid int; + item table_of_contents_items; + source_id int; + copied_source_id int; + acl_type access_control_list_type; + acl_id int; + orig_acl_id int; + new_toc_id int; + new_interactivity_settings_id int; +begin + -- check permissions + if session_is_admin("projectId") = false then + raise 'Permission denied. Must be a project admin'; + end if; + + -- delete existing published table of contents items, layers, sources, and interactivity settings + -- (published overlay_data_tables cascade-delete with their TOC rows) + delete from + interactivity_settings + where + id in ( + select + data_layers.interactivity_settings_id + from + data_layers + inner join + table_of_contents_items + on + data_layers.id = table_of_contents_items.data_layer_id where - id in ( - select - data_layers.interactivity_settings_id - from - data_layers - inner JOIN - table_of_contents_items - on - data_layers.id = table_of_contents_items.data_layer_id - where - table_of_contents_items.project_id = "projectId" and - is_draft = false - ); + table_of_contents_items.project_id = "projectId" and + is_draft = false + ); - delete from data_sources where data_sources.id in ( - select - data_source_id + delete from data_sources where data_sources.id in ( + select + data_source_id + from + data_layers + inner join + table_of_contents_items + on + data_layers.id = table_of_contents_items.data_layer_id + where + table_of_contents_items.project_id = "projectId" and + is_draft = false + ); + delete from data_layers where id in ( + select + data_layer_id + from + table_of_contents_items + where + project_id = "projectId" and + is_draft = false + ); + delete from + table_of_contents_items + where + project_id = "projectId" and + is_draft = false; + + -- one-by-one, copy related layers and link table of contents items + for item in + select + * + from + table_of_contents_items + where + is_draft = true and + project_id = "projectId" + loop + if item.is_folder = false then + -- copy interactivity settings first + insert into interactivity_settings ( + type, + short_template, + long_template, + cursor, + title + ) select + type, + short_template, + long_template, + cursor, + title from - data_layers - inner JOIN - table_of_contents_items - on - data_layers.id = table_of_contents_items.data_layer_id + interactivity_settings where - table_of_contents_items.project_id = "projectId" and - is_draft = false - ); - delete from data_layers where id in ( - select - data_layer_id - from - table_of_contents_items - where - project_id = "projectId" and - is_draft = false - ); - delete from - table_of_contents_items - where - project_id = "projectId" and - is_draft = false; - - -- one-by-one, copy related layers and link table of contents items - for item in - select - * - from - table_of_contents_items - where - is_draft = true and - project_id = "projectId" - loop - if item.is_folder = false then - -- copy interactivity settings first - insert into interactivity_settings ( - type, - short_template, - long_template, - cursor, - title - ) select - type, - short_template, - long_template, - cursor, - title - from - interactivity_settings - where - interactivity_settings.id = ( - select interactivity_settings_id from data_layers where data_layers.id = item.data_layer_id - ) - returning - id - into - new_interactivity_settings_id; + interactivity_settings.id = ( + select interactivity_settings_id from data_layers where data_layers.id = item.data_layer_id + ) + returning + id + into + new_interactivity_settings_id; - insert into data_layers ( - project_id, - data_source_id, - source_layer, - sublayer, - sublayer_type, - render_under, - mapbox_gl_styles, - interactivity_settings_id, - z_index + insert into data_layers ( + project_id, + data_source_id, + source_layer, + sublayer, + sublayer_type, + render_under, + mapbox_gl_styles, + interactivity_settings_id, + z_index + ) + select "projectId", + data_source_id, + source_layer, + sublayer, + sublayer_type, + render_under, + mapbox_gl_styles, + new_interactivity_settings_id, + z_index + from + data_layers + where + id = item.data_layer_id + returning id into lid; + else + lid = item.data_layer_id; + end if; + -- TODO: this will have to be modified with the addition of any columns + insert into table_of_contents_items ( + is_draft, + project_id, + path, + stable_id, + parent_stable_id, + title, + is_folder, + show_radio_children, + is_click_off_only, + metadata, + bounds, + data_layer_id, + sort_index, + hide_children, + geoprocessing_reference_id, + translated_props, + enable_download, + enable_data_tables, + data_table_join_column + ) values ( + false, + "projectId", + item.path, + item.stable_id, + item.parent_stable_id, + item.title, + item.is_folder, + item.show_radio_children, + item.is_click_off_only, + item.metadata, + item.bounds, + lid, + item.sort_index, + item.hide_children, + item.geoprocessing_reference_id, + item.translated_props, + item.enable_download, + item.enable_data_tables, + item.data_table_join_column + ) returning id into new_toc_id; + select + type, id into acl_type, orig_acl_id + from + access_control_lists + where + table_of_contents_item_id = ( + select + id + from + table_of_contents_items + where is_draft = true and stable_id = item.stable_id + ); + -- copy access control list settings + if acl_type != 'public' then + update + access_control_lists + set type = acl_type + where table_of_contents_item_id = new_toc_id + returning id into acl_id; + if acl_type = 'group' then + insert into + access_control_list_groups ( + access_control_list_id, + group_id ) - select "projectId", - data_source_id, - source_layer, - sublayer, - sublayer_type, - render_under, - mapbox_gl_styles, - new_interactivity_settings_id, - z_index - from - data_layers - where - id = item.data_layer_id - returning id into lid; - else - lid = item.data_layer_id; - end if; - -- TODO: this will have to be modified with the addition of any columns - insert into table_of_contents_items ( - is_draft, - project_id, - path, - stable_id, - parent_stable_id, - title, - is_folder, - show_radio_children, - is_click_off_only, - metadata, - bounds, - data_layer_id, - sort_index, - hide_children, - geoprocessing_reference_id, - translated_props, - enable_download - ) values ( - false, - "projectId", - item.path, - item.stable_id, - item.parent_stable_id, - item.title, - item.is_folder, - item.show_radio_children, - item.is_click_off_only, - item.metadata, - item.bounds, - lid, - item.sort_index, - item.hide_children, - item.geoprocessing_reference_id, - item.translated_props, - item.enable_download - ) returning id into new_toc_id; - select - type, id into acl_type, orig_acl_id - from - access_control_lists - where - table_of_contents_item_id = ( - select - id - from - table_of_contents_items - where is_draft = true and stable_id = item.stable_id - ); - -- copy access control list settings - if acl_type != 'public' then - update - access_control_lists - set type = acl_type - where table_of_contents_item_id = new_toc_id - returning id into acl_id; - if acl_type = 'group' then - insert into - access_control_list_groups ( - access_control_list_id, - group_id - ) - select - acl_id, - group_id - from - access_control_list_groups - where - access_control_list_id = orig_acl_id; - end if; - end if; - end loop; - -- one-by-one, copy related sources and update foreign keys of layers - for source_id in - select distinct(data_source_id) from data_layers where id in ( - select - data_layer_id - from - table_of_contents_items - where - is_draft = false and - project_id = "projectId" and - is_folder = false - ) - loop - -- TODO: This function will have to be updated whenever the schema - -- changes since these columns are hard coded... no way around it. - insert into data_sources ( - project_id, - type, - attribution, - bounds, - maxzoom, - minzoom, - url, - scheme, - tiles, - tile_size, - encoding, - buffer, - cluster, - cluster_max_zoom, - cluster_properties, - cluster_radius, - generate_id, - line_metrics, - promote_id, - tolerance, - coordinates, - urls, - query_parameters, - use_device_pixel_ratio, - import_type, - original_source_url, - enhanced_security, - byte_length, - supports_dynamic_layers, - uploaded_source_filename, - uploaded_source_layername, - normalized_source_object_key, - normalized_source_bytes, - geostats, - upload_task_id, - translated_props, - arcgis_fetch_strategy, - created_by - ) - select - "projectId", - type, - attribution, - bounds, - maxzoom, - minzoom, - url, - scheme, - tiles, - tile_size, - encoding, - buffer, - cluster, - cluster_max_zoom, - cluster_properties, - cluster_radius, - generate_id, - line_metrics, - promote_id, - tolerance, - coordinates, - urls, - query_parameters, - use_device_pixel_ratio, - import_type, - original_source_url, - enhanced_security, - byte_length, - supports_dynamic_layers, - uploaded_source_filename, - uploaded_source_layername, - normalized_source_object_key, - normalized_source_bytes, - geostats, - upload_task_id, - translated_props, - arcgis_fetch_strategy, - created_by - from - data_sources - where - id = source_id - returning id into copied_source_id; - -- copy data_upload_outputs - insert into data_upload_outputs ( - data_source_id, - project_id, - type, - created_at, - url, - remote, - is_original, - size, - filename, - original_filename, - source_processing_job_key, - epsg - ) select - copied_source_id, - project_id, - type, - created_at, - url, - remote, - is_original, - size, - filename, - original_filename, - source_processing_job_key, - epsg - from - data_upload_outputs - where - data_source_id = source_id; - -- update data_layers that should now reference the copy - update - data_layers - set data_source_id = copied_source_id - where - data_source_id = source_id and - data_layers.id in (( - select distinct(data_layer_id) from table_of_contents_items where is_draft = false and - project_id = "projectId" and - is_folder = false - )); - end loop; - update - projects - set - draft_table_of_contents_has_changes = false, - table_of_contents_last_published = now() - where - id = "projectId"; + select + acl_id, + group_id + from + access_control_list_groups + where + access_control_list_id = orig_acl_id; + end if; + end if; + end loop; - v_editor := nullif(current_setting('session.user_id', true), '')::int; - if v_editor is not null then - select count(*)::int into v_layer_count - from table_of_contents_items - where project_id = "projectId" - and is_draft = true - and is_folder = false; + -- Copy active draft overlay data tables onto newly published TOC items in one + -- statement (matched by TOC stable_id). Soft-deleted draft history is skipped. + -- Data table stable_id is preserved so bookmarks/prefs survive publish. + insert into overlay_data_tables ( + table_of_contents_item_id, + project_id, + name, + join_column, + overlay_join_column, + row_count, + created_by, + version, + parquet_remote, + column_stats_remote, + visualization_columns, + visualization_ops, + required_filter_columns, + stable_id + ) + select + published_toc.id, + odt.project_id, + odt.name, + odt.join_column, + odt.overlay_join_column, + odt.row_count, + odt.created_by, + odt.version, + odt.parquet_remote, + odt.column_stats_remote, + odt.visualization_columns, + odt.visualization_ops, + odt.required_filter_columns, + odt.stable_id + from overlay_data_tables odt + inner join table_of_contents_items draft_toc + on draft_toc.id = odt.table_of_contents_item_id + inner join table_of_contents_items published_toc + on published_toc.project_id = draft_toc.project_id + and published_toc.stable_id = draft_toc.stable_id + and published_toc.is_draft = false + where draft_toc.project_id = "projectId" + and draft_toc.is_draft = true + and draft_toc.is_folder = false + and odt.deleted_at is null; + + -- one-by-one, copy related sources and update foreign keys of layers + for source_id in + select distinct(data_source_id) from data_layers where id in ( + select + data_layer_id + from + table_of_contents_items + where + is_draft = false and + project_id = "projectId" and + is_folder = false + ) + loop + -- TODO: This function will have to be updated whenever the schema + -- changes since these columns are hard coded... no way around it. + insert into data_sources ( + project_id, + type, + attribution, + bounds, + maxzoom, + minzoom, + url, + scheme, + tiles, + tile_size, + encoding, + buffer, + cluster, + cluster_max_zoom, + cluster_properties, + cluster_radius, + generate_id, + line_metrics, + promote_id, + tolerance, + coordinates, + urls, + query_parameters, + use_device_pixel_ratio, + import_type, + original_source_url, + enhanced_security, + byte_length, + supports_dynamic_layers, + uploaded_source_filename, + uploaded_source_layername, + normalized_source_object_key, + normalized_source_bytes, + geostats, + upload_task_id, + translated_props, + arcgis_fetch_strategy, + created_by + ) + select + "projectId", + type, + attribution, + bounds, + maxzoom, + minzoom, + url, + scheme, + tiles, + tile_size, + encoding, + buffer, + cluster, + cluster_max_zoom, + cluster_properties, + cluster_radius, + generate_id, + line_metrics, + promote_id, + tolerance, + coordinates, + urls, + query_parameters, + use_device_pixel_ratio, + import_type, + original_source_url, + enhanced_security, + byte_length, + supports_dynamic_layers, + uploaded_source_filename, + uploaded_source_layername, + normalized_source_object_key, + normalized_source_bytes, + geostats, + upload_task_id, + translated_props, + arcgis_fetch_strategy, + created_by + from + data_sources + where + id = source_id + returning id into copied_source_id; + -- copy data_upload_outputs + insert into data_upload_outputs ( + data_source_id, + project_id, + type, + created_at, + url, + remote, + is_original, + size, + filename, + original_filename, + source_processing_job_key, + epsg + ) select + copied_source_id, + project_id, + type, + created_at, + url, + remote, + is_original, + size, + filename, + original_filename, + source_processing_job_key, + epsg + from + data_upload_outputs + where + data_source_id = source_id; + -- update data_layers that should now reference the copy + update + data_layers + set data_source_id = copied_source_id + where + data_source_id = source_id and + data_layers.id in (( + select distinct(data_layer_id) from table_of_contents_items where is_draft = false and + project_id = "projectId" and + is_folder = false + )); + end loop; + update + projects + set + draft_table_of_contents_has_changes = false, + table_of_contents_last_published = now() + where + id = "projectId"; - perform record_changelog( - "projectId", - v_editor, - 'projects', - "projectId", - 'layers:published'::change_log_field_group, - '{}'::jsonb, - jsonb_build_object('layer_count', v_layer_count), - null, - null, - null - ); - end if; - -- return items - return query select * from table_of_contents_items - where project_id = "projectId" and is_draft = false; - end; - $$; + v_editor := nullif(current_setting('session.user_id', true), '')::int; + if v_editor is not null then + select count(*)::int into v_layer_count + from table_of_contents_items + where project_id = "projectId" + and is_draft = true + and is_folder = false; + + perform record_changelog( + "projectId", + v_editor, + 'projects', + "projectId", + 'layers:published'::change_log_field_group, + '{}'::jsonb, + jsonb_build_object('layer_count', v_layer_count), + null, + null, + null + ); + end if; + -- return items + return query select * from table_of_contents_items + where project_id = "projectId" and is_draft = false; +end; +$$; -- @@ -18454,6 +18757,56 @@ Remove a SketchClass from the list of valid children for a Collection. '; +-- +-- Name: rename_overlay_data_table(integer, text); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.rename_overlay_data_table(table_id integer, new_name text) RETURNS public.overlay_data_tables + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +declare + row overlay_data_tables; + old_name text; + editor_id int; +begin + select * into row from overlay_data_tables where id = table_id and deleted_at is null; + if row is null then + raise exception 'Active data table not found'; + end if; + if not session_is_admin(row.project_id) then + raise exception 'permission denied'; + end if; + if not exists ( + select 1 from table_of_contents_items + where id = row.table_of_contents_item_id and is_draft = true + ) then + raise exception 'Can only rename data tables on draft layers'; + end if; + old_name := row.name; + update overlay_data_tables + set name = new_name, updated_at = now() + where id = table_id + returning * into row; + + editor_id := nullif(current_setting('session.user_id', true), '')::int; + if editor_id is not null then + perform record_changelog( + row.project_id, + editor_id, + 'overlay_data_table', + row.id, + 'data_table:renamed'::change_log_field_group, + jsonb_build_object('name', old_name), + jsonb_build_object('name', new_name), + null, null, + jsonb_build_object('table_of_contents_item_id', row.table_of_contents_item_id, 'version', row.version) + ); + end if; + return row; +end; +$$; + + -- -- Name: rename_report_tab(integer, text, jsonb); Type: FUNCTION; Schema: public; Owner: - -- @@ -18725,34 +19078,6 @@ CREATE FUNCTION public.replace_data_source(data_layer_id integer, data_source_id $$; --- --- Name: replace_toc_references_in_body(jsonb, jsonb); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.replace_toc_references_in_body(body jsonb, published_toc_counterparts jsonb) RETURNS jsonb - LANGUAGE plpgsql - AS $$ - declare - published_toc_counterpart_record record; - begin - -- first, check that published_toc_counterparts is a valid jsonb object with the correct keys and values - if jsonb_typeof(published_toc_counterparts) != 'object' then - raise exception 'published_toc_counterparts is not a valid jsonb object'; - end if; - if jsonb_typeof(published_toc_counterparts) = 'object' then - for published_toc_counterpart_record in select * from jsonb_each(published_toc_counterparts) loop - if published_toc_counterpart_record.value is null or jsonb_typeof(published_toc_counterpart_record.value) = 'null' then - raise exception 'This report references data layers that have not yet been published. Please publish the layer list first.'; - end if; - end loop; - end if; - -- now, replace the references to draft table of contents item ids with published table of contents item ids, referencing the published_toc_counterparts map. This will require walking through all nodes in the prosemirror document. - body := replace_toc_references_in_body_recursive(body, published_toc_counterparts); - return body; - end; - $$; - - -- -- Name: report_card_ids_for_report(integer); Type: FUNCTION; Schema: public; Owner: - -- @@ -19045,76 +19370,6 @@ CREATE FUNCTION public.reprocess_legacy_data_source(table_of_contents_item_id in $$; --- --- Name: resolvable_layer_comment_enforce_draft_toc(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.resolvable_layer_comment_enforce_draft_toc() RETURNS trigger - LANGUAGE plpgsql - AS $$ -begin - if not exists ( - select 1 - from public.table_of_contents_items t - where t.id = new.table_of_contents_item_id - and t.is_draft = true - ) then - raise exception - 'Resolvable layer comments may only reference draft table_of_contents_items rows'; - end if; - return new; -end; -$$; - - --- --- Name: resolvable_layer_comment_set_audit_fields(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.resolvable_layer_comment_set_audit_fields() RETURNS trigger - LANGUAGE plpgsql SECURITY DEFINER - SET search_path TO 'public', 'pg_temp' - AS $$ -declare - v_uid int; -begin - v_uid := nullif(current_setting('session.user_id', true), '')::int; - if v_uid is null then - raise exception 'Authentication required'; - end if; - - if tg_op = 'INSERT' then - if not session_is_admin(new.project_id) then - raise exception 'Admin privileges required'; - end if; - new.author_id := v_uid; - elsif tg_op = 'UPDATE' then - if not session_is_admin(new.project_id) then - raise exception 'Admin privileges required'; - end if; - if new.project_id is distinct from old.project_id - or new.table_of_contents_item_id is distinct from old.table_of_contents_item_id - or new.parent_comment_id is distinct from old.parent_comment_id - or new.author_id is distinct from old.author_id - then - raise exception 'Cannot reassign comment metadata'; - end if; - if new.comment is distinct from old.comment and v_uid is distinct from old.author_id then - raise exception 'Only the author may edit comment text'; - end if; - if old.resolved_at is null and new.resolved_at is not null then - new.resolved_by_id := v_uid; - elsif new.resolved_at is null then - new.resolved_by_id := null; - end if; - end if; - - new.updated_at := now(); - return new; -end; -$$; - - -- -- Name: resolvable_layer_comments_author_profile(public.resolvable_layer_comments); Type: FUNCTION; Schema: public; Owner: - -- @@ -19603,6 +19858,81 @@ CREATE FUNCTION public.revoke_api_key(id uuid) RETURNS void $$; +-- +-- Name: rollback_overlay_data_table_version(integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.rollback_overlay_data_table_version(table_id integer) RETURNS public.overlay_data_tables + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +declare + target overlay_data_tables; + successor overlay_data_tables; + editor_id int; +begin + select * into target from overlay_data_tables where id = table_id; + if target is null then + raise exception 'Data table not found'; + end if; + if target.deleted_at is null then + select * into target + from overlay_data_tables + where replaced_by_id = table_id + and deleted_at is not null + order by version desc + limit 1; + if target is null then + raise exception 'No previous version found to rollback to'; + end if; + end if; + if not session_is_admin(target.project_id) then + raise exception 'permission denied'; + end if; + if not exists ( + select 1 from table_of_contents_items + where id = target.table_of_contents_item_id and is_draft = true + ) then + raise exception 'Can only rollback data tables on draft layers'; + end if; + + select * into successor + from overlay_data_tables + where id = target.replaced_by_id + and deleted_at is null + limit 1; + if successor is null then + raise exception 'No successor version found to rollback from'; + end if; + + delete from overlay_data_tables where id = successor.id; + + update overlay_data_tables + set deleted_at = null, replaced_by_id = null, updated_at = now() + where id = target.id + returning * into target; + + editor_id := nullif(current_setting('session.user_id', true), '')::int; + if editor_id is not null then + perform record_changelog( + target.project_id, + editor_id, + 'overlay_data_table', + target.id, + 'data_table:rollback'::change_log_field_group, + jsonb_build_object('removed_version', successor.version), + jsonb_build_object('name', target.name, 'version', target.version), + null, null, + jsonb_build_object( + 'table_of_contents_item_id', target.table_of_contents_item_id, + 'removed_id', successor.id + ) + ); + end if; + return target; +end; +$$; + + -- -- Name: rollback_to_archived_source(integer, boolean); Type: FUNCTION; Schema: public; Owner: - -- @@ -20927,6 +21257,85 @@ Set the order in which discussion forums will be displayed. Provide a list of fo '; +-- +-- Name: set_overlay_data_table_visualization_settings(integer, text[], text[], text[]); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.set_overlay_data_table_visualization_settings(table_id integer, visualization_columns text[], visualization_ops text[], required_filter_columns text[] DEFAULT '{}'::text[]) RETURNS public.overlay_data_tables + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +declare + row overlay_data_tables; + old_columns text[]; + old_ops text[]; + old_required text[]; + editor_id int; + invalid_ops text[]; +begin + select * into row from overlay_data_tables where id = table_id and deleted_at is null; + if row is null then + raise exception 'Active data table not found'; + end if; + if not session_is_admin(row.project_id) then + raise exception 'permission denied'; + end if; + if not exists ( + select 1 from table_of_contents_items + where id = row.table_of_contents_item_id and is_draft = true + ) then + raise exception 'Can only update visualization settings on draft layers'; + end if; + + select array_agg(op) into invalid_ops + from unnest(coalesce(set_overlay_data_table_visualization_settings.visualization_ops, '{}')) op + where op not in ('count', 'sum', 'mean', 'min', 'max', 'median'); + if invalid_ops is not null and array_length(invalid_ops, 1) > 0 then + raise exception 'Invalid visualization op(s): %', array_to_string(invalid_ops, ', '); + end if; + + old_columns := row.visualization_columns; + old_ops := row.visualization_ops; + old_required := row.required_filter_columns; + + update overlay_data_tables + set visualization_columns = coalesce(set_overlay_data_table_visualization_settings.visualization_columns, '{}'), + visualization_ops = coalesce(set_overlay_data_table_visualization_settings.visualization_ops, '{}'), + required_filter_columns = coalesce(set_overlay_data_table_visualization_settings.required_filter_columns, '{}'), + updated_at = now() + where id = table_id + returning * into row; + + editor_id := nullif(current_setting('session.user_id', true), '')::int; + if editor_id is not null then + perform record_changelog( + row.project_id, + editor_id, + 'overlay_data_table', + row.id, + 'data_table:visualization_settings_updated'::change_log_field_group, + jsonb_build_object( + 'name', row.name, + 'version', row.version, + 'visualizationColumns', old_columns, + 'visualizationOps', old_ops, + 'requiredFilterColumns', old_required + ), + jsonb_build_object( + 'name', row.name, + 'version', row.version, + 'visualizationColumns', row.visualization_columns, + 'visualizationOps', row.visualization_ops, + 'requiredFilterColumns', row.required_filter_columns + ), + null, null, + jsonb_build_object('table_of_contents_item_id', row.table_of_contents_item_id, 'version', row.version) + ); + end if; + return row; +end; +$$; + + -- -- Name: set_post_hidden_by_moderator(integer, boolean); Type: FUNCTION; Schema: public; Owner: - -- @@ -21753,6 +22162,55 @@ CREATE FUNCTION public.slugify(value text, allow_unicode boolean) RETURNS text $$; +-- +-- Name: soft_delete_overlay_data_table(integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.soft_delete_overlay_data_table(table_id integer) RETURNS public.overlay_data_tables + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +declare + row overlay_data_tables; + editor_id int; +begin + select * into row from overlay_data_tables where id = table_id and deleted_at is null; + if row is null then + raise exception 'Active data table not found'; + end if; + if not session_is_admin(row.project_id) then + raise exception 'permission denied'; + end if; + if not exists ( + select 1 from table_of_contents_items + where id = row.table_of_contents_item_id and is_draft = true + ) then + raise exception 'Can only delete data tables on draft layers'; + end if; + + update overlay_data_tables + set deleted_at = now(), updated_at = now() + where id = table_id + returning * into row; + + editor_id := nullif(current_setting('session.user_id', true), '')::int; + if editor_id is not null then + perform record_changelog( + row.project_id, + editor_id, + 'overlay_data_table', + row.id, + 'data_table:deleted'::change_log_field_group, + jsonb_build_object('name', row.name, 'version', row.version), + '{}'::jsonb, + null, null, + jsonb_build_object('table_of_contents_item_id', row.table_of_contents_item_id) + ); + end if; + return row; +end; +$$; + + -- -- Name: soft_delete_sprite(integer); Type: FUNCTION; Schema: public; Owner: - -- @@ -21904,6 +22362,46 @@ CREATE FUNCTION public.submit_data_upload(id uuid, enable_ai_data_analyst boolea $$; +-- +-- Name: submit_overlay_data_table_upload(uuid); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.submit_overlay_data_table_upload(job_id uuid) RETURNS public.project_background_jobs + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +declare + job project_background_jobs; + pid int; +begin + select project_id into pid + from project_background_jobs + where id = submit_overlay_data_table_upload.job_id; + if not session_is_admin(pid) then + raise exception 'permission denied'; + end if; + if not exists ( + select 1 from overlay_data_table_uploads where project_background_job_id = job_id + ) then + raise exception 'Data table upload not found for job'; + end if; + update project_background_jobs + set + state = 'running', + progress_message = 'uploaded', + started_at = now(), + timeout_at = timezone('utc', now()) + interval '60 seconds' + where id = job_id + returning * into job; + perform graphile_worker.add_job( + 'processDataTableUpload', + json_build_object('jobId', job.id), + max_attempts := 1 + ); + return job; +end; +$$; + + -- -- Name: survey_group_ids(integer); Type: FUNCTION; Schema: public; Owner: - -- @@ -22384,6 +22882,28 @@ CREATE FUNCTION public.table_of_contents_items_contained_by(t public.table_of_co COMMENT ON FUNCTION public.table_of_contents_items_contained_by(t public.table_of_contents_items) IS '@simpleCollections only'; +-- +-- Name: table_of_contents_items_data_table_change_logs(public.table_of_contents_items); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.table_of_contents_items_data_table_change_logs(item public.table_of_contents_items) RETURNS SETOF public.change_logs + LANGUAGE sql STABLE SECURITY DEFINER + AS $$ + select * from change_logs + where entity_type = 'overlay_data_table' + and net_zero_changes = false + and (meta->>'table_of_contents_item_id')::int = item.id + order by last_at desc; + $$; + + +-- +-- Name: FUNCTION table_of_contents_items_data_table_change_logs(item public.table_of_contents_items); Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON FUNCTION public.table_of_contents_items_data_table_change_logs(item public.table_of_contents_items) IS '@simpleCollections only'; + + -- -- Name: table_of_contents_items_download_options(public.table_of_contents_items); Type: FUNCTION; Schema: public; Owner: - -- @@ -22633,6 +23153,31 @@ CREATE FUNCTION public.table_of_contents_items_metadata_xml(item public.table_of $$; +-- +-- Name: table_of_contents_items_overlay_data_tables(public.table_of_contents_items); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.table_of_contents_items_overlay_data_tables(item public.table_of_contents_items) RETURNS SETOF public.overlay_data_tables + LANGUAGE sql STABLE SECURITY DEFINER + AS $$ + select odt.* + from overlay_data_tables odt + where odt.table_of_contents_item_id = item.id + and odt.deleted_at is null + and ( + session_is_admin(item.project_id) + or (item.is_draft = false and _session_on_toc_item_acl(item.path)) + ); +$$; + + +-- +-- Name: FUNCTION table_of_contents_items_overlay_data_tables(item public.table_of_contents_items); Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON FUNCTION public.table_of_contents_items_overlay_data_tables(item public.table_of_contents_items) IS '@simpleCollections only'; + + -- -- Name: table_of_contents_items_primary_download_url(public.table_of_contents_items); Type: FUNCTION; Schema: public; Owner: - -- @@ -22710,20 +23255,23 @@ CREATE FUNCTION public.table_of_contents_items_project(t public.table_of_content CREATE FUNCTION public.table_of_contents_items_project_background_jobs(item public.table_of_contents_items) RETURNS SETOF public.project_background_jobs LANGUAGE sql STABLE SECURITY DEFINER AS $$ - select - * - from - project_background_jobs - where + select * + from project_background_jobs + where session_is_admin(item.project_id) + and ( id = ( - select - project_background_job_id - from - esri_feature_layer_conversion_tasks - where - table_of_contents_item_id = item.id - ) and session_is_admin(item.project_id); - $$; + select project_background_job_id + from esri_feature_layer_conversion_tasks + where table_of_contents_item_id = item.id + limit 1 + ) + or id in ( + select odtu.project_background_job_id + from overlay_data_table_uploads odtu + where odtu.table_of_contents_item_id = item.id + ) + ); +$$; -- @@ -23016,23 +23564,6 @@ CREATE FUNCTION public.table_of_contents_items_uses_dynamic_metadata(t public.ta $$; --- --- Name: tableofcontentsitembystableid(text); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.tableofcontentsitembystableid(stableid text) RETURNS public.table_of_contents_items - LANGUAGE sql STABLE - AS $$ - -- get the table of contents item by stable id and return the first - -- available of published (is_draft = false) or draft (is_draft = true) - select * from table_of_contents_items - where stable_id = stableId - and (is_draft = false or is_draft = true) - order by is_draft asc -- false (published) comes before true (draft) - limit 1; - $$; - - -- -- Name: template_forms(); Type: FUNCTION; Schema: public; Owner: - -- @@ -23464,51 +23995,6 @@ $$; COMMENT ON FUNCTION public.trg_changelog_access_control_lists_type_for_toc() IS 'Draft TOC ACL type change → layer:acl or folder:acl on linked table_of_contents_items (session.user_id).'; --- --- Name: trg_changelog_data_layers_attribution(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.trg_changelog_data_layers_attribution() RETURNS trigger - LANGUAGE plpgsql SECURITY DEFINER - SET search_path TO 'public', 'pg_temp' - AS $$ -declare - v_editor int; -begin - if old.attribution is not distinct from new.attribution then - return new; - end if; - - v_editor := nullif(current_setting('session.user_id', true), '')::int; - if v_editor is null then - return new; - end if; - - perform record_changelog( - new.project_id, - v_editor, - 'data_layers', - new.id, - 'layer:attribution'::change_log_field_group, - jsonb_build_object('attribution', old.attribution), - jsonb_build_object('attribution', new.attribution), - null, - null, - null - ); - - return new; -end; -$$; - - --- --- Name: FUNCTION trg_changelog_data_layers_attribution(); Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON FUNCTION public.trg_changelog_data_layers_attribution() IS 'Records admin changelog entries when a data_layers.attribution value is updated (session.user_id).'; - - -- -- Name: trg_changelog_data_layers_data_source_layer_uploaded(); Type: FUNCTION; Schema: public; Owner: - -- @@ -24636,65 +25122,6 @@ CREATE FUNCTION public.trigger_update_collection_updated_at_for_sketch_folder() $$; --- --- Name: try_record_layer_interactivity_changelog(integer, jsonb, jsonb, jsonb, jsonb); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.try_record_layer_interactivity_changelog(p_acl_id integer, p_from_summary jsonb, p_to_summary jsonb, p_from_blob jsonb, p_to_blob jsonb) RETURNS void - LANGUAGE plpgsql SECURITY DEFINER - SET search_path TO 'public', 'pg_temp' - AS $$ -declare - v_editor int; - v_toc_id int; - v_project_id int; -begin - select toc.id, toc.project_id - into v_toc_id, v_project_id - from access_control_lists acl - join table_of_contents_items toc on toc.id = acl.table_of_contents_item_id - where acl.id = p_acl_id - and acl.table_of_contents_item_id is not null - and toc.is_draft = true - and toc.is_folder = false; - - if v_toc_id is null then - return; - end if; - - if p_from_summary is not distinct from p_to_summary - and p_from_blob is not distinct from p_to_blob then - return; - end if; - - v_editor := nullif(current_setting('session.user_id', true), '')::int; - if v_editor is null then - return; - end if; - - perform record_changelog( - v_project_id, - v_editor, - 'table_of_contents_items', - v_toc_id, - 'layer:interactivity'::change_log_field_group, - p_from_summary, - p_to_summary, - p_from_blob, - p_to_blob, - null - ); -end; -$$; - - --- --- Name: FUNCTION try_record_layer_interactivity_changelog(p_acl_id integer, p_from_summary jsonb, p_to_summary jsonb, p_from_blob jsonb, p_to_blob jsonb); Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON FUNCTION public.try_record_layer_interactivity_changelog(p_acl_id integer, p_from_summary jsonb, p_to_summary jsonb, p_from_blob jsonb, p_to_blob jsonb) IS 'Writes layer:interactivity change_logs for draft layer TOC rows linked to the ACL (session.user_id); no-op if summaries and blobs match.'; - - -- -- Name: unsubscribed(integer); Type: FUNCTION; Schema: public; Owner: - -- @@ -25970,66 +26397,18 @@ CREATE TABLE public.visitor_metrics ( -- CREATE FUNCTION public.visitor_metrics(period public.activity_stats_period) RETURNS SETOF public.visitor_metrics - LANGUAGE plpgsql STABLE SECURITY DEFINER + LANGUAGE sql STABLE SECURITY DEFINER AS $$ -DECLARE - lookback interval; -BEGIN - IF NOT session_is_superuser() THEN - RETURN; - END IF; - - IF period IN ('6-months', '1-year') THEN - lookback := CASE period - WHEN '6-months' THEN '6 months'::interval - ELSE '1 year'::interval - END; - - RETURN QUERY SELECT - '30 days'::interval, - now()::timestamptz, - date_part('month', timezone('UTC', now()))::int, - (SELECT coalesce(jsonb_agg(jsonb_build_object('label', s.label, 'count', s.total) ORDER BY s.total DESC), '[]'::jsonb) - FROM (SELECT elem->>'label' as label, SUM((elem->>'count')::int) as total - FROM public.visitor_metrics vm, jsonb_array_elements(vm.top_referrers) elem - WHERE vm.interval = '30 days'::interval AND vm.timestamp >= now() - lookback - GROUP BY elem->>'label' ORDER BY total DESC LIMIT 15) s), - (SELECT coalesce(jsonb_agg(jsonb_build_object('label', s.label, 'count', s.total) ORDER BY s.total DESC), '[]'::jsonb) - FROM (SELECT elem->>'label' as label, SUM((elem->>'count')::int) as total - FROM public.visitor_metrics vm, jsonb_array_elements(vm.top_operating_systems) elem - WHERE vm.interval = '30 days'::interval AND vm.timestamp >= now() - lookback - GROUP BY elem->>'label' ORDER BY total DESC LIMIT 15) s), - (SELECT coalesce(jsonb_agg(jsonb_build_object('label', s.label, 'count', s.total) ORDER BY s.total DESC), '[]'::jsonb) - FROM (SELECT elem->>'label' as label, SUM((elem->>'count')::int) as total - FROM public.visitor_metrics vm, jsonb_array_elements(vm.top_browsers) elem - WHERE vm.interval = '30 days'::interval AND vm.timestamp >= now() - lookback - GROUP BY elem->>'label' ORDER BY total DESC LIMIT 15) s), - (SELECT coalesce(jsonb_agg(jsonb_build_object('label', s.label, 'count', s.total) ORDER BY s.total DESC), '[]'::jsonb) - FROM (SELECT elem->>'label' as label, SUM((elem->>'count')::int) as total - FROM public.visitor_metrics vm, jsonb_array_elements(vm.top_device_types) elem - WHERE vm.interval = '30 days'::interval AND vm.timestamp >= now() - lookback - GROUP BY elem->>'label' ORDER BY total DESC LIMIT 15) s), - (SELECT coalesce(jsonb_agg(jsonb_build_object('label', s.label, 'count', s.total) ORDER BY s.total DESC), '[]'::jsonb) - FROM (SELECT elem->>'label' as label, SUM((elem->>'count')::int) as total - FROM public.visitor_metrics vm, jsonb_array_elements(vm.top_countries) elem - WHERE vm.interval = '30 days'::interval AND vm.timestamp >= now() - lookback - GROUP BY elem->>'label' ORDER BY total DESC LIMIT 15) s); - ELSE - RETURN QUERY - SELECT * FROM public.visitor_metrics - WHERE session_is_superuser() - AND interval = ( - CASE period - WHEN '24hrs' THEN '24 hours'::interval - WHEN '7-days' THEN '7 days'::interval - WHEN '30-days' THEN '30 days'::interval - ELSE '1 day'::interval - END - ) - ORDER BY timestamp DESC LIMIT 1; - END IF; -END; -$$; + select * from visitor_metrics where session_is_superuser() and + interval = ( + case period + when '24hrs' then '24 hours'::interval + when '7-days' then '7 days'::interval + when '30-days' then '30 days'::interval + else '1 day'::interval + end + ) order by timestamp desc limit 1; + $$; -- @@ -26870,11 +27249,16 @@ ALTER TABLE public.optional_basemap_layers ALTER COLUMN id ADD GENERATED BY DEFA -- --- Name: original_source_id; Type: TABLE; Schema: public; Owner: - +-- Name: overlay_data_tables_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- -CREATE TABLE public.original_source_id ( - data_source_id integer +ALTER TABLE public.overlay_data_tables ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY ( + SEQUENCE NAME public.overlay_data_tables_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 ); @@ -27073,24 +27457,6 @@ CREATE TABLE public.projects_shared_basemaps ( ); --- --- Name: published_toc_item_id; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.published_toc_item_id ( - id integer -); - - --- --- Name: referenced_stable_ids; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.referenced_stable_ids ( - extract_stable_ids_from_body text[] -); - - -- -- Name: report_cards_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- @@ -27972,6 +28338,30 @@ ALTER TABLE ONLY public.optional_basemap_layers ADD CONSTRAINT optional_basemap_layers_pkey PRIMARY KEY (id); +-- +-- Name: overlay_data_table_uploads overlay_data_table_uploads_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.overlay_data_table_uploads + ADD CONSTRAINT overlay_data_table_uploads_pkey PRIMARY KEY (id); + + +-- +-- Name: overlay_data_table_uploads overlay_data_table_uploads_unique_project_background_job; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.overlay_data_table_uploads + ADD CONSTRAINT overlay_data_table_uploads_unique_project_background_job UNIQUE (project_background_job_id); + + +-- +-- Name: overlay_data_tables overlay_data_tables_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.overlay_data_tables + ADD CONSTRAINT overlay_data_tables_pkey PRIMARY KEY (id); + + -- -- Name: pending_topic_notifications pending_topic_notifications_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -28926,6 +29316,27 @@ CREATE UNIQUE INDEX offline_tile_settings_unique_project_default ON public.offli CREATE INDEX optional_basemap_layers_basemap_id_idx ON public.optional_basemap_layers USING btree (basemap_id); +-- +-- Name: overlay_data_tables_active_stable_id_per_toc; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX overlay_data_tables_active_stable_id_per_toc ON public.overlay_data_tables USING btree (table_of_contents_item_id, stable_id) WHERE (deleted_at IS NULL); + + +-- +-- Name: overlay_data_tables_project_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX overlay_data_tables_project_idx ON public.overlay_data_tables USING btree (project_id); + + +-- +-- Name: overlay_data_tables_toc_active_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX overlay_data_tables_toc_active_idx ON public.overlay_data_tables USING btree (table_of_contents_item_id) WHERE (deleted_at IS NULL); + + -- -- Name: posts_topic_id_idx; Type: INDEX; Schema: public; Owner: - -- @@ -29850,6 +30261,13 @@ CREATE TRIGGER on_delete_offline_tile_package_001 AFTER DELETE ON public.offline CREATE TRIGGER on_map_bookmark_update_trigger AFTER UPDATE ON public.map_bookmarks FOR EACH ROW EXECUTE FUNCTION public.on_map_bookmark_update(); +-- +-- Name: overlay_data_tables overlay_data_tables_draft_toc_has_changes; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER overlay_data_tables_draft_toc_has_changes AFTER INSERT OR DELETE OR UPDATE ON public.overlay_data_tables FOR EACH ROW EXECUTE FUNCTION public.overlay_data_tables_draft_toc_has_changes(); + + -- -- Name: posts post_after_insert_notify_subscriptions; Type: TRIGGER; Schema: public; Owner: - -- @@ -30789,6 +31207,62 @@ ALTER TABLE ONLY public.optional_basemap_layers ADD CONSTRAINT optional_basemap_layers_basemap_id_fkey FOREIGN KEY (basemap_id) REFERENCES public.basemaps(id) ON DELETE CASCADE; +-- +-- Name: overlay_data_table_uploads overlay_data_table_uploads_project_background_job_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.overlay_data_table_uploads + ADD CONSTRAINT overlay_data_table_uploads_project_background_job_id_fkey FOREIGN KEY (project_background_job_id) REFERENCES public.project_background_jobs(id) ON DELETE CASCADE; + + +-- +-- Name: overlay_data_table_uploads overlay_data_table_uploads_replace_overlay_data_table_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.overlay_data_table_uploads + ADD CONSTRAINT overlay_data_table_uploads_replace_overlay_data_table_id_fkey FOREIGN KEY (replace_overlay_data_table_id) REFERENCES public.overlay_data_tables(id); + + +-- +-- Name: overlay_data_table_uploads overlay_data_table_uploads_table_of_contents_item_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.overlay_data_table_uploads + ADD CONSTRAINT overlay_data_table_uploads_table_of_contents_item_id_fkey FOREIGN KEY (table_of_contents_item_id) REFERENCES public.table_of_contents_items(id) ON DELETE CASCADE; + + +-- +-- Name: overlay_data_tables overlay_data_tables_created_by_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.overlay_data_tables + ADD CONSTRAINT overlay_data_tables_created_by_fkey FOREIGN KEY (created_by) REFERENCES public.users(id); + + +-- +-- Name: overlay_data_tables overlay_data_tables_project_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.overlay_data_tables + ADD CONSTRAINT overlay_data_tables_project_id_fkey FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE CASCADE; + + +-- +-- Name: overlay_data_tables overlay_data_tables_replaced_by_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.overlay_data_tables + ADD CONSTRAINT overlay_data_tables_replaced_by_id_fkey FOREIGN KEY (replaced_by_id) REFERENCES public.overlay_data_tables(id); + + +-- +-- Name: overlay_data_tables overlay_data_tables_table_of_contents_item_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.overlay_data_tables + ADD CONSTRAINT overlay_data_tables_table_of_contents_item_id_fkey FOREIGN KEY (table_of_contents_item_id) REFERENCES public.table_of_contents_items(id) ON DELETE CASCADE; + + -- -- Name: pending_topic_notifications pending_topic_notifications_topic_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -31164,6 +31638,14 @@ ALTER TABLE ONLY public.sketch_classes COMMENT ON CONSTRAINT sketch_classes_project_id_fkey ON public.sketch_classes IS '@simpleCollections only'; +-- +-- Name: sketch_classes sketch_classes_report_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.sketch_classes + ADD CONSTRAINT sketch_classes_report_id_fkey FOREIGN KEY (report_id) REFERENCES public.reports(id) ON DELETE SET NULL; + + -- -- Name: sketch_classes_valid_children sketch_classes_valid_children_child_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -32143,6 +32625,66 @@ CREATE POLICY optional_basemap_layers_admin ON public.optional_basemap_layers US CREATE POLICY optional_basemap_layers_select ON public.optional_basemap_layers FOR SELECT USING (true); +-- +-- Name: overlay_data_table_uploads; Type: ROW SECURITY; Schema: public; Owner: - +-- + +ALTER TABLE public.overlay_data_table_uploads ENABLE ROW LEVEL SECURITY; + +-- +-- Name: overlay_data_table_uploads overlay_data_table_uploads_insert; Type: POLICY; Schema: public; Owner: - +-- + +CREATE POLICY overlay_data_table_uploads_insert ON public.overlay_data_table_uploads FOR INSERT WITH CHECK (public.session_is_admin(( SELECT table_of_contents_items.project_id + FROM public.table_of_contents_items + WHERE (table_of_contents_items.id = overlay_data_table_uploads.table_of_contents_item_id)))); + + +-- +-- Name: overlay_data_table_uploads overlay_data_table_uploads_select; Type: POLICY; Schema: public; Owner: - +-- + +CREATE POLICY overlay_data_table_uploads_select ON public.overlay_data_table_uploads FOR SELECT USING (public.session_is_admin(( SELECT table_of_contents_items.project_id + FROM public.table_of_contents_items + WHERE (table_of_contents_items.id = overlay_data_table_uploads.table_of_contents_item_id)))); + + +-- +-- Name: overlay_data_tables; Type: ROW SECURITY; Schema: public; Owner: - +-- + +ALTER TABLE public.overlay_data_tables ENABLE ROW LEVEL SECURITY; + +-- +-- Name: overlay_data_tables overlay_data_tables_delete; Type: POLICY; Schema: public; Owner: - +-- + +CREATE POLICY overlay_data_tables_delete ON public.overlay_data_tables FOR DELETE USING ((public.session_is_admin(project_id) AND public.overlay_data_table_linked_toc_is_draft(table_of_contents_item_id, project_id))); + + +-- +-- Name: overlay_data_tables overlay_data_tables_insert; Type: POLICY; Schema: public; Owner: - +-- + +CREATE POLICY overlay_data_tables_insert ON public.overlay_data_tables FOR INSERT WITH CHECK ((public.session_is_admin(project_id) AND public.overlay_data_table_linked_toc_is_draft(table_of_contents_item_id, project_id))); + + +-- +-- Name: overlay_data_tables overlay_data_tables_select; Type: POLICY; Schema: public; Owner: - +-- + +CREATE POLICY overlay_data_tables_select ON public.overlay_data_tables FOR SELECT USING ((public.session_is_admin(project_id) OR (EXISTS ( SELECT 1 + FROM public.table_of_contents_items toc + WHERE ((toc.id = overlay_data_tables.table_of_contents_item_id) AND (toc.is_draft = false) AND public._session_on_toc_item_acl(toc.path)))))); + + +-- +-- Name: overlay_data_tables overlay_data_tables_update; Type: POLICY; Schema: public; Owner: - +-- + +CREATE POLICY overlay_data_tables_update ON public.overlay_data_tables FOR UPDATE USING ((public.session_is_admin(project_id) AND public.overlay_data_table_linked_toc_is_draft(table_of_contents_item_id, project_id))); + + -- -- Name: posts posts_delete; Type: POLICY; Schema: public; Owner: - -- @@ -32180,6 +32722,12 @@ ALTER TABLE public.project_background_jobs ENABLE ROW LEVEL SECURITY; CREATE POLICY project_background_jobs_select ON public.project_background_jobs FOR SELECT USING (public.session_is_admin(project_id)); +-- +-- Name: project_geography; Type: ROW SECURITY; Schema: public; Owner: - +-- + +ALTER TABLE public.project_geography ENABLE ROW LEVEL SECURITY; + -- -- Name: project_groups; Type: ROW SECURITY; Schema: public; Owner: - -- @@ -34811,6 +35359,13 @@ GRANT SELECT(image_id) ON TABLE public.map_bookmarks TO anon; GRANT SELECT(sketch_names) ON TABLE public.map_bookmarks TO anon; +-- +-- Name: COLUMN map_bookmarks.data_table_states; Type: ACL; Schema: public; Owner: - +-- + +GRANT SELECT(data_table_states) ON TABLE public.map_bookmarks TO anon; + + -- -- Name: FUNCTION bookmark_by_id(id uuid); Type: ACL; Schema: public; Owner: - -- @@ -34879,13 +35434,6 @@ REVOKE ALL ON FUNCTION public.box3d(public.geometry) FROM PUBLIC; REVOKE ALL ON FUNCTION public.box3dtobox(public.box3d) FROM PUBLIC; --- --- Name: FUNCTION build_layer_interactivity_acl_payload(p_acl_id integer, p_exclude_group_id integer, p_include_group_id integer, p_type_override text, OUT acl_summary jsonb, OUT acl_blob jsonb); Type: ACL; Schema: public; Owner: - --- - -REVOKE ALL ON FUNCTION public.build_layer_interactivity_acl_payload(p_acl_id integer, p_exclude_group_id integer, p_include_group_id integer, p_type_override text, OUT acl_summary jsonb, OUT acl_blob jsonb) FROM PUBLIC; - - -- -- Name: FUNCTION bump_parent_collection_updated_at(folderid integer, collectionid integer); Type: ACL; Schema: public; Owner: - -- @@ -35290,6 +35838,21 @@ REVOKE ALL ON FUNCTION public.collection_as_geojson(id integer) FROM PUBLIC; GRANT ALL ON FUNCTION public.collection_as_geojson(id integer) TO anon; +-- +-- Name: TABLE overlay_data_tables; Type: ACL; Schema: public; Owner: - +-- + +GRANT SELECT ON TABLE public.overlay_data_tables TO anon; +GRANT SELECT ON TABLE public.overlay_data_tables TO seasketch_user; + + +-- +-- Name: FUNCTION complete_overlay_data_table_upload(job_id uuid, p_name text, p_join_column text, p_overlay_join_column text, p_row_count integer, p_parquet_remote text, p_column_stats_remote text); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.complete_overlay_data_table_upload(job_id uuid, p_name text, p_join_column text, p_overlay_join_column text, p_row_count integer, p_parquet_remote text, p_column_stats_remote text) FROM PUBLIC; + + -- -- Name: FUNCTION compute_project_geography_hash(geog_id integer); Type: ACL; Schema: public; Owner: - -- @@ -35535,6 +36098,22 @@ GRANT SELECT(translated_props) ON TABLE public.table_of_contents_items TO anon; GRANT UPDATE(translated_props) ON TABLE public.table_of_contents_items TO seasketch_user; +-- +-- Name: COLUMN table_of_contents_items.enable_data_tables; Type: ACL; Schema: public; Owner: - +-- + +GRANT SELECT(enable_data_tables) ON TABLE public.table_of_contents_items TO anon; +GRANT UPDATE(enable_data_tables) ON TABLE public.table_of_contents_items TO seasketch_user; + + +-- +-- Name: COLUMN table_of_contents_items.data_table_join_column; Type: ACL; Schema: public; Owner: - +-- + +GRANT SELECT(data_table_join_column) ON TABLE public.table_of_contents_items TO anon; +GRANT UPDATE(data_table_join_column) ON TABLE public.table_of_contents_items TO seasketch_user; + + -- -- Name: FUNCTION copy_data_library_template_item(template_id text, project_slug text); Type: ACL; Schema: public; Owner: - -- @@ -35785,11 +36364,11 @@ GRANT SELECT(content_type) ON TABLE public.data_upload_tasks TO seasketch_user; -- --- Name: FUNCTION create_data_upload(filename text, project_id integer, content_type text, replace_table_of_contents_item_id integer); Type: ACL; Schema: public; Owner: - +-- Name: FUNCTION create_data_upload(filename text, project_id integer, content_type text, replace_table_of_contents_item_id integer, processing_options jsonb); Type: ACL; Schema: public; Owner: - -- -REVOKE ALL ON FUNCTION public.create_data_upload(filename text, project_id integer, content_type text, replace_table_of_contents_item_id integer) FROM PUBLIC; -GRANT ALL ON FUNCTION public.create_data_upload(filename text, project_id integer, content_type text, replace_table_of_contents_item_id integer) TO seasketch_user; +REVOKE ALL ON FUNCTION public.create_data_upload(filename text, project_id integer, content_type text, replace_table_of_contents_item_id integer, processing_options jsonb) FROM PUBLIC; +GRANT ALL ON FUNCTION public.create_data_upload(filename text, project_id integer, content_type text, replace_table_of_contents_item_id integer, processing_options jsonb) TO seasketch_user; -- @@ -35847,11 +36426,11 @@ GRANT ALL ON FUNCTION public.create_inaturalist_table_of_contents_item(slug text -- --- Name: FUNCTION create_map_bookmark(slug text, "isPublic" boolean, style jsonb, "visibleDataLayers" text[], "selectedBasemap" integer, "basemapOptionalLayerStates" jsonb, "cameraOptions" jsonb, "mapDimensions" integer[], "visibleSketches" integer[], "sidebarState" jsonb, "basemapName" text, "layerNames" jsonb, "sketchNames" jsonb, "clientGeneratedThumbnail" text); Type: ACL; Schema: public; Owner: - +-- Name: FUNCTION create_map_bookmark(slug text, "isPublic" boolean, style jsonb, "visibleDataLayers" text[], "selectedBasemap" integer, "basemapOptionalLayerStates" jsonb, "cameraOptions" jsonb, "mapDimensions" integer[], "visibleSketches" integer[], "sidebarState" jsonb, "basemapName" text, "layerNames" jsonb, "sketchNames" jsonb, "clientGeneratedThumbnail" text, "dataTableStates" jsonb); Type: ACL; Schema: public; Owner: - -- -REVOKE ALL ON FUNCTION public.create_map_bookmark(slug text, "isPublic" boolean, style jsonb, "visibleDataLayers" text[], "selectedBasemap" integer, "basemapOptionalLayerStates" jsonb, "cameraOptions" jsonb, "mapDimensions" integer[], "visibleSketches" integer[], "sidebarState" jsonb, "basemapName" text, "layerNames" jsonb, "sketchNames" jsonb, "clientGeneratedThumbnail" text) FROM PUBLIC; -GRANT ALL ON FUNCTION public.create_map_bookmark(slug text, "isPublic" boolean, style jsonb, "visibleDataLayers" text[], "selectedBasemap" integer, "basemapOptionalLayerStates" jsonb, "cameraOptions" jsonb, "mapDimensions" integer[], "visibleSketches" integer[], "sidebarState" jsonb, "basemapName" text, "layerNames" jsonb, "sketchNames" jsonb, "clientGeneratedThumbnail" text) TO seasketch_user; +REVOKE ALL ON FUNCTION public.create_map_bookmark(slug text, "isPublic" boolean, style jsonb, "visibleDataLayers" text[], "selectedBasemap" integer, "basemapOptionalLayerStates" jsonb, "cameraOptions" jsonb, "mapDimensions" integer[], "visibleSketches" integer[], "sidebarState" jsonb, "basemapName" text, "layerNames" jsonb, "sketchNames" jsonb, "clientGeneratedThumbnail" text, "dataTableStates" jsonb) FROM PUBLIC; +GRANT ALL ON FUNCTION public.create_map_bookmark(slug text, "isPublic" boolean, style jsonb, "visibleDataLayers" text[], "selectedBasemap" integer, "basemapOptionalLayerStates" jsonb, "cameraOptions" jsonb, "mapDimensions" integer[], "visibleSketches" integer[], "sidebarState" jsonb, "basemapName" text, "layerNames" jsonb, "sketchNames" jsonb, "clientGeneratedThumbnail" text, "dataTableStates" jsonb) TO seasketch_user; -- @@ -35869,6 +36448,21 @@ REVOKE ALL ON FUNCTION public.create_metadata_xml_output(data_source_id integer, GRANT ALL ON FUNCTION public.create_metadata_xml_output(data_source_id integer, url text, remote text, size bigint, filename text, metadata_type text) TO seasketch_user; +-- +-- Name: TABLE overlay_data_table_uploads; Type: ACL; Schema: public; Owner: - +-- + +GRANT SELECT ON TABLE public.overlay_data_table_uploads TO seasketch_user; + + +-- +-- Name: FUNCTION create_overlay_data_table_upload(toc_item_id integer, filename text, content_type text, processing_options jsonb, replace_overlay_data_table_id integer); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.create_overlay_data_table_upload(toc_item_id integer, filename text, content_type text, processing_options jsonb, replace_overlay_data_table_id integer) FROM PUBLIC; +GRANT ALL ON FUNCTION public.create_overlay_data_table_upload(toc_item_id integer, filename text, content_type text, processing_options jsonb, replace_overlay_data_table_id integer) TO seasketch_user; + + -- -- Name: FUNCTION extract_post_bookmark_attachments(doc jsonb); Type: ACL; Schema: public; Owner: - -- @@ -36853,14 +37447,6 @@ REVOKE ALL ON FUNCTION public.enforce_api_key_limit() FROM PUBLIC; REVOKE ALL ON FUNCTION public.enforce_api_keys_admin_required() FROM PUBLIC; --- --- Name: FUNCTION enforce_resolvable_layer_comment_parent(); Type: ACL; Schema: public; Owner: - --- - -REVOKE ALL ON FUNCTION public.enforce_resolvable_layer_comment_parent() FROM PUBLIC; -GRANT ALL ON FUNCTION public.enforce_resolvable_layer_comment_parent() TO seasketch_user; - - -- -- Name: FUNCTION enqueue_metric_calculations_for_sketch(sketch_id integer); Type: ACL; Schema: public; Owner: - -- @@ -36931,6 +37517,13 @@ REVOKE ALL ON FUNCTION public.fail_data_upload(id uuid, msg text) FROM PUBLIC; GRANT ALL ON FUNCTION public.fail_data_upload(id uuid, msg text) TO seasketch_user; +-- +-- Name: FUNCTION fail_overlay_data_table_upload(job_id uuid, error_message text, error_details jsonb); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.fail_overlay_data_table_upload(job_id uuid, error_message text, error_details jsonb) FROM PUBLIC; + + -- -- Name: FUNCTION filter_state_to_search_string(filters jsonb, sketch_class_id integer); Type: ACL; Schema: public; Owner: - -- @@ -38135,14 +38728,6 @@ REVOKE ALL ON FUNCTION public.get_public_jwk(id uuid) FROM PUBLIC; GRANT ALL ON FUNCTION public.get_public_jwk(id uuid) TO anon; --- --- Name: FUNCTION get_published_card_id_from_draft(draft_report_card_id integer); Type: ACL; Schema: public; Owner: - --- - -REVOKE ALL ON FUNCTION public.get_published_card_id_from_draft(draft_report_card_id integer) FROM PUBLIC; -GRANT ALL ON FUNCTION public.get_published_card_id_from_draft(draft_report_card_id integer) TO seasketch_user; - - -- -- Name: FUNCTION get_referenced_stable_ids_for_report(_report_id integer); Type: ACL; Schema: public; Owner: - -- @@ -39075,6 +39660,29 @@ REVOKE ALL ON FUNCTION public.overlaps_nd(public.gidx, public.geometry) FROM PUB REVOKE ALL ON FUNCTION public.overlaps_nd(public.gidx, public.gidx) FROM PUBLIC; +-- +-- Name: FUNCTION overlay_data_table_linked_toc_is_draft(toc_item_id integer, pid integer); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.overlay_data_table_linked_toc_is_draft(toc_item_id integer, pid integer) FROM PUBLIC; +GRANT ALL ON FUNCTION public.overlay_data_table_linked_toc_is_draft(toc_item_id integer, pid integer) TO seasketch_user; + + +-- +-- Name: FUNCTION overlay_data_table_parquet_public_url(p_remote text); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.overlay_data_table_parquet_public_url(p_remote text) FROM PUBLIC; +GRANT ALL ON FUNCTION public.overlay_data_table_parquet_public_url(p_remote text) TO seasketch_user; + + +-- +-- Name: FUNCTION overlay_data_tables_draft_toc_has_changes(); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.overlay_data_tables_draft_toc_has_changes() FROM PUBLIC; + + -- -- Name: FUNCTION path(public.geometry); Type: ACL; Schema: public; Owner: - -- @@ -40417,6 +41025,14 @@ REVOKE ALL ON FUNCTION public.remove_valid_child_sketch_class(parent integer, ch GRANT ALL ON FUNCTION public.remove_valid_child_sketch_class(parent integer, child integer) TO seasketch_user; +-- +-- Name: FUNCTION rename_overlay_data_table(table_id integer, new_name text); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.rename_overlay_data_table(table_id integer, new_name text) FROM PUBLIC; +GRANT ALL ON FUNCTION public.rename_overlay_data_table(table_id integer, new_name text) TO seasketch_user; + + -- -- Name: FUNCTION rename_report_tab(tab_id integer, title text, alternate_language_settings jsonb); Type: ACL; Schema: public; Owner: - -- @@ -40463,13 +41079,6 @@ REVOKE ALL ON FUNCTION public.replace(public.citext, public.citext, public.citex REVOKE ALL ON FUNCTION public.replace_data_source(data_layer_id integer, data_source_id integer, source_layer text, bounds numeric[], gl_styles jsonb, stableid text) FROM PUBLIC; --- --- Name: FUNCTION replace_toc_references_in_body(body jsonb, published_toc_counterparts jsonb); Type: ACL; Schema: public; Owner: - --- - -REVOKE ALL ON FUNCTION public.replace_toc_references_in_body(body jsonb, published_toc_counterparts jsonb) FROM PUBLIC; - - -- -- Name: FUNCTION report_card_ids_for_report(rid integer); Type: ACL; Schema: public; Owner: - -- @@ -40558,21 +41167,6 @@ REVOKE ALL ON FUNCTION public.reprocess_legacy_data_source(table_of_contents_ite GRANT ALL ON FUNCTION public.reprocess_legacy_data_source(table_of_contents_item_id integer) TO seasketch_user; --- --- Name: FUNCTION resolvable_layer_comment_enforce_draft_toc(); Type: ACL; Schema: public; Owner: - --- - -REVOKE ALL ON FUNCTION public.resolvable_layer_comment_enforce_draft_toc() FROM PUBLIC; - - --- --- Name: FUNCTION resolvable_layer_comment_set_audit_fields(); Type: ACL; Schema: public; Owner: - --- - -REVOKE ALL ON FUNCTION public.resolvable_layer_comment_set_audit_fields() FROM PUBLIC; -GRANT ALL ON FUNCTION public.resolvable_layer_comment_set_audit_fields() TO seasketch_user; - - -- -- Name: FUNCTION resolvable_layer_comments_author_profile(comment public.resolvable_layer_comments); Type: ACL; Schema: public; Owner: - -- @@ -40660,6 +41254,14 @@ REVOKE ALL ON FUNCTION public.revoke_api_key(id uuid) FROM PUBLIC; GRANT ALL ON FUNCTION public.revoke_api_key(id uuid) TO seasketch_user; +-- +-- Name: FUNCTION rollback_overlay_data_table_version(table_id integer); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.rollback_overlay_data_table_version(table_id integer) FROM PUBLIC; +GRANT ALL ON FUNCTION public.rollback_overlay_data_table_version(table_id integer) TO seasketch_user; + + -- -- Name: FUNCTION rollback_to_archived_source(source_id integer, rollback_gl_style boolean); Type: ACL; Schema: public; Owner: - -- @@ -40936,6 +41538,14 @@ REVOKE ALL ON FUNCTION public.set_forum_order("forumIds" integer[]) FROM PUBLIC; GRANT ALL ON FUNCTION public.set_forum_order("forumIds" integer[]) TO seasketch_user; +-- +-- Name: FUNCTION set_overlay_data_table_visualization_settings(table_id integer, visualization_columns text[], visualization_ops text[], required_filter_columns text[]); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.set_overlay_data_table_visualization_settings(table_id integer, visualization_columns text[], visualization_ops text[], required_filter_columns text[]) FROM PUBLIC; +GRANT ALL ON FUNCTION public.set_overlay_data_table_visualization_settings(table_id integer, visualization_columns text[], visualization_ops text[], required_filter_columns text[]) TO seasketch_user; + + -- -- Name: FUNCTION set_post_hidden_by_moderator("postId" integer, value boolean); Type: ACL; Schema: public; Owner: - -- @@ -41231,6 +41841,14 @@ REVOKE ALL ON FUNCTION public.slugify(value text, allow_unicode boolean) FROM PU GRANT ALL ON FUNCTION public.slugify(value text, allow_unicode boolean) TO anon; +-- +-- Name: FUNCTION soft_delete_overlay_data_table(table_id integer); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.soft_delete_overlay_data_table(table_id integer) FROM PUBLIC; +GRANT ALL ON FUNCTION public.soft_delete_overlay_data_table(table_id integer) TO seasketch_user; + + -- -- Name: FUNCTION soft_delete_sprite(id integer); Type: ACL; Schema: public; Owner: - -- @@ -44054,6 +44672,14 @@ REVOKE ALL ON FUNCTION public.submit_data_upload(id uuid, enable_ai_data_analyst GRANT ALL ON FUNCTION public.submit_data_upload(id uuid, enable_ai_data_analyst boolean) TO seasketch_user; +-- +-- Name: FUNCTION submit_overlay_data_table_upload(job_id uuid); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.submit_overlay_data_table_upload(job_id uuid) FROM PUBLIC; +GRANT ALL ON FUNCTION public.submit_overlay_data_table_upload(job_id uuid) TO seasketch_user; + + -- -- Name: FUNCTION subpath(public.ltree, integer); Type: ACL; Schema: public; Owner: - -- @@ -44250,6 +44876,14 @@ REVOKE ALL ON FUNCTION public.table_of_contents_items_contained_by(t public.tabl GRANT ALL ON FUNCTION public.table_of_contents_items_contained_by(t public.table_of_contents_items) TO seasketch_user; +-- +-- Name: FUNCTION table_of_contents_items_data_table_change_logs(item public.table_of_contents_items); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.table_of_contents_items_data_table_change_logs(item public.table_of_contents_items) FROM PUBLIC; +GRANT ALL ON FUNCTION public.table_of_contents_items_data_table_change_logs(item public.table_of_contents_items) TO seasketch_user; + + -- -- Name: FUNCTION table_of_contents_items_download_options(item public.table_of_contents_items); Type: ACL; Schema: public; Owner: - -- @@ -44330,6 +44964,15 @@ REVOKE ALL ON FUNCTION public.table_of_contents_items_metadata_xml(item public.t GRANT ALL ON FUNCTION public.table_of_contents_items_metadata_xml(item public.table_of_contents_items) TO anon; +-- +-- Name: FUNCTION table_of_contents_items_overlay_data_tables(item public.table_of_contents_items); Type: ACL; Schema: public; Owner: - +-- + +REVOKE ALL ON FUNCTION public.table_of_contents_items_overlay_data_tables(item public.table_of_contents_items) FROM PUBLIC; +GRANT ALL ON FUNCTION public.table_of_contents_items_overlay_data_tables(item public.table_of_contents_items) TO seasketch_user; +GRANT ALL ON FUNCTION public.table_of_contents_items_overlay_data_tables(item public.table_of_contents_items) TO anon; + + -- -- Name: FUNCTION table_of_contents_items_primary_download_url(item public.table_of_contents_items); Type: ACL; Schema: public; Owner: - -- @@ -44433,13 +45076,6 @@ REVOKE ALL ON FUNCTION public.table_of_contents_items_uses_dynamic_metadata(t pu GRANT ALL ON FUNCTION public.table_of_contents_items_uses_dynamic_metadata(t public.table_of_contents_items) TO anon; --- --- Name: FUNCTION tableofcontentsitembystableid(stableid text); Type: ACL; Schema: public; Owner: - --- - -REVOKE ALL ON FUNCTION public.tableofcontentsitembystableid(stableid text) FROM PUBLIC; - - -- -- Name: FUNCTION template_forms(); Type: ACL; Schema: public; Owner: - -- @@ -44645,13 +45281,6 @@ REVOKE ALL ON FUNCTION public.trg_changelog_access_control_list_groups_for_toc() REVOKE ALL ON FUNCTION public.trg_changelog_access_control_lists_type_for_toc() FROM PUBLIC; --- --- Name: FUNCTION trg_changelog_data_layers_attribution(); Type: ACL; Schema: public; Owner: - --- - -REVOKE ALL ON FUNCTION public.trg_changelog_data_layers_attribution() FROM PUBLIC; - - -- -- Name: FUNCTION trg_changelog_data_layers_data_source_layer_uploaded(); Type: ACL; Schema: public; Owner: - -- @@ -44799,13 +45428,6 @@ REVOKE ALL ON FUNCTION public.trigger_update_collection_updated_at() FROM PUBLIC REVOKE ALL ON FUNCTION public.trigger_update_collection_updated_at_for_sketch_folder() FROM PUBLIC; --- --- Name: FUNCTION try_record_layer_interactivity_changelog(p_acl_id integer, p_from_summary jsonb, p_to_summary jsonb, p_from_blob jsonb, p_to_blob jsonb); Type: ACL; Schema: public; Owner: - --- - -REVOKE ALL ON FUNCTION public.try_record_layer_interactivity_changelog(p_acl_id integer, p_from_summary jsonb, p_to_summary jsonb, p_from_blob jsonb, p_to_blob jsonb) FROM PUBLIC; - - -- -- Name: FUNCTION unaccent(text); Type: ACL; Schema: public; Owner: - -- diff --git a/packages/api/src/graphileOptions.ts b/packages/api/src/graphileOptions.ts index a28a8c31e..8fc8c53c2 100644 --- a/packages/api/src/graphileOptions.ts +++ b/packages/api/src/graphileOptions.ts @@ -51,6 +51,7 @@ import ReportsPlugin from "./plugins/reportsPlugin"; import FeatureFlagsPlugin from "./plugins/featureFlagsPlugin"; import UserActivityPlugin from "./plugins/userActivityPlugin"; import DataUploadPiiClassifierWarmPlugin from "./plugins/dataUploadPiiClassifierWarmPlugin"; +import OverlayDataTablePlugin from "./plugins/overlayDataTablePlugin"; import PublishTilesAclPlugin from "./plugins/publishTilesAclPlugin"; import HostedTileUuidsRequiringAuthPlugin from "./plugins/hostedTileUuidsRequiringAuthPlugin"; @@ -117,6 +118,7 @@ export default function graphileOptions(): PostGraphileOptions { ReportsPlugin, FeatureFlagsPlugin, UserActivityPlugin, + OverlayDataTablePlugin, PublishTilesAclPlugin, HostedTileUuidsRequiringAuthPlugin, // reorderSchemaFields(graphqlSchemaModifiers.fieldOrder), diff --git a/packages/api/src/plugins/featureFlagsPlugin.ts b/packages/api/src/plugins/featureFlagsPlugin.ts index 2ec9085dd..b62f8c71c 100644 --- a/packages/api/src/plugins/featureFlagsPlugin.ts +++ b/packages/api/src/plugins/featureFlagsPlugin.ts @@ -8,6 +8,11 @@ const FeatureFlagsPlugin = makeExtendSchemaPlugin((build) => { typeDefs: gql` type FeatureFlags { iNaturalistLayers: Boolean + """ + When true, project admins see the Data Tables tab in layer editors + and related map UI. Controlled from SeaSketch developer settings. + """ + dataTables: Boolean } extend type Project { diff --git a/packages/api/src/plugins/overlayDataTablePlugin.ts b/packages/api/src/plugins/overlayDataTablePlugin.ts new file mode 100644 index 000000000..f4ddce3e6 --- /dev/null +++ b/packages/api/src/plugins/overlayDataTablePlugin.ts @@ -0,0 +1,70 @@ +import { makeExtendSchemaPlugin, gql } from "graphile-utils"; +import { S3 } from "aws-sdk"; + +export const DATA_TABLE_UPLOAD_PRESIGNED_URL_TTL = 60 * 120; + +function remoteKey(remote: string): string { + return remote.replace(/^r2:\/\//, "").split("/").slice(1).join("/"); +} + +const UPLOADS_PUBLIC_BASE_URL = "https://uploads.seasketch.org"; + +function remoteToPublicUrl(remote: string | null | undefined): string | null { + if (!remote) return null; + return `${UPLOADS_PUBLIC_BASE_URL}/${remoteKey(remote)}`; +} + +function tablePathFromParquetRemote( + remote: string | null | undefined +): string | null { + if (!remote) return null; + const key = remoteKey(remote); + const suffix = "/data.parquet"; + if (!key.endsWith(suffix)) return null; + return key.slice(0, -suffix.length); +} + +function remoteToQueryUrl(remote: string | null | undefined): string | null { + const tablePath = tablePathFromParquetRemote(remote); + if (!tablePath) return null; + // Served by pmtiles-server DataTablesBackend on the uploads host (same ACL + // as the parent layer's published UUID). + return `${UPLOADS_PUBLIC_BASE_URL}/${tablePath}/query`; +} + +const OverlayDataTablePlugin = makeExtendSchemaPlugin(() => ({ + typeDefs: gql` + extend type OverlayDataTable { + parquetUrl: String @requires(columns: ["parquet_remote"]) + columnStatsUrl: String @requires(columns: ["column_stats_remote"]) + queryUrl: String @requires(columns: ["parquet_remote"]) + } + + extend type OverlayDataTableUpload { + presignedUploadUrl: String + @requires(columns: ["id", "filename", "contentType"]) + } + `, + resolvers: { + OverlayDataTable: { + parquetUrl: (table) => remoteToPublicUrl(table.parquetRemote), + columnStatsUrl: (table) => remoteToPublicUrl(table.columnStatsRemote), + queryUrl: (table) => remoteToQueryUrl(table.parquetRemote), + }, + OverlayDataTableUpload: { + presignedUploadUrl: async (upload) => { + const region = process.env.S3_REGION; + const bucket = process.env.SPATIAL_UPLOADS_BUCKET; + const s3 = new S3({ region }); + return s3.getSignedUrlPromise("putObject", { + Bucket: bucket, + Key: `${upload.id}/${upload.filename}`, + Expires: DATA_TABLE_UPLOAD_PRESIGNED_URL_TTL, + ContentType: upload.contentType, + }); + }, + }, + }, +})); + +export default OverlayDataTablePlugin; diff --git a/packages/api/tasks/cleanupProjectBackgroundJobs.ts b/packages/api/tasks/cleanupProjectBackgroundJobs.ts index f8a6fa91e..9ffb4323a 100644 --- a/packages/api/tasks/cleanupProjectBackgroundJobs.ts +++ b/packages/api/tasks/cleanupProjectBackgroundJobs.ts @@ -11,7 +11,14 @@ import { Helpers } from "graphile-worker"; async function cleanupProjectBackgroundJobs(payload: {}, helpers: Helpers) { await helpers.withPgClient(async (client) => { await client.query(` - update project_background_jobs set state = 'failed', error_message = 'Timed out', progress_message = 'timeout' where (state = 'queued' or state = 'running') and now() >= timeout_at + select fail_overlay_data_table_upload(pbj.id, 'Timed out') + from project_background_jobs pbj + where pbj.type = 'data_table_upload' + and pbj.state in ('queued', 'running') + and now() >= pbj.timeout_at + `); + await client.query(` + update project_background_jobs set state = 'failed', error_message = 'Timed out', progress_message = 'timeout' where (state = 'queued' or state = 'running') and now() >= timeout_at and type != 'data_table_upload' `); await client.query(` delete from project_background_jobs where state = 'queued' and created_at < now() - interval '1 day' diff --git a/packages/api/tasks/processDataTableUpload.ts b/packages/api/tasks/processDataTableUpload.ts new file mode 100644 index 000000000..4036925fa --- /dev/null +++ b/packages/api/tasks/processDataTableUpload.ts @@ -0,0 +1,128 @@ +import { Helpers } from "graphile-worker"; +import AWS from "aws-sdk"; +import { extractHostedTilesUuid } from "../src/tilesAcl/writeProjectAclDoc"; + +interface DataTablesHandlerRequest { + taskId: string; + uploadId: string; + objectKey: string; + /** Project slug; first path segment of the R2 key. */ + slug: string; + /** Parent layer hosted-tiles UUID; R2 key prefix for ACL classification. */ + sourceUuid: string; + skipLoggingProgress?: boolean; +} + +const client = new AWS.Lambda({ + region: process.env.AWS_REGION || "us-west-2", + httpOptions: { timeout: 60000 * 5.5 }, +}); + +export async function runDataTablesLambda(event: DataTablesHandlerRequest) { + if (process.env.DATA_TABLES_LAMBDA_DEV_HANDLER) { + const url = process.env.DATA_TABLES_LAMBDA_DEV_HANDLER; + const response = await fetch(url, { + method: "POST", + body: JSON.stringify(event), + headers: { "Content-Type": "application/json" }, + }); + const text = await response.text(); + if (!response.ok) { + throw new Error( + `Data tables dev handler HTTP ${response.status}: ${text.slice(0, 500)}`, + ); + } + const data = JSON.parse(text); + if (data && data.error) { + throw new Error(data.error); + } + return data; + } + if (process.env.DATA_TABLES_HANDLER_LAMBDA_ARN) { + // The production lambda can only reach the production RDS instance. When + // this API runs against any other database, tell the lambda not to write + // progress updates (they would go to the wrong DB). + const apiUsesProductionDb = + !!process.env.PGHOST && /rds\.amazonaws\.com/.test(process.env.PGHOST); + await client + .invoke({ + InvocationType: "Event", + FunctionName: process.env.DATA_TABLES_HANDLER_LAMBDA_ARN, + Payload: JSON.stringify({ + ...event, + skipLoggingProgress: !apiUsesProductionDb, + }), + }) + .promise(); + return; + } + throw new Error( + "Neither DATA_TABLES_HANDLER_LAMBDA_ARN nor DATA_TABLES_LAMBDA_DEV_HANDLER are defined. " + + "For local dev, run `npm run dev` in packages/data-tables-handler and set " + + "DATA_TABLES_LAMBDA_DEV_HANDLER=http://localhost:3006 on the API server.", + ); +} + +export default async function processDataTableUpload( + payload: { jobId: string }, + helpers: Helpers, +) { + const { jobId } = payload; + helpers.logger.info(`Handling data table upload: ${jobId}`); + await helpers.withPgClient(async (client) => { + await client.query( + `update project_background_jobs set progress_message = 'processing', state = 'running', started_at = now(), timeout_at = timezone('utc', now()) + interval '60 seconds' where id = $1`, + [jobId], + ); + const q = await client.query( + `select + odtu.id as upload_id, + odtu.id || '/' || odtu.filename as object_key, + p.slug, + ds.url as data_source_url + from overlay_data_table_uploads odtu + join project_background_jobs pbj on pbj.id = odtu.project_background_job_id + join projects p on p.id = pbj.project_id + join table_of_contents_items toc on toc.id = odtu.table_of_contents_item_id + join data_layers dl on dl.id = toc.data_layer_id + join data_sources ds on ds.id = dl.data_source_id + where odtu.project_background_job_id = $1 + limit 1`, + [jobId], + ); + if (!q.rows[0]) { + throw new Error("Could not find upload for job " + jobId); + } + const sourceUuid = extractHostedTilesUuid(q.rows[0].data_source_url); + if (!sourceUuid) { + const message = + "Parent layer has no hosted tiles UUID; data tables must be stored under the layer path for ACL"; + await client.query(`select fail_overlay_data_table_upload($1, $2)`, [ + jobId, + message, + ]); + throw new Error(message); + } + try { + await runDataTablesLambda({ + taskId: jobId, + uploadId: q.rows[0].upload_id, + objectKey: q.rows[0].object_key, + slug: q.rows[0].slug, + sourceUuid, + }); + } catch (e) { + if (process.env.DATA_TABLES_LAMBDA_DEV_HANDLER) { + await client.query(`select fail_overlay_data_table_upload($1, $2)`, [ + jobId, + (e as Error).message, + ]); + } else { + await client.query(`select fail_overlay_data_table_upload($1, $2)`, [ + jobId, + "Lambda invocation failure", + ]); + } + } + }); +} diff --git a/packages/api/tasks/processDataTableUploadOutputs.ts b/packages/api/tasks/processDataTableUploadOutputs.ts new file mode 100644 index 000000000..a935ba4ce --- /dev/null +++ b/packages/api/tasks/processDataTableUploadOutputs.ts @@ -0,0 +1,48 @@ +import { Helpers } from "graphile-worker"; + +export default async function processDataTableUploadOutputs( + payload: { + jobId: string; + data: { + uploadId: string; + name: string; + joinColumn: string; + overlayJoinColumn: string; + rowCount: number; + parquetRemote: string; + columnStatsRemote: string; + }; + }, + helpers: Helpers, +) { + const { jobId, data } = payload; + helpers.logger.info(`Completing data table upload: ${jobId}`); + await helpers.withPgClient(async (client) => { + try { + if (!data?.parquetRemote || !data?.columnStatsRemote) { + await client.query(`select fail_overlay_data_table_upload($1, $2)`, [ + jobId, + "Missing output artifacts from processor", + ]); + return; + } + await client.query( + `select complete_overlay_data_table_upload($1, $2, $3, $4, $5, $6, $7)`, + [ + jobId, + data.name, + data.joinColumn, + data.overlayJoinColumn, + data.rowCount, + data.parquetRemote, + data.columnStatsRemote, + ], + ); + } catch (e) { + await client.query(`select fail_overlay_data_table_upload($1, $2)`, [ + jobId, + (e as Error).message, + ]); + } + }); +} diff --git a/packages/api/tasks/processDataUpload.ts b/packages/api/tasks/processDataUpload.ts index 058c9f183..37f05a6d7 100644 --- a/packages/api/tasks/processDataUpload.ts +++ b/packages/api/tasks/processDataUpload.ts @@ -29,7 +29,8 @@ export default async function processDataUpload( const q = await client.query( ` select - id || '/' || filename as object_key + id || '/' || filename as object_key, + processing_options from data_upload_tasks where @@ -42,6 +43,7 @@ export default async function processDataUpload( throw new Error("Could not find objectKey for job with ID=" + jobId); } const objectKey = q.rows[0].object_key; + const processingOptions = q.rows[0].processing_options ?? undefined; if (!job) { throw new Error("Could not find job with ID=" + job.id); } @@ -75,6 +77,7 @@ export default async function processDataUpload( requestingUser: user.fullname ? `${user.fullname} <${user.email || user.canonical_email}>` : user.email || user.canonical_email, + processingOptions, }); } catch (e) { console.error("error!!", e); diff --git a/packages/api/tests/changelogs.test.ts b/packages/api/tests/changelogs.test.ts index cc7b0bc33..3e6c5f773 100644 --- a/packages/api/tests/changelogs.test.ts +++ b/packages/api/tests/changelogs.test.ts @@ -270,6 +270,12 @@ describe("change_logs", () => { expect(groups.map((row: any) => row.enumlabel).sort()).toEqual( [ + "data_table:created", + "data_table:deleted", + "data_table:renamed", + "data_table:replaced", + "data_table:rollback", + "data_table:visualization_settings_updated", "folder:acl", "folder:created", "folder:deleted", diff --git a/packages/api/tests/overlayDataTables.test.ts b/packages/api/tests/overlayDataTables.test.ts new file mode 100644 index 000000000..db89a7942 --- /dev/null +++ b/packages/api/tests/overlayDataTables.test.ts @@ -0,0 +1,466 @@ +import { sql } from "slonik"; +import { createPool } from "./pool"; +import { + createUser, + createProject, + createSession, + clearSession, + projectTransaction, +} from "./helpers"; +// @ts-ignore +import nanoid from "nanoid"; + +const id = nanoid.customAlphabet( + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz", + 9, +); + +const pool = createPool("test"); + +async function asPostgres( + conn: any, + fn: () => Promise, + restore?: { userId: number; projectId: number }, +) { + await conn.any(sql`SAVEPOINT as_postgres`); + await conn.any(sql`set role postgres`); + try { + await fn(); + await conn.any(sql`set role seasketch_user`); + await conn.any(sql`RELEASE SAVEPOINT as_postgres`); + if (restore) { + await createSession(conn, restore.userId, true, false, restore.projectId); + } + } catch (error) { + await conn.any(sql`ROLLBACK TO SAVEPOINT as_postgres`); + await conn.any(sql`RELEASE SAVEPOINT as_postgres`); + if (restore) { + await createSession(conn, restore.userId, true, false, restore.projectId); + } + throw error; + } +} + +async function createDraftLayer( + conn: any, + projectId: number, + adminId: number, +) { + const sourceId = await conn.oneFirst( + sql`insert into data_sources (project_id, type, url, attribution) + values (${projectId}, 'vector', 'https://example.com/vector-tiles/{z}/{x}/{y}.pbf', 'test') + returning id`, + ); + const layerId = await conn.oneFirst( + sql`insert into data_layers (project_id, data_source_id, source_layer, mapbox_gl_styles) + values (${projectId}, ${sourceId}, 'test-layer', ${sql.json([ + { type: "circle", paint: { "circle-color": "#0000ff" } }, + ])}) returning id`, + ); + const stableId = id(); + const tocId = await conn.oneFirst( + sql`insert into table_of_contents_items (project_id, title, is_folder, data_layer_id, stable_id) + values (${projectId}, 'Sites', false, ${layerId}, ${stableId}) returning id`, + ); + return { sourceId, layerId, tocId }; +} + +describe("overlay_data_tables", () => { + test("admin can insert and soft delete draft data table", async () => { + await projectTransaction( + pool, + "public", + async (conn, projectId, adminId) => { + await createSession(conn, adminId, true, false, projectId); + const { tocId } = await createDraftLayer(conn, projectId, adminId); + + let jobId: string; + await asPostgres( + conn, + async () => { + jobId = (await conn.oneFirst(sql` + insert into project_background_jobs (project_id, title, type, user_id) + values (${projectId}, 'test', 'data_table_upload', ${adminId}) returning id`)) as string; + await conn.any(sql` + insert into overlay_data_table_uploads ( + project_background_job_id, table_of_contents_item_id, filename, content_type, + overlay_geostats + ) values ( + ${jobId}, ${tocId}, 'fish.csv', 'text/csv', '{"layers":[]}'::jsonb + )`); + await conn.any(sql` + select complete_overlay_data_table_upload( + ${jobId}, 'fish', 'site_id', 'id', 10, + 'r2://bucket/projects/test/public/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/dataTables/u1/data.parquet', + 'r2://bucket/projects/test/public/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/dataTables/u1/column-stats.json' + )`); + }, + { userId: adminId, projectId }, + ); + + const row = await conn.one( + sql`select name from overlay_data_tables where table_of_contents_item_id = ${tocId} and deleted_at is null`, + ); + expect(row.name).toBe("fish"); + + const tableId = await conn.oneFirst( + sql`select id from overlay_data_tables where table_of_contents_item_id = ${tocId} and deleted_at is null`, + ); + await conn.any( + sql`select soft_delete_overlay_data_table(${tableId})`, + ); + const deleted = await conn.one( + sql`select deleted_at from overlay_data_tables where id = ${tableId}`, + ); + expect(deleted.deleted_at).not.toBeNull(); + }, + ); + }); + + test("non-admin cannot insert overlay data tables", async () => { + await projectTransaction( + pool, + "public", + async (conn, projectId, adminId, [userA]) => { + await createSession(conn, adminId, true, false, projectId); + const { tocId } = await createDraftLayer(conn, projectId, adminId); + await createSession(conn, userA, true, false, projectId); + await expect( + conn.oneFirst(sql` + insert into overlay_data_tables ( + table_of_contents_item_id, project_id, name, join_column, overlay_join_column, + row_count, created_by, parquet_remote, column_stats_remote + ) values ( + ${tocId}, ${projectId}, 'fish', 'site_id', 'id', 10, ${userA}, + 'r2://bucket/a.parquet', 'r2://bucket/a.json' + ) returning id`), + ).rejects.toThrow(); + }, + ); + }); + + test("publish copies active draft tables to published toc item", async () => { + await projectTransaction( + pool, + "public", + async (conn, projectId, adminId) => { + await createSession(conn, adminId, true, false, projectId); + const { tocId } = await createDraftLayer(conn, projectId, adminId); + + await asPostgres( + conn, + async () => { + await conn.any(sql` + update table_of_contents_items + set enable_data_tables = true, data_table_join_column = 'id' + where id = ${tocId}`); + await conn.any(sql` + insert into overlay_data_tables ( + table_of_contents_item_id, project_id, name, join_column, overlay_join_column, + row_count, created_by, parquet_remote, column_stats_remote, + visualization_columns, visualization_ops, required_filter_columns + ) values ( + ${tocId}, ${projectId}, 'fish', 'site_id', 'id', 10, ${adminId}, + 'r2://bucket/projects/test/public/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/dataTables/u1/data.parquet', + 'r2://bucket/projects/test/public/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/dataTables/u1/column-stats.json', + ${sql.array(['count'], 'text')}, ${sql.array(['sum'], 'text')}, + ${sql.array(['year'], 'text')} + )`); + // Soft-deleted draft history must not be published. + await conn.any(sql` + insert into overlay_data_tables ( + table_of_contents_item_id, project_id, name, join_column, overlay_join_column, + row_count, created_by, parquet_remote, column_stats_remote, deleted_at + ) values ( + ${tocId}, ${projectId}, 'old-fish', 'site_id', 'id', 1, ${adminId}, + 'r2://bucket/old.parquet', 'r2://bucket/old.json', now() + )`); + }, + { userId: adminId, projectId }, + ); + + await conn.any(sql`select publish_table_of_contents(${projectId})`); + + const publishedToc = await conn.one(sql` + select enable_data_tables, data_table_join_column + from table_of_contents_items + where project_id = ${projectId} and is_draft = false and is_folder = false`); + expect(publishedToc.enable_data_tables).toBe(true); + expect(publishedToc.data_table_join_column).toBe("id"); + + const draftStableId = await conn.oneFirst(sql` + select stable_id from overlay_data_tables + where table_of_contents_item_id = ${tocId} and deleted_at is null`); + + const published = await conn.many(sql` + select odt.name, toc.is_draft, odt.visualization_columns, odt.visualization_ops, + odt.required_filter_columns, odt.parquet_remote, odt.stable_id + from overlay_data_tables odt + inner join table_of_contents_items toc on toc.id = odt.table_of_contents_item_id + where odt.project_id = ${projectId} and toc.is_draft = false + order by odt.name`); + + expect(published).toHaveLength(1); + expect(published[0].name).toBe("fish"); + expect(published[0].is_draft).toBe(false); + expect(published[0].visualization_columns).toEqual(["count"]); + expect(published[0].visualization_ops).toEqual(["sum"]); + expect(published[0].required_filter_columns).toEqual(["year"]); + expect(published[0].stable_id).toBe(draftStableId); + + const draftStillThere = await conn.oneFirst(sql` + select count(*) from overlay_data_tables odt + inner join table_of_contents_items toc on toc.id = odt.table_of_contents_item_id + where odt.project_id = ${projectId} and toc.is_draft = true and odt.deleted_at is null`); + expect(draftStillThere).toBe(1); + + // Republish replaces prior published copies (CASCADE delete of old published TOC). + await asPostgres( + conn, + async () => { + await conn.any(sql` + update overlay_data_tables + set name = 'fish-v2', visualization_ops = ${sql.array(['mean'], 'text')} + where table_of_contents_item_id = ${tocId} and deleted_at is null`); + }, + { userId: adminId, projectId }, + ); + await conn.any(sql`select publish_table_of_contents(${projectId})`); + + const republished = await conn.many(sql` + select odt.name, odt.visualization_ops + from overlay_data_tables odt + inner join table_of_contents_items toc on toc.id = odt.table_of_contents_item_id + where odt.project_id = ${projectId} and toc.is_draft = false`); + expect(republished).toHaveLength(1); + expect(republished[0].name).toBe("fish-v2"); + expect(republished[0].visualization_ops).toEqual(["mean"]); + }, + ); + }); + + test("allows duplicate active table names per toc item", async () => { + await projectTransaction( + pool, + "public", + async (conn, projectId, adminId) => { + await createSession(conn, adminId, true, false, projectId); + const { tocId } = await createDraftLayer(conn, projectId, adminId); + + await asPostgres( + conn, + async () => { + await conn.any(sql` + insert into overlay_data_tables ( + table_of_contents_item_id, project_id, name, join_column, overlay_join_column, + row_count, created_by, parquet_remote, column_stats_remote + ) values ( + ${tocId}, ${projectId}, 'fish', 'site_id', 'id', 10, ${adminId}, + 'r2://bucket/a.parquet', 'r2://bucket/a.json' + )`); + await conn.any(sql` + insert into overlay_data_tables ( + table_of_contents_item_id, project_id, name, join_column, overlay_join_column, + row_count, created_by, parquet_remote, column_stats_remote + ) values ( + ${tocId}, ${projectId}, 'fish', 'site_id', 'id', 5, ${adminId}, + 'r2://bucket/b.parquet', 'r2://bucket/b.json' + )`); + }, + { userId: adminId, projectId }, + ); + + const count = await conn.oneFirst(sql` + select count(*) from overlay_data_tables + where table_of_contents_item_id = ${tocId} + and name = 'fish' + and deleted_at is null`); + expect(count).toBe(2); + }, + ); + }); + + test("replace increments version and soft deletes previous", async () => { + await projectTransaction( + pool, + "public", + async (conn, projectId, adminId) => { + await createSession(conn, adminId, true, false, projectId); + const { tocId } = await createDraftLayer(conn, projectId, adminId); + + let jobId: string; + let oldId: number; + await asPostgres( + conn, + async () => { + jobId = (await conn.oneFirst(sql` + insert into project_background_jobs (project_id, title, type, user_id) + values (${projectId}, 'test', 'data_table_upload', ${adminId}) returning id`)) as string; + + oldId = Number(await conn.oneFirst(sql` + insert into overlay_data_tables ( + table_of_contents_item_id, project_id, name, join_column, overlay_join_column, + row_count, created_by, version, parquet_remote, column_stats_remote + ) values ( + ${tocId}, ${projectId}, 'fish', 'site_id', 'id', 10, ${adminId}, 1, + 'r2://bucket/old.parquet', 'r2://bucket/old.json' + ) returning id`)); + + await conn.any(sql` + insert into overlay_data_table_uploads ( + project_background_job_id, table_of_contents_item_id, filename, content_type, + overlay_geostats, replace_overlay_data_table_id + ) values ( + ${jobId}, ${tocId}, 'fish.csv', 'text/csv', + '{"layers":[{"attributes":[]}]}'::jsonb, + ${oldId} + )`); + await conn.any(sql` + select complete_overlay_data_table_upload( + ${jobId}, 'fish', 'site_id', 'id', 20, + 'r2://bucket/new.parquet', 'r2://bucket/new.json' + )`); + }, + { userId: adminId, projectId }, + ); + + const oldRow = await conn.one( + sql`select deleted_at, replaced_by_id, version, stable_id from overlay_data_tables where id = ${oldId!}`, + ); + expect(oldRow.deleted_at).not.toBeNull(); + expect(oldRow.replaced_by_id).not.toBeNull(); + + const newRow = await conn.one( + sql`select version, deleted_at, stable_id from overlay_data_tables where id = ${oldRow.replaced_by_id}`, + ); + expect(newRow.version).toBe(2); + expect(newRow.deleted_at).toBeNull(); + expect(newRow.stable_id).toBe(oldRow.stable_id); + }, + ); + }); + + test("replace records changelog when completed without session", async () => { + await projectTransaction( + pool, + "public", + async (conn, projectId, adminId) => { + await createSession(conn, adminId, true, false, projectId); + const { tocId } = await createDraftLayer(conn, projectId, adminId); + + let jobId: string; + let oldId: number; + await asPostgres( + conn, + async () => { + jobId = (await conn.oneFirst(sql` + insert into project_background_jobs (project_id, title, type, user_id) + values (${projectId}, 'test', 'data_table_upload', ${adminId}) returning id`)) as string; + + oldId = Number(await conn.oneFirst(sql` + insert into overlay_data_tables ( + table_of_contents_item_id, project_id, name, join_column, overlay_join_column, + row_count, created_by, version, parquet_remote, column_stats_remote + ) values ( + ${tocId}, ${projectId}, 'fish', 'site_id', 'id', 10, ${adminId}, 1, + 'r2://bucket/old.parquet', 'r2://bucket/old.json' + ) returning id`)); + + await conn.any(sql` + insert into overlay_data_table_uploads ( + project_background_job_id, table_of_contents_item_id, filename, content_type, + overlay_geostats, replace_overlay_data_table_id + ) values ( + ${jobId}, ${tocId}, 'fish.csv', 'text/csv', + '{"layers":[{"attributes":[]}]}'::jsonb, + ${oldId} + )`); + await clearSession(conn); + await conn.any(sql` + select set_config('seasketch.uploads_base_url', 'https://uploads.example.org', true) + `); + await conn.any(sql` + select complete_overlay_data_table_upload( + ${jobId}, 'fish', 'site_id', 'id', 20, + 'r2://bucket/new.parquet', 'r2://bucket/new.json' + )`); + }, + { userId: adminId, projectId }, + ); + + const changelog = await conn.one(sql` + select field_group, editor_id, from_summary, to_summary + from change_logs + where entity_type = 'overlay_data_table' + and field_group = 'data_table:replaced' + and (meta->>'table_of_contents_item_id')::int = ${tocId} + order by last_at desc + limit 1`); + expect(changelog.field_group).toBe("data_table:replaced"); + expect(changelog.editor_id).toBe(adminId); + expect(changelog.from_summary).toEqual( + expect.objectContaining({ name: "fish", version: 1 }), + ); + expect(changelog.to_summary).toEqual( + expect.objectContaining({ name: "fish", version: 2 }), + ); + expect(changelog.from_summary).toEqual( + expect.objectContaining({ + name: "fish", + version: 1, + parquet_url: "https://uploads.example.org/old.parquet", + }), + ); + }, + ); + }); + + test("create_map_bookmark accepts dataTableStates", async () => { + await projectTransaction( + pool, + "public", + async (conn, projectId, adminId) => { + await createSession(conn, adminId, true, false, projectId); + const slug = (await conn.oneFirst( + sql`select slug from projects where id = ${projectId}`, + )) as string; + const basemapId = (await conn.oneFirst(sql` + insert into basemaps (project_id, name, type, url, thumbnail) + values ( + ${projectId}, 'basemap a', 'MAPBOX', + 'mapbox://my-map/id', 'https://thumbnail.org/1.png' + ) returning id`)) as number; + + const dataTableStates = { + "toc-stable-id": { + stableId: "11111111-2222-3333-4444-555555555555", + column: "biomass", + op: "sum", + }, + }; + + const bookmark = await conn.one(sql` + select data_table_states + from create_map_bookmark( + ${slug}, + true, + ${sql.json({})}, + ${sql.array([], "text")}, + ${basemapId}, + ${sql.json({})}, + ${sql.json({ center: [0, 0], zoom: 1 })}, + ${sql.array([800, 600], "int4")}, + ${sql.array([], "int4")}, + null, + 'basemap a', + ${sql.json({})}, + ${sql.json({})}, + 'data:image/jpeg;base64,abc', + ${sql.json(dataTableStates)} + )`); + + expect(bookmark.data_table_states).toEqual(dataTableStates); + }, + ); + }); +}); diff --git a/packages/client/src/admin/Settings.tsx b/packages/client/src/admin/Settings.tsx index 3dd130f6b..9bde0c946 100644 --- a/packages/client/src/admin/Settings.tsx +++ b/packages/client/src/admin/Settings.tsx @@ -992,6 +992,13 @@ function SuperUserSettings() { const [updateFeatureFlags, updateFeatureFlagsState] = useUpdateFeatureFlagsMutation({ optimisticResponse: (input: UpdateFeatureFlagsMutationVariables) => { + const current = data?.project?.featureFlags; + const nextFlags = + input.flags && + typeof input.flags === "object" && + !Array.isArray(input.flags) + ? (input.flags as { [key: string]: boolean }) + : {}; return { __typename: "Mutation", updateFeatureFlags: { @@ -1001,7 +1008,12 @@ function SuperUserSettings() { id: data!.project!.id, featureFlags: { __typename: "FeatureFlags", - iNaturalistLayers: input.flags.iNaturalistLayers, + iNaturalistLayers: + nextFlags.iNaturalistLayers ?? + current?.iNaturalistLayers ?? + false, + dataTables: + nextFlags.dataTables ?? current?.dataTables ?? false, }, }, }, @@ -1235,6 +1247,30 @@ function SuperUserSettings() { title={t("Enable iNaturalist Layers")} description={t("Enable iNaturalist layers for this project.")} /> */} + { + updateFeatureFlags({ + variables: { + slug, + flags: { dataTables: enabled }, + }, + }); + }} + /> + } + title={t("Enable Data Tables")} + description={t( + "When enabled, project administrators can attach related data tables to layers and display them on the map. Alpha quality." + )} + /> + {updateFeatureFlagsState.error && ( +

+ {updateFeatureFlagsState.error.message} +

+ )} diff --git a/packages/client/src/admin/changelogs/ChangeLogListItem.tsx b/packages/client/src/admin/changelogs/ChangeLogListItem.tsx index eaa3e1849..b8bbf2dc1 100644 --- a/packages/client/src/admin/changelogs/ChangeLogListItem.tsx +++ b/packages/client/src/admin/changelogs/ChangeLogListItem.tsx @@ -10,11 +10,16 @@ export default function ChangeLogListItem({ last, itemTitle, missingProfileLabel, + dataTableActions, }: { changeLog: ChangeLogDetailsFragment; last?: boolean; itemTitle?: ReactNode; missingProfileLabel?: string; + dataTableActions?: { + tableOfContentsItemId: number; + activeTables: { id: number; version: number }[]; + }; }) { const FieldGroupListItem = FIELD_GROUP_LIST_ITEM_COMPONENTS[changeLog.fieldGroup] || @@ -25,6 +30,7 @@ export default function ChangeLogListItem({ last={last} itemTitle={itemTitle} missingProfileLabel={missingProfileLabel} + dataTableActions={dataTableActions} /> ); } diff --git a/packages/client/src/admin/changelogs/DataTablesChangeLogList.tsx b/packages/client/src/admin/changelogs/DataTablesChangeLogList.tsx new file mode 100644 index 000000000..938b1b331 --- /dev/null +++ b/packages/client/src/admin/changelogs/DataTablesChangeLogList.tsx @@ -0,0 +1,179 @@ +import { useLayoutEffect, useMemo, useRef, useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; +import { ChevronDownIcon } from "@heroicons/react/outline"; +import { + DataTableChangeLogQuery, + useDataTableChangeLogQuery, +} from "../../generated/graphql"; +import ChangeLogListItem from "./ChangeLogListItem"; +import { + DATA_TABLE_CHANGE_LOG_COLLAPSED_FIRST, + DATA_TABLE_CHANGE_LOG_EXPANDED_FIRST, + DATA_TABLE_CHANGE_LOG_PAGE_SIZE, +} from "./dataTableChangeLogRefetch"; +import { summary } from "./fieldGroups/FieldGroupListItemBase"; +import { tableNameFromSummary } from "./fieldGroups/dataTableSummary"; + +function findScrollParent(from: HTMLElement | null): HTMLElement | null { + if (!from) return null; + let el: HTMLElement | null = from.parentElement; + while (el) { + const { overflowY } = window.getComputedStyle(el); + if ( + overflowY === "auto" || + overflowY === "scroll" || + overflowY === "overlay" + ) { + if (el.scrollHeight > el.clientHeight) { + return el; + } + } + el = el.parentElement; + } + return null; +} + +export default function DataTablesChangeLogList({ + tableOfContentsItemId, +}: { + tableOfContentsItemId: number; +}) { + const { t } = useTranslation("admin:data"); + const [showAllHistory, setShowAllHistory] = useState(false); + const scrollRestoreRef = useRef<{ el: HTMLElement; top: number } | null>( + null, + ); + const lastTocRef = useRef< + NonNullable | undefined + >(undefined); + + const variables = useMemo( + () => ({ + id: tableOfContentsItemId, + first: showAllHistory + ? DATA_TABLE_CHANGE_LOG_EXPANDED_FIRST + : DATA_TABLE_CHANGE_LOG_COLLAPSED_FIRST, + }), + [tableOfContentsItemId, showAllHistory], + ); + + const { data, loading } = useDataTableChangeLogQuery({ + variables, + skip: !tableOfContentsItemId, + fetchPolicy: "cache-and-network", + nextFetchPolicy: "cache-first", + }); + + const tocQuery = data?.tableOfContentsItem; + if (tocQuery) { + lastTocRef.current = tocQuery; + } + + const toc = + tocQuery ?? + (showAllHistory && loading ? lastTocRef.current : undefined) ?? + lastTocRef.current; + + const fetchedLogs = toc?.dataTableChangeLogs + ? [...toc.dataTableChangeLogs] + : []; + // Collapsed query peeks one past the page size so we know whether more exist. + const changeLogs = showAllHistory + ? fetchedLogs + : fetchedLogs.slice(0, DATA_TABLE_CHANGE_LOG_PAGE_SIZE); + const activeTables = (toc?.overlayDataTables || []).map((table) => ({ + id: table.id, + version: table.version, + })); + const dataTableNameById = useMemo(() => { + const next = new Map(); + for (const table of toc?.overlayDataTables || []) { + if (table.name) { + next.set(table.id, table.name); + } + } + return next; + }, [toc?.overlayDataTables]); + + const canShowMore = + !showAllHistory && fetchedLogs.length > DATA_TABLE_CHANGE_LOG_PAGE_SIZE; + + const initialLoading = loading && !tocQuery && !showAllHistory; + + useLayoutEffect(() => { + const pending = scrollRestoreRef.current; + if (!pending || !showAllHistory) return; + pending.el.scrollTop = pending.top; + scrollRestoreRef.current = null; + }, [showAllHistory, changeLogs.length]); + + if ((initialLoading && !toc) || changeLogs.length === 0) { + return null; + } + + return ( +
+

+ History +

+
    + {changeLogs.map((changeLog, index) => ( + + ))} +
+ {canShowMore && ( +
+ +
+ )} +
+ ); +} + +function titleForDataTableChangeLog( + changeLog: NonNullable< + NonNullable< + DataTableChangeLogQuery["tableOfContentsItem"] + >["dataTableChangeLogs"] + >[number], + dataTableNameById: Map, + untitledLabel: string +) { + const tableName = + tableNameFromSummary(summary(changeLog.toSummary)) || + tableNameFromSummary(summary(changeLog.fromSummary)) || + dataTableNameById.get(changeLog.entityId) || + ""; + return tableName || untitledLabel; +} diff --git a/packages/client/src/admin/changelogs/LayerSettingsChangeLogList.tsx b/packages/client/src/admin/changelogs/LayerSettingsChangeLogList.tsx index 6e6c8956c..26b67c058 100644 --- a/packages/client/src/admin/changelogs/LayerSettingsChangeLogList.tsx +++ b/packages/client/src/admin/changelogs/LayerSettingsChangeLogList.tsx @@ -12,6 +12,7 @@ import { } from "./creationAnchorLogic"; import SourceCreationAnchorItem from "./SourceCreationAnchorItem"; import { + LAYER_SETTINGS_CHANGE_LOG_COLLAPSED_FIRST, LAYER_SETTINGS_CHANGE_LOG_EXPANDED_FIRST, LAYER_SETTINGS_CHANGE_LOG_PAGE_SIZE, } from "./layerSettingsChangeLogRefetch"; @@ -54,7 +55,7 @@ export default function LayerSettingsChangeLogList({ id: tableOfContentsItemId, first: showAllHistory ? LAYER_SETTINGS_CHANGE_LOG_EXPANDED_FIRST - : LAYER_SETTINGS_CHANGE_LOG_PAGE_SIZE, + : LAYER_SETTINGS_CHANGE_LOG_COLLAPSED_FIRST, }), [tableOfContentsItemId, showAllHistory] ); @@ -80,9 +81,12 @@ export default function LayerSettingsChangeLogList({ const relatedPublishChangeLogs = toc?.relatedPublishChangeLogs ? [...toc.relatedPublishChangeLogs] : []; - const changeLogs = [...directChangeLogs, ...relatedPublishChangeLogs] - .sort((a, b) => new Date(b.lastAt).getTime() - new Date(a.lastAt).getTime()) - .slice(0, variables.first); + const fetchedLogs = [...directChangeLogs, ...relatedPublishChangeLogs].sort( + (a, b) => new Date(b.lastAt).getTime() - new Date(a.lastAt).getTime() + ); + const changeLogs = showAllHistory + ? fetchedLogs + : fetchedLogs.slice(0, LAYER_SETTINGS_CHANGE_LOG_PAGE_SIZE); const rawCreatedAt = toc?.dataLayer?.dataSource?.createdAt; const authorProfile = toc?.dataLayer?.dataSource?.authorProfile ?? undefined; const dataLibraryTemplateId = toc?.dataLayer?.dataSource?.dataLibraryTemplateId; @@ -117,9 +121,7 @@ export default function LayerSettingsChangeLogList({ const canShowMore = !showAllHistory && - (directChangeLogs.length >= LAYER_SETTINGS_CHANGE_LOG_PAGE_SIZE || - relatedPublishChangeLogs.length >= LAYER_SETTINGS_CHANGE_LOG_PAGE_SIZE) && - itemCount > 0; + fetchedLogs.length > LAYER_SETTINGS_CHANGE_LOG_PAGE_SIZE; const initialLoading = loading && !tocQuery && !showAllHistory; diff --git a/packages/client/src/admin/changelogs/creationAnchorLogic.ts b/packages/client/src/admin/changelogs/creationAnchorLogic.ts index b7f5cd501..3d087bb56 100644 --- a/packages/client/src/admin/changelogs/creationAnchorLogic.ts +++ b/packages/client/src/admin/changelogs/creationAnchorLogic.ts @@ -6,13 +6,18 @@ import { /** Max difference between LayerUploaded.lastAt and dataSource.createdAt to treat upload as creation. */ export const CREATION_UPLOAD_MATCH_MS = 5 * 60 * 1000; +/** + * Whether the collapsed history fetch exhausted the list. Callers request + * `pageSize + 1` when collapsed, so a result of `pageSize` or fewer means + * there is nothing more to load. + */ export function hasLoadedFullHistory( changeLogsLength: number, pageSize: number, showAllHistory: boolean ): boolean { if (changeLogsLength === 0) return true; - if (changeLogsLength < pageSize) return true; + if (changeLogsLength <= pageSize) return true; return showAllHistory; } diff --git a/packages/client/src/admin/changelogs/dataTableChangeLogRefetch.ts b/packages/client/src/admin/changelogs/dataTableChangeLogRefetch.ts new file mode 100644 index 000000000..0666c8a37 --- /dev/null +++ b/packages/client/src/admin/changelogs/dataTableChangeLogRefetch.ts @@ -0,0 +1,51 @@ +import { RefetchQueriesInclude } from "@apollo/client"; +import { + DataTableChangeLogDocument, + GetLayerItemDocument, +} from "../../generated/graphql"; +import { + LAYER_SETTINGS_CHANGE_LOG_COLLAPSED_FIRST, + LAYER_SETTINGS_CHANGE_LOG_EXPANDED_FIRST, + LAYER_SETTINGS_CHANGE_LOG_PAGE_SIZE, +} from "./layerSettingsChangeLogRefetch"; + +export { + LAYER_SETTINGS_CHANGE_LOG_PAGE_SIZE as DATA_TABLE_CHANGE_LOG_PAGE_SIZE, + LAYER_SETTINGS_CHANGE_LOG_COLLAPSED_FIRST as DATA_TABLE_CHANGE_LOG_COLLAPSED_FIRST, + LAYER_SETTINGS_CHANGE_LOG_EXPANDED_FIRST as DATA_TABLE_CHANGE_LOG_EXPANDED_FIRST, +}; + +/** Refetches both paginated variants of {@link DataTableChangeLogDocument}. */ +export function dataTableChangeLogRefetchQueries( + tableOfContentsItemId: number, +): RefetchQueriesInclude { + return [ + { + query: DataTableChangeLogDocument, + variables: { + id: tableOfContentsItemId, + first: LAYER_SETTINGS_CHANGE_LOG_COLLAPSED_FIRST, + }, + }, + { + query: DataTableChangeLogDocument, + variables: { + id: tableOfContentsItemId, + first: LAYER_SETTINGS_CHANGE_LOG_EXPANDED_FIRST, + }, + }, + ] as unknown as RefetchQueriesInclude; +} + +/** Refetch data table history and the layer item table list after a mutation. */ +export function dataTableMutationRefetchQueries( + tableOfContentsItemId: number, +): RefetchQueriesInclude { + return [ + ...dataTableChangeLogRefetchQueries(tableOfContentsItemId), + { + query: GetLayerItemDocument, + variables: { id: tableOfContentsItemId }, + }, + ] as unknown as RefetchQueriesInclude; +} diff --git a/packages/client/src/admin/changelogs/fieldGroups/DataTableCreatedFieldGroupListItem.tsx b/packages/client/src/admin/changelogs/fieldGroups/DataTableCreatedFieldGroupListItem.tsx new file mode 100644 index 000000000..023e2b74c --- /dev/null +++ b/packages/client/src/admin/changelogs/fieldGroups/DataTableCreatedFieldGroupListItem.tsx @@ -0,0 +1,28 @@ +import { TableIcon } from "@heroicons/react/outline"; +import { Trans, useTranslation } from "react-i18next"; +import BaseFieldGroupListItem, { + ChangeValue, + FieldGroupListItemProps, + summary, +} from "./FieldGroupListItemBase"; +import { tableLabel } from "./dataTableSummary"; + +export default function DataTableCreatedFieldGroupListItem( + props: FieldGroupListItemProps, +) { + const to = summary(props.changeLog.toSummary); + const { t } = useTranslation("admin:data"); + const label = tableLabel(to, t("Untitled table")); + + return ( + } + iconClassName="bg-green-50 text-green-500" + > + + uploaded data table {label} + + + ); +} diff --git a/packages/client/src/admin/changelogs/fieldGroups/DataTableDeletedFieldGroupListItem.tsx b/packages/client/src/admin/changelogs/fieldGroups/DataTableDeletedFieldGroupListItem.tsx new file mode 100644 index 000000000..e8f1d4058 --- /dev/null +++ b/packages/client/src/admin/changelogs/fieldGroups/DataTableDeletedFieldGroupListItem.tsx @@ -0,0 +1,28 @@ +import { TrashIcon } from "@heroicons/react/outline"; +import { Trans, useTranslation } from "react-i18next"; +import BaseFieldGroupListItem, { + ChangeValue, + FieldGroupListItemProps, + summary, +} from "./FieldGroupListItemBase"; +import { tableLabel } from "./dataTableSummary"; + +export default function DataTableDeletedFieldGroupListItem( + props: FieldGroupListItemProps, +) { + const from = summary(props.changeLog.fromSummary); + const { t } = useTranslation("admin:data"); + + return ( + } + iconClassName="bg-red-50 text-red-500" + > + + deleted data table{" "} + {tableLabel(from, t("Untitled table"))} + + + ); +} diff --git a/packages/client/src/admin/changelogs/fieldGroups/DataTableRenamedFieldGroupListItem.tsx b/packages/client/src/admin/changelogs/fieldGroups/DataTableRenamedFieldGroupListItem.tsx new file mode 100644 index 000000000..27dd2983e --- /dev/null +++ b/packages/client/src/admin/changelogs/fieldGroups/DataTableRenamedFieldGroupListItem.tsx @@ -0,0 +1,38 @@ +import { PencilIcon } from "@heroicons/react/outline"; +import { Trans, useTranslation } from "react-i18next"; +import BaseFieldGroupListItem, { + ChangeValue, + FieldGroupListItemProps, + summary, + valueText, +} from "./FieldGroupListItemBase"; +import { tableVersionFromSummary } from "./dataTableSummary"; + +export default function DataTableRenamedFieldGroupListItem( + props: FieldGroupListItemProps, +) { + const from = summary(props.changeLog.fromSummary); + const to = summary(props.changeLog.toSummary); + const { t } = useTranslation("admin:data"); + const version = tableVersionFromSummary(to); + + return ( + } + iconClassName="bg-gray-50 text-gray-500" + > + + renamed data table from{" "} + + {valueText(from.name, t("Untitled table"))} + {" "} + {" -> "} + {valueText(to.name, t("Untitled table"))} + + {version != null ? ( + <> {t("(v{{version}})", { version })} + ) : null} + + ); +} diff --git a/packages/client/src/admin/changelogs/fieldGroups/DataTableReplacedFieldGroupListItem.tsx b/packages/client/src/admin/changelogs/fieldGroups/DataTableReplacedFieldGroupListItem.tsx new file mode 100644 index 000000000..66259ee1d --- /dev/null +++ b/packages/client/src/admin/changelogs/fieldGroups/DataTableReplacedFieldGroupListItem.tsx @@ -0,0 +1,89 @@ +import { RefreshIcon } from "@heroicons/react/outline"; +import { Trans, useTranslation } from "react-i18next"; +import { useRollbackOverlayDataTableVersionMutation } from "../../../generated/graphql"; +import BaseFieldGroupListItem, { + ChangeValue, + FieldGroupListItemProps, + summary, +} from "./FieldGroupListItemBase"; +import { dataTableMutationRefetchQueries } from "../dataTableChangeLogRefetch"; +import DataTableReplacementVersionMenu from "./DataTableReplacementVersionMenu"; +import { + parquetUrlFromSummary, + tableLabel, + tableNameFromSummary, + tableVersionFromSummary, +} from "./dataTableSummary"; + +export default function DataTableReplacedFieldGroupListItem( + props: FieldGroupListItemProps, +) { + const from = summary(props.changeLog.fromSummary); + const to = summary(props.changeLog.toSummary); + const { t } = useTranslation("admin:data"); + const name = + tableNameFromSummary(to) || + tableNameFromSummary(from) || + t("Untitled table"); + const fromVersion = tableVersionFromSummary(from); + const toVersion = tableVersionFromSummary(to); + const downloadUrl = parquetUrlFromSummary(from); + const tableId = props.changeLog.entityId; + const actions = props.dataTableActions; + const activeTable = actions?.activeTables.find((table) => table.id === tableId); + const canRollback = + actions != null && + activeTable != null && + toVersion != null && + activeTable.version === toVersion && + toVersion > 1; + const refetchQueries = actions + ? dataTableMutationRefetchQueries(actions.tableOfContentsItemId) + : undefined; + const [rollbackTable, { loading: rollbackLoading }] = + useRollbackOverlayDataTableVersionMutation({ + refetchQueries, + }); + const versionLabel = + fromVersion != null && toVersion != null + ? t("v{{fromVersion}} → v{{toVersion}}", { fromVersion, toVersion }) + : null; + const hasVersionMenu = + versionLabel != null && fromVersion != null && (downloadUrl || canRollback); + const versionMenu = hasVersionMenu ? ( + { + void rollbackTable({ variables: { id: tableId } }); + }} + /> + ) : null; + + return ( + } + iconClassName="bg-indigo-50 text-indigo-500" + > + {fromVersion != null && toVersion != null ? ( + <> + replaced data table{" "} + {name}{" "} + {versionMenu ?? ( + {versionLabel} + )} + + ) : ( + + replaced data table{" "} + {tableLabel(to, t("Untitled table"))} + + )} + + ); +} diff --git a/packages/client/src/admin/changelogs/fieldGroups/DataTableReplacementVersionMenu.tsx b/packages/client/src/admin/changelogs/fieldGroups/DataTableReplacementVersionMenu.tsx new file mode 100644 index 000000000..ea029cc4e --- /dev/null +++ b/packages/client/src/admin/changelogs/fieldGroups/DataTableReplacementVersionMenu.tsx @@ -0,0 +1,140 @@ +import clsx from "clsx"; +import { DownloadIcon, ReplyIcon } from "@heroicons/react/outline"; +import * as Popover from "@radix-ui/react-popover"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { useTranslation } from "react-i18next"; + +const CLOSE_DELAY_MS = 120; + +function downloadWithFilename(url: string, filename: string) { + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = filename; + anchor.rel = "noopener noreferrer"; + anchor.target = "_blank"; + document.body.appendChild(anchor); + anchor.click(); + document.body.removeChild(anchor); +} + +export default function DataTableReplacementVersionMenu({ + versionLabel, + fromVersion, + tableName, + downloadUrl, + canRollback, + onRollback, + rollbackLoading, +}: { + versionLabel: string; + fromVersion: number; + tableName: string; + downloadUrl?: string; + canRollback: boolean; + onRollback: () => void; + rollbackLoading: boolean; +}) { + const { t } = useTranslation("admin:data"); + const [open, setOpen] = useState(false); + const closeTimerRef = useRef(); + + const clearCloseTimer = useCallback(() => { + if (closeTimerRef.current != null) { + window.clearTimeout(closeTimerRef.current); + closeTimerRef.current = undefined; + } + }, []); + + const scheduleClose = useCallback(() => { + clearCloseTimer(); + closeTimerRef.current = window.setTimeout(() => { + setOpen(false); + }, CLOSE_DELAY_MS); + }, [clearCloseTimer]); + + useEffect(() => clearCloseTimer, [clearCloseTimer]); + + const downloadFilename = + // eslint-disable-next-line i18next/no-literal-string -- download filename + `${tableName}_v${fromVersion}.parquet`; + + const triggerClassName = clsx( + "inline-flex max-w-full cursor-pointer items-center gap-1 align-baseline rounded px-1 py-0.5 text-sm font-medium leading-5 text-blue-600 underline decoration-blue-400 decoration-dotted underline-offset-4 transition-colors hover:bg-blue-50 hover:text-blue-700 focus:outline-none focus-visible:ring-2 focus-visible:ring-primary-500", + open && "bg-blue-50 text-blue-700", + ); + + return ( + + + + + + event.preventDefault()} + onMouseEnter={clearCloseTimer} + onMouseLeave={scheduleClose} + > +
+ {t("Version {{version}}", { version: fromVersion })} +
+
+ {downloadUrl ? ( + + ) : null} + {canRollback ? ( + + ) : null} + + + + + ); +} diff --git a/packages/client/src/admin/changelogs/fieldGroups/DataTableRollbackFieldGroupListItem.tsx b/packages/client/src/admin/changelogs/fieldGroups/DataTableRollbackFieldGroupListItem.tsx new file mode 100644 index 000000000..663bd9f1a --- /dev/null +++ b/packages/client/src/admin/changelogs/fieldGroups/DataTableRollbackFieldGroupListItem.tsx @@ -0,0 +1,33 @@ +import { ReplyIcon } from "@heroicons/react/outline"; +import { Trans, useTranslation } from "react-i18next"; +import BaseFieldGroupListItem, { + ChangeValue, + FieldGroupListItemProps, + summary, +} from "./FieldGroupListItemBase"; +import { removedVersionFromSummary, tableLabel } from "./dataTableSummary"; + +export default function DataTableRollbackFieldGroupListItem( + props: FieldGroupListItemProps, +) { + const from = summary(props.changeLog.fromSummary); + const to = summary(props.changeLog.toSummary); + const { t } = useTranslation("admin:data"); + const removedVersion = removedVersionFromSummary(from); + const restored = tableLabel(to, t("Untitled table")); + + return ( + } + iconClassName="bg-amber-50 text-amber-600" + > + + rolled back to {restored} + + {removedVersion != null ? ( + <> {t("(removed v{{version}})", { version: removedVersion })} + ) : null} + + ); +} diff --git a/packages/client/src/admin/changelogs/fieldGroups/DataTableVisualizationSettingsUpdatedFieldGroupListItem.tsx b/packages/client/src/admin/changelogs/fieldGroups/DataTableVisualizationSettingsUpdatedFieldGroupListItem.tsx new file mode 100644 index 000000000..d703770c7 --- /dev/null +++ b/packages/client/src/admin/changelogs/fieldGroups/DataTableVisualizationSettingsUpdatedFieldGroupListItem.tsx @@ -0,0 +1,203 @@ +import { MixerHorizontalIcon } from "@radix-ui/react-icons"; +import { Trans, useTranslation } from "react-i18next"; +import BaseFieldGroupListItem, { + ChangeValue, + FieldGroupListItemProps, + Summary, + summary, + valueText, +} from "./FieldGroupListItemBase"; +import { tableLabel, tableNameFromSummary } from "./dataTableSummary"; + +function stringList(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + return value + .map((entry) => valueText(entry).trim()) + .filter(Boolean); +} + +function listsEqual(a: string[], b: string[]) { + if (a.length !== b.length) { + return false; + } + return a.every((entry, index) => entry === b[index]); +} + +function SettingsDetails({ + from, + to, + columnsLabel, + calculationsLabel, + requiredFiltersLabel, + allColumnsLabel, + allCalculationsLabel, + noneLabel, +}: { + from: Summary; + to: Summary; + columnsLabel: string; + calculationsLabel: string; + requiredFiltersLabel: string; + allColumnsLabel: string; + allCalculationsLabel: string; + noneLabel: string; +}) { + const rows = [ + { + label: columnsLabel, + before: formatColumnList(stringList(from.visualizationColumns), allColumnsLabel), + after: formatColumnList(stringList(to.visualizationColumns), allColumnsLabel), + }, + { + label: calculationsLabel, + before: formatColumnList(stringList(from.visualizationOps), allCalculationsLabel), + after: formatColumnList(stringList(to.visualizationOps), allCalculationsLabel), + }, + { + label: requiredFiltersLabel, + before: formatColumnList(stringList(from.requiredFilterColumns), noneLabel), + after: formatColumnList(stringList(to.requiredFilterColumns), noneLabel), + }, + ]; + + return ( +
+
+

+ Map display settings +

+
+
+ {rows.map((row) => ( +
+
+ {row.label} +
+
+ + {row.before} + + {/* eslint-disable-next-line i18next/no-literal-string */} + + {"->"} + + + {row.after} + +
+
+ ))} +
+
+ ); +} + +function formatColumnList(values: string[], emptyLabel: string) { + return values.length ? values.join(", ") : emptyLabel; +} + +export default function DataTableVisualizationSettingsUpdatedFieldGroupListItem( + props: FieldGroupListItemProps +) { + const from = summary(props.changeLog.fromSummary); + const to = summary(props.changeLog.toSummary); + const { t } = useTranslation("admin:data"); + + const fromColumns = stringList(from.visualizationColumns); + const toColumns = stringList(to.visualizationColumns); + const fromOps = stringList(from.visualizationOps); + const toOps = stringList(to.visualizationOps); + const fromRequired = stringList(from.requiredFilterColumns); + const toRequired = stringList(to.requiredFilterColumns); + + const columnsChanged = !listsEqual(fromColumns, toColumns); + const opsChanged = !listsEqual(fromOps, toOps); + const requiredChanged = !listsEqual(fromRequired, toRequired); + const changedCount = + Number(columnsChanged) + Number(opsChanged) + Number(requiredChanged); + + const namedSummary = tableNameFromSummary(to) + ? to + : tableNameFromSummary(from) + ? from + : null; + const tableLabelText = namedSummary + ? tableLabel(namedSummary, t("Untitled table")) + : ""; + + const allColumnsLabel = t("all columns"); + const allCalculationsLabel = t("all calculations"); + const noneLabel = t("none"); + const details = ( + + ); + + // When the publish UI already shows overlay · table as itemTitle, skip + // repeating the table name in the summary line. + const showTableInSummary = Boolean(tableLabelText) && !props.itemTitle; + + return ( + } + iconClassName="bg-blue-50 text-blue-500" + > + {changedCount === 1 && columnsChanged ? ( + + updated map data columns from{" "} + + {formatColumnList(fromColumns, allColumnsLabel)} + {" "} + {" -> "} + + {formatColumnList(toColumns, allColumnsLabel)} + + + ) : changedCount === 1 && opsChanged ? ( + + updated map calculations from{" "} + + {formatColumnList(fromOps, allCalculationsLabel)} + {" "} + {" -> "} + + {formatColumnList(toOps, allCalculationsLabel)} + + + ) : changedCount === 1 && requiredChanged ? ( + + updated required filters from{" "} + + {formatColumnList(fromRequired, noneLabel)} + {" "} + {" -> "} + + {formatColumnList(toRequired, noneLabel)} + + + ) : showTableInSummary ? ( + + updated{" "} + map display settings for{" "} + {tableLabelText} + + ) : ( + + updated{" "} + map display settings + + )} + + ); +} diff --git a/packages/client/src/admin/changelogs/fieldGroups/FieldGroupListItemBase.css b/packages/client/src/admin/changelogs/fieldGroups/FieldGroupListItemBase.css index 7e329b6c6..70f0c5bd0 100644 --- a/packages/client/src/admin/changelogs/fieldGroups/FieldGroupListItemBase.css +++ b/packages/client/src/admin/changelogs/fieldGroups/FieldGroupListItemBase.css @@ -25,3 +25,7 @@ 0 20px 25px -5px rgba(17, 24, 39, 0.1), 0 8px 10px -6px rgba(17, 24, 39, 0.1); } + +.change-log-data-table-version-menu { + pointer-events: auto; +} diff --git a/packages/client/src/admin/changelogs/fieldGroups/FieldGroupListItemBase.tsx b/packages/client/src/admin/changelogs/fieldGroups/FieldGroupListItemBase.tsx index 4896aad74..c453508b0 100644 --- a/packages/client/src/admin/changelogs/fieldGroups/FieldGroupListItemBase.tsx +++ b/packages/client/src/admin/changelogs/fieldGroups/FieldGroupListItemBase.tsx @@ -16,6 +16,11 @@ export interface FieldGroupListItemProps { itemTitle?: ReactNode; /** When the change log has no editor profile, show this instead of "Unknown user". */ missingProfileLabel?: string; + /** When set, replacement entries may offer rollback in a details tooltip. */ + dataTableActions?: { + tableOfContentsItemId: number; + activeTables: { id: number; version: number }[]; + }; } export type Summary = Record; diff --git a/packages/client/src/admin/changelogs/fieldGroups/dataTableSummary.ts b/packages/client/src/admin/changelogs/fieldGroups/dataTableSummary.ts new file mode 100644 index 000000000..10fcd0279 --- /dev/null +++ b/packages/client/src/admin/changelogs/fieldGroups/dataTableSummary.ts @@ -0,0 +1,161 @@ +import { summary, Summary, valueText } from "./FieldGroupListItemBase"; + +export function tocItemIdFromMeta(meta: unknown): number | undefined { + if (!meta || typeof meta !== "object" || Array.isArray(meta)) { + return undefined; + } + const value = (meta as Record).table_of_contents_item_id; + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value === "string") { + const parsed = parseInt(value, 10); + return Number.isFinite(parsed) ? parsed : undefined; + } + return undefined; +} + +export function tableNameFromSummary(s: Summary): string { + return valueText(s.name); +} + +export function tableIdFromSummary(s: Summary): number | undefined { + const value = s.id; + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value === "string") { + const parsed = parseInt(value, 10); + return Number.isFinite(parsed) ? parsed : undefined; + } + return undefined; +} + +export function parquetUrlFromSummary(s: Summary): string | undefined { + const url = s.parquet_url; + if (typeof url === "string" && url.length > 0) { + return url; + } + return undefined; +} + +export function tableVersionFromSummary(s: Summary): number | undefined { + const version = s.version; + if (typeof version === "number" && Number.isFinite(version)) { + return version; + } + if (typeof version === "string") { + const parsed = parseInt(version, 10); + return Number.isFinite(parsed) ? parsed : undefined; + } + return undefined; +} + +export function tableLabel( + s: Summary, + fallback: string, + options?: { includeVersion?: boolean }, +): string { + const name = tableNameFromSummary(s) || fallback; + const version = tableVersionFromSummary(s); + if (options?.includeVersion === false || version == null) { + return name; + } + // eslint-disable-next-line i18next/no-literal-string -- version suffix formatted for display inside Trans/ChangeValue + return `${name} (v${version})`; +} + +export function tableLabelFromChangeLog( + fromSummary: unknown, + toSummary: unknown, + prefer: "from" | "to", + fallback: string, +): string { + const from = summary(fromSummary); + const to = summary(toSummary); + const s = prefer === "from" ? from : to; + return tableLabel(s, fallback); +} + +export function removedVersionFromSummary(s: Summary): number | undefined { + const removed = s.removed_version; + if (typeof removed === "number" && Number.isFinite(removed)) { + return removed; + } + if (typeof removed === "string") { + const parsed = parseInt(removed, 10); + return Number.isFinite(parsed) ? parsed : undefined; + } + return undefined; +} + +export function dataTableEventDescription( + fieldGroup: string, + fromSummary: unknown, + toSummary: unknown, + t: (key: string, opts?: Record) => string, +): string { + const from = summary(fromSummary); + const to = summary(toSummary); + const fallback = t("Untitled table"); + + switch (fieldGroup) { + case "DATA_TABLE_CREATED": + return t("Uploaded {{table}}", { + table: tableLabel(to, fallback), + }); + case "DATA_TABLE_REPLACED": { + const fromVersion = tableVersionFromSummary(from); + const toVersion = tableVersionFromSummary(to); + const name = tableNameFromSummary(to) || tableNameFromSummary(from) || fallback; + if (fromVersion != null && toVersion != null) { + return t("Replaced {{name}} v{{fromVersion}} → v{{toVersion}}", { + name, + fromVersion, + toVersion, + }); + } + return t("Replaced {{table}}", { table: tableLabel(to, fallback) }); + } + case "DATA_TABLE_RENAMED": { + const version = tableVersionFromSummary(to); + const fromName = tableNameFromSummary(from) || fallback; + const toName = tableNameFromSummary(to) || fallback; + if (version != null) { + return t("Renamed {{fromName}} → {{toName}} (v{{version}})", { + fromName, + toName, + version, + }); + } + return t("Renamed {{fromName}} → {{toName}}", { fromName, toName }); + } + case "DATA_TABLE_DELETED": + return t("Deleted {{table}}", { + table: tableLabel(from, fallback), + }); + case "DATA_TABLE_ROLLBACK": { + const removedVersion = removedVersionFromSummary(from); + const restored = tableLabel(to, fallback); + if (removedVersion != null) { + return t("Rolled back to {{restored}} (removed v{{removedVersion}})", { + restored, + removedVersion, + }); + } + return t("Rolled back to {{restored}}", { restored }); + } + case "DATA_TABLE_VISUALIZATION_SETTINGS_UPDATED": { + const name = + tableNameFromSummary(to) || tableNameFromSummary(from) || ""; + if (name) { + return t("Updated map display settings for {{table}}", { + table: tableLabel(to.name ? to : from, fallback), + }); + } + return t("Updated map display settings"); + } + default: + return t("Updated data table"); + } +} diff --git a/packages/client/src/admin/changelogs/fieldGroups/index.ts b/packages/client/src/admin/changelogs/fieldGroups/index.ts index d53732ebc..0366a099b 100644 --- a/packages/client/src/admin/changelogs/fieldGroups/index.ts +++ b/packages/client/src/admin/changelogs/fieldGroups/index.ts @@ -18,6 +18,12 @@ import LayerUploadedFieldGroupListItem from "./LayerUploadedFieldGroupListItem"; import LayersPublishedFieldGroupListItem from "./LayersPublishedFieldGroupListItem"; import LayersZOrderChangeFieldGroupListItem from "./LayersZOrderChangeFieldGroupListItem"; import ResolvableLayerCommentFieldGroupListItem from "./ResolvableLayerCommentFieldGroupListItem"; +import DataTableCreatedFieldGroupListItem from "./DataTableCreatedFieldGroupListItem"; +import DataTableDeletedFieldGroupListItem from "./DataTableDeletedFieldGroupListItem"; +import DataTableRenamedFieldGroupListItem from "./DataTableRenamedFieldGroupListItem"; +import DataTableReplacedFieldGroupListItem from "./DataTableReplacedFieldGroupListItem"; +import DataTableRollbackFieldGroupListItem from "./DataTableRollbackFieldGroupListItem"; +import DataTableVisualizationSettingsUpdatedFieldGroupListItem from "./DataTableVisualizationSettingsUpdatedFieldGroupListItem"; export { GenericFieldGroupListItem }; @@ -50,4 +56,11 @@ export const FIELD_GROUP_LIST_ITEM_COMPONENTS: Partial< ResolvableLayerCommentFieldGroupListItem, [ChangeLogFieldGroup.ResolvableLayerCommentsReopened]: ResolvableLayerCommentFieldGroupListItem, + [ChangeLogFieldGroup.DataTableCreated]: DataTableCreatedFieldGroupListItem, + [ChangeLogFieldGroup.DataTableDeleted]: DataTableDeletedFieldGroupListItem, + [ChangeLogFieldGroup.DataTableRenamed]: DataTableRenamedFieldGroupListItem, + [ChangeLogFieldGroup.DataTableReplaced]: DataTableReplacedFieldGroupListItem, + [ChangeLogFieldGroup.DataTableRollback]: DataTableRollbackFieldGroupListItem, + [ChangeLogFieldGroup.DataTableVisualizationSettingsUpdated]: + DataTableVisualizationSettingsUpdatedFieldGroupListItem, }; diff --git a/packages/client/src/admin/changelogs/layerSettingsChangeLogRefetch.ts b/packages/client/src/admin/changelogs/layerSettingsChangeLogRefetch.ts index d261d5af8..fd295f3cf 100644 --- a/packages/client/src/admin/changelogs/layerSettingsChangeLogRefetch.ts +++ b/packages/client/src/admin/changelogs/layerSettingsChangeLogRefetch.ts @@ -1,8 +1,14 @@ -import type { RefetchQueriesInclude } from "@apollo/client"; +import { RefetchQueriesInclude } from "@apollo/client"; import { LayerSettingsChangeLogDocument } from "../../generated/graphql"; -/** Recent slice for layer settings history query (matches refetch mutation cache keys). */ +/** How many history items to show before "View full history". */ export const LAYER_SETTINGS_CHANGE_LOG_PAGE_SIZE = 5; +/** + * Collapsed fetch size: one more than the visible page so we can tell + * "exactly PAGE_SIZE items" from "there are more". + */ +export const LAYER_SETTINGS_CHANGE_LOG_COLLAPSED_FIRST = + LAYER_SETTINGS_CHANGE_LOG_PAGE_SIZE + 1; /** Cap when user expands history (newest-first; unlikely to exceed in practice). */ export const LAYER_SETTINGS_CHANGE_LOG_EXPANDED_FIRST = 500; @@ -15,7 +21,7 @@ export function layerSettingsChangeLogRefetchQueries( query: LayerSettingsChangeLogDocument, variables: { id: tableOfContentsItemId, - first: LAYER_SETTINGS_CHANGE_LOG_PAGE_SIZE, + first: LAYER_SETTINGS_CHANGE_LOG_COLLAPSED_FIRST, }, }, { diff --git a/packages/client/src/admin/data/DataSettings.tsx b/packages/client/src/admin/data/DataSettings.tsx index 9e44787c3..1a16343f7 100644 --- a/packages/client/src/admin/data/DataSettings.tsx +++ b/packages/client/src/admin/data/DataSettings.tsx @@ -16,12 +16,12 @@ import useMapData from "../../dataLayers/useMapData"; import DataUploadDropzone from "../uploads/ProjectBackgroundJobContext"; import { DndProvider } from "react-dnd"; import { HTML5Backend } from "react-dnd-html5-backend"; -import Legend from "../../dataLayers/Legend"; +import Legend, { LegendFocusRequest } from "../../dataLayers/Legend"; import useCommonLegendProps from "../../dataLayers/useCommonLegendProps"; import { TableOfContentsMetadataModalProvider } from "../../dataLayers/TableOfContentsMetadataModal"; import { LayerEditingContextProvider } from "./LayerEditingContext"; import { DataDownloadModalProvider } from "../../dataLayers/DataDownloadModal"; -import { useContext } from "react"; +import { useCallback, useContext, useState } from "react"; /** * Reads legend state from context so it stays in sync with the manager. @@ -30,6 +30,8 @@ function DataSettingsLegend() { const { manager } = useContext(MapManagerContext); const overlayState = useContext(MapOverlayContext); const legendsState = useContext(LegendsContext); + const [legendFocusRequest, setLegendFocusRequest] = + useState(null); const legendProps = useCommonLegendProps( { layerStatesByTocStaticId: overlayState.layerStatesByTocStaticId, @@ -39,6 +41,13 @@ function DataSettingsLegend() { {} ); + const onDataTableActivated = useCallback((layerId: string) => { + setLegendFocusRequest({ layerId, requestId: Date.now() }); + }, []); + const onLegendFocusComplete = useCallback(() => { + setLegendFocusRequest(null); + }, []); + if (legendProps.items.length === 0) return null; return ( ); } diff --git a/packages/client/src/admin/data/EnableDataTables.tsx b/packages/client/src/admin/data/EnableDataTables.tsx new file mode 100644 index 000000000..5c3e16e5d --- /dev/null +++ b/packages/client/src/admin/data/EnableDataTables.tsx @@ -0,0 +1,113 @@ +import { useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; +import { + FullAdminOverlayFragment, + useUpdateDataTablesSettingsMutation, +} from "../../generated/graphql"; +import { GeostatsLayer } from "@seasketch/geostats-types"; +import Switch from "../../components/Switch"; +import { layerSettingsChangeLogRefetchQueries } from "../changelogs/layerSettingsChangeLogRefetch"; +import DataTableJoinColumnModal from "./overlayDataTables/DataTableJoinColumnModal"; + +export default function EnableDataTables({ + className, + item, + geostatsLayer, + aiBestIdColumnHint, +}: { + className?: string; + item: Pick< + FullAdminOverlayFragment, + "id" | "enableDataTables" | "dataTableJoinColumn" + >; + geostatsLayer: GeostatsLayer | undefined; + aiBestIdColumnHint?: string | null; +}) { + const { t } = useTranslation("admin:data"); + const [modalOpen, setModalOpen] = useState(false); + const [updateSettings, updateSettingsState] = + useUpdateDataTablesSettingsMutation(); + + const saveSettings = async ( + enableDataTables: boolean, + dataTableJoinColumn?: string | null + ) => { + await updateSettings({ + variables: { + id: item.id, + enableDataTables, + dataTableJoinColumn, + }, + refetchQueries: [...layerSettingsChangeLogRefetchQueries(item.id)], + }); + }; + + const onToggle = () => { + if (item.enableDataTables) { + saveSettings(false, item.dataTableJoinColumn); + return; + } + setModalOpen(true); + }; + + const openJoinColumnModal = () => setModalOpen(true); + + return ( + <> +
+
+
+
+
+ Enable Data Tables +
+

+ + Attach multidimensional data about features in this layer + using a shared ID column. Great for monitoring data. For + example, add annual species observations for survey sites. + +

+
+ +
+ + {item.enableDataTables && item.dataTableJoinColumn ? ( +
+ + {t("Overlay join column")} + + +
+ ) : null} +
+
+ + {modalOpen ? ( + setModalOpen(false)} + geostatsLayer={geostatsLayer} + aiBestIdColumnHint={aiBestIdColumnHint} + initialJoinColumn={item.dataTableJoinColumn} + saving={updateSettingsState.loading} + onSave={async (joinColumn) => { + await saveSettings(true, joinColumn); + setModalOpen(false); + }} + /> + ) : null} + + ); +} diff --git a/packages/client/src/admin/data/LayerTableOfContentsItemEditor.tsx b/packages/client/src/admin/data/LayerTableOfContentsItemEditor.tsx index d97ae3882..b6532711a 100644 --- a/packages/client/src/admin/data/LayerTableOfContentsItemEditor.tsx +++ b/packages/client/src/admin/data/LayerTableOfContentsItemEditor.tsx @@ -12,9 +12,12 @@ import InteractivitySettings from "./InteractivitySettings"; import { gql, useApolloClient } from "@apollo/client"; import useDebounce from "../../useDebounce"; import GLStyleEditor from "./GLStyleEditor/GLStyleEditor"; -import Tabs, { NonLinkTabItem } from "../../components/Tabs"; import { MapManagerContext } from "../../dataLayers/MapContextManager"; -import { CaretRightIcon, ExclamationTriangleIcon } from "@radix-ui/react-icons"; +import LayerEditorTabs from "./TableOfContentsItemEditor/LayerEditorTabs"; +import { + CaretRightIcon, + ExclamationTriangleIcon, +} from "@radix-ui/react-icons"; import Skeleton from "../../components/Skeleton"; import FolderIcon from "../../components/FolderIcon"; import OverlayMetataEditor from "./OverlayMetadataEditor"; @@ -22,7 +25,9 @@ import useDialog from "../../components/useDialog"; import LayerSettings from "./TableOfContentsItemEditor/LayerSettings"; import { XIcon } from "@heroicons/react/outline"; import LayerVersioning from "./TableOfContentsItemEditor/LayerVersioning"; +import DataTablesEditor from "./overlayDataTables/DataTablesEditor"; import { layerSettingsChangeLogRefetchQueries } from "../changelogs/layerSettingsChangeLogRefetch"; +import useCurrentProjectMetadata from "../../useCurrentProjectMetadata"; interface LayerTableOfContentsItemEditorProps { itemId: number; @@ -102,6 +107,8 @@ export default function LayerTableOfContentsItemEditor( }); const item = data?.tableOfContentsItem; + const [selectedTab, setSelectedTab] = useState("settings"); + const activeTab = selectedTab; useEffect(() => { if (item?.stableId && item.dataLayer && manager) { @@ -133,8 +140,6 @@ export default function LayerTableOfContentsItemEditor( // eslint-disable-next-line react-hooks/exhaustive-deps }, [debouncedStyle]); - const [selectedTab, setSelectedTab] = useState("settings"); - const geostats = (source?.geostats?.layers || []).find( (l: { layer: string }) => { if (!Boolean(layer?.sourceLayer)) { @@ -145,35 +150,56 @@ export default function LayerTableOfContentsItemEditor( } ); - const tabs: NonLinkTabItem[] = useMemo(() => { - return [ + const { data: projectMeta } = useCurrentProjectMetadata(); + const dataTablesFeatureEnabled = Boolean( + projectMeta?.project?.featureFlags?.dataTables + ); + const supportsDataTables = Boolean(geostats) && dataTablesFeatureEnabled; + + const tabs = useMemo(() => { + const items: { + id: string; + name: string; + title?: string; + current: boolean; + }[] = [ { - name: "Settings", + name: t("Settings"), id: "settings", - current: selectedTab === "settings", + current: activeTab === "settings", }, { - name: "Data Source", + name: t("Source"), + title: t("Data Source"), id: "versions", - current: selectedTab === "versions", + current: activeTab === "versions", }, { - name: "Metadata", - id: "metadata", - current: selectedTab === "metadata", - }, - { - name: "Style", + name: t("Style"), id: "style", - current: selectedTab === "style", + current: activeTab === "style", }, { - name: "Interactivity", + name: t("Interact"), id: "interactivity", - current: selectedTab === "interactivity", + current: activeTab === "interactivity", + }, + { + name: t("Metadata"), + id: "metadata", + current: activeTab === "metadata", }, ]; - }, [selectedTab]); + if (supportsDataTables) { + items.push({ + name: t("Tables"), + title: t("Data Tables"), + id: "dataTables", + current: activeTab === "dataTables", + }); + } + return items; + }, [activeTab, supportsDataTables, t]); const isArcGISCustomSource = source?.type === DataSourceTypes.ArcgisDynamicMapserver || @@ -185,26 +211,29 @@ export default function LayerTableOfContentsItemEditor( className="bg-white z-30 absolute bottom-0 w-128 flex flex-col" style={{ height: "calc(100vh)" }} > -
-

- {error ? t("Error") : item?.title || props.title || t("Loading")} -

- -
- {item?.containedBy && item.containedBy.length > 0 && ( -
- +
+
+

+ {error ? t("Error") : item?.title || props.title || t("Loading")} +

+
- )} -
- setSelectedTab(id)} /> + {item?.containedBy && item.containedBy.length > 0 && ( +
+ +
+ )} +
{error && (
@@ -213,12 +242,15 @@ export default function LayerTableOfContentsItemEditor(
)} {!item && !error && } - {item && selectedTab === "settings" && } - {item && selectedTab === "versions" && } + {item && activeTab === "settings" && } + {item && activeTab === "dataTables" && supportsDataTables && ( + + )} + {item && activeTab === "versions" && } {item && (
)} {item && - selectedTab === "interactivity" && + activeTab === "interactivity" && source && source.type === DataSourceTypes.Inaturalist && (
@@ -246,7 +278,7 @@ export default function LayerTableOfContentsItemEditor(
)} {item && - selectedTab === "interactivity" && + activeTab === "interactivity" && source && source.type !== DataSourceTypes.Inaturalist && (
@@ -305,7 +337,7 @@ export default function LayerTableOfContentsItemEditor(
)} - {item && selectedTab === "style" && ( + {item && activeTab === "style" && (
{source && source.type === DataSourceTypes.Inaturalist && (
diff --git a/packages/client/src/admin/data/PublishSummarizedChangesPanel.tsx b/packages/client/src/admin/data/PublishSummarizedChangesPanel.tsx index a30f5c3b9..76ec08943 100644 --- a/packages/client/src/admin/data/PublishSummarizedChangesPanel.tsx +++ b/packages/client/src/admin/data/PublishSummarizedChangesPanel.tsx @@ -114,6 +114,8 @@ function badgeLabel(key: PublishBadgeKey, t: (s: string) => string): string { return t("moved"); case "source": return t("source"); + case "dataTables": + return t("data tables"); case "folderBehavior": return t("folder behavior"); case "comments": diff --git a/packages/client/src/admin/data/PublishTableOfContentsModal.tsx b/packages/client/src/admin/data/PublishTableOfContentsModal.tsx index 2c775602a..66733879d 100644 --- a/packages/client/src/admin/data/PublishTableOfContentsModal.tsx +++ b/packages/client/src/admin/data/PublishTableOfContentsModal.tsx @@ -20,6 +20,10 @@ import { summary, valueText, } from "../changelogs/fieldGroups/FieldGroupListItemBase"; +import { + tableNameFromSummary, + tocItemIdFromMeta, +} from "../changelogs/fieldGroups/dataTableSummary"; import useProjectId from "../../useProjectId"; import { CHANGE_LOG_INTRODUCTION_DATE } from "../changelogs/constants"; import PublishSummarizedChangesPanel from "./PublishSummarizedChangesPanel"; @@ -113,6 +117,18 @@ export default function PublishTableOfContentsModal(props: { ]) ); }, [publishProject?.draftTableOfContentsItems]); + /** Active data-table id → name, for publish changelog titles. */ + const dataTableNameById = useMemo(() => { + const next = new Map(); + for (const item of publishProject?.draftTableOfContentsItems || []) { + for (const table of item.overlayDataTables || []) { + if (table.name) { + next.set(table.id, table.name); + } + } + } + return next; + }, [publishProject?.draftTableOfContentsItems]); /** TOC item ids whose current draft data source is tied to the Data Library (for changelog attribution). */ const tocItemIdsWithDataLibrarySource = useMemo(() => { const ids = new Set(); @@ -306,6 +322,7 @@ export default function PublishTableOfContentsModal(props: { loading={loading} changeLogs={changeLogs} itemTitleById={itemTitleById} + dataTableNameById={dataTableNameById} tocItemIdsWithDataLibrarySource={tocItemIdsWithDataLibrarySource} />
@@ -681,6 +698,7 @@ function AllChangesPanel({ loading, changeLogs, itemTitleById, + dataTableNameById, tocItemIdsWithDataLibrarySource, }: { loading: boolean; @@ -690,6 +708,7 @@ function AllChangesPanel({ >["changeLogsSinceLastPublish"] >; itemTitleById: Map; + dataTableNameById: Map; tocItemIdsWithDataLibrarySource: ReadonlySet; }) { const { t } = useTranslation("admin:data"); @@ -746,7 +765,12 @@ function AllChangesPanel({ ["changeLogsSinceLastPublish"] >[number], itemTitleById: Map, + dataTableNameById: Map, fallback: (key: string) => string ) { + if (changeLog.entityType === "overlay_data_table") { + const tocItemId = tocItemIdFromMeta(changeLog.meta); + const overlayTitle = + tocItemId != null + ? itemTitleById.get(tocItemId)?.title ?? fallback("Unknown layer") + : fallback("Unknown layer"); + const tableName = + tableNameFromSummary(summary(changeLog.toSummary)) || + tableNameFromSummary(summary(changeLog.fromSummary)) || + dataTableNameById.get(changeLog.entityId) || + ""; + if (!tableName) { + return overlayTitle; + } + return ( + + {overlayTitle} + {/* eslint-disable-next-line i18next/no-literal-string */} + + · + + {tableName} + + ); + } + if (changeLog.entityType !== "table_of_contents_items") { return undefined; } diff --git a/packages/client/src/admin/data/QuotaUsageTreemap.tsx b/packages/client/src/admin/data/QuotaUsageTreemap.tsx index db8e3008a..6ef30ee8f 100644 --- a/packages/client/src/admin/data/QuotaUsageTreemap.tsx +++ b/packages/client/src/admin/data/QuotaUsageTreemap.tsx @@ -400,6 +400,8 @@ export function humanizeOutputType( return "PNG Image"; case DataUploadOutputType.NetCdf: return "NetCDF"; + case DataUploadOutputType.Csv: + return "CSV"; case "Archives": return "Archived Versions"; case DataUploadOutputType.Xmlmetadata: diff --git a/packages/client/src/admin/data/TableOfContentsItemEditor/HostedLayerInfo.tsx b/packages/client/src/admin/data/TableOfContentsItemEditor/HostedLayerInfo.tsx index 2caff8cf8..4956fdb14 100644 --- a/packages/client/src/admin/data/TableOfContentsItemEditor/HostedLayerInfo.tsx +++ b/packages/client/src/admin/data/TableOfContentsItemEditor/HostedLayerInfo.tsx @@ -285,6 +285,8 @@ export function humanizeSourceType(type: DataUploadOutputType) { return "PNG"; case DataUploadOutputType.ZippedShapefile: return "Shapefile"; + case DataUploadOutputType.Csv: + return "CSV"; default: return type; } diff --git a/packages/client/src/admin/data/TableOfContentsItemEditor/LayerEditorTabs.tsx b/packages/client/src/admin/data/TableOfContentsItemEditor/LayerEditorTabs.tsx new file mode 100644 index 000000000..9745ae225 --- /dev/null +++ b/packages/client/src/admin/data/TableOfContentsItemEditor/LayerEditorTabs.tsx @@ -0,0 +1,47 @@ +import { useTranslation } from "react-i18next"; + +export type LayerEditorTab = { + id: string; + name: string; + title?: string; + current: boolean; +}; + +export default function LayerEditorTabs({ + tabs, + onSelect, +}: { + tabs: LayerEditorTab[]; + onSelect: (id: string) => void; +}) { + const { t } = useTranslation("admin"); + + return ( +
+ +
+ ); +} diff --git a/packages/client/src/admin/data/overlayDataTables/DataTableJoinColumnModal.tsx b/packages/client/src/admin/data/overlayDataTables/DataTableJoinColumnModal.tsx new file mode 100644 index 000000000..2a20d0d71 --- /dev/null +++ b/packages/client/src/admin/data/overlayDataTables/DataTableJoinColumnModal.tsx @@ -0,0 +1,314 @@ +import { useEffect, useMemo, useState } from "react"; +import { Trans, useTranslation, TFunction } from "react-i18next"; +import { QuestionMarkCircledIcon } from "@radix-ui/react-icons"; +import * as Tooltip from "@radix-ui/react-tooltip"; +import { GeostatsLayer } from "@seasketch/geostats-types"; +import Modal from "../../../components/Modal"; +import Warning from "../../../components/Warning"; +import { + OverlayJoinColumnOption, + partitionOverlayJoinColumnOptions, +} from "./overlayJoinColumnOptions"; + +export default function DataTableJoinColumnModal({ + open, + onClose, + geostatsLayer, + aiBestIdColumnHint, + initialJoinColumn, + onSave, + saving, +}: { + open: boolean; + onClose: () => void; + geostatsLayer: GeostatsLayer | undefined; + aiBestIdColumnHint?: string | null; + initialJoinColumn?: string | null; + onSave: (joinColumn: string) => void; + saving?: boolean; +}) { + const { t } = useTranslation("admin:data"); + const { suggested, other, featureCount } = useMemo( + () => partitionOverlayJoinColumnOptions(geostatsLayer, aiBestIdColumnHint), + [geostatsLayer, aiBestIdColumnHint] + ); + const validOptions = useMemo( + () => + [...suggested, ...other].filter((option) => option.status === "valid"), + [suggested, other] + ); + const [selected, setSelected] = useState(""); + + useEffect(() => { + if (!open) { + return; + } + const preferred = + initialJoinColumn && + validOptions.some((option) => option.attribute === initialJoinColumn) + ? initialJoinColumn + : validOptions[0]?.attribute || ""; + setSelected(preferred); + }, [open, initialJoinColumn, validOptions]); + + const hasAnyOptions = suggested.length + other.length > 0; + + return ( + selected && onSave(selected), + disabled: saving || !selected, + loading: saving, + }, + ]} + > + +

+ + Data tables you upload will need to be joined to spatial features in + this layer using a common ID column (e.g. site_id). When you upload + new tables, distinct values in all columns will be compared to this + column in order to determine how to join the two datasets. + +

+ + {!hasAnyOptions ? ( + + + No attribute statistics are available for this layer. Upload or + reprocess the source data so geostats can be computed before + enabling data tables. + + + ) : ( +
+ {suggested.length > 0 ? ( +
+

+ {t("Best columns")} +

+
+ {suggested.map((option) => ( + setSelected(option.attribute)} + /> + ))} +
+ {hasAnyOptions && validOptions.length === 0 ? ( + + + None of the suggested columns uniquely identify every + feature. Fix the source data in desktop GIS and re-upload + in order to enable data tables. + + + ) : null} +
+ ) : null} + + {other.length > 0 ? ( +
+

+ {t("Unsuitable columns")} +

+
+ {other.map((option) => ( + + ))} +
+
+ ) : null} +
+ )} + + + ); +} + +function getJoinColumnReasonText( + option: OverlayJoinColumnOption, + t: TFunction<"admin:data"> +): string | null { + switch (option.reason) { + case "unsupported_type": + return t("Only text or numeric columns can be used as join keys."); + case "too_few_values": + return t("This column has too few distinct values to identify features."); + case "duplicate_values": + return t( + "{{distinct}} distinct values for {{features}} features. {{shared}} value(s) appear on multiple features, so rows cannot be linked unambiguously.", + { + distinct: option.distinctCount.toLocaleString(), + features: option.featureCount.toLocaleString(), + shared: option.duplicateValueNames ?? 0, + } + ); + case "not_unique": + return t( + "This column does not have exactly one value per feature in the layer." + ); + default: + return option.status === "valid" + ? t("Unique value for every feature.") + : null; + } +} + +function UnsuitableColumnRow({ option }: { option: OverlayJoinColumnOption }) { + const { t } = useTranslation("admin:data"); + const sampleText = option.sampleValues.join(", "); + const reasonText = getJoinColumnReasonText(option, t); + + return ( +
+ + {option.attribute} + + {sampleText ? ( + + {sampleText} + + ) : ( + + )} + + {t("{{distinct}} distinct values", { + distinct: option.distinctCount.toLocaleString(), + })} + + {reasonText ? ( + + + + + + + {reasonText} + + + + + ) : null} +
+ ); +} + +function JoinColumnOptionRow({ + option, + selected, + onSelect, +}: { + option: OverlayJoinColumnOption; + selected: boolean; + onSelect: () => void; +}) { + const { t } = useTranslation("admin:data"); + const disabled = option.status !== "valid"; + const sampleText = option.sampleValues.join(", "); + const reasonText = getJoinColumnReasonText(option, t); + + const rowClassName = disabled + ? "border-gray-200 bg-gray-50 cursor-not-allowed select-none" + : selected + ? "border-primary-500 bg-primary-50 cursor-pointer" + : "border-gray-200 bg-white hover:border-gray-300 cursor-pointer"; + + const body = ( +
+ +
+
+ + {option.attribute} + + + {t("{{distinct}} / {{features}} distinct", { + distinct: option.distinctCount.toLocaleString(), + features: option.featureCount.toLocaleString(), + })} + +
+ {sampleText ? ( +

+ {t("Examples: {{values}}", { values: sampleText })} +

+ ) : null} + {option.status === "valid" ? ( +

{reasonText}

+ ) : reasonText ? ( +

+ {reasonText} +

+ ) : null} +
+
+ ); + + if (disabled) { + return ( +
+ {body} +
+ ); + } + + return ( + + ); +} diff --git a/packages/client/src/admin/data/overlayDataTables/DataTableUploadJobProgress.tsx b/packages/client/src/admin/data/overlayDataTables/DataTableUploadJobProgress.tsx new file mode 100644 index 000000000..7b8843c20 --- /dev/null +++ b/packages/client/src/admin/data/overlayDataTables/DataTableUploadJobProgress.tsx @@ -0,0 +1,85 @@ +import { useTranslation } from "react-i18next"; +import { + JobDetailsFragment, + ProjectBackgroundJobState, +} from "../../../generated/graphql"; +import ProgressBar from "../../../components/ProgressBar"; +import Spinner from "../../../components/Spinner"; + +export default function DataTableUploadJobProgress({ + job, + onDismiss, +}: { + job: Pick< + JobDetailsFragment, + "state" | "progress" | "progressMessage" | "errorMessage" + >; + onDismiss?: () => void; +}) { + const { t } = useTranslation("admin:data"); + const failed = job.state === ProjectBackgroundJobState.Failed; + const active = + job.state === ProjectBackgroundJobState.Queued || + job.state === ProjectBackgroundJobState.Running; + const showBar = + active || job.state === ProjectBackgroundJobState.Complete; + + const progressLabel = failed + ? job.errorMessage || t("Upload failed") + : job.progressMessage === "uploading" + ? t("Uploading file…") + : job.progressMessage || t("Processing…"); + + if (failed) { + return ( +
+
+ {onDismiss ? ( + + ) : null} +
+ 
+          {progressLabel}
+
+
+ ); + } + + return ( +
+ {showBar ? ( +
+ +
+ ) : null} +
+ {active ? : null} + {progressLabel} + {active && job.progressMessage === "uploading" ? ( + + {t("{{percent}}%", { + percent: Math.round((job.progress ?? 0) * 100), + })} + + ) : null} +
+
+ ); +} diff --git a/packages/client/src/admin/data/overlayDataTables/DataTableUploadModal.tsx b/packages/client/src/admin/data/overlayDataTables/DataTableUploadModal.tsx new file mode 100644 index 000000000..b95ab1713 --- /dev/null +++ b/packages/client/src/admin/data/overlayDataTables/DataTableUploadModal.tsx @@ -0,0 +1,484 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { createPortal } from "react-dom"; +import { Trans, useTranslation } from "react-i18next"; +import Papa from "papaparse"; +import { + ChevronDownIcon, + DocumentTextIcon, + RefreshIcon, + TableIcon, + UploadIcon, +} from "@heroicons/react/outline"; +import { CheckIcon } from "@radix-ui/react-icons"; +import * as DropdownMenu from "@radix-ui/react-dropdown-menu"; +import { GeostatsLayer } from "@seasketch/geostats-types"; +import Modal from "../../../components/Modal"; +import Spinner from "../../../components/Spinner"; +import Warning from "../../../components/Warning"; +import { + detectJoinColumnCandidates, + JoinColumnCandidate, + pickJoinColumn, +} from "./detectJoinColumn"; +import { DELIMITED_SAMPLE_BYTES } from "../../uploads/delimitedSpatial/detectDelimitedGeometry"; +import { DataTableUploadProcessingOptions } from "./types"; +import ProjectBackgroundJobManager from "../../uploads/ProjectBackgroundJobManager"; + +const CSV_ACCEPT = ".csv,.tsv,.txt"; + +function DataTableCsvJoinColumnPicker({ + joinColumn, + candidates, + overlayJoinColumn, + onSelect, + disabled, +}: { + joinColumn: string; + candidates: JoinColumnCandidate[]; + overlayJoinColumn: string; + onSelect: (csvColumn: string) => void; + disabled?: boolean; +}) { + const { t } = useTranslation("admin:data"); + const hasAlternatives = candidates.length > 1; + + return ( + + + + + + e.preventDefault()} + className="z-[60] min-w-[14rem] max-w-xs rounded-md border border-black/5 bg-white p-1 text-sm shadow-lg" + > + {candidates.map((candidate) => ( + onSelect(candidate.csvColumn)} + > + {candidate.csvColumn} + {candidate.csvColumn === joinColumn ? ( + + ) : null} + + ))} + {!hasAlternatives ? ( + <> + +

+ {t( + "No other columns in this file match the ID column chosen for this spatial layer.", + { overlayJoinColumn }, + )} +

+ + ) : null} + + + + ); +} + +function defaultName(filename: string) { + return filename.replace(/\.[^.]+$/, ""); +} + +type PreviewData = { + headers: string[]; + rows: string[][]; +}; + +export default function DataTableUploadModal({ + open, + onClose, + tableOfContentsItemId, + geostatsLayer, + canonicalOverlayJoinColumn, + replaceTableId, + onUploadStarted, + uploadOverlayDataTable, +}: { + open: boolean; + onClose: () => void; + tableOfContentsItemId: number; + geostatsLayer: GeostatsLayer | undefined; + canonicalOverlayJoinColumn: string; + replaceTableId?: number; + onUploadStarted: () => void; + uploadOverlayDataTable?: ProjectBackgroundJobManager["uploadOverlayDataTable"]; +}) { + const { t } = useTranslation("admin:data"); + const fileInputRef = useRef(null); + const [file, setFile] = useState(null); + const [preview, setPreview] = useState(null); + const [error, setError] = useState(null); + const [joinColumn, setJoinColumn] = useState(""); + const [overlayJoinColumn, setOverlayJoinColumn] = useState(""); + const [candidates, setCandidates] = useState([]); + const [analyzing, setAnalyzing] = useState(false); + const [startingUpload, setStartingUpload] = useState(false); + // Delimiter detected by Papa.parse during analysis; passed to the server so + // TSV/tab-delimited files are processed with the same delimiter used for + // the preview. + const [detectedDelimiter, setDetectedDelimiter] = useState(","); + // Incremented per analyzeFile call so a slow parse of a previously chosen + // file can't clobber results from a newer selection. + const analysisGeneration = useRef(0); + + const formatFileSize = useCallback( + (bytes: number) => { + if (bytes < 1024) { + return t("{{size}} B", { size: bytes }); + } + if (bytes < 1024 * 1024) { + return t("{{size}} KB", { size: (bytes / 1024).toFixed(1) }); + } + return t("{{size}} MB", { + size: (bytes / (1024 * 1024)).toFixed(1), + }); + }, + [t], + ); + + const resetForm = useCallback(() => { + analysisGeneration.current++; + setFile(null); + setPreview(null); + setError(null); + setJoinColumn(""); + setOverlayJoinColumn(""); + setCandidates([]); + setAnalyzing(false); + setStartingUpload(false); + setDetectedDelimiter(","); + }, []); + + useEffect(() => { + if (!open) { + resetForm(); + } + }, [open, resetForm]); + + const analyzeFile = useCallback( + async (selected: File) => { + const generation = ++analysisGeneration.current; + const isCurrent = () => analysisGeneration.current === generation; + setAnalyzing(true); + setError(null); + setPreview(null); + setJoinColumn(""); + setOverlayJoinColumn(""); + setCandidates([]); + try { + const chunk = selected.slice(0, DELIMITED_SAMPLE_BYTES); + const text = await chunk.text(); + if (!isCurrent()) { + return; + } + const parsed = Papa.parse(text, { + header: false, + skipEmptyLines: true, + }); + const rows = (parsed.data as string[][]).filter((r) => r.length > 0); + if (rows.length < 2) { + setError(t("Could not parse CSV file")); + return; + } + setDetectedDelimiter(parsed.meta?.delimiter || ","); + const headers = rows[0].map((h) => h.trim()); + const dataRows = rows.slice(1, 6); + setPreview({ headers, rows: dataRows }); + const detected = detectJoinColumnCandidates( + headers, + rows.slice(1), + geostatsLayer, + canonicalOverlayJoinColumn, + ); + setCandidates(detected); + const picked = pickJoinColumn(detected); + if (!picked) { + setError( + t("Could not detect a join column matching the overlay layer"), + ); + return; + } + setJoinColumn(picked.joinColumn); + setOverlayJoinColumn(canonicalOverlayJoinColumn); + } finally { + if (isCurrent()) { + setAnalyzing(false); + } + } + }, + [geostatsLayer, canonicalOverlayJoinColumn, t], + ); + + const onFileSelected = useCallback( + (selected: File | null) => { + setFile(selected); + if (selected) { + analyzeFile(selected); + } else { + setPreview(null); + setCandidates([]); + setJoinColumn(""); + setOverlayJoinColumn(""); + setError(null); + } + }, + [analyzeFile], + ); + + const onInputChange = useCallback( + (e: React.ChangeEvent) => { + const selected = e.target.files?.[0] ?? null; + onFileSelected(selected); + e.target.value = ""; + }, + [onFileSelected], + ); + + const openFilePicker = useCallback(() => { + fileInputRef.current?.click(); + }, []); + + const onSubmit = async () => { + if (!file || !joinColumn || !overlayJoinColumn || !uploadOverlayDataTable) { + return; + } + setStartingUpload(true); + setError(null); + try { + const processingOptions: DataTableUploadProcessingOptions = { + delimiter: detectedDelimiter, + hasHeaderRow: true, + joinColumn, + overlayJoinColumn, + name: defaultName(file.name), + }; + await uploadOverlayDataTable({ + tableOfContentsItemId, + file, + processingOptions, + replaceOverlayDataTableId: replaceTableId, + }); + onUploadStarted(); + onClose(); + } catch (e) { + setError((e as Error).message); + } finally { + setStartingUpload(false); + } + }; + + const title = replaceTableId + ? t("Replace Data Table") + : t("Upload Data Table"); + + return createPortal( + +
+ {replaceTableId ? ( +
+ + Upload a new CSV to replace the existing table. The previous + version will be kept in history and can be rolled back. + +
+ ) : null} + +
+
+ +
+

+ + Attach a CSV of observations or measurements to this map + layer. Rows are linked to overlay features using a shared ID + column, then stored as an optimized table for reports and + analysis. + +

+
    +
  • + + CSV, TSV, or tab-delimited text with a header row + +
    • +
  • + + One column must match an attribute on this layer + +
    • +
  • + + Processing converts the file to a fast columnar format + +
    • +
+
+
+
+ + + + {!file ? ( +
+ +

+ {t("Choose a CSV file")} +

+

+ {t(".csv, .tsv, or .txt with a header row")} +

+ +
+ ) : ( +
+
+ +
+

+ {file.name} +

+

+ {formatFileSize(file.size)} +

+
+ {analyzing ? ( + + ) : ( + + )} +
+ + {joinColumn && !analyzing ? ( +
+ { + setJoinColumn(csvColumn); + setOverlayJoinColumn(canonicalOverlayJoinColumn); + }} + /> +
+ ) : null} + + {preview && !analyzing ? ( +
+ + {t("Preview")} + +
+ + + + {preview.headers.map((header) => ( + + ))} + + + + {preview.rows.map((row, i) => ( + + {row.map((cell, j) => ( + + ))} + + ))} + +
+ {header} +
+ {cell} +
+
+
+ ) : null} +
+ )} + + {error ? {error} : null} +
+ , + document.body, + ); +} diff --git a/packages/client/src/admin/data/overlayDataTables/DataTablesEditor.tsx b/packages/client/src/admin/data/overlayDataTables/DataTablesEditor.tsx new file mode 100644 index 000000000..721b77b11 --- /dev/null +++ b/packages/client/src/admin/data/overlayDataTables/DataTablesEditor.tsx @@ -0,0 +1,58 @@ +import { useMemo } from "react"; +import { Trans, useTranslation } from "react-i18next"; +import { GeostatsLayer } from "@seasketch/geostats-types"; +import { FullAdminOverlayFragment } from "../../../generated/graphql"; +import EnableDataTables from "../EnableDataTables"; +import RelatedDataTables from "./RelatedDataTables"; +import DataTablesChangeLogList from "../../changelogs/DataTablesChangeLogList"; + +export default function DataTablesEditor({ + item, +}: { + item: FullAdminOverlayFragment; +}) { + const { t } = useTranslation("admin:data"); + const layer = item.dataLayer; + const source = layer?.dataSource; + + const geostatsLayer: GeostatsLayer | undefined = useMemo(() => { + const layers = (source?.geostats?.layers || []) as GeostatsLayer[]; + if (!layer) { + return undefined; + } + return ( + layers.find((entry) => + layer.sourceLayer ? entry.layer === layer.sourceLayer : true + ) || layers[0] + ); + }, [layer, source?.geostats?.layers]); + + if (!layer || !geostatsLayer) { + return ( +
+
+ + Data tables are not available for this layer type. + +
+
+ ); + } + + return ( +
+ + {item.enableDataTables && item.dataTableJoinColumn ? ( + <> + + + + ) : null} +
+ ); +} diff --git a/packages/client/src/admin/data/overlayDataTables/RelatedDataTables.tsx b/packages/client/src/admin/data/overlayDataTables/RelatedDataTables.tsx new file mode 100644 index 000000000..62cea4d68 --- /dev/null +++ b/packages/client/src/admin/data/overlayDataTables/RelatedDataTables.tsx @@ -0,0 +1,867 @@ +import { useCallback, useContext, useEffect, useMemo, useRef, useState } from "react"; +import { useApolloClient } from "@apollo/client"; +import { useTranslation } from "react-i18next"; +import { createPortal } from "react-dom"; +import { + CheckIcon, + CogIcon, + TrashIcon, + UploadIcon, +} from "@heroicons/react/outline"; +import { + FullAdminOverlayFragment, + OverlayDataTableDetailsFragment, + ProjectBackgroundJobState, + ProjectBackgroundJobType, + useGetLayerItemQuery, + useRenameOverlayDataTableMutation, + useSetOverlayDataTableVisualizationSettingsMutation, + useSoftDeleteOverlayDataTableMutation, +} from "../../../generated/graphql"; +import { GeostatsLayer } from "@seasketch/geostats-types"; +import { ProjectBackgroundJobContext } from "../../uploads/ProjectBackgroundJobContext"; +import { dataTableChangeLogRefetchQueries } from "../../changelogs/dataTableChangeLogRefetch"; +import DataTableUploadJobProgress from "./DataTableUploadJobProgress"; +import DataTableUploadModal from "./DataTableUploadModal"; +import { DATA_TABLE_AGGREGATIONS } from "../../../dataLayers/dataTableQueryApi"; +import { + columnStatsUrlForTable, + numericColumnNames, + useDataTableColumnStats, +} from "../../../dataLayers/useDataTableColumnStats"; +import useCurrentProjectMetadata from "../../../useCurrentProjectMetadata"; +import useDialog from "../../../components/useDialog"; +import Modal from "../../../components/Modal"; + +type RelatedDataTablesProps = { + item: FullAdminOverlayFragment; +}; + +type DataTableJob = NonNullable< + FullAdminOverlayFragment["projectBackgroundJobs"] +>[number]; + +function isActiveDataTableJob(job: DataTableJob) { + return ( + job.type === ProjectBackgroundJobType.DataTableUpload && + (job.state === ProjectBackgroundJobState.Queued || + job.state === ProjectBackgroundJobState.Running || + job.state === ProjectBackgroundJobState.Failed) + ); +} + +function getJobForTable(tableId: number, jobs: DataTableJob[]) { + return jobs.find( + (job) => + isActiveDataTableJob(job) && + job.overlayDataTableUpload?.replaceOverlayDataTableId === tableId + ); +} + +/** + * Map visualization limits (`visualization_columns` / `visualization_ops`). + * Empty lists mean end users may choose freely. + */ +function MapDisplaySettings({ + table, + selectedColumns, + selectedOps, + selectedRequiredFilters, + onColumnsChange, + onOpsChange, + onRequiredFiltersChange, +}: { + table: OverlayDataTableDetailsFragment; + selectedColumns: string[]; + selectedOps: string[]; + selectedRequiredFilters: string[]; + onColumnsChange: (columns: string[]) => void; + onOpsChange: (ops: string[]) => void; + onRequiredFiltersChange: (columns: string[]) => void; +}) { + const { t } = useTranslation("admin:data"); + const { data: projectMeta } = useCurrentProjectMetadata(); + const mapAccessToken = projectMeta?.project?.mapAccessToken; + const { columnStats, loading } = useDataTableColumnStats( + columnStatsUrlForTable(table), + mapAccessToken + ); + const numericColumns = numericColumnNames(columnStats); + const filterableColumns = useMemo( + () => + (columnStats?.columns || []) + .map((column) => column.attribute) + .filter( + (column) => column && column !== table.joinColumn + ) + .sort((a, b) => + a.localeCompare(b, undefined, { sensitivity: "base" }) + ), + [columnStats?.columns, table.joinColumn] + ); + + const toggleColumn = (column: string) => { + onColumnsChange( + selectedColumns.includes(column) + ? selectedColumns.filter((c) => c !== column) + : [...selectedColumns, column] + ); + }; + + const toggleOp = (op: string) => { + onOpsChange( + selectedOps.includes(op) + ? selectedOps.filter((o) => o !== op) + : [...selectedOps, op] + ); + }; + + const toggleRequiredFilter = (column: string) => { + onRequiredFiltersChange( + selectedRequiredFilters.includes(column) + ? selectedRequiredFilters.filter((c) => c !== column) + : [...selectedRequiredFilters, column] + ); + }; + + return ( +
+
+
+

+ {t("Data columns")} +

+

+ {t("Choose which measurements users can map.")} +

+
+ {loading ? ( +

+ {t("Loading column metadata...")} +

+ ) : numericColumns.length === 0 ? ( +

+ {t("No numeric columns found.")} +

+ ) : ( +
+ + {numericColumns.map((column: string) => ( + + ))} +
+ )} +
+
+
+

+ {t("Calculations")} +

+

+ {t("Choose how users can summarize values for each feature.")} +

+
+
+ + {DATA_TABLE_AGGREGATIONS.map((op) => ( + + ))} +
+
+
+
+

+ {t("Required filters")} +

+

+ {t( + "These filters always appear in the legend. Users must choose a value and cannot remove them." + )} +

+
+ {loading ? ( +

+ {t("Loading column metadata...")} +

+ ) : filterableColumns.length === 0 ? ( +

+ {t("No filterable columns found.")} +

+ ) : ( +
+ + {filterableColumns.map((column: string) => ( + + ))} +
+ )} +
+
+ ); +} + +function DataTableSettingsModal({ + table, + onClose, + onRename, + onDelete, + onReplace, + onSetVisualizationSettings, +}: { + table: OverlayDataTableDetailsFragment; + onClose: () => void; + onRename: (id: number, name: string) => void | Promise; + onDelete: (id: number) => void; + onReplace: (id: number) => void; + onSetVisualizationSettings: ( + id: number, + visualizationColumns: string[], + visualizationOps: string[], + requiredFilterColumns: string[] + ) => void | Promise; +}) { + const { t } = useTranslation("admin:data"); + const [draftName, setDraftName] = useState(table.name); + const [nameError, setNameError] = useState(); + const [draftColumns, setDraftColumns] = useState( + (table.visualizationColumns || []).filter(Boolean) as string[] + ); + const [draftOps, setDraftOps] = useState( + (table.visualizationOps || []).filter(Boolean) as string[] + ); + const [draftRequiredFilters, setDraftRequiredFilters] = useState( + (table.requiredFilterColumns || []).filter(Boolean) as string[] + ); + const [saving, setSaving] = useState(false); + + useEffect(() => { + setDraftName(table.name); + setNameError(undefined); + setDraftColumns( + (table.visualizationColumns || []).filter(Boolean) as string[] + ); + setDraftOps((table.visualizationOps || []).filter(Boolean) as string[]); + setDraftRequiredFilters( + (table.requiredFilterColumns || []).filter(Boolean) as string[] + ); + }, [ + table.name, + table.visualizationColumns, + table.visualizationOps, + table.requiredFilterColumns, + ]); + + const originalColumns = (table.visualizationColumns || []).filter( + Boolean + ) as string[]; + const originalOps = (table.visualizationOps || []).filter( + Boolean + ) as string[]; + const originalRequiredFilters = (table.requiredFilterColumns || []).filter( + Boolean + ) as string[]; + const nameDirty = draftName.trim() !== table.name; + const displayDirty = + JSON.stringify(draftColumns) !== JSON.stringify(originalColumns) || + JSON.stringify(draftOps) !== JSON.stringify(originalOps) || + JSON.stringify(draftRequiredFilters) !== + JSON.stringify(originalRequiredFilters); + const dirty = nameDirty || displayDirty; + + const saveChanges = async () => { + const next = draftName.trim(); + if (!next) { + setNameError(t("Name is required")); + return; + } + setNameError(undefined); + setSaving(true); + try { + if (nameDirty) { + await onRename(table.id, next); + } + if (displayDirty) { + await onSetVisualizationSettings( + table.id, + draftColumns, + draftOps, + draftRequiredFilters + ); + } + onClose(); + } catch (e) { + setNameError((e as Error).message || t("Could not rename table")); + } finally { + setSaving(false); + } + }; + + return createPortal( + void saveChanges(), + disabled: !dirty || saving, + loading: saving, + }, + ]} + > +
+
+ + { + setDraftName(event.target.value); + setNameError(undefined); + }} + className={`block w-full rounded-md border px-3 py-2 text-sm shadow-sm focus:border-primary-500 focus:outline-none focus:ring-1 focus:ring-primary-500 ${ + nameError ? "border-red-400" : "border-gray-300" + }`} + /> + {nameError ? ( +

{nameError}

+ ) : null} +
+ +
+
+

+ {t("Map display options")} +

+

+ {t( + "Control the choices available when someone displays this table on the map." + )} +

+
+ +
+ +
+
+

+ {t("Table data")} +

+

+ {t("Upload a new version or remove this table from the layer.")} +

+
+
+ + +
+
+
+ , + document.body + ); +} + +function DataTableRow({ + table, + job, + onDismissJob, + onRename, + onDelete, + onReplace, + onSetVisualizationSettings, +}: { + table: OverlayDataTableDetailsFragment; + job?: DataTableJob; + onDismissJob: (jobId: string) => void; + onRename: (id: number, name: string) => void | Promise; + onDelete: (id: number) => void; + onReplace: (id: number) => void; + onSetVisualizationSettings: ( + id: number, + visualizationColumns: string[], + visualizationOps: string[], + requiredFilterColumns: string[] + ) => void | Promise; +}) { + const { t } = useTranslation("admin:data"); + const [settingsOpen, setSettingsOpen] = useState(false); + const sameJoinColumn = table.joinColumn === table.overlayJoinColumn; + + return ( +
  • +
    +
    + {table.name}{" "} + + {t("v{{version}}", { version: table.version })} + +
    + {!job && ( + + )} +
    + {job ? ( +
    + onDismissJob(job.id) + : undefined + } + /> +
    + ) : ( +

    + {t("{{rowCount}} rows · joined by", { + rowCount: table.rowCount.toLocaleString(), + })}{" "} + {/* eslint-disable-next-line i18next/no-literal-string -- column identifiers */} + + {table.joinColumn} + + {!sameJoinColumn && ( + <> + {/* eslint-disable-next-line i18next/no-literal-string */} + → + {/* eslint-disable-next-line i18next/no-literal-string -- column identifiers */} + + {table.overlayJoinColumn} + + + )} +

    + )} + {settingsOpen ? ( + setSettingsOpen(false)} + onRename={onRename} + onDelete={onDelete} + onReplace={onReplace} + onSetVisualizationSettings={onSetVisualizationSettings} + /> + ) : null} +
    • + ); +} + +function PendingDataTableUploadRow({ + job, + onDismissJob, +}: { + job: DataTableJob; + onDismissJob: (jobId: string) => void; +}) { + const filename = + job.overlayDataTableUpload?.filename || + job.title + ?.replace(/^Replacement data table /, "") + .replace(/^Data table /, ""); + + return ( +
  • +

    {filename}

    + onDismissJob(job.id) + : undefined + } + /> +
    • + ); +} + +export default function RelatedDataTables({ item }: RelatedDataTablesProps) { + const { t } = useTranslation("admin:data"); + const [uploadOpen, setUploadOpen] = useState(false); + const [replaceId, setReplaceId] = useState(); + const { manager } = useContext(ProjectBackgroundJobContext); + const client = useApolloClient(); + const { confirmDelete } = useDialog(); + + const { data, refetch } = useGetLayerItemQuery({ + variables: { id: item.id }, + fetchPolicy: "cache-and-network", + }); + + const layerItem = data?.tableOfContentsItem ?? item; + const tables = layerItem.overlayDataTables || []; + const layer = layerItem.dataLayer; + const source = layer?.dataSource; + const geostatsLayer: GeostatsLayer | undefined = useMemo(() => { + const layers = (source?.geostats?.layers || []) as GeostatsLayer[]; + if (!layer) { + return undefined; + } + return ( + layers.find((entry) => + layer.sourceLayer ? entry.layer === layer.sourceLayer : true + ) || layers[0] + ); + }, [layer, source?.geostats?.layers]); + const overlayJoinColumn = layerItem.dataTableJoinColumn || ""; + + const changeLogRefetchQueries = useMemo( + () => [...dataTableChangeLogRefetchQueries(layerItem.id)], + [layerItem.id], + ); + + const refetchTablesAndHistory = useCallback(async () => { + await refetch(); + await client.refetchQueries({ + include: changeLogRefetchQueries as Parameters< + typeof client.refetchQueries + >[0]["include"], + }); + }, [refetch, client, changeLogRefetchQueries]); + + const [renameTable] = useRenameOverlayDataTableMutation({ + refetchQueries: changeLogRefetchQueries, + }); + const [deleteTable] = useSoftDeleteOverlayDataTableMutation({ + refetchQueries: changeLogRefetchQueries, + }); + const [setVisualizationSettingsMutation] = + useSetOverlayDataTableVisualizationSettingsMutation({ + refetchQueries: changeLogRefetchQueries, + }); + + const onSetVisualizationSettings = useCallback( + async ( + id: number, + visualizationColumns: string[], + visualizationOps: string[], + requiredFilterColumns: string[] + ) => { + await setVisualizationSettingsMutation({ + variables: { + id, + visualizationColumns, + visualizationOps, + requiredFilterColumns, + }, + }); + }, + [setVisualizationSettingsMutation] + ); + + const dataTableJobs = useMemo( + () => + (layerItem.projectBackgroundJobs || []).filter( + (job) => job.type === ProjectBackgroundJobType.DataTableUpload + ), + [layerItem.projectBackgroundJobs] + ); + + const activeDataTableJobs = useMemo( + () => dataTableJobs.filter(isActiveDataTableJob), + [dataTableJobs] + ); + + const pendingNewUploadJobs = useMemo( + () => + activeDataTableJobs.filter( + (job) => !job.overlayDataTableUpload?.replaceOverlayDataTableId + ), + [activeDataTableJobs] + ); + + const onUploadStarted = useCallback(() => { + void refetchTablesAndHistory(); + }, [refetchTablesAndHistory]); + + const onDismissJob = useCallback( + async (jobId: string) => { + if (manager) { + await manager.dismissFailedUpload(jobId); + await refetchTablesAndHistory(); + } + }, + [manager, refetchTablesAndHistory], + ); + + const hasActiveUploads = activeDataTableJobs.some( + (job) => job.state !== ProjectBackgroundJobState.Failed, + ); + + const trackedJobStatesRef = useRef>( + new Map(), + ); + + useEffect(() => { + let shouldRefetch = false; + for (const job of dataTableJobs) { + const previousState = trackedJobStatesRef.current.get(job.id); + if ( + previousState && + previousState !== ProjectBackgroundJobState.Complete && + previousState !== ProjectBackgroundJobState.Failed && + (job.state === ProjectBackgroundJobState.Complete || + job.state === ProjectBackgroundJobState.Failed) + ) { + shouldRefetch = true; + } + trackedJobStatesRef.current.set(job.id, job.state); + } + if (shouldRefetch) { + void refetchTablesAndHistory(); + } + }, [dataTableJobs, refetchTablesAndHistory]); + + useEffect(() => { + if (!hasActiveUploads) { + return; + } + const intervalId = window.setInterval(() => { + void refetchTablesAndHistory(); + }, 3000); + return () => window.clearInterval(intervalId); + }, [hasActiveUploads, refetchTablesAndHistory]); + + const openUploadModal = useCallback(() => { + setReplaceId(undefined); + setUploadOpen(true); + }, []); + + return ( +
    +
    +
    +

    + {t("Related Data Tables")} +

    +

    + {t( + "Upload CSV tables linked to this layer by a shared ID column—for example species counts per site or survey results per polygon." + )} +

    +
    + +
    + {tables.length === 0 && pendingNewUploadJobs.length === 0 ? ( +
    +

    + {t("No data tables associated with this layer.")} +

    + +
    + ) : ( +
      + {pendingNewUploadJobs.map((job) => ( + + ))} + {tables.map((table) => ( + + renameTable({ variables: { id, name } }).then(() => + refetchTablesAndHistory(), + ) + } + onDelete={(id) => + confirmDelete({ + message: t("Delete data table?"), + description: t( + "\"{{name}}\" will no longer be available for display or analysis. Previous uploads remain in the change history.", + { name: table.name }, + ), + onDelete: async () => { + await deleteTable({ variables: { id } }); + await refetchTablesAndHistory(); + }, + }) + } + onSetVisualizationSettings={onSetVisualizationSettings} + onReplace={(id) => { + setReplaceId(id); + setUploadOpen(true); + }} + /> + ))} +
    + )} + setUploadOpen(false)} + tableOfContentsItemId={layerItem.id} + geostatsLayer={geostatsLayer} + canonicalOverlayJoinColumn={overlayJoinColumn} + replaceTableId={replaceId} + onUploadStarted={onUploadStarted} + uploadOverlayDataTable={manager?.uploadOverlayDataTable.bind(manager)} + /> +
    + ); +} diff --git a/packages/client/src/admin/data/overlayDataTables/detectJoinColumn.test.ts b/packages/client/src/admin/data/overlayDataTables/detectJoinColumn.test.ts new file mode 100644 index 000000000..65056d9f4 --- /dev/null +++ b/packages/client/src/admin/data/overlayDataTables/detectJoinColumn.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it } from "@jest/globals"; +import { + detectJoinColumnCandidates, + isOverlayIdAttribute, + pickJoinColumn, +} from "./detectJoinColumn"; +import { GeostatsLayer } from "@seasketch/geostats-types"; + +const numericIdLayer: GeostatsLayer = { + layer: "sites", + count: 3, + geometry: "Point", + hasZ: false, + attributeCount: 1, + attributes: [ + { + attribute: "id", + type: "number", + count: 3, + countDistinct: 3, + values: { "1": 1, "2": 1, "3": 1 }, + }, + ], +}; + +const kelpSiteLayer: GeostatsLayer = { + layer: "site_points", + count: 3, + geometry: "Point", + hasZ: false, + attributeCount: 3, + attributes: [ + { + attribute: "site", + type: "string", + count: 3, + countDistinct: 3, + values: { HOPKINS_DC: 1, HOPKINS_UC: 1, ASILOMAR_DC: 1 }, + }, + { + attribute: "campus", + type: "string", + count: 3, + countDistinct: 1, + values: { UCSC: 3 }, + }, + { + attribute: "CA_MPA_Name_Short", + type: "string", + count: 3, + countDistinct: 2, + values: { "Lovers Point - Julia Platt SMR": 2, "Asilomar SMR": 1 }, + }, + ], +}; + +describe("isOverlayIdAttribute", () => { + it("requires one distinct value per feature", () => { + expect( + isOverlayIdAttribute(kelpSiteLayer.attributes[0], kelpSiteLayer.count), + ).toBe(true); + expect( + isOverlayIdAttribute(kelpSiteLayer.attributes[1], kelpSiteLayer.count), + ).toBe(false); + expect( + isOverlayIdAttribute(kelpSiteLayer.attributes[2], kelpSiteLayer.count), + ).toBe(false); + }); +}); + +describe("detectJoinColumn", () => { + it("finds site_id as join to numeric id", () => { + const candidates = detectJoinColumnCandidates( + ["site_id", "species", "count"], + [ + ["1", "bass", "99"], + ["2", "bass", "88"], + ], + numericIdLayer, + "id", + ); + expect(candidates).toEqual([ + { + csvColumn: "site_id", + overlayAttribute: "id", + score: 2, + }, + ]); + }); + + it("prefers the csv column with the most matching id values", () => { + const layer: GeostatsLayer = { + ...numericIdLayer, + count: 4, + attributes: [ + { + attribute: "id", + type: "number", + count: 4, + countDistinct: 4, + values: { "1": 1, "2": 1, "3": 1, "4": 1 }, + }, + ], + }; + const candidates = detectJoinColumnCandidates( + ["partial_id", "full_id", "species"], + [ + ["1", "1", "bass"], + ["2", "2", "bass"], + ["3", "3", "bass"], + ["", "4", "bass"], + ], + layer, + "id", + ); + expect(candidates[0]).toMatchObject({ + csvColumn: "full_id", + overlayAttribute: "id", + score: 4, + }); + expect(candidates[1]).toMatchObject({ + csvColumn: "partial_id", + overlayAttribute: "id", + score: 3, + }); + }); + + it("matches swath observations to overlay site ids only", () => { + const candidates = detectJoinColumnCandidates( + [ + "campus", + "method", + "survey_year", + "year", + "month", + "day", + "site", + "zone", + ], + [ + ["UCSC", "SBTL_SWATH_PISCO", "1999", "1999", "9", "7", "HOPKINS_DC", "INNER"], + ["UCSC", "SBTL_SWATH_PISCO", "1999", "1999", "9", "7", "HOPKINS_DC", "MID"], + ["UCSC", "SBTL_SWATH_PISCO", "1999", "1999", "9", "7", "HOPKINS_UC", "INNER"], + ], + kelpSiteLayer, + "site", + ); + expect(candidates).toEqual([ + { + csvColumn: "site", + overlayAttribute: "site", + score: 2, + }, + ]); + }); + + it("rejects csv columns with values missing from the overlay id set", () => { + const candidates = detectJoinColumnCandidates( + ["site", "site_name_old"], + [ + ["HOPKINS_DC", "UNKNOWN_SITE"], + ["HOPKINS_UC", "UNKNOWN_SITE"], + ["MYSTERY_SITE", "UNKNOWN_SITE"], + ], + kelpSiteLayer, + "site", + ); + expect(candidates).toEqual([]); + }); + + it("auto-picks when only one strong candidate", () => { + const candidates = detectJoinColumnCandidates( + ["site_id", "species"], + [["1", "bass"]], + numericIdLayer, + "id", + ); + const picked = pickJoinColumn(candidates); + expect(picked?.needsPrompt).toBe(false); + expect(picked?.joinColumn).toBe("site_id"); + }); + + it("prompts when multiple candidates tie for the top score", () => { + const layer: GeostatsLayer = { + layer: "sites", + count: 2, + geometry: "Point", + hasZ: false, + attributeCount: 2, + attributes: [ + { + attribute: "site_code", + type: "string", + count: 2, + countDistinct: 2, + values: { A: 1, B: 1 }, + }, + ], + }; + const candidates = detectJoinColumnCandidates( + ["site", "site_alias"], + [ + ["A", "A"], + ["B", "B"], + ], + layer, + "site_code", + ); + const picked = pickJoinColumn(candidates); + expect(candidates).toHaveLength(2); + expect(picked?.needsPrompt).toBe(true); + }); +}); diff --git a/packages/client/src/admin/data/overlayDataTables/detectJoinColumn.ts b/packages/client/src/admin/data/overlayDataTables/detectJoinColumn.ts new file mode 100644 index 000000000..031307c37 --- /dev/null +++ b/packages/client/src/admin/data/overlayDataTables/detectJoinColumn.ts @@ -0,0 +1,160 @@ +import { + GeostatsAttribute, + GeostatsAttributeType, + GeostatsLayer, +} from "@seasketch/geostats-types"; + +export type JoinColumnCandidate = { + csvColumn: string; + overlayAttribute: string; + /** Number of distinct CSV values that match the overlay ID column */ + score: number; +}; + +export function isOverlayIdAttribute( + attr: GeostatsAttribute, + featureCount: number, +): boolean { + if (featureCount <= 0) { + return false; + } + if (attr.type !== "string" && attr.type !== "number") { + return false; + } + const distinct = attr.countDistinct; + if (distinct !== featureCount) { + return false; + } + const valueEntries = Object.entries(attr.values || {}); + if (valueEntries.length === 0) { + return false; + } + if (valueEntries.length !== distinct) { + return false; + } + const total = valueEntries.reduce((sum, [, count]) => sum + count, 0); + if (total !== featureCount) { + return false; + } + return valueEntries.every(([, count]) => count === 1); +} + +function normalizeValue(value: string, type: GeostatsAttributeType): string { + const trimmed = value.trim(); + if (type === "number") { + const n = Number(trimmed); + if (!Number.isNaN(n)) { + return String(n); + } + } + return trimmed; +} + +function getColumnDistinctValues( + headers: string[], + rows: string[][], + columnName: string, +): Set { + const colIndex = headers.indexOf(columnName); + if (colIndex < 0) { + return new Set(); + } + const values = new Set(); + for (const row of rows) { + const raw = (row[colIndex] ?? "").trim(); + if (raw) { + values.add(raw); + } + } + return values; +} + +function typesCompatible( + csvValues: Set, + overlayAttr: GeostatsAttribute, +): boolean { + if (overlayAttr.type === "number") { + for (const v of csvValues) { + if (Number.isNaN(Number(v))) { + return false; + } + } + } + return true; +} + +function csvValuesSubsetOfOverlay( + csvValues: Set, + overlayAttr: GeostatsAttribute, +): boolean { + const overlayKeys = new Set(Object.keys(overlayAttr.values || {})); + if (overlayKeys.size === 0) { + return false; + } + for (const csvValue of csvValues) { + const normalized = normalizeValue(csvValue, overlayAttr.type); + if (!overlayKeys.has(normalized)) { + return false; + } + } + return true; +} + +export function detectJoinColumnCandidates( + headers: string[], + rows: string[][], + geostatsLayer: GeostatsLayer | undefined, + overlayJoinColumn: string, +): JoinColumnCandidate[] { + if ( + !geostatsLayer?.attributes?.length || + headers.length === 0 || + !overlayJoinColumn + ) { + return []; + } + + const overlayAttr = geostatsLayer.attributes.find( + (attr) => attr.attribute === overlayJoinColumn, + ); + if (!overlayAttr) { + return []; + } + + const candidates: JoinColumnCandidate[] = []; + + for (const header of headers) { + const csvValues = getColumnDistinctValues(headers, rows, header); + if (csvValues.size === 0) { + continue; + } + if (!typesCompatible(csvValues, overlayAttr)) { + continue; + } + if (!csvValuesSubsetOfOverlay(csvValues, overlayAttr)) { + continue; + } + candidates.push({ + csvColumn: header, + overlayAttribute: overlayJoinColumn, + score: csvValues.size, + }); + } + + return candidates.sort((a, b) => b.score - a.score); +} + +export function pickJoinColumn( + candidates: JoinColumnCandidate[], +): { joinColumn: string; overlayJoinColumn: string; needsPrompt: boolean } | null { + if (candidates.length === 0) { + return null; + } + const top = candidates[0]; + const tiedTop = candidates.filter((c) => c.score === top.score); + return { + joinColumn: top.csvColumn, + overlayJoinColumn: top.overlayAttribute, + needsPrompt: tiedTop.length > 1, + }; +} diff --git a/packages/client/src/admin/data/overlayDataTables/overlayJoinColumnOptions.test.ts b/packages/client/src/admin/data/overlayDataTables/overlayJoinColumnOptions.test.ts new file mode 100644 index 000000000..b28aee109 --- /dev/null +++ b/packages/client/src/admin/data/overlayDataTables/overlayJoinColumnOptions.test.ts @@ -0,0 +1,91 @@ +/* eslint-disable i18next/no-literal-string */ +import { describe, expect, it } from "@jest/globals"; +import { GeostatsLayer } from "@seasketch/geostats-types"; +import { partitionOverlayJoinColumnOptions } from "./overlayJoinColumnOptions"; + +const kelpSiteLayer: GeostatsLayer = { + layer: "site_points", + count: 401, + geometry: "Point", + hasZ: false, + attributeCount: 4, + attributes: [ + { + attribute: "site", + type: "string", + count: 401, + countDistinct: 396, + values: Object.fromEntries( + Array.from({ length: 396 }, (_, i) => [`SITE_${i}`, i < 391 ? 1 : 2]), + ), + }, + { + attribute: "CA_MPA_Name_Short", + type: "string", + count: 401, + countDistinct: 59, + values: Object.fromEntries( + Array.from({ length: 59 }, (_, i) => [`MPA_${i}`, Math.ceil(401 / 59)]), + ), + }, + { + attribute: "campus", + type: "string", + count: 401, + countDistinct: 1, + values: { UCSC: 401 }, + }, + { + attribute: "source_row_count", + type: "number", + count: 401, + countDistinct: 80, + values: Object.fromEntries( + Array.from({ length: 80 }, (_, i) => [String(i + 1), 5]), + ), + }, + ], +}; + +describe("partitionOverlayJoinColumnOptions", () => { + it("puts ai bestIdColumn first in suggested even when close", () => { + const { suggested, other } = partitionOverlayJoinColumnOptions( + kelpSiteLayer, + "site", + ); + expect(suggested[0]?.attribute).toBe("site"); + expect(suggested[0]?.hintSource).toBe("ai"); + expect(suggested[0]?.status).toBe("close"); + expect(other.map((option) => option.attribute)).not.toContain("site"); + }); + + it("uses cardinality heuristics when ai hint is unavailable", () => { + const { suggested, primaryHint } = partitionOverlayJoinColumnOptions( + kelpSiteLayer, + ); + expect(primaryHint).toBe("site"); + expect(suggested[0]?.attribute).toBe("site"); + expect(suggested[0]?.hintSource).toBe("computed"); + }); + + it("includes valid columns in suggested ahead of low-cardinality fields", () => { + const layer: GeostatsLayer = { + ...kelpSiteLayer, + count: 3, + attributes: [ + { + attribute: "site_id", + type: "string", + count: 3, + countDistinct: 3, + values: { A: 1, B: 1, C: 1 }, + }, + kelpSiteLayer.attributes[1], + ], + }; + const { suggested } = partitionOverlayJoinColumnOptions(layer); + expect(suggested.map((option) => option.attribute)).toContain("site_id"); + expect(suggested.find((option) => option.attribute === "site_id")?.status) + .toBe("valid"); + }); +}); diff --git a/packages/client/src/admin/data/overlayDataTables/overlayJoinColumnOptions.ts b/packages/client/src/admin/data/overlayDataTables/overlayJoinColumnOptions.ts new file mode 100644 index 000000000..4c0312649 --- /dev/null +++ b/packages/client/src/admin/data/overlayDataTables/overlayJoinColumnOptions.ts @@ -0,0 +1,265 @@ +import { + GeostatsAttribute, + GeostatsLayer, +} from "@seasketch/geostats-types"; +import { isOverlayIdAttribute } from "./detectJoinColumn"; + +export type OverlayJoinColumnOptionStatus = "valid" | "close" | "invalid"; + +export type OverlayJoinColumnOptionReason = + | "unsupported_type" + | "too_few_values" + | "duplicate_values" + | "not_unique"; + +export type OverlayJoinColumnHintSource = "ai" | "computed"; + +export type OverlayJoinColumnOption = { + attribute: string; + distinctCount: number; + featureCount: number; + sampleValues: string[]; + status: OverlayJoinColumnOptionStatus; + reason?: OverlayJoinColumnOptionReason; + duplicateValueNames?: number; + hintSource?: OverlayJoinColumnHintSource; + isPrimaryHint?: boolean; + rankScore: number; +}; + +export type OverlayJoinColumnSections = { + suggested: OverlayJoinColumnOption[]; + other: OverlayJoinColumnOption[]; + featureCount: number; + primaryHint?: string; +}; + +const CLOSE_SUGGESTED_RATIO = 0.9; + +function sampleAttributeValues(attr: GeostatsAttribute, limit = 12): string[] { + return Object.keys(attr.values || {}) + .sort((a, b) => a.localeCompare(b)) + .slice(0, limit); +} + +function duplicateValueCount(attr: GeostatsAttribute): number { + return Object.values(attr.values || {}).filter((count) => count > 1).length; +} + +function cardinalityRatio(distinctCount: number, featureCount: number): number { + if (featureCount <= 0) { + return 0; + } + return distinctCount / featureCount; +} + +function computeRankScore( + status: OverlayJoinColumnOptionStatus, + distinctCount: number, + featureCount: number, +): number { + const ratio = cardinalityRatio(distinctCount, featureCount); + if (status === "valid") { + return 10000 + distinctCount; + } + if (status === "close") { + return 5000 + ratio * 1000 + distinctCount * 0.01; + } + return ratio * 100 + distinctCount * 0.01; +} + +function analyzeAttribute( + attr: GeostatsAttribute, + featureCount: number, +): OverlayJoinColumnOption { + const distinctCount = + attr.countDistinct ?? Object.keys(attr.values || {}).length; + const sampleValues = sampleAttributeValues(attr); + + if (attr.type !== "string" && attr.type !== "number") { + return { + attribute: attr.attribute, + distinctCount, + featureCount, + sampleValues, + status: "invalid", + reason: "unsupported_type", + rankScore: computeRankScore("invalid", distinctCount, featureCount), + }; + } + + if (distinctCount <= 1) { + return { + attribute: attr.attribute, + distinctCount, + featureCount, + sampleValues, + status: "invalid", + reason: "too_few_values", + rankScore: computeRankScore("invalid", distinctCount, featureCount), + }; + } + + if (isOverlayIdAttribute(attr, featureCount)) { + return { + attribute: attr.attribute, + distinctCount, + featureCount, + sampleValues, + status: "valid", + rankScore: computeRankScore("valid", distinctCount, featureCount), + }; + } + + if (distinctCount < featureCount) { + const duplicateValueNames = duplicateValueCount(attr); + return { + attribute: attr.attribute, + distinctCount, + featureCount, + sampleValues, + status: "close", + reason: "duplicate_values", + duplicateValueNames, + rankScore: computeRankScore("close", distinctCount, featureCount), + }; + } + + return { + attribute: attr.attribute, + distinctCount, + featureCount, + sampleValues, + status: "invalid", + reason: "not_unique", + rankScore: computeRankScore("invalid", distinctCount, featureCount), + }; +} + +function normalizeHint(hint?: string | null): string | undefined { + const trimmed = hint?.trim(); + return trimmed ? trimmed : undefined; +} + +function findMatchingOption( + options: OverlayJoinColumnOption[], + hint?: string, +): OverlayJoinColumnOption | undefined { + if (!hint) { + return undefined; + } + const normalized = hint.toLowerCase(); + return options.find( + (option) => option.attribute.toLowerCase() === normalized, + ); +} + +function isHighlyRankedCandidate(option: OverlayJoinColumnOption): boolean { + if (option.status === "valid") { + return true; + } + if (option.status === "close") { + return ( + cardinalityRatio(option.distinctCount, option.featureCount) >= + CLOSE_SUGGESTED_RATIO + ); + } + return false; +} + +export function getOverlayJoinColumnOptions( + geostatsLayer: GeostatsLayer | undefined, +): OverlayJoinColumnOption[] { + if (!geostatsLayer?.attributes?.length) { + return []; + } + + const featureCount = geostatsLayer.count; + return geostatsLayer.attributes + .map((attr) => analyzeAttribute(attr, featureCount)) + .sort((a, b) => b.rankScore - a.rankScore); +} + +export function partitionOverlayJoinColumnOptions( + geostatsLayer: GeostatsLayer | undefined, + aiBestIdColumnHint?: string | null, +): OverlayJoinColumnSections { + const options = getOverlayJoinColumnOptions(geostatsLayer); + const featureCount = geostatsLayer?.count ?? 0; + + if (options.length === 0) { + return { suggested: [], other: [], featureCount }; + } + + const aiHint = normalizeHint(aiBestIdColumnHint); + const aiMatch = findMatchingOption(options, aiHint); + const computedBest = + aiMatch || + [...options].sort((a, b) => b.rankScore - a.rankScore)[0]; + const primaryHint = aiMatch?.attribute ?? computedBest?.attribute; + + const annotated = options.map((option) => { + const isPrimaryHint = option.attribute === primaryHint; + let hintSource: OverlayJoinColumnHintSource | undefined; + if (isPrimaryHint) { + hintSource = aiMatch ? "ai" : "computed"; + } + return { + ...option, + isPrimaryHint, + hintSource, + }; + }); + + const suggestedAttributes = new Set(); + const suggested: OverlayJoinColumnOption[] = []; + + const pushSuggested = (option: OverlayJoinColumnOption | undefined) => { + if (!option || suggestedAttributes.has(option.attribute)) { + return; + } + suggestedAttributes.add(option.attribute); + suggested.push(option); + }; + + if (primaryHint) { + pushSuggested(annotated.find((option) => option.attribute === primaryHint)); + } + + for (const option of annotated) { + if (option.status === "valid") { + pushSuggested(option); + } + } + + for (const option of annotated) { + if (isHighlyRankedCandidate(option)) { + pushSuggested(option); + } + } + + const other = annotated + .filter((option) => !suggestedAttributes.has(option.attribute)) + .sort((a, b) => b.rankScore - a.rankScore); + + return { + suggested, + other, + featureCount, + primaryHint, + }; +} + +export function typesCompatibleWithOverlayAttribute( + csvValues: Set, + type: "string" | "number" | GeostatsAttribute["type"], +): boolean { + if (type === "number") { + for (const value of csvValues) { + if (Number.isNaN(Number(value))) { + return false; + } + } + } + return true; +} diff --git a/packages/client/src/admin/data/overlayDataTables/types.ts b/packages/client/src/admin/data/overlayDataTables/types.ts new file mode 100644 index 000000000..3d43a32f0 --- /dev/null +++ b/packages/client/src/admin/data/overlayDataTables/types.ts @@ -0,0 +1,7 @@ +export type DataTableUploadProcessingOptions = { + delimiter: string; + hasHeaderRow: boolean; + joinColumn: string; + overlayJoinColumn: string; + name: string; +}; diff --git a/packages/client/src/admin/data/publishBadgeDetails/PublishBadgeDetailContent.tsx b/packages/client/src/admin/data/publishBadgeDetails/PublishBadgeDetailContent.tsx index a38c8cdec..a0a037d23 100644 --- a/packages/client/src/admin/data/publishBadgeDetails/PublishBadgeDetailContent.tsx +++ b/packages/client/src/admin/data/publishBadgeDetails/PublishBadgeDetailContent.tsx @@ -35,6 +35,7 @@ import { Summary, valueText, } from "../../changelogs/fieldGroups/FieldGroupListItemBase"; +import { dataTableEventDescription } from "../../changelogs/fieldGroups/dataTableSummary"; import { formatTimeAgo } from "../../changelogs/ChangeLogTimelineItem"; import { ResolvableCommentBody } from "../TableOfContentsItemEditor/ResolvableCommentEditor"; import { oldestChangeLogId, PublishBadgeKey } from "../publishChangelogSummary"; @@ -418,6 +419,37 @@ function CommentPreviewItem({ ); } +function DataTableActivityDetails({ + logs, + t, +}: { + logs: ChangeLogDetailsFragment[]; + t: TFunction; +}) { + const sorted = [...logs].sort( + (a, b) => new Date(b.lastAt).getTime() - new Date(a.lastAt).getTime(), + ); + return ( +
      + {sorted.map((log) => { + const date = new Date(log.lastAt); + return ( +
    • +

      {dataTableEventDescription(log.fieldGroup, log.fromSummary, log.toSummary, t)}

      + +
      • + ); + })} +
    + ); +} + export default function PublishBadgeDetailContent( props: PublishBadgeDetailContentProps ) { @@ -524,6 +556,10 @@ export default function PublishBadgeDetailContent( ); break; } + case "dataTables": { + main = ; + break; + } case "metadata": { main = (

    diff --git a/packages/client/src/admin/data/publishBadgeDetails/presentation.ts b/packages/client/src/admin/data/publishBadgeDetails/presentation.ts index 35fa0d42d..7ac2a3a93 100644 --- a/packages/client/src/admin/data/publishBadgeDetails/presentation.ts +++ b/packages/client/src/admin/data/publishBadgeDetails/presentation.ts @@ -11,6 +11,8 @@ export function badgePopoverContentClassName(badgeKey: PublishBadgeKey): string return "max-w-xs"; case "comments": return "w-[30rem] max-w-[min(calc(100vw-1rem),30rem)]"; + case "dataTables": + return "max-w-md"; default: return "max-w-sm"; } diff --git a/packages/client/src/admin/data/publishChangelogSummary.ts b/packages/client/src/admin/data/publishChangelogSummary.ts index 560b09945..6bdb25c94 100644 --- a/packages/client/src/admin/data/publishChangelogSummary.ts +++ b/packages/client/src/admin/data/publishChangelogSummary.ts @@ -5,8 +5,10 @@ import { ChangeLogsSinceLastPublishQuery, } from "../../generated/graphql"; import { summary } from "../changelogs/fieldGroups/FieldGroupListItemBase"; +import { tocItemIdFromMeta } from "../changelogs/fieldGroups/dataTableSummary"; export const TOC_ENTITY_TYPE = "table_of_contents_items"; +export const DATA_TABLE_ENTITY_TYPE = "overlay_data_table"; export type PublishBadgeKey = | "title" @@ -18,6 +20,7 @@ export type PublishBadgeKey = | "interactivity" | "moved" | "source" + | "dataTables" | "folderBehavior" | "comments"; @@ -30,6 +33,7 @@ export const PUBLISH_BADGE_ORDER: PublishBadgeKey[] = [ "metadata", "cartography", "downloads", + "dataTables", "interactivity", "comments", "moved", @@ -50,6 +54,12 @@ const FIELD_GROUP_TO_BADGE: Partial< [ChangeLogFieldGroup.LayerInteractivity]: "interactivity", [ChangeLogFieldGroup.LayerParentChanged]: "moved", [ChangeLogFieldGroup.LayerUploaded]: "source", + [ChangeLogFieldGroup.DataTableCreated]: "dataTables", + [ChangeLogFieldGroup.DataTableDeleted]: "dataTables", + [ChangeLogFieldGroup.DataTableRenamed]: "dataTables", + [ChangeLogFieldGroup.DataTableReplaced]: "dataTables", + [ChangeLogFieldGroup.DataTableRollback]: "dataTables", + [ChangeLogFieldGroup.DataTableVisualizationSettingsUpdated]: "dataTables", [ChangeLogFieldGroup.FolderType]: "folderBehavior", [ChangeLogFieldGroup.ResolvableLayerCommentsCreated]: "comments", [ChangeLogFieldGroup.ResolvableLayerCommentsResponded]: "comments", @@ -295,6 +305,19 @@ export function buildPublishChangeSummary({ byEntity.set(log.entityId, list); } + const dataTableLogs = changeLogs.filter( + (l) => l.entityType === DATA_TABLE_ENTITY_TYPE, + ); + for (const log of dataTableLogs) { + const tocItemId = tocItemIdFromMeta(log.meta); + if (tocItemId == null) { + continue; + } + const list = byEntity.get(tocItemId) ?? []; + list.push(log); + byEntity.set(tocItemId, list); + } + const draftById = new Map(draftItems.map((i) => [i.id, i])); const added: PublishSummaryRow[] = []; diff --git a/packages/client/src/admin/uploads/ProjectBackgroundJobContext.tsx b/packages/client/src/admin/uploads/ProjectBackgroundJobContext.tsx index d15f0ccdf..d4fed90be 100644 --- a/packages/client/src/admin/uploads/ProjectBackgroundJobContext.tsx +++ b/packages/client/src/admin/uploads/ProjectBackgroundJobContext.tsx @@ -35,9 +35,13 @@ import ProjectBackgroundJobManager, { DataUploadErrorEvent, DataUploadProcessingCompleteEvent, } from "./ProjectBackgroundJobManager"; +import { dataTableChangeLogRefetchQueries } from "../changelogs/dataTableChangeLogRefetch"; import sleep from "../../sleep"; import ConvertFeatureLayerToHostedModal from "../data/arcgis/ConvertFeatureLayerToHostedModal"; import AiDataAnalystUploadPromptModal from "./AiDataAnalystUploadPromptModal"; +import DelimitedUploadConfigModal from "./delimitedSpatial/DelimitedUploadConfigModal"; +import { resolveDelimitedUploads } from "./delimitedSpatial/resolveDelimitedUploads"; +import { DelimitedUploadProcessingOptions } from "./delimitedSpatial/types"; import AiDataAnalystUploadReminderModal from "./AiDataAnalystUploadReminderModal"; export type UploadType = "create" | "replace"; @@ -47,7 +51,8 @@ type SupportedSpatialFormat = | "shapefileZip" | "geotiff" | "netcdf" - | "flatgeobuf"; + | "flatgeobuf" + | "delimited"; const SUPPORTED_FORMATS: { id: SupportedSpatialFormat; @@ -85,6 +90,12 @@ const SUPPORTED_FORMATS: { extensions: ".fgb", tag: "FGB", }, + { + id: "delimited", + label: "CSV / Delimited Text", + extensions: ".csv, .tsv, .txt", + tag: "CSV", + }, ]; // A generic "document" glyph (a page with a folded corner) used to represent @@ -166,11 +177,21 @@ function detectSupportedFormat( return "netcdf"; case ".fgb": return "flatgeobuf"; + case ".csv": + case ".tsv": + case ".txt": + return "delimited"; default: return null; } } +/** CSV/TSV/TXT files require column-mapping configuration before upload. */ +function isDelimitedSpatialFile(fileName: string): boolean { + const ext = fileExtension(fileName); + return ext === ".csv" || ext === ".tsv" || ext === ".txt"; +} + type DroppedFileInfo = { name: string; format: SupportedSpatialFormat | null; @@ -226,6 +247,16 @@ export default function DataUploadDropzone({ finishedWithChangelog: boolean; changelog?: string; aiDataAnalystUploadPromptOpen: boolean; + /** + * Set when one or more dropped files require delimited-text column + * mapping. Upload of the entire batch (delimited and non-delimited + * files dropped together) is held until the user confirms or cancels. + */ + pendingDelimitedUpload: { + delimitedFiles: File[]; + otherFiles: File[]; + autoConfigsByFile: Map; + } | null; aiDataAnalystUploadReminderOpen: boolean; }>({ droppedFiles: 0, @@ -234,6 +265,7 @@ export default function DataUploadDropzone({ disabled: false, uploadType: "create", isUploadingReplacement: false, + pendingDelimitedUpload: null, replaceTableOfContentsItemId: null, finishedWithChangelog: true, aiDataAnalystUploadPromptOpen: false, @@ -297,6 +329,18 @@ export default function DataUploadDropzone({ }); } ); + manager.on( + "data-table-upload-complete", + (event: { jobId: string; tableOfContentsItemId: number }) => { + client.refetchQueries({ + include: [ + GetLayerItemDocument, + DraftTableOfContentsDocument, + ...dataTableChangeLogRefetchQueries(event.tableOfContentsItemId), + ], + }); + } + ); manager.on("upload-error", (event: DataUploadErrorEvent) => { if (dismissTimerRef.current) { clearTimeout(dismissTimerRef.current); @@ -381,6 +425,80 @@ export default function DataUploadDropzone({ useEffect(() => clearDismissTimer, [clearDismissTimer]); + // Kicks off the actual upload (S3 PUT + createDataUpload/submitDataUpload) + // for a batch of files. Used both for the immediate-upload path (no + // delimited files present) and after the delimited config modal confirms. + const startUpload = useCallback( + ( + filesToUpload: File[], + processingOptionsByFile?: Map + ) => { + const droppedFileInfos: DroppedFileInfo[] = filesToUpload.map((file) => ({ + name: file.name, + format: detectSupportedFormat(file.name), + })); + + setState((prev) => ({ + ...prev, + droppedFiles: filesToUpload.length, + droppedFileInfos, + error: undefined, + })); + + if (state.manager) { + if (state.uploadType === "replace") { + setState((prev) => ({ + ...prev, + isUploadingReplacement: true, + finishedWithChangelog: false, + })); + } + // Keep the confirmation visible briefly, then fade out and let the + // background job queue UI report on progress from here. + scheduleDismiss(OVERLAY_DISMISS_DELAY); + state.manager + .uploadFiles( + filesToUpload, + state.uploadType === "replace" && state.replaceTableOfContentsItemId + ? { + replaceTableOfContentsItemId: + state.replaceTableOfContentsItemId, + processingOptionsByFile, + } + : { processingOptionsByFile } + ) + .catch((e) => { + clearDismissTimer(); + const error: ReactNode = /quota exceeded/.test(e.message) ? ( + + This project has exceeded its data storage quota. Please delete + some data to make room for new uploads. You can see how much + space your layers are using by selecting{" "} + View {"->"} Data Hosting Quota from the toolbar. + + ) : ( + e.message + ); + setState((prev) => ({ + ...prev, + droppedFiles: 0, + droppedFileInfos: [], + isUploadingReplacement: false, + finishedWithChangelog: true, + error, + })); + }); + } + }, + [ + state.manager, + state.uploadType, + state.replaceTableOfContentsItemId, + scheduleDismiss, + clearDismissTimer, + ] + ); + const onDrop = useCallback( async (acceptedFiles: File[]) => { clearDismissTimer(); @@ -392,12 +510,8 @@ export default function DataUploadDropzone({ message = t(`"${file.name}" is a Word document.`); } else if (file.name.endsWith(".xlsx")) { message = t(`"${file.name}" is an Excel spreadsheet.`); - } else if (file.name.endsWith(".csv")) { - message = t(`"${file.name}" is a CSV file.`); } else if (file.name.endsWith(".pdf")) { message = t(`"${file.name}" is a PDF file.`); - } else if (file.name.endsWith(".txt")) { - message = t(`"${file.name}" is a text file.`); } else if (file.name.endsWith(".dbf")) { message = t(`"${file.name}" is a database file.`); isPartOfShapefile = true; @@ -496,72 +610,57 @@ export default function DataUploadDropzone({ return; } - const droppedFileInfos: DroppedFileInfo[] = filteredFiles.map((file) => ({ - name: file.name, - format: detectSupportedFormat(file.name), - })); + // CSV/TSV/TXT files need column-mapping configuration before they can + // be uploaded. Hold the entire batch (including any non-delimited + // files dropped alongside them) until the user confirms or cancels. + const delimitedFiles = filteredFiles.filter((f) => + isDelimitedSpatialFile(f.name) + ); + const otherFiles = filteredFiles.filter( + (f) => !isDelimitedSpatialFile(f.name) + ); + if (delimitedFiles.length > 0) { + const resolved = await resolveDelimitedUploads(delimitedFiles); + const blockingErrors = resolved + .map((entry) => entry.blockingError) + .filter((message): message is string => Boolean(message)); + if (blockingErrors.length > 0) { + alert(blockingErrors.join("\n\n")); + return; + } - setState((prev) => ({ - ...prev, - droppedFiles: filteredFiles.length, - droppedFileInfos, - error: undefined, - })); + const autoConfigsByFile = new Map< + File, + DelimitedUploadProcessingOptions + >(); + const filesNeedingConfig: File[] = []; + for (const entry of resolved) { + if (entry.needsConfig) { + filesNeedingConfig.push(entry.file); + } else if (entry.processingOptions) { + autoConfigsByFile.set(entry.file, entry.processingOptions); + } + } - if (state.manager) { - if (state.uploadType === "replace") { - setState((prev) => ({ - ...prev, - isUploadingReplacement: true, - finishedWithChangelog: false, - })); + if (filesNeedingConfig.length === 0) { + startUpload([...otherFiles, ...delimitedFiles], autoConfigsByFile); + return; } - // Keep the confirmation visible briefly, then fade out and let the - // background job queue UI report on progress from here. - scheduleDismiss(OVERLAY_DISMISS_DELAY); - state.manager - .uploadFiles( - filteredFiles, - state.uploadType === "replace" && state.replaceTableOfContentsItemId - ? { - replaceTableOfContentsItemId: - state.replaceTableOfContentsItemId, - } - : undefined - ) - .catch((e) => { - clearDismissTimer(); - const error: ReactNode = /quota exceeded/.test(e.message) ? ( - - This project has exceeded its data storage quota. Please delete - some data to make room for new uploads. You can see how much - space your layers are using by selecting{" "} - View {"->"} Data Hosting Quota from the toolbar. - - ) : ( - e.message - ); - setState((prev) => ({ - ...prev, - droppedFiles: 0, - droppedFileInfos: [], - isUploadingReplacement: false, - finishedWithChangelog: true, - error, - })); - }); + + setState((prev) => ({ + ...prev, + pendingDelimitedUpload: { + delimitedFiles: filesNeedingConfig, + otherFiles, + autoConfigsByFile, + }, + })); + return; } + + startUpload(filteredFiles); }, - [ - alert, - confirm, - state.manager, - t, - state.uploadType, - state.replaceTableOfContentsItemId, - scheduleDismiss, - clearDismissTimer, - ] + [alert, confirm, t, state.uploadType, clearDismissTimer, startUpload] ); const { getRootProps, getInputProps, isDragActive } = useDropzone({ @@ -569,6 +668,27 @@ export default function DataUploadDropzone({ noClick: true, }); + const onCancelDelimitedConfig = useCallback(() => { + setState((prev) => ({ ...prev, pendingDelimitedUpload: null })); + }, []); + + const onSubmitDelimitedConfig = useCallback( + (configsByFile: Map) => { + const pending = state.pendingDelimitedUpload; + if (!pending) return; + setState((prev) => ({ ...prev, pendingDelimitedUpload: null })); + const processingOptionsByFile = new Map(pending.autoConfigsByFile); + configsByFile.forEach((options, file) => { + processingOptionsByFile.set(file, options); + }); + startUpload( + [...pending.otherFiles, ...Array.from(processingOptionsByFile.keys())], + processingOptionsByFile + ); + }, + [state.pendingDelimitedUpload, startUpload] + ); + const showOverlay = isDragActive || state.droppedFiles > 0 || Boolean(state.error); @@ -607,7 +727,8 @@ export default function DataUploadDropzone({ browseForFiles: (multiple?: boolean) => { const fileInput = document.createElement("input"); fileInput.type = "file"; - fileInput.accept = ".zip,.json,.geojson,.fgb,.tif,.tiff"; + fileInput.accept = + ".zip,.json,.geojson,.fgb,.tif,.tiff,.csv,.tsv,.txt"; fileInput.multiple = multiple || false; fileInput.onchange = async (e) => { const files = (e.target as HTMLInputElement).files; @@ -670,6 +791,13 @@ export default function DataUploadDropzone({ }} /> )} + {state.pendingDelimitedUpload && ( + + )} {state.manager && state.aiDataAnalystUploadReminderOpen && ( ; } ) { if (files.length > 1 && options?.replaceTableOfContentsItemId) { @@ -305,6 +314,7 @@ export default class ProjectBackgroundJobManager extends EventEmitter<{ contentType: file.type, replaceTableOfContentsItemId: options?.replaceTableOfContentsItemId, + processingOptions: options?.processingOptionsByFile?.get(file), }, }); if ( @@ -447,6 +457,208 @@ export default class ProjectBackgroundJobManager extends EventEmitter<{ } } + addJobToTableOfContentsItemCache( + tocItemId: number, + job: JobDetailsFragment, + ) { + try { + this.client.cache.updateFragment( + { + // eslint-disable-next-line i18next/no-literal-string + id: `TableOfContentsItem:${tocItemId}`, + // eslint-disable-next-line i18next/no-literal-string + fragment: gql` + fragment UpdateTocItemDataTableJobs on TableOfContentsItem { + id + projectBackgroundJobs { + id + type + title + state + progress + progressMessage + errorMessage + overlayDataTableUpload { + id + tableOfContentsItemId + filename + replaceOverlayDataTableId + } + } + } + `, + }, + (data) => { + if (!data) { + return data; + } + const existing = data.projectBackgroundJobs || []; + return { + ...data, + projectBackgroundJobs: [ + ...existing.filter( + (entry: { id: number }) => entry.id !== job.id + ), + job, + ], + }; + }, + ); + } catch (e) { + console.error(e); + } + } + + private updateDataTableUploadProgress(jobId: string, progress: number) { + this.client.cache.writeFragment({ + // eslint-disable-next-line i18next/no-literal-string + id: `ProjectBackgroundJob:${jobId}`, + fragment: gql` + fragment DataTableUploadProgress on ProjectBackgroundJob { + progress + progressMessage + } + `, + data: { + progress, + progressMessage: "uploading", + }, + }); + } + + private async runDataTableFileUpload( + jobId: string, + presignedUrl: string, + file: File, + ) { + const signal = this.abortControllers[jobId]?.signal; + const response = await axios({ + url: presignedUrl, + method: "PUT", + data: file, + signal, + headers: { + "Content-Type": file.type || "text/csv", + }, + onUploadProgress: (progressEvent) => { + if (!signal?.aborted && progressEvent.total) { + this.updateDataTableUploadProgress( + jobId, + progressEvent.loaded / progressEvent.total, + ); + } + }, + }); + if (response.status !== 200) { + throw new Error("Non-200 response code"); + } + this.updateDataTableUploadProgress(jobId, 1); + await this.client.mutate({ + mutation: SubmitOverlayDataTableUploadDocument, + variables: { jobId }, + }); + this.client.cache.writeFragment({ + // eslint-disable-next-line i18next/no-literal-string + id: `ProjectBackgroundJob:${jobId}`, + fragment: gql` + fragment DataTableProcessingStart on ProjectBackgroundJob { + progress + progressMessage + } + `, + data: { + progress: 0, + progressMessage: "processing", + }, + }); + delete this.abortControllers[jobId]; + } + + async uploadOverlayDataTable({ + tableOfContentsItemId, + file, + processingOptions, + replaceOverlayDataTableId, + }: { + tableOfContentsItemId: number; + file: File; + processingOptions: DataTableUploadProcessingOptions; + replaceOverlayDataTableId?: number; + }) { + const upload = await this.client.mutate( + { + mutation: CreateOverlayDataTableUploadDocument, + variables: { + tableOfContentsItemId, + filename: file.name, + contentType: file.type || "text/csv", + replaceOverlayDataTableId, + processingOptions, + }, + }, + ); + if (upload.errors || !upload.data?.createOverlayDataTableUpload) { + throw new Error(upload.errors?.toString() || "Upload could not be created"); + } + const payload = upload.data.createOverlayDataTableUpload; + const overlayUpload = payload.overlayDataTableUpload; + const job = payload.projectBackgroundJob; + const presignedUrl = overlayUpload?.presignedUploadUrl; + const jobId = job?.id ?? overlayUpload?.projectBackgroundJobId; + if (!presignedUrl || !jobId || !job) { + throw new Error("Upload could not be created"); + } + + this.addJobToQueryCache(job); + this.addJobToTableOfContentsItemCache(tableOfContentsItemId, job); + this.sessionUploadJobIds.push(jobId); + this.activeJobs.add(jobId); + this.abortControllers[jobId] = new AbortController(); + + // Note: "upload-error" is reserved for spatial dropzone uploads (it + // clears dropzone state and shows the fullscreen error overlay). Data + // table upload failures surface on the job entry itself. + void this.runDataTableFileUpload(jobId, presignedUrl, file).catch((e) => { + delete this.abortControllers[jobId]; + if (axios.isCancel(e)) { + // User-initiated abort; abortUpload handles server-side cancellation. + return; + } + const message = e instanceof Error ? e.message : String(e); + this.markDataTableUploadFailed(jobId, message); + }); + + return { jobId }; + } + + /** + * Writes a failed state to the cached job entry when the browser-side PUT + * or submit mutation fails. Server-side failures arrive via subscription; + * this covers failures before the server ever hears about the upload. + */ + private markDataTableUploadFailed(jobId: string, message: string) { + try { + this.client.cache.writeFragment({ + // eslint-disable-next-line i18next/no-literal-string + id: `ProjectBackgroundJob:${jobId}`, + fragment: gql` + fragment DataTableUploadFailed on ProjectBackgroundJob { + state + progressMessage + errorMessage + } + `, + data: { + state: ProjectBackgroundJobState.Failed, + progressMessage: "failed", + errorMessage: message, + }, + }); + } catch (e) { + console.error(e); + } + } + isUploadFromMySession(taskId: string) { return this.sessionUploadJobIds.includes(taskId); } @@ -501,6 +713,21 @@ export default class ProjectBackgroundJobManager extends EventEmitter<{ ?.stableId, }); } + const dataTableUpload = event.job.overlayDataTableUpload; + if ( + dataTableUpload && + event.job.type === ProjectBackgroundJobType.DataTableUpload && + !this.completedTasks.has(event.job.id) && + event.previousState && + event.previousState !== ProjectBackgroundJobState.Complete && + event.job.state === ProjectBackgroundJobState.Complete + ) { + this.completedTasks.add(event.job.id); + this.emit("data-table-upload-complete", { + jobId: event.job.id, + tableOfContentsItemId: dataTableUpload.tableOfContentsItemId, + }); + } } /** @@ -544,13 +771,19 @@ export default class ProjectBackgroundJobManager extends EventEmitter<{ id } } + overlayDataTableUpload { + tableOfContentsItemId + } } `, }); const tocId = // @ts-ignore - cachedItem?.esriFeatureLayerConversionTask?.tableOfContentsItem?.id; - this.client.cache.updateFragment( + cachedItem?.esriFeatureLayerConversionTask?.tableOfContentsItem?.id ?? + // @ts-ignore + cachedItem?.overlayDataTableUpload?.tableOfContentsItemId; + if (tocId) { + this.client.cache.updateFragment( { // eslint-disable-next-line i18next/no-literal-string id: `TableOfContentsItem:${tocId}`, @@ -577,8 +810,9 @@ export default class ProjectBackgroundJobManager extends EventEmitter<{ } else { return data; } - } + }, ); + } // Remove from job list query this.client.cache.updateQuery( { diff --git a/packages/client/src/admin/uploads/delimitedSpatial/DelimitedUploadConfigModal.tsx b/packages/client/src/admin/uploads/delimitedSpatial/DelimitedUploadConfigModal.tsx new file mode 100644 index 000000000..50fbb47cf --- /dev/null +++ b/packages/client/src/admin/uploads/delimitedSpatial/DelimitedUploadConfigModal.tsx @@ -0,0 +1,413 @@ +import { useEffect, useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; +import Modal from "../../../components/Modal"; +import RadioGroup from "../../../components/RadioGroup"; +import Warning from "../../../components/Warning"; +import Spinner from "../../../components/Spinner"; +import { detectDelimitedGeometry, validateWgs84PointColumns, DELIMITED_SAMPLE_BYTES } from "./detectDelimitedGeometry"; +import { + DelimitedUploadProcessingOptions, + DetectDelimitedGeometryResult, +} from "./types"; + +// Only the first chunk of each file is read to detect columns and preview +// rows. This keeps detection fast even for very large CSVs. + +const selectClassName = + "block w-full rounded-md border-0 py-1.5 pl-3 pr-10 text-gray-900 ring-1 ring-inset ring-gray-300 focus:ring-2 focus:ring-indigo-600 sm:text-sm sm:leading-6"; + +type FileConfigState = { + file: File; + loading: boolean; + error: string | null; + detection: DetectDelimitedGeometryResult | null; + geometryMode: "point_xy" | "wkt"; + xField: string; + yField: string; + geometryField: string; + delimiter: DelimitedUploadProcessingOptions["delimiter"]; + hasHeaderRow: boolean; + validationError: string | null; +}; + +function initialFileConfigState(file: File): FileConfigState { + return { + file, + loading: true, + error: null, + detection: null, + geometryMode: "point_xy", + xField: "", + yField: "", + geometryField: "", + delimiter: ",", + hasHeaderRow: true, + validationError: null, + }; +} + +function getPointColumnValidationError( + state: FileConfigState +): string | null { + if ( + !state.detection || + state.geometryMode !== "point_xy" || + !state.xField || + !state.yField + ) { + return null; + } + return validateWgs84PointColumns( + state.detection.headers, + state.detection.rows, + state.xField, + state.yField + ); +} + +function isFileConfigValid(state: FileConfigState) { + if (state.loading || state.error || state.validationError) return false; + if (state.geometryMode === "wkt") return Boolean(state.geometryField); + return Boolean(state.xField) && Boolean(state.yField); +} + +function toProcessingOptions( + state: FileConfigState +): DelimitedUploadProcessingOptions { + return { + kind: "delimited", + geometryMode: state.geometryMode, + xField: state.geometryMode === "point_xy" ? state.xField : undefined, + yField: state.geometryMode === "point_xy" ? state.yField : undefined, + geometryField: + state.geometryMode === "wkt" ? state.geometryField : undefined, + crs: "EPSG:4326", + delimiter: state.delimiter, + hasHeaderRow: state.hasHeaderRow, + }; +} + +/** + * Presented when one or more dropped CSV/TSV/TXT files need the user to + * confirm (or correct) which columns contain location data before they are + * uploaded. Detection runs client-side via `detectDelimitedGeometry`; this + * modal is skipped entirely for files where detection is high-confidence. + */ +export default function DelimitedUploadConfigModal({ + files, + onSubmit, + onCancel, +}: { + files: File[]; + onSubmit: ( + configsByFile: Map + ) => void; + onCancel: () => void; +}) { + const { t } = useTranslation("admin:data"); + const [activeIndex, setActiveIndex] = useState(0); + const [fileStates, setFileStates] = useState(() => + files.map(initialFileConfigState) + ); + + useEffect(() => { + let cancelled = false; + files.forEach((file, i) => { + file + .slice(0, DELIMITED_SAMPLE_BYTES) + .text() + .then((sample) => { + if (cancelled) return; + const detection = detectDelimitedGeometry(sample); + setFileStates((prev) => { + const next = [...prev]; + const nextState: FileConfigState = { + ...next[i], + loading: false, + detection, + geometryMode: detection.geometryMode || "point_xy", + xField: detection.xField || "", + yField: detection.yField || "", + geometryField: detection.geometryField || "", + delimiter: detection.delimiter, + hasHeaderRow: detection.hasHeaderRow, + error: detection.error || null, + validationError: null, + }; + nextState.validationError = getPointColumnValidationError(nextState); + next[i] = nextState; + return next; + }); + }) + .catch((e) => { + if (cancelled) return; + setFileStates((prev) => { + const next = [...prev]; + next[i] = { + ...next[i], + loading: false, + error: e instanceof Error ? e.message : String(e), + }; + return next; + }); + }); + }); + return () => { + cancelled = true; + }; + // Detection only needs to run once per set of files. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + const updateActive = (update: Partial) => { + setFileStates((prev) => { + const next = [...prev]; + const merged = { ...next[activeIndex], ...update }; + merged.validationError = getPointColumnValidationError(merged); + next[activeIndex] = merged; + return next; + }); + }; + + const active = fileStates[activeIndex]; + const allValid = fileStates.every(isFileConfigValid); + const anyLoading = fileStates.some((f) => f.loading); + + return ( + 1 ? files.map((f) => f.name) : undefined} + onTabChange={files.length > 1 ? setActiveIndex : undefined} + footer={[ + { + label: t("Cancel"), + variant: "secondary", + onClick: onCancel, + }, + { + label: + files.length > 1 + ? t("Upload {{count}} files", { count: files.length }) + : t("Upload"), + variant: "primary", + autoFocus: true, + disabled: !allValid || anyLoading, + onClick: () => { + const configs = new Map(); + fileStates.forEach((state) => { + configs.set(state.file, toProcessingOptions(state)); + }); + onSubmit(configs); + }, + }, + ]} + > +

    + {files.length === 1 && ( +

    + {active.file.name} +

    + )} + {active.loading && ( +
    + +
    + )} + {active.error && ( + {active.error} + )} + {!active.loading && !active.error && active.detection && ( + + )} +
    + + ); +} + +function FileConfigForm({ + state, + detection, + onChange, +}: { + state: FileConfigState; + detection: DetectDelimitedGeometryResult; + onChange: (update: Partial) => void; +}) { + const { t } = useTranslation("admin:data"); + + return ( +
    +

    + + SeaSketch needs to know which columns in this file contain location + data. Review the detected mapping below, or choose different + columns. + +

    + + {detection.warnings.map((warning, i) => ( + + {warning} + + ))} + + {(state.error || state.validationError) && ( + {state.validationError || state.error} + )} + +

    + + Coordinates must be in WGS 84 (EPSG:4326) decimal degrees. + +

    + + + legend={t("Geometry type")} + value={state.geometryMode} + onChange={(geometryMode) => onChange({ geometryMode })} + items={[ + { + value: "point_xy", + label: t("Point coordinates"), + description: t( + "Use separate columns for latitude (Y) and longitude (X)" + ), + }, + { + value: "wkt", + label: t("Well-Known Text (WKT)"), + description: t( + "Use a single column containing WKT geometry strings (points, lines, or polygons)" + ), + }, + ]} + /> + + {state.geometryMode === "point_xy" ? ( +
    + + +
    + ) : ( + + )} + + + + +
    + ); +} + +function PreviewTable({ + detection, +}: { + detection: DetectDelimitedGeometryResult; +}) { + const { t } = useTranslation("admin:data"); + if (!detection.rows.length) return null; + return ( +
    + + {t("Preview")} + +
    + + + + {detection.headers.map((header) => ( + + ))} + + + + {detection.rows.map((row, i) => ( + + {row.map((cell, j) => ( + + ))} + + ))} + +
    + {header} +
    + {cell} +
    +
    +
    + ); +} diff --git a/packages/client/src/admin/uploads/delimitedSpatial/detectDelimitedGeometry.test.ts b/packages/client/src/admin/uploads/delimitedSpatial/detectDelimitedGeometry.test.ts new file mode 100644 index 000000000..345f06610 --- /dev/null +++ b/packages/client/src/admin/uploads/delimitedSpatial/detectDelimitedGeometry.test.ts @@ -0,0 +1,125 @@ +import { + detectDelimitedGeometry, + processingOptionsFromDetectionResult, +} from "./detectDelimitedGeometry"; + +test("standard latitude/longitude headers, comma delimited", () => { + const csv = [ + "name,latitude,longitude", + "Site A,42.686744,-73.822852", + "Site B,42.718588,-73.755328", + "Site C,42.814403,-73.930967", + ].join("\n"); + const result = detectDelimitedGeometry(csv); + expect(result.delimiter).toBe(","); + expect(result.geometryMode).toBe("point_xy"); + expect(result.xField).toBe("longitude"); + expect(result.yField).toBe("latitude"); + expect(result.confidence).toBe("high"); + expect(processingOptionsFromDetectionResult(result)).toEqual({ + kind: "delimited", + geometryMode: "point_xy", + xField: "longitude", + yField: "latitude", + geometryField: undefined, + crs: "EPSG:4326", + delimiter: ",", + hasHeaderRow: true, + }); +}); + +test("x/y headers, tab delimited", () => { + const tsv = [ + "name\tx\ty", + "Site A\t-73.822852\t42.686744", + "Site B\t-73.755328\t42.718588", + "Site C\t-73.930967\t42.814403", + ].join("\n"); + const result = detectDelimitedGeometry(tsv); + expect(result.delimiter).toBe("\t"); + expect(result.geometryMode).toBe("point_xy"); + expect(result.xField).toBe("x"); + expect(result.yField).toBe("y"); + expect(result.confidence).toBe("high"); +}); + +test("WKT geometry column", () => { + const csv = [ + "id,name,wkt", + '1,Region A,"POLYGON((-122.5 37.7, -122.4 37.7, -122.4 37.8, -122.5 37.8, -122.5 37.7))"', + '2,Region B,"POLYGON((-122.6 37.6, -122.5 37.6, -122.5 37.7, -122.6 37.7, -122.6 37.6))"', + '3,Region C,"POLYGON((-122.7 37.5, -122.6 37.5, -122.6 37.6, -122.7 37.6, -122.7 37.5))"', + ].join("\n"); + const result = detectDelimitedGeometry(csv); + expect(result.geometryMode).toBe("wkt"); + expect(result.geometryField).toBe("wkt"); + expect(result.confidence).toBe("high"); +}); + +test("ambiguous numeric columns trigger range-heuristic fallback with low confidence", () => { + const csv = [ + "id,col_a,col_b", + "1,42.686744,-73.822852", + "2,42.718588,-73.755328", + "3,42.814403,-73.930967", + ].join("\n"); + const result = detectDelimitedGeometry(csv); + expect(result.geometryMode).toBe("point_xy"); + expect(result.yField).toBe("col_a"); + expect(result.xField).toBe("col_b"); + expect(result.confidence).toBe("low"); + expect(result.warnings.length).toBeGreaterThan(0); +}); + +test("no detectable geometry columns", () => { + const csv = [ + "name,description", + "Site A,some text", + "Site B,more text", + ].join("\n"); + const result = detectDelimitedGeometry(csv); + expect(result.geometryMode).toBeNull(); + expect(result.confidence).toBe("low"); + expect(processingOptionsFromDetectionResult(result)).toBeNull(); +}); + +test("out-of-range coordinates are rejected as non-WGS 84", () => { + // Latitude values here exceed the valid [-90, 90] range, suggesting the + // columns may be swapped or the data is not in decimal degrees. + const csv = [ + "name,latitude,longitude", + "Site A,142.686744,-73.822852", + "Site B,142.718588,-73.755328", + "Site C,142.814403,-73.930967", + ].join("\n"); + const result = detectDelimitedGeometry(csv); + expect(result.geometryMode).toBe("point_xy"); + expect(result.error).toMatch(/WGS 84/); + expect(result.confidence).toBe("low"); + expect(processingOptionsFromDetectionResult(result)).toBeNull(); +}); + +test("semicolon delimited file", () => { + const csv = [ + "name;lat;lon", + "Site A;42.686744;-73.822852", + "Site B;42.718588;-73.755328", + ].join("\n"); + const result = detectDelimitedGeometry(csv); + expect(result.delimiter).toBe(";"); + expect(result.geometryMode).toBe("point_xy"); +}); + +test("file with no header row", () => { + const csv = [ + "42.686744,-73.822852,Site A", + "42.718588,-73.755328,Site B", + "42.814403,-73.930967,Site C", + ].join("\n"); + const result = detectDelimitedGeometry(csv); + expect(result.hasHeaderRow).toBe(false); + // Without recognizable headers, detection falls back to the numeric-range + // heuristic. + expect(result.geometryMode).toBe("point_xy"); + expect(result.confidence).toBe("low"); +}); diff --git a/packages/client/src/admin/uploads/delimitedSpatial/detectDelimitedGeometry.ts b/packages/client/src/admin/uploads/delimitedSpatial/detectDelimitedGeometry.ts new file mode 100644 index 000000000..2477ca853 --- /dev/null +++ b/packages/client/src/admin/uploads/delimitedSpatial/detectDelimitedGeometry.ts @@ -0,0 +1,396 @@ +/* eslint-disable i18next/no-literal-string */ +import Papa from "papaparse"; +import { + DelimitedColumnPreview, + DelimitedUploadProcessingOptions, + DetectDelimitedGeometryResult, +} from "./types"; + +const SAMPLE_ROW_LIMIT = 100; +export const DELIMITED_SAMPLE_BYTES = 200_000; +const CANDIDATE_DELIMITERS: DelimitedUploadProcessingOptions["delimiter"][] = [ + ",", + "\t", + ";", + "|", +]; + +// Header synonyms borrowed from how ArcGIS Online and Mapbox Studio detect +// CSV coordinate columns (see plan doc for sources). +const LAT_HEADER_SYNONYMS = new Set([ + "latitude", + "lat", + "y", + "ycenter", + "pointy", + "latdd", + "latdecdeg", + "latitude83", +]); +const LON_HEADER_SYNONYMS = new Set([ + "longitude", + "lon", + "long", + "lng", + "x", + "xcenter", + "pointx", + "londd", + "longdecdeg", + "longitude83", +]); +const WKT_HEADER_NAMES = new Set([ + "wkt", + "geom", + "geometry", + "thegeom", + "shape", +]); +const WKT_VALUE_PATTERN = + /^(SRID=\d+;)?\s*(POINT|LINESTRING|POLYGON|MULTIPOINT|MULTILINESTRING|MULTIPOLYGON|GEOMETRYCOLLECTION)\s*\(/i; + +function normalizeHeader(header: string): string { + return header + .trim() + .toLowerCase() + .replace(/[^a-z0-9]/g, ""); +} + +function isNumeric(value: string): boolean { + if (value.trim() === "") return false; + return Number.isFinite(Number(value)); +} + +const WGS84_ERROR = + "Coordinates must be in WGS 84 (EPSG:4326) decimal degrees. This file appears to use a different coordinate system or column mapping."; + +/** Returns an error message when sample rows are not valid WGS 84 lon/lat. */ +export function validateWgs84PointColumns( + headers: string[], + dataRows: string[][], + xField: string, + yField: string +): string | null { + const xIndex = headers.indexOf(xField); + const yIndex = headers.indexOf(yField); + if (xIndex < 0 || yIndex < 0) return null; + const sample = dataRows.slice(0, 50); + if (!sample.length) return null; + const validRows = sample.filter((r) => { + const x = Number(r[xIndex]); + const y = Number(r[yIndex]); + return ( + Number.isFinite(x) && + Number.isFinite(y) && + Math.abs(x) <= 180 && + Math.abs(y) <= 90 + ); + }); + const validRatio = validRows.length / sample.length; + if (validRatio < 0.9) { + return WGS84_ERROR; + } + return null; +} + +function wgs84ValidRatio( + dataRows: string[][], + xIndex: number, + yIndex: number +): number { + const sample = dataRows.slice(0, 50); + if (!sample.length) return 0; + const validRows = sample.filter((r) => { + const x = Number(r[xIndex]); + const y = Number(r[yIndex]); + return ( + Number.isFinite(x) && + Number.isFinite(y) && + Math.abs(x) <= 180 && + Math.abs(y) <= 90 + ); + }); + return validRows.length / sample.length; +} + +/** Picks the delimiter that produces the most consistent column count across sample rows. */ +function detectDelimiter(sampleText: string): { + delimiter: DelimitedUploadProcessingOptions["delimiter"]; + rows: string[][]; +} { + let best: { + delimiter: DelimitedUploadProcessingOptions["delimiter"]; + rows: string[][]; + score: number; + } | null = null; + + for (const delimiter of CANDIDATE_DELIMITERS) { + const result = Papa.parse(sampleText, { + delimiter, + skipEmptyLines: true, + preview: SAMPLE_ROW_LIMIT, + }); + const rows = result.data; + if (rows.length < 2) continue; + const columnCount = rows[0].length; + if (columnCount < 2) continue; + const consistentRows = rows.filter((r) => r.length === columnCount).length; + const consistency = consistentRows / rows.length; + // Prefer delimiters that extract more columns with high row-to-row + // consistency, and penalize parse errors. + const score = columnCount * consistency - result.errors.length * 0.5; + if (!best || score > best.score) { + best = { delimiter, rows, score }; + } + } + + if (!best) { + const result = Papa.parse(sampleText, { + skipEmptyLines: true, + preview: SAMPLE_ROW_LIMIT, + }); + return { delimiter: ",", rows: result.data }; + } + + return { delimiter: best.delimiter, rows: best.rows }; +} + +/** + * Compares the first row against the rest of the sample to decide whether it + * is a header row. For columns that are consistently numeric in the data + * rows, a non-numeric value in the first row is a strong signal that it's a + * header label rather than data (a first row that is itself numeric in those + * columns means the file likely has no header at all). + */ +function looksLikeHeaderRow(rows: string[][]): boolean { + if (rows.length < 2) return true; + const [first, ...rest] = rows; + let numericColumnCount = 0; + let numericColumnWithNonNumericHeader = 0; + first.forEach((cell, i) => { + const restValues = rest + .map((r) => r[i] ?? "") + .filter((v) => v.trim() !== ""); + if (!restValues.length) return; + const numericRatio = + restValues.filter(isNumeric).length / restValues.length; + if (numericRatio >= 0.8) { + numericColumnCount++; + if (!isNumeric(cell)) numericColumnWithNonNumericHeader++; + } + }); + if (numericColumnCount === 0) { + // No reliably numeric columns to compare against; fall back to a simple + // check for any non-numeric cell in the first row. + return first.some( + (cell) => cell.trim() !== "" && Number.isNaN(Number(cell)) + ); + } + return numericColumnWithNonNumericHeader > 0; +} + +/** + * Inspects a sample of delimited text (CSV/TSV/etc.) and attempts to detect + * the delimiter, header row, and a geometry column or coordinate column pair. + * + * Detection proceeds in priority order: WKT geometry column, then header-name + * based lat/lon columns (the approach used by ArcGIS Online and Mapbox + * Studio), then a numeric-range heuristic fallback for unrecognized headers. + * Confidence is downgraded whenever the result relied on heuristics or + * sampled coordinates fall outside valid WGS 84 ranges, signaling that the + * upload UI should ask the user to confirm or correct the mapping. + */ +export function detectDelimitedGeometry( + sampleText: string +): DetectDelimitedGeometryResult { + const { delimiter, rows: allRows } = detectDelimiter(sampleText); + const warnings: string[] = []; + + if (allRows.length < 2) { + return { + delimiter, + hasHeaderRow: true, + headers: allRows[0] || [], + rows: [], + columns: [], + geometryMode: null, + crs: "EPSG:4326", + confidence: "low", + error: "Could not parse enough rows to detect columns.", + warnings: [], + }; + } + + const hasHeaderRow = looksLikeHeaderRow(allRows); + // Match GDAL's CSV driver naming for headerless files (HEADERS=NO) so that + // the field names selected here line up with what ogr2ogr will produce. + const headers = hasHeaderRow + ? allRows[0] + : allRows[0].map((_, i) => `field_${i + 1}`); + const dataRows = hasHeaderRow ? allRows.slice(1) : allRows; + + const columns: DelimitedColumnPreview[] = headers.map((name, i) => ({ + name, + sampleValues: dataRows.slice(0, 10).map((r) => r[i] ?? ""), + })); + + const base = { + delimiter, + hasHeaderRow, + headers, + rows: dataRows.slice(0, 10), + columns, + }; + + // --- 1. WKT geometry column --- + let geometryField: string | undefined; + for (let i = 0; i < headers.length; i++) { + if (!WKT_HEADER_NAMES.has(normalizeHeader(headers[i]))) continue; + const values = dataRows.slice(0, 20).map((r) => r[i] ?? ""); + const matches = values.filter((v) => WKT_VALUE_PATTERN.test(v)).length; + if ( + values.length > 0 && + matches >= Math.min(3, values.length) && + matches / values.length >= 0.8 + ) { + geometryField = headers[i]; + break; + } + } + + if (geometryField) { + return { + ...base, + geometryMode: "wkt", + geometryField, + crs: "EPSG:4326", + confidence: "high", + warnings, + }; + } + + // --- 2. Header-name based lat/lon detection --- + let xField: string | undefined; + let yField: string | undefined; + for (const header of headers) { + const normalized = normalizeHeader(header); + if (!xField && LON_HEADER_SYNONYMS.has(normalized)) xField = header; + if (!yField && LAT_HEADER_SYNONYMS.has(normalized)) yField = header; + } + + if (xField && yField) { + const xIndex = headers.indexOf(xField); + const yIndex = headers.indexOf(yField); + const validRatio = wgs84ValidRatio(dataRows, xIndex, yIndex); + if (validRatio < 0.9) { + return { + ...base, + geometryMode: "point_xy", + xField, + yField, + crs: "EPSG:4326", + confidence: "low", + error: WGS84_ERROR, + warnings: [], + }; + } + return { + ...base, + geometryMode: "point_xy", + xField, + yField, + crs: "EPSG:4326", + confidence: "high", + warnings, + }; + } + + // --- 3. Numeric-range heuristic fallback --- + const numericColumnIndices = headers + .map((_, i) => i) + .filter((i) => { + const sample = dataRows.slice(0, 100).map((r) => r[i] ?? ""); + const numeric = sample.filter(isNumeric); + return sample.length > 0 && numeric.length / sample.length >= 0.9; + }); + + const inRangeRatio = (i: number, max: number) => { + const sample = dataRows.slice(0, 100).map((r) => Number(r[i])); + const inRange = sample.filter( + (v) => Number.isFinite(v) && Math.abs(v) <= max + ); + return sample.length ? inRange.length / sample.length : 0; + }; + + // Coordinates are almost always fractional, while incidental numeric + // columns (row IDs, counts, etc.) tend to be whole numbers. Prefer + // candidates with decimal values when any are available. + const decimalRatio = (i: number) => { + const sample = dataRows + .slice(0, 100) + .map((r) => r[i] ?? "") + .filter((v) => v.trim() !== ""); + if (!sample.length) return 0; + return sample.filter((v) => v.includes(".")).length / sample.length; + }; + const preferDecimalValues = (indices: number[]) => { + const withDecimals = indices.filter((i) => decimalRatio(i) >= 0.5); + return withDecimals.length ? withDecimals : indices; + }; + + const latCandidateIndices = preferDecimalValues( + numericColumnIndices.filter((i) => inRangeRatio(i, 90) >= 0.85) + ); + const lonCandidateIndices = preferDecimalValues( + numericColumnIndices.filter((i) => inRangeRatio(i, 180) >= 0.85) + ); + + // Prefer a lat candidate that isn't also the only lon candidate, so a + // single ambiguous column isn't used for both axes. + const latIndex = latCandidateIndices.find( + (i) => lonCandidateIndices.length > 1 || lonCandidateIndices[0] !== i + ); + const lonIndex = lonCandidateIndices.find((i) => i !== latIndex); + + if (latIndex !== undefined && lonIndex !== undefined) { + warnings.push( + "Coordinate columns were guessed from numeric value ranges because no recognized header names (e.g. latitude/longitude) were found. Please confirm the mapping." + ); + return { + ...base, + geometryMode: "point_xy", + xField: headers[lonIndex], + yField: headers[latIndex], + crs: "EPSG:4326", + confidence: "low", + warnings, + }; + } + + warnings.push( + "Could not automatically detect coordinate or geometry columns. Please select them manually." + ); + return { + ...base, + geometryMode: null, + crs: "EPSG:4326", + confidence: "low", + warnings, + }; +} + +export function processingOptionsFromDetectionResult( + result: DetectDelimitedGeometryResult +): DelimitedUploadProcessingOptions | null { + if (!result.geometryMode || result.error) return null; + return { + kind: "delimited", + geometryMode: result.geometryMode, + xField: result.xField, + yField: result.yField, + geometryField: result.geometryField, + crs: result.crs, + delimiter: result.delimiter, + hasHeaderRow: result.hasHeaderRow, + }; +} diff --git a/packages/client/src/admin/uploads/delimitedSpatial/resolveDelimitedUploads.ts b/packages/client/src/admin/uploads/delimitedSpatial/resolveDelimitedUploads.ts new file mode 100644 index 000000000..17b905009 --- /dev/null +++ b/packages/client/src/admin/uploads/delimitedSpatial/resolveDelimitedUploads.ts @@ -0,0 +1,70 @@ +import { + DELIMITED_SAMPLE_BYTES, + detectDelimitedGeometry, + processingOptionsFromDetectionResult, +} from "./detectDelimitedGeometry"; +import { + DelimitedUploadProcessingOptions, + DetectDelimitedGeometryResult, +} from "./types"; + +/** Bytes read from each file for client-side column detection. */ +export { DELIMITED_SAMPLE_BYTES } from "./detectDelimitedGeometry"; + +export async function readDelimitedFileSample(file: File): Promise { + return file.slice(0, DELIMITED_SAMPLE_BYTES).text(); +} + +export async function detectDelimitedGeometryFromFile( + file: File +): Promise { + const sample = await readDelimitedFileSample(file); + return detectDelimitedGeometry(sample); +} + +/** True when detection is confident enough to upload without the config modal. */ +export function isAutoUploadReady(result: DetectDelimitedGeometryResult): boolean { + return ( + !result.error && + result.confidence === "high" && + Boolean(processingOptionsFromDetectionResult(result)) + ); +} + +export function getDelimitedUploadBlockingError( + result: DetectDelimitedGeometryResult, + fileName?: string +): string | null { + if (!result.error) return null; + return fileName ? `${fileName}: ${result.error}` : result.error; +} + +export type ResolvedDelimitedUpload = { + file: File; + detection: DetectDelimitedGeometryResult; + processingOptions: DelimitedUploadProcessingOptions | null; + needsConfig: boolean; + blockingError: string | null; +}; + +export async function resolveDelimitedUploads( + files: File[] +): Promise { + return Promise.all( + files.map(async (file) => { + const detection = await detectDelimitedGeometryFromFile(file); + const processingOptions = processingOptionsFromDetectionResult(detection); + const blockingError = getDelimitedUploadBlockingError( + detection, + file.name + ); + return { + file, + detection, + processingOptions, + needsConfig: !isAutoUploadReady(detection), + blockingError, + }; + }) + ); +} diff --git a/packages/client/src/admin/uploads/delimitedSpatial/types.ts b/packages/client/src/admin/uploads/delimitedSpatial/types.ts new file mode 100644 index 000000000..3eab6feec --- /dev/null +++ b/packages/client/src/admin/uploads/delimitedSpatial/types.ts @@ -0,0 +1,44 @@ +/** + * Processing instructions for delimited text (CSV/TSV/TXT) spatial uploads. + * + * This shape mirrors `DelimitedUploadProcessingOptions` defined in the + * `spatial-uploads-handler` package, which is the source of truth consumed + * server-side. It is duplicated here (rather than imported) because that + * package pulls in Node/GDAL-only dependencies that should not be bundled + * into the client. Keep both definitions in sync when making changes. + */ +export type DelimitedUploadProcessingOptions = { + kind: "delimited"; + geometryMode: "point_xy" | "wkt"; + xField?: string; + yField?: string; + geometryField?: string; + crs: string; + delimiter: "," | "\t" | ";" | "|"; + hasHeaderRow: boolean; +}; + +export type DelimitedGeometryConfidence = "high" | "medium" | "low"; + +export type DelimitedColumnPreview = { + name: string; + sampleValues: string[]; +}; + +export type DetectDelimitedGeometryResult = { + delimiter: DelimitedUploadProcessingOptions["delimiter"]; + hasHeaderRow: boolean; + headers: string[]; + rows: string[][]; + columns: DelimitedColumnPreview[]; + geometryMode: "point_xy" | "wkt" | null; + xField?: string; + yField?: string; + geometryField?: string; + crs: string; + confidence: DelimitedGeometryConfidence; + /** When set, the file cannot be uploaded until the issue is resolved. */ + error?: string | null; + /** Human-readable reasons the result has medium/low confidence. */ + warnings: string[]; +}; diff --git a/packages/client/src/components/icons/DataTableIcon.tsx b/packages/client/src/components/icons/DataTableIcon.tsx new file mode 100644 index 000000000..56f552274 --- /dev/null +++ b/packages/client/src/components/icons/DataTableIcon.tsx @@ -0,0 +1,74 @@ +/** + * Landscape-oriented data table glyph for legend / TOC controls. + * Header row uses a filled band; body shows a 2x2 grid with light cell tinting. + */ +export default function DataTableIcon({ className }: { className?: string }) { + return ( + + + + + + + + + + ); +} diff --git a/packages/client/src/dataLayers/ActivatedDataTableButton.tsx b/packages/client/src/dataLayers/ActivatedDataTableButton.tsx new file mode 100644 index 000000000..494f84db1 --- /dev/null +++ b/packages/client/src/dataLayers/ActivatedDataTableButton.tsx @@ -0,0 +1,90 @@ +import { useContext, useState } from "react"; +import * as Popover from "@radix-ui/react-popover"; +import { useTranslation } from "react-i18next"; +import clsx from "clsx"; +import { ClientOverlayDataTableFragment } from "../generated/graphql"; +import DataTableIcon from "../components/icons/DataTableIcon"; +import { MapOverlayContext } from "./MapContextManager"; +import ActivatedDataTablePanel from "./ActivatedDataTablePanel"; + +/** + * Button + popover for choosing which OverlayDataTable (if any) is + * "activated" for a layer and configuring its map display. Used in the + * map Legend, and eventually the overlay Table of Contents as well. + */ +export default function ActivatedDataTableButton({ + layerId, + tocItemId, + layerName, + tables, + className, + onDataTableActivated, +}: { + /** Table of contents stableId for the associated layer */ + layerId: string; + /** Numeric TableOfContentsItem id, used to prefetch table metadata in one query */ + tocItemId?: number; + /** Used as a subtitle in the popover header */ + layerName?: string; + tables?: ClientOverlayDataTableFragment[] | null; + className?: string; + /** Called when a table is activated (not when cleared), e.g. to focus the legend. */ + onDataTableActivated?: (layerId: string) => void; +}) { + const { t } = useTranslation("homepage"); + const { layerStatesByTocStaticId } = useContext(MapOverlayContext); + const [open, setOpen] = useState(false); + + if (!tables || tables.length === 0) { + return null; + } + + const activeStableId = layerStatesByTocStaticId[layerId]?.dataTable?.stableId; + const activeTable = tables.find((table) => table.stableId === activeStableId); + const title = activeTable + ? // eslint-disable-next-line i18next/no-literal-string + `${t("Data table")}: ${activeTable.name}` + : t("Show data tables"); + + return ( + + + + + + setOpen(false)} + onDataTableActivated={onDataTableActivated} + /> + + + ); +} diff --git a/packages/client/src/dataLayers/ActivatedDataTablePanel.tsx b/packages/client/src/dataLayers/ActivatedDataTablePanel.tsx new file mode 100644 index 000000000..5c2491f6d --- /dev/null +++ b/packages/client/src/dataLayers/ActivatedDataTablePanel.tsx @@ -0,0 +1,141 @@ +import { useContext, useEffect, useMemo } from "react"; +import * as Popover from "@radix-ui/react-popover"; +import { CheckIcon } from "@radix-ui/react-icons"; +import { useTranslation } from "react-i18next"; +import clsx from "clsx"; +import { + ClientOverlayDataTableFragment, + useOverlayDataTableVisualizationMetadataForLayerQuery, +} from "../generated/graphql"; +import { MapManagerContext, MapOverlayContext } from "./MapContextManager"; +import { DataTableVisualizationMetadata } from "./dataTableQueryApi"; +import { + columnStatsUrlForTable, + fetchDataTableColumnStats, +} from "./useDataTableColumnStats"; +import useCurrentProjectMetadata from "../useCurrentProjectMetadata"; + +/** + * Popover panel for choosing which OverlayDataTable is active for a layer. + * Display settings and filters are configured in the map legend. + */ +export default function ActivatedDataTablePanel({ + layerId, + tocItemId, + layerName, + tables, + onTableSelected, + onDataTableActivated, +}: { + layerId: string; + tocItemId?: number; + layerName?: string; + tables: ClientOverlayDataTableFragment[]; + onTableSelected?: () => void; + onDataTableActivated?: (layerId: string) => void; +}) { + const { t } = useTranslation("homepage"); + const { manager } = useContext(MapManagerContext); + const { layerStatesByTocStaticId } = useContext(MapOverlayContext); + const { data: projectMeta } = useCurrentProjectMetadata(); + const mapAccessToken = projectMeta?.project?.mapAccessToken; + const metadataQuery = useOverlayDataTableVisualizationMetadataForLayerQuery({ + variables: { tocItemId: tocItemId || -1 }, + skip: tocItemId === undefined, + fetchPolicy: "cache-first", + }); + + const activeStableId = layerStatesByTocStaticId[layerId]?.dataTable?.stableId; + const metadataByTableId = useMemo(() => { + const next: { + [tableId: number]: DataTableVisualizationMetadata | undefined; + } = {}; + for (const table of metadataQuery.data?.tableOfContentsItem + ?.overlayDataTables || []) { + next[table.id] = { + queryUrl: table.queryUrl, + columnStatsUrl: table.columnStatsUrl, + visualizationColumns: table.visualizationColumns, + visualizationOps: table.visualizationOps, + requiredFilterColumns: table.requiredFilterColumns, + }; + } + return next; + }, [metadataQuery.data?.tableOfContentsItem?.overlayDataTables]); + + useEffect(() => { + for (const table of tables) { + const metadata = metadataByTableId[table.id] || table; + const columnStatsUrl = columnStatsUrlForTable(metadata); + if (columnStatsUrl) { + void fetchDataTableColumnStats(columnStatsUrl, mapAccessToken); + } + } + }, [metadataByTableId, tables, mapAccessToken]); + + return ( + +
    +

    + {t("Data Tables")} +

    + {layerName && ( +

    {layerName}

    + )} +
    + {metadataQuery.loading && ( +
    + {t("Loading table metadata...")} +
    + )} +
      + {tables.map((table) => { + const isActive = table.stableId === activeStableId; + return ( +
    • + +
      • + ); + })} +
    + +     + ); +} diff --git a/packages/client/src/dataLayers/DataDownloadModal.tsx b/packages/client/src/dataLayers/DataDownloadModal.tsx index 4b0d33b67..4203ef416 100644 --- a/packages/client/src/dataLayers/DataDownloadModal.tsx +++ b/packages/client/src/dataLayers/DataDownloadModal.tsx @@ -335,6 +335,10 @@ function DownloadFormatDescription({ type }: { type: DataUploadOutputType }) { are a compressed binary format compatible with most software. ); + case DataUploadOutputType.Csv: + return t( + "Delimited text file (e.g. CSV), compatible with excel and other spreadsheet software." + ); default: return null; } diff --git a/packages/client/src/dataLayers/DataTableFilterControls.tsx b/packages/client/src/dataLayers/DataTableFilterControls.tsx new file mode 100644 index 000000000..49d3be833 --- /dev/null +++ b/packages/client/src/dataLayers/DataTableFilterControls.tsx @@ -0,0 +1,477 @@ +import { useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Cross2Icon } from "@radix-ui/react-icons"; +import * as Popover from "@radix-ui/react-popover"; +import { GeostatsAttribute } from "@seasketch/geostats-types"; +import { DataTableFilter } from "./dataTableQueryApi"; +import DataTableNumericFilter, { + defaultNumericFilters, +} from "./DataTableNumericFilter"; +import DataTableStringFilter from "./DataTableStringFilter"; + +function filtersForColumn(filters: DataTableFilter[], column: string) { + return filters.filter((filter) => filter.column === column); +} + +function replaceColumnFilters( + filters: DataTableFilter[], + column: string, + replacements: DataTableFilter[] +) { + const next: DataTableFilter[] = []; + let inserted = false; + for (const filter of filters) { + if (filter.column === column) { + if (!inserted) { + next.push(...replacements); + inserted = true; + } + continue; + } + next.push(filter); + } + if (!inserted) { + next.push(...replacements); + } + return next; +} + +function nullMode(filters: DataTableFilter[]) { + if (filters.some((filter) => filter.op === "isNull")) { + return "isNull"; + } + if (filters.some((filter) => filter.op === "notNull")) { + return "notNull"; + } + return "value"; +} + +function BooleanColumnFilter({ + column, + filters, + onChange, +}: { + column: GeostatsAttribute; + filters: DataTableFilter[]; + onChange: (filters: DataTableFilter[]) => void; +}) { + const { t } = useTranslation("homepage"); + const filter = filters[0]; + const value = + filter?.op === "eq" && filter.value !== undefined ? filter.value : filter?.op; + return ( + + ); +} + +function DateColumnFilter({ + column, + filters, + onChange, +}: { + column: GeostatsAttribute; + filters: DataTableFilter[]; + onChange: (filters: DataTableFilter[]) => void; +}) { + const { t } = useTranslation("homepage"); + const mode = nullMode(filters); + const startValue = filters.find((filter) => filter.op === "gte")?.value || ""; + const endValue = filters.find((filter) => filter.op === "lte")?.value || ""; + + const setRange = (nextStart: string, nextEnd: string) => { + const next: DataTableFilter[] = []; + if (nextStart !== "") { + next.push({ column: column.attribute, op: "gte", value: nextStart }); + } + if (nextEnd !== "") { + next.push({ column: column.attribute, op: "lte", value: nextEnd }); + } + onChange(next.length ? next : [{ column: column.attribute, op: "notNull" }]); + }; + + return ( +
    + + {mode === "value" && ( +
    + setRange(e.target.value, endValue)} + /> + setRange(startValue, e.target.value)} + /> +
    + )} +
    + ); +} + +function isStringLikeColumn(column: GeostatsAttribute) { + const type = column.type as string; + return ( + type !== "number" && + type !== "boolean" && + type !== "date" && + type !== "timestamp" + ); +} + +export function defaultFiltersForColumn( + column: GeostatsAttribute +): DataTableFilter[] { + if (column.type === "number") { + return defaultNumericFilters(column); + } + if (isStringLikeColumn(column)) { + const firstValue = Object.keys(column.values || {}).sort((a, b) => + a.localeCompare(b, undefined, { sensitivity: "base" }) + )[0]; + if (firstValue) { + return [{ column: column.attribute, op: "eq", value: firstValue }]; + } + } + return [{ column: column.attribute, op: "notNull" }]; +} + +/** + * Ensure every required filter column has at least one filter entry, using + * sensible defaults (first string choice, full numeric range, etc.). Existing + * filters for those columns are kept. Optional filters are preserved. + */ +export function ensureRequiredDataTableFilters( + filters: DataTableFilter[] | undefined, + requiredColumns: string[], + columns: GeostatsAttribute[], + excludedColumns: string[] = [] +): DataTableFilter[] { + const excluded = new Set(excludedColumns.filter(Boolean)); + const columnsByName = new Map( + columns.map((column) => [column.attribute, column]) + ); + const current = filters || []; + const next = [...current]; + for (const columnName of requiredColumns) { + if (!columnName || excluded.has(columnName)) { + continue; + } + const column = columnsByName.get(columnName); + if (!column) { + continue; + } + if (next.some((filter) => filter.column === columnName)) { + continue; + } + next.push(...defaultFiltersForColumn(column)); + } + return next; +} + +function FilterValueEditor({ + column, + filters, + onChange, +}: { + column: GeostatsAttribute; + filters: DataTableFilter[]; + onChange: (filters: DataTableFilter[]) => void; +}) { + if (column.type === "number") { + return ( + + ); + } + if (column.type === "boolean") { + return ( + + ); + } + if ((column.type as string) === "date" || (column.type as string) === "timestamp") { + return ( + + ); + } + return ( + + ); +} + +export default function DataTableFilterControls({ + columns, + filters, + visualizedColumns, + requiredColumns = [], + onChange, +}: { + columns: GeostatsAttribute[]; + filters: DataTableFilter[]; + visualizedColumns: string[]; + /** Admin-required filter columns; shown first and not removable. */ + requiredColumns?: string[]; + onChange: (filters: DataTableFilter[]) => void; +}) { + const { t } = useTranslation("homepage"); + const excludedColumns = useMemo( + () => new Set(visualizedColumns.filter(Boolean)), + [visualizedColumns] + ); + const requiredColumnSet = useMemo( + () => + new Set( + requiredColumns.filter( + (column) => Boolean(column) && !excludedColumns.has(column) + ) + ), + [excludedColumns, requiredColumns] + ); + const columnsByName = useMemo( + () => new Map(columns.map((column) => [column.attribute, column])), + [columns] + ); + // Required columns first (admin order), then other active filters A–Z. + const activeColumnNames = useMemo(() => { + const active = new Set( + filters + .map((filter) => filter.column) + .filter( + (column) => columnsByName.has(column) && !excludedColumns.has(column) + ) + ); + const requiredActive = requiredColumns.filter( + (column) => active.has(column) && columnsByName.has(column) + ); + const optionalActive = Array.from(active) + .filter((column) => !requiredColumnSet.has(column)) + .sort((a, b) => a.localeCompare(b, undefined, { sensitivity: "base" })); + return [...requiredActive, ...optionalActive]; + }, [ + columnsByName, + excludedColumns, + filters, + requiredColumnSet, + requiredColumns, + ]); + const availableColumns = useMemo( + () => + columns + .filter( + (column) => + !excludedColumns.has(column.attribute) && + !activeColumnNames.includes(column.attribute) + ) + .sort((a, b) => + a.attribute.localeCompare(b.attribute, undefined, { + sensitivity: "base", + }) + ), + [activeColumnNames, columns, excludedColumns] + ); + const [addFilterOpen, setAddFilterOpen] = useState(false); + + if (columns.length === 0) { + return null; + } + + return ( +
    + {activeColumnNames.length > 0 && ( +
    {t("Filters")}
    + )} + {activeColumnNames.map((columnName) => { + const column = columnsByName.get(columnName)!; + const columnFilters = filtersForColumn(filters, columnName); + const isRequired = requiredColumnSet.has(columnName); + const compact = + isStringLikeColumn(column) || + column.type === "boolean" || + column.type === "number"; + return ( +
    + {compact ? ( + <> + + {column.attribute} + + + onChange( + replaceColumnFilters(filters, columnName, nextFilters) + ) + } + /> + {!isRequired && ( + + )} + + ) : ( + <> +
    +
    +
    + {column.attribute} +
    +
    {column.type}
    +
    + {!isRequired && ( + + )} +
    + + onChange( + replaceColumnFilters(filters, columnName, nextFilters) + ) + } + /> + + )} +
    + ); + })} + {availableColumns.length > 0 && ( + + + + + + +
    + {availableColumns.map((column) => ( + + ))} +
    + +     +     +     + )} + {availableColumns.length === 0 && activeColumnNames.length === 0 && ( +

    + {t("No filterable columns available.")} +

    + )} +
    + ); +} diff --git a/packages/client/src/dataLayers/DataTableNumericFilter.tsx b/packages/client/src/dataLayers/DataTableNumericFilter.tsx new file mode 100644 index 000000000..0f763aa07 --- /dev/null +++ b/packages/client/src/dataLayers/DataTableNumericFilter.tsx @@ -0,0 +1,910 @@ +import { + KeyboardEvent, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import { useTranslation } from "react-i18next"; +import { + CaretDownIcon, + CheckIcon, + MagnifyingGlassIcon, +} from "@radix-ui/react-icons"; +import * as Popover from "@radix-ui/react-popover"; +import * as Slider from "@radix-ui/react-slider"; +import { GeostatsAttribute } from "@seasketch/geostats-types"; +import { + DataTableFilter, + dataTableInFilterValues, +} from "./dataTableQueryApi"; +import clsx from "clsx"; + +/** Prefer discrete equality when unique values stay within this bound. */ +export const NUMERIC_DISCRETE_VALUE_LIMIT = 40; + +type NumericUiMode = "values" | "range" | "isNull" | "notNull"; + +function numericKeys(column: GeostatsAttribute) { + return Object.keys(column.values || {}) + .map((value) => ({ + raw: value, + num: Number(value), + })) + .filter((entry) => Number.isFinite(entry.num)) + .sort((a, b) => a.num - b.num); +} + +/** + * Discrete when there are a modest number of unique values (years, survey + * sites codes, etc.). Continuous/range when the value space is large or + * poorly enumerated (depth, temperature). + */ +export function prefersDiscreteNumericFilter(column: GeostatsAttribute) { + const keys = numericKeys(column); + if (keys.length === 0) { + return false; + } + return keys.length <= NUMERIC_DISCRETE_VALUE_LIMIT; +} + +export function defaultNumericFilters( + column: GeostatsAttribute +): DataTableFilter[] { + const keys = numericKeys(column); + if (prefersDiscreteNumericFilter(column)) { + const highest = keys[Math.max(keys.length - 1, 0)]?.raw; + if (highest !== undefined) { + return [{ column: column.attribute, op: "eq", value: highest }]; + } + } + const min = + column.min !== undefined && Number.isFinite(column.min) + ? String(column.min) + : keys[0]?.raw; + const max = + column.max !== undefined && Number.isFinite(column.max) + ? String(column.max) + : keys[Math.max(keys.length - 1, 0)]?.raw; + const next: DataTableFilter[] = []; + if (min !== undefined) { + next.push({ column: column.attribute, op: "gte", value: min }); + } + if (max !== undefined) { + next.push({ column: column.attribute, op: "lte", value: max }); + } + return next.length ? next : [{ column: column.attribute, op: "notNull" }]; +} + +function parseNumericFilterState( + filters: DataTableFilter[], + preferDiscrete: boolean +) { + if (filters.some((filter) => filter.op === "isNull")) { + return { + mode: "isNull" as NumericUiMode, + selected: [] as string[], + multi: false, + min: "", + max: "", + }; + } + if (filters.some((filter) => filter.op === "notNull") && filters.length === 1) { + return { + mode: "notNull" as NumericUiMode, + selected: [] as string[], + multi: false, + min: "", + max: "", + }; + } + const inFilter = filters.find((filter) => filter.op === "in"); + if (inFilter) { + return { + mode: "values" as NumericUiMode, + selected: dataTableInFilterValues(inFilter), + multi: true, + min: "", + max: "", + }; + } + const eqFilter = filters.find((filter) => filter.op === "eq"); + if (eqFilter?.value !== undefined && eqFilter.value !== "") { + return { + mode: "values" as NumericUiMode, + selected: [eqFilter.value], + multi: false, + min: "", + max: "", + }; + } + const min = filters.find((filter) => filter.op === "gte")?.value || ""; + const max = filters.find((filter) => filter.op === "lte")?.value || ""; + if (min !== "" || max !== "") { + return { + mode: "range" as NumericUiMode, + selected: [] as string[], + multi: false, + min, + max, + }; + } + return { + mode: (preferDiscrete ? "values" : "range") as NumericUiMode, + selected: [] as string[], + multi: false, + min: "", + max: "", + }; +} + +function emitNumericFilters( + column: string, + mode: NumericUiMode, + selected: string[], + multi: boolean, + min: string, + max: string +): DataTableFilter[] { + if (mode === "isNull") { + return [{ column, op: "isNull" }]; + } + if (mode === "notNull") { + return [{ column, op: "notNull" }]; + } + if (mode === "values") { + if (selected.length === 0) { + return [{ column, op: "eq", value: "" }]; + } + if (multi || selected.length > 1) { + return [{ column, op: "in", values: selected }]; + } + return [{ column, op: "eq", value: selected[0] }]; + } + const next: DataTableFilter[] = []; + if (min !== "") { + next.push({ column, op: "gte", value: min }); + } + if (max !== "") { + next.push({ column, op: "lte", value: max }); + } + return next.length ? next : [{ column, op: "notNull" }]; +} + +function formatNumberLabel(value: string | number) { + const num = typeof value === "number" ? value : Number(value); + if (!Number.isFinite(num)) { + return String(value); + } + // Never add thousands separators — years like 1999 must stay "1999", + // and filter equality values should match the source data exactly. + return num.toLocaleString(undefined, { + maximumFractionDigits: 6, + useGrouping: false, + }); +} + +function parseFiniteNumber(value: string, fallback: number) { + if (value === "") { + return fallback; + } + const num = Number(value); + return Number.isFinite(num) ? num : fallback; +} + +function niceStep(min: number, max: number, preferInteger: boolean) { + const span = Math.abs(max - min); + if (!Number.isFinite(span) || span === 0) { + return preferInteger ? 1 : 0.01; + } + if (preferInteger || (Number.isInteger(min) && Number.isInteger(max))) { + return 1; + } + // Aim for ~100 steps across the span. + const rough = span / 100; + const power = Math.pow(10, Math.floor(Math.log10(rough))); + const normalized = rough / power; + if (normalized <= 1) return power; + if (normalized <= 2) return 2 * power; + if (normalized <= 5) return 5 * power; + return 10 * power; +} + +function clampRange( + nextMin: number, + nextMax: number, + boundMin: number, + boundMax: number +): [number, number] { + let lo = Math.min(nextMin, nextMax); + let hi = Math.max(nextMin, nextMax); + lo = Math.min(Math.max(lo, boundMin), boundMax); + hi = Math.min(Math.max(hi, boundMin), boundMax); + if (lo > hi) { + return [hi, hi]; + } + return [lo, hi]; +} + +function serializeBound(value: number, preferInteger: boolean) { + if (preferInteger) { + return String(Math.round(value)); + } + // Avoid long float noise in query params / labels. + const rounded = Number(value.toPrecision(10)); + return String(rounded); +} + +/** + * Compact single-line numeric filter for the data-table legend. + * Discrete columns (few unique values) default to equality selection; + * continuous columns default to a min/max range. + */ +export default function DataTableNumericFilter({ + column, + filters, + onChange, +}: { + column: GeostatsAttribute; + filters: DataTableFilter[]; + onChange: (filters: DataTableFilter[]) => void; +}) { + const { t } = useTranslation("homepage"); + const preferDiscrete = useMemo( + () => prefersDiscreteNumericFilter(column), + [column] + ); + const parsed = useMemo( + () => parseNumericFilterState(filters, preferDiscrete), + [filters, preferDiscrete] + ); + const [open, setOpen] = useState(false); + const [query, setQuery] = useState(""); + const [mode, setMode] = useState(parsed.mode); + const [multi, setMulti] = useState(parsed.multi); + const [selected, setSelected] = useState(parsed.selected); + const [min, setMin] = useState(parsed.min); + const [max, setMax] = useState(parsed.max); + const [draftRange, setDraftRange] = useState<[number, number] | null>(null); + const searchRef = useRef(null); + const listRef = useRef(null); + const selectedOptionRef = useRef(null); + const pendingTypeRef = useRef(""); + + const allValues = useMemo( + () => + numericKeys(column) + .map((entry) => entry.raw) + .reverse(), + [column] + ); + const bounds = useMemo(() => { + const keys = numericKeys(column); + const boundMin = + column.min !== undefined && Number.isFinite(column.min) + ? column.min + : keys[0]?.num; + const boundMax = + column.max !== undefined && Number.isFinite(column.max) + ? column.max + : keys[Math.max(keys.length - 1, 0)]?.num; + const preferInteger = + (boundMin === undefined || Number.isInteger(boundMin)) && + (boundMax === undefined || Number.isInteger(boundMax)) && + keys.every((entry) => Number.isInteger(entry.num)); + return { + min: boundMin, + max: boundMax, + minLabel: boundMin === undefined ? "" : String(boundMin), + maxLabel: boundMax === undefined ? "" : String(boundMax), + preferInteger, + step: + boundMin !== undefined && boundMax !== undefined + ? niceStep(boundMin, boundMax, preferInteger) + : preferInteger + ? 1 + : 0.01, + }; + }, [column]); + + const filteredValues = useMemo(() => { + const q = query.trim().toLowerCase(); + if (!q) { + return allValues; + } + return allValues.filter((value) => value.toLowerCase().includes(q)); + }, [allValues, query]); + + /** First selected value in list order — used to scroll into view on open. */ + const scrollTargetValue = useMemo(() => { + if (mode !== "values" || selected.length === 0) { + return undefined; + } + return filteredValues.find((value) => selected.includes(value)); + }, [filteredValues, mode, selected]); + + const committedRange = useMemo((): [number, number] | null => { + if (bounds.min === undefined || bounds.max === undefined) { + return null; + } + return clampRange( + parseFiniteNumber(min, bounds.min), + parseFiniteNumber(max, bounds.max), + bounds.min, + bounds.max + ); + }, [bounds.max, bounds.min, max, min]); + + const sliderValue = draftRange || committedRange; + + useEffect(() => { + if (!open) { + setMode(parsed.mode); + setMulti(parsed.multi); + setSelected(parsed.selected); + setMin(parsed.min); + setMax(parsed.max); + setDraftRange(null); + setQuery(""); + pendingTypeRef.current = ""; + } + }, [open, parsed]); + + useEffect(() => { + if (!open) { + return; + } + // Wait for the popover list to lay out, then focus search and bring the + // current selection into view (important for long lists like years). + const frame = window.requestAnimationFrame(() => { + if (mode !== "values") { + return; + } + searchRef.current?.focus(); + if (pendingTypeRef.current) { + setQuery(pendingTypeRef.current); + pendingTypeRef.current = ""; + return; + } + const list = listRef.current; + const item = selectedOptionRef.current; + if (list && item) { + // Scroll only the options list — avoid scrolling the map behind. + const itemOffset = + item.getBoundingClientRect().top - + list.getBoundingClientRect().top + + list.scrollTop; + list.scrollTop = + itemOffset - list.clientHeight / 2 + item.clientHeight / 2; + } + }); + return () => window.cancelAnimationFrame(frame); + }, [open, mode]); + + const commit = ( + nextMode: NumericUiMode, + nextSelected: string[], + nextMulti: boolean, + nextMin: string, + nextMax: string + ) => { + onChange( + emitNumericFilters( + column.attribute, + nextMode, + nextSelected, + nextMulti, + nextMin, + nextMax + ) + ); + }; + + const displayLabel = (() => { + if (mode === "isNull") { + return t("Is blank"); + } + if (mode === "notNull") { + return t("Has a value"); + } + if (mode === "range") { + const live = draftRange || committedRange; + if (live) { + // eslint-disable-next-line i18next/no-literal-string + return `${formatNumberLabel(live[0])} – ${formatNumberLabel(live[1])}`; + } + if (min !== "" && max !== "") { + // eslint-disable-next-line i18next/no-literal-string + return `${formatNumberLabel(min)} – ${formatNumberLabel(max)}`; + } + if (min !== "") { + // eslint-disable-next-line i18next/no-literal-string + return `≥ ${formatNumberLabel(min)}`; + } + if (max !== "") { + // eslint-disable-next-line i18next/no-literal-string + return `≤ ${formatNumberLabel(max)}`; + } + return t("Any range"); + } + if (selected.length === 0) { + return t("Select a value"); + } + if (selected.length === 1) { + return formatNumberLabel(selected[0]); + } + // eslint-disable-next-line i18next/no-literal-string + return `${selected.length} ${t("selected")}`; + })(); + + const selectSingle = (value: string) => { + setMode("values"); + setSelected([value]); + commit("values", [value], false, min, max); + setOpen(false); + }; + + const toggleMultiValue = (value: string) => { + let nextSelected = selected.includes(value) + ? selected.filter((entry) => entry !== value) + : [...selected, value]; + if (nextSelected.length === 0 && allValues[0]) { + nextSelected = [allValues[0]]; + } + setMode("values"); + setSelected(nextSelected); + commit("values", nextSelected, true, min, max); + }; + + const switchToValues = () => { + const nextSelected = + selected.length > 0 + ? selected + : allValues[0] + ? [allValues[0]] + : []; + setMode("values"); + setSelected(nextSelected); + commit("values", nextSelected, multi, min, max); + }; + + const switchToRange = () => { + if (bounds.min === undefined || bounds.max === undefined) { + return; + } + const [nextMinNum, nextMaxNum] = clampRange( + parseFiniteNumber(min, bounds.min), + parseFiniteNumber(max, bounds.max), + bounds.min, + bounds.max + ); + const nextMin = serializeBound(nextMinNum, bounds.preferInteger); + const nextMax = serializeBound(nextMaxNum, bounds.preferInteger); + setMode("range"); + setMin(nextMin); + setMax(nextMax); + setDraftRange(null); + commit("range", selected, multi, nextMin, nextMax); + }; + + const setNullMode = (nextMode: "isNull" | "notNull") => { + setMode(nextMode); + setSelected([]); + setDraftRange(null); + commit(nextMode, [], multi, min, max); + setOpen(false); + }; + + const onMultiToggle = (enabled: boolean) => { + setMulti(enabled); + if (!enabled) { + const nextSelected = selected.slice(0, 1); + setSelected(nextSelected); + if (mode === "values") { + commit("values", nextSelected, false, min, max); + } + return; + } + if (mode === "values") { + commit("values", selected, true, min, max); + } + }; + + const onRangeChange = (nextMin: string, nextMax: string) => { + setMode("range"); + setMin(nextMin); + setMax(nextMax); + setDraftRange(null); + commit("range", selected, multi, nextMin, nextMax); + }; + + const onSliderDrag = (values: number[]) => { + if (bounds.min === undefined || bounds.max === undefined) { + return; + } + const [lo, hi] = clampRange( + values[0], + values[1] ?? values[0], + bounds.min, + bounds.max + ); + setMode("range"); + setDraftRange([lo, hi]); + }; + + const onSliderCommit = (values: number[]) => { + if (bounds.min === undefined || bounds.max === undefined) { + return; + } + const [lo, hi] = clampRange( + values[0], + values[1] ?? values[0], + bounds.min, + bounds.max + ); + const nextMin = serializeBound(lo, bounds.preferInteger); + const nextMax = serializeBound(hi, bounds.preferInteger); + setDraftRange(null); + onRangeChange(nextMin, nextMax); + }; + + const onExactMinChange = (raw: string) => { + if (bounds.min === undefined || bounds.max === undefined) { + onRangeChange(raw, max); + return; + } + if (raw === "") { + onRangeChange("", max); + return; + } + const parsedMin = Number(raw); + if (!Number.isFinite(parsedMin)) { + setMin(raw); + return; + } + const currentMax = parseFiniteNumber(max, bounds.max); + const [lo, hi] = clampRange(parsedMin, currentMax, bounds.min, bounds.max); + onRangeChange( + serializeBound(lo, bounds.preferInteger), + serializeBound(hi, bounds.preferInteger) + ); + }; + + const onExactMaxChange = (raw: string) => { + if (bounds.min === undefined || bounds.max === undefined) { + onRangeChange(min, raw); + return; + } + if (raw === "") { + onRangeChange(min, ""); + return; + } + const parsedMax = Number(raw); + if (!Number.isFinite(parsedMax)) { + setMax(raw); + return; + } + const currentMin = parseFiniteNumber(min, bounds.min); + const [lo, hi] = clampRange(currentMin, parsedMax, bounds.min, bounds.max); + onRangeChange( + serializeBound(lo, bounds.preferInteger), + serializeBound(hi, bounds.preferInteger) + ); + }; + + const onTriggerKeyDown = (event: KeyboardEvent) => { + if ( + event.key.length === 1 && + !event.metaKey && + !event.ctrlKey && + !event.altKey + ) { + pendingTypeRef.current = event.key; + if (mode !== "values") { + setMode("values"); + } + setOpen(true); + } + }; + + return ( + + + + + + { + event.preventDefault(); + if (mode === "values") { + searchRef.current?.focus(); + } + }} + onKeyDown={(event) => { + if (event.key === "Escape") { + setOpen(false); + } + }} + > +
    +
    + + +
    +
    + + {(mode === "isNull" || mode === "notNull") && ( +
    + {mode === "isNull" + ? t("Filtering to blank values. Pick a value or range to switch.") + : t( + "Filtering to any non-blank value. Pick a value or range to switch." + )} +
    + )} + + {mode === "range" || + ((mode === "isNull" || mode === "notNull") && !preferDiscrete) ? ( +
    + {sliderValue && + bounds.min !== undefined && + bounds.max !== undefined ? ( + <> +
    + + {formatNumberLabel(sliderValue[0])} + + + {t("to")} + + + {formatNumberLabel(sliderValue[1])} + +
    + { + if (mode !== "range") { + setMode("range"); + } + }} + > + + + + + + +
    + {formatNumberLabel(bounds.min)} + {formatNumberLabel(bounds.max)} +
    +
    + + +
    + + ) : ( +

    + {t("Range bounds are not available for this column.")} +

    + )} +
    + ) : ( + <> +
    +
    + + setQuery(e.target.value)} + placeholder={t("Search {{count}} options...", { + count: allValues.length, + })} + className="w-full rounded border border-gray-200 bg-gray-50 pl-7 pr-2 py-1 text-xs text-gray-800 placeholder:text-gray-400 focus:outline-none focus:ring-0 focus:border-gray-300 focus-visible:ring-1 focus-visible:ring-primary-500 focus-visible:border-primary-500" + /> +
    +
    +
    + {filteredValues.length === 0 ? ( +

    + {t("No matching values")} +

    + ) : ( + filteredValues.map((value) => { + const isSelected = + mode === "values" && selected.includes(value); + return ( + + ); + }) + )} +
    + + )} + +
    + {mode === "values" && ( + + )} +
    + + +
    +
    +     +     +     + ); +} diff --git a/packages/client/src/dataLayers/DataTableQueryManager.ts b/packages/client/src/dataLayers/DataTableQueryManager.ts new file mode 100644 index 000000000..70d392ba4 --- /dev/null +++ b/packages/client/src/dataLayers/DataTableQueryManager.ts @@ -0,0 +1,429 @@ +import mapboxgl from "mapbox-gl"; +import { + buildDataTableQuerySearchParams, + DataTableQuerySettings, + ParsedDataTableQueryValues, +} from "./dataTableQueryApi"; +import { parseDataTableQueryGroups } from "./dataTableQueryApi"; +import { fetchDataTableColumnStats } from "./useDataTableColumnStats"; +import { ClientOverlayDataTableFragment } from "../generated/graphql"; +import { shouldSendTilesAclNamespace, tilesAclNamespace } from "./tilesAuth"; +import { DATA_TABLE_ZERO_SENTINEL } from "./dataTableMapStyle"; + +/** Catalog row + admin-resolved query, derived on demand from LayerState.dataTable. */ +export type ResolvedDataTableVisualizationSettings = { + table: ClientOverlayDataTableFragment; + query: DataTableQuerySettings; +}; + +/** Lightweight query status for legend bubble scale — not the full values map. */ +export type DataTableLegendSummary = { + loading: boolean; + error?: string; + scaleMin: number; + scaleMax: number; + hasZero: boolean; +}; + +export class DataTableQueryManager { + private mapAccessToken: string | null = null; + private map: mapboxgl.Map | null = null; + /** sourceId → queryKey that was successfully written to feature-state. */ + private appliedQueryKeys = new Map(); + /** + * One feature id per source used to detect whether feature-state survived + * a style update. Style diffs often preserve sources (and their state); + * full source rebuilds do not. + */ + private appliedWitnesses = new Map< + string, + { featureId: string; sourceLayer?: string } + >(); + /** sourceId → legend bubble extents / loading (no per-feature values). */ + private legendSummaries = new Map(); + private onLegendSummaryChange: (() => void) | null = null; + private inFlightRequests = new Map< + string, + { + abortController: AbortController; + queryKey: string; + promise: Promise; + error?: Error; + } + >(); + + constructor(mapAccessToken: string | null) { + this.mapAccessToken = mapAccessToken; + } + + setMapAccessToken(mapAccessToken: string | null) { + this.mapAccessToken = mapAccessToken; + } + + setMap(map: mapboxgl.Map) { + this.map = map; + this.abortAllInFlight(); + this.clearAppliedMarkers(); + } + + /** Notify when legend summaries change (loading / extents / error). */ + setOnLegendSummaryChange(callback: (() => void) | null) { + this.onLegendSummaryChange = callback; + } + + getLegendSummary(sourceId: string): DataTableLegendSummary | undefined { + return this.legendSummaries.get(sourceId); + } + + clearLegendSummary(sourceId: string) { + if (this.legendSummaries.delete(sourceId)) { + this.onLegendSummaryChange?.(); + } + } + + /** + * Drop "already applied" markers (and abort in-flight work). Prefer + * {@link reconcileAppliedState} after style updates — not every + * `setStyle` clears feature-state. + */ + invalidateApplied() { + this.abortAllInFlight(); + this.clearAppliedMarkers(); + } + + private clearAppliedMarkers() { + this.appliedQueryKeys.clear(); + this.appliedWitnesses.clear(); + } + + private abortAllInFlight() { + for (const inFlight of this.inFlightRequests.values()) { + inFlight.abortController.abort(); + } + this.inFlightRequests.clear(); + } + + private publishLegendSummary( + sourceId: string, + summary: DataTableLegendSummary + ) { + this.legendSummaries.set(sourceId, summary); + this.onLegendSummaryChange?.(); + } + + /** + * If we marked this source as applied but the map no longer has our + * feature-state (source removed or rebuilt), clear the marker so the next + * {@link applyFeatureState} will run again. + */ + reconcileAppliedState(sourceId: string, sourceLayerId?: string) { + if (!this.appliedQueryKeys.has(sourceId)) { + return; + } + if (!this.map?.getSource(sourceId)) { + this.appliedQueryKeys.delete(sourceId); + this.appliedWitnesses.delete(sourceId); + return; + } + const witness = this.appliedWitnesses.get(sourceId); + if (!witness) { + this.appliedQueryKeys.delete(sourceId); + return; + } + const state = this.map.getFeatureState({ + source: sourceId, + id: witness.featureId, + ...(sourceLayerId || witness.sourceLayer + ? { sourceLayer: sourceLayerId || witness.sourceLayer } + : {}), + }); + if (!("loading" in state) && !("scaledValue" in state)) { + this.appliedQueryKeys.delete(sourceId); + this.appliedWitnesses.delete(sourceId); + } + } + + /** + * Key that can be used to cache the result of a data table query. + * + * @param url - The URL of the data table query. + * @param query - The query settings for the data table query. + * @returns The key that can be used to cache the result of a data table query. + */ + private getQueryKey(url: string, query: DataTableQuerySettings) { + return JSON.stringify({ + url, + query, + }); + } + + /** + * Fetch query results (relying on browser HTTP cache) and write + * feature-state for every join id. Skips when this queryKey is already + * applied or in flight for `sourceId`. + */ + async applyFeatureState( + sourceId: string, + sourceLayerId: string | undefined, + settings: ResolvedDataTableVisualizationSettings, + tokenRequired: boolean = false + ) { + if (!settings.table.queryUrl) { + throw new Error( + "Query URL is required for data table proportional symbol layer" + ); + } + if (!this.map || (tokenRequired && !this.mapAccessToken)) { + if (tokenRequired) { + console.warn("Map or map access token is not set"); + } + return; + } + const queryKey = this.getQueryKey(settings.table.queryUrl, settings.query); + + if (this.appliedQueryKeys.get(sourceId) === queryKey) { + return; + } + + const inFlight = this.inFlightRequests.get(sourceId); + if (inFlight) { + if (inFlight.queryKey === queryKey) { + return; + } + inFlight.abortController.abort(); + this.inFlightRequests.delete(sourceId); + } + + const map = this.map; + const abortController = new AbortController(); + const { signal } = abortController; + + const promise = (async () => { + const stillCurrent = () => + this.inFlightRequests.get(sourceId)?.abortController === + abortController; + /** Source gone mid-flight (style rebuild) — leave unmarked for retry. */ + const sourceReady = () => Boolean(map.getSource(sourceId)); + + try { + // Mark loading as soon as join ids are known (column-stats, usually cached). + const loadingIds = await this.getFeatureIds(settings, tokenRequired); + if (signal.aborted || !stillCurrent()) { + return; + } + if (!sourceReady()) { + // Style still settling; caller will schedule another sync. Don't + // publish loading:true here or the legend can stick with nothing + // in flight after this request is cleared. + return; + } + + const previousSummary = this.legendSummaries.get(sourceId); + this.publishLegendSummary(sourceId, { + loading: true, + scaleMin: previousSummary?.scaleMin ?? 0, + scaleMax: previousSummary?.scaleMax ?? 0, + hasZero: previousSummary?.hasZero ?? false, + }); + + for (const id of loadingIds) { + map.setFeatureState( + { + source: sourceId, + sourceLayer: sourceLayerId, + id, + }, + { loading: true } + ); + } + + const parsed = await this.query( + settings.table, + settings.query, + tokenRequired, + signal + ); + if (signal.aborted || !stillCurrent()) { + return; + } + if (!sourceReady()) { + return; + } + + // Normalize positive values 0–1 against the positive extents + // (scaleMin/scaleMax) so map radii line up with the legend's bubble + // scale numbers. True zeros get a sentinel so paint expressions can + // give them distinct symbology (smallest positive also scales to 0). + const scaleMin = parsed.scaleMin; + const range = parsed.scaleMax - scaleMin; + const scaleValue = (value: number) => { + if (value === 0) { + return DATA_TABLE_ZERO_SENTINEL; + } + if (value < 0) { + return 0; + } + if (range === 0) { + // Single unique positive value renders at full size, matching + // the legend's single-bubble presentation. + return 1; + } + return Math.min(Math.max((value - scaleMin) / range, 0), 1); + }; + + const featureIds = await this.getFeatureIds(settings, tokenRequired); + if (signal.aborted || !stillCurrent()) { + return; + } + if (!sourceReady()) { + return; + } + + for (const id of featureIds) { + const value = id in parsed.values ? parsed.values[id] : null; + map.setFeatureState( + { + source: sourceId, + sourceLayer: sourceLayerId, + id, + }, + { + loading: false, + scaledValue: value !== null ? scaleValue(value) : null, + rawValue: value, + } + ); + } + + if (stillCurrent()) { + this.appliedQueryKeys.set(sourceId, queryKey); + if (featureIds.length > 0) { + this.appliedWitnesses.set(sourceId, { + featureId: featureIds[0], + sourceLayer: sourceLayerId, + }); + } else { + this.appliedWitnesses.delete(sourceId); + } + this.publishLegendSummary(sourceId, { + loading: false, + scaleMin: parsed.scaleMin, + scaleMax: parsed.scaleMax, + hasZero: parsed.hasZero, + }); + } + } catch (error) { + if ( + signal.aborted || + (error instanceof DOMException && error.name === "AbortError") + ) { + return; + } + console.error(error); + if (stillCurrent()) { + const current = this.inFlightRequests.get(sourceId); + if (current) { + current.error = error as Error; + } + const prev = this.legendSummaries.get(sourceId); + this.publishLegendSummary(sourceId, { + loading: false, + error: error instanceof Error ? error.message : String(error), + scaleMin: prev?.scaleMin ?? 0, + scaleMax: prev?.scaleMax ?? 0, + hasZero: prev?.hasZero ?? false, + }); + } + } finally { + if (stillCurrent()) { + this.inFlightRequests.delete(sourceId); + } + } + })(); + + this.inFlightRequests.set(sourceId, { + abortController, + queryKey, + promise, + }); + } + + /** + * Fetches and parses query results. Always resolves to parsed values, or + * rejects (including AbortError when `signal` is aborted). + */ + private async query( + table: ClientOverlayDataTableFragment, + query: DataTableQuerySettings, + tokenRequired: boolean, + signal: AbortSignal + ): Promise { + const url = new URL(table.queryUrl!); + const params = buildDataTableQuerySearchParams({ + ...query, + groupBy: table.joinColumn, + }); + params.set("f", "json"); + if (tokenRequired) { + params.set("access_token", this.mapAccessToken!); + } + if (shouldSendTilesAclNamespace()) { + params.set("ns", tilesAclNamespace()); + } + url.search = params.toString(); + const response = await fetch(url.toString(), { + headers: { + accept: "application/json", + }, + signal, + }); + if (!response.ok) { + throw new Error( + `Failed to fetch data table query: ${response.statusText}` + ); + } + const data = await response.json(); + if (signal.aborted) { + throw new DOMException("AbortError", "AbortError"); + } + // Group keys in the response use the table join column (groupBy), not + // the overlay property name. Values still match promoteId / feature ids. + return parseDataTableQueryGroups(data.groups, table.joinColumn, query.op); + } + + /** + * Returns an exhaustive list of feature IDs, derived from column stats and + * referencing the overlay join column. Can be used to set loading state and + * proportional symbol values on all features of a data-table layer. + * @param settings - The resolved data table visualization settings. + * @returns A list of feature IDs (strings). + */ + private async getFeatureIds( + settings: ResolvedDataTableVisualizationSettings, + tokenRequired: boolean = false + ) { + if (!settings.table.columnStatsUrl) { + throw new Error( + "Column stats URL is required for data table proportional symbol layer" + ); + } + const columnStats = await fetchDataTableColumnStats( + settings.table.columnStatsUrl, + tokenRequired ? this.mapAccessToken : null + ); + if (!columnStats) { + throw new Error("Failed to fetch column stats"); + } + // column-stats describes the data table, so look up joinColumn (not the + // overlay property name). Distinct values are the feature ids used with + // promoteId / setFeatureState. + const column = columnStats.columns.find( + (column) => column.attribute === settings.table.joinColumn + ); + if (!column) { + throw new Error("Join column not found in column stats"); + } + const featureIds = Object.keys(column.values); + return featureIds; + } +} diff --git a/packages/client/src/dataLayers/DataTableStringFilter.tsx b/packages/client/src/dataLayers/DataTableStringFilter.tsx new file mode 100644 index 000000000..dfc248765 --- /dev/null +++ b/packages/client/src/dataLayers/DataTableStringFilter.tsx @@ -0,0 +1,398 @@ +import { + KeyboardEvent, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import { useTranslation } from "react-i18next"; +import { + CaretDownIcon, + CheckIcon, + MagnifyingGlassIcon, +} from "@radix-ui/react-icons"; +import * as Popover from "@radix-ui/react-popover"; +import { GeostatsAttribute } from "@seasketch/geostats-types"; +import { + DataTableFilter, + dataTableInFilterValues, +} from "./dataTableQueryApi"; +import clsx from "clsx"; + +type StringFilterMode = "value" | "isNull" | "notNull"; + +function parseStringFilterState(filters: DataTableFilter[]) { + if (filters.some((filter) => filter.op === "isNull")) { + return { + mode: "isNull" as StringFilterMode, + selected: [] as string[], + multi: false, + }; + } + if (filters.some((filter) => filter.op === "notNull")) { + return { + mode: "notNull" as StringFilterMode, + selected: [] as string[], + multi: false, + }; + } + const inFilter = filters.find((filter) => filter.op === "in"); + if (inFilter) { + return { + mode: "value" as StringFilterMode, + selected: dataTableInFilterValues(inFilter), + multi: true, + }; + } + const eqFilter = filters.find((filter) => filter.op === "eq"); + if (eqFilter?.value) { + return { + mode: "value" as StringFilterMode, + selected: [eqFilter.value], + multi: false, + }; + } + return { + mode: "value" as StringFilterMode, + selected: [] as string[], + multi: false, + }; +} + +function emitStringFilters( + column: string, + mode: StringFilterMode, + selected: string[], + multi: boolean +): DataTableFilter[] { + if (mode === "isNull") { + return [{ column, op: "isNull" }]; + } + if (mode === "notNull") { + return [{ column, op: "notNull" }]; + } + if (selected.length === 0) { + // Callers should avoid committing an empty value selection. + return [{ column, op: "eq", value: "" }]; + } + if (multi || selected.length > 1) { + return [{ column, op: "in", values: selected }]; + } + return [{ column, op: "eq", value: selected[0] }]; +} + +function alphabetize(values: string[]) { + return [...values].sort((a, b) => + a.localeCompare(b, undefined, { sensitivity: "base" }) + ); +} + +/** + * Compact single-line string filter for the data-table legend. + * Trigger shows the current selection; popover handles search, single/multi + * equality, and null / has-value modes. + */ +export default function DataTableStringFilter({ + column, + filters, + onChange, +}: { + column: GeostatsAttribute; + filters: DataTableFilter[]; + onChange: (filters: DataTableFilter[]) => void; +}) { + const { t } = useTranslation("homepage"); + const parsed = useMemo(() => parseStringFilterState(filters), [filters]); + const [open, setOpen] = useState(false); + const [query, setQuery] = useState(""); + const [multi, setMulti] = useState(parsed.multi); + const [mode, setMode] = useState(parsed.mode); + const [selected, setSelected] = useState(parsed.selected); + const searchRef = useRef(null); + const listRef = useRef(null); + const selectedOptionRef = useRef(null); + const pendingTypeRef = useRef(""); + + const allValues = useMemo( + () => alphabetize(Object.keys(column.values || {})), + [column.values] + ); + + const filteredValues = useMemo(() => { + const q = query.trim().toLowerCase(); + if (!q) { + return allValues; + } + return allValues.filter((value) => value.toLowerCase().includes(q)); + }, [allValues, query]); + + /** First selected value in list order — used to scroll into view on open. */ + const scrollTargetValue = useMemo(() => { + if (mode !== "value" || selected.length === 0) { + return undefined; + } + return filteredValues.find((value) => selected.includes(value)); + }, [filteredValues, mode, selected]); + + useEffect(() => { + if (!open) { + setMulti(parsed.multi); + setMode(parsed.mode); + setSelected(parsed.selected); + setQuery(""); + pendingTypeRef.current = ""; + } + }, [open, parsed]); + + useEffect(() => { + if (!open) { + return; + } + // Wait for the popover list to lay out, then focus search and bring the + // current selection into view (important for long lists like years). + const frame = window.requestAnimationFrame(() => { + searchRef.current?.focus(); + if (pendingTypeRef.current) { + setQuery(pendingTypeRef.current); + pendingTypeRef.current = ""; + return; + } + const list = listRef.current; + const item = selectedOptionRef.current; + if (list && item) { + // Scroll only the options list — avoid scrolling the map behind. + const itemOffset = + item.getBoundingClientRect().top - + list.getBoundingClientRect().top + + list.scrollTop; + list.scrollTop = + itemOffset - list.clientHeight / 2 + item.clientHeight / 2; + } + }); + return () => window.cancelAnimationFrame(frame); + }, [open]); + + const commit = ( + nextMode: StringFilterMode, + nextSelected: string[], + nextMulti: boolean + ) => { + onChange( + emitStringFilters(column.attribute, nextMode, nextSelected, nextMulti) + ); + }; + + const displayLabel = (() => { + if (mode === "isNull") { + return t("Is blank"); + } + if (mode === "notNull") { + return t("Has a value"); + } + if (selected.length === 0) { + return t("Select a value"); + } + if (selected.length === 1) { + return selected[0]; + } + // eslint-disable-next-line i18next/no-literal-string + return `${selected.length} ${t("selected")}`; + })(); + + const selectSingle = (value: string) => { + const nextSelected = [value]; + setMode("value"); + setSelected(nextSelected); + commit("value", nextSelected, false); + setOpen(false); + }; + + const toggleMultiValue = (value: string) => { + let nextSelected = selected.includes(value) + ? selected.filter((entry) => entry !== value) + : [...selected, value]; + // Keep at least one value selected in multi mode so we don't fall back + // to a "has a value" filter. + if (nextSelected.length === 0 && allValues[0]) { + nextSelected = [allValues[0]]; + } + setMode("value"); + setSelected(nextSelected); + commit("value", nextSelected, true); + }; + + const setNullMode = (nextMode: "isNull" | "notNull") => { + setMode(nextMode); + setSelected([]); + commit(nextMode, [], multi); + setOpen(false); + }; + + const onMultiToggle = (enabled: boolean) => { + setMulti(enabled); + if (!enabled) { + const nextSelected = selected.slice(0, 1); + setSelected(nextSelected); + if (mode === "value") { + commit("value", nextSelected, false); + } + return; + } + if (mode === "value") { + commit("value", selected, true); + } + }; + + const onTriggerKeyDown = (event: KeyboardEvent) => { + if (event.key.length === 1 && !event.metaKey && !event.ctrlKey && !event.altKey) { + pendingTypeRef.current = event.key; + setOpen(true); + } + }; + + return ( + + + + + + { + event.preventDefault(); + searchRef.current?.focus(); + }} + onKeyDown={(event) => { + if (event.key === "Escape") { + setOpen(false); + } + }} + > +
    +
    + + setQuery(e.target.value)} + placeholder={t("Search {{count}} options...", { + count: allValues.length, + })} + className="w-full rounded border border-gray-200 bg-gray-50 pl-7 pr-2 py-1 text-xs text-gray-800 placeholder:text-gray-400 focus:outline-none focus:ring-0 focus:border-gray-300 focus-visible:ring-1 focus-visible:ring-primary-500 focus-visible:border-primary-500" + /> +
    +
    + + {mode !== "value" && ( +
    + {mode === "isNull" + ? t("Filtering to blank values. Pick a value to switch.") + : t("Filtering to any non-blank value. Pick a value to switch.")} +
    + )} + +
    + {filteredValues.length === 0 ? ( +

    + {t("No matching values")} +

    + ) : ( + filteredValues.map((value) => { + const isSelected = mode === "value" && selected.includes(value); + return ( + + ); + }) + )} +
    + +
    + +
    + + +
    +
    +     +     +     + ); +} diff --git a/packages/client/src/dataLayers/DataTableVisualizationControls.tsx b/packages/client/src/dataLayers/DataTableVisualizationControls.tsx new file mode 100644 index 000000000..e5fb0d090 --- /dev/null +++ b/packages/client/src/dataLayers/DataTableVisualizationControls.tsx @@ -0,0 +1,309 @@ +import { useContext, useEffect, useMemo } from "react"; +import { CaretDownIcon, CheckIcon } from "@radix-ui/react-icons"; +import * as Select from "@radix-ui/react-select"; +import { useTranslation } from "react-i18next"; +import { MapManagerContext, MapOverlayContext } from "./MapContextManager"; +import { + DATA_TABLE_AGGREGATIONS, + DataTableAggregation, + DataTableVisualizationMetadata, + pickDefaultDataTableColumn, + resolveDataTableVisualizationSettings, +} from "./dataTableQueryApi"; +import { + DataTableColumnStatsState, + columnStatsUrlForTable, + numericColumnNames, +} from "./useDataTableColumnStats"; +import clsx from "clsx"; + +export function DataTableVisualizationLabel({ + op, + column, + className, +}: { + op: DataTableAggregation; + column?: string; + className?: string; +}) { + const { t } = useTranslation("homepage"); + if (op === "count" && !column) { + return {t("Count")}; + } + if (column) { + return ( + + {t("{{op}} of {{column}}", { op, column })} + + ); + } + return {op}; +} + +/** + * Compact non-native select styled like the filter value chips. + * Options can later grow a `description` field without changing the trigger. + */ +function InlineSelect({ + value, + options, + onChange, + ariaLabel, +}: { + value: string; + options: { value: string; label: string; description?: string }[]; + onChange: (value: string) => void; + ariaLabel: string; +}) { + const selected = options.find((option) => option.value === value); + return ( + + + + {selected?.label || value} + + + + + + + + + {options.map((option) => ( + +
    + + + {option.label} + + + {option.description && ( + + {option.description} + + )} +
    + + + +     + ))} +     +     +     +     + ); +} + +/** + * Inline column/op controls for an activated data table visualization. + * Renders as a compact subheading, e.g. "▼ mean of ▼ count". + */ +export default function DataTableVisualizationControls({ + layerId, + metadata, + columnStatsState, + metadataLoading, +}: { + layerId: string; + metadata: DataTableVisualizationMetadata; + columnStatsState?: DataTableColumnStatsState; + metadataLoading?: boolean; +}) { + const { t } = useTranslation("homepage"); + const { manager } = useContext(MapManagerContext); + const { layerStatesByTocStaticId } = useContext(MapOverlayContext); + const dataTable = layerStatesByTocStaticId[layerId]?.dataTable; + + const allowedColumns = useMemo( + () => (metadata.visualizationColumns || []).filter(Boolean) as string[], + [metadata.visualizationColumns] + ); + const allowedOps = useMemo( + () => + (metadata.visualizationOps || []).filter( + Boolean + ) as DataTableAggregation[], + [metadata.visualizationOps] + ); + const columnStats = columnStatsState?.columnStats; + const error = columnStatsState?.error; + // Prefer GraphQL columnStatsUrl; fall back to rewriting queryUrl + // (/query → /column-stats.json) when the API host env is missing. + const resolvedColumnStatsUrl = columnStatsUrlForTable(metadata); + const loading = Boolean( + metadataLoading || + columnStatsState?.loading || + (resolvedColumnStatsUrl && !columnStatsState) + ); + + // Empty admin visualizationColumns ⇒ all numeric columns are valid choices. + const columnChoices: string[] = useMemo( + () => + allowedColumns.length > 0 + ? allowedColumns + : numericColumnNames(columnStats), + [allowedColumns, columnStats] + ); + const opChoices = + allowedOps.length > 0 ? allowedOps : DATA_TABLE_AGGREGATIONS; + + const userChoice = useMemo( + () => ({ + column: dataTable?.column, + op: dataTable?.op, + filters: dataTable?.filters, + }), + [dataTable?.column, dataTable?.filters, dataTable?.op] + ); + const resolved = resolveDataTableVisualizationSettings(metadata, userChoice); + const defaultColumn = + resolved.column || + (allowedColumns.length > 0 + ? columnChoices[0] + : pickDefaultDataTableColumn(columnChoices)); + const effectiveColumn = userChoice.column || defaultColumn; + const showColumn = resolved.op !== "count" || Boolean(effectiveColumn); + const tableStableId = dataTable?.stableId; + + // Persist a default column/op once column-stats (or admin constraints) + // make one available, so the map query can start without a separate + // settings step. Call the manager directly — don't route through a + // render-scoped helper omitted from the dependency list. + useEffect(() => { + if (loading || error || !effectiveColumn || !tableStableId || !manager) { + return; + } + if ( + userChoice.column === effectiveColumn && + userChoice.op === resolved.op + ) { + return; + } + manager.setLayerDataTable(layerId, { + stableId: tableStableId, + column: effectiveColumn, + op: resolved.op, + filters: userChoice.filters, + }); + }, [ + manager, + layerId, + tableStableId, + effectiveColumn, + resolved.op, + userChoice.column, + userChoice.op, + userChoice.filters, + loading, + error, + ]); + + const showColumnSelect = columnChoices.length > 1; + const showOpSelect = opChoices.length > 1; + const opOptions = opChoices.map((op) => ({ value: op, label: op })); + const columnOptions = columnChoices.map((column) => ({ + value: column, + label: column, + })); + + return ( +
    +
    + {t("Showing")} + {showOpSelect ? ( + { + if (!tableStableId) return; + manager?.setLayerDataTable(layerId, { + stableId: tableStableId, + column: userChoice.column, + op: value as DataTableAggregation, + filters: userChoice.filters, + }); + }} + /> + ) : ( + {resolved.op} + )} + {showColumn && ( + <> + {t("of")} + {showColumnSelect ? ( + { + if (!tableStableId) return; + manager?.setLayerDataTable(layerId, { + stableId: tableStableId, + column: value, + op: userChoice.op, + filters: userChoice.filters, + }); + }} + /> + ) : ( + + {effectiveColumn} + + )} + + )} +
    + {allowedColumns.length === 0 && loading && ( +

    + {t("Loading column metadata...")} +

    + )} + {allowedColumns.length === 0 && !loading && error && ( +

    + {t("Unable to load column metadata.")} +

    + )} + {allowedColumns.length === 0 && + !loading && + !error && + !resolvedColumnStatsUrl && ( +

    + {t("Column metadata is not available for this table.")} +

    + )} + {allowedColumns.length === 0 && + !loading && + !error && + resolvedColumnStatsUrl && + columnChoices.length === 0 && ( +

    + {t("No numeric columns available to visualize.")} +

    + )} +
    + ); +} diff --git a/packages/client/src/dataLayers/LayerInteractivityManager.ts b/packages/client/src/dataLayers/LayerInteractivityManager.ts index f0a28cfbe..1af229d1b 100644 --- a/packages/client/src/dataLayers/LayerInteractivityManager.ts +++ b/packages/client/src/dataLayers/LayerInteractivityManager.ts @@ -41,6 +41,16 @@ export type InteractivityUIUpdate = Partial<{ sidebarPopupTitle: string | undefined; }>; +/** Active data-table proportional-symbol layer for value tooltips. */ +export type DataTableInteractiveLayer = { + /** Mapbox GL style layer id (`idForLayer(layer, 0)`). */ + glLayerId: string; + sourceId: string; + sourceLayer?: string; + column: string; + op: string; +}; + /** * LayerInteractivityManager works in tandem with the MapContextManager to react * to user interaction which clicks, hovers, or otherwise interacts with map content @@ -65,6 +75,10 @@ export default class LayerInteractivityManager extends EventEmitter { private popupAbortController: AbortController | undefined; private interactiveVectorLayerIds: string[] = []; private interactiveImageLayerIds: string[] = []; + /** glLayerId → data-table value tooltip config (independent of admin interactivity). */ + private dataTableLayersByGlId: { + [glLayerId: string]: DataTableInteractiveLayer; + } = {}; private basemap: BasemapDetailsFragment | undefined; private sketchLayerIds: string[] = []; private focusedSketchId?: number; @@ -109,17 +123,43 @@ export default class LayerInteractivityManager extends EventEmitter { setSketchLayerIds(ids: string[]) { this.sketchLayerIds = ids; + this.syncMouseMoveListener(); + } + + setFocusedSketchId(id: number | null) { + this.focusedSketchId = id || undefined; + } + + /** + * Layers with an active data-table visualization. Hover shows a value + * tooltip from `rawValue` feature-state. Works alongside popup / sidebar / + * banner interactivity; overrides an admin Tooltip (and SidebarOverlay + * short-template hover tooltip) because both compete for the same UI slot. + */ + setDataTableLayers(layers: DataTableInteractiveLayer[]) { + const next: { [glLayerId: string]: DataTableInteractiveLayer } = {}; + for (const layer of layers) { + next[layer.glLayerId] = layer; + } + this.dataTableLayersByGlId = next; + this.syncMouseMoveListener(); + } + + private syncMouseMoveListener() { this.map.off("mousemove", this.debouncedMouseMoveListener); if ( this.interactiveVectorLayerIds.length > 0 || - this.sketchLayerIds.length > 0 + this.sketchLayerIds.length > 0 || + this.inaturalistConfigs.length > 0 || + Object.keys(this.dataTableLayersByGlId).length > 0 ) { this.map.on("mousemove", this.debouncedMouseMoveListener); } } - setFocusedSketchId(id: number | null) { - this.focusedSketchId = id || undefined; + /** Layer ids that currently exist on the map (queryRenderedFeatures throws otherwise). */ + private existingLayerIds(ids: string[]) { + return ids.filter((id) => Boolean(this.map.getLayer(id))); } /** @@ -226,14 +266,7 @@ export default class LayerInteractivityManager extends EventEmitter { } this.layers = newActiveLayers; this.imageSources = newActiveImageSources; - this.map.off("mousemove", this.debouncedMouseMoveListener); - if ( - this.interactiveVectorLayerIds.length > 0 || - this.sketchLayerIds.length > 0 || - this.inaturalistConfigs.length > 0 - ) { - this.map.on("mousemove", this.debouncedMouseMoveListener); - } + this.syncMouseMoveListener(); } /** @@ -349,12 +382,20 @@ export default class LayerInteractivityManager extends EventEmitter { this.popupAbortController.abort(); delete this.popupAbortController; } - const sketchFeatures = this.map!.queryRenderedFeatures(e.point, { - layers: this.sketchLayerIds || [], - }); - const features = this.map!.queryRenderedFeatures(e.point, { - layers: this.interactiveVectorLayerIds, - }); + const sketchLayerIds = this.existingLayerIds(this.sketchLayerIds || []); + const vectorLayerIds = this.existingLayerIds( + this.interactiveVectorLayerIds + ); + const sketchFeatures = sketchLayerIds.length + ? this.map!.queryRenderedFeatures(e.point, { + layers: sketchLayerIds, + }) + : []; + const features = vectorLayerIds.length + ? this.map!.queryRenderedFeatures(e.point, { + layers: vectorLayerIds, + }) + : []; const allFeatures = [...sketchFeatures, ...features]; sortFeaturesByLayer(allFeatures, this.map.getStyle()); const top = allFeatures[0]; @@ -775,9 +816,12 @@ export default class LayerInteractivityManager extends EventEmitter { const inatDrawIndex = inatHit ? inatHit.layerIndex : -Infinity; // First, check sketch layers - const sketchFeatures = this.map!.queryRenderedFeatures(e.point, { - layers: this.sketchLayerIds, - }); + const sketchLayerIds = this.existingLayerIds(this.sketchLayerIds); + const sketchFeatures = sketchLayerIds.length + ? this.map!.queryRenderedFeatures(e.point, { + layers: sketchLayerIds, + }) + : []; if (sketchFeatures.length) { clear(); if ( @@ -788,15 +832,34 @@ export default class LayerInteractivityManager extends EventEmitter { } return; } - const layerIds = this.interactiveVectorLayerIds; - const features = this.map!.queryRenderedFeatures(e.point, { - layers: layerIds, + const dataTableLayerIds = Object.keys(this.dataTableLayersByGlId); + const layerIds = this.existingLayerIds([ + ...dataTableLayerIds, + ...this.interactiveVectorLayerIds, + ]); + // Deduplicate while preserving order (data-table ids first for lookup). + const seen = new Set(); + const uniqueLayerIds = layerIds.filter((id) => { + if (seen.has(id)) { + return false; + } + seen.add(id); + return true; }); + const features = uniqueLayerIds.length + ? this.map!.queryRenderedFeatures(e.point, { + layers: uniqueLayerIds, + }) + : []; let topVectorIndex = -Infinity; - if (features.length && layerIds.indexOf(features[0].layer.id) > -1) { + if (features.length && uniqueLayerIds.indexOf(features[0].layer.id) > -1) { const top = features[0]; topVectorIndex = this.getLayerDrawIndex(top.layer.id); this.setHoveredFeature(top); + const dataTableConfig = this.dataTableLayersByGlId[top.layer.id]; + const dataTableTooltip = dataTableConfig + ? this.buildDataTableValueTooltip(top, dataTableConfig, e) + : undefined; const interactivitySetting = this.getInteractivitySettingForFeature(top); if (interactivitySetting) { let cursor = ""; @@ -813,25 +876,33 @@ export default class LayerInteractivityManager extends EventEmitter { ...(top.properties || {}), }), ]; + // Data-table value tooltip sits alongside the banner. + tooltip = dataTableTooltip; break; case InteractivityType.Tooltip: cursor = "default"; - tooltip = { - x: e.originalEvent.x, - y: e.originalEvent.y, - messages: [ - Mustache.render(interactivitySetting.shortTemplate || "", { - ...mustacheHelpers, - ...(top.properties || {}), - }), - ], - }; + // Data-table value wins over an admin Tooltip template. + tooltip = + dataTableTooltip ?? + ({ + x: e.originalEvent.x, + y: e.originalEvent.y, + messages: [ + Mustache.render(interactivitySetting.shortTemplate || "", { + ...mustacheHelpers, + ...(top.properties || {}), + }), + ], + } as Tooltip); break; case InteractivityType.Popup: case InteractivityType.AllPropertiesPopup: case InteractivityType.SidebarOverlay: cursor = "pointer"; - if ( + if (dataTableTooltip) { + // Hover value inspection alongside click popup / sidebar. + tooltip = dataTableTooltip; + } else if ( interactivitySetting.type === InteractivityType.SidebarOverlay && interactivitySetting.shortTemplate?.length ) { @@ -855,8 +926,10 @@ export default class LayerInteractivityManager extends EventEmitter { ...(top.properties || {}), }), ]; + tooltip = dataTableTooltip; break; default: + tooltip = dataTableTooltip; break; } if (interactivitySetting.cursor !== "AUTO") { @@ -864,8 +937,11 @@ export default class LayerInteractivityManager extends EventEmitter { } this.map!.getCanvas().style.cursor = cursor; const currentInteractionTarget = `${top.id}-${interactivitySetting.id}`; + // Skip redundant banner/popup updates for the same feature — but always + // refresh when a data-table tooltip needs to follow the pointer. if ( this.previousInteractionTarget === currentInteractionTarget && + !dataTableTooltip && (interactivitySetting.type === InteractivityType.Banner || interactivitySetting.type === InteractivityType.FixedBlock || interactivitySetting.type === InteractivityType.Popup || @@ -880,6 +956,14 @@ export default class LayerInteractivityManager extends EventEmitter { fixedBlocks, }); } + } else if (dataTableTooltip) { + this.map!.getCanvas().style.cursor = "default"; + this.previousInteractionTarget = `${top.id}-data-table`; + this.updateUI({ + bannerMessages: [], + tooltip: dataTableTooltip, + fixedBlocks: [], + }); } else { clear(); } @@ -893,6 +977,48 @@ export default class LayerInteractivityManager extends EventEmitter { } }; + /** + * Hover tooltip for a data-table proportional symbol, using `rawValue` + * feature-state written by DataTableQueryManager. + */ + private buildDataTableValueTooltip( + feature: MapboxGeoJSONFeature, + config: DataTableInteractiveLayer, + e: MapMouseEvent + ): Tooltip | undefined { + if (feature.id === undefined || feature.id === null) { + return undefined; + } + const state = this.map.getFeatureState({ + source: config.sourceId, + id: feature.id, + ...(config.sourceLayer ? { sourceLayer: config.sourceLayer } : {}), + }) as { + rawValue?: number | null; + loading?: boolean; + scaledValue?: number | null; + }; + const position = { + x: e.originalEvent.x, + y: e.originalEvent.y, + }; + if (state.loading) { + return { ...position, messages: ["Loading…"] }; + } + if (!("rawValue" in state) && !("scaledValue" in state)) { + // Feature-state not applied yet (style still settling). + return undefined; + } + if (state.rawValue === null || state.rawValue === undefined) { + return { ...position, messages: ["No data"] }; + } + const formatted = PopupNumberFormatter.format(state.rawValue); + return { + ...position, + messages: [`${config.op}(${config.column}): ${formatted}`], + }; + } + private async getTopInaturalistHit(e: MapMouseEvent) { if (!this.inaturalistConfigs.length) { return null; diff --git a/packages/client/src/dataLayers/LayerStateManager.dataTable.test.ts b/packages/client/src/dataLayers/LayerStateManager.dataTable.test.ts new file mode 100644 index 000000000..919f083e6 --- /dev/null +++ b/packages/client/src/dataLayers/LayerStateManager.dataTable.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "@jest/globals"; +import { LayerStateManager } from "./LayerStateManager"; +import { LayerState } from "./MapContextManager"; + +describe("LayerStateManager dataTable regression", () => { + it("preserves dataTable across visibility and loading updates", () => { + const manager = new LayerStateManager("overlays"); + const dataTable = { + stableId: "table-uuid", + column: "biomass", + op: "sum" as const, + }; + + manager.addLayer("toc-1", { + visible: true, + loading: false, + }); + manager.patch("toc-1", { dataTable }); + + const afterPatch = manager.getState()["toc-1"]; + expect(afterPatch.dataTable).toEqual(dataTable); + const dataTableRef = afterPatch.dataTable; + + manager.setVisible("toc-1", false); + manager.setLoading("toc-1", true); + manager.setLoading("toc-1", false); + manager.setVisible("toc-1", true); + + const after = manager.getState()["toc-1"]; + expect(after.dataTable).toBe(dataTableRef); + expect(after.dataTable).toEqual(dataTable); + expect(after.visible).toBe(true); + expect(after.loading).toBe(false); + }); + + it("replacing dataTable via patch updates the object reference", () => { + const manager = new LayerStateManager("overlays"); + manager.addLayer("toc-1", { visible: true, loading: false }); + manager.patch("toc-1", { + dataTable: { stableId: "a", column: "x" }, + }); + const first = manager.getState()["toc-1"].dataTable; + manager.patch("toc-1", { + dataTable: { stableId: "a", column: "y" }, + }); + const second = manager.getState()["toc-1"].dataTable; + expect(second).not.toBe(first); + expect(second?.column).toBe("y"); + }); +}); diff --git a/packages/client/src/dataLayers/Legend.tsx b/packages/client/src/dataLayers/Legend.tsx index d06e5ff00..cd2633ccb 100644 --- a/packages/client/src/dataLayers/Legend.tsx +++ b/packages/client/src/dataLayers/Legend.tsx @@ -43,6 +43,16 @@ import { SketchClassMenuItemType, } from "./SketchClassItemMenu"; import InaturalistLegendPanel from "./legends/INaturalistLegend"; +import ActivatedDataTableButton from "./ActivatedDataTableButton"; +import DataTableLegendPanel from "./legends/DataTableLegendPanel"; +import { ClientOverlayDataTableFragment } from "../generated/graphql"; +import { DataTableAggregation } from "./dataTableQueryApi"; +import useCurrentProjectMetadata from "../useCurrentProjectMetadata"; + +export interface LegendFocusRequest { + layerId: string; + requestId: number; +} require("../admin/data/arcgis/Accordion.css"); @@ -51,6 +61,8 @@ interface BaseLegendItem { label: string; tableOfContentsItemDetails?: TocMenuItemType; isSketchClass?: boolean; + /** Data tables available for use in a thematic map or other visualization of this layer */ + overlayDataTables?: ClientOverlayDataTableFragment[]; } interface CustomGLSourceSymbolLegend extends BaseLegendItem { @@ -77,10 +89,27 @@ export type InaturalistLegendItem = BaseLegendItem & { }; }; +export type DataTableLegendItem = BaseLegendItem & { + type: "DataTableLegendItem"; + tableStableId: string; + tableName: string; + column?: string; + op: DataTableAggregation; + min: number; + max: number; + hasZero?: boolean; + showValueScale?: boolean; + /** Query in flight for circle values / bubble scale extents. */ + loading?: boolean; + /** Failed to load circle values / bubble scale extents. */ + error?: string; +}; + export type LegendItem = | GLStyleLegendItem | CustomGLSourceSymbolLegend - | InaturalistLegendItem; + | InaturalistLegendItem + | DataTableLegendItem; const PANEL_WIDTH = 180; @@ -99,6 +128,9 @@ export type LegendProps = { editable?: boolean; /** Default visibility state of the legend. Only used if no persisted state exists. Defaults to true (hidden) */ defaultToHidden?: boolean; + legendFocusRequest?: LegendFocusRequest | null; + onLegendFocusComplete?: () => void; + onDataTableActivated?: (layerId: string) => void; }; export default function Legend({ @@ -113,6 +145,9 @@ export default function Legend({ persistedStateKey, editable, defaultToHidden = true, + legendFocusRequest, + onLegendFocusComplete, + onDataTableActivated, }: LegendProps) { const { t } = useTranslation("homepage"); maxHeight = maxHeight || undefined; @@ -122,6 +157,23 @@ export default function Legend({ true ); + useEffect(() => { + if (!legendFocusRequest) { + return; + } + setHidden(false); + const frame = window.requestAnimationFrame(() => { + const element = document.getElementById( + `legend-item-${legendFocusRequest.layerId}` + ); + if (element) { + element.scrollIntoView({ behavior: "smooth", block: "nearest" }); + } + onLegendFocusComplete?.(); + }); + return () => window.cancelAnimationFrame(frame); + }, [legendFocusRequest, onLegendFocusComplete, setHidden]); + const layerEditingContext = useContext(LayerEditingContext); return (
    ); })} @@ -205,6 +258,7 @@ function LegendListItem({ editable, top, bottom, + onDataTableActivated, }: { item: LegendItem; visible: boolean; @@ -214,18 +268,25 @@ function LegendListItem({ editable?: boolean; top?: boolean; bottom?: boolean; + onDataTableActivated?: (layerId: string) => void; }) { const [contextMenuIsOpen, setContextMenuIsOpen] = useState(false); + const { data: projectMeta } = useCurrentProjectMetadata(); + const dataTablesFeatureEnabled = Boolean( + projectMeta?.project?.featureFlags?.dataTables + ); const isSingleSymbol = - (item.type === "GLStyleLegendItem" && + item.type !== "DataTableLegendItem" && + ((item.type === "GLStyleLegendItem" && item.legend?.type === "SimpleGLLegend" && item.legend.symbol) || - (item.type === "CustomGLSourceSymbolLegend" && item.symbols.length <= 1); + (item.type === "CustomGLSourceSymbolLegend" && item.symbols.length <= 1)); const inatLegendType = useInaturalistLegendType(item, map); return (
  • )} + {dataTablesFeatureEnabled && + item.overlayDataTables && + item.overlayDataTables.length > 0 && ( + + )} { @@ -334,7 +406,26 @@ function LegendListItem({ />
    • - {!isSingleSymbol && ( + {item.type === "DataTableLegendItem" && + item.overlayDataTables && + item.overlayDataTables.length > 0 && ( + + )} + {!isSingleSymbol && item.type !== "DataTableLegendItem" && (
      {item.type === "GLStyleLegendItem" && item.legend?.type === "MultipleSymbolGLLegend" && diff --git a/packages/client/src/dataLayers/MapContextManager.ts b/packages/client/src/dataLayers/MapContextManager.ts index b921be1de..3f4049e3e 100644 --- a/packages/client/src/dataLayers/MapContextManager.ts +++ b/packages/client/src/dataLayers/MapContextManager.ts @@ -20,6 +20,7 @@ import { ArchivedSourceFragment, BasemapDetailsFragment, BasemapType, + ClientOverlayDataTableFragment, DataLayerDetailsFragment, DataSourceDetailsFragment, DataSourceTypes, @@ -47,6 +48,39 @@ import debounce from "lodash.debounce"; import { currentSidebarState } from "../projects/ProjectAppSidebar"; import { ApolloClient, NormalizedCacheObject } from "@apollo/client"; import { compileLegendFromGLStyleLayers } from "./legends/compileLegend"; +import { + DataTableAggregation, + resolveDataTableVisualizationSettings, +} from "./dataTableQueryApi"; +import { + DataTableStatesMap, + LayerDataTableState, + buildDataTableStatesFromLayers, + layerStatesForPreferences, +} from "./dataTableLayerState"; +import { + DATA_TABLE_ACTIVE_COLOR, + DATA_TABLE_CIRCLE_FILL_OPACITY, + DATA_TABLE_CIRCLE_STROKE_OPACITY, + DATA_TABLE_LOADING_COLOR, + DATA_TABLE_LOADING_FILL_OPACITY, + DATA_TABLE_LOADING_STROKE_OPACITY, + DATA_TABLE_NO_DATA_FILL_OPACITY, + DATA_TABLE_NO_DATA_VALUE, + DATA_TABLE_PAINT_TRANSITION, + DATA_TABLE_VALUE_PROPERTY, + DATA_TABLE_ZERO_FILL_OPACITY, + DATA_TABLE_ZERO_SENTINEL, + DATA_TABLE_NO_DATA_COLOR, + buildDataTableCircleRadiusExpression, + buildDataTableValueExpression, +} from "./dataTableMapStyle"; +import { + columnStatsUrlForTable, + fetchDataTableColumnStats, + getCachedDataTableColumnStats, +} from "./useDataTableColumnStats"; +import { DataTablesColumnStats } from "@seasketch/geostats-types"; import { LegendItem } from "./Legend"; import { EventEmitter } from "eventemitter3"; import cloneDeep from "lodash.clonedeep"; @@ -68,11 +102,20 @@ import { buildInaturalistSourcesAndLayers, normalizeInaturalistParams, } from "./inaturalist"; -import type { +import { BasemapContextState, OptionalBasemapLayerValue, } from "./BasemapContext"; import { LayerStateManager } from "./LayerStateManager"; +import { + DataTableQueryManager, + ResolvedDataTableVisualizationSettings, +} from "./DataTableQueryManager"; + +export type { + LayerDataTableState, + DataTableStatesMap, +} from "./dataTableLayerState"; export const MeasureEventTypes = { Started: "measure_started", @@ -158,6 +201,12 @@ export interface LayerState { * legend. */ hidden?: boolean; + /** + * User intent for an activated overlay data table on this layer. + * Identified by overlay_data_tables.stable_id. Persisted with map prefs + * and map bookmarks. + */ + dataTable?: LayerDataTableState; } export interface SketchClassLayerState extends LayerState { @@ -206,6 +255,35 @@ class MapContextManager extends EventEmitter { private _setLegendsState?: Dispatch>; /** Basemap state from BasemapContext, set via updateBasemapState by MapManagerContextProvider */ private basemapState: BasemapContextState | null = null; + /** + * Cached query results for activated data tables (keyed by TOC stableId). + * User intent lives on LayerState.dataTable; query params are resolved on + * demand from the TOC catalog + admin constraints. + */ + private activatedDataTableValues: { + [tocStableId: string]: { + queryKey: string; + min: number; + max: number; + scaleMin: number; + scaleMax: number; + hasZero: boolean; + values: { [featureId: string]: number }; + }; + } = {}; + private activatedDataTableAppliedFeatureIds: { + [tocStableId: string]: Set | undefined; + } = {}; + private activatedDataTableRequests: { + [tocStableId: string]: string | undefined; + } = {}; + /** True while a data-table query is in flight for a layer. Drives gray/faded paint. */ + private activatedDataTableLoading: { + [tocStableId: string]: boolean | undefined; + } = {}; + private activatedDataTableSourceListeners: { + [tocStableId: string]: (event: mapboxgl.MapSourceDataEvent) => void; + } = {}; private _ready = false; /** Cached result of computeInatCtas for referential stability. */ @@ -241,6 +319,7 @@ class MapContextManager extends EventEmitter { | "basemapOptionalLayerStates" | "visibleDataLayers" | "selectedBasemap" + | "dataTableStates" > = undefined; /** * The manager needs an up-to-date list of geoprocessing reference ids so that @@ -260,6 +339,12 @@ class MapContextManager extends EventEmitter { }; } = {}; + private dataTableQueryManager: DataTableQueryManager; + /** tocStableIds with LayerState.dataTable intent — keeps sync O(active). */ + private dataTableActiveTocIds = new Set(); + /** Dedupes stacked `idle` listeners for data-table feature-state sync. */ + private dataTableFeatureStateSyncIdleQueued = false; + constructor( initialState: MapContextInterface, initialSketchState: SketchLayerContextState, @@ -304,6 +389,18 @@ class MapContextManager extends EventEmitter { // vice-versa). this.overlayStates.on("stateChanged", this.onOverlayStateChanged); this.sketchStates.on("stateChanged", this.onSketchStateChanged); + this.dataTableQueryManager = new DataTableQueryManager( + this.mapAccessToken ?? null + ); + this.dataTableQueryManager.setOnLegendSummaryChange(() => { + this.updateLegends(); + }); + // Seed from restored preferences / initial layer states. + for (const tocStableId of this.overlayStates.keys()) { + if (this.overlayStates.getRaw(tocStableId)?.dataTable?.stableId) { + this.dataTableActiveTocIds.add(tocStableId); + } + } } getCustomGLSource(sourceId: number) { @@ -330,6 +427,623 @@ class MapContextManager extends EventEmitter { } } + /** + * Set or clear data-table user intent on a layer. Source of truth is + * LayerState.dataTable (persisted with prefs / bookmarks). + */ + setLayerDataTable(tocStableId: string, state: LayerDataTableState | null) { + if (!this.overlayStates.has(tocStableId)) { + return; + } + if (state === null || !state.stableId) { + this.overlayStates.patch(tocStableId, { dataTable: undefined }); + this.dataTableActiveTocIds.delete(tocStableId); + const layer = this.layers[tocStableId]; + if (layer) { + this.dataTableQueryManager.clearLegendSummary( + this.overlayStates.prefixedSourceId(layer.dataSourceId) + ); + } + } else { + this.overlayStates.patch(tocStableId, { + dataTable: { ...state }, + }); + this.dataTableActiveTocIds.add(tocStableId); + } + this.scheduleDataTableFeatureStateSync(); + this.debouncedUpdateStyle(); + this.debouncedUpdatePreferences(); + this.updateLegends(); + } + + /** + * Apply a bookmark-style dataTableStates map onto current overlay layers. + * Clears dataTable only on layers that currently have one and are absent + * from the bookmark map; applies only keys present in dataTableStates. + */ + applyDataTableStates(dataTableStates: DataTableStatesMap | null | undefined) { + const states = dataTableStates || {}; + // Bookmark apply must turn off data tables that aren't in the map. + for (const tocStableId of [...this.dataTableActiveTocIds]) { + if (!states[tocStableId]?.stableId) { + this.overlayStates.patch(tocStableId, { dataTable: undefined }); + this.dataTableActiveTocIds.delete(tocStableId); + const layer = this.layers[tocStableId]; + if (layer) { + this.dataTableQueryManager.clearLegendSummary( + this.overlayStates.prefixedSourceId(layer.dataSourceId) + ); + } + } + } + for (const [tocStableId, next] of Object.entries(states)) { + if (!next?.stableId || !this.overlayStates.has(tocStableId)) { + continue; + } + this.overlayStates.patch(tocStableId, { + dataTable: { ...next }, + }); + this.dataTableActiveTocIds.add(tocStableId); + } + this.scheduleDataTableFeatureStateSync(); + this.debouncedUpdateStyle(); + this.debouncedUpdatePreferences(); + this.updateLegends(); + } + + /** + * Idempotent apply path plus a follow-up on the next map `idle`. + * + * Important: if the map is *already* idle when we register `once("idle")`, + * Mapbox will not fire until it becomes busy again — so we also sync + * immediately and once more after the next paint when the style is loaded. + * That covers reload races where the TOC catalog / sources arrive after the + * first style settle, and style-hash early returns that never call setStyle. + */ + private scheduleDataTableFeatureStateSync() { + this.syncDataTableFeatureStates(); + if (!this.map) { + return; + } + const map = this.map; + if (!this.dataTableFeatureStateSyncIdleQueued) { + this.dataTableFeatureStateSyncIdleQueued = true; + map.once("idle", () => { + this.dataTableFeatureStateSyncIdleQueued = false; + if (this.map === map) { + this.syncDataTableFeatureStates(); + } + }); + } + // Already-idle maps never re-emit `idle` until something else dirties + // them. Retry once the current frame finishes if the style is ready. + requestAnimationFrame(() => { + if (this.map !== map || !map.isStyleLoaded()) { + return; + } + this.syncDataTableFeatureStates(); + }); + } + + /** + * Active data-table proportional-symbol layers for hover value tooltips. + * Consumed by LayerInteractivityManager via MapUIContext. + */ + getDataTableInteractiveLayers(): { + glLayerId: string; + sourceId: string; + sourceLayer?: string; + column: string; + op: string; + }[] { + const out: { + glLayerId: string; + sourceId: string; + sourceLayer?: string; + column: string; + op: string; + }[] = []; + for (const tocStableId of this.dataTableActiveTocIds) { + const overlayState = this.overlayStates.getRaw(tocStableId); + if ( + !overlayState?.dataTable?.stableId || + !overlayState.visible || + overlayState.hidden + ) { + continue; + } + const activation = + this.resolveDataTableVisualizationSettings(tocStableId); + if (!activation?.query.column || !activation.query.op) { + continue; + } + const layer = this.layers[tocStableId]; + if (!layer) { + continue; + } + const op = Array.isArray(activation.query.op) + ? activation.query.op[0] + : activation.query.op; + const sourceLayer = + this.archivedSource && this.archivedSource.dataLayerId === layer.id + ? this.archivedSource.sourceLayer || undefined + : layer.sourceLayer || undefined; + out.push({ + glLayerId: idForLayer(layer, 0), + sourceId: this.overlayStates.prefixedSourceId(layer.dataSourceId), + sourceLayer, + column: activation.query.column, + op, + }); + } + return out; + } + + /** + * Drive data-table queries / feature-state from LayerState.dataTable. + * Style computation stays pure — this is the side-effect path. + * + * Only iterates layers with active data-table intent (not the full TOC). + * After style updates, reconciles applied markers against the live map: + * style diffs often keep feature-state on reused sources; only rebuilds + * that drop state (or missing sources) trigger a re-fetch via browser cache. + */ + private syncDataTableFeatureStates() { + if (!this.map || this.dataTableActiveTocIds.size === 0) { + return; + } + for (const tocStableId of this.dataTableActiveTocIds) { + if (!this.overlayStates.getRaw(tocStableId)?.dataTable?.stableId) { + this.dataTableActiveTocIds.delete(tocStableId); + continue; + } + const activation = + this.resolveDataTableVisualizationSettings(tocStableId); + if ( + !activation?.query.column || + !activation.query.op || + !activation.table.joinColumn || + !activation.table.queryUrl + ) { + continue; + } + const layer = this.layers[tocStableId]; + if (!layer) { + continue; + } + const sourceId = this.overlayStates.prefixedSourceId(layer.dataSourceId); + const sourceLayer = + this.archivedSource && this.archivedSource.dataLayerId === layer.id + ? this.archivedSource.sourceLayer || undefined + : layer.sourceLayer || undefined; + // Drop stale "applied" markers when the source/state didn't survive. + this.dataTableQueryManager.reconcileAppliedState(sourceId, sourceLayer); + if (!this.map.getSource(sourceId)) { + // Source not on the map yet (style still building). A scheduled + // idle / rAF retry will pick this up once sources exist. + continue; + } + const uuid = extractTilesUuidFromUrl(activation.table.queryUrl); + const requiresToken = uuid + ? this.hostedUuidNeedsMapAccessToken(uuid) + : false; + void this.dataTableQueryManager.applyFeatureState( + sourceId, + sourceLayer, + activation, + requiresToken + ); + } + } + + /** + * Resolve LayerState.dataTable against the TOC catalog + admin constraints. + * Returns undefined when intent is missing, the catalog isn't ready, or the + * stableId no longer exists. Does not mutate LayerState. + */ + private resolveDataTableVisualizationSettings( + tocStableId: string + ): ResolvedDataTableVisualizationSettings | undefined { + const dataTable = this.overlayStates.getRaw(tocStableId)?.dataTable; + if (!dataTable?.stableId) { + return undefined; + } + const tables = this.tocItems[tocStableId]?.overlayDataTables; + if (!tables) { + return undefined; + } + const table = tables.find((entry) => entry.stableId === dataTable.stableId); + if (!table) { + return undefined; + } + const resolved = resolveDataTableVisualizationSettings(table, { + column: dataTable.column, + op: dataTable.op, + filters: dataTable.filters, + }); + return { + table, + query: { + column: resolved.column, + op: resolved.op, + filters: resolved.filters, + }, + }; + } + + private removeActivatedDataTableSourceListener(tocStableId: string) { + const listener = this.activatedDataTableSourceListeners[tocStableId]; + if (listener && this.map) { + this.map.off("sourcedata", listener); + } + delete this.activatedDataTableSourceListeners[tocStableId]; + } + + /** + * Promoted feature IDs for a data-table join come from column-stats for the + * table join column (same values used as Mapbox promoteId). Not from + * currently loaded tiles, and not from overlay geostats. + */ + private getJoinFeatureIdsFromColumnStats( + columnStats: DataTablesColumnStats | undefined, + joinColumn: string + ): string[] { + if (!columnStats?.columns?.length || !joinColumn) { + return []; + } + const attribute = columnStats.columns.find( + (column) => column.attribute === joinColumn + ); + if (!attribute?.values) { + return []; + } + return Object.keys(attribute.values); + } + + private scheduleActivatedDataTableFeatureStateApply(tocStableId: string) { + if (!this.map) { + return; + } + const cached = this.activatedDataTableValues[tocStableId]; + const activation = this.resolveDataTableVisualizationSettings(tocStableId); + if (!cached?.values || !activation) { + return; + } + const layer = this.layers[tocStableId]; + const table = activation.table; + if (!layer || !table.overlayJoinColumn || !table.joinColumn) { + return; + } + const sourceId = this.overlayStates.prefixedSourceId(layer.dataSourceId); + const columnStatsUrl = columnStatsUrlForTable(table); + const queryKeyAtStart = cached.queryKey; + const applyWithColumnStats = async () => { + if (!this.map) { + return; + } + const currentValues = this.activatedDataTableValues[tocStableId]?.values; + const currentLayer = this.layers[tocStableId]; + const currentActivation = + this.resolveDataTableVisualizationSettings(tocStableId); + if ( + !currentValues || + !currentLayer || + !currentActivation?.table.overlayJoinColumn || + !currentActivation.table.joinColumn + ) { + return; + } + let columnStats = columnStatsUrl + ? getCachedDataTableColumnStats(columnStatsUrl) + : undefined; + if (!columnStats && columnStatsUrl) { + try { + columnStats = await fetchDataTableColumnStats( + columnStatsUrl, + this.mapAccessToken + ); + } catch (error) { + console.error(error); + } + } + // Re-check after the async column-stats fetch; activation may have changed. + if ( + this.activatedDataTableValues[tocStableId]?.values !== currentValues || + this.activatedDataTableValues[tocStableId]?.queryKey !== queryKeyAtStart + ) { + return; + } + const previousIds = + this.activatedDataTableAppliedFeatureIds[tocStableId] || + new Set(); + const joinIds = this.getJoinFeatureIdsFromColumnStats( + columnStats, + currentActivation.table.joinColumn + ); + // Prefer the complete join-column ID set from column-stats. Fall back to + // query result keys only if column-stats is missing. + const allFeatureIds = + joinIds.length > 0 ? joinIds : Object.keys(currentValues); + const nextIds = new Set(); + for (const featureId of allFeatureIds) { + const stateValue = + currentValues[featureId] !== undefined + ? currentValues[featureId] + : DATA_TABLE_NO_DATA_VALUE; + this.map.setFeatureState( + { + source: sourceId, + id: featureId, + ...(currentLayer.sourceLayer + ? { sourceLayer: currentLayer.sourceLayer } + : {}), + }, + { [DATA_TABLE_VALUE_PROPERTY]: stateValue } + ); + nextIds.add(featureId); + } + for (const featureId of previousIds) { + if (!nextIds.has(featureId)) { + this.map.removeFeatureState( + { + source: sourceId, + id: featureId, + ...(currentLayer.sourceLayer + ? { sourceLayer: currentLayer.sourceLayer } + : {}), + }, + DATA_TABLE_VALUE_PROPERTY + ); + } + } + this.activatedDataTableAppliedFeatureIds[tocStableId] = nextIds; + }; + // Apply once when the source exists. Do not re-apply on sourcedata / + // pan / zoom — feature-state is keyed by id and survives tile unload. + if (this.map.getSource(sourceId)) { + void applyWithColumnStats(); + return; + } + if (this.activatedDataTableSourceListeners[tocStableId]) { + return; + } + const listener = (event: mapboxgl.MapSourceDataEvent) => { + if (event.sourceId !== sourceId || !event.isSourceLoaded) { + return; + } + this.removeActivatedDataTableSourceListener(tocStableId); + void applyWithColumnStats(); + }; + this.activatedDataTableSourceListeners[tocStableId] = listener; + this.map.on("sourcedata", listener); + } + + /** + * Live map circle layer for an activated data table. ID and paint must stay + * in sync with {@link setLayerOpacity} so the legend opacity slider can + * call `setPaintProperty` on the same layer the style put on the map. + */ + private buildDataTableProportionalSymbolLayer( + layer: { tocId: string } & DataLayerDetailsFragment, + sourceId: string, + sourceLayer?: string + ): AnyLayer { + const valueExpression = ["feature-state", "scaledValue"] as Expression; + // Unset feature-state is null; typeof !== "number" covers that and any + // non-numeric sentinel in one check. + const isNoData = [ + "!=", + ["typeof", valueExpression], + "number", + ] as Expression; + // Safe to compare numerically: case chains check isNoData before isZero. + const isZero = [ + "==", + valueExpression, + DATA_TABLE_ZERO_SENTINEL, + ] as Expression; + const isLoading = [ + "boolean", + ["feature-state", "loading"], + false, + ] as Expression; + return { + id: idForLayer(layer, 0), + type: "circle", + source: sourceId, + ...(sourceLayer ? { "source-layer": sourceLayer } : {}), + metadata: { + "s:data-table-proporional-symbol": true, + }, + paint: { + "circle-color": [ + "case", + isLoading, + DATA_TABLE_LOADING_COLOR, + isNoData, + DATA_TABLE_NO_DATA_COLOR, + DATA_TABLE_ACTIVE_COLOR, + ], + "circle-opacity": [ + "case", + isLoading, + DATA_TABLE_LOADING_FILL_OPACITY, + isNoData, + DATA_TABLE_NO_DATA_FILL_OPACITY, + isZero, + DATA_TABLE_ZERO_FILL_OPACITY, + DATA_TABLE_CIRCLE_FILL_OPACITY, + ], + "circle-stroke-opacity": [ + "case", + isLoading, + DATA_TABLE_LOADING_STROKE_OPACITY, + isNoData, + DATA_TABLE_CIRCLE_STROKE_OPACITY, + 1, + ], + "circle-stroke-color": [ + "case", + isNoData, + DATA_TABLE_NO_DATA_COLOR, + DATA_TABLE_ACTIVE_COLOR, + ], + "circle-stroke-width": ["case", isNoData, 2, isZero, 2, 0], + "circle-radius": buildDataTableCircleRadiusExpression({ + valueExpression, + scaleMin: 0, + scaleMax: 1, + zoomDependent: true, + hideWhenMissing: false, + }), + }, + }; + } + + private dataTableCircleLayer( + tocStableId: string, + layer: { tocId: string } & DataLayerDetailsFragment, + sourceId: string, + sourceLayer?: string, + legendOnly = false + ): AnyLayer | undefined { + const activation = this.resolveDataTableVisualizationSettings(tocStableId); + if (!activation?.query.column || !activation.query.op) { + return undefined; + } + // Note: query execution is orchestrated exclusively by + // updateActivatedDataTableFeatureStates; style builders must stay pure. + const values = this.activatedDataTableValues[tocStableId]; + const loading = + !legendOnly && Boolean(this.activatedDataTableLoading[tocStableId]); + const fillOpacity = loading + ? DATA_TABLE_LOADING_FILL_OPACITY + : DATA_TABLE_CIRCLE_FILL_OPACITY; + const strokeOpacity = loading + ? DATA_TABLE_LOADING_STROKE_OPACITY + : DATA_TABLE_CIRCLE_STROKE_OPACITY; + const op = Array.isArray(activation.query.op) + ? activation.query.op[0] + : activation.query.op; + const label = `${op || "value"}(${activation.query.column})`; + const valueExpression = buildDataTableValueExpression( + (legendOnly + ? ["get", DATA_TABLE_VALUE_PROPERTY] + : ["feature-state", DATA_TABLE_VALUE_PROPERTY]) as Expression + ); + const isNoData = [ + "==", + valueExpression, + DATA_TABLE_NO_DATA_VALUE, + ] as Expression; + const isZero = ["==", valueExpression, 0] as Expression; + const isPositive = [">", valueExpression, 0] as Expression; + // UNSET (-2 via coalesce) matches none of these, so missing feature-state + // stays hidden without Mapbox evaluating `>` / interpolate on null. + const hasSymbolExpression = [ + "any", + isNoData, + isZero, + isPositive, + ] as Expression; + const scaleMin = + values && values.scaleMax > values.scaleMin ? values.scaleMin : 0; + const scaleMax = + values && values.scaleMax > values.scaleMin + ? values.scaleMax + : // Single positive value (or empty): keep interpolate valid and + // clamp that value to the max radius stop. + Math.max(scaleMin + 1, values?.scaleMax ?? 1); + const symbolRadius = buildDataTableCircleRadiusExpression({ + valueExpression, + scaleMin, + scaleMax, + zoomDependent: !legendOnly, + hideWhenMissing: !legendOnly, + }); + const activeFillColor = loading + ? DATA_TABLE_LOADING_COLOR + : DATA_TABLE_ACTIVE_COLOR; + const activeStrokeColor = loading + ? "rgba(255, 255, 255, 0.25)" + : "rgba(255, 255, 255, 0.5)"; + const symbolFillOpacity = loading + ? fillOpacity + : ([ + "case", + isNoData, + DATA_TABLE_NO_DATA_FILL_OPACITY, + isZero, + DATA_TABLE_ZERO_FILL_OPACITY, + fillOpacity, + ] as Expression); + const symbolStrokeOpacity = loading + ? strokeOpacity + : ([ + "case", + isNoData, + DATA_TABLE_CIRCLE_STROKE_OPACITY, + isZero, + DATA_TABLE_CIRCLE_STROKE_OPACITY, + strokeOpacity, + ] as Expression); + const symbolColor = loading + ? DATA_TABLE_LOADING_COLOR + : ([ + "case", + isNoData, + DATA_TABLE_NO_DATA_COLOR, + activeFillColor, + ] as Expression); + const symbolStrokeColor = loading + ? activeStrokeColor + : ([ + "case", + isNoData, + DATA_TABLE_NO_DATA_COLOR, + activeStrokeColor, + ] as Expression); + const symbolStrokeWidth = loading + ? 1 + : (["case", isNoData, 1.5, 1] as Expression); + return { + // eslint-disable-next-line i18next/no-literal-string + id: `${idForLayer(layer, 0)}-data-table`, + type: "circle", + source: sourceId, + ...(sourceLayer ? { "source-layer": sourceLayer } : {}), + metadata: { + "s:legend-labels": { + [DATA_TABLE_VALUE_PROPERTY]: label, + }, + }, + paint: { + "circle-color": symbolColor, + "circle-color-transition": DATA_TABLE_PAINT_TRANSITION, + "circle-opacity": legendOnly + ? symbolFillOpacity + : (["case", hasSymbolExpression, symbolFillOpacity, 0] as Expression), + "circle-opacity-transition": DATA_TABLE_PAINT_TRANSITION, + "circle-stroke-color": symbolStrokeColor, + "circle-stroke-color-transition": DATA_TABLE_PAINT_TRANSITION, + "circle-stroke-opacity": legendOnly + ? symbolStrokeOpacity + : ([ + "case", + hasSymbolExpression, + symbolStrokeOpacity, + 0, + ] as Expression), + "circle-stroke-opacity-transition": DATA_TABLE_PAINT_TRANSITION, + "circle-stroke-width": symbolStrokeWidth, + "circle-radius": symbolRadius, + // Radius can animate when the expression itself changes (new min/max), + // but feature-state value updates do not interpolate. + "circle-radius-transition": DATA_TABLE_PAINT_TRANSITION, + }, + }; + } + private setState = (action: SetStateAction) => { if (typeof action === "function") { this.internalState = { @@ -399,7 +1113,16 @@ class MapContextManager extends EventEmitter { } setMapAccessToken(token: string | null | undefined) { + const changed = this.mapAccessToken !== token; + if (changed) { + this.dataTableQueryManager.setMapAccessToken(token ?? null); + } this.mapAccessToken = token; + // Hosted query/column-stats apply bails when the token is missing. Retry + // once it arrives so a reload race can't leave the legend stuck loading. + if (changed && token) { + this.scheduleDataTableFeatureStateSync(); + } } setHostedTileUuidsRequiringAuth(uuids: string[] | null | undefined) { @@ -584,6 +1307,7 @@ class MapContextManager extends EventEmitter { throw new Error("Both initialBounds and initialCameraOptions are empty"); } this.map = new Map(mapOptions); + this.dataTableQueryManager.setMap(this.map); this.addSprites(sprites, this.map); @@ -812,6 +1536,13 @@ class MapContextManager extends EventEmitter { }); } + /** + * Persist the current map session so a reload can restore it. + * + * Writes overlay visibility (including data-table intent), camera, and + * sketch-class display overrides to localStorage under `preferencesKey`. + * Ephemeral fields like loading/error are omitted. No-op when no key is set. + */ private updatePreferences() { const sketchClassLayerStates = cloneDeep( this.internalState.sketchClassLayerStates || {} @@ -840,7 +1571,7 @@ class MapContextManager extends EventEmitter { } if (this.preferencesKey) { const prefs = { - layers: this.overlayStates.getState(), + layers: layerStatesForPreferences(this.overlayStates.getState()), ...(this.map ? { cameraOptions: { @@ -857,6 +1588,7 @@ class MapContextManager extends EventEmitter { } } + /** Coalesce rapid map-state changes into a single preferences write. */ private debouncedUpdatePreferences = debounce(() => { this.updatePreferences(); }, 200); @@ -948,6 +1680,9 @@ class MapContextManager extends EventEmitter { this.emit("sketchLayerIdsChanged", sketchLayerIds); const styleHash = md5(JSON.stringify(style)); if (styleHash === this.internalState.styleHash) { + // Style is unchanged, but catalog / token / sources may have become + // ready since the last settle — still retry feature-state apply. + this.scheduleDataTableFeatureStateSync(); return; } this.addSprites(sprites, this.map); @@ -1003,6 +1738,9 @@ class MapContextManager extends EventEmitter { } this.emit("customSourcesChanged", sources); this.setState((prev) => ({ ...prev, styleHash })); + // Style diffs may preserve feature-state on reused sources. Sync + // reconciles markers and only re-fetches when state was actually lost. + this.scheduleDataTableFeatureStateSync(); }; if (!this.mapIsLoaded) { setTimeout(update, 20); @@ -1604,6 +2342,17 @@ class MapContextManager extends EventEmitter { const overlaySourceKey = this.overlayStates.prefixedSourceId( source.id ); + + let dataTableVisualizationSettings = + this.resolveDataTableVisualizationSettings(layerId); + + if (!dataTableVisualizationSettings?.table.joinColumn) { + dataTableVisualizationSettings = undefined; + } + + const promoteId = + dataTableVisualizationSettings?.table.overlayJoinColumn; + if (!baseStyle.sources[overlaySourceKey]) { switch (source.type) { case DataSourceTypes.Vector: @@ -1611,6 +2360,7 @@ class MapContextManager extends EventEmitter { type: "vector", attribution: source.attribution || "", tiles: source.tiles as string[], + ...(promoteId ? { promoteId } : {}), ...(source.bounds ? { bounds: source.bounds.map((b) => parseFloat(b)) } : {}), @@ -1639,6 +2389,7 @@ class MapContextManager extends EventEmitter { type: "vector", url, attribution: source.attribution || "", + ...(promoteId ? { promoteId } : {}), }; break; case DataSourceTypes.SeasketchVector: @@ -1647,6 +2398,7 @@ class MapContextManager extends EventEmitter { type: "geojson", data: source.url!, attribution: source.attribution || "", + ...(promoteId ? { promoteId } : {}), }; break; case DataSourceTypes.SeasketchRaster: @@ -1831,6 +2583,28 @@ class MapContextManager extends EventEmitter { } as AnyLayer; }); const _staticOverlay = this.overlayStates.getRaw(layerId); + const sourceLayer = shouldHaveSourceLayer + ? this.archivedSource && + this.archivedSource.dataLayerId === layer.id + ? this.archivedSource.sourceLayer || undefined + : layer.sourceLayer || undefined + : undefined; + /** + * # Data Table proportional symbol layer calculation + */ + if ( + dataTableVisualizationSettings && + dataTableVisualizationSettings.query.column && + dataTableVisualizationSettings.query.op + ) { + glLayers = [ + this.buildDataTableProportionalSymbolLayer( + layer, + overlaySourceKey, + sourceLayer + ), + ]; + } if ( _staticOverlay && "opacity" in _staticOverlay && @@ -2189,6 +2963,7 @@ class MapContextManager extends EventEmitter { parentStableId?: string; enableDownload: boolean; primaryDownloadUrl?: string; + overlayDataTables?: ClientOverlayDataTableFragment[]; }; } = {}; @@ -2207,6 +2982,7 @@ class MapContextManager extends EventEmitter { | "parentStableId" | "enableDownload" | "primaryDownloadUrl" + | "overlayDataTables" | "acl" >[] ) { @@ -2230,6 +3006,9 @@ class MapContextManager extends EventEmitter { id: item.id, enableDownload: Boolean(item.enableDownload), primaryDownloadUrl: item.primaryDownloadUrl || undefined, + // Always store an array once the TOC item is in the catalog so sync + // can distinguish "not loaded" (missing toc item) from "no tables". + overlayDataTables: item.overlayDataTables || [], }; if (item.dataLayerId) { const layer = layersById[item.dataLayerId]; @@ -2251,6 +3030,11 @@ class MapContextManager extends EventEmitter { // Cleanup entries in overlayStates that no longer exist const validKeys = new Set(tocItems.map((i) => i.stableId)); this.overlayStates.retainOnly(validKeys); + for (const id of [...this.dataTableActiveTocIds]) { + if (!validKeys.has(id)) { + this.dataTableActiveTocIds.delete(id); + } + } for (const stableId of this.overlayStates.keys()) { const layer = this.layers[stableId]; if (layer) { @@ -2259,9 +3043,16 @@ class MapContextManager extends EventEmitter { this.overlayStates.prefixedSourceId(layer.dataSourceId) ); } + // Rebuild active set from restored LayerState (e.g. preferences). + if (this.overlayStates.getRaw(stableId)?.dataTable?.stableId) { + this.dataTableActiveTocIds.add(stableId); + } } this.debouncedUpdatePreferences(); } + // TOC catalog (including overlayDataTables) may have just arrived or + // changed — resolve LayerState.dataTable and (re)start queries. + this.scheduleDataTableFeatureStateSync(); this.debouncedUpdateStyle(); this.updateLegends(); return; @@ -2380,66 +3171,6 @@ class MapContextManager extends EventEmitter { ); } - // measure = () => { - // if (this.interactivityManager) { - // this.interactivityManager.pause(); - // } - // this.emit(MeasureEventTypes.Started); - // if (!this.map) { - // throw new Error("Map not initialized"); - // } - // if (this.MeasureControl) { - // this.MeasureControl.destroy(); - // } - // this.MeasureControl = new MeasureControl(this.map); - // this.MeasureControl.on("update", (measureControlState: any) => { - // this.setState((prev) => { - // return { - // ...prev, - // measureControlState, - // }; - // }); - // }); - // this.MeasureControl.start(); - // }; - - // resetMeasurement = () => { - // if (this.MeasureControl) { - // this.MeasureControl.reset(); - // } - // }; - - // cancelMeasurement = () => { - // this.MeasureControl?.destroy(); - // this.MeasureControl = undefined; - // this.emit(MeasureEventTypes.Stopped); - // this.setState((prev) => ({ - // ...prev, - // measureControlState: undefined, - // })); - // if (this.interactivityManager) { - // this.interactivityManager.resume(); - // } - // }; - - // pauseMeasurementTools = () => { - // if (this.MeasureControl) { - // this.MeasureControl.setPaused(true); - // } - // if (this.interactivityManager) { - // this.interactivityManager.resume(); - // } - // }; - - // resumeMeasurementTools = () => { - // if (this.MeasureControl) { - // if (this.interactivityManager) { - // this.interactivityManager.pause(); - // } - // this.MeasureControl.setPaused(false); - // } - // }; - private spritesById: { [id: string]: SpriteDetailsFragment } = {}; private async addSprites( @@ -2694,6 +3425,9 @@ class MapContextManager extends EventEmitter { ? undefined : await this.getMapThumbnail(sidebarState); const selectedId = this.basemapState?.selectedBasemap; + const dataTableStates = buildDataTableStatesFromLayers( + this.overlayStates.getState() + ); return { cameraOptions: { center: this.map.getCenter().toArray(), @@ -2711,9 +3445,74 @@ class MapContextManager extends EventEmitter { visibleSketches, basemapName: selectedId ? this.basemaps[selectedId]?.name || "" : "", clientGeneratedThumbnail, + dataTableStates, }; } + /** + * Wait briefly for in-flight data-table queries so thumbnails aren't gray. + * Always resolves so bookmark creation is not blocked. + */ + private async waitForDataTableQueriesIdle(timeoutMs = 2500) { + const started = Date.now(); + while (Date.now() - started < timeoutMs) { + const loading = Object.values(this.activatedDataTableLoading).some( + Boolean + ); + if (!loading) { + return; + } + await new Promise((r) => setTimeout(r, 50)); + } + } + + /** + * Re-apply in-memory data-table feature-state onto a map (e.g. thumbnail + * clone). Style paint uses feature-state which is not part of getStyle(). + */ + private applyActivatedDataTableFeatureStatesToMap(targetMap: Map) { + for (const tocStableId of Object.keys(this.activatedDataTableValues)) { + const values = this.activatedDataTableValues[tocStableId]?.values; + const activation = + this.resolveDataTableVisualizationSettings(tocStableId); + if (!values || !activation) { + continue; + } + const layer = this.layers[tocStableId]; + const table = activation.table; + if (!layer || !table.overlayJoinColumn || !table.joinColumn) { + continue; + } + const sourceId = this.overlayStates.prefixedSourceId(layer.dataSourceId); + if (!targetMap.getSource(sourceId)) { + continue; + } + const columnStatsUrl = columnStatsUrlForTable(table); + const columnStats = columnStatsUrl + ? getCachedDataTableColumnStats(columnStatsUrl) + : undefined; + const joinIds = this.getJoinFeatureIdsFromColumnStats( + columnStats, + table.joinColumn + ); + const allFeatureIds = joinIds.length > 0 ? joinIds : Object.keys(values); + for (const featureId of allFeatureIds) { + const stateValue = + values[featureId] !== undefined + ? values[featureId] + : DATA_TABLE_NO_DATA_VALUE; + targetMap.setFeatureState( + { + source: sourceId, + id: featureId, + ...(layer.sourceLayer ? { sourceLayer: layer.sourceLayer } : {}), + }, + { [DATA_TABLE_VALUE_PROPERTY]: stateValue } + ); + } + } + } + getMapThumbnail = withTimeout( 15000, async (sidebarState: { @@ -2726,6 +3525,7 @@ class MapContextManager extends EventEmitter { if (!this.map) { throw new Error("Map not ready to create thumbnail"); } + await this.waitForDataTableQueriesIdle(); const { sprites } = await this.getComputedStyle(); const style = this.map.getStyle(); const div = document.createElement("div"); @@ -2747,7 +3547,12 @@ class MapContextManager extends EventEmitter { transformRequest: this.requestTransformer, }); this.addSprites(sprites, newMap); - newMap.on("load", () => { + let captured = false; + const capture = () => { + if (captured) { + return; + } + captured = true; let clip: null | { x: number; width: number } = null; if (sidebarState.open && width >= 1080) { clip = { @@ -2785,6 +3590,17 @@ class MapContextManager extends EventEmitter { newMap.remove(); div.remove(); return resolve(data); + }; + newMap.on("load", () => { + this.applyActivatedDataTableFeatureStatesToMap(newMap); + // Wait for feature-state paint to commit before reading the canvas. + newMap.once("idle", () => { + capture(); + }); + // Fallback if idle never fires (e.g. no tiles for some sources). + setTimeout(() => { + capture(); + }, 2000); }); } catch (e) { reject(e); @@ -2800,6 +3616,7 @@ class MapContextManager extends EventEmitter { | "basemapOptionalLayerStates" | "visibleDataLayers" | "selectedBasemap" + | "dataTableStates" > & { id?: string }, savePreviousState = true, client?: ApolloClient @@ -2831,6 +3648,9 @@ class MapContextManager extends EventEmitter { } } this.setVisibleTocItems((bookmark.visibleDataLayers || []) as string[]); + const dataTableStates = + (bookmark.dataTableStates as DataTableStatesMap | null | undefined) || {}; + this.applyDataTableStates(dataTableStates); this.map.flyTo({ ...bookmark.cameraOptions, animate: true, @@ -2969,6 +3789,7 @@ class MapContextManager extends EventEmitter { title: this.tocItems[id].label || "Unknown", primaryDownloadUrl: this.tocItems[id]?.primaryDownloadUrl, }; + const overlayDataTables = this.tocItems[id]?.overlayDataTables; const layer = this.layers[id]; const source = this.clientDataSources[layer?.dataSourceId]; if (layer && source && layer.mapboxGlStyles) { @@ -3012,29 +3833,86 @@ class MapContextManager extends EventEmitter { (l: any) => l.metadata?.["s:respect-scale-and-offset"] ) ); - const legend = compileLegendFromGLStyleLayers( - layer.mapboxGlStyles, - sourceType, - source.rasterRepresentativeColors, - respectRasterOffsetAndScale - ? (source.rasterScale as number | undefined) - : undefined, - respectRasterOffsetAndScale - ? (source.rasterOffset as number | undefined) - : undefined - ); - if (legend) { + const dataTableActivation = + this.resolveDataTableVisualizationSettings(id); + const dataTableOp = dataTableActivation?.query.op + ? Array.isArray(dataTableActivation.query.op) + ? dataTableActivation.query.op[0] + : dataTableActivation.query.op + : undefined; + // Show the data-table legend panel as soon as a table is + // activated — even before a column is resolved. Column/op + // controls live in that panel and pick a default numeric + // column from column-stats when admin constraints are empty. + if (dataTableActivation && dataTableActivation.table.stableId) { + const sourceId = this.overlayStates.prefixedSourceId( + layer.dataSourceId + ); + const legendSummary = + this.dataTableQueryManager.getLegendSummary(sourceId); newLegendState[id] = { id, - type: "GLStyleLegendItem", - legend: legend, + type: "DataTableLegendItem", label: this.tocItems[layer.tocId]?.label || "", tableOfContentsItemDetails, + overlayDataTables, + loading: legendSummary?.loading ?? true, + error: legendSummary?.error, + tableStableId: dataTableActivation.table.stableId, + tableName: dataTableActivation.table.name, + column: dataTableActivation.query.column, + op: (dataTableOp || "mean") as DataTableAggregation, + min: + legendSummary && + legendSummary.scaleMax > legendSummary.scaleMin + ? legendSummary.scaleMin + : legendSummary?.scaleMax ?? 0, + max: + legendSummary && + legendSummary.scaleMax > legendSummary.scaleMin + ? legendSummary.scaleMax + : legendSummary?.scaleMax ?? 1, + hasZero: legendSummary?.hasZero ?? false, + // Show the bubble scale for any positive values, including + // a single unique value where scaleMin === scaleMax. + // Keep it during refetch (loading) so the legend dims the + // stale scale instead of swapping layouts. + showValueScale: (legendSummary?.scaleMax ?? 0) > 0, }; changes = true; } else { - newLegendState[id] = null; - changes = true; + const dataTableLayer = this.dataTableCircleLayer( + id, + layer, + this.overlayStates.prefixedSourceId(layer.dataSourceId), + layer.sourceLayer || undefined, + true + ); + const legend = compileLegendFromGLStyleLayers( + dataTableLayer ? [dataTableLayer] : layer.mapboxGlStyles, + sourceType, + source.rasterRepresentativeColors, + respectRasterOffsetAndScale + ? (source.rasterScale as number | undefined) + : undefined, + respectRasterOffsetAndScale + ? (source.rasterOffset as number | undefined) + : undefined + ); + if (legend) { + newLegendState[id] = { + id, + type: "GLStyleLegendItem", + legend: legend, + label: this.tocItems[layer.tocId]?.label || "", + tableOfContentsItemDetails, + overlayDataTables, + }; + changes = true; + } else { + newLegendState[id] = null; + changes = true; + } } } catch (e) { console.error(e); @@ -3043,6 +3921,7 @@ class MapContextManager extends EventEmitter { type: "GLStyleLegendItem", label: this.tocItems[layer.tocId]?.label || "", tableOfContentsItemDetails, + overlayDataTables, }; changes = true; } @@ -3082,6 +3961,7 @@ class MapContextManager extends EventEmitter { label: this.tocItems[layer.tocId]?.label || "", modes, tableOfContentsItemDetails, + overlayDataTables, } as LegendItem; changes = true; } else if (source && isCustomSourceType(source.type)) { @@ -3107,6 +3987,7 @@ class MapContextManager extends EventEmitter { ), label: this.tocItems[layer.tocId]?.label || "", tableOfContentsItemDetails, + overlayDataTables, }; } else if (item.legend) { newLegendState[id] = { @@ -3120,6 +4001,7 @@ class MapContextManager extends EventEmitter { symbols: item.legend, id, tableOfContentsItemDetails, + overlayDataTables, }; } changes = true; @@ -3353,12 +4235,39 @@ class MapContextManager extends EventEmitter { } this.overlayStates.setOpacity(stableId, opacity); const layer = this.layers[stableId]; + if (!layer || !this.map) { + this.debouncedUpdateStyle(); + return; + } const source = this.clientDataSources[layer.dataSourceId]; + if (!source) { + this.debouncedUpdateStyle(); + return; + } const shouldHaveSourceLayer = source.type === DataSourceTypes.SeasketchMvt || source.type === DataSourceTypes.Vector; // || // source.type === DataSourceTypes.SeasketchRaster; - if (layer && source && this.map && layer.mapboxGlStyles?.length > 0) { + const dataTableActivation = + this.resolveDataTableVisualizationSettings(stableId); + const dataTableActive = Boolean( + dataTableActivation?.query.column && dataTableActivation?.query.op + ); + if (dataTableActive) { + // Must use the same layer id/paint as getComputedStyle. The old + // dataTableCircleLayer helper used a `-data-table` id suffix, so + // setPaintProperty was a no-op on a layer that isn't on the map. + const dataTableLayer = this.buildDataTableProportionalSymbolLayer( + layer, + this.overlayStates.prefixedSourceId(layer.dataSourceId), + shouldHaveSourceLayer ? layer.sourceLayer || undefined : undefined + ); + if (this.map.getLayer(dataTableLayer.id)) { + adjustLayerOpacities([dataTableLayer], opacity, this.map); + } else { + this.debouncedUpdateStyle(); + } + } else if (layer.mapboxGlStyles?.length > 0) { let glLayers = (layer.mapboxGlStyles as any[]).map((lyr, i) => { return { ...lyr, diff --git a/packages/client/src/dataLayers/MapManagerContextProvider.tsx b/packages/client/src/dataLayers/MapManagerContextProvider.tsx index 95bf22ce8..447deee39 100644 --- a/packages/client/src/dataLayers/MapManagerContextProvider.tsx +++ b/packages/client/src/dataLayers/MapManagerContextProvider.tsx @@ -1,22 +1,19 @@ import React, { useContext, useEffect, useMemo, useRef, useState } from "react"; import { useParams } from "react-router"; import { BBox } from "geojson"; -import type { CameraOptions } from "mapbox-gl"; +import { CameraOptions } from "mapbox-gl"; import bytes from "bytes"; import useAccessToken from "../useAccessToken"; import useCurrentProjectMetadata from "../useCurrentProjectMetadata"; import useMapAccessTokenRollover from "./useMapAccessTokenRollover"; import { useProjectRegionQuery } from "../generated/graphql"; -import { BasemapContext } from "./BasemapContext"; -import type { BasemapContextState } from "./BasemapContext"; +import { BasemapContext, BasemapContextState } from "./BasemapContext"; import MapContextManager, { DigitizingLockState, MapManagerContext, SketchLayerContext, MapOverlayContext, LegendsContext, -} from "./MapContextManager"; -import type { MapContextInterface, MapManagerState, SketchLayerContextState, @@ -198,7 +195,16 @@ export default function MapManagerContextProvider({ const parentTocItems = parentOverlay.tableOfContentsItems; useEffect(() => { const manager = managerRef.current; - if (manager && parentDataSources && parentDataLayers && parentTocItems) { + // useMapData initializes these as [] before the GraphQL response arrives. + // Empty arrays are truthy, so guard on length — otherwise reset()+sync can + // run against an empty TOC catalog and drop restored LayerState.dataTable. + if ( + manager && + parentDataSources && + parentDataLayers && + parentTocItems && + parentTocItems.length > 0 + ) { manager.reset(parentDataSources, parentDataLayers, parentTocItems); } }, [parentDataLayers, parentDataSources, parentTocItems]); diff --git a/packages/client/src/dataLayers/MapUIContext.tsx b/packages/client/src/dataLayers/MapUIContext.tsx index dfdef53af..a4cc68bea 100644 --- a/packages/client/src/dataLayers/MapUIContext.tsx +++ b/packages/client/src/dataLayers/MapUIContext.tsx @@ -414,6 +414,13 @@ export default function MapUIProvider({ tocItemLabels ); } + // Data-table value tooltips (feature-state rawValue) — independent of + // admin interactivity type, but refreshed on the same overlay cadence. + if (manager) { + interactivityManager.setDataTableLayers( + manager.getDataTableInteractiveLayers() + ); + } }, [ mapOverlay.layerStatesByTocStaticId, mapOverlay.dataLayers, @@ -424,6 +431,7 @@ export default function MapUIProvider({ interactivityManager, basemap, mapOverlay, + manager, ]); useEffect(() => { diff --git a/packages/client/src/dataLayers/dataTableLayerState.restore.test.ts b/packages/client/src/dataLayers/dataTableLayerState.restore.test.ts new file mode 100644 index 000000000..fea14d14e --- /dev/null +++ b/packages/client/src/dataLayers/dataTableLayerState.restore.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "@jest/globals"; +import { + applyDataTableStatesToLayerStates, + buildDataTableStatesFromLayers, + layerStatesForPreferences, + LayerStateWithDataTable, +} from "./dataTableLayerState"; + +/** + * Regression: prefs restore must keep dataTable across a "catalog not ready" + * round-trip. The manager should not treat missing TOC tables as deletion. + */ +describe("dataTable prefs restore semantics", () => { + it("keeps dataTable in preferences when layer is visible", () => { + const layers: { [id: string]: LayerStateWithDataTable } = { + sites: { + visible: true, + loading: false, + dataTable: { + stableId: "table-uuid", + column: "biomass", + op: "sum", + }, + }, + }; + const prefs = layerStatesForPreferences(layers); + const hydrated = JSON.parse(JSON.stringify(prefs)) as { + [id: string]: LayerStateWithDataTable; + }; + expect(hydrated.sites.dataTable?.stableId).toBe("table-uuid"); + }); + + it("bookmark apply restores dataTable onto visible layers", () => { + const before: { [id: string]: LayerStateWithDataTable } = { + sites: { + visible: true, + loading: false, + dataTable: { stableId: "table-uuid", column: "biomass", op: "sum" }, + }, + }; + const bookmark = buildDataTableStatesFromLayers(before); + const layersAfterReload: { [id: string]: LayerStateWithDataTable } = { + sites: { visible: true, loading: false }, + }; + const afterReload = applyDataTableStatesToLayerStates( + layersAfterReload, + bookmark + ); + expect(afterReload.sites.dataTable).toEqual(before.sites.dataTable); + }); +}); diff --git a/packages/client/src/dataLayers/dataTableLayerState.test.ts b/packages/client/src/dataLayers/dataTableLayerState.test.ts new file mode 100644 index 000000000..ee31f536a --- /dev/null +++ b/packages/client/src/dataLayers/dataTableLayerState.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "@jest/globals"; +import { + applyDataTableStatesToLayerStates, + buildDataTableStatesFromLayers, + layerStatesForPreferences, + LayerStateWithDataTable, +} from "./dataTableLayerState"; + +describe("buildDataTableStatesFromLayers", () => { + it("includes only visible layers with dataTable", () => { + const layers: { [id: string]: LayerStateWithDataTable } = { + tocA: { + visible: true, + loading: false, + dataTable: { stableId: "table-a", column: "biomass", op: "sum" }, + }, + tocB: { + visible: false, + loading: false, + dataTable: { stableId: "table-b", column: "count", op: "count" }, + }, + tocC: { + visible: true, + loading: false, + }, + }; + expect(buildDataTableStatesFromLayers(layers)).toEqual({ + tocA: { stableId: "table-a", column: "biomass", op: "sum" }, + }); + }); +}); + +describe("applyDataTableStatesToLayerStates", () => { + it("merges bookmark states and clears missing entries", () => { + const layers: { [id: string]: LayerStateWithDataTable } = { + tocA: { + visible: true, + loading: false, + dataTable: { stableId: "old", column: "x" }, + }, + tocB: { + visible: true, + loading: false, + dataTable: { stableId: "keep-me" }, + }, + tocC: { visible: true, loading: false }, + }; + const next = applyDataTableStatesToLayerStates(layers, { + tocA: { stableId: "new-a", column: "y", op: "avg" }, + tocC: { stableId: "new-c" }, + }); + expect(next.tocA.dataTable).toEqual({ + stableId: "new-a", + column: "y", + op: "avg", + }); + expect(next.tocB.dataTable).toBeUndefined(); + expect(next.tocC.dataTable).toEqual({ stableId: "new-c" }); + }); + + it("does not invent layers absent from the current map", () => { + const next = applyDataTableStatesToLayerStates( + { tocA: { visible: true, loading: false } }, + { tocMissing: { stableId: "x" } } + ); + expect(Object.keys(next)).toEqual(["tocA"]); + }); +}); + +describe("layerStatesForPreferences", () => { + it("persists dataTable and strips loading/error", () => { + const layers: { [id: string]: LayerStateWithDataTable } = { + tocA: { + visible: true, + loading: true, + error: new Error("boom"), + opacity: 0.5, + dataTable: { + stableId: "table-a", + column: "biomass", + op: "sum", + filters: [{ column: "year", op: "eq", value: "2020" }], + }, + }, + }; + const prefs = layerStatesForPreferences(layers); + expect(prefs.tocA).toEqual({ + visible: true, + loading: false, + opacity: 0.5, + dataTable: { + stableId: "table-a", + column: "biomass", + op: "sum", + filters: [{ column: "year", op: "eq", value: "2020" }], + }, + }); + expect((prefs.tocA as any).error).toBeUndefined(); + + const roundTrip = JSON.parse(JSON.stringify(prefs)); + expect(roundTrip.tocA.dataTable.stableId).toBe("table-a"); + }); +}); + +describe("bookmark prefs round-trip", () => { + it("build then apply restores the same dataTable fields", () => { + const layers: { [id: string]: LayerStateWithDataTable } = { + tocA: { + visible: true, + loading: false, + dataTable: { stableId: "t1", column: "c", op: "sum" }, + }, + tocB: { visible: true, loading: false }, + }; + const bookmark = buildDataTableStatesFromLayers(layers); + const restored = applyDataTableStatesToLayerStates( + { + tocA: { visible: true, loading: false }, + tocB: { + visible: true, + loading: false, + dataTable: { stableId: "stale" }, + }, + }, + bookmark + ); + expect(restored.tocA.dataTable).toEqual(layers.tocA.dataTable); + expect(restored.tocB.dataTable).toBeUndefined(); + }); +}); diff --git a/packages/client/src/dataLayers/dataTableLayerState.ts b/packages/client/src/dataLayers/dataTableLayerState.ts new file mode 100644 index 000000000..8a0af888d --- /dev/null +++ b/packages/client/src/dataLayers/dataTableLayerState.ts @@ -0,0 +1,108 @@ +import { + DataTableAggregation, + DataTableFilter, +} from "./dataTableQueryApi"; + +/** + * Persisted / bookmarked data-table intent for a single overlay layer. + * Identifies the table by overlay_data_tables.stable_id (not row id). + */ +export interface LayerDataTableState { + stableId: string; + column?: string; + op?: DataTableAggregation; + filters?: DataTableFilter[]; +} + +/** Minimal layer fields needed for prefs/bookmark helpers. */ +export interface LayerStateWithDataTable { + visible: boolean; + opacity?: number; + zOrderOverride?: number; + loading?: boolean; + error?: Error; + hidden?: boolean; + dataTable?: LayerDataTableState; +} + +/** Bookmark / API payload: TOC stableId → table viz intent. */ +export type DataTableStatesMap = { + [tocStableId: string]: LayerDataTableState; +}; + +/** + * Build bookmark/API dataTableStates from overlay layer states. + * Only includes visible layers that have an active data table. + */ +export function buildDataTableStatesFromLayers( + layerStates: { [tocStableId: string]: LayerStateWithDataTable } +): DataTableStatesMap { + const out: DataTableStatesMap = {}; + for (const [tocStableId, state] of Object.entries(layerStates)) { + if (!state?.visible || !state.dataTable?.stableId) { + continue; + } + out[tocStableId] = { ...state.dataTable }; + } + return out; +} + +/** + * Merge bookmark dataTableStates onto a copy of layer states. + * Clears dataTable on layers not present in the bookmark map. + * Does not add layers that are missing from `layers`. + */ +export function applyDataTableStatesToLayerStates< + T extends LayerStateWithDataTable +>( + layers: { [tocStableId: string]: T }, + dataTableStates: DataTableStatesMap | null | undefined +): { [tocStableId: string]: T } { + const next: { [tocStableId: string]: T } = {}; + const states = dataTableStates || {}; + for (const [tocStableId, state] of Object.entries(layers)) { + const dataTable = states[tocStableId]; + if (dataTable?.stableId) { + next[tocStableId] = { + ...state, + dataTable: { ...dataTable }, + }; + } else if (state.dataTable) { + const { dataTable: _removed, ...rest } = state; + next[tocStableId] = rest as T; + } else { + next[tocStableId] = state; + } + } + return next; +} + +/** + * Strip ephemeral LayerState fields before persisting to localStorage. + * Keeps dataTable (user intent) and visibility/opacity/z/hidden. + */ +export function layerStatesForPreferences( + layerStates: { [tocStableId: string]: LayerStateWithDataTable } +): { [tocStableId: string]: Partial } { + const out: { [tocStableId: string]: Partial } = {}; + for (const [tocStableId, state] of Object.entries(layerStates)) { + const next: Partial = { + visible: state.visible, + loading: false, + }; + if (state.opacity !== undefined) { + next.opacity = state.opacity; + } + if (state.zOrderOverride !== undefined) { + next.zOrderOverride = state.zOrderOverride; + } + if (state.hidden !== undefined) { + next.hidden = state.hidden; + } + if (state.dataTable?.stableId) { + next.dataTable = { ...state.dataTable }; + } + out[tocStableId] = next; + } + return out; +} diff --git a/packages/client/src/dataLayers/dataTableMapStyle.ts b/packages/client/src/dataLayers/dataTableMapStyle.ts new file mode 100644 index 000000000..04e57857b --- /dev/null +++ b/packages/client/src/dataLayers/dataTableMapStyle.ts @@ -0,0 +1,217 @@ +import { Expression } from "mapbox-gl"; + +/** Feature-state key for aggregated data-table values on overlay features. */ +export const DATA_TABLE_VALUE_PROPERTY = "__dataTableValue"; + +/** + * Sentinel written to feature-state when a join site has no matching + * table rows for the current filters (distinct from an explicit zero). + * Paint expressions treat non-numeric `scaledValue` as no-data. + */ +export const DATA_TABLE_NO_DATA_VALUE = null; + +/** + * Stand-in used when feature-state / get returns null (feature not yet + * joined). Same null as {@link DATA_TABLE_NO_DATA_VALUE} in the scaledValue + * model — distinguish loading vs no-data via the `loading` feature-state. + */ +export const DATA_TABLE_UNSET_VALUE = null; + +/** + * Coalesce a possibly-null feature-state/get expression. With null sentinels + * this is a no-op; kept for call sites that still wrap value expressions. + */ +export function buildDataTableValueExpression( + rawValueExpression: Expression +): Expression { + return ["coalesce", rawValueExpression, DATA_TABLE_UNSET_VALUE] as Expression; +} + +/** + * Sentinel written to `scaledValue` feature-state for a true zero value. + * Positive values scale 0–1 against scaleMin/scaleMax, so the smallest + * positive value also lands on 0 — zero needs its own marker. + */ +export const DATA_TABLE_ZERO_SENTINEL = -1; + +export const DATA_TABLE_ACTIVE_COLOR = "#2563eb"; +export const DATA_TABLE_NO_DATA_COLOR = "#9ca3af"; +export const DATA_TABLE_LOADING_COLOR = "#6b7280"; + +export const DATA_TABLE_CIRCLE_FILL_OPACITY = 0.8; +export const DATA_TABLE_CIRCLE_STROKE_OPACITY = Math.min( + DATA_TABLE_CIRCLE_FILL_OPACITY + 0.2, + 1 +); +export const DATA_TABLE_LOADING_FILL_OPACITY = 0.35; +export const DATA_TABLE_LOADING_STROKE_OPACITY = 0.25; + +/** Full-size radii used at high zoom (and in the legend). */ +export const DATA_TABLE_VALUE_MIN_RADIUS = 10; +export const DATA_TABLE_VALUE_MAX_RADIUS = 65; +/** No-data sites render slightly smaller than the min positive-value symbol. */ +export const DATA_TABLE_NO_DATA_RADIUS = DATA_TABLE_VALUE_MIN_RADIUS * 0.9; +/** Zero-value sites match the no-data footprint exactly, but in active blue. */ +export const DATA_TABLE_ZERO_RADIUS = DATA_TABLE_NO_DATA_RADIUS; + +/** Light fill shared by the no-data and zero-value symbols. */ +export const DATA_TABLE_NO_DATA_FILL_OPACITY = 0.35; +export const DATA_TABLE_ZERO_FILL_OPACITY = DATA_TABLE_NO_DATA_FILL_OPACITY; + +export const DATA_TABLE_PAINT_TRANSITION = { duration: 450, delay: 0 }; + +/** Zoom at which circle radii reach DATA_TABLE_VALUE_MIN/MAX_RADIUS. */ +export const DATA_TABLE_RADIUS_FULL_ZOOM = 14; + +/** + * Scales all zoom-dependent radii. At 1, stops at DATA_TABLE_RADIUS_FULL_ZOOM + * use the full MIN/MAX constants; lower values shrink symbols more when + * zoomed out. + */ +export const DATA_TABLE_RADIUS_ZOOM_MULTIPLIER = 0.8; + +/** Zoom levels at which radius stops are defined; values interpolate between. */ +const DATA_TABLE_RADIUS_ZOOM_LEVELS = [5, 8, 11, DATA_TABLE_RADIUS_FULL_ZOOM]; + +function dataTableRadiusAtZoom(baseRadius: number, zoom: number): number { + return ( + baseRadius * + (zoom / DATA_TABLE_RADIUS_FULL_ZOOM) * + DATA_TABLE_RADIUS_ZOOM_MULTIPLIER + ); +} + +const DATA_TABLE_RADIUS_ZOOM_STOPS = DATA_TABLE_RADIUS_ZOOM_LEVELS.map( + (zoom) => ({ + zoom, + min: dataTableRadiusAtZoom(DATA_TABLE_VALUE_MIN_RADIUS, zoom), + max: dataTableRadiusAtZoom(DATA_TABLE_VALUE_MAX_RADIUS, zoom), + }) +); + +function valueRadiusExpression( + valueExpression: Expression, + scaleMin: number, + scaleMax: number, + minRadius: number, + maxRadius: number +): Expression { + return [ + "interpolate", + ["linear"], + valueExpression, + scaleMin, + minRadius, + scaleMax, + maxRadius, + ] as Expression; +} + +function radiusForStop( + valueExpression: Expression, + scaleMin: number, + scaleMax: number, + minRadius: number, + maxRadius: number, + hideWhenMissing: boolean +): Expression { + // Keep zoom-stop radii in the same ratio as the full-size legend constants. + const noDataRadius = + minRadius * (DATA_TABLE_NO_DATA_RADIUS / DATA_TABLE_VALUE_MIN_RADIUS); + const zeroRadius = + minRadius * (DATA_TABLE_ZERO_RADIUS / DATA_TABLE_VALUE_MIN_RADIUS); + // scaledValue is a number when present; null / missing feature-state is + // non-numeric. (UNSET and NO_DATA are both null in the current model.) + const isMissing = [ + "!=", + ["typeof", valueExpression], + "number", + ] as Expression; + // True zeros carry the sentinel; the smallest positive value scales to 0, + // so a plain `== 0` check would misclassify it. + const isZero = [ + "==", + valueExpression, + DATA_TABLE_ZERO_SENTINEL, + ] as Expression; + const isPositive = [">=", valueExpression, 0] as Expression; + // Guard missing/zero before interpolate so Mapbox never runs `>` / + // interpolate on null. + const sized = [ + "case", + isMissing, + hideWhenMissing ? 0 : noDataRadius, + isZero, + zeroRadius, + isPositive, + valueRadiusExpression( + valueExpression, + scaleMin, + scaleMax, + minRadius, + maxRadius + ), + 0, + ] as Expression; + if (!hideWhenMissing) { + return sized; + } + return [ + "case", + ["any", isZero, isPositive], + sized, + // Missing / no-data: hidden when hideWhenMissing is set. + 0, + ] as Expression; +} + +/** + * Build a `circle-radius` paint expression that scales with both the + * aggregated value (via feature-state / get) and map zoom. + * + * Mapbox requires `["zoom"]` to be the input of a *top-level* interpolate + * or step in paint properties, so value/feature-state logic lives inside + * each zoom stop's output — never wrapping the zoom interpolate. + */ +export function buildDataTableCircleRadiusExpression({ + valueExpression, + scaleMin, + scaleMax, + zoomDependent, + hideWhenMissing, +}: { + valueExpression: Expression; + scaleMin: number; + scaleMax: number; + /** When false (legend preview), use full-size radii with no zoom scaling. */ + zoomDependent: boolean; + /** When true, features without a known state value get radius 0. */ + hideWhenMissing: boolean; +}): Expression { + if (!zoomDependent) { + return radiusForStop( + valueExpression, + scaleMin, + scaleMax, + DATA_TABLE_VALUE_MIN_RADIUS, + DATA_TABLE_VALUE_MAX_RADIUS, + hideWhenMissing + ); + } + + const expression: unknown[] = ["interpolate", ["linear"], ["zoom"]]; + for (const stop of DATA_TABLE_RADIUS_ZOOM_STOPS) { + expression.push( + stop.zoom, + radiusForStop( + valueExpression, + scaleMin, + scaleMax, + stop.min, + stop.max, + hideWhenMissing + ) + ); + } + return expression as Expression; +} diff --git a/packages/client/src/dataLayers/dataTableQueryApi.parse.test.ts b/packages/client/src/dataLayers/dataTableQueryApi.parse.test.ts new file mode 100644 index 000000000..1921df248 --- /dev/null +++ b/packages/client/src/dataLayers/dataTableQueryApi.parse.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "@jest/globals"; +import { parseDataTableQueryGroups } from "./dataTableQueryApi"; + +describe("parseDataTableQueryGroups", () => { + it("maps join keys to aggregation values and computes extents", () => { + const parsed = parseDataTableQueryGroups( + [ + { site: "A", sum: 10 }, + { site: "B", sum: 0 }, + { site: "C", sum: 4 }, + { site: "D", sum: null }, + { site: "E", sum: "skip" }, + ], + "site", + "sum" + ); + expect(parsed.values).toEqual({ A: 10, B: 0, C: 4 }); + expect(parsed.min).toBe(0); + expect(parsed.max).toBe(10); + expect(parsed.scaleMin).toBe(4); + expect(parsed.scaleMax).toBe(10); + expect(parsed.hasZero).toBe(true); + }); + + it("handles empty groups and array ops", () => { + expect(parseDataTableQueryGroups([], "site", "mean")).toEqual({ + values: {}, + min: 0, + max: 0, + scaleMin: 0, + scaleMax: 0, + hasZero: false, + }); + const parsed = parseDataTableQueryGroups( + [{ site: 12, mean: 3.5 }], + "site", + ["mean", "sum"] + ); + expect(parsed.values).toEqual({ "12": 3.5 }); + }); +}); diff --git a/packages/client/src/dataLayers/dataTableQueryApi.ts b/packages/client/src/dataLayers/dataTableQueryApi.ts new file mode 100644 index 000000000..f29d8c5c1 --- /dev/null +++ b/packages/client/src/dataLayers/dataTableQueryApi.ts @@ -0,0 +1,364 @@ +/* eslint-disable i18next/no-literal-string -- query URL serialization, not UI copy */ + +/** + * Data table query API — types and semantics for GET `/query` requests. + * + * Each overlay data table lives at an immutable R2 prefix under the parent + * layer hosted UUID + * (`projects/{slug}/public/{sourceUuid}/dataTables/{uploadId}`), served by + * the Overlay Data Server (`uploads.seasketch.org` / pmtiles-server). Map + * clients compile {@link DataTableQuerySettings} into query-string + * parameters, fetch aggregated JSON (with `access_token` when required by + * the parent layer ACL), and join results to vector features using + * `column-stats.json` join metadata. + * + * **Endpoints** (relative to `{tablePath}` or `OverlayDataTable.queryUrl`): + * + * - `GET /{tablePath}/query` — DataTablesBackend aggregation (`f=json` or HTML UI) + * - `GET /{tablePath}/column-stats.json` — column metadata and join stats + * - `GET /{tablePath}/data.parquet` — download underlying parquet + * + * **Query modes** + * + * 1. *Aggregated* — set {@link DataTableQuerySettings.groupBy} and + * {@link DataTableQuerySettings.op}. Response contains a `groups` array; each + * object has the group key column(s) plus one property per aggregation. + * 2. *Raw rows* — omit `groupBy` and `op`. Response contains a `rows` array + * (all columns). Rarely used for map joins. + * + * **Built-in query parameters** + * + * | Param | Description | + * | ----- | ----------- | + * | `f` | `json` (default) or `html` (interactive query UI) | + * | `groupBy` | Comma-separated columns, e.g. `site` or `site,year` | + * | `op` | Comma-separated: `count`, `sum`, `mean`, `min`, `max`, `median` | + * | `column` | Numeric column to aggregate; required unless only `count` | + * | `orderBy` | Sort key, optional `:desc`, e.g. `mean:desc` or `site` | + * | `limit` | Max groups/rows (omit for no limit) | + * | `offset` | Skip N groups/rows after sorting (default 0) | + * + * **Column filters** use a `q.{columnName}` prefix with PostgREST-style + * operators in the value. See {@link DataTableFilter} and + * {@link serializeDataTableFilter}. + * + * **JSON response** (aggregated): `{ table, totalRows, rowsScanned, rowsMatched, + * rowGroups, timing, groups }`. Raw mode returns `rows` instead of `groups`. + * + * Implementation: `packages/pmtiles-server/src/dataTables/params.ts`. + * Extended reference: `packages/pmtiles-server/README.md`. + * + * @module dataTableQueryApi + */ + +/** Aggregation operations supported by the query endpoint. */ +export type DataTableAggregation = + | "count" + | "sum" + | "mean" + | "min" + | "max" + | "median"; + +/** All aggregation operations, in the order they should be presented in UI. */ +export const DATA_TABLE_AGGREGATIONS: DataTableAggregation[] = [ + "mean", + "sum", + "count", + "min", + "max", + "median", +]; + +/** + * Filter operator for structured settings. Matches `FilterOperator` in + * `packages/pmtiles-server/src/dataTables/params.ts`. + * + * URL encoding: `isNull` → `q.{col}=is.null`, `notNull` → `q.{col}=not.null`. + * There is no `not.in`; use `neq` or multiple filters instead. + */ +export type DataTableFilterOperator = + | "eq" + | "neq" + | "gt" + | "gte" + | "lt" + | "lte" + | "in" + | "isNull" + | "notNull"; + +/** + * A single column filter. Serializes to `q.{column}=…` via + * {@link serializeDataTableFilter}. + */ +export interface DataTableFilter { + column: string; + op: DataTableFilterOperator; + /** Required for all ops except `isNull`, `notNull`, and `in`. */ + value?: string; + /** List items for `in`. Items may contain commas; they are quoted on serialization. */ + values?: string[]; +} + +/** + * Returns the list items of an `in` filter, tolerating the legacy + * comma-joined `value` representation. + */ +export function dataTableInFilterValues(filter: DataTableFilter): string[] { + if (filter.values) { + return filter.values; + } + return (filter.value || "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean); +} + +/** + * Structured settings for querying a data table for map display. Compile with + * {@link buildDataTableQuerySearchParams} and append to `{queryUrl}`. + * + * Applied on top of any base data table settings set by the admin. + */ +export interface DataTableQuerySettings { + /** Aggregation(s) to compute per group. Multiple ops share one `column`. */ + op?: DataTableAggregation | DataTableAggregation[]; + /** Numeric column to aggregate. Omit when using `count` alone (row count). */ + column?: string; + /** Group key column(s), e.g. the join column for a thematic map. */ + groupBy?: string | string[]; + filters?: DataTableFilter[]; +} + +/** + * Admin-configured constraints on how a data table may be visualized, as + * stored on `overlay_data_tables.visualization_columns` / + * `.visualization_ops` / `.required_filter_columns`. Empty/null + * visualization columns/ops means "no constraint -- let the end user + * choose". Empty required filter columns means no filters are forced. + */ +export interface DataTableVisualizationConstraints { + visualizationColumns?: (string | null)[] | null; + visualizationOps?: (string | null)[] | null; + /** Columns that must always appear as map filters (values are user-chosen). */ + requiredFilterColumns?: (string | null)[] | null; +} + +/** Metadata needed by the legend display settings UI before query/style work begins. */ +export interface DataTableVisualizationMetadata + extends DataTableVisualizationConstraints { + queryUrl?: string | null; + columnStatsUrl?: string | null; +} + +/** Raw user picks from the "Display settings" UI, before reconciling with admin constraints. */ +export interface DataTableUserVisualizationChoice { + column?: string; + op?: DataTableAggregation; + filters?: DataTableFilter[]; +} + +/** Result of {@link resolveDataTableVisualizationSettings}: a valid, ready-to-query column/op pair. */ +export interface ResolvedDataTableVisualization { + column?: string; + op: DataTableAggregation; + filters?: DataTableFilter[]; + requiredFilterColumns: string[]; +} + +/** + * Combines admin-set constraints ({@link DataTableVisualizationConstraints}) + * with the end user's choice in "Display settings" to produce a valid + * column/op pair to pass to {@link buildDataTableQuerySearchParams}: + * + * - `op` must be one of `visualizationOps` when that list is non-empty; + * otherwise falls back to the user's choice, or `"mean"`. + * - `column` must be one of `visualizationColumns` when that list is + * non-empty; otherwise falls back to the user's choice. When neither + * admin constraints nor a user choice supply a column, callers should + * treat **all numeric columns** (from column-stats) as valid and pick a + * default — see {@link DataTableVisualizationControls}. + */ +export function resolveDataTableVisualizationSettings( + constraints: DataTableVisualizationConstraints, + userChoice: DataTableUserVisualizationChoice +): ResolvedDataTableVisualization { + const allowedOps = (constraints.visualizationOps?.filter( + (op): op is DataTableAggregation => + Boolean(op) && (DATA_TABLE_AGGREGATIONS as string[]).includes(op!) + ) || []) as DataTableAggregation[]; + + const op = + allowedOps.length > 0 + ? userChoice.op && allowedOps.includes(userChoice.op) + ? userChoice.op + : allowedOps[0] + : userChoice.op || "mean"; + + const allowedColumns = (constraints.visualizationColumns?.filter( + (column): column is string => Boolean(column) + ) || []) as string[]; + + const column = + allowedColumns.length > 0 + ? userChoice.column && allowedColumns.includes(userChoice.column) + ? userChoice.column + : allowedColumns[0] + : userChoice.column; + + const requiredFilterColumns = (constraints.requiredFilterColumns?.filter( + (column): column is string => Boolean(column) + ) || []) as string[]; + + return { + column, + op, + filters: userChoice.filters, + requiredFilterColumns, + }; +} + +/** Normalize admin-required filter column names (drop empties, preserve order). */ +export function requiredDataTableFilterColumns( + constraints: DataTableVisualizationConstraints +): string[] { + return (constraints.requiredFilterColumns?.filter( + (column): column is string => Boolean(column) + ) || []) as string[]; +} + +/** + * Prefer a sensible default numeric column when admin visualization + * constraints are empty. Prefers a column named `count` when present. + */ +export function pickDefaultDataTableColumn( + numericColumns: string[] +): string | undefined { + if (numericColumns.length === 0) { + return undefined; + } + const countColumn = numericColumns.find( + (column) => column.toLowerCase() === "count" + ); + return countColumn || numericColumns[0]; +} + +/** + * Serialize one {@link DataTableFilter} to a `q.{column}` query parameter value. + */ +export function serializeDataTableFilter(filter: DataTableFilter): string { + const { column, op, value } = filter; + switch (op) { + case "isNull": + return "is.null"; + case "notNull": + return "not.null"; + case "in": { + const rawItems = dataTableInFilterValues(filter); + if (rawItems.length === 0) { + throw new Error(`Filter on "${column}" with op "in" requires values`); + } + const items = rawItems.map((item) => { + const trimmed = item.trim(); + if (trimmed.includes(",") || trimmed.includes('"')) { + return `"${trimmed.replace(/"/g, '""')}"`; + } + return trimmed; + }); + return `in.(${items.join(",")})`; + } + case "eq": + return value ?? ""; + default: + if (value === undefined) { + throw new Error(`Filter on "${column}" with op "${op}" requires value`); + } + return `${op}.${value}`; + } +} + +/** + * Compile {@link DataTableQuerySettings} into URLSearchParams for a GET query. + * Does not set `f`; callers should request JSON via `Accept: application/json` + * or `f=json`. + */ +export function buildDataTableQuerySearchParams( + settings: DataTableQuerySettings +): URLSearchParams { + const params = new URLSearchParams(); + + if (settings.groupBy !== undefined) { + const groupBy = Array.isArray(settings.groupBy) + ? settings.groupBy.join(",") + : settings.groupBy; + params.set("groupBy", groupBy); + } + + if (settings.op !== undefined) { + const op = Array.isArray(settings.op) ? settings.op.join(",") : settings.op; + params.set("op", op); + } + + if (settings.column !== undefined) { + params.set("column", settings.column); + } + + for (const filter of settings.filters ?? []) { + params.append(`q.${filter.column}`, serializeDataTableFilter(filter)); + } + + return params; +} + +/** One row/group object from an aggregated `/query` JSON response. */ +export type DataTableQueryResultGroup = { + [key: string]: string | number | null | undefined; +}; + +/** Parsed join values + scale extents from an aggregated query response. */ +export interface ParsedDataTableQueryValues { + values: { [featureId: string]: number }; + min: number; + max: number; + /** Min of positive values only (for bubble scale); 0 when none. */ + scaleMin: number; + /** Max of positive values only (for bubble scale); 0 when none. */ + scaleMax: number; + hasZero: boolean; +} + +/** + * Turn aggregated `/query` JSON into a featureId → numeric value map plus + * extents used for data-table circle symbology. + */ +export function parseDataTableQueryGroups( + groups: DataTableQueryResultGroup[] | null | undefined, + joinColumn: string, + op: DataTableAggregation | DataTableAggregation[] | undefined +): ParsedDataTableQueryValues { + const resolvedOp = Array.isArray(op) ? op[0] : op; + const values: { [featureId: string]: number } = {}; + for (const group of groups || []) { + const featureId = group[joinColumn]; + const value = resolvedOp ? group[resolvedOp] : undefined; + if ( + featureId !== null && + featureId !== undefined && + typeof value === "number" && + Number.isFinite(value) + ) { + values[String(featureId)] = value; + } + } + const numericValues = Object.values(values); + const positiveValues = numericValues.filter((value) => value > 0); + return { + values, + min: numericValues.length ? Math.min(...numericValues) : 0, + max: numericValues.length ? Math.max(...numericValues) : 0, + scaleMin: positiveValues.length ? Math.min(...positiveValues) : 0, + scaleMax: positiveValues.length ? Math.max(...positiveValues) : 0, + hasZero: numericValues.some((value) => value === 0), + }; +} diff --git a/packages/client/src/dataLayers/legends/DataTableLegendBubble.tsx b/packages/client/src/dataLayers/legends/DataTableLegendBubble.tsx new file mode 100644 index 000000000..a73ab407f --- /dev/null +++ b/packages/client/src/dataLayers/legends/DataTableLegendBubble.tsx @@ -0,0 +1,323 @@ +import { ReactNode } from "react"; +import { useTranslation } from "react-i18next"; +import { + DATA_TABLE_ACTIVE_COLOR, + DATA_TABLE_NO_DATA_COLOR, + DATA_TABLE_NO_DATA_FILL_OPACITY, + DATA_TABLE_NO_DATA_RADIUS, + DATA_TABLE_ZERO_FILL_OPACITY, + DATA_TABLE_ZERO_RADIUS, +} from "../dataTableMapStyle"; + +function formatLegendNumber(value: number) { + return value.toLocaleString(undefined, { + maximumFractionDigits: 1, + useGrouping: false, + }); +} + +const MAX_RADIUS = 32; +const MID_RADIUS = 19; +const MIN_RADIUS = 10; +const FILL = "rgba(37, 99, 235, 0.92)"; +const RING_STROKE = "rgba(255, 255, 255, 0.95)"; +const EMPTY_FILL = "rgba(156, 163, 175, 0.45)"; +const EMPTY_RING_STROKE = "rgba(255, 255, 255, 0.9)"; + +const WIDTH = MAX_RADIUS * 2 + 12; +const HEIGHT = MAX_RADIUS * 2 + 16; +const CENTER_X = WIDTH / 2; +const BASELINE_Y = HEIGHT - 8; + +function labelFontSize(text: string, radius: number) { + if (radius <= MIN_RADIUS && text.length > 4) { + return 8; + } + if (radius <= MID_RADIUS && text.length > 5) { + return 9; + } + return 11; +} + +/** Grey outline + light grey fill, matching map no-data symbology. */ +function NoDataSymbol() { + const size = Math.max(DATA_TABLE_NO_DATA_RADIUS * 2, 7); + return ( + + ); +} + +/** Same footprint and outline as no-data, in active blue. Matches map paint. */ +function ZeroSymbol() { + const size = Math.max(DATA_TABLE_ZERO_RADIUS * 2, 7); + return ( + + ); +} + +/** + * Nested bubble scale in a fixed WIDTH×HEIGHT footprint. + * + * `active` renders blue bubbles with value labels; otherwise a grey, + * label-free version keeps the exact same geometry (optionally pulsing + * while the first query loads) so state changes never shift layout. + */ +function ValueScaleBubble({ + min, + max, + active, + pulse = false, +}: { + min: number; + max: number; + active: boolean; + pulse?: boolean; +}) { + if (!active) { + return ( + + + + + + ); + } + + // Single unique positive value: one max-size bubble (matches map paint, + // which renders that value at the top radius stop). + if (!(max > min)) { + const label = formatLegendNumber(max); + const labelY = BASELINE_Y - MAX_RADIUS; + return ( + + + + {label} + + + ); + } + + const displayMid = min + (max - min) / 2; + const stops = [ + { value: max, radius: MAX_RADIUS }, + { value: displayMid, radius: MID_RADIUS }, + { value: min, radius: MIN_RADIUS }, + ]; + + return ( + + + + + {stops.map((stop) => { + const centerY = BASELINE_Y - stop.radius; + const label = formatLegendNumber(stop.value); + const labelY = centerY - stop.radius + 11; + return ( + + {label} + + ); + })} + + ); +} + +function SpecialSymbolEntry({ + symbol, + label, + labelClassName = "text-gray-700", +}: { + symbol: ReactNode; + label: ReactNode; + labelClassName?: string; +}) { + return ( + + {symbol} + + {label} + + + ); +} + +/** + * Data-table map legend: nested bubble scale on the left, zero-value and + * no-data symbols stacked to the right. + * + * All three symbols are always rendered in a fixed footprint so loading, + * errors, or new extents change content (colors, labels, dimming) but never + * shift layout: + * - data settled → blue scale with numbers + * - refetch with previous extents → same scale, dimmed + * - first load / no data → grey scale, no numbers (pulsing while loading) + * - a reserved status line shows "Loading values…" without inserting rows + */ +export default function DataTableLegendBubble({ + min, + max, + showValueScale = true, + loading = false, + error, +}: { + min: number; + max: number; + /** Retained for API compatibility; all symbols are always shown. */ + hasZero?: boolean; + /** True when there are real positive extents to label the scale with. */ + showValueScale?: boolean; + loading?: boolean; + error?: string; +}) { + const { t } = useTranslation("homepage"); + const hasError = Boolean(error); + const hasScale = showValueScale && max > 0; + const muted = loading || hasError; + + return ( +
      +
      +
      + +
      +
        +
      • + } + label={0} + labelClassName="font-medium tabular-nums text-gray-700" + /> +
        • +
      • + } + label={t("No data")} + labelClassName="text-gray-600" + /> +
        • +
      +
      + {/* Reserved line: fades in/out without inserting or removing rows. */} +
      + + {t("Loading values…")} + +
      +
      + ); +} diff --git a/packages/client/src/dataLayers/legends/DataTableLegendPanel.tsx b/packages/client/src/dataLayers/legends/DataTableLegendPanel.tsx new file mode 100644 index 000000000..f7bec9162 --- /dev/null +++ b/packages/client/src/dataLayers/legends/DataTableLegendPanel.tsx @@ -0,0 +1,289 @@ +import { useContext, useEffect, useMemo } from "react"; +import { Cross2Icon, ExclamationTriangleIcon } from "@radix-ui/react-icons"; +import { useTranslation } from "react-i18next"; +import DataTableIcon from "../../components/icons/DataTableIcon"; +import { + ClientOverlayDataTableFragment, + useOverlayDataTableVisualizationMetadataForLayerQuery, +} from "../../generated/graphql"; +import { MapManagerContext, MapOverlayContext } from "../MapContextManager"; +import { + DataTableAggregation, + DataTableVisualizationMetadata, + requiredDataTableFilterColumns, + resolveDataTableVisualizationSettings, +} from "../dataTableQueryApi"; +import DataTableFilterControls, { + ensureRequiredDataTableFilters, +} from "../DataTableFilterControls"; +import DataTableVisualizationControls from "../DataTableVisualizationControls"; +import { + columnStatsUrlForTable, + useDataTableColumnStats, +} from "../useDataTableColumnStats"; +import useCurrentProjectMetadata from "../../useCurrentProjectMetadata"; +import DataTableLegendBubble from "./DataTableLegendBubble"; + +export default function DataTableLegendPanel({ + layerId, + tableStableId, + tableName, + column, + op, + min, + max, + hasZero = false, + showValueScale = true, + loading = false, + error, + tables, + tocItemId, +}: { + layerId: string; + tableStableId: string; + tableName: string; + column?: string; + op: DataTableAggregation; + min: number; + max: number; + hasZero?: boolean; + showValueScale?: boolean; + loading?: boolean; + error?: string; + tables: ClientOverlayDataTableFragment[]; + tocItemId?: number; +}) { + const { t } = useTranslation("homepage"); + const { manager } = useContext(MapManagerContext); + const { layerStatesByTocStaticId } = useContext(MapOverlayContext); + const dataTable = layerStatesByTocStaticId[layerId]?.dataTable; + const { data: projectMeta } = useCurrentProjectMetadata(); + const mapAccessToken = projectMeta?.project?.mapAccessToken; + const table = tables.find((entry) => entry.stableId === tableStableId); + + const metadataQuery = useOverlayDataTableVisualizationMetadataForLayerQuery({ + variables: { tocItemId: tocItemId || -1 }, + skip: tocItemId === undefined, + fetchPolicy: "cache-first", + }); + + const metadataByTableId = useMemo(() => { + const next: { + [tableId: number]: DataTableVisualizationMetadata | undefined; + } = {}; + for (const entry of metadataQuery.data?.tableOfContentsItem + ?.overlayDataTables || []) { + next[entry.id] = { + queryUrl: entry.queryUrl, + columnStatsUrl: entry.columnStatsUrl, + visualizationColumns: entry.visualizationColumns, + visualizationOps: entry.visualizationOps, + requiredFilterColumns: entry.requiredFilterColumns, + }; + } + return next; + }, [metadataQuery.data?.tableOfContentsItem?.overlayDataTables]); + + const tableMetadata = useMemo(() => { + if (!table) { + return undefined; + } + return ( + metadataByTableId[table.id] || { + queryUrl: table.queryUrl, + columnStatsUrl: table.columnStatsUrl, + visualizationColumns: table.visualizationColumns, + visualizationOps: table.visualizationOps, + requiredFilterColumns: table.requiredFilterColumns, + } + ); + }, [metadataByTableId, table]); + + const columnStatsUrl = tableMetadata + ? columnStatsUrlForTable(tableMetadata) + : undefined; + const columnStatsState = useDataTableColumnStats( + columnStatsUrl, + mapAccessToken + ); + const columnStats = columnStatsState.columnStats; + const userChoice = useMemo( + () => ({ + column: dataTable?.column, + op: dataTable?.op, + filters: dataTable?.filters, + }), + [dataTable?.column, dataTable?.filters, dataTable?.op] + ); + const resolved = tableMetadata + ? resolveDataTableVisualizationSettings(tableMetadata, userChoice) + : { op, column, requiredFilterColumns: [] as string[] }; + const effectiveColumn = userChoice.column || resolved.column || column; + const visualizedColumns = useMemo( + () => (effectiveColumn ? [effectiveColumn] : []), + [effectiveColumn] + ); + const requiredFilterColumns = useMemo( + () => + tableMetadata ? requiredDataTableFilterColumns(tableMetadata) : [], + [tableMetadata] + ); + const validFilterColumns = useMemo( + () => + new Set( + (columnStats?.columns || []) + .map((entry: { attribute: string }) => entry.attribute) + .filter((entry: string) => !visualizedColumns.includes(entry)) + ), + [columnStats?.columns, visualizedColumns] + ); + + // Persist required filters into layer state so map queries include them + // (and the legend shows them) as soon as column-stats are available. + useEffect(() => { + if (!tableMetadata || !columnStats?.columns?.length || !manager) { + return; + } + if (requiredFilterColumns.length === 0 || !dataTable?.stableId) { + return; + } + const ensured = ensureRequiredDataTableFilters( + userChoice.filters, + requiredFilterColumns, + columnStats.columns, + visualizedColumns + ); + const current = userChoice.filters || []; + if (JSON.stringify(ensured) === JSON.stringify(current)) { + return; + } + manager.setLayerDataTable(layerId, { + stableId: dataTable.stableId, + column: effectiveColumn, + op: userChoice.op || op, + filters: ensured, + }); + }, [ + manager, + layerId, + dataTable?.stableId, + tableMetadata, + columnStats?.columns, + requiredFilterColumns, + visualizedColumns, + userChoice.filters, + userChoice.op, + effectiveColumn, + op, + ]); + + const activeFilters = useMemo(() => { + const base = (userChoice.filters || []).filter((filter) => + validFilterColumns.has(filter.column) + ); + if (!columnStats?.columns?.length || requiredFilterColumns.length === 0) { + return base; + } + return ensureRequiredDataTableFilters( + base, + requiredFilterColumns, + columnStats.columns, + visualizedColumns + ).filter((filter) => validFilterColumns.has(filter.column)); + }, [ + columnStats?.columns, + requiredFilterColumns, + userChoice.filters, + validFilterColumns, + visualizedColumns, + ]); + + if (!table || !tableMetadata) { + return null; + } + + return ( +
      +
      +
      + +

      + {tableName || table.name} +

      + {error && ( + + )} + +
      + +
      + + {error && ( +
      + + {error} +
      + )} + + {(loading || error || showValueScale || Boolean(column)) && ( +
      + +
      + )} + + {!columnStatsState.loading && + !columnStatsState.error && + columnStats?.columns && ( + { + if (!tableStableId) return; + manager?.setLayerDataTable(layerId, { + stableId: tableStableId, + column: effectiveColumn, + op: userChoice.op || op, + filters: ensureRequiredDataTableFilters( + filters, + requiredFilterColumns, + columnStats.columns, + visualizedColumns + ), + }); + }} + /> + )} +
      + ); +} diff --git a/packages/client/src/dataLayers/useDataTableColumnStats.ts b/packages/client/src/dataLayers/useDataTableColumnStats.ts new file mode 100644 index 000000000..ed3811921 --- /dev/null +++ b/packages/client/src/dataLayers/useDataTableColumnStats.ts @@ -0,0 +1,206 @@ +import { useEffect, useState } from "react"; +import { DataTablesColumnStats } from "@seasketch/geostats-types"; +import { withHostedAuthParams } from "./tilesAuth"; + +export interface DataTableColumnStatsState { + columnStats?: DataTablesColumnStats; + loading: boolean; + error?: Error; +} + +const columnStatsCache: { [url: string]: DataTablesColumnStats | undefined } = + {}; +const columnStatsErrors: { [url: string]: Error | undefined } = {}; +const columnStatsRequests: { + [url: string]: Promise | undefined; +} = {}; + +export function columnStatsUrlForTable(table: { + columnStatsUrl?: string | null; + queryUrl?: string | null; +}) { + if (table.columnStatsUrl) { + return table.columnStatsUrl; + } + if (!table.queryUrl) { + return undefined; + } + try { + const url = new URL(table.queryUrl); + url.pathname = url.pathname.replace(/\/query$/, "/column-stats.json"); + url.search = ""; + return url.toString(); + } catch { + return table.queryUrl.replace(/\/query(?:\?.*)?$/, "/column-stats.json"); + } +} + +export function getCachedDataTableColumnStats( + columnStatsUrl?: string | null +) { + return columnStatsUrl ? columnStatsCache[columnStatsUrl] : undefined; +} + +/** + * Fetch column-stats.json for a data table. When `accessToken` is provided, + * appends hosted-auth query params (same ACL as the parent layer tiles). + * Cache keys use the bare URL so token rollover does not fragment the cache. + */ +export function fetchDataTableColumnStats( + columnStatsUrl: string, + accessToken?: string | null +) { + if (columnStatsCache[columnStatsUrl]) { + return Promise.resolve(columnStatsCache[columnStatsUrl]); + } + if (columnStatsRequests[columnStatsUrl]) { + return columnStatsRequests[columnStatsUrl]!; + } + const authorizedUrl = withHostedAuthParams(columnStatsUrl, { + accessToken, + }); + const request = fetch(authorizedUrl) + .then((response) => { + if (!response.ok) { + // eslint-disable-next-line i18next/no-literal-string + throw new Error(`Failed to fetch column stats (${response.status})`); + } + return response.json(); + }) + .then((columnStats: DataTablesColumnStats) => { + columnStatsCache[columnStatsUrl] = columnStats; + columnStatsErrors[columnStatsUrl] = undefined; + return columnStats; + }) + .catch((error: Error) => { + columnStatsErrors[columnStatsUrl] = error; + throw error; + }) + .finally(() => { + columnStatsRequests[columnStatsUrl] = undefined; + }); + columnStatsRequests[columnStatsUrl] = request; + return request; +} + +/** + * Fetches `column-stats.json` for a data table (see `columnStatsUrl` on + * `OverlayDataTable`) so admin and end-user forms can present real column + * names and types rather than free text. + * + * @see packages/pmtiles-server/README.md + */ +export function useDataTableColumnStats( + columnStatsUrl?: string | null, + accessToken?: string | null +): DataTableColumnStatsState { + const [state, setState] = useState({ + columnStats: getCachedDataTableColumnStats(columnStatsUrl), + error: columnStatsUrl ? columnStatsErrors[columnStatsUrl] : undefined, + // Uncached stats always trigger a fetch in the effect below, so start in + // loading state to avoid an empty-state flash on first render. + loading: Boolean(columnStatsUrl && !columnStatsCache[columnStatsUrl]), + }); + + useEffect(() => { + if (!columnStatsUrl) { + setState({ loading: false }); + return; + } + const cached = columnStatsCache[columnStatsUrl]; + if (cached) { + setState({ columnStats: cached, loading: false }); + return; + } + let cancelled = false; + setState({ + loading: true, + error: columnStatsErrors[columnStatsUrl], + }); + fetchDataTableColumnStats(columnStatsUrl, accessToken) + .then((columnStats: DataTablesColumnStats) => { + if (!cancelled) { + setState({ columnStats, loading: false }); + } + }) + .catch((error: Error) => { + if (!cancelled) { + setState({ loading: false, error }); + } + }); + return () => { + cancelled = true; + }; + }, [columnStatsUrl, accessToken]); + + return state; +} + +export function useDataTableColumnStatsMap( + columnStatsUrls: string[], + accessToken?: string | null +) { + const [state, setState] = useState<{ + [url: string]: DataTableColumnStatsState | undefined; + }>(() => + Object.fromEntries( + columnStatsUrls.map((url) => [ + url, + { + columnStats: columnStatsCache[url], + error: columnStatsErrors[url], + loading: Boolean(!columnStatsCache[url] && columnStatsRequests[url]), + }, + ]) + ) + ); + + useEffect(() => { + let cancelled = false; + setState((prev) => { + const next = { ...prev }; + for (const url of columnStatsUrls) { + next[url] = { + columnStats: columnStatsCache[url], + error: columnStatsErrors[url], + loading: Boolean(!columnStatsCache[url]), + }; + } + return next; + }); + for (const url of columnStatsUrls) { + if (columnStatsCache[url]) { + continue; + } + fetchDataTableColumnStats(url, accessToken) + .then((columnStats) => { + if (!cancelled) { + setState((prev) => ({ + ...prev, + [url]: { columnStats, loading: false }, + })); + } + }) + .catch((error: Error) => { + if (!cancelled) { + setState((prev) => ({ + ...prev, + [url]: { error, loading: false }, + })); + } + }); + } + return () => { + cancelled = true; + }; + }, [columnStatsUrls, accessToken]); + + return state; +} + +/** Column names with numeric values, suitable for `sum`/`mean`/`min`/`max`/`median` aggregation. */ +export function numericColumnNames(columnStats?: DataTablesColumnStats) { + return (columnStats?.columns || []) + .filter((column) => column.type === "number") + .map((column) => column.attribute); +} diff --git a/packages/client/src/generated/graphql.ts b/packages/client/src/generated/graphql.ts index 06fbff361..aa4a07fdf 100644 --- a/packages/client/src/generated/graphql.ts +++ b/packages/client/src/generated/graphql.ts @@ -956,6 +956,12 @@ export type ChangeLogCondition = { }; export enum ChangeLogFieldGroup { + DataTableCreated = 'DATA_TABLE_CREATED', + DataTableDeleted = 'DATA_TABLE_DELETED', + DataTableRenamed = 'DATA_TABLE_RENAMED', + DataTableReplaced = 'DATA_TABLE_REPLACED', + DataTableRollback = 'DATA_TABLE_ROLLBACK', + DataTableVisualizationSettingsUpdated = 'DATA_TABLE_VISUALIZATION_SETTINGS_UPDATED', FolderAcl = 'FOLDER_ACL', FolderCreated = 'FOLDER_CREATED', FolderDeleted = 'FOLDER_DELETED', @@ -1510,6 +1516,7 @@ export type CreateDataUploadInput = { clientMutationId?: Maybe; contentType?: Maybe; filename?: Maybe; + processingOptions?: Maybe; projectId?: Maybe; replaceTableOfContentsItemId?: Maybe; }; @@ -1904,6 +1911,7 @@ export type CreateMapBookmarkInput = { * payload verbatim. May be used to track mutations by the client. */ clientMutationId?: Maybe; + dataTableStates?: Maybe; isPublic?: Maybe; layerNames?: Maybe; mapDimensions?: Maybe>>; @@ -1995,37 +2003,41 @@ export type CreateOptionalBasemapLayerPayload = { query?: Maybe; }; -/** All input for the create `OriginalSourceId` mutation. */ -export type CreateOriginalSourceIdInput = { +/** All input for the `createOverlayDataTableUpload` mutation. */ +export type CreateOverlayDataTableUploadInput = { /** * An arbitrary string value with no semantic meaning. Will be included in the * payload verbatim. May be used to track mutations by the client. */ clientMutationId?: Maybe; - /** The `OriginalSourceId` to be created by this mutation. */ - originalSourceId: OriginalSourceIdInput; + contentType?: Maybe; + filename?: Maybe; + processingOptions?: Maybe; + replaceOverlayDataTableId?: Maybe; + tocItemId?: Maybe; }; -/** The output of our create `OriginalSourceId` mutation. */ -export type CreateOriginalSourceIdPayload = { - __typename?: 'CreateOriginalSourceIdPayload'; +/** The output of our `createOverlayDataTableUpload` mutation. */ +export type CreateOverlayDataTableUploadPayload = { + __typename?: 'CreateOverlayDataTableUploadPayload'; /** * The exact same `clientMutationId` that was provided in the mutation input, * unchanged and unused. May be used by a client to track mutations. */ clientMutationId?: Maybe; - /** The `OriginalSourceId` that was created by this mutation. */ - originalSourceId?: Maybe; - /** An edge for our `OriginalSourceId`. May be used by Relay 1. */ - originalSourceIdEdge?: Maybe; + overlayDataTableUpload?: Maybe; + /** An edge for our `OverlayDataTableUpload`. May be used by Relay 1. */ + overlayDataTableUploadEdge?: Maybe; + /** Reads a single `ProjectBackgroundJob` that is related to this `OverlayDataTableUpload`. */ + projectBackgroundJob?: Maybe; /** Our root query field type. Allows us to run any query from our mutation payload. */ query?: Maybe; }; -/** The output of our create `OriginalSourceId` mutation. */ -export type CreateOriginalSourceIdPayloadOriginalSourceIdEdgeArgs = { - orderBy?: Maybe>; +/** The output of our `createOverlayDataTableUpload` mutation. */ +export type CreateOverlayDataTableUploadPayloadOverlayDataTableUploadEdgeArgs = { + orderBy?: Maybe>; }; export type CreateProjectGeographyClippingLayerInput = { @@ -2199,39 +2211,6 @@ export type CreateProjectsSharedBasemapPayloadProjectsSharedBasemapEdgeArgs = { orderBy?: Maybe>; }; -/** All input for the create `PublishedTocItemId` mutation. */ -export type CreatePublishedTocItemIdInput = { - /** - * An arbitrary string value with no semantic meaning. Will be included in the - * payload verbatim. May be used to track mutations by the client. - */ - clientMutationId?: Maybe; - /** The `PublishedTocItemId` to be created by this mutation. */ - publishedTocItemId: PublishedTocItemIdInput; -}; - -/** The output of our create `PublishedTocItemId` mutation. */ -export type CreatePublishedTocItemIdPayload = { - __typename?: 'CreatePublishedTocItemIdPayload'; - /** - * The exact same `clientMutationId` that was provided in the mutation input, - * unchanged and unused. May be used by a client to track mutations. - */ - clientMutationId?: Maybe; - /** The `PublishedTocItemId` that was created by this mutation. */ - publishedTocItemId?: Maybe; - /** An edge for our `PublishedTocItemId`. May be used by Relay 1. */ - publishedTocItemIdEdge?: Maybe; - /** Our root query field type. Allows us to run any query from our mutation payload. */ - query?: Maybe; -}; - - -/** The output of our create `PublishedTocItemId` mutation. */ -export type CreatePublishedTocItemIdPayloadPublishedTocItemIdEdgeArgs = { - orderBy?: Maybe>; -}; - /** All input for the `createRemoteGeojsonSource` mutation. */ export type CreateRemoteGeojsonSourceInput = { bounds?: Maybe>>; @@ -3649,6 +3628,7 @@ export type DataUploadOutput = Node & { }; export enum DataUploadOutputType { + Csv = 'CSV', FlatGeobuf = 'FLAT_GEOBUF', GeoJson = 'GEO_JSON', GeoTiff = 'GEO_TIFF', @@ -3680,6 +3660,12 @@ export type DataUploadTask = Node & { outputs?: Maybe; /** Use to upload source data to s3. Must be an admin. */ presignedUploadUrl?: Maybe; + /** + * Format-specific processing instructions supplied by the client at upload time + * (e.g. column mapping and CRS for delimited text uploads). Consumed by the + * spatial-uploads-handler. + */ + processingOptions?: Maybe; /** Reads a single `ProjectBackgroundJob` that is related to this `DataUploadTask`. */ projectBackgroundJob?: Maybe; projectBackgroundJobId: Scalars['UUID']; @@ -5587,6 +5573,11 @@ export type FailDataUploadPayloadDataUploadTaskEdgeArgs = { export type FeatureFlags = { __typename?: 'FeatureFlags'; + /** + * When true, project admins see the Data Tables tab in layer editors + * and related map UI. Controlled from SeaSketch developer settings. + */ + dataTables?: Maybe; iNaturalistLayers?: Maybe; }; @@ -7023,29 +7014,6 @@ export type GetChildFoldersRecursivePayload = { query?: Maybe; }; -/** All input for the `getPublishedCardIdFromDraft` mutation. */ -export type GetPublishedCardIdFromDraftInput = { - /** - * An arbitrary string value with no semantic meaning. Will be included in the - * payload verbatim. May be used to track mutations by the client. - */ - clientMutationId?: Maybe; - draftReportCardId?: Maybe; -}; - -/** The output of our `getPublishedCardIdFromDraft` mutation. */ -export type GetPublishedCardIdFromDraftPayload = { - __typename?: 'GetPublishedCardIdFromDraftPayload'; - /** - * The exact same `clientMutationId` that was provided in the mutation input, - * unchanged and unused. May be used by a client to track mutations. - */ - clientMutationId?: Maybe; - integer?: Maybe; - /** Our root query field type. Allows us to run any query from our mutation payload. */ - query?: Maybe; -}; - export type GoogleMapsTileApiSession = Node & { __typename?: 'GoogleMapsTileApiSession'; expiresAt: Scalars['Datetime']; @@ -7695,6 +7663,8 @@ export type MapBookmark = { /** Generated by clients. Should not be used if authorative thumbnail (image_id) is available. */ clientGeneratedThumbnail?: Maybe; createdAt: Scalars['Datetime']; + /** JSON map of TOC stableId -> { stableId, column?, op?, filters? } for activated overlay data tables. */ + dataTableStates: Scalars['JSON']; id: Scalars['UUID']; imageId?: Maybe; isPublic: Scalars['Boolean']; @@ -7981,8 +7951,7 @@ export type Mutation = { createOfflineTileSetting?: Maybe; /** Creates a single `OptionalBasemapLayer`. */ createOptionalBasemapLayer?: Maybe; - /** Creates a single `OriginalSourceId`. */ - createOriginalSourceId?: Maybe; + createOverlayDataTableUpload?: Maybe; createPost: Post; /** * Users with verified emails can create new projects by choosing a unique name @@ -8012,8 +7981,6 @@ export type Mutation = { * layers have not yet been created. */ createProjectWithGeographies?: Maybe; - /** Creates a single `PublishedTocItemId`. */ - createPublishedTocItemId?: Maybe; createRemoteGeojsonSource?: Maybe; createRemoteMvtSource?: Maybe; /** Creates a single `Report`. */ @@ -8207,7 +8174,6 @@ export type Mutation = { */ getOrCreateSprite?: Maybe; getPresignedPMTilesUploadUrl: PresignedUrl; - getPublishedCardIdFromDraft?: Maybe; /** Give a user admin access to a project. User must have already joined the project and shared their user profile. */ grantAdminAccess?: Maybe; importArcgisServices?: Maybe; @@ -8266,6 +8232,7 @@ export type Mutation = { removeUserFromGroup?: Maybe; /** Remove a SketchClass from the list of valid children for a Collection. */ removeValidChildSketchClass?: Maybe; + renameOverlayDataTable?: Maybe; renameReportTab?: Maybe; reopenResolvableLayerComment?: Maybe; reorderReportTabCards?: Maybe; @@ -8285,6 +8252,7 @@ export type Mutation = { /** Remove participant admin privileges. */ revokeAdminAccess?: Maybe; revokeApiKey?: Maybe; + rollbackOverlayDataTableVersion?: Maybe; rollbackToArchivedSource?: Maybe; /** Send all UNSENT invites in the current project. */ sendAllProjectInvites?: Maybe; @@ -8320,6 +8288,7 @@ export type Mutation = { * forum IDs in the correct order. Missing ids will be added to the end of the list. */ setForumOrder?: Maybe; + setOverlayDataTableVisualizationSettings?: Maybe; /** * Admins can use this function to hide the contents of a message. Message will * still appear in the client with the missing content, and should link to the @@ -8344,9 +8313,11 @@ export type Mutation = { setUserGroups?: Maybe; /** Superusers only. Promote a sprite to be globally available. */ shareSprite?: Maybe; + softDeleteOverlayDataTable?: Maybe; /** Superusers only. "Deletes" a sprite but keeps it in the DB in case layers are already referencing it. */ softDeleteSprite?: Maybe; submitDataUpload?: Maybe; + submitOverlayDataTableUpload?: Maybe; /** * Toggle admin access for the given project and user. User must have already * joined the project and shared their user profile. @@ -8816,8 +8787,8 @@ export type MutationCreateOptionalBasemapLayerArgs = { /** The root mutation type which contains root level fields which mutate data. */ -export type MutationCreateOriginalSourceIdArgs = { - input: CreateOriginalSourceIdInput; +export type MutationCreateOverlayDataTableUploadArgs = { + input: CreateOverlayDataTableUploadInput; }; @@ -8858,12 +8829,6 @@ export type MutationCreateProjectWithGeographiesArgs = { }; -/** The root mutation type which contains root level fields which mutate data. */ -export type MutationCreatePublishedTocItemIdArgs = { - input: CreatePublishedTocItemIdInput; -}; - - /** The root mutation type which contains root level fields which mutate data. */ export type MutationCreateRemoteGeojsonSourceArgs = { input: CreateRemoteGeojsonSourceInput; @@ -9454,12 +9419,6 @@ export type MutationGetPresignedPmTilesUploadUrlArgs = { }; -/** The root mutation type which contains root level fields which mutate data. */ -export type MutationGetPublishedCardIdFromDraftArgs = { - input: GetPublishedCardIdFromDraftInput; -}; - - /** The root mutation type which contains root level fields which mutate data. */ export type MutationGrantAdminAccessArgs = { input: GrantAdminAccessInput; @@ -9598,6 +9557,12 @@ export type MutationRemoveValidChildSketchClassArgs = { }; +/** The root mutation type which contains root level fields which mutate data. */ +export type MutationRenameOverlayDataTableArgs = { + input: RenameOverlayDataTableInput; +}; + + /** The root mutation type which contains root level fields which mutate data. */ export type MutationRenameReportTabArgs = { input: RenameReportTabInput; @@ -9671,6 +9636,12 @@ export type MutationRevokeApiKeyArgs = { }; +/** The root mutation type which contains root level fields which mutate data. */ +export type MutationRollbackOverlayDataTableVersionArgs = { + input: RollbackOverlayDataTableVersionInput; +}; + + /** The root mutation type which contains root level fields which mutate data. */ export type MutationRollbackToArchivedSourceArgs = { input: RollbackToArchivedSourceInput; @@ -9746,6 +9717,12 @@ export type MutationSetForumOrderArgs = { }; +/** The root mutation type which contains root level fields which mutate data. */ +export type MutationSetOverlayDataTableVisualizationSettingsArgs = { + input: SetOverlayDataTableVisualizationSettingsInput; +}; + + /** The root mutation type which contains root level fields which mutate data. */ export type MutationSetPostHiddenByModeratorArgs = { input: SetPostHiddenByModeratorInput; @@ -9797,6 +9774,12 @@ export type MutationShareSpriteArgs = { }; +/** The root mutation type which contains root level fields which mutate data. */ +export type MutationSoftDeleteOverlayDataTableArgs = { + input: SoftDeleteOverlayDataTableInput; +}; + + /** The root mutation type which contains root level fields which mutate data. */ export type MutationSoftDeleteSpriteArgs = { input: SoftDeleteSpriteInput; @@ -9809,6 +9792,12 @@ export type MutationSubmitDataUploadArgs = { }; +/** The root mutation type which contains root level fields which mutate data. */ +export type MutationSubmitOverlayDataTableUploadArgs = { + input: SubmitOverlayDataTableUploadInput; +}; + + /** The root mutation type which contains root level fields which mutate data. */ export type MutationToggleAdminAccessArgs = { input: ToggleAdminAccessInput; @@ -10658,50 +10647,162 @@ export enum OptionalBasemapLayersOrderBy { PrimaryKeyDesc = 'PRIMARY_KEY_DESC' } -export type OriginalSourceId = { - __typename?: 'OriginalSourceId'; - dataSourceId?: Maybe; +export type OutstandingSurveyInvites = { + __typename?: 'OutstandingSurveyInvites'; + projectId: Scalars['Int']; + surveyId: Scalars['Int']; + token: Scalars['String']; }; -/** An input for mutations affecting `OriginalSourceId` */ -export type OriginalSourceIdInput = { - dataSourceId?: Maybe; +export type OverlayDataTable = Node & { + __typename?: 'OverlayDataTable'; + columnStatsRemote: Scalars['String']; + columnStatsUrl?: Maybe; + createdAt?: Maybe; + createdBy: Scalars['Int']; + deletedAt?: Maybe; + id: Scalars['Int']; + joinColumn: Scalars['String']; + name: Scalars['String']; + /** A globally unique identifier. Can be used in various places throughout the system to identify this single value. */ + nodeId: Scalars['ID']; + overlayJoinColumn: Scalars['String']; + parquetRemote: Scalars['String']; + parquetUrl?: Maybe; + /** Reads a single `Project` that is related to this `OverlayDataTable`. */ + project?: Maybe; + projectId: Scalars['Int']; + queryUrl?: Maybe; + replacedById?: Maybe; + /** + * Columns that must appear as filters when this table is displayed on the map. + * End users can change the filter values but cannot remove these filters. + */ + requiredFilterColumns?: Maybe>>; + rowCount: Scalars['Int']; + /** + * Stable logical identity for a data table across version replace and TOC + * publish. Draft and published copies share the same UUID. + */ + stableId: Scalars['UUID']; + tableOfContentsItemId: Scalars['Int']; + updatedAt?: Maybe; + version: Scalars['Int']; + /** Columns that may/should be used for creating thematic maps. For example `count` or `density` */ + visualizationColumns?: Maybe>>; + /** Operations that may/should be used for creating thematic maps. For example `mean` or `max` */ + visualizationOps?: Maybe>>; +}; + +/** + * A condition to be used against `OverlayDataTable` object types. All fields are + * tested for equality and combined with a logical ‘and.’ + */ +export type OverlayDataTableCondition = { + /** Checks for equality with the object’s `id` field. */ + id?: Maybe; + /** Checks for equality with the object’s `projectId` field. */ + projectId?: Maybe; }; -/** A connection to a list of `OriginalSourceId` values. */ -export type OriginalSourceIdsConnection = { - __typename?: 'OriginalSourceIdsConnection'; - /** A list of edges which contains the `OriginalSourceId` and cursor to aid in pagination. */ - edges: Array; - /** A list of `OriginalSourceId` objects. */ - nodes: Array; +export type OverlayDataTableUpload = Node & { + __typename?: 'OverlayDataTableUpload'; + contentType: Scalars['String']; + createdAt?: Maybe; + errorDetails?: Maybe; + filename: Scalars['String']; + id: Scalars['UUID']; + /** A globally unique identifier. Can be used in various places throughout the system to identify this single value. */ + nodeId: Scalars['ID']; + overlayGeostats: Scalars['JSON']; + overlayJoinColumn?: Maybe; + presignedUploadUrl?: Maybe; + processingOptions: Scalars['JSON']; + /** Reads a single `ProjectBackgroundJob` that is related to this `OverlayDataTableUpload`. */ + projectBackgroundJob?: Maybe; + projectBackgroundJobId: Scalars['UUID']; + replaceOverlayDataTableId?: Maybe; + tableOfContentsItemId: Scalars['Int']; + updatedAt?: Maybe; +}; + +/** + * A condition to be used against `OverlayDataTableUpload` object types. All fields + * are tested for equality and combined with a logical ‘and.’ + */ +export type OverlayDataTableUploadCondition = { + /** Checks for equality with the object’s `id` field. */ + id?: Maybe; + /** Checks for equality with the object’s `projectBackgroundJobId` field. */ + projectBackgroundJobId?: Maybe; +}; + +/** A connection to a list of `OverlayDataTableUpload` values. */ +export type OverlayDataTableUploadsConnection = { + __typename?: 'OverlayDataTableUploadsConnection'; + /** A list of edges which contains the `OverlayDataTableUpload` and cursor to aid in pagination. */ + edges: Array; + /** A list of `OverlayDataTableUpload` objects. */ + nodes: Array; /** Information to aid in pagination. */ pageInfo: PageInfo; - /** The count of *all* `OriginalSourceId` you could get from the connection. */ + /** The count of *all* `OverlayDataTableUpload` you could get from the connection. */ totalCount: Scalars['Int']; }; -/** A `OriginalSourceId` edge in the connection. */ -export type OriginalSourceIdsEdge = { - __typename?: 'OriginalSourceIdsEdge'; +/** A `OverlayDataTableUpload` edge in the connection. */ +export type OverlayDataTableUploadsEdge = { + __typename?: 'OverlayDataTableUploadsEdge'; /** A cursor for use in pagination. */ cursor?: Maybe; - /** The `OriginalSourceId` at the end of the edge. */ - node: OriginalSourceId; + /** The `OverlayDataTableUpload` at the end of the edge. */ + node: OverlayDataTableUpload; }; -/** Methods to use when ordering `OriginalSourceId`. */ -export enum OriginalSourceIdsOrderBy { - Natural = 'NATURAL' +/** Methods to use when ordering `OverlayDataTableUpload`. */ +export enum OverlayDataTableUploadsOrderBy { + IdAsc = 'ID_ASC', + IdDesc = 'ID_DESC', + Natural = 'NATURAL', + PrimaryKeyAsc = 'PRIMARY_KEY_ASC', + PrimaryKeyDesc = 'PRIMARY_KEY_DESC', + ProjectBackgroundJobIdAsc = 'PROJECT_BACKGROUND_JOB_ID_ASC', + ProjectBackgroundJobIdDesc = 'PROJECT_BACKGROUND_JOB_ID_DESC' } -export type OutstandingSurveyInvites = { - __typename?: 'OutstandingSurveyInvites'; - projectId: Scalars['Int']; - surveyId: Scalars['Int']; - token: Scalars['String']; +/** A connection to a list of `OverlayDataTable` values. */ +export type OverlayDataTablesConnection = { + __typename?: 'OverlayDataTablesConnection'; + /** A list of edges which contains the `OverlayDataTable` and cursor to aid in pagination. */ + edges: Array; + /** A list of `OverlayDataTable` objects. */ + nodes: Array; + /** Information to aid in pagination. */ + pageInfo: PageInfo; + /** The count of *all* `OverlayDataTable` you could get from the connection. */ + totalCount: Scalars['Int']; }; +/** A `OverlayDataTable` edge in the connection. */ +export type OverlayDataTablesEdge = { + __typename?: 'OverlayDataTablesEdge'; + /** A cursor for use in pagination. */ + cursor?: Maybe; + /** The `OverlayDataTable` at the end of the edge. */ + node: OverlayDataTable; +}; + +/** Methods to use when ordering `OverlayDataTable`. */ +export enum OverlayDataTablesOrderBy { + IdAsc = 'ID_ASC', + IdDesc = 'ID_DESC', + Natural = 'NATURAL', + PrimaryKeyAsc = 'PRIMARY_KEY_ASC', + PrimaryKeyDesc = 'PRIMARY_KEY_DESC', + ProjectIdAsc = 'PROJECT_ID_ASC', + ProjectIdDesc = 'PROJECT_ID_DESC' +} + /** Information about pagination in a connection. */ export type PageInfo = { __typename?: 'PageInfo'; @@ -11028,10 +11129,11 @@ export type Project = Node & { hideOverlays: Scalars['Boolean']; hideSketches: Scalars['Boolean']; /** - * Content-addressed tileset UUIDs whose /v2 requests must include a map - * access token. Equals hosted UUIDs from published TOC items (and draft - * TOC items for project admins) that are not in the published ACL - * document's public list. Empty for projects with no protected overlays. + * Content-addressed tileset UUIDs whose hosted tiles/uploads requests + * must include a map access token. Equals hosted UUIDs from published + * TOC items (and draft TOC items for project admins) that are not in the + * published ACL document's public list. Empty for projects with no + * protected overlays. */ hostedTileUuidsRequiringAuth: Array; id: Scalars['Int']; @@ -11079,7 +11181,7 @@ export type Project = Node & { logoUrl?: Maybe; /** * Short-lived (90m) JWT for authorized overlay tile requests on - * tiles.seasketch.org/v2/{ns}/.... Returns null if the user is not signed in. + * hosted tiles/uploads URLs with access_token (and optional ns). Returns null if the user is not signed in. * Claims include role (admin|user) and project group ids. */ mapAccessToken?: Maybe; @@ -11100,6 +11202,8 @@ export type Project = Node & { offlineTilePackagesConnection: OfflineTilePackagesConnection; /** Reads and enables pagination through a set of `OfflineTileSetting`. */ offlineTileSettings: Array; + /** Reads and enables pagination through a set of `OverlayDataTable`. */ + overlayDataTablesConnection: OverlayDataTablesConnection; /** Count of all users who have opted into participating in the project, sharing their profile with project administrators. */ participantCount?: Maybe; /** @@ -11519,6 +11623,21 @@ export type ProjectOfflineTileSettingsArgs = { }; +/** + * SeaSketch Project type. This root type contains most of the fields and queries + * needed to drive the application. + */ +export type ProjectOverlayDataTablesConnectionArgs = { + after?: Maybe; + before?: Maybe; + condition?: Maybe; + first?: Maybe; + last?: Maybe; + offset?: Maybe; + orderBy?: Maybe>; +}; + + /** * SeaSketch Project type. This root type contains most of the fields and queries * needed to drive the application. @@ -11730,6 +11849,13 @@ export type ProjectBackgroundJob = Node & { id: Scalars['UUID']; /** A globally unique identifier. Can be used in various places throughout the system to identify this single value. */ nodeId: Scalars['ID']; + /** Reads a single `OverlayDataTableUpload` that is related to this `ProjectBackgroundJob`. */ + overlayDataTableUpload?: Maybe; + /** + * Reads and enables pagination through a set of `OverlayDataTableUpload`. + * @deprecated Please use overlayDataTableUpload instead + */ + overlayDataTableUploadsConnection: OverlayDataTableUploadsConnection; progress?: Maybe; progressMessage: Scalars['String']; /** Reads a single `Project` that is related to this `ProjectBackgroundJob`. */ @@ -11754,6 +11880,17 @@ export type ProjectBackgroundJobDataUploadTasksConnectionArgs = { orderBy?: Maybe>; }; + +export type ProjectBackgroundJobOverlayDataTableUploadsConnectionArgs = { + after?: Maybe; + before?: Maybe; + condition?: Maybe; + first?: Maybe; + last?: Maybe; + offset?: Maybe; + orderBy?: Maybe>; +}; + /** * A condition to be used against `ProjectBackgroundJob` object types. All fields * are tested for equality and combined with a logical ‘and.’ @@ -11784,6 +11921,7 @@ export type ProjectBackgroundJobSubscriptionPayload = { export enum ProjectBackgroundJobType { ArcgisImport = 'ARCGIS_IMPORT', ConsolidateDataSources = 'CONSOLIDATE_DATA_SOURCES', + DataTableUpload = 'DATA_TABLE_UPLOAD', DataUpload = 'DATA_UPLOAD', ReplacementUpload = 'REPLACEMENT_UPLOAD' } @@ -12282,43 +12420,6 @@ export type PublishTableOfContentsPayload = { tableOfContentsItems?: Maybe>; }; -export type PublishedTocItemId = { - __typename?: 'PublishedTocItemId'; - id?: Maybe; -}; - -/** An input for mutations affecting `PublishedTocItemId` */ -export type PublishedTocItemIdInput = { - id?: Maybe; -}; - -/** A connection to a list of `PublishedTocItemId` values. */ -export type PublishedTocItemIdsConnection = { - __typename?: 'PublishedTocItemIdsConnection'; - /** A list of edges which contains the `PublishedTocItemId` and cursor to aid in pagination. */ - edges: Array; - /** A list of `PublishedTocItemId` objects. */ - nodes: Array; - /** Information to aid in pagination. */ - pageInfo: PageInfo; - /** The count of *all* `PublishedTocItemId` you could get from the connection. */ - totalCount: Scalars['Int']; -}; - -/** A `PublishedTocItemId` edge in the connection. */ -export type PublishedTocItemIdsEdge = { - __typename?: 'PublishedTocItemIdsEdge'; - /** A cursor for use in pagination. */ - cursor?: Maybe; - /** The `PublishedTocItemId` at the end of the edge. */ - node: PublishedTocItemId; -}; - -/** Methods to use when ordering `PublishedTocItemId`. */ -export enum PublishedTocItemIdsOrderBy { - Natural = 'NATURAL' -} - /** The root query type which gives access points into the data universe. */ export type Query = Node & { __typename?: 'Query'; @@ -12498,8 +12599,19 @@ export type Query = Node & { optionalBasemapLayer?: Maybe; /** Reads a single `OptionalBasemapLayer` using its globally unique `ID`. */ optionalBasemapLayerByNodeId?: Maybe; - /** Reads and enables pagination through a set of `OriginalSourceId`. */ - originalSourceIdsConnection?: Maybe; + overlayDataTable?: Maybe; + /** Reads a single `OverlayDataTable` using its globally unique `ID`. */ + overlayDataTableByNodeId?: Maybe; + overlayDataTableLinkedTocIsDraft?: Maybe; + overlayDataTableParquetPublicUrl?: Maybe; + /** Reads and enables pagination through a set of `OverlayDataTable`. */ + overlayDataTablesConnection?: Maybe; + overlayDataTableUpload?: Maybe; + /** Reads a single `OverlayDataTableUpload` using its globally unique `ID`. */ + overlayDataTableUploadByNodeId?: Maybe; + overlayDataTableUploadByProjectBackgroundJobId?: Maybe; + /** Reads and enables pagination through a set of `OverlayDataTableUpload`. */ + overlayDataTableUploadsConnection?: Maybe; post?: Maybe; /** Reads a single `Post` using its globally unique `ID`. */ postByNodeId?: Maybe; @@ -12531,8 +12643,6 @@ export type Query = Node & { projectsSharedBasemapsConnection?: Maybe; /** Used by project administrators to access a list of public sprites promoted by the SeaSketch development team. */ publicSprites?: Maybe>; - /** Reads and enables pagination through a set of `PublishedTocItemId`. */ - publishedTocItemIdsConnection?: Maybe; /** * Exposes the root query type nested one level down. This is helpful for Relay 1 * which can only query top level fields if they are in a particular form. @@ -13387,13 +13497,69 @@ export type QueryOptionalBasemapLayerByNodeIdArgs = { /** The root query type which gives access points into the data universe. */ -export type QueryOriginalSourceIdsConnectionArgs = { +export type QueryOverlayDataTableArgs = { + id: Scalars['Int']; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableByNodeIdArgs = { + nodeId: Scalars['ID']; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableLinkedTocIsDraftArgs = { + pid?: Maybe; + tocItemId?: Maybe; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableParquetPublicUrlArgs = { + pRemote?: Maybe; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTablesConnectionArgs = { + after?: Maybe; + before?: Maybe; + condition?: Maybe; + first?: Maybe; + last?: Maybe; + offset?: Maybe; + orderBy?: Maybe>; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableUploadArgs = { + id: Scalars['UUID']; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableUploadByNodeIdArgs = { + nodeId: Scalars['ID']; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableUploadByProjectBackgroundJobIdArgs = { + projectBackgroundJobId: Scalars['UUID']; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableUploadsConnectionArgs = { after?: Maybe; before?: Maybe; + condition?: Maybe; first?: Maybe; last?: Maybe; offset?: Maybe; - orderBy?: Maybe>; + orderBy?: Maybe>; }; @@ -13554,17 +13720,6 @@ export type QueryPublicSpritesArgs = { }; -/** The root query type which gives access points into the data universe. */ -export type QueryPublishedTocItemIdsConnectionArgs = { - after?: Maybe; - before?: Maybe; - first?: Maybe; - last?: Maybe; - offset?: Maybe; - orderBy?: Maybe>; -}; - - /** The root query type which gives access points into the data universe. */ export type QueryReportArgs = { id: Scalars['Int']; @@ -14097,6 +14252,40 @@ export type RemoveValidChildSketchClassPayload = { query?: Maybe; }; +/** All input for the `renameOverlayDataTable` mutation. */ +export type RenameOverlayDataTableInput = { + /** + * An arbitrary string value with no semantic meaning. Will be included in the + * payload verbatim. May be used to track mutations by the client. + */ + clientMutationId?: Maybe; + newName?: Maybe; + tableId?: Maybe; +}; + +/** The output of our `renameOverlayDataTable` mutation. */ +export type RenameOverlayDataTablePayload = { + __typename?: 'RenameOverlayDataTablePayload'; + /** + * The exact same `clientMutationId` that was provided in the mutation input, + * unchanged and unused. May be used by a client to track mutations. + */ + clientMutationId?: Maybe; + overlayDataTable?: Maybe; + /** An edge for our `OverlayDataTable`. May be used by Relay 1. */ + overlayDataTableEdge?: Maybe; + /** Reads a single `Project` that is related to this `OverlayDataTable`. */ + project?: Maybe; + /** Our root query field type. Allows us to run any query from our mutation payload. */ + query?: Maybe; +}; + + +/** The output of our `renameOverlayDataTable` mutation. */ +export type RenameOverlayDataTablePayloadOverlayDataTableEdgeArgs = { + orderBy?: Maybe>; +}; + /** All input for the `renameReportTab` mutation. */ export type RenameReportTabInput = { alternateLanguageSettings?: Maybe; @@ -14633,6 +14822,39 @@ export type RevokeApiKeyPayload = { query?: Maybe; }; +/** All input for the `rollbackOverlayDataTableVersion` mutation. */ +export type RollbackOverlayDataTableVersionInput = { + /** + * An arbitrary string value with no semantic meaning. Will be included in the + * payload verbatim. May be used to track mutations by the client. + */ + clientMutationId?: Maybe; + tableId?: Maybe; +}; + +/** The output of our `rollbackOverlayDataTableVersion` mutation. */ +export type RollbackOverlayDataTableVersionPayload = { + __typename?: 'RollbackOverlayDataTableVersionPayload'; + /** + * The exact same `clientMutationId` that was provided in the mutation input, + * unchanged and unused. May be used by a client to track mutations. + */ + clientMutationId?: Maybe; + overlayDataTable?: Maybe; + /** An edge for our `OverlayDataTable`. May be used by Relay 1. */ + overlayDataTableEdge?: Maybe; + /** Reads a single `Project` that is related to this `OverlayDataTable`. */ + project?: Maybe; + /** Our root query field type. Allows us to run any query from our mutation payload. */ + query?: Maybe; +}; + + +/** The output of our `rollbackOverlayDataTableVersion` mutation. */ +export type RollbackOverlayDataTableVersionPayloadOverlayDataTableEdgeArgs = { + orderBy?: Maybe>; +}; + /** All input for the `rollbackToArchivedSource` mutation. */ export type RollbackToArchivedSourceInput = { /** @@ -14855,6 +15077,42 @@ export type SetForumOrderPayload = { query?: Maybe; }; +/** All input for the `setOverlayDataTableVisualizationSettings` mutation. */ +export type SetOverlayDataTableVisualizationSettingsInput = { + /** + * An arbitrary string value with no semantic meaning. Will be included in the + * payload verbatim. May be used to track mutations by the client. + */ + clientMutationId?: Maybe; + requiredFilterColumns?: Maybe>>; + tableId?: Maybe; + visualizationColumns?: Maybe>>; + visualizationOps?: Maybe>>; +}; + +/** The output of our `setOverlayDataTableVisualizationSettings` mutation. */ +export type SetOverlayDataTableVisualizationSettingsPayload = { + __typename?: 'SetOverlayDataTableVisualizationSettingsPayload'; + /** + * The exact same `clientMutationId` that was provided in the mutation input, + * unchanged and unused. May be used by a client to track mutations. + */ + clientMutationId?: Maybe; + overlayDataTable?: Maybe; + /** An edge for our `OverlayDataTable`. May be used by Relay 1. */ + overlayDataTableEdge?: Maybe; + /** Reads a single `Project` that is related to this `OverlayDataTable`. */ + project?: Maybe; + /** Our root query field type. Allows us to run any query from our mutation payload. */ + query?: Maybe; +}; + + +/** The output of our `setOverlayDataTableVisualizationSettings` mutation. */ +export type SetOverlayDataTableVisualizationSettingsPayloadOverlayDataTableEdgeArgs = { + orderBy?: Maybe>; +}; + /** All input for the `setPostHiddenByModerator` mutation. */ export type SetPostHiddenByModeratorInput = { /** @@ -15519,6 +15777,39 @@ export type SketchesRelatedFragmentsRecord = { sketches?: Maybe>>; }; +/** All input for the `softDeleteOverlayDataTable` mutation. */ +export type SoftDeleteOverlayDataTableInput = { + /** + * An arbitrary string value with no semantic meaning. Will be included in the + * payload verbatim. May be used to track mutations by the client. + */ + clientMutationId?: Maybe; + tableId?: Maybe; +}; + +/** The output of our `softDeleteOverlayDataTable` mutation. */ +export type SoftDeleteOverlayDataTablePayload = { + __typename?: 'SoftDeleteOverlayDataTablePayload'; + /** + * The exact same `clientMutationId` that was provided in the mutation input, + * unchanged and unused. May be used by a client to track mutations. + */ + clientMutationId?: Maybe; + overlayDataTable?: Maybe; + /** An edge for our `OverlayDataTable`. May be used by Relay 1. */ + overlayDataTableEdge?: Maybe; + /** Reads a single `Project` that is related to this `OverlayDataTable`. */ + project?: Maybe; + /** Our root query field type. Allows us to run any query from our mutation payload. */ + query?: Maybe; +}; + + +/** The output of our `softDeleteOverlayDataTable` mutation. */ +export type SoftDeleteOverlayDataTablePayloadOverlayDataTableEdgeArgs = { + orderBy?: Maybe>; +}; + /** All input for the `softDeleteSprite` mutation. */ export type SoftDeleteSpriteInput = { /** @@ -15742,6 +16033,31 @@ export type SubmitDataUploadPayload = { query?: Maybe; }; +/** All input for the `submitOverlayDataTableUpload` mutation. */ +export type SubmitOverlayDataTableUploadInput = { + /** + * An arbitrary string value with no semantic meaning. Will be included in the + * payload verbatim. May be used to track mutations by the client. + */ + clientMutationId?: Maybe; + jobId?: Maybe; +}; + +/** The output of our `submitOverlayDataTableUpload` mutation. */ +export type SubmitOverlayDataTableUploadPayload = { + __typename?: 'SubmitOverlayDataTableUploadPayload'; + /** + * The exact same `clientMutationId` that was provided in the mutation input, + * unchanged and unused. May be used by a client to track mutations. + */ + clientMutationId?: Maybe; + /** Reads a single `Project` that is related to this `ProjectBackgroundJob`. */ + project?: Maybe; + projectBackgroundJob?: Maybe; + /** Our root query field type. Allows us to run any query from our mutation payload. */ + query?: Maybe; +}; + /** The root subscription type: contains realtime events you can subscribe to with the `subscription` operation. */ export type Subscription = { __typename?: 'Subscription'; @@ -16289,8 +16605,14 @@ export type TableOfContentsItem = Node & { /** If is_folder=false, a DataLayers visibility will be controlled by this item */ dataLayerId?: Maybe; dataSourceType?: Maybe; + /** Reads and enables pagination through a set of `ChangeLog`. */ + dataTableChangeLogs?: Maybe>; + /** Overlay attribute name used as the canonical feature ID for linked data tables. Required when enable_data_tables is true. */ + dataTableJoinColumn?: Maybe; /** Reads and enables pagination through a set of `DownloadOption`. */ downloadOptions?: Maybe>; + /** When true, admins can attach CSV data tables linked to this layer by a canonical join column. */ + enableDataTables: Scalars['Boolean']; enableDownload: Scalars['Boolean']; ftsAr?: Maybe; ftsDa?: Maybe; @@ -16339,6 +16661,8 @@ export type TableOfContentsItem = Node & { metadataXml?: Maybe; /** A globally unique identifier. Can be used in various places throughout the system to identify this single value. */ nodeId: Scalars['ID']; + /** Reads and enables pagination through a set of `OverlayDataTable`. */ + overlayDataTables?: Maybe>; /** * stable_id of the parent folder, if any. This property cannot be changed * directly. To rearrange items into folders, use the @@ -16429,6 +16753,22 @@ export type TableOfContentsItemChangeLogsArgs = { }; +/** + * TableOfContentsItems represent a tree-view of folders and operational layers + * that can be added to the map. Both layers and folders may be nested into other + * folders for organization, and each folder has its own access control list. + * + * Items that represent data layers have a `DataLayer` relation, which in turn has + * a reference to a `DataSource`. Usually these relations should be fetched in + * batch only once the layer is turned on, using the + * `dataLayersAndSourcesByLayerId` query. + */ +export type TableOfContentsItemDataTableChangeLogsArgs = { + first?: Maybe; + offset?: Maybe; +}; + + /** * TableOfContentsItems represent a tree-view of folders and operational layers * that can be added to the map. Both layers and folders may be nested into other @@ -16461,6 +16801,22 @@ export type TableOfContentsItemMetadataChangeLogsArgs = { }; +/** + * TableOfContentsItems represent a tree-view of folders and operational layers + * that can be added to the map. Both layers and folders may be nested into other + * folders for organization, and each folder has its own access control list. + * + * Items that represent data layers have a `DataLayer` relation, which in turn has + * a reference to a `DataSource`. Usually these relations should be fetched in + * batch only once the layer is turned on, using the + * `dataLayersAndSourcesByLayerId` query. + */ +export type TableOfContentsItemOverlayDataTablesArgs = { + first?: Maybe; + offset?: Maybe; +}; + + /** * TableOfContentsItems represent a tree-view of folders and operational layers * that can be added to the map. Both layers and folders may be nested into other @@ -16633,6 +16989,10 @@ export type TableOfContentsItemPatch = { bounds?: Maybe>>; /** If is_folder=false, a DataLayers visibility will be controlled by this item */ dataLayerId?: Maybe; + /** Overlay attribute name used as the canonical feature ID for linked data tables. Required when enable_data_tables is true. */ + dataTableJoinColumn?: Maybe; + /** When true, admins can attach CSV data tables linked to this layer by a canonical join column. */ + enableDataTables?: Maybe; enableDownload?: Maybe; geoprocessingReferenceId?: Maybe; hideChildren?: Maybe; @@ -20545,6 +20905,7 @@ export type CreateDataUploadMutationVariables = Exact<{ filename: Scalars['String']; contentType: Scalars['String']; replaceTableOfContentsItemId?: Maybe; + processingOptions?: Maybe; }>; @@ -20573,6 +20934,9 @@ export type JobDetailsFragment = ( { __typename?: 'TableOfContentsItem' } & Pick )> } + )>, overlayDataTableUpload?: Maybe<( + { __typename?: 'OverlayDataTableUpload' } + & Pick )> } ); @@ -21173,6 +21537,27 @@ export type LayerSettingsChangeLogQuery = ( )> } ); +export type DataTableChangeLogQueryVariables = Exact<{ + id: Scalars['Int']; + first: Scalars['Int']; +}>; + + +export type DataTableChangeLogQuery = ( + { __typename?: 'Query' } + & { tableOfContentsItem?: Maybe<( + { __typename?: 'TableOfContentsItem' } + & Pick + & { overlayDataTables?: Maybe + )>>, dataTableChangeLogs?: Maybe> } + )> } +); + export type ResolvableLayerCommentDetailsFragment = ( { __typename?: 'ResolvableLayerComment' } & Pick @@ -21209,7 +21594,7 @@ export type ResolvableLayerCommentThreadForChangelogQuery = ( export type FullAdminOverlayFragment = ( { __typename?: 'TableOfContentsItem' } - & Pick + & Pick & { acl?: Maybe<( { __typename?: 'Acl' } & Pick @@ -21222,11 +21607,14 @@ export type FullAdminOverlayFragment = ( & Pick )>>>, projectBackgroundJobs?: Maybe + & JobDetailsFragment )>>, dataLayer?: Maybe<( { __typename?: 'DataLayer' } & FullAdminDataLayerFragment - )>, relatedReportCardDetails?: Maybe, overlayDataTables?: Maybe>, relatedReportCardDetails?: Maybe & { sketchClass: ( @@ -21296,6 +21684,24 @@ export type UpdateEnableDownloadMutation = ( )> } ); +export type UpdateDataTablesSettingsMutationVariables = Exact<{ + id: Scalars['Int']; + enableDataTables?: Maybe; + dataTableJoinColumn?: Maybe; +}>; + + +export type UpdateDataTablesSettingsMutation = ( + { __typename?: 'Mutation' } + & { updateTableOfContentsItem?: Maybe<( + { __typename?: 'UpdateTableOfContentsItemPayload' } + & { tableOfContentsItem?: Maybe<( + { __typename?: 'TableOfContentsItem' } + & Pick + )> } + )> } +); + export type UpdateLayerMutationVariables = Exact<{ id: Scalars['Int']; renderUnder?: Maybe; @@ -22071,7 +22477,10 @@ export type ChangeLogsSinceLastPublishQuery = ( & { unresolvedComment?: Maybe<( { __typename?: 'ResolvableLayerComment' } & ResolvableLayerCommentThreadFragment - )>, dataLayer?: Maybe<( + )>, overlayDataTables?: Maybe + )>>, dataLayer?: Maybe<( { __typename?: 'DataLayer' } & Pick & { dataSource?: Maybe<( @@ -22530,7 +22939,7 @@ export type JobFragment = ( export type MapBookmarkDetailsFragment = ( { __typename?: 'MapBookmark' } - & Pick + & Pick & { job?: Maybe<( { __typename?: 'WorkerJob' } & JobFragment @@ -22565,6 +22974,7 @@ export type CreateMapBookmarkMutationVariables = Exact<{ layerNames: Scalars['JSON']; sketchNames: Scalars['JSON']; clientGeneratedThumbnail: Scalars['String']; + dataTableStates?: Maybe; }>; @@ -23279,6 +23689,158 @@ export type GetTilePackageQuery = ( )> } ); +export type ClientOverlayDataTableFragment = ( + { __typename?: 'OverlayDataTable' } + & Pick +); + +export type OverlayDataTableDetailsFragment = ( + { __typename?: 'OverlayDataTable' } + & Pick +); + +export type OverlayDataTableVisualizationMetadataQueryVariables = Exact<{ + id: Scalars['Int']; +}>; + + +export type OverlayDataTableVisualizationMetadataQuery = ( + { __typename?: 'Query' } + & { overlayDataTable?: Maybe<( + { __typename?: 'OverlayDataTable' } + & Pick + )> } +); + +export type OverlayDataTableVisualizationMetadataForLayerQueryVariables = Exact<{ + tocItemId: Scalars['Int']; +}>; + + +export type OverlayDataTableVisualizationMetadataForLayerQuery = ( + { __typename?: 'Query' } + & { tableOfContentsItem?: Maybe<( + { __typename?: 'TableOfContentsItem' } + & Pick + & { overlayDataTables?: Maybe + )>> } + )> } +); + +export type OverlayDataTableUploadDetailsFragment = ( + { __typename?: 'OverlayDataTableUpload' } + & Pick +); + +export type CreateOverlayDataTableUploadMutationVariables = Exact<{ + tableOfContentsItemId: Scalars['Int']; + filename: Scalars['String']; + contentType: Scalars['String']; + processingOptions?: Maybe; + replaceOverlayDataTableId?: Maybe; +}>; + + +export type CreateOverlayDataTableUploadMutation = ( + { __typename?: 'Mutation' } + & { createOverlayDataTableUpload?: Maybe<( + { __typename?: 'CreateOverlayDataTableUploadPayload' } + & { overlayDataTableUpload?: Maybe<( + { __typename?: 'OverlayDataTableUpload' } + & OverlayDataTableUploadDetailsFragment + )>, projectBackgroundJob?: Maybe<( + { __typename?: 'ProjectBackgroundJob' } + & JobDetailsFragment + )> } + )> } +); + +export type SubmitOverlayDataTableUploadMutationVariables = Exact<{ + jobId: Scalars['UUID']; +}>; + + +export type SubmitOverlayDataTableUploadMutation = ( + { __typename?: 'Mutation' } + & { submitOverlayDataTableUpload?: Maybe<( + { __typename?: 'SubmitOverlayDataTableUploadPayload' } + & { projectBackgroundJob?: Maybe<( + { __typename?: 'ProjectBackgroundJob' } + & Pick + )> } + )> } +); + +export type RenameOverlayDataTableMutationVariables = Exact<{ + id: Scalars['Int']; + name: Scalars['String']; +}>; + + +export type RenameOverlayDataTableMutation = ( + { __typename?: 'Mutation' } + & { renameOverlayDataTable?: Maybe<( + { __typename?: 'RenameOverlayDataTablePayload' } + & { overlayDataTable?: Maybe<( + { __typename?: 'OverlayDataTable' } + & OverlayDataTableDetailsFragment + )> } + )> } +); + +export type SoftDeleteOverlayDataTableMutationVariables = Exact<{ + id: Scalars['Int']; +}>; + + +export type SoftDeleteOverlayDataTableMutation = ( + { __typename?: 'Mutation' } + & { softDeleteOverlayDataTable?: Maybe<( + { __typename?: 'SoftDeleteOverlayDataTablePayload' } + & { overlayDataTable?: Maybe<( + { __typename?: 'OverlayDataTable' } + & Pick + )> } + )> } +); + +export type RollbackOverlayDataTableVersionMutationVariables = Exact<{ + id: Scalars['Int']; +}>; + + +export type RollbackOverlayDataTableVersionMutation = ( + { __typename?: 'Mutation' } + & { rollbackOverlayDataTableVersion?: Maybe<( + { __typename?: 'RollbackOverlayDataTableVersionPayload' } + & { overlayDataTable?: Maybe<( + { __typename?: 'OverlayDataTable' } + & OverlayDataTableDetailsFragment + )> } + )> } +); + +export type SetOverlayDataTableVisualizationSettingsMutationVariables = Exact<{ + id: Scalars['Int']; + visualizationColumns: Array> | Maybe; + visualizationOps: Array> | Maybe; + requiredFilterColumns: Array> | Maybe; +}>; + + +export type SetOverlayDataTableVisualizationSettingsMutation = ( + { __typename?: 'Mutation' } + & { setOverlayDataTableVisualizationSettings?: Maybe<( + { __typename?: 'SetOverlayDataTableVisualizationSettingsPayload' } + & { overlayDataTable?: Maybe<( + { __typename?: 'OverlayDataTable' } + & OverlayDataTableDetailsFragment + )> } + )> } +); + export type ProjectAccessControlSettingsQueryVariables = Exact<{ slug: Scalars['String']; }>; @@ -23492,7 +24054,7 @@ export type ProjectMetadataFragment = ( & Pick )>>>, featureFlags?: Maybe<( { __typename?: 'FeatureFlags' } - & Pick + & Pick )> } ); @@ -23629,7 +24191,10 @@ export type OverlayFragment = ( & { acl?: Maybe<( { __typename?: 'Acl' } & Pick - )> } + )>, overlayDataTables?: Maybe> } ); export type PublishedTableOfContentsQueryVariables = Exact<{ @@ -26527,7 +27092,7 @@ export type UpdateFeatureFlagsMutation = ( & Pick & { featureFlags?: Maybe<( { __typename?: 'FeatureFlags' } - & Pick + & Pick )> } )> } )> } @@ -27568,6 +28133,12 @@ export const JobDetailsFragmentDoc = gql` title } } + overlayDataTableUpload { + id + tableOfContentsItemId + filename + replaceOverlayDataTableId + } } ${DataUploadDetailsFragmentDoc}`; export const DataUploadExtendedDetailsFragmentDoc = gql` @@ -27600,6 +28171,22 @@ export const BackgroundJobSubscriptionEventFragmentDoc = gql` } } ${JobDetailsFragmentDoc}`; +export const ClientOverlayDataTableFragmentDoc = gql` + fragment ClientOverlayDataTable on OverlayDataTable { + id + stableId + name + version + rowCount + joinColumn + overlayJoinColumn + queryUrl + columnStatsUrl + visualizationColumns + visualizationOps + requiredFilterColumns +} + `; export const OverlayFragmentDoc = gql` fragment Overlay on TableOfContentsItem { id @@ -27623,8 +28210,11 @@ export const OverlayFragmentDoc = gql` hasMetadata primaryDownloadUrl dataSourceType + overlayDataTables { + ...ClientOverlayDataTable + } } - `; + ${ClientOverlayDataTableFragmentDoc}`; export const AdminOverlayFragmentDoc = gql` fragment AdminOverlay on TableOfContentsItem { ...Overlay @@ -27845,6 +28435,29 @@ export const FullAdminDataLayerFragmentDoc = gql` } ${FullAdminSourceFragmentDoc} ${ArchivedSourceFragmentDoc}`; +export const OverlayDataTableDetailsFragmentDoc = gql` + fragment OverlayDataTableDetails on OverlayDataTable { + id + stableId + name + version + joinColumn + overlayJoinColumn + rowCount + parquetRemote + columnStatsRemote + parquetUrl + columnStatsUrl + queryUrl + deletedAt + replacedById + createdAt + updatedAt + visualizationColumns + visualizationOps + requiredFilterColumns +} + `; export const UserProfileDetailsFragmentDoc = gql` fragment UserProfileDetails on Profile { userId @@ -27908,22 +28521,21 @@ export const FullAdminOverlayFragmentDoc = gql` stableId title enableDownload + enableDataTables + dataTableJoinColumn geoprocessingReferenceId copiedFromDataLibraryTemplateId primaryDownloadUrl projectBackgroundJobs { - id - type - title - state - progress - progressMessage - errorMessage + ...JobDetails } hasOriginalSourceUpload dataLayer { ...FullAdminDataLayer } + overlayDataTables { + ...OverlayDataTableDetails + } relatedReportCardDetails { isDraft title @@ -27940,7 +28552,9 @@ export const FullAdminOverlayFragmentDoc = gql` ...ResolvableLayerCommentThread } } - ${FullAdminDataLayerFragmentDoc} + ${JobDetailsFragmentDoc} +${FullAdminDataLayerFragmentDoc} +${OverlayDataTableDetailsFragmentDoc} ${ResolvableLayerCommentThreadFragmentDoc}`; export const MetadataXmlFileFragmentDoc = gql` fragment MetadataXmlFile on DataUploadOutput { @@ -28005,6 +28619,7 @@ export const MapBookmarkDetailsFragmentDoc = gql` basemapName sketchNames clientGeneratedThumbnail + dataTableStates } ${JobFragmentDoc}`; export const FileUploadDetailsFragmentDoc = gql` @@ -28251,6 +28866,20 @@ export const OfflineTileSettingsFragmentDoc = gql` } } `; +export const OverlayDataTableUploadDetailsFragmentDoc = gql` + fragment OverlayDataTableUploadDetails on OverlayDataTableUpload { + id + tableOfContentsItemId + filename + contentType + processingOptions + overlayJoinColumn + errorDetails + presignedUploadUrl + replaceOverlayDataTableId + projectBackgroundJobId +} + `; export const ProjectMetadataFragmentDoc = gql` fragment ProjectMetadata on Project { id @@ -28299,6 +28928,7 @@ export const ProjectMetadataFragmentDoc = gql` showLegendByDefault featureFlags { iNaturalistLayers + dataTables } } `; @@ -31546,9 +32176,9 @@ export type DashboardBannerStatsQueryHookResult = ReturnType; export type DashboardBannerStatsQueryResult = Apollo.QueryResult; export const CreateDataUploadDocument = gql` - mutation createDataUpload($projectId: Int!, $filename: String!, $contentType: String!, $replaceTableOfContentsItemId: Int) { + mutation createDataUpload($projectId: Int!, $filename: String!, $contentType: String!, $replaceTableOfContentsItemId: Int, $processingOptions: JSON) { createDataUpload( - input: {filename: $filename, projectId: $projectId, contentType: $contentType, replaceTableOfContentsItemId: $replaceTableOfContentsItemId} + input: {filename: $filename, projectId: $projectId, contentType: $contentType, replaceTableOfContentsItemId: $replaceTableOfContentsItemId, processingOptions: $processingOptions} ) { dataUploadTask { ...DataUploadExtendedDetails @@ -31576,6 +32206,7 @@ export type CreateDataUploadMutationFn = Apollo.MutationFunction; export type LayerSettingsChangeLogLazyQueryHookResult = ReturnType; export type LayerSettingsChangeLogQueryResult = Apollo.QueryResult; +export const DataTableChangeLogDocument = gql` + query DataTableChangeLog($id: Int!, $first: Int!) { + tableOfContentsItem(id: $id) { + id + overlayDataTables { + id + name + version + } + dataTableChangeLogs(first: $first) { + ...ChangeLogDetails + } + } +} + ${ChangeLogDetailsFragmentDoc}`; + +/** + * __useDataTableChangeLogQuery__ + * + * To run a query within a React component, call `useDataTableChangeLogQuery` and pass it any options that fit your needs. + * When your component renders, `useDataTableChangeLogQuery` returns an object from Apollo Client that contains loading, error, and data properties + * you can use to render your UI. + * + * @param baseOptions options that will be passed into the query, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options; + * + * @example + * const { data, loading, error } = useDataTableChangeLogQuery({ + * variables: { + * id: // value for 'id' + * first: // value for 'first' + * }, + * }); + */ +export function useDataTableChangeLogQuery(baseOptions: Apollo.QueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useQuery(DataTableChangeLogDocument, options); + } +export function useDataTableChangeLogLazyQuery(baseOptions?: Apollo.LazyQueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useLazyQuery(DataTableChangeLogDocument, options); + } +export type DataTableChangeLogQueryHookResult = ReturnType; +export type DataTableChangeLogLazyQueryHookResult = ReturnType; +export type DataTableChangeLogQueryResult = Apollo.QueryResult; export const ResolvableLayerCommentThreadForChangelogDocument = gql` query ResolvableLayerCommentThreadForChangelog($commentId: Int!) { getResolvableLayerComment(commentId: $commentId) { @@ -32883,6 +33558,47 @@ export function useUpdateEnableDownloadMutation(baseOptions?: Apollo.MutationHoo export type UpdateEnableDownloadMutationHookResult = ReturnType; export type UpdateEnableDownloadMutationResult = Apollo.MutationResult; export type UpdateEnableDownloadMutationOptions = Apollo.BaseMutationOptions; +export const UpdateDataTablesSettingsDocument = gql` + mutation UpdateDataTablesSettings($id: Int!, $enableDataTables: Boolean, $dataTableJoinColumn: String) { + updateTableOfContentsItem( + input: {id: $id, patch: {enableDataTables: $enableDataTables, dataTableJoinColumn: $dataTableJoinColumn}} + ) { + tableOfContentsItem { + id + enableDataTables + dataTableJoinColumn + } + } +} + `; +export type UpdateDataTablesSettingsMutationFn = Apollo.MutationFunction; + +/** + * __useUpdateDataTablesSettingsMutation__ + * + * To run a mutation, you first call `useUpdateDataTablesSettingsMutation` within a React component and pass it any options that fit your needs. + * When your component renders, `useUpdateDataTablesSettingsMutation` returns a tuple that includes: + * - A mutate function that you can call at any time to execute the mutation + * - An object with fields that represent the current status of the mutation's execution + * + * @param baseOptions options that will be passed into the mutation, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options-2; + * + * @example + * const [updateDataTablesSettingsMutation, { data, loading, error }] = useUpdateDataTablesSettingsMutation({ + * variables: { + * id: // value for 'id' + * enableDataTables: // value for 'enableDataTables' + * dataTableJoinColumn: // value for 'dataTableJoinColumn' + * }, + * }); + */ +export function useUpdateDataTablesSettingsMutation(baseOptions?: Apollo.MutationHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useMutation(UpdateDataTablesSettingsDocument, options); + } +export type UpdateDataTablesSettingsMutationHookResult = ReturnType; +export type UpdateDataTablesSettingsMutationResult = Apollo.MutationResult; +export type UpdateDataTablesSettingsMutationOptions = Apollo.BaseMutationOptions; export const UpdateLayerDocument = gql` mutation UpdateLayer($id: Int!, $renderUnder: RenderUnderType, $mapboxGlStyles: JSON, $sublayer: String, $staticId: String) { updateDataLayer( @@ -34614,6 +35330,10 @@ export const ChangeLogsSinceLastPublishDocument = gql` unresolvedComment { ...ResolvableLayerCommentThread } + overlayDataTables { + id + name + } dataLayer { id dataSource { @@ -35403,9 +36123,9 @@ export type GetBookmarkQueryHookResult = ReturnType; export type GetBookmarkLazyQueryHookResult = ReturnType; export type GetBookmarkQueryResult = Apollo.QueryResult; export const CreateMapBookmarkDocument = gql` - mutation CreateMapBookmark($slug: String!, $isPublic: Boolean!, $basemapOptionalLayerStates: JSON, $visibleDataLayers: [String!]!, $cameraOptions: JSON!, $selectedBasemap: Int!, $style: JSON!, $mapDimensions: [Int!]!, $visibleSketches: [Int!]!, $sidebarState: JSON, $basemapName: String!, $layerNames: JSON!, $sketchNames: JSON!, $clientGeneratedThumbnail: String!) { + mutation CreateMapBookmark($slug: String!, $isPublic: Boolean!, $basemapOptionalLayerStates: JSON, $visibleDataLayers: [String!]!, $cameraOptions: JSON!, $selectedBasemap: Int!, $style: JSON!, $mapDimensions: [Int!]!, $visibleSketches: [Int!]!, $sidebarState: JSON, $basemapName: String!, $layerNames: JSON!, $sketchNames: JSON!, $clientGeneratedThumbnail: String!, $dataTableStates: JSON) { createMapBookmark( - input: {isPublic: $isPublic, slug: $slug, basemapOptionalLayerStates: $basemapOptionalLayerStates, visibleDataLayers: $visibleDataLayers, cameraOptions: $cameraOptions, selectedBasemap: $selectedBasemap, style: $style, mapDimensions: $mapDimensions, visibleSketches: $visibleSketches, sidebarState: $sidebarState, basemapName: $basemapName, layerNames: $layerNames, sketchNames: $sketchNames, clientGeneratedThumbnail: $clientGeneratedThumbnail} + input: {isPublic: $isPublic, slug: $slug, basemapOptionalLayerStates: $basemapOptionalLayerStates, visibleDataLayers: $visibleDataLayers, cameraOptions: $cameraOptions, selectedBasemap: $selectedBasemap, style: $style, mapDimensions: $mapDimensions, visibleSketches: $visibleSketches, sidebarState: $sidebarState, basemapName: $basemapName, layerNames: $layerNames, sketchNames: $sketchNames, clientGeneratedThumbnail: $clientGeneratedThumbnail, dataTableStates: $dataTableStates} ) { mapBookmark { ...MapBookmarkDetails @@ -35442,6 +36162,7 @@ export type CreateMapBookmarkMutationFn = Apollo.MutationFunction; export type GetTilePackageLazyQueryHookResult = ReturnType; export type GetTilePackageQueryResult = Apollo.QueryResult; +export const OverlayDataTableVisualizationMetadataDocument = gql` + query OverlayDataTableVisualizationMetadata($id: Int!) { + overlayDataTable(id: $id) { + id + queryUrl + columnStatsUrl + visualizationColumns + visualizationOps + requiredFilterColumns + } +} + `; + +/** + * __useOverlayDataTableVisualizationMetadataQuery__ + * + * To run a query within a React component, call `useOverlayDataTableVisualizationMetadataQuery` and pass it any options that fit your needs. + * When your component renders, `useOverlayDataTableVisualizationMetadataQuery` returns an object from Apollo Client that contains loading, error, and data properties + * you can use to render your UI. + * + * @param baseOptions options that will be passed into the query, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options; + * + * @example + * const { data, loading, error } = useOverlayDataTableVisualizationMetadataQuery({ + * variables: { + * id: // value for 'id' + * }, + * }); + */ +export function useOverlayDataTableVisualizationMetadataQuery(baseOptions: Apollo.QueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useQuery(OverlayDataTableVisualizationMetadataDocument, options); + } +export function useOverlayDataTableVisualizationMetadataLazyQuery(baseOptions?: Apollo.LazyQueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useLazyQuery(OverlayDataTableVisualizationMetadataDocument, options); + } +export type OverlayDataTableVisualizationMetadataQueryHookResult = ReturnType; +export type OverlayDataTableVisualizationMetadataLazyQueryHookResult = ReturnType; +export type OverlayDataTableVisualizationMetadataQueryResult = Apollo.QueryResult; +export const OverlayDataTableVisualizationMetadataForLayerDocument = gql` + query OverlayDataTableVisualizationMetadataForLayer($tocItemId: Int!) { + tableOfContentsItem(id: $tocItemId) { + id + overlayDataTables { + id + queryUrl + columnStatsUrl + visualizationColumns + visualizationOps + requiredFilterColumns + } + } +} + `; + +/** + * __useOverlayDataTableVisualizationMetadataForLayerQuery__ + * + * To run a query within a React component, call `useOverlayDataTableVisualizationMetadataForLayerQuery` and pass it any options that fit your needs. + * When your component renders, `useOverlayDataTableVisualizationMetadataForLayerQuery` returns an object from Apollo Client that contains loading, error, and data properties + * you can use to render your UI. + * + * @param baseOptions options that will be passed into the query, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options; + * + * @example + * const { data, loading, error } = useOverlayDataTableVisualizationMetadataForLayerQuery({ + * variables: { + * tocItemId: // value for 'tocItemId' + * }, + * }); + */ +export function useOverlayDataTableVisualizationMetadataForLayerQuery(baseOptions: Apollo.QueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useQuery(OverlayDataTableVisualizationMetadataForLayerDocument, options); + } +export function useOverlayDataTableVisualizationMetadataForLayerLazyQuery(baseOptions?: Apollo.LazyQueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useLazyQuery(OverlayDataTableVisualizationMetadataForLayerDocument, options); + } +export type OverlayDataTableVisualizationMetadataForLayerQueryHookResult = ReturnType; +export type OverlayDataTableVisualizationMetadataForLayerLazyQueryHookResult = ReturnType; +export type OverlayDataTableVisualizationMetadataForLayerQueryResult = Apollo.QueryResult; +export const CreateOverlayDataTableUploadDocument = gql` + mutation CreateOverlayDataTableUpload($tableOfContentsItemId: Int!, $filename: String!, $contentType: String!, $processingOptions: JSON, $replaceOverlayDataTableId: Int) { + createOverlayDataTableUpload( + input: {tocItemId: $tableOfContentsItemId, filename: $filename, contentType: $contentType, processingOptions: $processingOptions, replaceOverlayDataTableId: $replaceOverlayDataTableId} + ) { + overlayDataTableUpload { + ...OverlayDataTableUploadDetails + } + projectBackgroundJob { + ...JobDetails + } + } +} + ${OverlayDataTableUploadDetailsFragmentDoc} +${JobDetailsFragmentDoc}`; +export type CreateOverlayDataTableUploadMutationFn = Apollo.MutationFunction; + +/** + * __useCreateOverlayDataTableUploadMutation__ + * + * To run a mutation, you first call `useCreateOverlayDataTableUploadMutation` within a React component and pass it any options that fit your needs. + * When your component renders, `useCreateOverlayDataTableUploadMutation` returns a tuple that includes: + * - A mutate function that you can call at any time to execute the mutation + * - An object with fields that represent the current status of the mutation's execution + * + * @param baseOptions options that will be passed into the mutation, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options-2; + * + * @example + * const [createOverlayDataTableUploadMutation, { data, loading, error }] = useCreateOverlayDataTableUploadMutation({ + * variables: { + * tableOfContentsItemId: // value for 'tableOfContentsItemId' + * filename: // value for 'filename' + * contentType: // value for 'contentType' + * processingOptions: // value for 'processingOptions' + * replaceOverlayDataTableId: // value for 'replaceOverlayDataTableId' + * }, + * }); + */ +export function useCreateOverlayDataTableUploadMutation(baseOptions?: Apollo.MutationHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useMutation(CreateOverlayDataTableUploadDocument, options); + } +export type CreateOverlayDataTableUploadMutationHookResult = ReturnType; +export type CreateOverlayDataTableUploadMutationResult = Apollo.MutationResult; +export type CreateOverlayDataTableUploadMutationOptions = Apollo.BaseMutationOptions; +export const SubmitOverlayDataTableUploadDocument = gql` + mutation SubmitOverlayDataTableUpload($jobId: UUID!) { + submitOverlayDataTableUpload(input: {jobId: $jobId}) { + projectBackgroundJob { + id + state + progress + progressMessage + } + } +} + `; +export type SubmitOverlayDataTableUploadMutationFn = Apollo.MutationFunction; + +/** + * __useSubmitOverlayDataTableUploadMutation__ + * + * To run a mutation, you first call `useSubmitOverlayDataTableUploadMutation` within a React component and pass it any options that fit your needs. + * When your component renders, `useSubmitOverlayDataTableUploadMutation` returns a tuple that includes: + * - A mutate function that you can call at any time to execute the mutation + * - An object with fields that represent the current status of the mutation's execution + * + * @param baseOptions options that will be passed into the mutation, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options-2; + * + * @example + * const [submitOverlayDataTableUploadMutation, { data, loading, error }] = useSubmitOverlayDataTableUploadMutation({ + * variables: { + * jobId: // value for 'jobId' + * }, + * }); + */ +export function useSubmitOverlayDataTableUploadMutation(baseOptions?: Apollo.MutationHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useMutation(SubmitOverlayDataTableUploadDocument, options); + } +export type SubmitOverlayDataTableUploadMutationHookResult = ReturnType; +export type SubmitOverlayDataTableUploadMutationResult = Apollo.MutationResult; +export type SubmitOverlayDataTableUploadMutationOptions = Apollo.BaseMutationOptions; +export const RenameOverlayDataTableDocument = gql` + mutation RenameOverlayDataTable($id: Int!, $name: String!) { + renameOverlayDataTable(input: {tableId: $id, newName: $name}) { + overlayDataTable { + ...OverlayDataTableDetails + } + } +} + ${OverlayDataTableDetailsFragmentDoc}`; +export type RenameOverlayDataTableMutationFn = Apollo.MutationFunction; + +/** + * __useRenameOverlayDataTableMutation__ + * + * To run a mutation, you first call `useRenameOverlayDataTableMutation` within a React component and pass it any options that fit your needs. + * When your component renders, `useRenameOverlayDataTableMutation` returns a tuple that includes: + * - A mutate function that you can call at any time to execute the mutation + * - An object with fields that represent the current status of the mutation's execution + * + * @param baseOptions options that will be passed into the mutation, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options-2; + * + * @example + * const [renameOverlayDataTableMutation, { data, loading, error }] = useRenameOverlayDataTableMutation({ + * variables: { + * id: // value for 'id' + * name: // value for 'name' + * }, + * }); + */ +export function useRenameOverlayDataTableMutation(baseOptions?: Apollo.MutationHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useMutation(RenameOverlayDataTableDocument, options); + } +export type RenameOverlayDataTableMutationHookResult = ReturnType; +export type RenameOverlayDataTableMutationResult = Apollo.MutationResult; +export type RenameOverlayDataTableMutationOptions = Apollo.BaseMutationOptions; +export const SoftDeleteOverlayDataTableDocument = gql` + mutation SoftDeleteOverlayDataTable($id: Int!) { + softDeleteOverlayDataTable(input: {tableId: $id}) { + overlayDataTable { + id + deletedAt + } + } +} + `; +export type SoftDeleteOverlayDataTableMutationFn = Apollo.MutationFunction; + +/** + * __useSoftDeleteOverlayDataTableMutation__ + * + * To run a mutation, you first call `useSoftDeleteOverlayDataTableMutation` within a React component and pass it any options that fit your needs. + * When your component renders, `useSoftDeleteOverlayDataTableMutation` returns a tuple that includes: + * - A mutate function that you can call at any time to execute the mutation + * - An object with fields that represent the current status of the mutation's execution + * + * @param baseOptions options that will be passed into the mutation, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options-2; + * + * @example + * const [softDeleteOverlayDataTableMutation, { data, loading, error }] = useSoftDeleteOverlayDataTableMutation({ + * variables: { + * id: // value for 'id' + * }, + * }); + */ +export function useSoftDeleteOverlayDataTableMutation(baseOptions?: Apollo.MutationHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useMutation(SoftDeleteOverlayDataTableDocument, options); + } +export type SoftDeleteOverlayDataTableMutationHookResult = ReturnType; +export type SoftDeleteOverlayDataTableMutationResult = Apollo.MutationResult; +export type SoftDeleteOverlayDataTableMutationOptions = Apollo.BaseMutationOptions; +export const RollbackOverlayDataTableVersionDocument = gql` + mutation RollbackOverlayDataTableVersion($id: Int!) { + rollbackOverlayDataTableVersion(input: {tableId: $id}) { + overlayDataTable { + ...OverlayDataTableDetails + } + } +} + ${OverlayDataTableDetailsFragmentDoc}`; +export type RollbackOverlayDataTableVersionMutationFn = Apollo.MutationFunction; + +/** + * __useRollbackOverlayDataTableVersionMutation__ + * + * To run a mutation, you first call `useRollbackOverlayDataTableVersionMutation` within a React component and pass it any options that fit your needs. + * When your component renders, `useRollbackOverlayDataTableVersionMutation` returns a tuple that includes: + * - A mutate function that you can call at any time to execute the mutation + * - An object with fields that represent the current status of the mutation's execution + * + * @param baseOptions options that will be passed into the mutation, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options-2; + * + * @example + * const [rollbackOverlayDataTableVersionMutation, { data, loading, error }] = useRollbackOverlayDataTableVersionMutation({ + * variables: { + * id: // value for 'id' + * }, + * }); + */ +export function useRollbackOverlayDataTableVersionMutation(baseOptions?: Apollo.MutationHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useMutation(RollbackOverlayDataTableVersionDocument, options); + } +export type RollbackOverlayDataTableVersionMutationHookResult = ReturnType; +export type RollbackOverlayDataTableVersionMutationResult = Apollo.MutationResult; +export type RollbackOverlayDataTableVersionMutationOptions = Apollo.BaseMutationOptions; +export const SetOverlayDataTableVisualizationSettingsDocument = gql` + mutation SetOverlayDataTableVisualizationSettings($id: Int!, $visualizationColumns: [String]!, $visualizationOps: [String]!, $requiredFilterColumns: [String]!) { + setOverlayDataTableVisualizationSettings( + input: {tableId: $id, visualizationColumns: $visualizationColumns, visualizationOps: $visualizationOps, requiredFilterColumns: $requiredFilterColumns} + ) { + overlayDataTable { + ...OverlayDataTableDetails + } + } +} + ${OverlayDataTableDetailsFragmentDoc}`; +export type SetOverlayDataTableVisualizationSettingsMutationFn = Apollo.MutationFunction; + +/** + * __useSetOverlayDataTableVisualizationSettingsMutation__ + * + * To run a mutation, you first call `useSetOverlayDataTableVisualizationSettingsMutation` within a React component and pass it any options that fit your needs. + * When your component renders, `useSetOverlayDataTableVisualizationSettingsMutation` returns a tuple that includes: + * - A mutate function that you can call at any time to execute the mutation + * - An object with fields that represent the current status of the mutation's execution + * + * @param baseOptions options that will be passed into the mutation, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options-2; + * + * @example + * const [setOverlayDataTableVisualizationSettingsMutation, { data, loading, error }] = useSetOverlayDataTableVisualizationSettingsMutation({ + * variables: { + * id: // value for 'id' + * visualizationColumns: // value for 'visualizationColumns' + * visualizationOps: // value for 'visualizationOps' + * requiredFilterColumns: // value for 'requiredFilterColumns' + * }, + * }); + */ +export function useSetOverlayDataTableVisualizationSettingsMutation(baseOptions?: Apollo.MutationHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useMutation(SetOverlayDataTableVisualizationSettingsDocument, options); + } +export type SetOverlayDataTableVisualizationSettingsMutationHookResult = ReturnType; +export type SetOverlayDataTableVisualizationSettingsMutationResult = Apollo.MutationResult; +export type SetOverlayDataTableVisualizationSettingsMutationOptions = Apollo.BaseMutationOptions; export const ProjectAccessControlSettingsDocument = gql` query ProjectAccessControlSettings($slug: String!) { projectBySlug(slug: $slug) { @@ -42384,6 +43418,7 @@ export const UpdateFeatureFlagsDocument = gql` id featureFlags { iNaturalistLayers + dataTables } } } @@ -43620,6 +44655,7 @@ export const namedOperations = { layersAndSourcesForItems: 'layersAndSourcesForItems', GetFolder: 'GetFolder', LayerSettingsChangeLog: 'LayerSettingsChangeLog', + DataTableChangeLog: 'DataTableChangeLog', ResolvableLayerCommentThreadForChangelog: 'ResolvableLayerCommentThreadForChangelog', GetLayerItem: 'GetLayerItem', InteractivitySettingsForLayer: 'InteractivitySettingsForLayer', @@ -43658,6 +44694,8 @@ export const namedOperations = { OfflineSurveyMaps: 'OfflineSurveyMaps', BasemapOfflineSettings: 'BasemapOfflineSettings', getTilePackage: 'getTilePackage', + OverlayDataTableVisualizationMetadata: 'OverlayDataTableVisualizationMetadata', + OverlayDataTableVisualizationMetadataForLayer: 'OverlayDataTableVisualizationMetadataForLayer', ProjectAccessControlSettings: 'ProjectAccessControlSettings', ProjectDashboard: 'ProjectDashboard', ProjectDashboardBannerStats: 'ProjectDashboardBannerStats', @@ -43777,6 +44815,7 @@ export const namedOperations = { UpdateFolder: 'UpdateFolder', UpdateTableOfContentsItem: 'UpdateTableOfContentsItem', UpdateEnableDownload: 'UpdateEnableDownload', + UpdateDataTablesSettings: 'UpdateDataTablesSettings', UpdateLayer: 'UpdateLayer', UpdateDataSource: 'UpdateDataSource', UpdateInteractivitySettings: 'UpdateInteractivitySettings', @@ -43824,6 +44863,12 @@ export const namedOperations = { UpdateBasemapOfflineTileSettings: 'UpdateBasemapOfflineTileSettings', generateOfflineTilePackage: 'generateOfflineTilePackage', deleteTilePackage: 'deleteTilePackage', + CreateOverlayDataTableUpload: 'CreateOverlayDataTableUpload', + SubmitOverlayDataTableUpload: 'SubmitOverlayDataTableUpload', + RenameOverlayDataTable: 'RenameOverlayDataTable', + SoftDeleteOverlayDataTable: 'SoftDeleteOverlayDataTable', + RollbackOverlayDataTableVersion: 'RollbackOverlayDataTableVersion', + SetOverlayDataTableVisualizationSettings: 'SetOverlayDataTableVisualizationSettings', updateProjectAccessControlSettings: 'updateProjectAccessControlSettings', toggleLanguageSupport: 'toggleLanguageSupport', setTranslatedProps: 'setTranslatedProps', @@ -43998,6 +45043,9 @@ export const namedOperations = { OfflineBasemapDetails: 'OfflineBasemapDetails', OfflineTileSettingsForCalculation: 'OfflineTileSettingsForCalculation', OfflineTileSettings: 'OfflineTileSettings', + ClientOverlayDataTable: 'ClientOverlayDataTable', + OverlayDataTableDetails: 'OverlayDataTableDetails', + OverlayDataTableUploadDetails: 'OverlayDataTableUploadDetails', ProjectMetadata: 'ProjectMetadata', ProjectPublicDetailsMetadata: 'ProjectPublicDetailsMetadata', ProjectMetadataMeFrag: 'ProjectMetadataMeFrag', diff --git a/packages/client/src/generated/queries.ts b/packages/client/src/generated/queries.ts index dbdea2551..9d72b2a6b 100644 --- a/packages/client/src/generated/queries.ts +++ b/packages/client/src/generated/queries.ts @@ -954,6 +954,12 @@ export type ChangeLogCondition = { }; export enum ChangeLogFieldGroup { + DataTableCreated = 'DATA_TABLE_CREATED', + DataTableDeleted = 'DATA_TABLE_DELETED', + DataTableRenamed = 'DATA_TABLE_RENAMED', + DataTableReplaced = 'DATA_TABLE_REPLACED', + DataTableRollback = 'DATA_TABLE_ROLLBACK', + DataTableVisualizationSettingsUpdated = 'DATA_TABLE_VISUALIZATION_SETTINGS_UPDATED', FolderAcl = 'FOLDER_ACL', FolderCreated = 'FOLDER_CREATED', FolderDeleted = 'FOLDER_DELETED', @@ -1508,6 +1514,7 @@ export type CreateDataUploadInput = { clientMutationId?: Maybe; contentType?: Maybe; filename?: Maybe; + processingOptions?: Maybe; projectId?: Maybe; replaceTableOfContentsItemId?: Maybe; }; @@ -1902,6 +1909,7 @@ export type CreateMapBookmarkInput = { * payload verbatim. May be used to track mutations by the client. */ clientMutationId?: Maybe; + dataTableStates?: Maybe; isPublic?: Maybe; layerNames?: Maybe; mapDimensions?: Maybe>>; @@ -1993,37 +2001,41 @@ export type CreateOptionalBasemapLayerPayload = { query?: Maybe; }; -/** All input for the create `OriginalSourceId` mutation. */ -export type CreateOriginalSourceIdInput = { +/** All input for the `createOverlayDataTableUpload` mutation. */ +export type CreateOverlayDataTableUploadInput = { /** * An arbitrary string value with no semantic meaning. Will be included in the * payload verbatim. May be used to track mutations by the client. */ clientMutationId?: Maybe; - /** The `OriginalSourceId` to be created by this mutation. */ - originalSourceId: OriginalSourceIdInput; + contentType?: Maybe; + filename?: Maybe; + processingOptions?: Maybe; + replaceOverlayDataTableId?: Maybe; + tocItemId?: Maybe; }; -/** The output of our create `OriginalSourceId` mutation. */ -export type CreateOriginalSourceIdPayload = { - __typename?: 'CreateOriginalSourceIdPayload'; +/** The output of our `createOverlayDataTableUpload` mutation. */ +export type CreateOverlayDataTableUploadPayload = { + __typename?: 'CreateOverlayDataTableUploadPayload'; /** * The exact same `clientMutationId` that was provided in the mutation input, * unchanged and unused. May be used by a client to track mutations. */ clientMutationId?: Maybe; - /** The `OriginalSourceId` that was created by this mutation. */ - originalSourceId?: Maybe; - /** An edge for our `OriginalSourceId`. May be used by Relay 1. */ - originalSourceIdEdge?: Maybe; + overlayDataTableUpload?: Maybe; + /** An edge for our `OverlayDataTableUpload`. May be used by Relay 1. */ + overlayDataTableUploadEdge?: Maybe; + /** Reads a single `ProjectBackgroundJob` that is related to this `OverlayDataTableUpload`. */ + projectBackgroundJob?: Maybe; /** Our root query field type. Allows us to run any query from our mutation payload. */ query?: Maybe; }; -/** The output of our create `OriginalSourceId` mutation. */ -export type CreateOriginalSourceIdPayloadOriginalSourceIdEdgeArgs = { - orderBy?: Maybe>; +/** The output of our `createOverlayDataTableUpload` mutation. */ +export type CreateOverlayDataTableUploadPayloadOverlayDataTableUploadEdgeArgs = { + orderBy?: Maybe>; }; export type CreateProjectGeographyClippingLayerInput = { @@ -2197,39 +2209,6 @@ export type CreateProjectsSharedBasemapPayloadProjectsSharedBasemapEdgeArgs = { orderBy?: Maybe>; }; -/** All input for the create `PublishedTocItemId` mutation. */ -export type CreatePublishedTocItemIdInput = { - /** - * An arbitrary string value with no semantic meaning. Will be included in the - * payload verbatim. May be used to track mutations by the client. - */ - clientMutationId?: Maybe; - /** The `PublishedTocItemId` to be created by this mutation. */ - publishedTocItemId: PublishedTocItemIdInput; -}; - -/** The output of our create `PublishedTocItemId` mutation. */ -export type CreatePublishedTocItemIdPayload = { - __typename?: 'CreatePublishedTocItemIdPayload'; - /** - * The exact same `clientMutationId` that was provided in the mutation input, - * unchanged and unused. May be used by a client to track mutations. - */ - clientMutationId?: Maybe; - /** The `PublishedTocItemId` that was created by this mutation. */ - publishedTocItemId?: Maybe; - /** An edge for our `PublishedTocItemId`. May be used by Relay 1. */ - publishedTocItemIdEdge?: Maybe; - /** Our root query field type. Allows us to run any query from our mutation payload. */ - query?: Maybe; -}; - - -/** The output of our create `PublishedTocItemId` mutation. */ -export type CreatePublishedTocItemIdPayloadPublishedTocItemIdEdgeArgs = { - orderBy?: Maybe>; -}; - /** All input for the `createRemoteGeojsonSource` mutation. */ export type CreateRemoteGeojsonSourceInput = { bounds?: Maybe>>; @@ -3647,6 +3626,7 @@ export type DataUploadOutput = Node & { }; export enum DataUploadOutputType { + Csv = 'CSV', FlatGeobuf = 'FLAT_GEOBUF', GeoJson = 'GEO_JSON', GeoTiff = 'GEO_TIFF', @@ -3678,6 +3658,12 @@ export type DataUploadTask = Node & { outputs?: Maybe; /** Use to upload source data to s3. Must be an admin. */ presignedUploadUrl?: Maybe; + /** + * Format-specific processing instructions supplied by the client at upload time + * (e.g. column mapping and CRS for delimited text uploads). Consumed by the + * spatial-uploads-handler. + */ + processingOptions?: Maybe; /** Reads a single `ProjectBackgroundJob` that is related to this `DataUploadTask`. */ projectBackgroundJob?: Maybe; projectBackgroundJobId: Scalars['UUID']; @@ -5585,6 +5571,11 @@ export type FailDataUploadPayloadDataUploadTaskEdgeArgs = { export type FeatureFlags = { __typename?: 'FeatureFlags'; + /** + * When true, project admins see the Data Tables tab in layer editors + * and related map UI. Controlled from SeaSketch developer settings. + */ + dataTables?: Maybe; iNaturalistLayers?: Maybe; }; @@ -7021,29 +7012,6 @@ export type GetChildFoldersRecursivePayload = { query?: Maybe; }; -/** All input for the `getPublishedCardIdFromDraft` mutation. */ -export type GetPublishedCardIdFromDraftInput = { - /** - * An arbitrary string value with no semantic meaning. Will be included in the - * payload verbatim. May be used to track mutations by the client. - */ - clientMutationId?: Maybe; - draftReportCardId?: Maybe; -}; - -/** The output of our `getPublishedCardIdFromDraft` mutation. */ -export type GetPublishedCardIdFromDraftPayload = { - __typename?: 'GetPublishedCardIdFromDraftPayload'; - /** - * The exact same `clientMutationId` that was provided in the mutation input, - * unchanged and unused. May be used by a client to track mutations. - */ - clientMutationId?: Maybe; - integer?: Maybe; - /** Our root query field type. Allows us to run any query from our mutation payload. */ - query?: Maybe; -}; - export type GoogleMapsTileApiSession = Node & { __typename?: 'GoogleMapsTileApiSession'; expiresAt: Scalars['Datetime']; @@ -7693,6 +7661,8 @@ export type MapBookmark = { /** Generated by clients. Should not be used if authorative thumbnail (image_id) is available. */ clientGeneratedThumbnail?: Maybe; createdAt: Scalars['Datetime']; + /** JSON map of TOC stableId -> { stableId, column?, op?, filters? } for activated overlay data tables. */ + dataTableStates: Scalars['JSON']; id: Scalars['UUID']; imageId?: Maybe; isPublic: Scalars['Boolean']; @@ -7979,8 +7949,7 @@ export type Mutation = { createOfflineTileSetting?: Maybe; /** Creates a single `OptionalBasemapLayer`. */ createOptionalBasemapLayer?: Maybe; - /** Creates a single `OriginalSourceId`. */ - createOriginalSourceId?: Maybe; + createOverlayDataTableUpload?: Maybe; createPost: Post; /** * Users with verified emails can create new projects by choosing a unique name @@ -8010,8 +7979,6 @@ export type Mutation = { * layers have not yet been created. */ createProjectWithGeographies?: Maybe; - /** Creates a single `PublishedTocItemId`. */ - createPublishedTocItemId?: Maybe; createRemoteGeojsonSource?: Maybe; createRemoteMvtSource?: Maybe; /** Creates a single `Report`. */ @@ -8205,7 +8172,6 @@ export type Mutation = { */ getOrCreateSprite?: Maybe; getPresignedPMTilesUploadUrl: PresignedUrl; - getPublishedCardIdFromDraft?: Maybe; /** Give a user admin access to a project. User must have already joined the project and shared their user profile. */ grantAdminAccess?: Maybe; importArcgisServices?: Maybe; @@ -8264,6 +8230,7 @@ export type Mutation = { removeUserFromGroup?: Maybe; /** Remove a SketchClass from the list of valid children for a Collection. */ removeValidChildSketchClass?: Maybe; + renameOverlayDataTable?: Maybe; renameReportTab?: Maybe; reopenResolvableLayerComment?: Maybe; reorderReportTabCards?: Maybe; @@ -8283,6 +8250,7 @@ export type Mutation = { /** Remove participant admin privileges. */ revokeAdminAccess?: Maybe; revokeApiKey?: Maybe; + rollbackOverlayDataTableVersion?: Maybe; rollbackToArchivedSource?: Maybe; /** Send all UNSENT invites in the current project. */ sendAllProjectInvites?: Maybe; @@ -8318,6 +8286,7 @@ export type Mutation = { * forum IDs in the correct order. Missing ids will be added to the end of the list. */ setForumOrder?: Maybe; + setOverlayDataTableVisualizationSettings?: Maybe; /** * Admins can use this function to hide the contents of a message. Message will * still appear in the client with the missing content, and should link to the @@ -8342,9 +8311,11 @@ export type Mutation = { setUserGroups?: Maybe; /** Superusers only. Promote a sprite to be globally available. */ shareSprite?: Maybe; + softDeleteOverlayDataTable?: Maybe; /** Superusers only. "Deletes" a sprite but keeps it in the DB in case layers are already referencing it. */ softDeleteSprite?: Maybe; submitDataUpload?: Maybe; + submitOverlayDataTableUpload?: Maybe; /** * Toggle admin access for the given project and user. User must have already * joined the project and shared their user profile. @@ -8814,8 +8785,8 @@ export type MutationCreateOptionalBasemapLayerArgs = { /** The root mutation type which contains root level fields which mutate data. */ -export type MutationCreateOriginalSourceIdArgs = { - input: CreateOriginalSourceIdInput; +export type MutationCreateOverlayDataTableUploadArgs = { + input: CreateOverlayDataTableUploadInput; }; @@ -8856,12 +8827,6 @@ export type MutationCreateProjectWithGeographiesArgs = { }; -/** The root mutation type which contains root level fields which mutate data. */ -export type MutationCreatePublishedTocItemIdArgs = { - input: CreatePublishedTocItemIdInput; -}; - - /** The root mutation type which contains root level fields which mutate data. */ export type MutationCreateRemoteGeojsonSourceArgs = { input: CreateRemoteGeojsonSourceInput; @@ -9452,12 +9417,6 @@ export type MutationGetPresignedPmTilesUploadUrlArgs = { }; -/** The root mutation type which contains root level fields which mutate data. */ -export type MutationGetPublishedCardIdFromDraftArgs = { - input: GetPublishedCardIdFromDraftInput; -}; - - /** The root mutation type which contains root level fields which mutate data. */ export type MutationGrantAdminAccessArgs = { input: GrantAdminAccessInput; @@ -9596,6 +9555,12 @@ export type MutationRemoveValidChildSketchClassArgs = { }; +/** The root mutation type which contains root level fields which mutate data. */ +export type MutationRenameOverlayDataTableArgs = { + input: RenameOverlayDataTableInput; +}; + + /** The root mutation type which contains root level fields which mutate data. */ export type MutationRenameReportTabArgs = { input: RenameReportTabInput; @@ -9669,6 +9634,12 @@ export type MutationRevokeApiKeyArgs = { }; +/** The root mutation type which contains root level fields which mutate data. */ +export type MutationRollbackOverlayDataTableVersionArgs = { + input: RollbackOverlayDataTableVersionInput; +}; + + /** The root mutation type which contains root level fields which mutate data. */ export type MutationRollbackToArchivedSourceArgs = { input: RollbackToArchivedSourceInput; @@ -9744,6 +9715,12 @@ export type MutationSetForumOrderArgs = { }; +/** The root mutation type which contains root level fields which mutate data. */ +export type MutationSetOverlayDataTableVisualizationSettingsArgs = { + input: SetOverlayDataTableVisualizationSettingsInput; +}; + + /** The root mutation type which contains root level fields which mutate data. */ export type MutationSetPostHiddenByModeratorArgs = { input: SetPostHiddenByModeratorInput; @@ -9795,6 +9772,12 @@ export type MutationShareSpriteArgs = { }; +/** The root mutation type which contains root level fields which mutate data. */ +export type MutationSoftDeleteOverlayDataTableArgs = { + input: SoftDeleteOverlayDataTableInput; +}; + + /** The root mutation type which contains root level fields which mutate data. */ export type MutationSoftDeleteSpriteArgs = { input: SoftDeleteSpriteInput; @@ -9807,6 +9790,12 @@ export type MutationSubmitDataUploadArgs = { }; +/** The root mutation type which contains root level fields which mutate data. */ +export type MutationSubmitOverlayDataTableUploadArgs = { + input: SubmitOverlayDataTableUploadInput; +}; + + /** The root mutation type which contains root level fields which mutate data. */ export type MutationToggleAdminAccessArgs = { input: ToggleAdminAccessInput; @@ -10656,50 +10645,162 @@ export enum OptionalBasemapLayersOrderBy { PrimaryKeyDesc = 'PRIMARY_KEY_DESC' } -export type OriginalSourceId = { - __typename?: 'OriginalSourceId'; - dataSourceId?: Maybe; +export type OutstandingSurveyInvites = { + __typename?: 'OutstandingSurveyInvites'; + projectId: Scalars['Int']; + surveyId: Scalars['Int']; + token: Scalars['String']; }; -/** An input for mutations affecting `OriginalSourceId` */ -export type OriginalSourceIdInput = { - dataSourceId?: Maybe; +export type OverlayDataTable = Node & { + __typename?: 'OverlayDataTable'; + columnStatsRemote: Scalars['String']; + columnStatsUrl?: Maybe; + createdAt?: Maybe; + createdBy: Scalars['Int']; + deletedAt?: Maybe; + id: Scalars['Int']; + joinColumn: Scalars['String']; + name: Scalars['String']; + /** A globally unique identifier. Can be used in various places throughout the system to identify this single value. */ + nodeId: Scalars['ID']; + overlayJoinColumn: Scalars['String']; + parquetRemote: Scalars['String']; + parquetUrl?: Maybe; + /** Reads a single `Project` that is related to this `OverlayDataTable`. */ + project?: Maybe; + projectId: Scalars['Int']; + queryUrl?: Maybe; + replacedById?: Maybe; + /** + * Columns that must appear as filters when this table is displayed on the map. + * End users can change the filter values but cannot remove these filters. + */ + requiredFilterColumns?: Maybe>>; + rowCount: Scalars['Int']; + /** + * Stable logical identity for a data table across version replace and TOC + * publish. Draft and published copies share the same UUID. + */ + stableId: Scalars['UUID']; + tableOfContentsItemId: Scalars['Int']; + updatedAt?: Maybe; + version: Scalars['Int']; + /** Columns that may/should be used for creating thematic maps. For example `count` or `density` */ + visualizationColumns?: Maybe>>; + /** Operations that may/should be used for creating thematic maps. For example `mean` or `max` */ + visualizationOps?: Maybe>>; +}; + +/** + * A condition to be used against `OverlayDataTable` object types. All fields are + * tested for equality and combined with a logical ‘and.’ + */ +export type OverlayDataTableCondition = { + /** Checks for equality with the object’s `id` field. */ + id?: Maybe; + /** Checks for equality with the object’s `projectId` field. */ + projectId?: Maybe; }; -/** A connection to a list of `OriginalSourceId` values. */ -export type OriginalSourceIdsConnection = { - __typename?: 'OriginalSourceIdsConnection'; - /** A list of edges which contains the `OriginalSourceId` and cursor to aid in pagination. */ - edges: Array; - /** A list of `OriginalSourceId` objects. */ - nodes: Array; +export type OverlayDataTableUpload = Node & { + __typename?: 'OverlayDataTableUpload'; + contentType: Scalars['String']; + createdAt?: Maybe; + errorDetails?: Maybe; + filename: Scalars['String']; + id: Scalars['UUID']; + /** A globally unique identifier. Can be used in various places throughout the system to identify this single value. */ + nodeId: Scalars['ID']; + overlayGeostats: Scalars['JSON']; + overlayJoinColumn?: Maybe; + presignedUploadUrl?: Maybe; + processingOptions: Scalars['JSON']; + /** Reads a single `ProjectBackgroundJob` that is related to this `OverlayDataTableUpload`. */ + projectBackgroundJob?: Maybe; + projectBackgroundJobId: Scalars['UUID']; + replaceOverlayDataTableId?: Maybe; + tableOfContentsItemId: Scalars['Int']; + updatedAt?: Maybe; +}; + +/** + * A condition to be used against `OverlayDataTableUpload` object types. All fields + * are tested for equality and combined with a logical ‘and.’ + */ +export type OverlayDataTableUploadCondition = { + /** Checks for equality with the object’s `id` field. */ + id?: Maybe; + /** Checks for equality with the object’s `projectBackgroundJobId` field. */ + projectBackgroundJobId?: Maybe; +}; + +/** A connection to a list of `OverlayDataTableUpload` values. */ +export type OverlayDataTableUploadsConnection = { + __typename?: 'OverlayDataTableUploadsConnection'; + /** A list of edges which contains the `OverlayDataTableUpload` and cursor to aid in pagination. */ + edges: Array; + /** A list of `OverlayDataTableUpload` objects. */ + nodes: Array; /** Information to aid in pagination. */ pageInfo: PageInfo; - /** The count of *all* `OriginalSourceId` you could get from the connection. */ + /** The count of *all* `OverlayDataTableUpload` you could get from the connection. */ totalCount: Scalars['Int']; }; -/** A `OriginalSourceId` edge in the connection. */ -export type OriginalSourceIdsEdge = { - __typename?: 'OriginalSourceIdsEdge'; +/** A `OverlayDataTableUpload` edge in the connection. */ +export type OverlayDataTableUploadsEdge = { + __typename?: 'OverlayDataTableUploadsEdge'; /** A cursor for use in pagination. */ cursor?: Maybe; - /** The `OriginalSourceId` at the end of the edge. */ - node: OriginalSourceId; + /** The `OverlayDataTableUpload` at the end of the edge. */ + node: OverlayDataTableUpload; }; -/** Methods to use when ordering `OriginalSourceId`. */ -export enum OriginalSourceIdsOrderBy { - Natural = 'NATURAL' +/** Methods to use when ordering `OverlayDataTableUpload`. */ +export enum OverlayDataTableUploadsOrderBy { + IdAsc = 'ID_ASC', + IdDesc = 'ID_DESC', + Natural = 'NATURAL', + PrimaryKeyAsc = 'PRIMARY_KEY_ASC', + PrimaryKeyDesc = 'PRIMARY_KEY_DESC', + ProjectBackgroundJobIdAsc = 'PROJECT_BACKGROUND_JOB_ID_ASC', + ProjectBackgroundJobIdDesc = 'PROJECT_BACKGROUND_JOB_ID_DESC' } -export type OutstandingSurveyInvites = { - __typename?: 'OutstandingSurveyInvites'; - projectId: Scalars['Int']; - surveyId: Scalars['Int']; - token: Scalars['String']; +/** A connection to a list of `OverlayDataTable` values. */ +export type OverlayDataTablesConnection = { + __typename?: 'OverlayDataTablesConnection'; + /** A list of edges which contains the `OverlayDataTable` and cursor to aid in pagination. */ + edges: Array; + /** A list of `OverlayDataTable` objects. */ + nodes: Array; + /** Information to aid in pagination. */ + pageInfo: PageInfo; + /** The count of *all* `OverlayDataTable` you could get from the connection. */ + totalCount: Scalars['Int']; }; +/** A `OverlayDataTable` edge in the connection. */ +export type OverlayDataTablesEdge = { + __typename?: 'OverlayDataTablesEdge'; + /** A cursor for use in pagination. */ + cursor?: Maybe; + /** The `OverlayDataTable` at the end of the edge. */ + node: OverlayDataTable; +}; + +/** Methods to use when ordering `OverlayDataTable`. */ +export enum OverlayDataTablesOrderBy { + IdAsc = 'ID_ASC', + IdDesc = 'ID_DESC', + Natural = 'NATURAL', + PrimaryKeyAsc = 'PRIMARY_KEY_ASC', + PrimaryKeyDesc = 'PRIMARY_KEY_DESC', + ProjectIdAsc = 'PROJECT_ID_ASC', + ProjectIdDesc = 'PROJECT_ID_DESC' +} + /** Information about pagination in a connection. */ export type PageInfo = { __typename?: 'PageInfo'; @@ -11026,10 +11127,11 @@ export type Project = Node & { hideOverlays: Scalars['Boolean']; hideSketches: Scalars['Boolean']; /** - * Content-addressed tileset UUIDs whose /v2 requests must include a map - * access token. Equals hosted UUIDs from published TOC items (and draft - * TOC items for project admins) that are not in the published ACL - * document's public list. Empty for projects with no protected overlays. + * Content-addressed tileset UUIDs whose hosted tiles/uploads requests + * must include a map access token. Equals hosted UUIDs from published + * TOC items (and draft TOC items for project admins) that are not in the + * published ACL document's public list. Empty for projects with no + * protected overlays. */ hostedTileUuidsRequiringAuth: Array; id: Scalars['Int']; @@ -11077,7 +11179,7 @@ export type Project = Node & { logoUrl?: Maybe; /** * Short-lived (90m) JWT for authorized overlay tile requests on - * tiles.seasketch.org/v2/{ns}/.... Returns null if the user is not signed in. + * hosted tiles/uploads URLs with access_token (and optional ns). Returns null if the user is not signed in. * Claims include role (admin|user) and project group ids. */ mapAccessToken?: Maybe; @@ -11098,6 +11200,8 @@ export type Project = Node & { offlineTilePackagesConnection: OfflineTilePackagesConnection; /** Reads and enables pagination through a set of `OfflineTileSetting`. */ offlineTileSettings: Array; + /** Reads and enables pagination through a set of `OverlayDataTable`. */ + overlayDataTablesConnection: OverlayDataTablesConnection; /** Count of all users who have opted into participating in the project, sharing their profile with project administrators. */ participantCount?: Maybe; /** @@ -11517,6 +11621,21 @@ export type ProjectOfflineTileSettingsArgs = { }; +/** + * SeaSketch Project type. This root type contains most of the fields and queries + * needed to drive the application. + */ +export type ProjectOverlayDataTablesConnectionArgs = { + after?: Maybe; + before?: Maybe; + condition?: Maybe; + first?: Maybe; + last?: Maybe; + offset?: Maybe; + orderBy?: Maybe>; +}; + + /** * SeaSketch Project type. This root type contains most of the fields and queries * needed to drive the application. @@ -11728,6 +11847,13 @@ export type ProjectBackgroundJob = Node & { id: Scalars['UUID']; /** A globally unique identifier. Can be used in various places throughout the system to identify this single value. */ nodeId: Scalars['ID']; + /** Reads a single `OverlayDataTableUpload` that is related to this `ProjectBackgroundJob`. */ + overlayDataTableUpload?: Maybe; + /** + * Reads and enables pagination through a set of `OverlayDataTableUpload`. + * @deprecated Please use overlayDataTableUpload instead + */ + overlayDataTableUploadsConnection: OverlayDataTableUploadsConnection; progress?: Maybe; progressMessage: Scalars['String']; /** Reads a single `Project` that is related to this `ProjectBackgroundJob`. */ @@ -11752,6 +11878,17 @@ export type ProjectBackgroundJobDataUploadTasksConnectionArgs = { orderBy?: Maybe>; }; + +export type ProjectBackgroundJobOverlayDataTableUploadsConnectionArgs = { + after?: Maybe; + before?: Maybe; + condition?: Maybe; + first?: Maybe; + last?: Maybe; + offset?: Maybe; + orderBy?: Maybe>; +}; + /** * A condition to be used against `ProjectBackgroundJob` object types. All fields * are tested for equality and combined with a logical ‘and.’ @@ -11782,6 +11919,7 @@ export type ProjectBackgroundJobSubscriptionPayload = { export enum ProjectBackgroundJobType { ArcgisImport = 'ARCGIS_IMPORT', ConsolidateDataSources = 'CONSOLIDATE_DATA_SOURCES', + DataTableUpload = 'DATA_TABLE_UPLOAD', DataUpload = 'DATA_UPLOAD', ReplacementUpload = 'REPLACEMENT_UPLOAD' } @@ -12280,43 +12418,6 @@ export type PublishTableOfContentsPayload = { tableOfContentsItems?: Maybe>; }; -export type PublishedTocItemId = { - __typename?: 'PublishedTocItemId'; - id?: Maybe; -}; - -/** An input for mutations affecting `PublishedTocItemId` */ -export type PublishedTocItemIdInput = { - id?: Maybe; -}; - -/** A connection to a list of `PublishedTocItemId` values. */ -export type PublishedTocItemIdsConnection = { - __typename?: 'PublishedTocItemIdsConnection'; - /** A list of edges which contains the `PublishedTocItemId` and cursor to aid in pagination. */ - edges: Array; - /** A list of `PublishedTocItemId` objects. */ - nodes: Array; - /** Information to aid in pagination. */ - pageInfo: PageInfo; - /** The count of *all* `PublishedTocItemId` you could get from the connection. */ - totalCount: Scalars['Int']; -}; - -/** A `PublishedTocItemId` edge in the connection. */ -export type PublishedTocItemIdsEdge = { - __typename?: 'PublishedTocItemIdsEdge'; - /** A cursor for use in pagination. */ - cursor?: Maybe; - /** The `PublishedTocItemId` at the end of the edge. */ - node: PublishedTocItemId; -}; - -/** Methods to use when ordering `PublishedTocItemId`. */ -export enum PublishedTocItemIdsOrderBy { - Natural = 'NATURAL' -} - /** The root query type which gives access points into the data universe. */ export type Query = Node & { __typename?: 'Query'; @@ -12496,8 +12597,19 @@ export type Query = Node & { optionalBasemapLayer?: Maybe; /** Reads a single `OptionalBasemapLayer` using its globally unique `ID`. */ optionalBasemapLayerByNodeId?: Maybe; - /** Reads and enables pagination through a set of `OriginalSourceId`. */ - originalSourceIdsConnection?: Maybe; + overlayDataTable?: Maybe; + /** Reads a single `OverlayDataTable` using its globally unique `ID`. */ + overlayDataTableByNodeId?: Maybe; + overlayDataTableLinkedTocIsDraft?: Maybe; + overlayDataTableParquetPublicUrl?: Maybe; + /** Reads and enables pagination through a set of `OverlayDataTable`. */ + overlayDataTablesConnection?: Maybe; + overlayDataTableUpload?: Maybe; + /** Reads a single `OverlayDataTableUpload` using its globally unique `ID`. */ + overlayDataTableUploadByNodeId?: Maybe; + overlayDataTableUploadByProjectBackgroundJobId?: Maybe; + /** Reads and enables pagination through a set of `OverlayDataTableUpload`. */ + overlayDataTableUploadsConnection?: Maybe; post?: Maybe; /** Reads a single `Post` using its globally unique `ID`. */ postByNodeId?: Maybe; @@ -12529,8 +12641,6 @@ export type Query = Node & { projectsSharedBasemapsConnection?: Maybe; /** Used by project administrators to access a list of public sprites promoted by the SeaSketch development team. */ publicSprites?: Maybe>; - /** Reads and enables pagination through a set of `PublishedTocItemId`. */ - publishedTocItemIdsConnection?: Maybe; /** * Exposes the root query type nested one level down. This is helpful for Relay 1 * which can only query top level fields if they are in a particular form. @@ -13385,13 +13495,69 @@ export type QueryOptionalBasemapLayerByNodeIdArgs = { /** The root query type which gives access points into the data universe. */ -export type QueryOriginalSourceIdsConnectionArgs = { +export type QueryOverlayDataTableArgs = { + id: Scalars['Int']; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableByNodeIdArgs = { + nodeId: Scalars['ID']; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableLinkedTocIsDraftArgs = { + pid?: Maybe; + tocItemId?: Maybe; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableParquetPublicUrlArgs = { + pRemote?: Maybe; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTablesConnectionArgs = { + after?: Maybe; + before?: Maybe; + condition?: Maybe; + first?: Maybe; + last?: Maybe; + offset?: Maybe; + orderBy?: Maybe>; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableUploadArgs = { + id: Scalars['UUID']; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableUploadByNodeIdArgs = { + nodeId: Scalars['ID']; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableUploadByProjectBackgroundJobIdArgs = { + projectBackgroundJobId: Scalars['UUID']; +}; + + +/** The root query type which gives access points into the data universe. */ +export type QueryOverlayDataTableUploadsConnectionArgs = { after?: Maybe; before?: Maybe; + condition?: Maybe; first?: Maybe; last?: Maybe; offset?: Maybe; - orderBy?: Maybe>; + orderBy?: Maybe>; }; @@ -13552,17 +13718,6 @@ export type QueryPublicSpritesArgs = { }; -/** The root query type which gives access points into the data universe. */ -export type QueryPublishedTocItemIdsConnectionArgs = { - after?: Maybe; - before?: Maybe; - first?: Maybe; - last?: Maybe; - offset?: Maybe; - orderBy?: Maybe>; -}; - - /** The root query type which gives access points into the data universe. */ export type QueryReportArgs = { id: Scalars['Int']; @@ -14095,6 +14250,40 @@ export type RemoveValidChildSketchClassPayload = { query?: Maybe; }; +/** All input for the `renameOverlayDataTable` mutation. */ +export type RenameOverlayDataTableInput = { + /** + * An arbitrary string value with no semantic meaning. Will be included in the + * payload verbatim. May be used to track mutations by the client. + */ + clientMutationId?: Maybe; + newName?: Maybe; + tableId?: Maybe; +}; + +/** The output of our `renameOverlayDataTable` mutation. */ +export type RenameOverlayDataTablePayload = { + __typename?: 'RenameOverlayDataTablePayload'; + /** + * The exact same `clientMutationId` that was provided in the mutation input, + * unchanged and unused. May be used by a client to track mutations. + */ + clientMutationId?: Maybe; + overlayDataTable?: Maybe; + /** An edge for our `OverlayDataTable`. May be used by Relay 1. */ + overlayDataTableEdge?: Maybe; + /** Reads a single `Project` that is related to this `OverlayDataTable`. */ + project?: Maybe; + /** Our root query field type. Allows us to run any query from our mutation payload. */ + query?: Maybe; +}; + + +/** The output of our `renameOverlayDataTable` mutation. */ +export type RenameOverlayDataTablePayloadOverlayDataTableEdgeArgs = { + orderBy?: Maybe>; +}; + /** All input for the `renameReportTab` mutation. */ export type RenameReportTabInput = { alternateLanguageSettings?: Maybe; @@ -14631,6 +14820,39 @@ export type RevokeApiKeyPayload = { query?: Maybe; }; +/** All input for the `rollbackOverlayDataTableVersion` mutation. */ +export type RollbackOverlayDataTableVersionInput = { + /** + * An arbitrary string value with no semantic meaning. Will be included in the + * payload verbatim. May be used to track mutations by the client. + */ + clientMutationId?: Maybe; + tableId?: Maybe; +}; + +/** The output of our `rollbackOverlayDataTableVersion` mutation. */ +export type RollbackOverlayDataTableVersionPayload = { + __typename?: 'RollbackOverlayDataTableVersionPayload'; + /** + * The exact same `clientMutationId` that was provided in the mutation input, + * unchanged and unused. May be used by a client to track mutations. + */ + clientMutationId?: Maybe; + overlayDataTable?: Maybe; + /** An edge for our `OverlayDataTable`. May be used by Relay 1. */ + overlayDataTableEdge?: Maybe; + /** Reads a single `Project` that is related to this `OverlayDataTable`. */ + project?: Maybe; + /** Our root query field type. Allows us to run any query from our mutation payload. */ + query?: Maybe; +}; + + +/** The output of our `rollbackOverlayDataTableVersion` mutation. */ +export type RollbackOverlayDataTableVersionPayloadOverlayDataTableEdgeArgs = { + orderBy?: Maybe>; +}; + /** All input for the `rollbackToArchivedSource` mutation. */ export type RollbackToArchivedSourceInput = { /** @@ -14853,6 +15075,42 @@ export type SetForumOrderPayload = { query?: Maybe; }; +/** All input for the `setOverlayDataTableVisualizationSettings` mutation. */ +export type SetOverlayDataTableVisualizationSettingsInput = { + /** + * An arbitrary string value with no semantic meaning. Will be included in the + * payload verbatim. May be used to track mutations by the client. + */ + clientMutationId?: Maybe; + requiredFilterColumns?: Maybe>>; + tableId?: Maybe; + visualizationColumns?: Maybe>>; + visualizationOps?: Maybe>>; +}; + +/** The output of our `setOverlayDataTableVisualizationSettings` mutation. */ +export type SetOverlayDataTableVisualizationSettingsPayload = { + __typename?: 'SetOverlayDataTableVisualizationSettingsPayload'; + /** + * The exact same `clientMutationId` that was provided in the mutation input, + * unchanged and unused. May be used by a client to track mutations. + */ + clientMutationId?: Maybe; + overlayDataTable?: Maybe; + /** An edge for our `OverlayDataTable`. May be used by Relay 1. */ + overlayDataTableEdge?: Maybe; + /** Reads a single `Project` that is related to this `OverlayDataTable`. */ + project?: Maybe; + /** Our root query field type. Allows us to run any query from our mutation payload. */ + query?: Maybe; +}; + + +/** The output of our `setOverlayDataTableVisualizationSettings` mutation. */ +export type SetOverlayDataTableVisualizationSettingsPayloadOverlayDataTableEdgeArgs = { + orderBy?: Maybe>; +}; + /** All input for the `setPostHiddenByModerator` mutation. */ export type SetPostHiddenByModeratorInput = { /** @@ -15517,6 +15775,39 @@ export type SketchesRelatedFragmentsRecord = { sketches?: Maybe>>; }; +/** All input for the `softDeleteOverlayDataTable` mutation. */ +export type SoftDeleteOverlayDataTableInput = { + /** + * An arbitrary string value with no semantic meaning. Will be included in the + * payload verbatim. May be used to track mutations by the client. + */ + clientMutationId?: Maybe; + tableId?: Maybe; +}; + +/** The output of our `softDeleteOverlayDataTable` mutation. */ +export type SoftDeleteOverlayDataTablePayload = { + __typename?: 'SoftDeleteOverlayDataTablePayload'; + /** + * The exact same `clientMutationId` that was provided in the mutation input, + * unchanged and unused. May be used by a client to track mutations. + */ + clientMutationId?: Maybe; + overlayDataTable?: Maybe; + /** An edge for our `OverlayDataTable`. May be used by Relay 1. */ + overlayDataTableEdge?: Maybe; + /** Reads a single `Project` that is related to this `OverlayDataTable`. */ + project?: Maybe; + /** Our root query field type. Allows us to run any query from our mutation payload. */ + query?: Maybe; +}; + + +/** The output of our `softDeleteOverlayDataTable` mutation. */ +export type SoftDeleteOverlayDataTablePayloadOverlayDataTableEdgeArgs = { + orderBy?: Maybe>; +}; + /** All input for the `softDeleteSprite` mutation. */ export type SoftDeleteSpriteInput = { /** @@ -15740,6 +16031,31 @@ export type SubmitDataUploadPayload = { query?: Maybe; }; +/** All input for the `submitOverlayDataTableUpload` mutation. */ +export type SubmitOverlayDataTableUploadInput = { + /** + * An arbitrary string value with no semantic meaning. Will be included in the + * payload verbatim. May be used to track mutations by the client. + */ + clientMutationId?: Maybe; + jobId?: Maybe; +}; + +/** The output of our `submitOverlayDataTableUpload` mutation. */ +export type SubmitOverlayDataTableUploadPayload = { + __typename?: 'SubmitOverlayDataTableUploadPayload'; + /** + * The exact same `clientMutationId` that was provided in the mutation input, + * unchanged and unused. May be used by a client to track mutations. + */ + clientMutationId?: Maybe; + /** Reads a single `Project` that is related to this `ProjectBackgroundJob`. */ + project?: Maybe; + projectBackgroundJob?: Maybe; + /** Our root query field type. Allows us to run any query from our mutation payload. */ + query?: Maybe; +}; + /** The root subscription type: contains realtime events you can subscribe to with the `subscription` operation. */ export type Subscription = { __typename?: 'Subscription'; @@ -16287,8 +16603,14 @@ export type TableOfContentsItem = Node & { /** If is_folder=false, a DataLayers visibility will be controlled by this item */ dataLayerId?: Maybe; dataSourceType?: Maybe; + /** Reads and enables pagination through a set of `ChangeLog`. */ + dataTableChangeLogs?: Maybe>; + /** Overlay attribute name used as the canonical feature ID for linked data tables. Required when enable_data_tables is true. */ + dataTableJoinColumn?: Maybe; /** Reads and enables pagination through a set of `DownloadOption`. */ downloadOptions?: Maybe>; + /** When true, admins can attach CSV data tables linked to this layer by a canonical join column. */ + enableDataTables: Scalars['Boolean']; enableDownload: Scalars['Boolean']; ftsAr?: Maybe; ftsDa?: Maybe; @@ -16337,6 +16659,8 @@ export type TableOfContentsItem = Node & { metadataXml?: Maybe; /** A globally unique identifier. Can be used in various places throughout the system to identify this single value. */ nodeId: Scalars['ID']; + /** Reads and enables pagination through a set of `OverlayDataTable`. */ + overlayDataTables?: Maybe>; /** * stable_id of the parent folder, if any. This property cannot be changed * directly. To rearrange items into folders, use the @@ -16427,6 +16751,22 @@ export type TableOfContentsItemChangeLogsArgs = { }; +/** + * TableOfContentsItems represent a tree-view of folders and operational layers + * that can be added to the map. Both layers and folders may be nested into other + * folders for organization, and each folder has its own access control list. + * + * Items that represent data layers have a `DataLayer` relation, which in turn has + * a reference to a `DataSource`. Usually these relations should be fetched in + * batch only once the layer is turned on, using the + * `dataLayersAndSourcesByLayerId` query. + */ +export type TableOfContentsItemDataTableChangeLogsArgs = { + first?: Maybe; + offset?: Maybe; +}; + + /** * TableOfContentsItems represent a tree-view of folders and operational layers * that can be added to the map. Both layers and folders may be nested into other @@ -16459,6 +16799,22 @@ export type TableOfContentsItemMetadataChangeLogsArgs = { }; +/** + * TableOfContentsItems represent a tree-view of folders and operational layers + * that can be added to the map. Both layers and folders may be nested into other + * folders for organization, and each folder has its own access control list. + * + * Items that represent data layers have a `DataLayer` relation, which in turn has + * a reference to a `DataSource`. Usually these relations should be fetched in + * batch only once the layer is turned on, using the + * `dataLayersAndSourcesByLayerId` query. + */ +export type TableOfContentsItemOverlayDataTablesArgs = { + first?: Maybe; + offset?: Maybe; +}; + + /** * TableOfContentsItems represent a tree-view of folders and operational layers * that can be added to the map. Both layers and folders may be nested into other @@ -16631,6 +16987,10 @@ export type TableOfContentsItemPatch = { bounds?: Maybe>>; /** If is_folder=false, a DataLayers visibility will be controlled by this item */ dataLayerId?: Maybe; + /** Overlay attribute name used as the canonical feature ID for linked data tables. Required when enable_data_tables is true. */ + dataTableJoinColumn?: Maybe; + /** When true, admins can attach CSV data tables linked to this layer by a canonical join column. */ + enableDataTables?: Maybe; enableDownload?: Maybe; geoprocessingReferenceId?: Maybe; hideChildren?: Maybe; @@ -20543,6 +20903,7 @@ export type CreateDataUploadMutationVariables = Exact<{ filename: Scalars['String']; contentType: Scalars['String']; replaceTableOfContentsItemId?: Maybe; + processingOptions?: Maybe; }>; @@ -20571,6 +20932,9 @@ export type JobDetailsFragment = ( { __typename?: 'TableOfContentsItem' } & Pick )> } + )>, overlayDataTableUpload?: Maybe<( + { __typename?: 'OverlayDataTableUpload' } + & Pick )> } ); @@ -21171,6 +21535,27 @@ export type LayerSettingsChangeLogQuery = ( )> } ); +export type DataTableChangeLogQueryVariables = Exact<{ + id: Scalars['Int']; + first: Scalars['Int']; +}>; + + +export type DataTableChangeLogQuery = ( + { __typename?: 'Query' } + & { tableOfContentsItem?: Maybe<( + { __typename?: 'TableOfContentsItem' } + & Pick + & { overlayDataTables?: Maybe + )>>, dataTableChangeLogs?: Maybe> } + )> } +); + export type ResolvableLayerCommentDetailsFragment = ( { __typename?: 'ResolvableLayerComment' } & Pick @@ -21207,7 +21592,7 @@ export type ResolvableLayerCommentThreadForChangelogQuery = ( export type FullAdminOverlayFragment = ( { __typename?: 'TableOfContentsItem' } - & Pick + & Pick & { acl?: Maybe<( { __typename?: 'Acl' } & Pick @@ -21220,11 +21605,14 @@ export type FullAdminOverlayFragment = ( & Pick )>>>, projectBackgroundJobs?: Maybe + & JobDetailsFragment )>>, dataLayer?: Maybe<( { __typename?: 'DataLayer' } & FullAdminDataLayerFragment - )>, relatedReportCardDetails?: Maybe, overlayDataTables?: Maybe>, relatedReportCardDetails?: Maybe & { sketchClass: ( @@ -21294,6 +21682,24 @@ export type UpdateEnableDownloadMutation = ( )> } ); +export type UpdateDataTablesSettingsMutationVariables = Exact<{ + id: Scalars['Int']; + enableDataTables?: Maybe; + dataTableJoinColumn?: Maybe; +}>; + + +export type UpdateDataTablesSettingsMutation = ( + { __typename?: 'Mutation' } + & { updateTableOfContentsItem?: Maybe<( + { __typename?: 'UpdateTableOfContentsItemPayload' } + & { tableOfContentsItem?: Maybe<( + { __typename?: 'TableOfContentsItem' } + & Pick + )> } + )> } +); + export type UpdateLayerMutationVariables = Exact<{ id: Scalars['Int']; renderUnder?: Maybe; @@ -22069,7 +22475,10 @@ export type ChangeLogsSinceLastPublishQuery = ( & { unresolvedComment?: Maybe<( { __typename?: 'ResolvableLayerComment' } & ResolvableLayerCommentThreadFragment - )>, dataLayer?: Maybe<( + )>, overlayDataTables?: Maybe + )>>, dataLayer?: Maybe<( { __typename?: 'DataLayer' } & Pick & { dataSource?: Maybe<( @@ -22528,7 +22937,7 @@ export type JobFragment = ( export type MapBookmarkDetailsFragment = ( { __typename?: 'MapBookmark' } - & Pick + & Pick & { job?: Maybe<( { __typename?: 'WorkerJob' } & JobFragment @@ -22563,6 +22972,7 @@ export type CreateMapBookmarkMutationVariables = Exact<{ layerNames: Scalars['JSON']; sketchNames: Scalars['JSON']; clientGeneratedThumbnail: Scalars['String']; + dataTableStates?: Maybe; }>; @@ -23277,6 +23687,158 @@ export type GetTilePackageQuery = ( )> } ); +export type ClientOverlayDataTableFragment = ( + { __typename?: 'OverlayDataTable' } + & Pick +); + +export type OverlayDataTableDetailsFragment = ( + { __typename?: 'OverlayDataTable' } + & Pick +); + +export type OverlayDataTableVisualizationMetadataQueryVariables = Exact<{ + id: Scalars['Int']; +}>; + + +export type OverlayDataTableVisualizationMetadataQuery = ( + { __typename?: 'Query' } + & { overlayDataTable?: Maybe<( + { __typename?: 'OverlayDataTable' } + & Pick + )> } +); + +export type OverlayDataTableVisualizationMetadataForLayerQueryVariables = Exact<{ + tocItemId: Scalars['Int']; +}>; + + +export type OverlayDataTableVisualizationMetadataForLayerQuery = ( + { __typename?: 'Query' } + & { tableOfContentsItem?: Maybe<( + { __typename?: 'TableOfContentsItem' } + & Pick + & { overlayDataTables?: Maybe + )>> } + )> } +); + +export type OverlayDataTableUploadDetailsFragment = ( + { __typename?: 'OverlayDataTableUpload' } + & Pick +); + +export type CreateOverlayDataTableUploadMutationVariables = Exact<{ + tableOfContentsItemId: Scalars['Int']; + filename: Scalars['String']; + contentType: Scalars['String']; + processingOptions?: Maybe; + replaceOverlayDataTableId?: Maybe; +}>; + + +export type CreateOverlayDataTableUploadMutation = ( + { __typename?: 'Mutation' } + & { createOverlayDataTableUpload?: Maybe<( + { __typename?: 'CreateOverlayDataTableUploadPayload' } + & { overlayDataTableUpload?: Maybe<( + { __typename?: 'OverlayDataTableUpload' } + & OverlayDataTableUploadDetailsFragment + )>, projectBackgroundJob?: Maybe<( + { __typename?: 'ProjectBackgroundJob' } + & JobDetailsFragment + )> } + )> } +); + +export type SubmitOverlayDataTableUploadMutationVariables = Exact<{ + jobId: Scalars['UUID']; +}>; + + +export type SubmitOverlayDataTableUploadMutation = ( + { __typename?: 'Mutation' } + & { submitOverlayDataTableUpload?: Maybe<( + { __typename?: 'SubmitOverlayDataTableUploadPayload' } + & { projectBackgroundJob?: Maybe<( + { __typename?: 'ProjectBackgroundJob' } + & Pick + )> } + )> } +); + +export type RenameOverlayDataTableMutationVariables = Exact<{ + id: Scalars['Int']; + name: Scalars['String']; +}>; + + +export type RenameOverlayDataTableMutation = ( + { __typename?: 'Mutation' } + & { renameOverlayDataTable?: Maybe<( + { __typename?: 'RenameOverlayDataTablePayload' } + & { overlayDataTable?: Maybe<( + { __typename?: 'OverlayDataTable' } + & OverlayDataTableDetailsFragment + )> } + )> } +); + +export type SoftDeleteOverlayDataTableMutationVariables = Exact<{ + id: Scalars['Int']; +}>; + + +export type SoftDeleteOverlayDataTableMutation = ( + { __typename?: 'Mutation' } + & { softDeleteOverlayDataTable?: Maybe<( + { __typename?: 'SoftDeleteOverlayDataTablePayload' } + & { overlayDataTable?: Maybe<( + { __typename?: 'OverlayDataTable' } + & Pick + )> } + )> } +); + +export type RollbackOverlayDataTableVersionMutationVariables = Exact<{ + id: Scalars['Int']; +}>; + + +export type RollbackOverlayDataTableVersionMutation = ( + { __typename?: 'Mutation' } + & { rollbackOverlayDataTableVersion?: Maybe<( + { __typename?: 'RollbackOverlayDataTableVersionPayload' } + & { overlayDataTable?: Maybe<( + { __typename?: 'OverlayDataTable' } + & OverlayDataTableDetailsFragment + )> } + )> } +); + +export type SetOverlayDataTableVisualizationSettingsMutationVariables = Exact<{ + id: Scalars['Int']; + visualizationColumns: Array> | Maybe; + visualizationOps: Array> | Maybe; + requiredFilterColumns: Array> | Maybe; +}>; + + +export type SetOverlayDataTableVisualizationSettingsMutation = ( + { __typename?: 'Mutation' } + & { setOverlayDataTableVisualizationSettings?: Maybe<( + { __typename?: 'SetOverlayDataTableVisualizationSettingsPayload' } + & { overlayDataTable?: Maybe<( + { __typename?: 'OverlayDataTable' } + & OverlayDataTableDetailsFragment + )> } + )> } +); + export type ProjectAccessControlSettingsQueryVariables = Exact<{ slug: Scalars['String']; }>; @@ -23490,7 +24052,7 @@ export type ProjectMetadataFragment = ( & Pick )>>>, featureFlags?: Maybe<( { __typename?: 'FeatureFlags' } - & Pick + & Pick )> } ); @@ -23627,7 +24189,10 @@ export type OverlayFragment = ( & { acl?: Maybe<( { __typename?: 'Acl' } & Pick - )> } + )>, overlayDataTables?: Maybe> } ); export type PublishedTableOfContentsQueryVariables = Exact<{ @@ -26525,7 +27090,7 @@ export type UpdateFeatureFlagsMutation = ( & Pick & { featureFlags?: Maybe<( { __typename?: 'FeatureFlags' } - & Pick + & Pick )> } )> } )> } @@ -27566,6 +28131,12 @@ export const JobDetailsFragmentDoc = /*#__PURE__*/ gql` title } } + overlayDataTableUpload { + id + tableOfContentsItemId + filename + replaceOverlayDataTableId + } } ${DataUploadDetailsFragmentDoc}`; export const DataUploadExtendedDetailsFragmentDoc = /*#__PURE__*/ gql` @@ -27598,6 +28169,22 @@ export const BackgroundJobSubscriptionEventFragmentDoc = /*#__PURE__*/ gql` } } ${JobDetailsFragmentDoc}`; +export const ClientOverlayDataTableFragmentDoc = /*#__PURE__*/ gql` + fragment ClientOverlayDataTable on OverlayDataTable { + id + stableId + name + version + rowCount + joinColumn + overlayJoinColumn + queryUrl + columnStatsUrl + visualizationColumns + visualizationOps + requiredFilterColumns +} + `; export const OverlayFragmentDoc = /*#__PURE__*/ gql` fragment Overlay on TableOfContentsItem { id @@ -27621,8 +28208,11 @@ export const OverlayFragmentDoc = /*#__PURE__*/ gql` hasMetadata primaryDownloadUrl dataSourceType + overlayDataTables { + ...ClientOverlayDataTable + } } - `; + ${ClientOverlayDataTableFragmentDoc}`; export const AdminOverlayFragmentDoc = /*#__PURE__*/ gql` fragment AdminOverlay on TableOfContentsItem { ...Overlay @@ -27843,6 +28433,29 @@ export const FullAdminDataLayerFragmentDoc = /*#__PURE__*/ gql` } ${FullAdminSourceFragmentDoc} ${ArchivedSourceFragmentDoc}`; +export const OverlayDataTableDetailsFragmentDoc = /*#__PURE__*/ gql` + fragment OverlayDataTableDetails on OverlayDataTable { + id + stableId + name + version + joinColumn + overlayJoinColumn + rowCount + parquetRemote + columnStatsRemote + parquetUrl + columnStatsUrl + queryUrl + deletedAt + replacedById + createdAt + updatedAt + visualizationColumns + visualizationOps + requiredFilterColumns +} + `; export const UserProfileDetailsFragmentDoc = /*#__PURE__*/ gql` fragment UserProfileDetails on Profile { userId @@ -27906,22 +28519,21 @@ export const FullAdminOverlayFragmentDoc = /*#__PURE__*/ gql` stableId title enableDownload + enableDataTables + dataTableJoinColumn geoprocessingReferenceId copiedFromDataLibraryTemplateId primaryDownloadUrl projectBackgroundJobs { - id - type - title - state - progress - progressMessage - errorMessage + ...JobDetails } hasOriginalSourceUpload dataLayer { ...FullAdminDataLayer } + overlayDataTables { + ...OverlayDataTableDetails + } relatedReportCardDetails { isDraft title @@ -27938,7 +28550,9 @@ export const FullAdminOverlayFragmentDoc = /*#__PURE__*/ gql` ...ResolvableLayerCommentThread } } - ${FullAdminDataLayerFragmentDoc} + ${JobDetailsFragmentDoc} +${FullAdminDataLayerFragmentDoc} +${OverlayDataTableDetailsFragmentDoc} ${ResolvableLayerCommentThreadFragmentDoc}`; export const MetadataXmlFileFragmentDoc = /*#__PURE__*/ gql` fragment MetadataXmlFile on DataUploadOutput { @@ -28003,6 +28617,7 @@ export const MapBookmarkDetailsFragmentDoc = /*#__PURE__*/ gql` basemapName sketchNames clientGeneratedThumbnail + dataTableStates } ${JobFragmentDoc}`; export const FileUploadDetailsFragmentDoc = /*#__PURE__*/ gql` @@ -28249,6 +28864,20 @@ export const OfflineTileSettingsFragmentDoc = /*#__PURE__*/ gql` } } `; +export const OverlayDataTableUploadDetailsFragmentDoc = /*#__PURE__*/ gql` + fragment OverlayDataTableUploadDetails on OverlayDataTableUpload { + id + tableOfContentsItemId + filename + contentType + processingOptions + overlayJoinColumn + errorDetails + presignedUploadUrl + replaceOverlayDataTableId + projectBackgroundJobId +} + `; export const ProjectMetadataFragmentDoc = /*#__PURE__*/ gql` fragment ProjectMetadata on Project { id @@ -28297,6 +28926,7 @@ export const ProjectMetadataFragmentDoc = /*#__PURE__*/ gql` showLegendByDefault featureFlags { iNaturalistLayers + dataTables } } `; @@ -30006,9 +30636,9 @@ export const DashboardBannerStatsDocument = /*#__PURE__*/ gql` } `; export const CreateDataUploadDocument = /*#__PURE__*/ gql` - mutation createDataUpload($projectId: Int!, $filename: String!, $contentType: String!, $replaceTableOfContentsItemId: Int) { + mutation createDataUpload($projectId: Int!, $filename: String!, $contentType: String!, $replaceTableOfContentsItemId: Int, $processingOptions: JSON) { createDataUpload( - input: {filename: $filename, projectId: $projectId, contentType: $contentType, replaceTableOfContentsItemId: $replaceTableOfContentsItemId} + input: {filename: $filename, projectId: $projectId, contentType: $contentType, replaceTableOfContentsItemId: $replaceTableOfContentsItemId, processingOptions: $processingOptions} ) { dataUploadTask { ...DataUploadExtendedDetails @@ -30408,6 +31038,21 @@ export const LayerSettingsChangeLogDocument = /*#__PURE__*/ gql` } } ${ChangeLogDetailsFragmentDoc}`; +export const DataTableChangeLogDocument = /*#__PURE__*/ gql` + query DataTableChangeLog($id: Int!, $first: Int!) { + tableOfContentsItem(id: $id) { + id + overlayDataTables { + id + name + version + } + dataTableChangeLogs(first: $first) { + ...ChangeLogDetails + } + } +} + ${ChangeLogDetailsFragmentDoc}`; export const ResolvableLayerCommentThreadForChangelogDocument = /*#__PURE__*/ gql` query ResolvableLayerCommentThreadForChangelog($commentId: Int!) { getResolvableLayerComment(commentId: $commentId) { @@ -30456,6 +31101,19 @@ export const UpdateEnableDownloadDocument = /*#__PURE__*/ gql` } } `; +export const UpdateDataTablesSettingsDocument = /*#__PURE__*/ gql` + mutation UpdateDataTablesSettings($id: Int!, $enableDataTables: Boolean, $dataTableJoinColumn: String) { + updateTableOfContentsItem( + input: {id: $id, patch: {enableDataTables: $enableDataTables, dataTableJoinColumn: $dataTableJoinColumn}} + ) { + tableOfContentsItem { + id + enableDataTables + dataTableJoinColumn + } + } +} + `; export const UpdateLayerDocument = /*#__PURE__*/ gql` mutation UpdateLayer($id: Int!, $renderUnder: RenderUnderType, $mapboxGlStyles: JSON, $sublayer: String, $staticId: String) { updateDataLayer( @@ -31055,6 +31713,10 @@ export const ChangeLogsSinceLastPublishDocument = /*#__PURE__*/ gql` unresolvedComment { ...ResolvableLayerCommentThread } + overlayDataTables { + id + name + } dataLayer { id dataSource { @@ -31351,9 +32013,9 @@ export const GetBookmarkDocument = /*#__PURE__*/ gql` } ${MapBookmarkDetailsFragmentDoc}`; export const CreateMapBookmarkDocument = /*#__PURE__*/ gql` - mutation CreateMapBookmark($slug: String!, $isPublic: Boolean!, $basemapOptionalLayerStates: JSON, $visibleDataLayers: [String!]!, $cameraOptions: JSON!, $selectedBasemap: Int!, $style: JSON!, $mapDimensions: [Int!]!, $visibleSketches: [Int!]!, $sidebarState: JSON, $basemapName: String!, $layerNames: JSON!, $sketchNames: JSON!, $clientGeneratedThumbnail: String!) { + mutation CreateMapBookmark($slug: String!, $isPublic: Boolean!, $basemapOptionalLayerStates: JSON, $visibleDataLayers: [String!]!, $cameraOptions: JSON!, $selectedBasemap: Int!, $style: JSON!, $mapDimensions: [Int!]!, $visibleSketches: [Int!]!, $sidebarState: JSON, $basemapName: String!, $layerNames: JSON!, $sketchNames: JSON!, $clientGeneratedThumbnail: String!, $dataTableStates: JSON) { createMapBookmark( - input: {isPublic: $isPublic, slug: $slug, basemapOptionalLayerStates: $basemapOptionalLayerStates, visibleDataLayers: $visibleDataLayers, cameraOptions: $cameraOptions, selectedBasemap: $selectedBasemap, style: $style, mapDimensions: $mapDimensions, visibleSketches: $visibleSketches, sidebarState: $sidebarState, basemapName: $basemapName, layerNames: $layerNames, sketchNames: $sketchNames, clientGeneratedThumbnail: $clientGeneratedThumbnail} + input: {isPublic: $isPublic, slug: $slug, basemapOptionalLayerStates: $basemapOptionalLayerStates, visibleDataLayers: $visibleDataLayers, cameraOptions: $cameraOptions, selectedBasemap: $selectedBasemap, style: $style, mapDimensions: $mapDimensions, visibleSketches: $visibleSketches, sidebarState: $sidebarState, basemapName: $basemapName, layerNames: $layerNames, sketchNames: $sketchNames, clientGeneratedThumbnail: $clientGeneratedThumbnail, dataTableStates: $dataTableStates} ) { mapBookmark { ...MapBookmarkDetails @@ -31786,6 +32448,99 @@ export const GetTilePackageDocument = /*#__PURE__*/ gql` } } ${OfflineTilePackageDetailsFragmentDoc}`; +export const OverlayDataTableVisualizationMetadataDocument = /*#__PURE__*/ gql` + query OverlayDataTableVisualizationMetadata($id: Int!) { + overlayDataTable(id: $id) { + id + queryUrl + columnStatsUrl + visualizationColumns + visualizationOps + requiredFilterColumns + } +} + `; +export const OverlayDataTableVisualizationMetadataForLayerDocument = /*#__PURE__*/ gql` + query OverlayDataTableVisualizationMetadataForLayer($tocItemId: Int!) { + tableOfContentsItem(id: $tocItemId) { + id + overlayDataTables { + id + queryUrl + columnStatsUrl + visualizationColumns + visualizationOps + requiredFilterColumns + } + } +} + `; +export const CreateOverlayDataTableUploadDocument = /*#__PURE__*/ gql` + mutation CreateOverlayDataTableUpload($tableOfContentsItemId: Int!, $filename: String!, $contentType: String!, $processingOptions: JSON, $replaceOverlayDataTableId: Int) { + createOverlayDataTableUpload( + input: {tocItemId: $tableOfContentsItemId, filename: $filename, contentType: $contentType, processingOptions: $processingOptions, replaceOverlayDataTableId: $replaceOverlayDataTableId} + ) { + overlayDataTableUpload { + ...OverlayDataTableUploadDetails + } + projectBackgroundJob { + ...JobDetails + } + } +} + ${OverlayDataTableUploadDetailsFragmentDoc} +${JobDetailsFragmentDoc}`; +export const SubmitOverlayDataTableUploadDocument = /*#__PURE__*/ gql` + mutation SubmitOverlayDataTableUpload($jobId: UUID!) { + submitOverlayDataTableUpload(input: {jobId: $jobId}) { + projectBackgroundJob { + id + state + progress + progressMessage + } + } +} + `; +export const RenameOverlayDataTableDocument = /*#__PURE__*/ gql` + mutation RenameOverlayDataTable($id: Int!, $name: String!) { + renameOverlayDataTable(input: {tableId: $id, newName: $name}) { + overlayDataTable { + ...OverlayDataTableDetails + } + } +} + ${OverlayDataTableDetailsFragmentDoc}`; +export const SoftDeleteOverlayDataTableDocument = /*#__PURE__*/ gql` + mutation SoftDeleteOverlayDataTable($id: Int!) { + softDeleteOverlayDataTable(input: {tableId: $id}) { + overlayDataTable { + id + deletedAt + } + } +} + `; +export const RollbackOverlayDataTableVersionDocument = /*#__PURE__*/ gql` + mutation RollbackOverlayDataTableVersion($id: Int!) { + rollbackOverlayDataTableVersion(input: {tableId: $id}) { + overlayDataTable { + ...OverlayDataTableDetails + } + } +} + ${OverlayDataTableDetailsFragmentDoc}`; +export const SetOverlayDataTableVisualizationSettingsDocument = /*#__PURE__*/ gql` + mutation SetOverlayDataTableVisualizationSettings($id: Int!, $visualizationColumns: [String]!, $visualizationOps: [String]!, $requiredFilterColumns: [String]!) { + setOverlayDataTableVisualizationSettings( + input: {tableId: $id, visualizationColumns: $visualizationColumns, visualizationOps: $visualizationOps, requiredFilterColumns: $requiredFilterColumns} + ) { + overlayDataTable { + ...OverlayDataTableDetails + } + } +} + ${OverlayDataTableDetailsFragmentDoc}`; export const ProjectAccessControlSettingsDocument = /*#__PURE__*/ gql` query ProjectAccessControlSettings($slug: String!) { projectBySlug(slug: $slug) { @@ -33790,6 +34545,7 @@ export const UpdateFeatureFlagsDocument = /*#__PURE__*/ gql` id featureFlags { iNaturalistLayers + dataTables } } } @@ -34221,6 +34977,7 @@ export const namedOperations = { layersAndSourcesForItems: 'layersAndSourcesForItems', GetFolder: 'GetFolder', LayerSettingsChangeLog: 'LayerSettingsChangeLog', + DataTableChangeLog: 'DataTableChangeLog', ResolvableLayerCommentThreadForChangelog: 'ResolvableLayerCommentThreadForChangelog', GetLayerItem: 'GetLayerItem', InteractivitySettingsForLayer: 'InteractivitySettingsForLayer', @@ -34259,6 +35016,8 @@ export const namedOperations = { OfflineSurveyMaps: 'OfflineSurveyMaps', BasemapOfflineSettings: 'BasemapOfflineSettings', getTilePackage: 'getTilePackage', + OverlayDataTableVisualizationMetadata: 'OverlayDataTableVisualizationMetadata', + OverlayDataTableVisualizationMetadataForLayer: 'OverlayDataTableVisualizationMetadataForLayer', ProjectAccessControlSettings: 'ProjectAccessControlSettings', ProjectDashboard: 'ProjectDashboard', ProjectDashboardBannerStats: 'ProjectDashboardBannerStats', @@ -34378,6 +35137,7 @@ export const namedOperations = { UpdateFolder: 'UpdateFolder', UpdateTableOfContentsItem: 'UpdateTableOfContentsItem', UpdateEnableDownload: 'UpdateEnableDownload', + UpdateDataTablesSettings: 'UpdateDataTablesSettings', UpdateLayer: 'UpdateLayer', UpdateDataSource: 'UpdateDataSource', UpdateInteractivitySettings: 'UpdateInteractivitySettings', @@ -34425,6 +35185,12 @@ export const namedOperations = { UpdateBasemapOfflineTileSettings: 'UpdateBasemapOfflineTileSettings', generateOfflineTilePackage: 'generateOfflineTilePackage', deleteTilePackage: 'deleteTilePackage', + CreateOverlayDataTableUpload: 'CreateOverlayDataTableUpload', + SubmitOverlayDataTableUpload: 'SubmitOverlayDataTableUpload', + RenameOverlayDataTable: 'RenameOverlayDataTable', + SoftDeleteOverlayDataTable: 'SoftDeleteOverlayDataTable', + RollbackOverlayDataTableVersion: 'RollbackOverlayDataTableVersion', + SetOverlayDataTableVisualizationSettings: 'SetOverlayDataTableVisualizationSettings', updateProjectAccessControlSettings: 'updateProjectAccessControlSettings', toggleLanguageSupport: 'toggleLanguageSupport', setTranslatedProps: 'setTranslatedProps', @@ -34599,6 +35365,9 @@ export const namedOperations = { OfflineBasemapDetails: 'OfflineBasemapDetails', OfflineTileSettingsForCalculation: 'OfflineTileSettingsForCalculation', OfflineTileSettings: 'OfflineTileSettings', + ClientOverlayDataTable: 'ClientOverlayDataTable', + OverlayDataTableDetails: 'OverlayDataTableDetails', + OverlayDataTableUploadDetails: 'OverlayDataTableUploadDetails', ProjectMetadata: 'ProjectMetadata', ProjectPublicDetailsMetadata: 'ProjectPublicDetailsMetadata', ProjectMetadataMeFrag: 'ProjectMetadataMeFrag', diff --git a/packages/client/src/index.css b/packages/client/src/index.css index 678dadee8..d3b9c525a 100644 --- a/packages/client/src/index.css +++ b/packages/client/src/index.css @@ -6611,6 +6611,10 @@ select{ z-index:100 } +.z-\[110\]{ + z-index:110 +} + .z-\[1400\]{ z-index:1400 } @@ -6659,6 +6663,10 @@ select{ z-index:80 } +.z-\[999999\]{ + z-index:999999 +} + .z-auto{ z-index:auto } @@ -6919,6 +6927,11 @@ select{ margin-bottom:0px } +.my-0\.5{ + margin-top:0.125rem; + margin-bottom:0.125rem +} + .my-1{ margin-top:0.25rem; margin-bottom:0.25rem @@ -7882,6 +7895,10 @@ select{ max-height:45vh } +.max-h-\[70vh\]{ + max-height:70vh +} + .max-h-\[88vh\]{ max-height:88vh } @@ -8412,6 +8429,10 @@ select{ min-width:12rem } +.min-w-\[14rem\]{ + min-width:14rem +} + .min-w-\[160px\]{ min-width:160px } @@ -8472,6 +8493,10 @@ select{ min-width:var(--radix-dropdown-menu-trigger-width) } +.min-w-\[var\(--radix-select-trigger-width\)\]{ + min-width:var(--radix-select-trigger-width) +} + .min-w-full{ min-width:100% } @@ -8709,6 +8734,10 @@ select{ max-width:520px } +.max-w-\[58\%\]{ + max-width:58% +} + .max-w-\[620px\]{ max-width:620px } @@ -10198,6 +10227,10 @@ select{ border-width:8px } +.border-\[1\.5px\]{ + border-width:1.5px +} + .border-y{ border-top-width:1px; border-bottom-width:1px @@ -10909,6 +10942,10 @@ select{ background-color:rgb(255 251 235 / 0.6) } +.bg-amber-50\/70{ + background-color:rgb(255 251 235 / 0.7) +} + .bg-amber-50\/90{ background-color:rgb(255 251 235 / 0.9) } @@ -10943,6 +10980,10 @@ select{ background-color:rgb(0 0 0 / 0.5) } +.bg-black\/\[0\.07\]{ + background-color:rgb(0 0 0 / 0.07) +} + .bg-blue-100{ --tw-bg-opacity:1; background-color:rgb(219 234 254 / var(--tw-bg-opacity, 1)) @@ -11223,6 +11264,10 @@ select{ background-color:rgb(31 41 55 / 0.8) } +.bg-gray-800\/90{ + background-color:rgb(31 41 55 / 0.9) +} + .bg-gray-900{ --tw-bg-opacity:1; background-color:rgb(17 24 39 / var(--tw-bg-opacity, 1)) @@ -12733,6 +12778,10 @@ select{ padding-bottom:0.5rem } +.pb-2\.5{ + padding-bottom:0.625rem +} + .pb-20{ padding-bottom:5rem } @@ -15298,6 +15347,11 @@ input[type="checkbox"][indeterminate="true"]:checked { } } +.placeholder\:text-gray-400::placeholder{ + --tw-text-opacity:1; + color:rgb(156 163 175 / var(--tw-text-opacity, 1)) +} + .placeholder\:text-gray-500::placeholder{ --tw-text-opacity:1; color:rgb(107 114 128 / var(--tw-text-opacity, 1)) @@ -15353,6 +15407,12 @@ input[type="checkbox"][indeterminate="true"]:checked { --tw-ring-opacity:0.5 } +.hover\:scale-110:hover{ + --tw-scale-x:1.1; + --tw-scale-y:1.1; + transform:translate(var(--tw-translate-x), var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y)) +} + .hover\:scale-125:hover{ --tw-scale-x:1.25; --tw-scale-y:1.25; @@ -15542,6 +15602,10 @@ input[type="checkbox"][indeterminate="true"]:checked { background-color:rgb(55 65 81 / var(--tw-bg-opacity, 1)) } +.hover\:bg-gray-700\/80:hover{ + background-color:rgb(55 65 81 / 0.8) +} + .hover\:bg-gray-900:hover{ --tw-bg-opacity:1; background-color:rgb(17 24 39 / var(--tw-bg-opacity, 1)) @@ -15556,6 +15620,11 @@ input[type="checkbox"][indeterminate="true"]:checked { background-color:rgb(165 180 252 / 0.05) } +.hover\:bg-indigo-50:hover{ + --tw-bg-opacity:1; + background-color:rgb(238 242 255 / var(--tw-bg-opacity, 1)) +} + .hover\:bg-primary-300:hover{ --tw-bg-opacity:1; background-color:rgb(71 163 220 / var(--tw-bg-opacity, 1)) @@ -15812,6 +15881,16 @@ input[type="checkbox"][indeterminate="true"]:checked { color:rgb(99 102 241 / var(--tw-text-opacity, 1)) } +.hover\:text-indigo-700:hover{ + --tw-text-opacity:1; + color:rgb(67 56 202 / var(--tw-text-opacity, 1)) +} + +.hover\:text-primary-500:hover{ + --tw-text-opacity:1; + color:rgb(46 115 182 / var(--tw-text-opacity, 1)) +} + .hover\:text-primary-600:hover{ --tw-text-opacity:1; color:rgb(36 105 172 / var(--tw-text-opacity, 1)) @@ -16281,6 +16360,11 @@ input[type="checkbox"][indeterminate="true"]:checked { border-color:rgb(59 130 246 / var(--tw-border-opacity, 1)) } +.focus-visible\:border-primary-500:focus-visible{ + --tw-border-opacity:1; + border-color:rgb(46 115 182 / var(--tw-border-opacity, 1)) +} + .focus-visible\:bg-blue-100:focus-visible{ --tw-bg-opacity:1; background-color:rgb(219 234 254 / var(--tw-bg-opacity, 1)) @@ -16376,6 +16460,11 @@ input[type="checkbox"][indeterminate="true"]:checked { --tw-ring-color:rgb(59 130 246 / 0.25) } +.focus-visible\:ring-indigo-300:focus-visible{ + --tw-ring-opacity:1; + --tw-ring-color:rgb(165 180 252 / var(--tw-ring-opacity, 1)) +} + .focus-visible\:ring-primary-400:focus-visible{ --tw-ring-opacity:1; --tw-ring-color:rgb(61 153 210 / var(--tw-ring-opacity, 1)) @@ -16712,6 +16801,11 @@ input[type="checkbox"][indeterminate="true"]:checked { background-color:rgb(243 244 246 / var(--tw-bg-opacity, 1)) } +.data-\[highlighted\]\:bg-gray-50[data-highlighted]{ + --tw-bg-opacity:1; + background-color:rgb(249 250 251 / var(--tw-bg-opacity, 1)) +} + .data-\[highlighted\]\:bg-red-50[data-highlighted]{ --tw-bg-opacity:1; background-color:rgb(254 242 242 / var(--tw-bg-opacity, 1)) @@ -16732,6 +16826,11 @@ input[type="checkbox"][indeterminate="true"]:checked { background-color:rgb(243 244 246 / var(--tw-bg-opacity, 1)) } +.data-\[state\=open\]\:bg-gray-50[data-state="open"]{ + --tw-bg-opacity:1; + background-color:rgb(249 250 251 / var(--tw-bg-opacity, 1)) +} + .data-\[highlighted\]\:text-blue-900[data-highlighted]{ --tw-text-opacity:1; color:rgb(30 58 138 / var(--tw-text-opacity, 1)) @@ -16979,6 +17078,10 @@ input[type="checkbox"][indeterminate="true"]:checked { width:24rem } + .sm\:w-\[34rem\]{ + width:34rem + } + .sm\:w-auto{ width:auto } diff --git a/packages/client/src/projects/ProjectApp.tsx b/packages/client/src/projects/ProjectApp.tsx index 2c2cc9cbd..e69676360 100644 --- a/packages/client/src/projects/ProjectApp.tsx +++ b/packages/client/src/projects/ProjectApp.tsx @@ -157,7 +157,9 @@ export default function ProjectApp() { defaultShowScale={data?.project?.showScalebarByDefault || false} > - + (null); const loading = useMemo(() => { for (const key in layerTree.layerStatesByTocStaticId) { if ( @@ -30,12 +34,20 @@ export default function ProjectMapLegend({ }, [layerTree.layerStatesByTocStaticId]); const legendProps = useCommonLegendProps( - { layerStatesByTocStaticId: layerTree.layerStatesByTocStaticId, legends: legendsCtx.legends }, + { + layerStatesByTocStaticId: layerTree.layerStatesByTocStaticId, + legends: legendsCtx.legends, + }, manager, sketchClassLayerStates ); - const isSmall = useMediaQuery("(max-width: 1535px)"); + const onDataTableActivated = useCallback((layerId: string) => { + setLegendFocusRequest({ layerId, requestId: Date.now() }); + }, []); + const onLegendFocusComplete = useCallback(() => { + setLegendFocusRequest(null); + }, []); if (legendProps.items.length > 0) { return ( @@ -50,6 +62,9 @@ export default function ProjectMapLegend({ persistedStateKey="project-map-legend" {...legendProps} defaultToHidden={!showByDefault} + legendFocusRequest={legendFocusRequest} + onLegendFocusComplete={onLegendFocusComplete} + onDataTableActivated={onDataTableActivated} /> ); } else { diff --git a/packages/client/src/queries/DataUploads.graphql b/packages/client/src/queries/DataUploads.graphql index 4ab91c2d5..9c48ed266 100644 --- a/packages/client/src/queries/DataUploads.graphql +++ b/packages/client/src/queries/DataUploads.graphql @@ -19,6 +19,7 @@ mutation createDataUpload( $filename: String! $contentType: String! $replaceTableOfContentsItemId: Int + $processingOptions: JSON ) { createDataUpload( input: { @@ -26,6 +27,7 @@ mutation createDataUpload( projectId: $projectId contentType: $contentType replaceTableOfContentsItemId: $replaceTableOfContentsItemId + processingOptions: $processingOptions } ) { dataUploadTask { @@ -58,6 +60,12 @@ fragment JobDetails on ProjectBackgroundJob { title } } + overlayDataTableUpload { + id + tableOfContentsItemId + filename + replaceOverlayDataTableId + } } mutation submitDataUpload($jobId: UUID!, $enableAiDataAnalyst: Boolean) { diff --git a/packages/client/src/queries/DraftTableOfContents.graphql b/packages/client/src/queries/DraftTableOfContents.graphql index 45caa690a..4120309dc 100644 --- a/packages/client/src/queries/DraftTableOfContents.graphql +++ b/packages/client/src/queries/DraftTableOfContents.graphql @@ -462,6 +462,20 @@ query LayerSettingsChangeLog($id: Int!, $first: Int!) { } } +query DataTableChangeLog($id: Int!, $first: Int!) { + tableOfContentsItem(id: $id) { + id + overlayDataTables { + id + name + version + } + dataTableChangeLogs(first: $first) { + ...ChangeLogDetails + } + } +} + fragment ResolvableLayerCommentDetails on ResolvableLayerComment { id projectId @@ -516,22 +530,21 @@ fragment FullAdminOverlay on TableOfContentsItem { stableId title enableDownload + enableDataTables + dataTableJoinColumn geoprocessingReferenceId copiedFromDataLibraryTemplateId primaryDownloadUrl projectBackgroundJobs { - id - type - title - state - progress - progressMessage - errorMessage + ...JobDetails } hasOriginalSourceUpload dataLayer { ...FullAdminDataLayer } + overlayDataTables { + ...OverlayDataTableDetails + } relatedReportCardDetails { isDraft title @@ -601,6 +614,28 @@ mutation UpdateEnableDownload($id: Int!, $enableDownload: Boolean) { } } +mutation UpdateDataTablesSettings( + $id: Int! + $enableDataTables: Boolean + $dataTableJoinColumn: String +) { + updateTableOfContentsItem( + input: { + id: $id + patch: { + enableDataTables: $enableDataTables + dataTableJoinColumn: $dataTableJoinColumn + } + } + ) { + tableOfContentsItem { + id + enableDataTables + dataTableJoinColumn + } + } +} + mutation UpdateLayer( $id: Int! $renderUnder: RenderUnderType @@ -1270,6 +1305,10 @@ query ChangeLogsSinceLastPublish($slug: String!) { unresolvedComment { ...ResolvableLayerCommentThread } + overlayDataTables { + id + name + } dataLayer { id dataSource { diff --git a/packages/client/src/queries/Forums.graphql b/packages/client/src/queries/Forums.graphql index eb15d3db9..b7ed6748b 100644 --- a/packages/client/src/queries/Forums.graphql +++ b/packages/client/src/queries/Forums.graphql @@ -246,6 +246,7 @@ fragment MapBookmarkDetails on MapBookmark { basemapName sketchNames clientGeneratedThumbnail + dataTableStates } query GetBookmark($id: UUID!) { @@ -269,6 +270,7 @@ mutation CreateMapBookmark( $layerNames: JSON! $sketchNames: JSON! $clientGeneratedThumbnail: String! + $dataTableStates: JSON ) { createMapBookmark( input: { @@ -286,6 +288,7 @@ mutation CreateMapBookmark( layerNames: $layerNames sketchNames: $sketchNames clientGeneratedThumbnail: $clientGeneratedThumbnail + dataTableStates: $dataTableStates } ) { mapBookmark { diff --git a/packages/client/src/queries/OverlayDataTables.graphql b/packages/client/src/queries/OverlayDataTables.graphql new file mode 100644 index 000000000..c1b53a8c8 --- /dev/null +++ b/packages/client/src/queries/OverlayDataTables.graphql @@ -0,0 +1,158 @@ +# Lean, public-safe subset of OverlayDataTable fields for the map legend / +# table of contents (ActivatedDataTableButton). Deliberately omits internal +# remote paths that only make sense to admins. +fragment ClientOverlayDataTable on OverlayDataTable { + id + stableId + name + version + rowCount + joinColumn + overlayJoinColumn + queryUrl + columnStatsUrl + visualizationColumns + visualizationOps + requiredFilterColumns +} + +fragment OverlayDataTableDetails on OverlayDataTable { + id + stableId + name + version + joinColumn + overlayJoinColumn + rowCount + parquetRemote + columnStatsRemote + parquetUrl + columnStatsUrl + queryUrl + deletedAt + replacedById + createdAt + updatedAt + visualizationColumns + visualizationOps + requiredFilterColumns +} + +query OverlayDataTableVisualizationMetadata($id: Int!) { + overlayDataTable(id: $id) { + id + queryUrl + columnStatsUrl + visualizationColumns + visualizationOps + requiredFilterColumns + } +} + +query OverlayDataTableVisualizationMetadataForLayer($tocItemId: Int!) { + tableOfContentsItem(id: $tocItemId) { + id + overlayDataTables { + id + queryUrl + columnStatsUrl + visualizationColumns + visualizationOps + requiredFilterColumns + } + } +} + +fragment OverlayDataTableUploadDetails on OverlayDataTableUpload { + id + tableOfContentsItemId + filename + contentType + processingOptions + overlayJoinColumn + errorDetails + presignedUploadUrl + replaceOverlayDataTableId + projectBackgroundJobId +} + +mutation CreateOverlayDataTableUpload( + $tableOfContentsItemId: Int! + $filename: String! + $contentType: String! + $processingOptions: JSON + $replaceOverlayDataTableId: Int +) { + createOverlayDataTableUpload( + input: { + tocItemId: $tableOfContentsItemId + filename: $filename + contentType: $contentType + processingOptions: $processingOptions + replaceOverlayDataTableId: $replaceOverlayDataTableId + } + ) { + overlayDataTableUpload { + ...OverlayDataTableUploadDetails + } + projectBackgroundJob { + ...JobDetails + } + } +} + +mutation SubmitOverlayDataTableUpload($jobId: UUID!) { + submitOverlayDataTableUpload(input: { jobId: $jobId }) { + projectBackgroundJob { + id + state + progress + progressMessage + } + } +} + +mutation RenameOverlayDataTable($id: Int!, $name: String!) { + renameOverlayDataTable(input: { tableId: $id, newName: $name }) { + overlayDataTable { + ...OverlayDataTableDetails + } + } +} + +mutation SoftDeleteOverlayDataTable($id: Int!) { + softDeleteOverlayDataTable(input: { tableId: $id }) { + overlayDataTable { + id + deletedAt + } + } +} + +mutation RollbackOverlayDataTableVersion($id: Int!) { + rollbackOverlayDataTableVersion(input: { tableId: $id }) { + overlayDataTable { + ...OverlayDataTableDetails + } + } +} + +mutation SetOverlayDataTableVisualizationSettings( + $id: Int! + $visualizationColumns: [String]! + $visualizationOps: [String]! + $requiredFilterColumns: [String]! +) { + setOverlayDataTableVisualizationSettings( + input: { + tableId: $id + visualizationColumns: $visualizationColumns + visualizationOps: $visualizationOps + requiredFilterColumns: $requiredFilterColumns + } + ) { + overlayDataTable { + ...OverlayDataTableDetails + } + } +} diff --git a/packages/client/src/queries/ProjectMetadata.graphql b/packages/client/src/queries/ProjectMetadata.graphql index e2f4c9a6b..e019fdc9b 100644 --- a/packages/client/src/queries/ProjectMetadata.graphql +++ b/packages/client/src/queries/ProjectMetadata.graphql @@ -45,6 +45,7 @@ fragment ProjectMetadata on Project { showLegendByDefault featureFlags { iNaturalistLayers + dataTables } } diff --git a/packages/client/src/queries/PublishedTableOfContents.graphql b/packages/client/src/queries/PublishedTableOfContents.graphql index 6684d8898..a047b1829 100644 --- a/packages/client/src/queries/PublishedTableOfContents.graphql +++ b/packages/client/src/queries/PublishedTableOfContents.graphql @@ -20,6 +20,9 @@ fragment Overlay on TableOfContentsItem { hasMetadata primaryDownloadUrl dataSourceType + overlayDataTables { + ...ClientOverlayDataTable + } } query PublishedTableOfContents($slug: String!) { diff --git a/packages/client/src/queries/UpdateProjectSettings.graphql b/packages/client/src/queries/UpdateProjectSettings.graphql index 2c5561545..67acd5c2c 100644 --- a/packages/client/src/queries/UpdateProjectSettings.graphql +++ b/packages/client/src/queries/UpdateProjectSettings.graphql @@ -94,6 +94,7 @@ mutation UpdateFeatureFlags($slug: String!, $flags: JSON!) { id featureFlags { iNaturalistLayers + dataTables } } } diff --git a/packages/data-tables-handler/.env.template b/packages/data-tables-handler/.env.template new file mode 100644 index 000000000..fbb32441b --- /dev/null +++ b/packages/data-tables-handler/.env.template @@ -0,0 +1,21 @@ +# Copy to .env and fill in (see packages/spatial-uploads-handler/.env for values). +PGHOST=localhost +PGPORT=54321 +PGUSER=postgres +PGDATABASE=seasketch +PGPASSWORD=password +PGREGION=us-west-2 + +BUCKET= +AWS_REGION=us-west-2 +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= + +R2_ACCESS_KEY_ID= +R2_SECRET_ACCESS_KEY= +R2_ENDPOINT= +R2_TILES_BUCKET=ssn-tiles +TILES_REMOTE=r2://ssn-tiles +UPLOADS_BASE_URL=https://uploads.seasketch.org + +PORT=3006 diff --git a/packages/data-tables-handler/Dockerfile b/packages/data-tables-handler/Dockerfile new file mode 100644 index 000000000..d9b692e53 --- /dev/null +++ b/packages/data-tables-handler/Dockerfile @@ -0,0 +1,21 @@ +FROM amazon/aws-lambda-nodejs:20 as builder +WORKDIR /usr/app +COPY data-tables-handler/package.json data-tables-handler/package-lock.json ./ +RUN npm ci +WORKDIR /usr/geostats-types +COPY geostats-types/package.json geostats-types/package-lock.json ./ +RUN npm ci +COPY geostats-types/ ./ +RUN npm run build +WORKDIR /usr/app +RUN mkdir -p node_modules/@seasketch && cp -r /usr/geostats-types node_modules/@seasketch/geostats-types +COPY data-tables-handler/tsconfig.json data-tables-handler/handler.ts ./ +COPY data-tables-handler/src ./src +RUN npm run build + +FROM amazon/aws-lambda-nodejs:20 +WORKDIR ${LAMBDA_TASK_ROOT} +COPY --from=builder /usr/app/dist/* ./ +COPY --from=builder /usr/app/dist/src ./src +COPY --from=builder /usr/app/node_modules ./node_modules +CMD ["handler.processDataTableUpload"] diff --git a/packages/data-tables-handler/dist/handler.d.ts b/packages/data-tables-handler/dist/handler.d.ts new file mode 100644 index 000000000..668680154 --- /dev/null +++ b/packages/data-tables-handler/dist/handler.d.ts @@ -0,0 +1,3 @@ +import type { DataTablesHandlerRequest } from "./src/types"; +export type { DataTablesHandlerRequest, DataTablesHandlerResponse } from "./src/types"; +export declare const processDataTableUpload: (event: DataTablesHandlerRequest) => Promise; diff --git a/packages/data-tables-handler/dist/handler.js b/packages/data-tables-handler/dist/handler.js new file mode 100644 index 000000000..15a36c4bd --- /dev/null +++ b/packages/data-tables-handler/dist/handler.js @@ -0,0 +1,16 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.processDataTableUpload = void 0; +const handleDataTableUpload_1 = __importDefault(require("./src/handleDataTableUpload")); +const processDataTableUpload = async (event) => { + try { + return await (0, handleDataTableUpload_1.default)(event); + } + catch (e) { + return { error: e.message }; + } +}; +exports.processDataTableUpload = processDataTableUpload; diff --git a/packages/data-tables-handler/dist/server.d.ts b/packages/data-tables-handler/dist/server.d.ts new file mode 100644 index 000000000..bd6913440 --- /dev/null +++ b/packages/data-tables-handler/dist/server.d.ts @@ -0,0 +1,2 @@ +#!/usr/bin/env tsx +export {}; diff --git a/packages/data-tables-handler/dist/server.js b/packages/data-tables-handler/dist/server.js new file mode 100644 index 000000000..82fabb3a9 --- /dev/null +++ b/packages/data-tables-handler/dist/server.js @@ -0,0 +1,58 @@ +#!/usr/bin/env tsx +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const http_1 = require("http"); +const dotenv = __importStar(require("dotenv")); +dotenv.config(); +const PORT = process.env.PORT || 3006; +const server = (0, http_1.createServer)(function (req, res) { + if (req.method === "POST") { + let body = ""; + req.on("data", (chunk) => { + body += chunk.toString(); + }); + req.on("end", async () => { + try { + const data = JSON.parse(body); + const { processDataTableUpload } = await Promise.resolve().then(() => __importStar(require("./handler"))); + const outputs = await processDataTableUpload(data); + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify(outputs)); + } + catch (e) { + res.writeHead(500, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: e.message })); + } + }); + } + else { + res.writeHead(200); + res.end("POST to simulate data-tables-handler lambda"); + } +}); +server.listen(PORT); +console.log(`Data tables handler dev server on http://localhost:${PORT}`); +console.log(`Set DATA_TABLES_LAMBDA_DEV_HANDLER=http://localhost:${PORT}`); diff --git a/packages/data-tables-handler/dist/src/handleDataTableUpload.d.ts b/packages/data-tables-handler/dist/src/handleDataTableUpload.d.ts new file mode 100644 index 000000000..b3fad443b --- /dev/null +++ b/packages/data-tables-handler/dist/src/handleDataTableUpload.d.ts @@ -0,0 +1,2 @@ +import type { DataTablesHandlerRequest, DataTablesHandlerResponse } from "./types"; +export default function handleDataTableUpload(request: DataTablesHandlerRequest): Promise; diff --git a/packages/data-tables-handler/dist/src/handleDataTableUpload.js b/packages/data-tables-handler/dist/src/handleDataTableUpload.js new file mode 100644 index 000000000..83a5894b1 --- /dev/null +++ b/packages/data-tables-handler/dist/src/handleDataTableUpload.js @@ -0,0 +1,165 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.default = handleDataTableUpload; +const tmp_1 = require("tmp"); +const path = __importStar(require("path")); +const fs_1 = require("fs"); +const lambda_db_client_1 = require("./lambda-db-client"); +const remotes_1 = require("./remotes"); +const validateJoinColumn_1 = require("./validateJoinColumn"); +const processWithDuckDb_1 = require("./processWithDuckDb"); +const PARQUET_CONTENT_TYPE = "application/vnd.apache.parquet"; +const JSON_CONTENT_TYPE = "application/json; charset=utf-8"; +function defaultTableName(filename) { + return filename.replace(/\.[^.]+$/, ""); +} +function logDebug(message, details) { + const suffix = details ? ` ${JSON.stringify(details)}` : ""; + console.log(`[data-tables-handler] ${message}${suffix}`); +} +async function handleDataTableUpload(request) { + const { taskId, uploadId, objectKey, slug, sourceUuid, skipLoggingProgress, } = request; + logDebug("starting upload", { + taskId, + uploadId, + objectKey, + slug, + sourceUuid, + skipLoggingProgress: Boolean(skipLoggingProgress), + }); + const tmpobj = (0, tmp_1.dirSync)({ unsafeCleanup: true, keep: false, prefix: "dt-" }); + const csvPath = path.join(tmpobj.name, "input.csv"); + const parquetPath = path.join(tmpobj.name, "data.parquet"); + const statsPath = path.join(tmpobj.name, "column-stats.json"); + // Established inside try so that connection failures are still reported to + // the caller; if the DB is unreachable the job fails via timeout instead. + let pgClient = null; + const updateProgress = async (state, progressMessage, progress) => { + if (skipLoggingProgress || !pgClient) + return; + logDebug("updateProgress", { taskId, state, progressMessage, progress }); + if (progress !== undefined) { + await pgClient.query(`update project_background_jobs set state = $1, progress = least($2::numeric, 1.0::numeric), progress_message = $3 where id = $4`, [state, progress, progressMessage, taskId]); + } + else { + await pgClient.query(`update project_background_jobs set state = $1, progress_message = $2 where id = $3`, [state, progressMessage, taskId]); + } + }; + try { + if (!sourceUuid) { + throw new Error("sourceUuid is required to store data tables under the parent layer path"); + } + pgClient = await (0, lambda_db_client_1.getClient)(); + await updateProgress("running", "downloading", 0.05); + const uploadQ = await pgClient.query(`select filename, processing_options, overlay_geostats, overlay_join_column, replace_overlay_data_table_id + from overlay_data_table_uploads where id = $1`, [uploadId]); + if (!uploadQ.rows[0]) { + throw new Error("Upload record not found"); + } + const upload = uploadQ.rows[0]; + const processingOptions = (upload.processing_options || + {}); + const joinColumn = processingOptions.joinColumn; + const overlayJoinColumn = processingOptions.overlayJoinColumn || upload.overlay_join_column; + if (!joinColumn || !overlayJoinColumn) { + throw new Error("Join column and overlay join column are required"); + } + logDebug("downloading staging object", { objectKey, csvPath }); + await (0, remotes_1.getStagingObject)(csvPath, objectKey); + await updateProgress("running", "processing", 0.2); + logDebug("processing csv with duckdb", { csvPath, parquetPath }); + const { rowCount, headers } = await (0, processWithDuckDb_1.processCsvWithDuckDb)(csvPath, parquetPath, processingOptions); + logDebug("csv processed", { rowCount, columnCount: headers.length }); + const joinValues = await (0, processWithDuckDb_1.readJoinValues)(parquetPath, joinColumn); + const layer = (0, validateJoinColumn_1.getGeostatsLayer)(upload.overlay_geostats); + const joinValidation = (0, validateJoinColumn_1.validateJoinColumnChoice)(headers, joinColumn, overlayJoinColumn, layer, joinValues); + logDebug("join validation complete", { + joinColumn, + overlayJoinColumn, + matchRate: joinValidation.matchRate, + matchedRows: joinValidation.matchedRows, + unmatchedRows: joinValidation.unmatchedRows, + }); + await updateProgress("running", "computing stats", 0.6); + const tableName = processingOptions.name || defaultTableName(upload.filename); + const columnStats = await (0, processWithDuckDb_1.computeColumnStatsFromParquet)(parquetPath, tableName, { + column: joinColumn, + overlayAttribute: overlayJoinColumn, + matchRate: joinValidation.matchRate, + matchedRows: joinValidation.matchedRows, + unmatchedRows: joinValidation.unmatchedRows, + unmatchedOverlayValues: joinValidation.unmatchedOverlayValues, + }); + (0, fs_1.writeFileSync)(statsPath, JSON.stringify(columnStats)); + await updateProgress("running", "uploading", 0.8); + const parquetTarget = (0, remotes_1.buildR2Remote)(slug, sourceUuid, uploadId, "data.parquet"); + const statsTarget = (0, remotes_1.buildR2Remote)(slug, sourceUuid, uploadId, "column-stats.json"); + await (0, remotes_1.putObject)(parquetPath, parquetTarget.remote, PARQUET_CONTENT_TYPE); + await (0, remotes_1.putObject)(statsPath, statsTarget.remote, JSON_CONTENT_TYPE); + const result = { + uploadId, + name: tableName, + joinColumn, + overlayJoinColumn, + rowCount: Math.trunc(Number(rowCount)), + parquetRemote: parquetTarget.remote, + columnStatsRemote: statsTarget.remote, + }; + logDebug("upload processing complete, enqueueing outputs job", { + taskId, + uploadId, + rowCount: result.rowCount, + parquetRemote: result.parquetRemote, + }); + await pgClient.query(`SELECT graphile_worker.add_job('processDataTableUploadOutputs', $1::json)`, [JSON.stringify({ jobId: taskId, data: result })]); + return { success: result }; + } + catch (e) { + const error = e; + logDebug("upload failed", { + taskId, + uploadId, + message: error.message, + stack: error.stack, + }); + const errorDetails = { message: error.message }; + if (pgClient) { + try { + await pgClient.query(`select fail_overlay_data_table_upload($1, $2, $3)`, [taskId, error.message, JSON.stringify(errorDetails)]); + } + catch (failError) { + logDebug("failed to record job failure", { + taskId, + error: failError.message, + }); + } + } + return { error: error.message, errorDetails }; + } + finally { + tmpobj.removeCallback(); + } +} diff --git a/packages/data-tables-handler/dist/src/inferCsvColumnPlans.d.ts b/packages/data-tables-handler/dist/src/inferCsvColumnPlans.d.ts new file mode 100644 index 000000000..583dbfb94 --- /dev/null +++ b/packages/data-tables-handler/dist/src/inferCsvColumnPlans.d.ts @@ -0,0 +1,17 @@ +import duckdb from "duckdb"; +export declare const CSV_NULL_STRINGS_BASE: string[]; +export declare const CSV_NULL_STRINGS_WITH_NA: string[]; +export type CsvColumnPlan = { + name: string; + duckDbType: string; + nullifyNa: boolean; +}; +export declare function nullstrOption(nullStrings: string[]): string; +export declare function isNumericDuckDbType(type: string): boolean; +/** + * Compare two typed CSV reads (with vs without treating "NA" as null) to decide + * per-column whether bare "NA" is missing data or a valid string code. + */ +export declare function decideNaNullHandling(strictType: string, naNullType: string, naLiteralCount: number, naLooksLikeValidCode: boolean): Pick; +export declare function inferCsvColumnPlans(conn: duckdb.Connection, csvPath: string, readCsvOptionsSuffix: string): Promise; +export declare function buildTypedSelectSql(plans: CsvColumnPlan[]): string; diff --git a/packages/data-tables-handler/dist/src/inferCsvColumnPlans.js b/packages/data-tables-handler/dist/src/inferCsvColumnPlans.js new file mode 100644 index 000000000..96b2784e1 --- /dev/null +++ b/packages/data-tables-handler/dist/src/inferCsvColumnPlans.js @@ -0,0 +1,114 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.CSV_NULL_STRINGS_WITH_NA = exports.CSV_NULL_STRINGS_BASE = void 0; +exports.nullstrOption = nullstrOption; +exports.isNumericDuckDbType = isNumericDuckDbType; +exports.decideNaNullHandling = decideNaNullHandling; +exports.inferCsvColumnPlans = inferCsvColumnPlans; +exports.buildTypedSelectSql = buildTypedSelectSql; +exports.CSV_NULL_STRINGS_BASE = ["", "N/A", "#N/A", "NULL", "null"]; +exports.CSV_NULL_STRINGS_WITH_NA = [...exports.CSV_NULL_STRINGS_BASE, "NA"]; +function run(conn, sql) { + return new Promise((resolve, reject) => { + conn.run(sql, (err) => (err ? reject(err) : resolve())); + }); +} +function all(conn, sql) { + return new Promise((resolve, reject) => { + conn.all(sql, (err, rows) => (err ? reject(err) : resolve(rows))); + }); +} +function nullstrOption(nullStrings) { + return `nullstr=[${nullStrings.map((v) => `'${v.replace(/'/g, "''")}'`).join(", ")}]`; +} +function isNumericDuckDbType(type) { + return /INT|DOUBLE|FLOAT|DECIMAL|NUMERIC|REAL|HUGEINT/i.test(type); +} +function quoteColumn(name) { + return `"${name.replace(/"/g, '""')}"`; +} +/** + * Compare two typed CSV reads (with vs without treating "NA" as null) to decide + * per-column whether bare "NA" is missing data or a valid string code. + */ +function decideNaNullHandling(strictType, naNullType, naLiteralCount, naLooksLikeValidCode) { + const strictIsNumeric = isNumericDuckDbType(strictType); + const naNullIsNumeric = isNumericDuckDbType(naNullType); + if (strictIsNumeric) { + return { duckDbType: strictType, nullifyNa: true }; + } + if (naNullIsNumeric) { + // "NA" was blocking numeric inference (e.g. count column with R-style NA). + return { duckDbType: naNullType, nullifyNa: true }; + } + if (naLiteralCount > 0 && naLooksLikeValidCode) { + // Column stays text either way; preserve literal "NA" codes (e.g. species). + return { duckDbType: "VARCHAR", nullifyNa: false }; + } + return { duckDbType: "VARCHAR", nullifyNa: true }; +} +async function naLooksLikeValidCode(conn, quotedColumn, naLiteralCount) { + if (naLiteralCount === 0) { + return false; + } + const freqRows = await all(conn, `SELECT + MAX(c)::INTEGER as max_c, + MEDIAN(c)::DOUBLE as median_c + FROM ( + SELECT COUNT(*)::INTEGER as c + FROM _csv_probe_strict + WHERE CAST(${quotedColumn} AS VARCHAR) != 'NA' + AND ${quotedColumn} IS NOT NULL + GROUP BY CAST(${quotedColumn} AS VARCHAR) + ) freq`); + const maxC = freqRows[0]?.max_c ?? 0; + const medianC = freqRows[0]?.median_c ?? 0; + const peerFrequency = Math.max(medianC, 1); + // NA appears roughly as often as a typical category value, not a bulk missing marker. + return naLiteralCount <= peerFrequency * 2 && naLiteralCount <= maxC; +} +async function inferCsvColumnPlans(conn, csvPath, readCsvOptionsSuffix) { + const escapedPath = csvPath.replace(/'/g, "''"); + const baseNullstr = nullstrOption(exports.CSV_NULL_STRINGS_BASE); + const naNullstr = nullstrOption(exports.CSV_NULL_STRINGS_WITH_NA); + await run(conn, `CREATE OR REPLACE TEMP TABLE _csv_probe_strict AS SELECT * FROM read_csv('${escapedPath}', ${readCsvOptionsSuffix}, ${baseNullstr})`); + await run(conn, `CREATE OR REPLACE TEMP TABLE _csv_probe_na_null AS SELECT * FROM read_csv('${escapedPath}', ${readCsvOptionsSuffix}, ${naNullstr})`); + const strictSchema = await all(conn, `SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '_csv_probe_strict' ORDER BY ordinal_position`); + const naNullSchema = await all(conn, `SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '_csv_probe_na_null' ORDER BY ordinal_position`); + const naNullTypeByColumn = new Map(naNullSchema.map((col) => [col.column_name, col.data_type])); + const plans = []; + for (const col of strictSchema) { + const quoted = quoteColumn(col.column_name); + const naLiteralRows = await all(conn, `SELECT COUNT(*)::INTEGER as count FROM _csv_probe_strict WHERE CAST(${quoted} AS VARCHAR) = 'NA'`); + const naLiteralCount = naLiteralRows[0]?.count ?? 0; + const validNaCode = await naLooksLikeValidCode(conn, quoted, naLiteralCount); + const decision = decideNaNullHandling(col.data_type, naNullTypeByColumn.get(col.column_name) || "VARCHAR", naLiteralCount, validNaCode); + plans.push({ + name: col.column_name, + ...decision, + }); + if (!decision.nullifyNa && naLiteralCount > 0) { + console.log(`[data-tables-handler] preserving literal "NA" in column ${col.column_name} (${naLiteralCount} row(s); looks like a category code)`); + } + } + await run(conn, "DROP TABLE IF EXISTS _csv_probe_strict"); + await run(conn, "DROP TABLE IF EXISTS _csv_probe_na_null"); + return plans; +} +function buildTypedSelectSql(plans) { + return plans + .map((plan) => { + const quoted = quoteColumn(plan.name); + if (isNumericDuckDbType(plan.duckDbType)) { + const valueExpr = plan.nullifyNa + ? `NULLIF(${quoted}, 'NA')` + : quoted; + return `TRY_CAST(${valueExpr} AS ${plan.duckDbType}) AS ${quoted}`; + } + if (plan.nullifyNa) { + return `NULLIF(${quoted}, 'NA') AS ${quoted}`; + } + return `${quoted}`; + }) + .join(", "); +} diff --git a/packages/data-tables-handler/dist/src/lambda-db-client.d.ts b/packages/data-tables-handler/dist/src/lambda-db-client.d.ts new file mode 100644 index 000000000..14d41406c --- /dev/null +++ b/packages/data-tables-handler/dist/src/lambda-db-client.d.ts @@ -0,0 +1,2 @@ +import { Client } from "pg"; +export declare function getClient(): Promise; diff --git a/packages/data-tables-handler/dist/src/lambda-db-client.js b/packages/data-tables-handler/dist/src/lambda-db-client.js new file mode 100644 index 000000000..19ca599a4 --- /dev/null +++ b/packages/data-tables-handler/dist/src/lambda-db-client.js @@ -0,0 +1,82 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.getClient = getClient; +const aws_sdk_1 = require("aws-sdk"); +const pg_1 = require("pg"); +async function getClient() { + if (!dbClient) { + if (db.host && + (/localhost/.test(db.host) || + /127.0.0.1/.test(db.host) || + /host.docker.internal/.test(db.host))) { + dbClient = new Promise(async (resolve, reject) => { + const client = new pg_1.Client({ + database: db.database, + host: db.host, + port: parseInt(db.port.toString()), + user: db.user, + password: process.env.PGPASSWORD, + }); + try { + await client.connect(); + client.on("error", () => { + dbClient = null; + }); + resolve(client); + } + catch (e) { + reject(e); + } + }); + } + else { + dbClient = new Promise((resolve, reject) => { + signer.getAuthToken({ + region: db.region, + hostname: db.host, + port: parseInt(db.port.toString()), + username: db.user, + }, async function (err, token) { + if (err) { + reject(new Error(`could not get auth token: ${err}`)); + } + else { + const client = new pg_1.Client({ + database: db.database, + host: db.host, + port: parseInt(db.port.toString()), + user: db.user, + password: token, + ssl: { rejectUnauthorized: false }, + }); + try { + await client.connect(); + client.on("error", () => { + dbClient = null; + }); + resolve(client); + } + catch (e) { + reject(e); + } + } + }); + }); + } + } + return dbClient; +} +const signer = new aws_sdk_1.RDS.Signer(); +for (const param of ["PGREGION", "PGHOST", "PGPORT", "PGUSER", "PGDATABASE"]) { + if (!process.env[param]) { + throw new Error(`Environment variable "${param}" not set`); + } +} +const db = { + region: process.env.PGREGION, + host: process.env.PGHOST, + port: parseInt(process.env.PGPORT), + user: process.env.PGUSER, + database: process.env.PGDATABASE, +}; +let dbClient = null; diff --git a/packages/data-tables-handler/dist/src/normalizeCsvEncoding.d.ts b/packages/data-tables-handler/dist/src/normalizeCsvEncoding.d.ts new file mode 100644 index 000000000..28e4fbce1 --- /dev/null +++ b/packages/data-tables-handler/dist/src/normalizeCsvEncoding.d.ts @@ -0,0 +1,9 @@ +/** + * DuckDB's CSV reader requires valid UTF-8. Many legacy exports (Excel, R, etc.) + * are Latin-1 / Windows-1252. Reinterpret single-byte encodings as Latin-1 and + * write UTF-8 so every byte 0x00–0xFF round-trips without parse failures. + */ +export declare function normalizeCsvEncodingIfNeeded(csvPath: string, normalizedPath?: string): { + path: string; + normalized: boolean; +}; diff --git a/packages/data-tables-handler/dist/src/normalizeCsvEncoding.js b/packages/data-tables-handler/dist/src/normalizeCsvEncoding.js new file mode 100644 index 000000000..eaf84b9b7 --- /dev/null +++ b/packages/data-tables-handler/dist/src/normalizeCsvEncoding.js @@ -0,0 +1,52 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.normalizeCsvEncodingIfNeeded = normalizeCsvEncodingIfNeeded; +const fs_1 = require("fs"); +const path = __importStar(require("path")); +function isValidUtf8(buffer) { + try { + new TextDecoder("utf-8", { fatal: true }).decode(buffer); + return true; + } + catch { + return false; + } +} +/** + * DuckDB's CSV reader requires valid UTF-8. Many legacy exports (Excel, R, etc.) + * are Latin-1 / Windows-1252. Reinterpret single-byte encodings as Latin-1 and + * write UTF-8 so every byte 0x00–0xFF round-trips without parse failures. + */ +function normalizeCsvEncodingIfNeeded(csvPath, normalizedPath) { + const buffer = (0, fs_1.readFileSync)(csvPath); + if (isValidUtf8(buffer)) { + return { path: csvPath, normalized: false }; + } + const out = normalizedPath || + path.join(path.dirname(csvPath), `${path.basename(csvPath, path.extname(csvPath))}.utf8${path.extname(csvPath) || ".csv"}`); + (0, fs_1.writeFileSync)(out, buffer.toString("latin1"), "utf8"); + return { path: out, normalized: true }; +} diff --git a/packages/data-tables-handler/dist/src/processWithDuckDb.d.ts b/packages/data-tables-handler/dist/src/processWithDuckDb.d.ts new file mode 100644 index 000000000..1742e2d88 --- /dev/null +++ b/packages/data-tables-handler/dist/src/processWithDuckDb.d.ts @@ -0,0 +1,8 @@ +import { DataTablesColumnStats } from "@seasketch/geostats-types"; +import type { DataTableUploadProcessingOptions } from "./types"; +export declare function processCsvWithDuckDb(csvPath: string, parquetPath: string, options: DataTableUploadProcessingOptions): Promise<{ + rowCount: number; + headers: string[]; +}>; +export declare function readJoinValues(parquetPath: string, joinColumn: string): Promise>; +export declare function computeColumnStatsFromParquet(parquetPath: string, tableName: string, joinInfo: DataTablesColumnStats["join"]): Promise; diff --git a/packages/data-tables-handler/dist/src/processWithDuckDb.js b/packages/data-tables-handler/dist/src/processWithDuckDb.js new file mode 100644 index 000000000..b043e3929 --- /dev/null +++ b/packages/data-tables-handler/dist/src/processWithDuckDb.js @@ -0,0 +1,158 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.processCsvWithDuckDb = processCsvWithDuckDb; +exports.readJoinValues = readJoinValues; +exports.computeColumnStatsFromParquet = computeColumnStatsFromParquet; +const duckdb_1 = __importDefault(require("duckdb")); +const path = __importStar(require("path")); +const validateJoinColumn_1 = require("./validateJoinColumn"); +const normalizeCsvEncoding_1 = require("./normalizeCsvEncoding"); +const inferCsvColumnPlans_1 = require("./inferCsvColumnPlans"); +function run(conn, sql) { + return new Promise((resolve, reject) => { + conn.run(sql, (err) => (err ? reject(err) : resolve())); + }); +} +function all(conn, sql) { + return new Promise((resolve, reject) => { + conn.all(sql, (err, rows) => (err ? reject(err) : resolve(rows))); + }); +} +function escapePath(path) { + return path.replace(/'/g, "''"); +} +/** Runs fn against a fresh in-memory DuckDB, always closing it afterwards. */ +async function withDuckDb(fn) { + const db = new duckdb_1.default.Database(":memory:"); + const conn = db.connect(); + try { + return await fn(conn); + } + finally { + conn.close(); + db.close(); + } +} +function delimiterOption(delimiter) { + switch (delimiter) { + case "\t": + return "delim='\\t'"; + case ";": + return "delim=';'"; + case "|": + return "delim='|'"; + default: + return ""; + } +} +async function processCsvWithDuckDb(csvPath, parquetPath, options) { + const { path: duckDbCsvPath, normalized } = (0, normalizeCsvEncoding_1.normalizeCsvEncodingIfNeeded)(csvPath, path.join(path.dirname(csvPath), "input.utf8.csv")); + if (normalized) { + console.log(`[data-tables-handler] normalized csv encoding to utf-8: ${duckDbCsvPath}`); + } + const delimiter = options.delimiter || ","; + const hasHeader = options.hasHeaderRow !== false; + const forceNotNull = options.forceNotNull || []; + const forceNotNullSql = forceNotNull.length > 0 + ? `, force_not_null=[${forceNotNull.map((c) => `'${c.replace(/'/g, "''")}'`).join(", ")}]` + : ""; + const readCsvOptionsSuffix = [ + "header=" + (hasHeader ? "true" : "false"), + "sample_size=-1", + delimiterOption(delimiter), + ] + .filter(Boolean) + .join(", "); + return withDuckDb(async (conn) => { + const columnPlans = await (0, inferCsvColumnPlans_1.inferCsvColumnPlans)(conn, duckDbCsvPath, readCsvOptionsSuffix); + const baseNullstr = (0, inferCsvColumnPlans_1.nullstrOption)(inferCsvColumnPlans_1.CSV_NULL_STRINGS_BASE); + await run(conn, `CREATE OR REPLACE TEMP TABLE _csv_raw AS SELECT * FROM read_csv('${escapePath(duckDbCsvPath)}', ${readCsvOptionsSuffix}, ${baseNullstr}, all_varchar=true${forceNotNullSql})`); + await run(conn, `CREATE OR REPLACE TABLE observations AS SELECT ${(0, inferCsvColumnPlans_1.buildTypedSelectSql)(columnPlans)} FROM _csv_raw`); + await run(conn, "DROP TABLE IF EXISTS _csv_raw"); + const countRows = await all(conn, "SELECT COUNT(*)::INTEGER as count FROM observations"); + const rowCount = countRows[0]?.count ?? 0; + const columns = await all(conn, `SELECT column_name FROM information_schema.columns WHERE table_name = 'observations' ORDER BY ordinal_position`); + const headers = columns.map((c) => c.column_name); + await run(conn, `COPY observations TO '${escapePath(parquetPath)}' (FORMAT PARQUET)`); + return { rowCount, headers }; + }); +} +async function readJoinValues(parquetPath, joinColumn) { + const col = joinColumn.replace(/"/g, '""'); + return withDuckDb(async (conn) => { + const rows = await all(conn, `SELECT DISTINCT CAST("${col}" AS VARCHAR) as v FROM read_parquet('${escapePath(parquetPath)}') WHERE "${col}" IS NOT NULL`); + return new Set(rows.map((r) => r.v)); + }); +} +/** Cap on histogram entries stored per column in column-stats.json. */ +const VALUES_HISTOGRAM_LIMIT = 500; +async function computeColumnStatsFromParquet(parquetPath, tableName, joinInfo) { + return withDuckDb(async (conn) => { + await run(conn, `CREATE OR REPLACE TABLE observations AS SELECT * FROM read_parquet('${escapePath(parquetPath)}')`); + const countRows = await all(conn, "SELECT COUNT(*)::INTEGER as count FROM observations"); + const rowCount = countRows[0]?.count ?? 0; + const schema = await all(conn, `SELECT column_name, data_type FROM information_schema.columns WHERE table_name = 'observations' ORDER BY ordinal_position`); + const columns = []; + for (const col of schema) { + const colName = col.column_name.replace(/"/g, '""'); + const type = (0, validateJoinColumn_1.inferGeostatsType)(col.data_type); + const counts = await all(conn, `SELECT COUNT("${colName}")::INTEGER as count, COUNT(DISTINCT "${colName}")::INTEGER as count_distinct FROM observations`); + const count = counts[0]?.count ?? 0; + const countDistinct = counts[0]?.count_distinct ?? 0; + const distinctRows = await all(conn, `SELECT CAST("${colName}" AS VARCHAR) as v, COUNT(*)::INTEGER as c FROM observations WHERE "${colName}" IS NOT NULL GROUP BY 1 ORDER BY c DESC LIMIT ${VALUES_HISTOGRAM_LIMIT}`); + const values = {}; + for (const row of distinctRows) { + values[row.v] = row.c; + } + const attr = { + attribute: col.column_name, + count, + countDistinct, + type, + values, + }; + if (type === "number") { + const stats = await all(conn, `SELECT MIN(CAST("${colName}" AS DOUBLE)) as min, MAX(CAST("${colName}" AS DOUBLE)) as max FROM observations WHERE "${colName}" IS NOT NULL`); + if (stats[0]) { + attr.min = + stats[0].min; + attr.max = + stats[0].max; + } + } + columns.push(attr); + } + return { + table: tableName, + rowCount, + columns, + join: joinInfo, + }; + }); +} diff --git a/packages/data-tables-handler/dist/src/remotes.d.ts b/packages/data-tables-handler/dist/src/remotes.d.ts new file mode 100644 index 000000000..d15527388 --- /dev/null +++ b/packages/data-tables-handler/dist/src/remotes.d.ts @@ -0,0 +1,14 @@ +export declare function putObject(filepath: string, remote: string, contentType?: string): Promise; +/** Download the user's raw upload from the S3 staging bucket. */ +export declare function getStagingObject(filepath: string, objectKey: string): Promise; +/** + * Store data tables under the parent layer's hosted UUID so objects classify + * as `published` in the tiles ACL gateway and inherit the layer's ACL. + * + * `projects/{slug}/public/{sourceUuid}/dataTables/{uploadId}/{filename}` + */ +export declare function buildR2Remote(slug: string, sourceUuid: string, uploadId: string, filename: string): { + remote: string; + key: string; + bucket: string; +}; diff --git a/packages/data-tables-handler/dist/src/remotes.js b/packages/data-tables-handler/dist/src/remotes.js new file mode 100644 index 000000000..b750f2901 --- /dev/null +++ b/packages/data-tables-handler/dist/src/remotes.js @@ -0,0 +1,73 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.putObject = putObject; +exports.getStagingObject = getStagingObject; +exports.buildR2Remote = buildR2Remote; +const fs_1 = require("fs"); +const promises_1 = require("stream/promises"); +const client_s3_1 = require("@aws-sdk/client-s3"); +const lib_storage_1 = require("@aws-sdk/lib-storage"); +const bytes_1 = __importDefault(require("bytes")); +const s3Client = new client_s3_1.S3Client({ region: process.env.AWS_REGION }); +const r2Client = new client_s3_1.S3Client({ + region: "auto", + endpoint: process.env.R2_ENDPOINT, + credentials: { + accessKeyId: process.env.R2_ACCESS_KEY_ID, + secretAccessKey: process.env.R2_SECRET_ACCESS_KEY, + }, +}); +async function putObject(filepath, remote, contentType) { + if (!/r2:/.test(remote) && !/s3:/.test(remote)) { + throw new Error(`Invalid remote ${remote}`); + } + const parts = remote.replace(/\w+:\/\//, "").split("/"); + const client = /r2:/.test(remote) ? r2Client : s3Client; + const Bucket = parts[0]; + const Key = parts.slice(1).join("/"); + const fileSizeBytes = (0, fs_1.statSync)(filepath).size; + const fileStream = (0, fs_1.createReadStream)(filepath); + if (fileSizeBytes > 500 * 1024 * 1024) { + const upload = new lib_storage_1.Upload({ + client, + params: { Bucket, Key, Body: fileStream, ContentType: contentType }, + }); + await upload.done(); + } + else { + await client.send(new client_s3_1.PutObjectCommand({ + Bucket, + Key, + Body: fileStream, + ContentType: contentType, + })); + } + console.log(`putObject ${filepath} (${(0, bytes_1.default)(fileSizeBytes)}) to ${remote}`); +} +/** Download the user's raw upload from the S3 staging bucket. */ +async function getStagingObject(filepath, objectKey) { + const bucket = process.env.BUCKET; + const response = await s3Client.send(new client_s3_1.GetObjectCommand({ Bucket: bucket, Key: objectKey })); + const body = response.Body; + await (0, promises_1.pipeline)(body, (0, fs_1.createWriteStream)(filepath)); +} +/** + * Store data tables under the parent layer's hosted UUID so objects classify + * as `published` in the tiles ACL gateway and inherit the layer's ACL. + * + * `projects/{slug}/public/{sourceUuid}/dataTables/{uploadId}/{filename}` + */ +function buildR2Remote(slug, sourceUuid, uploadId, filename) { + const bucket = process.env.R2_TILES_BUCKET || process.env.TILES_REMOTE?.replace("r2://", "").split("/")[0]; + if (!bucket) { + throw new Error("R2_TILES_BUCKET or TILES_REMOTE must be set"); + } + if (!sourceUuid) { + throw new Error("sourceUuid is required to store data tables under the parent layer path"); + } + const key = `projects/${slug}/public/${sourceUuid}/dataTables/${uploadId}/${filename}`; + return { remote: `r2://${bucket}/${key}`, key, bucket }; +} diff --git a/packages/data-tables-handler/dist/src/types.d.ts b/packages/data-tables-handler/dist/src/types.d.ts new file mode 100644 index 000000000..1ef9b5017 --- /dev/null +++ b/packages/data-tables-handler/dist/src/types.d.ts @@ -0,0 +1,32 @@ +export interface DataTableUploadProcessingOptions { + delimiter?: "," | "\t" | ";" | "|"; + hasHeaderRow?: boolean; + joinColumn?: string; + overlayJoinColumn?: string; + forceNotNull?: string[]; + name?: string; +} +export interface DataTablesHandlerRequest { + taskId: string; + uploadId: string; + objectKey: string; + /** Project slug; first path segment of the R2 key. */ + slug: string; + /** Parent layer hosted-tiles UUID (`data_sources.url`); used for R2 key prefix. */ + sourceUuid: string; + skipLoggingProgress?: boolean; +} +export interface DataTablesHandlerSuccess { + uploadId: string; + name: string; + joinColumn: string; + overlayJoinColumn: string; + rowCount: number; + parquetRemote: string; + columnStatsRemote: string; +} +export interface DataTablesHandlerResponse { + success?: DataTablesHandlerSuccess; + error?: string; + errorDetails?: Record; +} diff --git a/packages/data-tables-handler/dist/src/types.js b/packages/data-tables-handler/dist/src/types.js new file mode 100644 index 000000000..c8ad2e549 --- /dev/null +++ b/packages/data-tables-handler/dist/src/types.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/packages/data-tables-handler/dist/src/validateJoinColumn.d.ts b/packages/data-tables-handler/dist/src/validateJoinColumn.d.ts new file mode 100644 index 000000000..456a63565 --- /dev/null +++ b/packages/data-tables-handler/dist/src/validateJoinColumn.d.ts @@ -0,0 +1,26 @@ +import { GeostatsAttribute, GeostatsAttributeType, GeostatsLayer } from "@seasketch/geostats-types"; +export declare function getGeostatsLayer(overlayGeostats: unknown): GeostatsLayer; +export declare function findOverlayAttribute(layer: GeostatsLayer, attributeName: string): GeostatsAttribute; +/** + * Validates that distinct values in the table's join column exist among the + * overlay layer's feature identifiers (from geostats). Comparison is an + * exact string match — join keys must match the overlay attribute exactly, + * as they would in any other analysis tooling. + * + * Geostats attribute `values` histograms are truncated (top ~500 keys), so + * when the overlay attribute has more distinct values than the histogram + * holds, unmatched CSV values are reported in the stats but do not fail the + * upload. With a complete histogram, any unmatched value is an error. + * + * Note: despite the names (kept for compatibility with the stored + * column-stats.json format), matchedRows/unmatchedRows count *distinct join + * values*, not table rows. + */ +export declare function validateJoinColumnChoice(headers: string[], joinColumn: string, overlayJoinColumn: string, layer: GeostatsLayer, joinValues: Set): { + overlayAttr: GeostatsAttribute; + matchRate: number; + matchedRows: number; + unmatchedRows: number; + unmatchedOverlayValues: number; +}; +export declare function inferGeostatsType(duckDbType: string): GeostatsAttributeType; diff --git a/packages/data-tables-handler/dist/src/validateJoinColumn.js b/packages/data-tables-handler/dist/src/validateJoinColumn.js new file mode 100644 index 000000000..de0a8036c --- /dev/null +++ b/packages/data-tables-handler/dist/src/validateJoinColumn.js @@ -0,0 +1,82 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.getGeostatsLayer = getGeostatsLayer; +exports.findOverlayAttribute = findOverlayAttribute; +exports.validateJoinColumnChoice = validateJoinColumnChoice; +exports.inferGeostatsType = inferGeostatsType; +function getGeostatsLayer(overlayGeostats) { + const data = overlayGeostats; + const layer = data?.layers?.[0]; + if (!layer?.attributes) { + throw new Error("Overlay geostats missing layer attributes"); + } + return layer; +} +function findOverlayAttribute(layer, attributeName) { + const attr = layer.attributes.find((a) => a.attribute === attributeName); + if (!attr) { + throw new Error(`Overlay attribute "${attributeName}" not found in geostats`); + } + return attr; +} +/** + * Validates that distinct values in the table's join column exist among the + * overlay layer's feature identifiers (from geostats). Comparison is an + * exact string match — join keys must match the overlay attribute exactly, + * as they would in any other analysis tooling. + * + * Geostats attribute `values` histograms are truncated (top ~500 keys), so + * when the overlay attribute has more distinct values than the histogram + * holds, unmatched CSV values are reported in the stats but do not fail the + * upload. With a complete histogram, any unmatched value is an error. + * + * Note: despite the names (kept for compatibility with the stored + * column-stats.json format), matchedRows/unmatchedRows count *distinct join + * values*, not table rows. + */ +function validateJoinColumnChoice(headers, joinColumn, overlayJoinColumn, layer, joinValues) { + if (!headers.includes(joinColumn)) { + throw new Error(`Join column "${joinColumn}" not found in CSV headers`); + } + const overlayAttr = findOverlayAttribute(layer, overlayJoinColumn); + const overlayKeys = new Set(Object.keys(overlayAttr.values || {})); + const histogramComplete = typeof overlayAttr.countDistinct !== "number" || + overlayAttr.countDistinct <= overlayKeys.size; + let matchedRows = 0; + for (const v of joinValues) { + if (overlayKeys.has(v)) { + matchedRows++; + } + } + const unmatchedRows = joinValues.size - matchedRows; + if (matchedRows === 0) { + throw new Error("No values in the join column match overlay feature identifiers"); + } + if (unmatchedRows > 0 && histogramComplete) { + throw new Error(`${unmatchedRows} value(s) in the join column are not present in the overlay layer`); + } + let unmatchedOverlayValues = 0; + for (const k of overlayKeys) { + if (!joinValues.has(k)) { + unmatchedOverlayValues++; + } + } + const matchRate = joinValues.size > 0 ? matchedRows / joinValues.size : 0; + return { + overlayAttr, + matchRate, + matchedRows, + unmatchedRows, + unmatchedOverlayValues, + }; +} +function inferGeostatsType(duckDbType) { + const t = duckDbType.toUpperCase(); + if (/INT|DOUBLE|FLOAT|DECIMAL|NUMERIC|REAL|HUGEINT/.test(t)) { + return "number"; + } + if (/BOOL/.test(t)) { + return "boolean"; + } + return "string"; +} diff --git a/packages/data-tables-handler/handler.ts b/packages/data-tables-handler/handler.ts new file mode 100644 index 000000000..1d1d7542b --- /dev/null +++ b/packages/data-tables-handler/handler.ts @@ -0,0 +1,14 @@ +import handleDataTableUpload from "./src/handleDataTableUpload"; +import type { DataTablesHandlerRequest } from "./src/types"; + +export type { DataTablesHandlerRequest, DataTablesHandlerResponse } from "./src/types"; + +export const processDataTableUpload = async ( + event: DataTablesHandlerRequest, +) => { + try { + return await handleDataTableUpload(event); + } catch (e) { + return { error: (e as Error).message }; + } +}; diff --git a/packages/data-tables-handler/package-lock.json b/packages/data-tables-handler/package-lock.json new file mode 100644 index 000000000..aa7070bdb --- /dev/null +++ b/packages/data-tables-handler/package-lock.json @@ -0,0 +1,2607 @@ +{ + "name": "data-tables-handler", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "data-tables-handler", + "version": "1.0.0", + "license": "BSD-3-Clause", + "dependencies": { + "@aws-sdk/client-s3": "^3.758.0", + "@aws-sdk/lib-storage": "^3.758.0", + "@seasketch/geostats-types": "^1.0.0", + "aws-sdk": "^2.1234.0", + "bytes": "^3.1.2", + "duckdb": "^1.4.4", + "pg": "^8.8.0", + "tmp": "^0.2.1", + "typescript": "5.5.3" + }, + "devDependencies": { + "@types/bytes": "^3.1.1", + "@types/node": "^18.11.4", + "@types/pg": "^8.6.5", + "@types/tmp": "^0.2.3", + "dotenv": "^16.0.3" + } + }, + "node_modules/@aws-sdk/checksums": { + "version": "3.1000.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.10.tgz", + "integrity": "sha512-OUNjNcyA8Ai2OdlRUxW5jHUf6XJmqqZk3UddL+mDiUCtXrVqdmIHHkdDFWBlBRjhore/3ZBMgRXgcS0ggtvD0Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.25", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.1077.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1077.0.tgz", + "integrity": "sha512-yWK6jOMrMUgGarXlrQaAopWXva20NaOqTPApuE68SzsBFQ57fQ5E13BKUPLwtPgymk+8L6vG/O4h7Q30IBYr2w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1000.10", + "@aws-sdk/core": "^3.974.25", + "@aws-sdk/credential-provider-node": "^3.972.60", + "@aws-sdk/middleware-sdk-s3": "^3.972.56", + "@aws-sdk/signature-v4-multi-region": "^3.996.37", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/fetch-http-handler": "^5.6.1", + "@smithy/node-http-handler": "^4.9.1", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.974.25", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.974.25.tgz", + "integrity": "sha512-fJFkx6u6wCqGMV/v6EAxiwa2UzEukbvr1hNPv4MrD3yj4IFz011jZg42/eSTOP/u5kJ0tlILqEjCWtT8GiKZvA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.14", + "@aws-sdk/xml-builder": "^3.972.32", + "@aws/lambda-invoke-store": "^0.2.2", + "@smithy/core": "^3.28.0", + "@smithy/signature-v4": "^5.6.0", + "@smithy/types": "^4.15.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.51", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.51.tgz", + "integrity": "sha512-Xo+/zf5k5pZdo53X8aVXN4MJGfU/M1P7yMM/GbNY/x9fyRZGEzjhKqW38GA0FSQQ9TYKs+bfPyz5ja4bi6pjTQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.25", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.53", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.53.tgz", + "integrity": "sha512-7E9oFUcf9YWe+ttGiWhe/cCSI+pswwelzgQMoKXgPJi1AIfS27TK6et5ZULqEqHu30zbN+jh1RqlwcXqY/aXyg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.25", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/fetch-http-handler": "^5.6.1", + "@smithy/node-http-handler": "^4.9.1", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.972.58", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.58.tgz", + "integrity": "sha512-MPr0hD8pyDGfF3dWXvFOILhcKTB9ptqJOJK9JEuDQzpc2HgKisY16eR7IrKUXxSbz8LZj+LHz/CS8Y5G1ai7yw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.25", + "@aws-sdk/credential-provider-env": "^3.972.51", + "@aws-sdk/credential-provider-http": "^3.972.53", + "@aws-sdk/credential-provider-login": "^3.972.57", + "@aws-sdk/credential-provider-process": "^3.972.51", + "@aws-sdk/credential-provider-sso": "^3.972.57", + "@aws-sdk/credential-provider-web-identity": "^3.972.57", + "@aws-sdk/nested-clients": "^3.997.25", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/credential-provider-imds": "^4.4.4", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.57", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.57.tgz", + "integrity": "sha512-kPWc/SCrl9agKeywxKwPEoQHanWag0LcNQrcZpEQpjNifkxq6tQENhgrrS9al317CF6yytyihlX+FhPHlk0QjA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.25", + "@aws-sdk/nested-clients": "^3.997.25", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.60", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.60.tgz", + "integrity": "sha512-hE2hIBJQjCDRx8TbSqpVQ+/o2mIrJZQZbQ3LlwE2bJf7z47x5GmhcvGwZPqJH7Oq//SzTXEBGSZ4qSpK3yPbhw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.51", + "@aws-sdk/credential-provider-http": "^3.972.53", + "@aws-sdk/credential-provider-ini": "^3.972.58", + "@aws-sdk/credential-provider-process": "^3.972.51", + "@aws-sdk/credential-provider-sso": "^3.972.57", + "@aws-sdk/credential-provider-web-identity": "^3.972.57", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/credential-provider-imds": "^4.4.4", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.51", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.51.tgz", + "integrity": "sha512-081dD2RlnmY+G05v6E73KfACvDjPjnttrLjGHE2SSglbID25UcuijbWpL4g+XR5T2Kl4oIJoVBXi64s+2f009Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.25", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.972.57", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.57.tgz", + "integrity": "sha512-dC7ZyX3EHKHLOeVUEDzzGvk0L1s6N06YDrau7P0rGXL/j1cO+DzN2w1x9vcEh7zljVCR3019f5mi1Th+GGTURw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.25", + "@aws-sdk/nested-clients": "^3.997.25", + "@aws-sdk/token-providers": "3.1077.0", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.57", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.57.tgz", + "integrity": "sha512-HtWM3FV2o7NJFJSUqFLBlxmV9RxQRHpzCvQaP1n1Qo4CxQSvwpJ8ERWHiLqXMFDgDXyELt+EZNFcpG6XQRcJbQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.25", + "@aws-sdk/nested-clients": "^3.997.25", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/lib-storage": { + "version": "3.1077.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.1077.0.tgz", + "integrity": "sha512-WwheNNdvCqdw5Z5Uvl/XB5e3pN5Q/b4N3yDYloUsrN5EiMw21UHHDd6Bkt3cC2ALUU/7+l+TXhtx5d9DAGkiDQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.28.0", + "@smithy/types": "^4.15.0", + "buffer": "5.6.0", + "events": "3.3.0", + "stream-browserify": "3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.1077.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.56", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.56.tgz", + "integrity": "sha512-VFLd7bT8ef48H2n2iDrY/w9EPpXLmC0v4NEriXy2vuTgEP/FrKqrcwi7eobhcOrdO37uLbMt5AWZlY55R2/xqg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.25", + "@aws-sdk/signature-v4-multi-region": "^3.996.37", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.25", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.25.tgz", + "integrity": "sha512-VpRQ3wR6l+fwRHV5veJL2ehtyQFrGyH/2CJG9DVtb8H3xyqqnZWSTSrq/CJJ7DvDlDgrPRiW2SkYA8pN6VWCFQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.25", + "@aws-sdk/signature-v4-multi-region": "^3.996.37", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/fetch-http-handler": "^5.6.1", + "@smithy/node-http-handler": "^4.9.1", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.37", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.37.tgz", + "integrity": "sha512-u8qd064XsHzM0Mk+yH4IPKn/ZC9rdniEKs+neBHNlsPZirw3rcLvmrH4ImoKC4yF7A0I/MbcC3dseARnJLiAhg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.14", + "@smithy/signature-v4": "^5.6.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1077.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1077.0.tgz", + "integrity": "sha512-sRUkfZ3fpOco95jZHsQUQiXvuIVLvCmWVclFg6dRFDyfsYs6Pdr/NuZ2+yJxeHN+6WAfDh2aZ8nlZntnvuhZUQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.25", + "@aws-sdk/nested-clients": "^3.997.25", + "@aws-sdk/types": "^3.973.14", + "@smithy/core": "^3.28.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.973.14", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.14.tgz", + "integrity": "sha512-vH4pEu9YBEwr67yT+GVcmKX0GzfIrIYUn+MF5vXg9OspouVnAekuyVyawFvZHEK7WlcwVDwNrqI3ZBDUAiyu9A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.32", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.32.tgz", + "integrity": "sha512-2loKuOMRFDg1nwdni5AtJ9S5juVbRNPNsPC7tWTfkHyycPwACMhxepspUHi8GhvfNlL2cQo3sPMod1uib+KZ0w==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.4.tgz", + "integrity": "sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@gar/promisify": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", + "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", + "license": "MIT" + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@isaacs/fs-minipass/node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/@mapbox/node-pre-gyp": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-2.0.3.tgz", + "integrity": "sha512-uwPAhccfFJlsfCxMYTwOdVfOz3xqyj8xYL3zJj8f0pb30tLohnnFPhLuqp4/qoEz8sNxe4SESZedcBojRefIzg==", + "license": "BSD-3-Clause", + "dependencies": { + "consola": "^3.2.3", + "detect-libc": "^2.0.0", + "https-proxy-agent": "^7.0.5", + "node-fetch": "^2.6.7", + "nopt": "^8.0.0", + "semver": "^7.5.3", + "tar": "^7.4.0" + }, + "bin": { + "node-pre-gyp": "bin/node-pre-gyp" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@npmcli/fs": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-2.1.2.tgz", + "integrity": "sha512-yOJKRvohFOaLqipNtwYB9WugyZKhC/DZC4VYPmpaCzDBrA8YpK3qHZ8/HGscMnE4GqbkLNuVcCnxkeQEdGt6LQ==", + "license": "ISC", + "dependencies": { + "@gar/promisify": "^1.1.3", + "semver": "^7.3.5" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/@npmcli/move-file": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-2.0.1.tgz", + "integrity": "sha512-mJd2Z5TjYWq/ttPLLGqArdtnC74J6bOzg4rMDnN+p1xTacZ2yPRCk2y0oSWQtygLR9YVQXgOcONrwtnk3JupxQ==", + "deprecated": "This functionality has been moved to @npmcli/fs", + "license": "MIT", + "dependencies": { + "mkdirp": "^1.0.4", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/@seasketch/geostats-types": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@seasketch/geostats-types/-/geostats-types-1.0.0.tgz", + "integrity": "sha512-vuUoQuPh4SdCjhxAXHU9UH8n1Nfq7hfv4hPO7OSjxqz+e77GW9QbqQduKON2sEpR0J0DrA1naMBi9hioUoud7w==", + "license": "BSD-3-Clause", + "dependencies": { + "@types/geojson": "^7946.0.14" + } + }, + "node_modules/@smithy/core": { + "version": "3.29.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.0.tgz", + "integrity": "sha512-sEvpvkBVoMxjoek35XyJFn2ZD3EJ1RpiZrT47WaZodxzAIWS44zkdvbqGE/ZlugtjiQp62cffYZ9ldyRkjAGnA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.15.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.4.5", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.5.tgz", + "integrity": "sha512-LnjUTNG0GgQlKIq7IioeOrPaEmC5xOd1WtAz24TLSiYQnWX2uHr53GrFuQhkrJBktPYCMga/NbUOW7hFbSA2Cg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.2.tgz", + "integrity": "sha512-q96PSDOAGw+X+nuELd7Cjebps0SYr+YlPbviEX9sLVw+VM4M7VV8hn1nL1mGS6urDu33eQ5A7WhlphaDO6kUyQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.2.tgz", + "integrity": "sha512-s0yAIRj6TVfHgl+QzVyqal1KMGZ9B5512IrxKc6+dOpw8fUmFL3CvuAhjv0J+aNjUPfVZ2IhqPEDvkB5Ncx9oA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.1.tgz", + "integrity": "sha512-SqvuP75p/DmgWWI7jv4kf/UW+V4LFmlUn19s604SgAcRuJRB1vDnWwzZMYCLUcmKxko9wDn6iLgGEIpTNgZbIQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.15.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.15.1.tgz", + "integrity": "sha512-x3L0XSACF6UYzKpa9biqiRMgvH5+wnFFew9Tm/grFYqgaupPwx/+ojDPpPJM8dZON3S9tjz5U+PQYsCBd1Mw5Q==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@tootallnate/once": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.1.tgz", + "integrity": "sha512-HqmEUIGRJ5fSXchkVgR5F7qn48bDBzv0kWj/Kfu5e6uci4UlEeng4331LnBkWffb++Ei3FOVLxo8JJWMFBDMeQ==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/@types/bytes": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@types/bytes/-/bytes-3.1.5.tgz", + "integrity": "sha512-VgZkrJckypj85YxEsEavcMmmSOIzkUHqWmM4CCyia5dc54YwsXzJ5uT4fYxBQNEXx+oF1krlhgCbvfubXqZYsQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/pg": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, + "node_modules/@types/tmp": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/@types/tmp/-/tmp-0.2.6.tgz", + "integrity": "sha512-chhaNf2oKHlRkDGt+tiKE2Z5aJ6qalm7Z9rlLdBwmOiAAf09YQvvoLXjWK4HWPF1xU/fqvMgfNfpVoBscA/tKA==", + "dev": true, + "license": "MIT" + }, + "node_modules/abbrev": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-3.0.1.tgz", + "integrity": "sha512-AO2ac6pjRB3SJmGJo+v5/aK6Omggp6fsLrs6wN9bd35ulu4cCwaAU9+7ZhXjeqHVkaHThLuzH0nZr0YpCDhygg==", + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "license": "MIT", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "license": "MIT", + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aproba": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.1.0.tgz", + "integrity": "sha512-tLIEcj5GuR2RSTnxNKdkK0dJ/GrC7P38sUkiDmDuHfsHmbagTFAxDVIBltoklXEVIQ/f14IL8IMJ5pn9Hez1Ew==", + "license": "ISC" + }, + "node_modules/are-we-there-yet": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz", + "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/available-typed-arrays": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", + "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", + "license": "MIT", + "dependencies": { + "possible-typed-array-names": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/aws-sdk": { + "version": "2.1693.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1693.0.tgz", + "integrity": "sha512-cJmb8xEnVLT+R6fBS5sn/EFJiX7tUnDaPtOPZ1vFbOJtd0fnZn/Ky2XGgsvvoeliWeH7mL3TWSX5zXXGSQV6gQ==", + "deprecated": "The AWS SDK for JavaScript (v2) has reached end-of-support, and no longer receives updates. Please migrate your code to use AWS SDK for JavaScript (v3). More info https://a.co/cUPnyil", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.16.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "util": "^0.12.4", + "uuid": "8.0.0", + "xml2js": "0.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/aws-sdk/node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "node_modules/aws-sdk/node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==", + "license": "MIT", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.15.tgz", + "integrity": "sha512-EwOCDEex4quD37XhqM3omwtMoJjr//isUZz1JopUNWms+4Z2ViyM/k1YIRePpoVNnQhENnxtFjLaxNHrT7xIUg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/cacache": { + "version": "16.1.3", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-16.1.3.tgz", + "integrity": "sha512-/+Emcj9DAXxX4cwlLmRI9c166RuL3w30zp4R7Joiv2cQTtTtA+jeuCAjH3ZlGnYS3tKENSrKhAzVVP9GVyzeYQ==", + "license": "ISC", + "dependencies": { + "@npmcli/fs": "^2.1.0", + "@npmcli/move-file": "^2.0.0", + "chownr": "^2.0.0", + "fs-minipass": "^2.1.0", + "glob": "^8.0.1", + "infer-owner": "^1.0.4", + "lru-cache": "^7.7.1", + "minipass": "^3.1.6", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "mkdirp": "^1.0.4", + "p-map": "^4.0.0", + "promise-inflight": "^1.0.1", + "rimraf": "^3.0.2", + "ssri": "^9.0.0", + "tar": "^6.1.11", + "unique-filename": "^2.0.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/cacache/node_modules/brace-expansion": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.1.tgz", + "integrity": "sha512-WR1cURNjuvBLMZBMbqM0UoE+WAfdUcEV1ccD8PVBVOI+Z3ND4+SZbN8RsfT2bMuG1qwz5RFvPukSZm5fF2D5eA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/cacache/node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/cacache/node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/cacache/node_modules/tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "deprecated": "Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/cacache/node_modules/tar/node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "license": "ISC", + "engines": { + "node": ">=8" + } + }, + "node_modules/call-bind": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.9.tgz", + "integrity": "sha512-a/hy+pNsFUTR+Iz8TCJvXudKVLAnz/DyeSUo10I5yvFDQJBFU2s9uqQpoSrJlroHUKoKqzg+epxyP9lqFdzfBQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "get-intrinsic": "^1.3.0", + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "license": "ISC", + "bin": { + "color-support": "bin.js" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT" + }, + "node_modules/consola": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz", + "integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==", + "license": "MIT", + "engines": { + "node": "^14.18.0 || >=16.10.0" + } + }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", + "license": "ISC" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/dotenv": { + "version": "16.6.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", + "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/duckdb": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/duckdb/-/duckdb-1.4.4.tgz", + "integrity": "sha512-hEJJ5hyVF0VLj3dmOLpsWrQR0b3d4Ykqtygm6iUXjHJDDnSqg/USKxcb5qvNrOhFYPgktayU6dXCtgPqcmmpog==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "@mapbox/node-pre-gyp": "^2.0.0", + "node-addon-api": "^7.0.0", + "node-gyp": "^9.4.1" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "license": "MIT", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/err-code": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", + "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==", + "license": "MIT" + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/exponential-backoff": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.3.tgz", + "integrity": "sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==", + "license": "Apache-2.0" + }, + "node_modules/for-each": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", + "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", + "license": "MIT", + "dependencies": { + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gauge": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz", + "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.3", + "console-control-strings": "^1.1.0", + "has-unicode": "^2.0.1", + "signal-exit": "^3.0.7", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.5" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/generator-function": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/generator-function/-/generator-function-2.0.1.tgz", + "integrity": "sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", + "license": "ISC" + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", + "license": "BSD-2-Clause" + }, + "node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "license": "MIT", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/http-proxy-agent/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==", + "license": "BSD-3-Clause" + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/infer-owner": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz", + "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==", + "license": "ISC" + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", + "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-arguments": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", + "integrity": "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-function": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.2.tgz", + "integrity": "sha512-upqt1SkGkODW9tsGNG5mtXTXtECizwtS2kA161M+gJPc1xdb/Ax629af6YrTwcOeQHbewrPNlE5Dx7kzvXTizA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.4", + "generator-function": "^2.0.0", + "get-proto": "^1.0.1", + "has-tostringtag": "^1.0.2", + "safe-regex-test": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-lambda": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz", + "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==", + "license": "MIT" + }, + "node_modules/is-regex": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", + "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-typed-array": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", + "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", + "license": "MIT", + "dependencies": { + "which-typed-array": "^1.1.16" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jmespath": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", + "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", + "license": "Apache-2.0", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/make-fetch-happen": { + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-10.2.1.tgz", + "integrity": "sha512-NgOPbRiaQM10DYXvN3/hhGVI2M5MtITFryzBGxHM5p4wnFxsVCbxkrBrDsk+EZ5OB4jEOT7AjDxtdF+KVEFT7w==", + "license": "ISC", + "dependencies": { + "agentkeepalive": "^4.2.1", + "cacache": "^16.1.0", + "http-cache-semantics": "^4.1.0", + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "is-lambda": "^1.0.1", + "lru-cache": "^7.7.1", + "minipass": "^3.1.6", + "minipass-collect": "^1.0.2", + "minipass-fetch": "^2.0.3", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^0.6.3", + "promise-retry": "^2.0.1", + "socks-proxy-agent": "^7.0.0", + "ssri": "^9.0.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/make-fetch-happen/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/make-fetch-happen/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-collect": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz", + "integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==", + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-fetch": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-2.1.2.tgz", + "integrity": "sha512-LT49Zi2/WMROHYoqGgdlQIZh8mLPZmOrN2NdJjMXxYe4nkN6FUyuPuOAOedNJDrx0IRGg9+4guZewtp8hE6TxA==", + "license": "MIT", + "dependencies": { + "minipass": "^3.1.6", + "minipass-sized": "^1.0.3", + "minizlib": "^2.1.2" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + }, + "optionalDependencies": { + "encoding": "^0.1.13" + } + }, + "node_modules/minipass-flush": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.7.tgz", + "integrity": "sha512-TbqTz9cUwWyHS2Dy89P3ocAGUGxKjjLuR9z8w4WUTGAVgEj17/4nhgo2Du56i0Fm3Pm30g4iA8Lcqctc76jCzA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz", + "integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==", + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "license": "MIT", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "license": "MIT", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-addon-api": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", + "license": "MIT" + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-gyp": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-9.4.1.tgz", + "integrity": "sha512-OQkWKbjQKbGkMf/xqI1jjy3oCTgMKJac58G2+bjZb3fza6gW2YrCSdMQYaoTb70crvE//Gngr4f0AgVHmqHvBQ==", + "license": "MIT", + "dependencies": { + "env-paths": "^2.2.0", + "exponential-backoff": "^3.1.1", + "glob": "^7.1.4", + "graceful-fs": "^4.2.6", + "make-fetch-happen": "^10.0.3", + "nopt": "^6.0.0", + "npmlog": "^6.0.0", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.2", + "which": "^2.0.2" + }, + "bin": { + "node-gyp": "bin/node-gyp.js" + }, + "engines": { + "node": "^12.13 || ^14.13 || >=16" + } + }, + "node_modules/node-gyp/node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "license": "ISC" + }, + "node_modules/node-gyp/node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "license": "ISC", + "engines": { + "node": ">=8" + } + }, + "node_modules/node-gyp/node_modules/nopt": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-6.0.0.tgz", + "integrity": "sha512-ZwLpbTgdhuZUnZzjd7nb1ZV+4DoiC6/sfiVKok72ym/4Tlf+DFdlHYmT2JPmcNNWV6Pi3SDf1kT+A4r9RTuT9g==", + "license": "ISC", + "dependencies": { + "abbrev": "^1.0.0" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/node-gyp/node_modules/tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "deprecated": "Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/nopt": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-8.1.0.tgz", + "integrity": "sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==", + "license": "ISC", + "dependencies": { + "abbrev": "^3.0.0" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npmlog": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz", + "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "dependencies": { + "are-we-there-yet": "^3.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^4.0.3", + "set-blocking": "^2.0.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "license": "MIT", + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pg": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz", + "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.15.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", + "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz", + "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/possible-typed-array-names": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", + "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/promise-inflight": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", + "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==", + "license": "ISC" + }, + "node_modules/promise-retry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", + "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", + "license": "MIT", + "dependencies": { + "err-code": "^2.0.2", + "retry": "^0.12.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==", + "license": "MIT" + }, + "node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==", + "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-regex-test": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", + "integrity": "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "is-regex": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT", + "optional": true + }, + "node_modules/sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==", + "license": "ISC" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "license": "ISC" + }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz", + "integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.1.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-7.0.0.tgz", + "integrity": "sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==", + "license": "MIT", + "dependencies": { + "agent-base": "^6.0.2", + "debug": "^4.3.3", + "socks": "^2.6.2" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/socks-proxy-agent/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/ssri": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-9.0.1.tgz", + "integrity": "sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q==", + "license": "ISC", + "dependencies": { + "minipass": "^3.1.1" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/stream-browserify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", + "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", + "license": "MIT", + "dependencies": { + "inherits": "~2.0.4", + "readable-stream": "^3.5.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tar": { + "version": "7.5.19", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.19.tgz", + "integrity": "sha512-4LeEWl96twnS2Q7Bz4MGqgazLqO+hJN63GZxXoIqh1T3VweYD997gbU1ItNsQafqqXTXd5WFyFdReLtwvRBNiw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tar/node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/tar/node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/tar/node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/tar/node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/tmp": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", + "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true, + "license": "MIT" + }, + "node_modules/unique-filename": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-2.0.1.tgz", + "integrity": "sha512-ODWHtkkdx3IAR+veKxFV+VBkUMcN+FaqzUUd7IZzt+0zhDZFPFxhlqwPF3YQvMHx1TD0tdgYl+kuPnJ8E6ql7A==", + "license": "ISC", + "dependencies": { + "unique-slug": "^3.0.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/unique-slug": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-3.0.0.tgz", + "integrity": "sha512-8EyMynh679x/0gqE9fT9oilG+qEt+ibFyqjuVTsZn1+CMxH+XLlpvr2UZx4nVcCwTpx81nICr2JQFkM+HPLq4w==", + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", + "license": "MIT", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/uuid": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", + "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/which-typed-array": { + "version": "1.1.22", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.22.tgz", + "integrity": "sha512-fvO4ExWMFsqyhG3AiPAObMuY1lxaqgYcxbc49CNdWDDECOJNgQyvsOWVwbZc+qf3rzRtxojBK+CMEv0Ld5CYpw==", + "license": "MIT", + "dependencies": { + "available-typed-arrays": "^1.0.7", + "call-bind": "^1.0.9", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "license": "ISC", + "dependencies": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/xml2js": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "license": "MIT", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "license": "MIT", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + } + } +} diff --git a/packages/data-tables-handler/package.json b/packages/data-tables-handler/package.json new file mode 100644 index 000000000..5cf4c547e --- /dev/null +++ b/packages/data-tables-handler/package.json @@ -0,0 +1,31 @@ +{ + "name": "data-tables-handler", + "version": "1.0.0", + "description": "Processes overlay-related CSV data table uploads to Parquet + column stats on R2. Intentionally omitted from lerna.json: the duckdb native dep compiles from source on unsupported Node ABIs and makes CI lerna bootstrap take ~40m. Install deps in this package directly (or via Docker). Track migration to @duckdb/node-api in https://github.com/seasketch/next/issues/971.", + "license": "BSD-3-Clause", + "main": "dist/handler.js", + "scripts": { + "build": "tsc", + "watch": "tsc -w", + "dev": "npx --yes tsx@4.19.3 watch server.ts", + "build:container": "docker build -f Dockerfile .. -t data-tables-handler:latest" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.758.0", + "@aws-sdk/lib-storage": "^3.758.0", + "@seasketch/geostats-types": "^1.0.0", + "aws-sdk": "^2.1234.0", + "bytes": "^3.1.2", + "duckdb": "^1.4.4", + "pg": "^8.8.0", + "tmp": "^0.2.1", + "typescript": "5.5.3" + }, + "devDependencies": { + "@types/bytes": "^3.1.1", + "@types/node": "^18.11.4", + "@types/pg": "^8.6.5", + "@types/tmp": "^0.2.3", + "dotenv": "^16.0.3" + } +} diff --git a/packages/data-tables-handler/server.ts b/packages/data-tables-handler/server.ts new file mode 100644 index 000000000..47991f472 --- /dev/null +++ b/packages/data-tables-handler/server.ts @@ -0,0 +1,34 @@ +#!/usr/bin/env tsx +import { createServer } from "http"; +import * as dotenv from "dotenv"; +dotenv.config(); + +const PORT = process.env.PORT || 3006; + +const server = createServer(function (req, res) { + if (req.method === "POST") { + let body = ""; + req.on("data", (chunk) => { + body += chunk.toString(); + }); + req.on("end", async () => { + try { + const data = JSON.parse(body); + const { processDataTableUpload } = await import("./handler"); + const outputs = await processDataTableUpload(data); + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify(outputs)); + } catch (e) { + res.writeHead(500, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: (e as Error).message })); + } + }); + } else { + res.writeHead(200); + res.end("POST to simulate data-tables-handler lambda"); + } +}); + +server.listen(PORT); +console.log(`Data tables handler dev server on http://localhost:${PORT}`); +console.log(`Set DATA_TABLES_LAMBDA_DEV_HANDLER=http://localhost:${PORT}`); diff --git a/packages/data-tables-handler/src/handleDataTableUpload.ts b/packages/data-tables-handler/src/handleDataTableUpload.ts new file mode 100644 index 000000000..2592f6ac6 --- /dev/null +++ b/packages/data-tables-handler/src/handleDataTableUpload.ts @@ -0,0 +1,226 @@ +import { dirSync } from "tmp"; +import * as path from "path"; +import { writeFileSync } from "fs"; +import { getClient } from "./lambda-db-client"; +import { + buildR2Remote, + getStagingObject, + putObject, +} from "./remotes"; +import { + getGeostatsLayer, + validateJoinColumnChoice, +} from "./validateJoinColumn"; +import { + computeColumnStatsFromParquet, + processCsvWithDuckDb, + readJoinValues, +} from "./processWithDuckDb"; +import type { + DataTableUploadProcessingOptions, + DataTablesHandlerRequest, + DataTablesHandlerResponse, +} from "./types"; + +const PARQUET_CONTENT_TYPE = "application/vnd.apache.parquet"; +const JSON_CONTENT_TYPE = "application/json; charset=utf-8"; + +function defaultTableName(filename: string): string { + return filename.replace(/\.[^.]+$/, ""); +} + +function logDebug(message: string, details?: Record) { + const suffix = details ? ` ${JSON.stringify(details)}` : ""; + console.log(`[data-tables-handler] ${message}${suffix}`); +} + +export default async function handleDataTableUpload( + request: DataTablesHandlerRequest, +): Promise { + const { + taskId, + uploadId, + objectKey, + slug, + sourceUuid, + skipLoggingProgress, + } = request; + logDebug("starting upload", { + taskId, + uploadId, + objectKey, + slug, + sourceUuid, + skipLoggingProgress: Boolean(skipLoggingProgress), + }); + + const tmpobj = dirSync({ unsafeCleanup: true, keep: false, prefix: "dt-" }); + const csvPath = path.join(tmpobj.name, "input.csv"); + const parquetPath = path.join(tmpobj.name, "data.parquet"); + const statsPath = path.join(tmpobj.name, "column-stats.json"); + + // Established inside try so that connection failures are still reported to + // the caller; if the DB is unreachable the job fails via timeout instead. + let pgClient: Awaited> | null = null; + + const updateProgress = async ( + state: "running" | "complete" | "failed", + progressMessage: string, + progress?: number, + ) => { + if (skipLoggingProgress || !pgClient) return; + logDebug("updateProgress", { taskId, state, progressMessage, progress }); + if (progress !== undefined) { + await pgClient.query( + `update project_background_jobs set state = $1, progress = least($2::numeric, 1.0::numeric), progress_message = $3 where id = $4`, + [state, progress, progressMessage, taskId], + ); + } else { + await pgClient.query( + `update project_background_jobs set state = $1, progress_message = $2 where id = $3`, + [state, progressMessage, taskId], + ); + } + }; + + try { + if (!sourceUuid) { + throw new Error( + "sourceUuid is required to store data tables under the parent layer path", + ); + } + pgClient = await getClient(); + await updateProgress("running", "downloading", 0.05); + + const uploadQ = await pgClient.query( + `select filename, processing_options, overlay_geostats, overlay_join_column, replace_overlay_data_table_id + from overlay_data_table_uploads where id = $1`, + [uploadId], + ); + if (!uploadQ.rows[0]) { + throw new Error("Upload record not found"); + } + const upload = uploadQ.rows[0]; + const processingOptions = (upload.processing_options || + {}) as DataTableUploadProcessingOptions; + const joinColumn = processingOptions.joinColumn; + const overlayJoinColumn = + processingOptions.overlayJoinColumn || upload.overlay_join_column; + if (!joinColumn || !overlayJoinColumn) { + throw new Error("Join column and overlay join column are required"); + } + + logDebug("downloading staging object", { objectKey, csvPath }); + await getStagingObject(csvPath, objectKey); + await updateProgress("running", "processing", 0.2); + + logDebug("processing csv with duckdb", { csvPath, parquetPath }); + const { rowCount, headers } = await processCsvWithDuckDb( + csvPath, + parquetPath, + processingOptions, + ); + logDebug("csv processed", { rowCount, columnCount: headers.length }); + + const joinValues = await readJoinValues(parquetPath, joinColumn); + const layer = getGeostatsLayer(upload.overlay_geostats); + const joinValidation = validateJoinColumnChoice( + headers, + joinColumn, + overlayJoinColumn, + layer, + joinValues, + ); + logDebug("join validation complete", { + joinColumn, + overlayJoinColumn, + matchRate: joinValidation.matchRate, + matchedRows: joinValidation.matchedRows, + unmatchedRows: joinValidation.unmatchedRows, + }); + + await updateProgress("running", "computing stats", 0.6); + + const tableName = + processingOptions.name || defaultTableName(upload.filename); + const columnStats = await computeColumnStatsFromParquet( + parquetPath, + tableName, + { + column: joinColumn, + overlayAttribute: overlayJoinColumn, + matchRate: joinValidation.matchRate, + matchedRows: joinValidation.matchedRows, + unmatchedRows: joinValidation.unmatchedRows, + unmatchedOverlayValues: joinValidation.unmatchedOverlayValues, + }, + ); + + writeFileSync(statsPath, JSON.stringify(columnStats)); + + await updateProgress("running", "uploading", 0.8); + + const parquetTarget = buildR2Remote( + slug, + sourceUuid, + uploadId, + "data.parquet", + ); + const statsTarget = buildR2Remote( + slug, + sourceUuid, + uploadId, + "column-stats.json", + ); + await putObject(parquetPath, parquetTarget.remote, PARQUET_CONTENT_TYPE); + await putObject(statsPath, statsTarget.remote, JSON_CONTENT_TYPE); + + const result = { + uploadId, + name: tableName, + joinColumn, + overlayJoinColumn, + rowCount: Math.trunc(Number(rowCount)), + parquetRemote: parquetTarget.remote, + columnStatsRemote: statsTarget.remote, + }; + logDebug("upload processing complete, enqueueing outputs job", { + taskId, + uploadId, + rowCount: result.rowCount, + parquetRemote: result.parquetRemote, + }); + + await pgClient.query( + `SELECT graphile_worker.add_job('processDataTableUploadOutputs', $1::json)`, + [JSON.stringify({ jobId: taskId, data: result })], + ); + + return { success: result }; + } catch (e) { + const error = e as Error; + logDebug("upload failed", { + taskId, + uploadId, + message: error.message, + stack: error.stack, + }); + const errorDetails = { message: error.message }; + if (pgClient) { + try { + await pgClient.query( + `select fail_overlay_data_table_upload($1, $2, $3)`, + [taskId, error.message, JSON.stringify(errorDetails)], + ); + } catch (failError) { + logDebug("failed to record job failure", { + taskId, + error: (failError as Error).message, + }); + } + } + return { error: error.message, errorDetails }; + } finally { + tmpobj.removeCallback(); + } +} diff --git a/packages/data-tables-handler/src/inferCsvColumnPlans.ts b/packages/data-tables-handler/src/inferCsvColumnPlans.ts new file mode 100644 index 000000000..0b6e88b62 --- /dev/null +++ b/packages/data-tables-handler/src/inferCsvColumnPlans.ts @@ -0,0 +1,178 @@ +import duckdb from "duckdb"; + +export const CSV_NULL_STRINGS_BASE = ["", "N/A", "#N/A", "NULL", "null"]; +export const CSV_NULL_STRINGS_WITH_NA = [...CSV_NULL_STRINGS_BASE, "NA"]; + +export type CsvColumnPlan = { + name: string; + duckDbType: string; + nullifyNa: boolean; +}; + +function run(conn: duckdb.Connection, sql: string): Promise { + return new Promise((resolve, reject) => { + conn.run(sql, (err) => (err ? reject(err) : resolve())); + }); +} + +function all(conn: duckdb.Connection, sql: string): Promise { + return new Promise((resolve, reject) => { + conn.all(sql, (err, rows) => (err ? reject(err) : resolve(rows as T[]))); + }); +} + +export function nullstrOption(nullStrings: string[]): string { + return `nullstr=[${nullStrings.map((v) => `'${v.replace(/'/g, "''")}'`).join(", ")}]`; +} + +export function isNumericDuckDbType(type: string): boolean { + return /INT|DOUBLE|FLOAT|DECIMAL|NUMERIC|REAL|HUGEINT/i.test(type); +} + +function quoteColumn(name: string): string { + return `"${name.replace(/"/g, '""')}"`; +} + +/** + * Compare two typed CSV reads (with vs without treating "NA" as null) to decide + * per-column whether bare "NA" is missing data or a valid string code. + */ +export function decideNaNullHandling( + strictType: string, + naNullType: string, + naLiteralCount: number, + naLooksLikeValidCode: boolean, +): Pick { + const strictIsNumeric = isNumericDuckDbType(strictType); + const naNullIsNumeric = isNumericDuckDbType(naNullType); + + if (strictIsNumeric) { + return { duckDbType: strictType, nullifyNa: true }; + } + + if (naNullIsNumeric) { + // "NA" was blocking numeric inference (e.g. count column with R-style NA). + return { duckDbType: naNullType, nullifyNa: true }; + } + + if (naLiteralCount > 0 && naLooksLikeValidCode) { + // Column stays text either way; preserve literal "NA" codes (e.g. species). + return { duckDbType: "VARCHAR", nullifyNa: false }; + } + + return { duckDbType: "VARCHAR", nullifyNa: true }; +} + +async function naLooksLikeValidCode( + conn: duckdb.Connection, + quotedColumn: string, + naLiteralCount: number, +): Promise { + if (naLiteralCount === 0) { + return false; + } + + const freqRows = await all<{ max_c: number; median_c: number }>( + conn, + `SELECT + MAX(c)::INTEGER as max_c, + MEDIAN(c)::DOUBLE as median_c + FROM ( + SELECT COUNT(*)::INTEGER as c + FROM _csv_probe_strict + WHERE CAST(${quotedColumn} AS VARCHAR) != 'NA' + AND ${quotedColumn} IS NOT NULL + GROUP BY CAST(${quotedColumn} AS VARCHAR) + ) freq`, + ); + const maxC = freqRows[0]?.max_c ?? 0; + const medianC = freqRows[0]?.median_c ?? 0; + const peerFrequency = Math.max(medianC, 1); + + // NA appears roughly as often as a typical category value, not a bulk missing marker. + return naLiteralCount <= peerFrequency * 2 && naLiteralCount <= maxC; +} + +export async function inferCsvColumnPlans( + conn: duckdb.Connection, + csvPath: string, + readCsvOptionsSuffix: string, +): Promise { + const escapedPath = csvPath.replace(/'/g, "''"); + const baseNullstr = nullstrOption(CSV_NULL_STRINGS_BASE); + const naNullstr = nullstrOption(CSV_NULL_STRINGS_WITH_NA); + + await run( + conn, + `CREATE OR REPLACE TEMP TABLE _csv_probe_strict AS SELECT * FROM read_csv('${escapedPath}', ${readCsvOptionsSuffix}, ${baseNullstr})`, + ); + await run( + conn, + `CREATE OR REPLACE TEMP TABLE _csv_probe_na_null AS SELECT * FROM read_csv('${escapedPath}', ${readCsvOptionsSuffix}, ${naNullstr})`, + ); + + const strictSchema = await all<{ column_name: string; data_type: string }>( + conn, + `SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '_csv_probe_strict' ORDER BY ordinal_position`, + ); + const naNullSchema = await all<{ column_name: string; data_type: string }>( + conn, + `SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '_csv_probe_na_null' ORDER BY ordinal_position`, + ); + const naNullTypeByColumn = new Map( + naNullSchema.map((col) => [col.column_name, col.data_type]), + ); + + const plans: CsvColumnPlan[] = []; + for (const col of strictSchema) { + const quoted = quoteColumn(col.column_name); + const naLiteralRows = await all<{ count: number }>( + conn, + `SELECT COUNT(*)::INTEGER as count FROM _csv_probe_strict WHERE CAST(${quoted} AS VARCHAR) = 'NA'`, + ); + const naLiteralCount = naLiteralRows[0]?.count ?? 0; + const validNaCode = await naLooksLikeValidCode( + conn, + quoted, + naLiteralCount, + ); + const decision = decideNaNullHandling( + col.data_type, + naNullTypeByColumn.get(col.column_name) || "VARCHAR", + naLiteralCount, + validNaCode, + ); + plans.push({ + name: col.column_name, + ...decision, + }); + if (!decision.nullifyNa && naLiteralCount > 0) { + console.log( + `[data-tables-handler] preserving literal "NA" in column ${col.column_name} (${naLiteralCount} row(s); looks like a category code)`, + ); + } + } + + await run(conn, "DROP TABLE IF EXISTS _csv_probe_strict"); + await run(conn, "DROP TABLE IF EXISTS _csv_probe_na_null"); + + return plans; +} + +export function buildTypedSelectSql(plans: CsvColumnPlan[]): string { + return plans + .map((plan) => { + const quoted = quoteColumn(plan.name); + if (isNumericDuckDbType(plan.duckDbType)) { + const valueExpr = plan.nullifyNa + ? `NULLIF(${quoted}, 'NA')` + : quoted; + return `TRY_CAST(${valueExpr} AS ${plan.duckDbType}) AS ${quoted}`; + } + if (plan.nullifyNa) { + return `NULLIF(${quoted}, 'NA') AS ${quoted}`; + } + return `${quoted}`; + }) + .join(", "); +} diff --git a/packages/data-tables-handler/src/lambda-db-client.ts b/packages/data-tables-handler/src/lambda-db-client.ts new file mode 100644 index 000000000..2b51ff5f3 --- /dev/null +++ b/packages/data-tables-handler/src/lambda-db-client.ts @@ -0,0 +1,85 @@ +import { RDS } from "aws-sdk"; +import { Client } from "pg"; + +export async function getClient() { + if (!dbClient) { + if ( + db.host && + (/localhost/.test(db.host) || + /127.0.0.1/.test(db.host) || + /host.docker.internal/.test(db.host)) + ) { + dbClient = new Promise(async (resolve, reject) => { + const client = new Client({ + database: db.database, + host: db.host, + port: parseInt(db.port.toString()), + user: db.user, + password: process.env.PGPASSWORD, + }); + try { + await client.connect(); + client.on("error", () => { + dbClient = null; + }); + resolve(client); + } catch (e) { + reject(e); + } + }); + } else { + dbClient = new Promise((resolve, reject) => { + signer.getAuthToken( + { + region: db.region, + hostname: db.host, + port: parseInt(db.port.toString()), + username: db.user, + }, + async function (err, token) { + if (err) { + reject(new Error(`could not get auth token: ${err}`)); + } else { + const client = new Client({ + database: db.database, + host: db.host, + port: parseInt(db.port.toString()), + user: db.user, + password: token, + ssl: { rejectUnauthorized: false }, + }); + try { + await client.connect(); + client.on("error", () => { + dbClient = null; + }); + resolve(client); + } catch (e) { + reject(e); + } + } + }, + ); + }); + } + } + return dbClient; +} + +const signer = new RDS.Signer(); + +for (const param of ["PGREGION", "PGHOST", "PGPORT", "PGUSER", "PGDATABASE"]) { + if (!process.env[param]) { + throw new Error(`Environment variable "${param}" not set`); + } +} + +const db = { + region: process.env.PGREGION!, + host: process.env.PGHOST!, + port: parseInt(process.env.PGPORT as string), + user: process.env.PGUSER!, + database: process.env.PGDATABASE!, +}; + +let dbClient: Promise | null = null; diff --git a/packages/data-tables-handler/src/normalizeCsvEncoding.ts b/packages/data-tables-handler/src/normalizeCsvEncoding.ts new file mode 100644 index 000000000..169959c18 --- /dev/null +++ b/packages/data-tables-handler/src/normalizeCsvEncoding.ts @@ -0,0 +1,35 @@ +import { readFileSync, writeFileSync } from "fs"; +import * as path from "path"; + +function isValidUtf8(buffer: Buffer): boolean { + try { + new TextDecoder("utf-8", { fatal: true }).decode(buffer); + return true; + } catch { + return false; + } +} + +/** + * DuckDB's CSV reader requires valid UTF-8. Many legacy exports (Excel, R, etc.) + * are Latin-1 / Windows-1252. Reinterpret single-byte encodings as Latin-1 and + * write UTF-8 so every byte 0x00–0xFF round-trips without parse failures. + */ +export function normalizeCsvEncodingIfNeeded( + csvPath: string, + normalizedPath?: string, +): { path: string; normalized: boolean } { + const buffer = readFileSync(csvPath); + if (isValidUtf8(buffer)) { + return { path: csvPath, normalized: false }; + } + + const out = + normalizedPath || + path.join( + path.dirname(csvPath), + `${path.basename(csvPath, path.extname(csvPath))}.utf8${path.extname(csvPath) || ".csv"}`, + ); + writeFileSync(out, buffer.toString("latin1"), "utf8"); + return { path: out, normalized: true }; +} diff --git a/packages/data-tables-handler/src/processWithDuckDb.ts b/packages/data-tables-handler/src/processWithDuckDb.ts new file mode 100644 index 000000000..0edcb8b82 --- /dev/null +++ b/packages/data-tables-handler/src/processWithDuckDb.ts @@ -0,0 +1,218 @@ +import duckdb from "duckdb"; +import * as path from "path"; +import { + DataTablesColumnStats, + GeostatsAttribute, +} from "@seasketch/geostats-types"; +import { inferGeostatsType } from "./validateJoinColumn"; +import type { DataTableUploadProcessingOptions } from "./types"; +import { normalizeCsvEncodingIfNeeded } from "./normalizeCsvEncoding"; +import { + buildTypedSelectSql, + CSV_NULL_STRINGS_BASE, + inferCsvColumnPlans, + nullstrOption, +} from "./inferCsvColumnPlans"; + +function run(conn: duckdb.Connection, sql: string): Promise { + return new Promise((resolve, reject) => { + conn.run(sql, (err) => (err ? reject(err) : resolve())); + }); +} + +function all(conn: duckdb.Connection, sql: string): Promise { + return new Promise((resolve, reject) => { + conn.all(sql, (err, rows) => (err ? reject(err) : resolve(rows as T[]))); + }); +} + +function escapePath(path: string): string { + return path.replace(/'/g, "''"); +} + +/** Runs fn against a fresh in-memory DuckDB, always closing it afterwards. */ +async function withDuckDb( + fn: (conn: duckdb.Connection) => Promise, +): Promise { + const db = new duckdb.Database(":memory:"); + const conn = db.connect(); + try { + return await fn(conn); + } finally { + conn.close(); + db.close(); + } +} + +function delimiterOption(delimiter: string): string { + switch (delimiter) { + case "\t": + return "delim='\\t'"; + case ";": + return "delim=';'"; + case "|": + return "delim='|'"; + default: + return ""; + } +} + +export async function processCsvWithDuckDb( + csvPath: string, + parquetPath: string, + options: DataTableUploadProcessingOptions, +): Promise<{ rowCount: number; headers: string[] }> { + const { path: duckDbCsvPath, normalized } = normalizeCsvEncodingIfNeeded( + csvPath, + path.join(path.dirname(csvPath), "input.utf8.csv"), + ); + if (normalized) { + console.log( + `[data-tables-handler] normalized csv encoding to utf-8: ${duckDbCsvPath}`, + ); + } + const delimiter = options.delimiter || ","; + const hasHeader = options.hasHeaderRow !== false; + const forceNotNull = options.forceNotNull || []; + const forceNotNullSql = + forceNotNull.length > 0 + ? `, force_not_null=[${forceNotNull.map((c) => `'${c.replace(/'/g, "''")}'`).join(", ")}]` + : ""; + + const readCsvOptionsSuffix = [ + "header=" + (hasHeader ? "true" : "false"), + "sample_size=-1", + delimiterOption(delimiter), + ] + .filter(Boolean) + .join(", "); + + return withDuckDb(async (conn) => { + const columnPlans = await inferCsvColumnPlans( + conn, + duckDbCsvPath, + readCsvOptionsSuffix, + ); + const baseNullstr = nullstrOption(CSV_NULL_STRINGS_BASE); + + await run( + conn, + `CREATE OR REPLACE TEMP TABLE _csv_raw AS SELECT * FROM read_csv('${escapePath(duckDbCsvPath)}', ${readCsvOptionsSuffix}, ${baseNullstr}, all_varchar=true${forceNotNullSql})`, + ); + await run( + conn, + `CREATE OR REPLACE TABLE observations AS SELECT ${buildTypedSelectSql(columnPlans)} FROM _csv_raw`, + ); + await run(conn, "DROP TABLE IF EXISTS _csv_raw"); + + const countRows = await all<{ count: number }>( + conn, + "SELECT COUNT(*)::INTEGER as count FROM observations", + ); + const rowCount = countRows[0]?.count ?? 0; + + const columns = await all<{ column_name: string }>( + conn, + `SELECT column_name FROM information_schema.columns WHERE table_name = 'observations' ORDER BY ordinal_position`, + ); + const headers = columns.map((c) => c.column_name); + + await run( + conn, + `COPY observations TO '${escapePath(parquetPath)}' (FORMAT PARQUET)`, + ); + + return { rowCount, headers }; + }); +} + +export async function readJoinValues( + parquetPath: string, + joinColumn: string, +): Promise> { + const col = joinColumn.replace(/"/g, '""'); + return withDuckDb(async (conn) => { + const rows = await all<{ v: string }>( + conn, + `SELECT DISTINCT CAST("${col}" AS VARCHAR) as v FROM read_parquet('${escapePath(parquetPath)}') WHERE "${col}" IS NOT NULL`, + ); + return new Set(rows.map((r) => r.v)); + }); +} + +/** Cap on histogram entries stored per column in column-stats.json. */ +const VALUES_HISTOGRAM_LIMIT = 500; + +export async function computeColumnStatsFromParquet( + parquetPath: string, + tableName: string, + joinInfo: DataTablesColumnStats["join"], +): Promise { + return withDuckDb(async (conn) => { + await run( + conn, + `CREATE OR REPLACE TABLE observations AS SELECT * FROM read_parquet('${escapePath(parquetPath)}')`, + ); + + const countRows = await all<{ count: number }>( + conn, + "SELECT COUNT(*)::INTEGER as count FROM observations", + ); + const rowCount = countRows[0]?.count ?? 0; + + const schema = await all<{ column_name: string; data_type: string }>( + conn, + `SELECT column_name, data_type FROM information_schema.columns WHERE table_name = 'observations' ORDER BY ordinal_position`, + ); + + const columns: GeostatsAttribute[] = []; + for (const col of schema) { + const colName = col.column_name.replace(/"/g, '""'); + const type = inferGeostatsType(col.data_type); + const counts = await all<{ count: number; count_distinct: number }>( + conn, + `SELECT COUNT("${colName}")::INTEGER as count, COUNT(DISTINCT "${colName}")::INTEGER as count_distinct FROM observations`, + ); + const count = counts[0]?.count ?? 0; + const countDistinct = counts[0]?.count_distinct ?? 0; + const distinctRows = await all<{ v: string; c: number }>( + conn, + `SELECT CAST("${colName}" AS VARCHAR) as v, COUNT(*)::INTEGER as c FROM observations WHERE "${colName}" IS NOT NULL GROUP BY 1 ORDER BY c DESC LIMIT ${VALUES_HISTOGRAM_LIMIT}`, + ); + const values: { [key: string]: number } = {}; + for (const row of distinctRows) { + values[row.v] = row.c; + } + const attr: GeostatsAttribute = { + attribute: col.column_name, + count, + countDistinct, + type, + values, + }; + if (type === "number") { + const stats = await all<{ + min: number; + max: number; + }>( + conn, + `SELECT MIN(CAST("${colName}" AS DOUBLE)) as min, MAX(CAST("${colName}" AS DOUBLE)) as max FROM observations WHERE "${colName}" IS NOT NULL`, + ); + if (stats[0]) { + (attr as GeostatsAttribute & { min?: number; max?: number }).min = + stats[0].min; + (attr as GeostatsAttribute & { min?: number; max?: number }).max = + stats[0].max; + } + } + columns.push(attr); + } + + return { + table: tableName, + rowCount, + columns, + join: joinInfo, + }; + }); +} diff --git a/packages/data-tables-handler/src/remotes.ts b/packages/data-tables-handler/src/remotes.ts new file mode 100644 index 000000000..b54638777 --- /dev/null +++ b/packages/data-tables-handler/src/remotes.ts @@ -0,0 +1,89 @@ +import { createReadStream, createWriteStream, statSync } from "fs"; +import { pipeline } from "stream/promises"; +import { + S3Client, + PutObjectCommand, + GetObjectCommand, +} from "@aws-sdk/client-s3"; +import { Upload } from "@aws-sdk/lib-storage"; +import bytes from "bytes"; +import { Readable } from "stream"; + +const s3Client = new S3Client({ region: process.env.AWS_REGION! }); +const r2Client = new S3Client({ + region: "auto", + endpoint: process.env.R2_ENDPOINT!, + credentials: { + accessKeyId: process.env.R2_ACCESS_KEY_ID!, + secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!, + }, +}); + +export async function putObject( + filepath: string, + remote: string, + contentType?: string, +) { + if (!/r2:/.test(remote) && !/s3:/.test(remote)) { + throw new Error(`Invalid remote ${remote}`); + } + const parts = remote.replace(/\w+:\/\//, "").split("/"); + const client = /r2:/.test(remote) ? r2Client : s3Client; + const Bucket = parts[0]; + const Key = parts.slice(1).join("/"); + const fileSizeBytes = statSync(filepath).size; + const fileStream = createReadStream(filepath); + if (fileSizeBytes > 500 * 1024 * 1024) { + const upload = new Upload({ + client, + params: { Bucket, Key, Body: fileStream, ContentType: contentType }, + }); + await upload.done(); + } else { + await client.send( + new PutObjectCommand({ + Bucket, + Key, + Body: fileStream, + ContentType: contentType, + }), + ); + } + console.log(`putObject ${filepath} (${bytes(fileSizeBytes)}) to ${remote}`); +} + +/** Download the user's raw upload from the S3 staging bucket. */ +export async function getStagingObject( + filepath: string, + objectKey: string, +): Promise { + const bucket = process.env.BUCKET!; + const response = await s3Client.send( + new GetObjectCommand({ Bucket: bucket, Key: objectKey }), + ); + const body = response.Body as Readable; + await pipeline(body, createWriteStream(filepath)); +} + +/** + * Store data tables under the parent layer's hosted UUID so objects classify + * as `published` in the tiles ACL gateway and inherit the layer's ACL. + * + * `projects/{slug}/public/{sourceUuid}/dataTables/{uploadId}/{filename}` + */ +export function buildR2Remote( + slug: string, + sourceUuid: string, + uploadId: string, + filename: string, +) { + const bucket = process.env.R2_TILES_BUCKET || process.env.TILES_REMOTE?.replace("r2://", "").split("/")[0]; + if (!bucket) { + throw new Error("R2_TILES_BUCKET or TILES_REMOTE must be set"); + } + if (!sourceUuid) { + throw new Error("sourceUuid is required to store data tables under the parent layer path"); + } + const key = `projects/${slug}/public/${sourceUuid}/dataTables/${uploadId}/${filename}`; + return { remote: `r2://${bucket}/${key}`, key, bucket }; +} diff --git a/packages/data-tables-handler/src/types.ts b/packages/data-tables-handler/src/types.ts new file mode 100644 index 000000000..135e7fb8e --- /dev/null +++ b/packages/data-tables-handler/src/types.ts @@ -0,0 +1,35 @@ +export interface DataTableUploadProcessingOptions { + delimiter?: "," | "\t" | ";" | "|"; + hasHeaderRow?: boolean; + joinColumn?: string; + overlayJoinColumn?: string; + forceNotNull?: string[]; + name?: string; +} + +export interface DataTablesHandlerRequest { + taskId: string; + uploadId: string; + objectKey: string; + /** Project slug; first path segment of the R2 key. */ + slug: string; + /** Parent layer hosted-tiles UUID (`data_sources.url`); used for R2 key prefix. */ + sourceUuid: string; + skipLoggingProgress?: boolean; +} + +export interface DataTablesHandlerSuccess { + uploadId: string; + name: string; + joinColumn: string; + overlayJoinColumn: string; + rowCount: number; + parquetRemote: string; + columnStatsRemote: string; +} + +export interface DataTablesHandlerResponse { + success?: DataTablesHandlerSuccess; + error?: string; + errorDetails?: Record; +} diff --git a/packages/data-tables-handler/src/validateJoinColumn.ts b/packages/data-tables-handler/src/validateJoinColumn.ts new file mode 100644 index 000000000..bb72ccfaa --- /dev/null +++ b/packages/data-tables-handler/src/validateJoinColumn.ts @@ -0,0 +1,102 @@ +import { + GeostatsAttribute, + GeostatsAttributeType, + GeostatsLayer, +} from "@seasketch/geostats-types"; + +export function getGeostatsLayer(overlayGeostats: unknown): GeostatsLayer { + const data = overlayGeostats as { layers?: GeostatsLayer[] }; + const layer = data?.layers?.[0]; + if (!layer?.attributes) { + throw new Error("Overlay geostats missing layer attributes"); + } + return layer; +} + +export function findOverlayAttribute( + layer: GeostatsLayer, + attributeName: string, +) { + const attr = layer.attributes.find((a) => a.attribute === attributeName); + if (!attr) { + throw new Error( + `Overlay attribute "${attributeName}" not found in geostats`, + ); + } + return attr; +} + +/** + * Validates that distinct values in the table's join column exist among the + * overlay layer's feature identifiers (from geostats). Comparison is an + * exact string match — join keys must match the overlay attribute exactly, + * as they would in any other analysis tooling. + * + * Geostats attribute `values` histograms are truncated (top ~500 keys), so + * when the overlay attribute has more distinct values than the histogram + * holds, unmatched CSV values are reported in the stats but do not fail the + * upload. With a complete histogram, any unmatched value is an error. + * + * Note: despite the names (kept for compatibility with the stored + * column-stats.json format), matchedRows/unmatchedRows count *distinct join + * values*, not table rows. + */ +export function validateJoinColumnChoice( + headers: string[], + joinColumn: string, + overlayJoinColumn: string, + layer: GeostatsLayer, + joinValues: Set, +): { overlayAttr: GeostatsAttribute; matchRate: number; matchedRows: number; unmatchedRows: number; unmatchedOverlayValues: number } { + if (!headers.includes(joinColumn)) { + throw new Error(`Join column "${joinColumn}" not found in CSV headers`); + } + const overlayAttr = findOverlayAttribute(layer, overlayJoinColumn); + const overlayKeys = new Set(Object.keys(overlayAttr.values || {})); + const histogramComplete = + typeof overlayAttr.countDistinct !== "number" || + overlayAttr.countDistinct <= overlayKeys.size; + + let matchedRows = 0; + for (const v of joinValues) { + if (overlayKeys.has(v)) { + matchedRows++; + } + } + const unmatchedRows = joinValues.size - matchedRows; + if (matchedRows === 0) { + throw new Error( + "No values in the join column match overlay feature identifiers", + ); + } + if (unmatchedRows > 0 && histogramComplete) { + throw new Error( + `${unmatchedRows} value(s) in the join column are not present in the overlay layer`, + ); + } + let unmatchedOverlayValues = 0; + for (const k of overlayKeys) { + if (!joinValues.has(k)) { + unmatchedOverlayValues++; + } + } + const matchRate = joinValues.size > 0 ? matchedRows / joinValues.size : 0; + return { + overlayAttr, + matchRate, + matchedRows, + unmatchedRows, + unmatchedOverlayValues, + }; +} + +export function inferGeostatsType(duckDbType: string): GeostatsAttributeType { + const t = duckDbType.toUpperCase(); + if (/INT|DOUBLE|FLOAT|DECIMAL|NUMERIC|REAL|HUGEINT/.test(t)) { + return "number"; + } + if (/BOOL/.test(t)) { + return "boolean"; + } + return "string"; +} diff --git a/packages/data-tables-handler/tsconfig.json b/packages/data-tables-handler/tsconfig.json new file mode 100644 index 000000000..ea087eee5 --- /dev/null +++ b/packages/data-tables-handler/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020"], + "outDir": "dist", + "rootDir": ".", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "declaration": true + }, + "include": ["handler.ts", "server.ts", "src/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/packages/fragment-worker/package-lock.json b/packages/fragment-worker/package-lock.json index 9eb7ac868..093055b11 100644 --- a/packages/fragment-worker/package-lock.json +++ b/packages/fragment-worker/package-lock.json @@ -10,10 +10,8 @@ "license": "BSD-3-Clause", "dependencies": { "@aws-sdk/client-secrets-manager": "^3.873.0", - "fgb-source": "^1.0.0", "geobuf": "^4.0.0", "lru-cache": "^11.2.2", - "overlay-engine": "^1.0.0", "undici": "^7.16.0" }, "devDependencies": { @@ -24,6 +22,7 @@ }, "../fgb-source": { "version": "1.0.0", + "extraneous": true, "license": "BSD-3-Clause", "dependencies": { "@turf/bbox": "^7.2.0", @@ -45,6 +44,7 @@ }, "../overlay-engine": { "version": "1.0.0", + "extraneous": true, "license": "BSD-3-Clause", "dependencies": { "@turf/along": "^7.3.0", @@ -486,10 +486,6 @@ "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "license": "MIT" }, - "node_modules/fgb-source": { - "resolved": "../fgb-source", - "link": true - }, "node_modules/file-source": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/file-source/-/file-source-0.6.1.tgz", @@ -522,10 +518,6 @@ "node": "20 || >=22" } }, - "node_modules/overlay-engine": { - "resolved": "../overlay-engine", - "link": true - }, "node_modules/path-source": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/path-source/-/path-source-0.1.3.tgz", diff --git a/packages/geostats-types/dist/lib/index.d.ts b/packages/geostats-types/dist/lib/index.d.ts index f0d25db5b..b2db94f90 100644 --- a/packages/geostats-types/dist/lib/index.d.ts +++ b/packages/geostats-types/dist/lib/index.d.ts @@ -229,4 +229,22 @@ export interface RasterInfo { } export declare function isRasterInfo(info: RasterInfo | GeostatsLayer | any): info is RasterInfo; export declare function isGeostatsLayer(data: RasterInfo | GeostatsLayer | any): data is GeostatsLayer; +/** + * Column statistics for overlay-related tabular data (CSV uploads joined to + * vector layers). Stored as JSON on R2 alongside Parquet artifacts. + */ +export interface DataTablesColumnStats { + /** Display name of the table (typically derived from filename). */ + table: string; + rowCount: number; + columns: GeostatsAttribute[]; + join: { + column: string; + overlayAttribute: string; + matchRate: number; + matchedRows: number; + unmatchedRows: number; + unmatchedOverlayValues: number; + }; +} //# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/packages/geostats-types/dist/lib/index.d.ts.map b/packages/geostats-types/dist/lib/index.d.ts.map index 81ac3fbf5..53faceafb 100644 --- a/packages/geostats-types/dist/lib/index.d.ts.map +++ b/packages/geostats-types/dist/lib/index.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../lib/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAE/C;;GAEG;AACH,MAAM,MAAM,qBAAqB,GAC7B,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,MAAM,GACN,OAAO,GACP,QAAQ,GACR,OAAO,CAAC;AAEZ;;;;GAIG;AACH,MAAM,MAAM,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;AAE7C;;;GAGG;AACH,MAAM,MAAM,OAAO,GAAG;IAAE,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;CAAE,CAAC;AAExD,MAAM,MAAM,eAAe,GACvB,OAAO,GACP,OAAO,GACP,eAAe,GACf,WAAW,GACX,MAAM,GACN,OAAO,CAAC;AACZ,MAAM,WAAW,qBAAqB;IACpC,4BAA4B;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,kEAAkE;IAClE,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,IAAI,EAAE,qBAAqB,CAAC;IAC5B;;;;;OAKG;IACH,WAAW,CAAC,EAAE,qBAAqB,CAAC;IACpC,2CAA2C;IAC3C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,MAAM,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IAClC;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iBAAiB,CAAC,EAAE,eAAe,EAAE,CAAC;CACvC;AAED;;;GAGG;AACH,MAAM,WAAW,wBAAyB,SAAQ,qBAAqB;IACrE,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE;QACL,kCAAkC;QAClC,GAAG,EAAE,MAAM,CAAC;QACZ;;WAEG;QACH,aAAa,EAAE,OAAO,CAAC;QACvB,8BAA8B;QAC9B,aAAa,EAAE,OAAO,CAAC;QACvB,sBAAsB;QACtB,SAAS,EAAE,OAAO,CAAC;QACnB,gCAAgC;QAChC,iBAAiB,EAAE,OAAO,CAAC;QAC3B,gCAAgC;QAChC,kBAAkB,EAAE,OAAO,CAAC;QAC5B;;;WAGG;QACH,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;QACrC,0CAA0C;QAC1C,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED,MAAM,MAAM,iBAAiB,GACzB,qBAAqB,GACrB,wBAAwB,CAAC;AAE7B,MAAM,MAAM,uBAAuB,GAAG,IAAI,CACxC,qBAAqB,EACrB,QAAQ,GAAG,eAAe,CAC3B,GAAG;IACF,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC;IAC7C,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB,CAAC;AAEF,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,iBAAiB,GACtB,IAAI,IAAI,wBAAwB,CAElC;AAED,MAAM,CAAC,OAAO,MAAM,YAAY;IAC9B,QAAQ,aAAa;IACrB,IAAI,SAAS;CACd;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;OAEG;IACH,GAAG,EAAE,GAAG,CAAC;IACT;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,YAAY,CAAC;IACnB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,QAAQ,EAAE,oBAAoB,GAAG,SAAS,CAAC;IAC3C,IAAI,EAAE,OAAO,CAAC;IACd;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,UAAU,EAAE,iBAAiB,EAAE,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,MAAM,mBAAmB,GAAG,IAAI,CACpC,aAAa,EACb,YAAY,GAAG,QAAQ,CACxB,GAAG;IACF,UAAU,EAAE,uBAAuB,EAAE,CAAC;CACvC,CAAC;AAEF,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,mBAAmB,GAAG,aAAa,GACzC,KAAK,IAAI,mBAAmB,CAM9B;AAED,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,uBAAuB,GAAG,iBAAiB,GAChD,IAAI,IAAI,uBAAuB,CAEjC;AAED;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;AAEnD;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG;IAAE,CAAC,SAAS,EAAE,MAAM,GAAG,YAAY,EAAE,CAAA;CAAE,CAAC;AAEpE;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE/C,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,mBAAmB,EACf,KAAK,GACL,OAAO,GACP,MAAM,GACN,OAAO,GACP,MAAM,GACN,MAAM,GACN,IAAI,CAAC;IAET,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,aAAa,CAAC;QAC7B,iBAAiB,EAAE,aAAa,CAAC;QACjC,aAAa,EAAE,aAAa,CAAC;QAC7B,SAAS,EAAE,aAAa,CAAC;QACzB,kBAAkB,EAAE,aAAa,CAAC;QAClC,SAAS,EAAE,YAAY,EAAE,CAAC;QAC1B,UAAU,EAAE,YAAY,EAAE,CAAC;KAC5B,CAAC;IACF,QAAQ,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACrC,UAAU,CAAC,EAAE,eAAe,EAAE,CAAC;IAC/B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB,CAAC;AAEF;;;;;;;;;GASG;AACH,oBAAY,2BAA2B;IACrC,aAAa,IAAA;IACb,YAAY,IAAA;IACZ,KAAK,IAAA;CACN;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,YAAY,EAAE,2BAA2B,CAAC;IAC1C,0BAA0B,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IACxD,QAAQ,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACrC;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,GAAG,GACrC,IAAI,IAAI,UAAU,CAEpB;AAED,wBAAgB,eAAe,CAC7B,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,GAAG,GACrC,IAAI,IAAI,aAAa,CAcvB"} \ No newline at end of file +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../lib/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAE/C;;GAEG;AACH,MAAM,MAAM,qBAAqB,GAC7B,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,MAAM,GACN,OAAO,GACP,QAAQ,GACR,OAAO,CAAC;AAEZ;;;;GAIG;AACH,MAAM,MAAM,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;AAE7C;;;GAGG;AACH,MAAM,MAAM,OAAO,GAAG;IAAE,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;CAAE,CAAC;AAExD,MAAM,MAAM,eAAe,GACvB,OAAO,GACP,OAAO,GACP,eAAe,GACf,WAAW,GACX,MAAM,GACN,OAAO,CAAC;AACZ,MAAM,WAAW,qBAAqB;IACpC,4BAA4B;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,kEAAkE;IAClE,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,IAAI,EAAE,qBAAqB,CAAC;IAC5B;;;;;OAKG;IACH,WAAW,CAAC,EAAE,qBAAqB,CAAC;IACpC,2CAA2C;IAC3C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,MAAM,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IAClC;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iBAAiB,CAAC,EAAE,eAAe,EAAE,CAAC;CACvC;AAED;;;GAGG;AACH,MAAM,WAAW,wBAAyB,SAAQ,qBAAqB;IACrE,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE;QACL,kCAAkC;QAClC,GAAG,EAAE,MAAM,CAAC;QACZ;;WAEG;QACH,aAAa,EAAE,OAAO,CAAC;QACvB,8BAA8B;QAC9B,aAAa,EAAE,OAAO,CAAC;QACvB,sBAAsB;QACtB,SAAS,EAAE,OAAO,CAAC;QACnB,gCAAgC;QAChC,iBAAiB,EAAE,OAAO,CAAC;QAC3B,gCAAgC;QAChC,kBAAkB,EAAE,OAAO,CAAC;QAC5B;;;WAGG;QACH,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;QACrC,0CAA0C;QAC1C,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED,MAAM,MAAM,iBAAiB,GACzB,qBAAqB,GACrB,wBAAwB,CAAC;AAE7B,MAAM,MAAM,uBAAuB,GAAG,IAAI,CACxC,qBAAqB,EACrB,QAAQ,GAAG,eAAe,CAC3B,GAAG;IACF,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC;IAC7C,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB,CAAC;AAEF,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,iBAAiB,GACtB,IAAI,IAAI,wBAAwB,CAElC;AAED,MAAM,CAAC,OAAO,MAAM,YAAY;IAC9B,QAAQ,aAAa;IACrB,IAAI,SAAS;CACd;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;OAEG;IACH,GAAG,EAAE,GAAG,CAAC;IACT;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,YAAY,CAAC;IACnB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,QAAQ,EAAE,oBAAoB,GAAG,SAAS,CAAC;IAC3C,IAAI,EAAE,OAAO,CAAC;IACd;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,UAAU,EAAE,iBAAiB,EAAE,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,MAAM,mBAAmB,GAAG,IAAI,CACpC,aAAa,EACb,YAAY,GAAG,QAAQ,CACxB,GAAG;IACF,UAAU,EAAE,uBAAuB,EAAE,CAAC;CACvC,CAAC;AAEF,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,mBAAmB,GAAG,aAAa,GACzC,KAAK,IAAI,mBAAmB,CAM9B;AAED,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,uBAAuB,GAAG,iBAAiB,GAChD,IAAI,IAAI,uBAAuB,CAEjC;AAED;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;AAEnD;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG;IAAE,CAAC,SAAS,EAAE,MAAM,GAAG,YAAY,EAAE,CAAA;CAAE,CAAC;AAEpE;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE/C,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,mBAAmB,EACf,KAAK,GACL,OAAO,GACP,MAAM,GACN,OAAO,GACP,MAAM,GACN,MAAM,GACN,IAAI,CAAC;IAET,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,aAAa,CAAC;QAC7B,iBAAiB,EAAE,aAAa,CAAC;QACjC,aAAa,EAAE,aAAa,CAAC;QAC7B,SAAS,EAAE,aAAa,CAAC;QACzB,kBAAkB,EAAE,aAAa,CAAC;QAClC,SAAS,EAAE,YAAY,EAAE,CAAC;QAC1B,UAAU,EAAE,YAAY,EAAE,CAAC;KAC5B,CAAC;IACF,QAAQ,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACrC,UAAU,CAAC,EAAE,eAAe,EAAE,CAAC;IAC/B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB,CAAC;AAEF;;;;;;;;;GASG;AACH,oBAAY,2BAA2B;IACrC,aAAa,IAAA;IACb,YAAY,IAAA;IACZ,KAAK,IAAA;CACN;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,YAAY,EAAE,2BAA2B,CAAC;IAC1C,0BAA0B,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IACxD,QAAQ,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACrC;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,GAAG,GACrC,IAAI,IAAI,UAAU,CAEpB;AAED,wBAAgB,eAAe,CAC7B,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,GAAG,GACrC,IAAI,IAAI,aAAa,CAcvB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,mEAAmE;IACnE,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,IAAI,EAAE;QACJ,MAAM,EAAE,MAAM,CAAC;QACf,gBAAgB,EAAE,MAAM,CAAC;QACzB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;CACH"} \ No newline at end of file diff --git a/packages/geostats-types/lib/index.ts b/packages/geostats-types/lib/index.ts index 9b5a82793..3a8e2c65e 100644 --- a/packages/geostats-types/lib/index.ts +++ b/packages/geostats-types/lib/index.ts @@ -306,3 +306,22 @@ export function isGeostatsLayer( } return true; } + +/** + * Column statistics for overlay-related tabular data (CSV uploads joined to + * vector layers). Stored as JSON on R2 alongside Parquet artifacts. + */ +export interface DataTablesColumnStats { + /** Display name of the table (typically derived from filename). */ + table: string; + rowCount: number; + columns: GeostatsAttribute[]; + join: { + column: string; + overlayAttribute: string; + matchRate: number; + matchedRows: number; + unmatchedRows: number; + unmatchedOverlayValues: number; + }; +} diff --git a/packages/infra/bin/infra.ts b/packages/infra/bin/infra.ts index 40dff93e0..163a23e1f 100644 --- a/packages/infra/bin/infra.ts +++ b/packages/infra/bin/infra.ts @@ -15,6 +15,7 @@ import { MailerLambdaStack } from "../lib/MailerLambdaStack"; import { OfflineTilePackageBucketStack } from "../lib/OfflineTilePackageUploadStack"; import { DataUploadsStack } from "../lib/DataUploadsStack"; import { UploadHandlerLambdaStack } from "../lib/UploadHandlerLambdaStack"; +import { DataTablesHandlerLambdaStack } from "../lib/DataTablesHandlerLambdaStack"; import { SQSStack } from "../lib/SQSStack"; import { SecretsStack } from "../lib/SecretsStack"; import { OverlayWorkerLambdaStack } from "../lib/OverlayWorkerLambdaStack"; @@ -174,6 +175,17 @@ const uploadHandler = new UploadHandlerLambdaStack( const secrets = new SecretsStack(app, "SeaSketchSecrets", { env }); +const dataTablesHandler = new DataTablesHandlerLambdaStack( + app, + "DataTablesHandler", + { + env, + vpc: db.vpc, + db: db.instance, + bucket: dataUploads.uploadsBucket, + } +); + const sqs = new SQSStack(app, "SeaSketchSQS", { env, }); @@ -228,6 +240,7 @@ new GraphQLStack(app, "SeaSketchGraphQLServer", { overlayWorkerArn: overlayWorker.fn.functionArn, normalizedOutputsBucket: dataUploads.normalizedUploadsBucket, uploadHandler: uploadHandler.fn, + dataTablesHandler: dataTablesHandler.fn, subdivisionWorkerLambdaArn: subdivideWorker.fn.functionArn, fragmentWorkerLambdaArn: fragmentWorker.fn.functionArn, geostatsPiiClassifierLambdaArn: piiClassifier.fn.functionArn, diff --git a/packages/infra/lib/DataTablesHandlerLambdaStack.ts b/packages/infra/lib/DataTablesHandlerLambdaStack.ts new file mode 100644 index 000000000..64b443906 --- /dev/null +++ b/packages/infra/lib/DataTablesHandlerLambdaStack.ts @@ -0,0 +1,76 @@ +/** + * This stack defines the lambda which processes overlay data table uploads. + */ +import * as cdk from "aws-cdk-lib"; +import * as ec2 from "aws-cdk-lib/aws-ec2"; +import * as iam from "aws-cdk-lib/aws-iam"; +import * as lambda from "aws-cdk-lib/aws-lambda"; +import * as logs from "aws-cdk-lib/aws-logs"; +import * as rds from "aws-cdk-lib/aws-rds"; +import * as s3 from "aws-cdk-lib/aws-s3"; +import { Platform } from "aws-cdk-lib/aws-ecr-assets"; +import * as path from "path"; +import { Construct } from "constructs"; + +export class DataTablesHandlerLambdaStack extends cdk.Stack { + fn: lambda.DockerImageFunction; + constructor( + scope: Construct, + id: string, + props: cdk.StackProps & { + vpc: ec2.Vpc; + db: rds.DatabaseInstance; + bucket: s3.Bucket; + }, + ) { + super(scope, id, props); + if (!process.env.TILES_REMOTE) { + throw new Error("TILES_REMOTE must be set in environment"); + } + + const fn = new lambda.DockerImageFunction(this, "DataTablesHandler", { + functionName: "DataTablesHandler", + vpc: props.vpc, + architecture: lambda.Architecture.X86_64, + code: lambda.DockerImageCode.fromImageAsset( + path.join(__dirname, "../.."), + { + file: "data-tables-handler/Dockerfile", + platform: Platform.LINUX_AMD64, + }, + ), + timeout: cdk.Duration.minutes(15), + logGroup: new logs.LogGroup(this, "DataTablesHandlerLogs", { + retention: logs.RetentionDays.ONE_MONTH, + }), + memorySize: 2048, + environment: { + NODE_ENV: "production", + BUCKET: props.bucket.bucketName, + TILES_REMOTE: process.env.TILES_REMOTE, + R2_ENDPOINT: process.env.R2_ENDPOINT!, + R2_ACCESS_KEY_ID: process.env.R2_ACCESS_KEY_ID!, + R2_SECRET_ACCESS_KEY: process.env.R2_SECRET_ACCESS_KEY!, + R2_TILES_BUCKET: process.env.R2_TILES_BUCKET!, + AWS_REGION: props.bucket.env.region, + PGHOST: props.db.instanceEndpoint.hostname, + PGUSER: "graphile", + PGREGION: props.db.env.region, + PGPORT: "5432", + PGDATABASE: "seasketch", + }, + }); + + props.bucket.grantRead(fn); + fn.addToRolePolicy( + new iam.PolicyStatement({ + actions: ["rds-db:connect"], + resources: [ + `arn:aws:rds-db:${props.db.env.region}:${props.db.env.account}:dbuser:${props.db.instanceResourceId}/graphile`, + ], + }), + ); + + this.fn = fn; + } +} diff --git a/packages/infra/lib/GraphQLStack.ts b/packages/infra/lib/GraphQLStack.ts index 8d54a8cba..d413fe8a2 100644 --- a/packages/infra/lib/GraphQLStack.ts +++ b/packages/infra/lib/GraphQLStack.ts @@ -39,6 +39,7 @@ export class GraphQLStack extends cdk.Stack { spatialUploadsHandlerArn: string; overlayWorkerArn: string; uploadHandler: lambda.DockerImageFunction; + dataTablesHandler: lambda.DockerImageFunction; subdivisionWorkerLambdaArn: string; fragmentWorkerLambdaArn: string; /** Same function the upload handler uses for PII scoring; warmed on createDataUpload. */ @@ -184,6 +185,7 @@ export class GraphQLStack extends cdk.Stack { GEOSTATS_PII_CLASSIFIER_ARN: props.geostatsPiiClassifierLambdaArn, OVERLAY_ENGINE_WORKER_SQS_QUEUE_URL: props.overlayEngineWorkerSqsQueue.queueUrl, + DATA_TABLES_HANDLER_LAMBDA_ARN: props.dataTablesHandler.functionArn, OVERLAY_ENGINE_ACCESS_TOKEN_SECRET_ARN: props.overlayEngineAccessTokenSecret.secretArn, }, @@ -294,6 +296,7 @@ export class GraphQLStack extends cdk.Stack { service.taskDefinition.taskRole ); props.uploadHandler.grantInvoke(service.taskDefinition.taskRole); + props.dataTablesHandler.grantInvoke(service.taskDefinition.taskRole); props.overlayEngineAccessTokenSecret.grantWrite( service.taskDefinition.taskRole, ); diff --git a/packages/overlay-worker/package-lock.json b/packages/overlay-worker/package-lock.json index b19e2cf0f..7d9f8dad8 100644 --- a/packages/overlay-worker/package-lock.json +++ b/packages/overlay-worker/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "BSD-3-Clause", "dependencies": { + "@aws-sdk/client-secrets-manager": "^3.873.0", "@aws-sdk/client-sqs": "^3.873.0", "@turf/along": "^7.3.0", "@turf/area": "^7.2.0", @@ -30,8 +31,7 @@ "polyclip-ts": "^0.16.8", "proj4": "^2.7.0", "transform-coordinates": "^1.0.0", - "undici": "^7.16.0", - "@aws-sdk/client-secrets-manager": "^3.873.0" + "undici": "^7.16.0" }, "devDependencies": { "@types/aws-lambda": "^8.10.119", @@ -221,6 +221,57 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/client-secrets-manager": { + "version": "3.873.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-secrets-manager/-/client-secrets-manager-3.873.0.tgz", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.873.0", + "@aws-sdk/credential-provider-node": "3.873.0", + "@aws-sdk/middleware-host-header": "3.873.0", + "@aws-sdk/middleware-logger": "3.873.0", + "@aws-sdk/middleware-recursion-detection": "3.873.0", + "@aws-sdk/middleware-user-agent": "3.873.0", + "@aws-sdk/region-config-resolver": "3.873.0", + "@aws-sdk/types": "3.862.0", + "@aws-sdk/util-endpoints": "3.873.0", + "@aws-sdk/util-user-agent-browser": "3.873.0", + "@aws-sdk/util-user-agent-node": "3.873.0", + "@smithy/config-resolver": "^4.1.5", + "@smithy/core": "^3.8.0", + "@smithy/fetch-http-handler": "^5.1.1", + "@smithy/hash-node": "^4.0.5", + "@smithy/invalid-dependency": "^4.0.5", + "@smithy/middleware-content-length": "^4.0.5", + "@smithy/middleware-endpoint": "^4.1.18", + "@smithy/middleware-retry": "^4.1.19", + "@smithy/middleware-serde": "^4.0.9", + "@smithy/middleware-stack": "^4.0.5", + "@smithy/node-config-provider": "^4.1.4", + "@smithy/node-http-handler": "^4.1.1", + "@smithy/protocol-http": "^5.1.3", + "@smithy/smithy-client": "^4.4.10", + "@smithy/types": "^4.3.2", + "@smithy/url-parser": "^4.0.5", + "@smithy/util-base64": "^4.0.0", + "@smithy/util-body-length-browser": "^4.0.0", + "@smithy/util-body-length-node": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.26", + "@smithy/util-defaults-mode-node": "^4.0.26", + "@smithy/util-endpoints": "^3.0.7", + "@smithy/util-middleware": "^4.0.5", + "@smithy/util-retry": "^4.0.7", + "@smithy/util-utf8": "^4.0.0", + "@types/uuid": "^9.0.1", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/client-sqs": { "version": "3.873.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sqs/-/client-sqs-3.873.0.tgz", @@ -2844,57 +2895,6 @@ "resolved": "https://registry.npmjs.org/wkt-parser/-/wkt-parser-1.5.5.tgz", "integrity": "sha512-/zMYi94/7D7fxcOSlVmWn6vnOMj3Gq5d1xvVjaYOS9n6h0qOJ4I7YYVxBWYcH1vq9+suhqzXkn05Yx47zQNUIA==", "license": "MIT" - }, - "node_modules/@aws-sdk/client-secrets-manager": { - "version": "3.873.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-secrets-manager/-/client-secrets-manager-3.873.0.tgz", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "3.873.0", - "@aws-sdk/credential-provider-node": "3.873.0", - "@aws-sdk/middleware-host-header": "3.873.0", - "@aws-sdk/middleware-logger": "3.873.0", - "@aws-sdk/middleware-recursion-detection": "3.873.0", - "@aws-sdk/middleware-user-agent": "3.873.0", - "@aws-sdk/region-config-resolver": "3.873.0", - "@aws-sdk/types": "3.862.0", - "@aws-sdk/util-endpoints": "3.873.0", - "@aws-sdk/util-user-agent-browser": "3.873.0", - "@aws-sdk/util-user-agent-node": "3.873.0", - "@smithy/config-resolver": "^4.1.5", - "@smithy/core": "^3.8.0", - "@smithy/fetch-http-handler": "^5.1.1", - "@smithy/hash-node": "^4.0.5", - "@smithy/invalid-dependency": "^4.0.5", - "@smithy/middleware-content-length": "^4.0.5", - "@smithy/middleware-endpoint": "^4.1.18", - "@smithy/middleware-retry": "^4.1.19", - "@smithy/middleware-serde": "^4.0.9", - "@smithy/middleware-stack": "^4.0.5", - "@smithy/node-config-provider": "^4.1.4", - "@smithy/node-http-handler": "^4.1.1", - "@smithy/protocol-http": "^5.1.3", - "@smithy/smithy-client": "^4.4.10", - "@smithy/types": "^4.3.2", - "@smithy/url-parser": "^4.0.5", - "@smithy/util-base64": "^4.0.0", - "@smithy/util-body-length-browser": "^4.0.0", - "@smithy/util-body-length-node": "^4.0.0", - "@smithy/util-defaults-mode-browser": "^4.0.26", - "@smithy/util-defaults-mode-node": "^4.0.26", - "@smithy/util-endpoints": "^3.0.7", - "@smithy/util-middleware": "^4.0.5", - "@smithy/util-retry": "^4.0.7", - "@smithy/util-utf8": "^4.0.0", - "@types/uuid": "^9.0.1", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=18.0.0" - } } } } diff --git a/packages/pmtiles-server/README.md b/packages/pmtiles-server/README.md index f2764a88b..b9b7c7b8b 100644 --- a/packages/pmtiles-server/README.md +++ b/packages/pmtiles-server/README.md @@ -3,17 +3,19 @@ Cloudflare Worker for serving SeaSketch overlay data from the `ssn-tiles` R2 bucket. -The Worker has three responsibilities: +The Worker has four responsibilities: 1. PMTiles-backed TileJSON, ZXY tiles, and browser previews. 2. Streaming whole-object downloads and efficient byte-range reads for FlatGeobuf, cloud-optimized GeoTIFF, PMTiles, and arbitrary object types. 3. FlatGeobuf property extraction through the legacy-compatible `/properties` API. +4. Overlay data-table aggregations (`…/dataTables/{uploadId}/query`) via + hyparquet, sharing the parent layer's published-UUID ACL. The default entrypoint is an uncached authorization and host-routing gateway. -It invokes isolated `TilesBackend`, `ObjectBackend`, and `PropertiesBackend` -entrypoints after credentials have been removed. +It invokes isolated `TilesBackend`, `ObjectBackend`, `PropertiesBackend`, and +`DataTablesBackend` entrypoints after credentials have been removed. ## Routes @@ -50,6 +52,156 @@ Consequently, a project `.json` path is raw JSON on the uploads host but TileJSON on the tiles host. An extensionless published UUID is raw on uploads and a preview on tiles. +### Data table queries + +Overlay monitoring tables are stored under the parent layer UUID: + +```text +projects/{slug}/public/{uuid}/dataTables/{uploadId}/data.parquet +projects/{slug}/public/{uuid}/dataTables/{uploadId}/column-stats.json +GET /projects/{slug}/public/{uuid}/dataTables/{uploadId}/query +``` + +Static parquet and `column-stats.json` are served by `ObjectBackend` (typically +on `uploads.seasketch.org`). The `/query` path is handled by +`DataTablesBackend` (hyparquet plan + execute). Paths classify as `published` +for the parent `{uuid}`, so map-access tokens and ACL docs apply the same way +as for tiles. + +#### Query API + +All queries are **GET** requests with parameters in the query string. There is +no request body. Parameter order does not matter; identical queries are +canonicalized for caching. When ACL enforcement is on, pass `access_token` +(and optional `ns`) like any other published overlay asset — credentials are +stripped before the backend runs. + +Response format depends only on the URL — the `Accept` header is ignored so +that cached responses can never be served to the wrong client type: + +| Request | Response | +| ------- | -------- | +| default (no `f`), or `f=json` | JSON query results | +| `f=html` | Interactive query builder UI | + +The HTML UI loads sibling `column-stats.json` from the same origin to populate +column pickers. + +**Built-in parameters** + +| Param | Required | Description | +| ----- | -------- | ----------- | +| `f` | No | `json` (default) or `html`. | +| `groupBy` | When aggregating | Comma-separated columns, e.g. `site` or `site,year`. Requires at least one `op`. | +| `op` | When `groupBy` is set | Comma-separated: `count`, `sum`, `mean`, `min`, `max`, `median`. Multiple ops share one `column` (except bare `count`). | +| `column` | When any `op` other than `count` alone | Numeric column to aggregate, e.g. `column=count`. | +| `orderBy` | No | Sort key with optional direction: `mean:desc`, `site` (asc). Valid keys are **groupBy columns** and **aggregation names** from `op`. | +| `limit` | No | Max groups or raw rows. `1`–`100000`. Aggregated queries default to no limit; raw-row queries default to `10000`. | +| `offset` | No | Skip first N groups/rows after sorting (default `0`). | + +**Query modes** + +1. **Aggregated** — `groupBy` + `op` (+ `column` when needed). Returns a + `groups` array. Each object has the group key column(s) plus one property + per aggregation (`mean`, `count`, etc.). +2. **Raw rows** — omit `groupBy` and `op`. Returns a `rows` array of matching + parquet records (all columns, subject to filters). Useful for inspection, + not typical map joins. + +**Aggregation semantics** + +| `op` | Behavior | +| ---- | -------- | +| `count` | With `column`: count non-null values in that column per group (`COUNT(col)`). Without `column`: count matching rows per group (`COUNT(*)`). | +| `sum`, `mean` | Over non-null values in `column`. Groups with no non-null values return `null` for these ops. | +| `min`, `max` | Extremes over non-null values in `column`. | +| `median` | Median of non-null values in `column` (even-length groups average the two middle values). | + +When multiple aggregations are requested (`op=mean,count&column=count`), each +group includes all of them, e.g. +`{ "site": "PINOS", "mean": 12.4, "count": 87 }`. + +**Column filters (`q.` prefix)** + +Filters restrict which parquet rows participate before aggregation or raw +output. Parameter names are `q.{columnName}`; operators are embedded in the +value (PostgREST-like): + +| URL value | Meaning | Column types | +| --------- | ------- | ------------ | +| `{value}` or `eq.{value}` | equals | all | +| `neq.{value}` | not equals | all | +| `gt.{n}`, `gte.{n}`, `lt.{n}`, `lte.{n}` | comparisons | number, timestamp | +| `in.(a,b,"Smith, John")` | value in list | all; quote items that contain commas; `""` escapes `"` | +| `is.null` | column is null | all | +| `not.null` | column is not null | all | + +Examples: + +```text +q.year=2018 +q.year=eq.2018 +q.count=gte.5 +q.observer=in.(CHAD BURT,BRAD BURT) +q.size=is.null +q.year=gte.2010&q.year=lte.2018 +``` + +Rules: + +- Unknown column names → `400` with + `{ error, validColumns: [{ name, type }] }`. +- Type-invalid operators (e.g. `gt` on a string) → `400`. +- Timestamp filters accept ISO 8601; comparisons use epoch milliseconds. +- Boolean filters accept `true`/`false` or `1`/`0`. +- There is no `not.in`; use `neq` or multiple filters instead. + +Implementation: `src/dataTables/params.ts`. + +**JSON response** + +```json +{ + "table": "projects/california/public/{uuid}/dataTables/{uploadId}", + "totalRows": 353253, + "rowsScanned": 353253, + "rowsMatched": 2328, + "rowGroups": { "total": 3, "scanned": 3 }, + "timing": { "metadataMs": 209, "executeMs": 230, "totalMs": 439 }, + "groups": [ + { "site": "PINOS", "mean": 1, "count": 1 } + ] +} +``` + +| Field | Description | +| ----- | ----------- | +| `table` | R2 prefix queried | +| `totalRows` | Rows in the parquet file | +| `rowsScanned` | Rows read from pruned row groups | +| `rowsMatched` | Rows passing all filters | +| `rowGroups.total` / `rowGroups.scanned` | Parquet row-group pruning stats | +| `timing` | Server-side phase timings (ms) | +| `groups` | Present when `groupBy` + `op` were requested | +| `rows` | Present for raw-row mode (no aggregation) | + +Responses include `ETag`, `Cache-Control`, and `Vary: Accept`. + +**Example (thematic map join)** + +```text +GET /projects/california/public/{uuid}/dataTables/{uploadId}/query + ?groupBy=site&op=mean,count&column=count + &q.year=2018&q.classcode=PYCHEL&f=json + &access_token=… +``` + +Map clients typically join `groups[]` to vector features using the data +table's join column (see `column-stats.json` → `join`). Structured UI +settings compile to these params via +`packages/client/src/dataLayers/dataTableQueryApi.ts` +(`buildDataTableQuerySearchParams`). + ### Properties ```text @@ -83,6 +235,8 @@ backend invocation and never enter a cache key. Keys outside `projects/` are public fixtures. For example, `/eez-land-joined.fgb` can be read without a token. Data-library keys below `projects/superuser/public/` are also always public on every host. +ACL documents (`acl/{ns}/projects/{slug}.json`) are stored in the same R2 +bucket for Worker-side authorization but are never served over HTTP. ### Map-access tokens diff --git a/packages/pmtiles-server/package.json b/packages/pmtiles-server/package.json index 061543550..729337f59 100644 --- a/packages/pmtiles-server/package.json +++ b/packages/pmtiles-server/package.json @@ -21,6 +21,7 @@ "dependencies": { "flatbuffers": "^25.9.23", "flatgeobuf": "^4.4.0", + "hyparquet": "^1.26.2", "jose": "^5.10.0", "pmtiles": "^4.4.1" } diff --git a/packages/pmtiles-server/src/dataTables/engine/asyncBuffer.ts b/packages/pmtiles-server/src/dataTables/engine/asyncBuffer.ts new file mode 100644 index 000000000..0c6f12e11 --- /dev/null +++ b/packages/pmtiles-server/src/dataTables/engine/asyncBuffer.ts @@ -0,0 +1,82 @@ +import { AsyncBuffer, cachedAsyncBuffer } from "hyparquet"; +import { ByteBudgetCache, createBlockReader } from "./blockReader"; + +/** Raw parquet bytes, keyed by file version. Persists for the isolate lifetime. */ +const BLOCK_MEMORY_BUDGET = 32 * 1024 * 1024; +const blockMemory = new ByteBudgetCache(BLOCK_MEMORY_BUDGET); + +interface FileStat { + byteLength: number; + etag: string; +} + +/** Avoid a repeated R2 head() within a warm isolate. Table paths are + * versioned, so etag/byteLength never change for a given key. */ +const statCache = new Map(); + +async function getFileStat( + bucket: R2Bucket, + key: string +): Promise { + const cached = statCache.get(key); + if (cached) { + return cached; + } + + const head = await bucket.head(key); + if (!head) { + return null; + } + const stat: FileStat = { byteLength: head.size, etag: head.httpEtag }; + if (statCache.size >= 500) { + statCache.clear(); + } + statCache.set(key, stat); + return stat; +} + +export interface R2FileSource { + buffer: AsyncBuffer; + byteLength: number; + etag: string; +} + +export interface OpenFileOptions { + bucket: R2Bucket; + key: string; +} + +/** + * Opens a parquet file as a hyparquet AsyncBuffer. Byte-range reads are + * block-aligned and cached in isolate memory (keyed by etag), so warm + * isolates serve repeat queries without contacting R2. + */ +export async function openR2File( + options: OpenFileOptions +): Promise { + const { bucket, key } = options; + + const stat = await getFileStat(bucket, key); + if (!stat) { + return null; + } + const { byteLength, etag } = stat; + + const slice = createBlockReader({ + byteLength, + cacheId: `${key}@${etag}`, + memory: blockMemory, + fetchRange: async (start, end) => { + const body = await bucket.get(key, { + range: { offset: start, length: end - start }, + }); + if (!body) { + throw new Error(`Range request to R2 for ${key} returned no body`); + } + return await body.arrayBuffer(); + }, + }); + + const buffer = cachedAsyncBuffer({ byteLength, slice }); + return { buffer, byteLength, etag }; +} diff --git a/packages/pmtiles-server/src/dataTables/engine/blockReader.ts b/packages/pmtiles-server/src/dataTables/engine/blockReader.ts new file mode 100644 index 000000000..065ffdadd --- /dev/null +++ b/packages/pmtiles-server/src/dataTables/engine/blockReader.ts @@ -0,0 +1,139 @@ +/** + * Block-aligned range reader with an isolate-global in-memory cache. + * + * Parquet reads many small, overlapping byte ranges. Rounding reads up to + * fixed-size blocks and caching those blocks in isolate memory means a second + * query against the same table (even with different filters) reuses raw bytes + * without another R2 round trip, as long as the Worker isolate is still warm. + */ + +export const DEFAULT_BLOCK_SIZE = 512 * 1024; + +/** Simple LRU keyed by string with a total-bytes budget. */ +export class ByteBudgetCache { + private map = new Map(); + private bytes = 0; + + constructor(private maxBytes: number) {} + + get(key: string): T | undefined { + const entry = this.map.get(key); + if (entry !== undefined) { + this.map.delete(key); + this.map.set(key, entry); + } + return entry?.value; + } + + set(key: string, value: T, bytes?: number): void { + const size = bytes ?? (value as unknown as ArrayBuffer).byteLength; + const existing = this.map.get(key); + if (existing) { + this.bytes -= existing.bytes; + this.map.delete(key); + } + this.map.set(key, { value, bytes: size }); + this.bytes += size; + for (const [oldKey, oldEntry] of this.map) { + if (this.bytes <= this.maxBytes) { + break; + } + this.map.delete(oldKey); + this.bytes -= oldEntry.bytes; + } + } + + get totalBytes(): number { + return this.bytes; + } +} + +/** Groups sorted block indices into contiguous [first, last] runs. */ +export function coalesceRuns(indices: number[]): Array<[number, number]> { + const runs: Array<[number, number]> = []; + for (const index of indices) { + const last = runs[runs.length - 1]; + if (last && index === last[1] + 1) { + last[1] = index; + } else { + runs.push([index, index]); + } + } + return runs; +} + +export interface BlockReaderOptions { + byteLength: number; + blockSize?: number; + /** Unique per file version (e.g. `${key}@${etag}`). */ + cacheId: string; + memory: ByteBudgetCache; + fetchRange(start: number, end: number): Promise; +} + +export function createBlockReader(options: BlockReaderOptions) { + const { + byteLength, + blockSize = DEFAULT_BLOCK_SIZE, + cacheId, + memory, + fetchRange, + } = options; + + const memoryKey = (index: number) => `${cacheId}#${blockSize}#${index}`; + + return async (start: number, end?: number): Promise => { + const rangeEnd = Math.min(end === undefined ? byteLength : end, byteLength); + if (rangeEnd <= start) { + return new ArrayBuffer(0); + } + + const firstBlock = Math.floor(start / blockSize); + const lastBlock = Math.floor((rangeEnd - 1) / blockSize); + const blocks = new Map(); + let missing: number[] = []; + + for (let i = firstBlock; i <= lastBlock; i++) { + const cached = memory.get(memoryKey(i)); + if (cached) { + blocks.set(i, cached); + } else { + missing.push(i); + } + } + + if (missing.length > 0) { + await Promise.all( + coalesceRuns(missing).map(async ([runFirst, runLast]) => { + const runStart = runFirst * blockSize; + const runEnd = Math.min((runLast + 1) * blockSize, byteLength); + const data = await fetchRange(runStart, runEnd); + for (let i = runFirst; i <= runLast; i++) { + const offset = (i - runFirst) * blockSize; + const block = data.slice( + offset, + Math.min(offset + blockSize, data.byteLength) + ); + blocks.set(i, block); + memory.set(memoryKey(i), block); + } + }) + ); + } + + const out = new Uint8Array(rangeEnd - start); + for (let i = firstBlock; i <= lastBlock; i++) { + const block = blocks.get(i)!; + const blockStart = i * blockSize; + const from = Math.max(start, blockStart); + const to = Math.min(rangeEnd, blockStart + block.byteLength); + if (to > from) { + out.set( + new Uint8Array(block, from - blockStart, to - from), + from - start + ); + } + } + return out.buffer; + }; +} diff --git a/packages/pmtiles-server/src/dataTables/engine/execute.ts b/packages/pmtiles-server/src/dataTables/engine/execute.ts new file mode 100644 index 000000000..aadeb6bfe --- /dev/null +++ b/packages/pmtiles-server/src/dataTables/engine/execute.ts @@ -0,0 +1,372 @@ +import { AsyncBuffer, FileMetaData, parquetReadObjects } from "hyparquet"; +import { parquetReadColumn } from "hyparquet/src/read.js"; +import { Aggregation, ParsedQuery, QueryError } from "../params"; +import { ByteBudgetCache } from "./blockReader"; +import { + ColumnKind, + CompiledFilter, + QueryPlan, + normalizeValue, +} from "./plan"; + +/** + * Decoded column arrays for a warm isolate. Parquet decode is CPU-heavy; + * caching by file version + column + row span lets filter sweeps (changing + * species/year against the same columns) skip decode on repeat queries. + * Identical queries should hit HTTP/Workers Cache instead and never reach here. + */ +const DECODED_COLUMN_BUDGET = 48 * 1024 * 1024; +const decodedColumns = new ByteBudgetCache(DECODED_COLUMN_BUDGET); + +function estimateDecodedBytes(data: unknown[]): number { + let bytes = 0; + for (const value of data) { + bytes += typeof value === "string" ? 16 + value.length * 2 : 16; + } + return bytes; +} + +export interface QueryResult { + /** Present for raw (non-aggregated) queries */ + rows?: Record[]; + /** Present for aggregated queries */ + groups?: Record[]; + rowsScanned: number; + rowsMatched: number; +} + +type Row = Record; +type Primitive = string | number | boolean | null; + +function matchesFilter(raw: unknown, filter: CompiledFilter): boolean { + const value = normalizeValue(raw, filter.kind); + switch (filter.op) { + case "isNull": + return value === null; + case "notNull": + return value !== null; + case "eq": + return value !== null && value === filter.value; + case "neq": + return value !== null && value !== filter.value; + case "in": + return value !== null && filter.values!.includes(value); + case "gt": + return value !== null && value > filter.value!; + case "gte": + return value !== null && value >= filter.value!; + case "lt": + return value !== null && value < filter.value!; + case "lte": + return value !== null && value <= filter.value!; + } +} + +function makePredicate(filters: CompiledFilter[]): (row: Row) => boolean { + if (filters.length === 0) { + return () => true; + } + return (row: Row) => { + for (const filter of filters) { + if (!matchesFilter(row[filter.column], filter)) return false; + } + return true; + }; +} + +/** Converts values to JSON-serializable output (BigInt, Date handling). */ +function jsonValue(value: unknown): unknown { + if (typeof value === "bigint") { + return Number(value); + } + if (value instanceof Date) { + return value.toISOString(); + } + if (value instanceof Uint8Array) { + return new TextDecoder().decode(value); + } + if (value === undefined) { + return null; + } + return value; +} + +function jsonRow(row: Row): Record { + const out: Record = {}; + for (const key of Object.keys(row)) { + out[key] = jsonValue(row[key]); + } + return out; +} + +interface GroupAccumulator { + keyValues: unknown[]; + rowCount: number; + valueCount: number; + sum: number; + min: Primitive; + max: Primitive; + /** buffered values for median */ + values?: number[]; +} + +function compareValues(a: unknown, b: unknown): number { + if (a === null || a === undefined) return b === null || b === undefined ? 0 : 1; + if (b === null || b === undefined) return -1; + if (typeof a === "number" && typeof b === "number") return a - b; + return String(a) < String(b) ? -1 : String(a) > String(b) ? 1 : 0; +} + +function median(values: number[]): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 + ? sorted[mid] + : (sorted[mid - 1] + sorted[mid]) / 2; +} + +function sortAndPage>( + items: T[], + orderBy: ParsedQuery["orderBy"], + offset: number, + limit: number | null, + validKeys: (key: string) => boolean +): T[] { + if (orderBy) { + if (!validKeys(orderBy.key)) { + throw new QueryError( + `orderBy key "${orderBy.key}" is not present in the output.` + ); + } + const dir = orderBy.direction === "desc" ? -1 : 1; + items.sort((a, b) => dir * compareValues(a[orderBy.key], b[orderBy.key])); + } + return limit === null + ? items.slice(offset) + : items.slice(offset, offset + limit); +} + +export async function executeQuery(options: { + file: AsyncBuffer; + metadata: FileMetaData; + query: ParsedQuery; + plan: QueryPlan; + /** Unique per file version (e.g. the object etag). Enables the decoded + * column cache; omit for one-shot reads. */ + cacheKey?: string; +}): Promise { + const { file, metadata, query, plan, cacheKey } = options; + const predicate = makePredicate(plan.filters); + + if (query.ops.length === 0) { + return await executeRawQuery(options, predicate); + } + + const aggColumn = query.column; + const aggKind: ColumnKind | undefined = aggColumn + ? plan.columns.get(aggColumn)?.kind + : undefined; + const needsMedian = query.ops.includes("median"); + const groups = new Map(); + let rowsScanned = 0; + let rowsMatched = 0; + + // Columnar execution: evaluate filters against just the filter columns to + // produce matched row indices, then read the remaining columns only for + // spans with matches. Avoids materializing an object per scanned row. + const readColumn = async ( + name: string, + span: { rowStart: number; rowEnd: number } + ): Promise => { + const key = cacheKey + ? `${cacheKey}#${name}#${span.rowStart}-${span.rowEnd}` + : null; + if (key) { + const cached = decodedColumns.get(key); + if (cached) return cached; + } + const data = (await parquetReadColumn({ + file, + metadata, + columns: [name], + rowStart: span.rowStart, + rowEnd: span.rowEnd, + })) as unknown[]; + if (key) { + decodedColumns.set(key, data, estimateDecodedBytes(data)); + } + return data; + }; + + for (const span of plan.spans) { + const spanRows = span.rowEnd - span.rowStart; + rowsScanned += spanRows; + + const filterColumns = new Map(); + await Promise.all( + [...new Set(plan.filters.map((f) => f.column))].map(async (name) => { + filterColumns.set(name, await readColumn(name, span)); + }) + ); + + const matched: number[] = []; + for (let i = 0; i < spanRows; i++) { + let ok = true; + for (const filter of plan.filters) { + if (!matchesFilter(filterColumns.get(filter.column)![i], filter)) { + ok = false; + break; + } + } + if (ok) matched.push(i); + } + rowsMatched += matched.length; + if (matched.length === 0) continue; + + const otherColumns = new Set(query.groupBy); + if (aggColumn) otherColumns.add(aggColumn); + const columnData = new Map(filterColumns); + await Promise.all( + [...otherColumns] + .filter((name) => !columnData.has(name)) + .map(async (name) => { + columnData.set(name, await readColumn(name, span)); + }) + ); + + const groupByData = query.groupBy.map((col) => columnData.get(col)!); + const aggData = aggColumn ? columnData.get(aggColumn)! : null; + + for (const i of matched) { + const keyValues = groupByData.map((data) => jsonValue(data[i])); + const key = JSON.stringify(keyValues); + let group = groups.get(key); + if (!group) { + group = { + keyValues, + rowCount: 0, + valueCount: 0, + sum: 0, + min: null, + max: null, + values: needsMedian ? [] : undefined, + }; + groups.set(key, group); + } + group.rowCount++; + + if (aggData) { + const value = normalizeValue(aggData[i], aggKind || "string"); + if (value !== null) { + group.valueCount++; + if (typeof value === "number") { + group.sum += value; + group.values?.push(value); + } + if (group.min === null || compareValues(value, group.min) < 0) { + group.min = value; + } + if (group.max === null || compareValues(value, group.max) > 0) { + group.max = value; + } + } + } + } + } + + const output: Record[] = []; + for (const group of groups.values()) { + const entry: Record = {}; + query.groupBy.forEach((col, i) => { + entry[col] = group.keyValues[i]; + }); + for (const op of query.ops) { + entry[op] = aggregateValue(op, group, aggColumn !== null); + } + output.push(entry); + } + + const validKeys = (key: string) => + query.groupBy.includes(key) || (query.ops as string[]).includes(key); + const paged = sortAndPage( + output, + query.orderBy, + query.offset, + query.limit, + validKeys + ); + + return { groups: paged, rowsScanned, rowsMatched }; +} + +function aggregateValue( + op: Aggregation, + group: GroupAccumulator, + hasColumn: boolean +): unknown { + switch (op) { + case "count": + // With an aggregation column, count non-null values (like SQL + // COUNT(col)); otherwise count matching rows (COUNT(*)). + return hasColumn ? group.valueCount : group.rowCount; + case "sum": + return group.valueCount > 0 ? group.sum : null; + case "mean": + return group.valueCount > 0 ? group.sum / group.valueCount : null; + case "min": + return group.min; + case "max": + return group.max; + case "median": + return median(group.values || []); + } +} + +async function executeRawQuery( + options: { + file: AsyncBuffer; + metadata: FileMetaData; + query: ParsedQuery; + plan: QueryPlan; + }, + predicate: (row: Row) => boolean +): Promise { + const { file, metadata, query, plan } = options; + const matched: Record[] = []; + let rowsScanned = 0; + let rowsMatched = 0; + // Without an orderBy we can stop reading as soon as the page is filled. + const target = query.orderBy + ? Infinity + : query.offset + (query.limit ?? Infinity); + + for (const span of plan.spans) { + const rows = (await parquetReadObjects({ + file, + metadata, + columns: plan.neededColumns, + rowStart: span.rowStart, + rowEnd: span.rowEnd, + })) as Row[]; + rowsScanned += rows.length; + for (const row of rows) { + if (!predicate(row)) continue; + rowsMatched++; + if (matched.length < target) { + matched.push(jsonRow(row)); + } + } + if (matched.length >= target) break; + } + + const columnNames = new Set(plan.columns.keys()); + const paged = sortAndPage( + matched, + query.orderBy, + query.offset, + query.limit, + (key) => columnNames.has(key) + ); + return { rows: paged, rowsScanned, rowsMatched }; +} diff --git a/packages/pmtiles-server/src/dataTables/engine/plan.ts b/packages/pmtiles-server/src/dataTables/engine/plan.ts new file mode 100644 index 000000000..833c373b5 --- /dev/null +++ b/packages/pmtiles-server/src/dataTables/engine/plan.ts @@ -0,0 +1,413 @@ +import { AsyncBuffer, FileMetaData, SchemaElement, parquetSchema } from "hyparquet"; +import { + hashParquetValue, + readBloomFilter, + sbbfContains, +} from "hyparquet/src/bloom.js"; +import { ParsedQuery, QueryError, RawFilter } from "../params"; + +export type ColumnKind = "string" | "number" | "boolean" | "timestamp"; + +export interface ColumnInfo { + name: string; + kind: ColumnKind; +} + +/** A filter with its value(s) coerced to the column's type. */ +export interface CompiledFilter extends Omit { + kind: ColumnKind; + value?: string | number | boolean; + values?: (string | number | boolean)[]; +} + +export interface ReadSpan { + /** Global row index (inclusive) */ + rowStart: number; + /** Global row index (exclusive) */ + rowEnd: number; +} + +export interface QueryPlan { + columns: Map; + filters: CompiledFilter[]; + /** Columns that must be read from the parquet file. undefined = all. */ + neededColumns: string[] | undefined; + spans: ReadSpan[]; + rowGroupsTotal: number; + rowGroupsScanned: number; + totalRows: number; +} + +function kindForSchemaElement(element: { + type?: string; + converted_type?: string; + logical_type?: { type?: string }; +}): ColumnKind { + const logical = element.logical_type?.type; + if (logical === "STRING" || element.converted_type === "UTF8") { + return "string"; + } + if ( + logical === "DATE" || + logical === "TIMESTAMP" || + element.converted_type === "DATE" || + element.converted_type === "TIMESTAMP_MILLIS" || + element.converted_type === "TIMESTAMP_MICROS" || + element.type === "INT96" + ) { + return "timestamp"; + } + if (element.type === "BOOLEAN") { + return "boolean"; + } + if ( + element.type === "INT32" || + element.type === "INT64" || + element.type === "FLOAT" || + element.type === "DOUBLE" || + element.converted_type === "DECIMAL" + ) { + return "number"; + } + // Unrecognized physical types (nested, byte arrays without UTF8, etc.) + // are treated as strings for equality purposes. + return "string"; +} + +export function columnsFromMetadata( + metadata: FileMetaData +): Map { + const columns = new Map(); + const schema = parquetSchema(metadata); + for (const child of schema.children) { + const name = child.element.name; + columns.set(name, { name, kind: kindForSchemaElement(child.element) }); + } + return columns; +} + +function unknownColumnError( + name: string, + columns: Map +): QueryError { + return new QueryError(`Unknown column "${name}".`, 400, { + validColumns: [...columns.values()].map((c) => ({ + name: c.name, + type: c.kind, + })), + }); +} + +function coerceValue( + raw: string, + column: ColumnInfo +): string | number | boolean { + switch (column.kind) { + case "number": { + const n = Number(raw); + if (raw.trim() === "" || Number.isNaN(n)) { + throw new QueryError( + `Invalid numeric value "${raw}" for column "${column.name}".` + ); + } + return n; + } + case "boolean": { + if (raw === "true" || raw === "1") return true; + if (raw === "false" || raw === "0") return false; + throw new QueryError( + `Invalid boolean value "${raw}" for column "${column.name}". Use true or false.` + ); + } + case "timestamp": { + const t = Date.parse(raw); + if (Number.isNaN(t)) { + throw new QueryError( + `Invalid date value "${raw}" for column "${column.name}". Use an ISO 8601 date.` + ); + } + // timestamps are compared as epoch millis + return t; + } + default: + return raw; + } +} + +const COMPARISON_OPS = new Set(["gt", "gte", "lt", "lte"]); + +export function compileFilters( + filters: RawFilter[], + columns: Map +): CompiledFilter[] { + return filters.map((filter) => { + const column = columns.get(filter.column); + if (!column) { + throw unknownColumnError(filter.column, columns); + } + if ( + COMPARISON_OPS.has(filter.op) && + column.kind !== "number" && + column.kind !== "timestamp" + ) { + throw new QueryError( + `Operator "${filter.op}" is only supported for numeric or date columns; "${column.name}" is a ${column.kind} column.` + ); + } + const compiled: CompiledFilter = { + column: filter.column, + op: filter.op, + kind: column.kind, + }; + if (filter.value !== undefined) { + compiled.value = coerceValue(filter.value, column); + } + if (filter.values !== undefined) { + compiled.values = filter.values.map((v) => coerceValue(v, column)); + } + return compiled; + }); +} + +/** + * Normalizes a runtime or statistics value into something directly comparable + * with compiled filter values (number | string | boolean), or null. + */ +export function normalizeValue( + value: unknown, + kind: ColumnKind +): string | number | boolean | null { + if (value === null || value === undefined) { + return null; + } + if (typeof value === "bigint") { + return Number(value); + } + if (value instanceof Date) { + return value.getTime(); + } + if (kind === "number" && typeof value !== "number") { + const n = Number(value); + return Number.isNaN(n) ? null : n; + } + if (kind === "timestamp" && typeof value === "number") { + return value; + } + if (value instanceof Uint8Array) { + return new TextDecoder().decode(value); + } + return value as string | number | boolean; +} + +/** + * Bloom-filter pruning for eq/in filters. Statistics min/max rarely help for + * high-cardinality string columns (e.g. species codes span the alphabet in + * every row group), but split-block bloom filters written by DuckDB prove a + * value's *absence*, letting us skip whole row groups. Filters are tiny + * (tens to hundreds of bytes) and live next to the footer, so reads are + * served from the cached tail blocks. + * + * Returns false if a bloom filter proves the row group cannot match. + */ +async function rowGroupPassesBloomFilters( + file: AsyncBuffer, + rowGroup: FileMetaData["row_groups"][number], + filters: CompiledFilter[], + schemaElements: Map +): Promise { + for (const filter of filters) { + if (filter.op !== "eq" && filter.op !== "in") continue; + const element = schemaElements.get(filter.column); + if (!element) continue; + const meta = rowGroup.columns.find( + (c) => c.meta_data?.path_in_schema.join(".") === filter.column + )?.meta_data; + if (!meta?.bloom_filter_offset || !meta.bloom_filter_length) continue; + + const values = filter.op === "eq" ? [filter.value!] : filter.values!; + const hashes = values.map((v) => hashParquetValue(v, element)); + // undefined = ambiguous type mapping; the bloom filter can't help. + if (hashes.some((h) => h === undefined)) continue; + + const start = Number(meta.bloom_filter_offset); + const buffer = await file.slice(start, start + meta.bloom_filter_length); + const bloom = readBloomFilter({ view: new DataView(buffer), offset: 0 }); + if (!bloom) continue; + if (hashes.every((h) => !sbbfContains(bloom.blocks, h!))) { + return false; + } + } + return true; +} + +/** + * Tests whether a row group could possibly contain rows matching the filter, + * based on column chunk statistics. Conservative: returns true when unsure. + */ +function rowGroupMayMatch( + filter: CompiledFilter, + stats: + | { + min_value?: unknown; + max_value?: unknown; + null_count?: bigint | number; + } + | undefined, + rowCount: number +): boolean { + if (!stats) return true; + const nullCount = + stats.null_count === undefined ? undefined : Number(stats.null_count); + + if (filter.op === "isNull") { + return nullCount === undefined || nullCount > 0; + } + if (filter.op === "notNull") { + return nullCount === undefined || nullCount < rowCount; + } + + // Timestamp statistics units vary by writer (millis/micros/DATE days). + // Filter values are epoch millis; only prune when stats decoded to Dates. + if ( + filter.kind === "timestamp" && + !(stats.min_value instanceof Date) && + !(stats.max_value instanceof Date) + ) { + return true; + } + const min = normalizeValue(stats.min_value, filter.kind); + const max = normalizeValue(stats.max_value, filter.kind); + if (min === null || max === null) return true; + + switch (filter.op) { + case "eq": + return filter.value! >= min && filter.value! <= max; + case "in": + return filter.values!.some((v) => v >= min && v <= max); + case "gt": + return max > filter.value!; + case "gte": + return max >= filter.value!; + case "lt": + return min < filter.value!; + case "lte": + return min <= filter.value!; + default: + // neq can only be pruned when min === max === value; rarely useful + return !( + filter.op === "neq" && + min === max && + min === filter.value + ); + } +} + +export async function planQuery( + metadata: FileMetaData, + query: ParsedQuery, + /** When provided, bloom filters are consulted to prune row groups that + * pass the min/max statistics check. */ + file?: AsyncBuffer +): Promise { + const columns = columnsFromMetadata(metadata); + + // Validate built-in column references + for (const name of query.groupBy) { + if (!columns.has(name)) { + throw unknownColumnError(name, columns); + } + } + if (query.column !== null) { + if (!columns.has(query.column)) { + throw unknownColumnError(query.column, columns); + } + const kind = columns.get(query.column)!.kind; + const numericOps = query.ops.filter( + (op) => op !== "count" && op !== "min" && op !== "max" + ); + if (numericOps.length > 0 && kind !== "number" && kind !== "timestamp") { + throw new QueryError( + `Aggregations ${numericOps.join(", ")} require a numeric column; "${ + query.column + }" is a ${kind} column.` + ); + } + } + + const filters = compileFilters(query.filters, columns); + + // Determine which columns need to be read + let neededColumns: string[] | undefined; + if (query.ops.length > 0) { + const needed = new Set(query.groupBy); + if (query.column) needed.add(query.column); + for (const f of filters) needed.add(f.column); + neededColumns = [...needed]; + } else { + // raw row output returns all columns + neededColumns = undefined; + } + + // Row group pruning, first via column chunk statistics (free -- already in + // the footer), then via bloom filters for surviving row groups. + const statsPass = metadata.row_groups.map((rowGroup) => { + const numRows = Number(rowGroup.num_rows); + if (numRows === 0) return false; + for (const filter of filters) { + const chunk = rowGroup.columns.find( + (c) => c.meta_data?.path_in_schema.join(".") === filter.column + ); + if (!chunk?.meta_data) continue; + if (!rowGroupMayMatch(filter, chunk.meta_data.statistics, numRows)) { + return false; + } + } + return true; + }); + + let mayMatch = statsPass; + const hasBloomEligibleFilter = filters.some( + (f) => f.op === "eq" || f.op === "in" + ); + if (file && hasBloomEligibleFilter && statsPass.some((p) => p)) { + const schemaElements = new Map(); + for (const child of parquetSchema(metadata).children) { + schemaElements.set(child.element.name, child.element); + } + mayMatch = await Promise.all( + metadata.row_groups.map((rowGroup, i) => + statsPass[i] + ? rowGroupPassesBloomFilters(file, rowGroup, filters, schemaElements) + : Promise.resolve(false) + ) + ); + } + + const spans: ReadSpan[] = []; + let rowStart = 0; + let scanned = 0; + metadata.row_groups.forEach((rowGroup, i) => { + const numRows = Number(rowGroup.num_rows); + if (mayMatch[i]) { + scanned++; + const last = spans[spans.length - 1]; + if (last && last.rowEnd === rowStart) { + // merge contiguous row groups into a single read + last.rowEnd = rowStart + numRows; + } else { + spans.push({ rowStart, rowEnd: rowStart + numRows }); + } + } + rowStart += numRows; + }); + + return { + columns, + filters, + neededColumns, + spans, + rowGroupsTotal: metadata.row_groups.length, + rowGroupsScanned: scanned, + totalRows: Number(metadata.num_rows), + }; +} diff --git a/packages/pmtiles-server/src/dataTables/params.ts b/packages/pmtiles-server/src/dataTables/params.ts new file mode 100644 index 000000000..d8e7b16b3 --- /dev/null +++ b/packages/pmtiles-server/src/dataTables/params.ts @@ -0,0 +1,291 @@ +/** + * Parses query string parameters for the /query endpoint. + * + * Built-in parameters use bare names (f, groupBy, op, column, limit, offset, + * orderBy). Column filters use a `q.` prefix with PostgREST-style operators + * embedded in the value: + * + * q.year=2018 equality (bare value) + * q.year=eq.2018 equality (explicit) + * q.count=gte.5 gt / gte / lt / lte / neq + * q.observer=in.(Chad Burt,Lyal B) IN list ("quoted" items may contain commas) + * q.size=is.null / q.size=not.null null tests + * + * Repeating the same q. param ANDs the conditions together. + */ + +export type OutputFormat = "json" | "html"; + +export const AGGREGATIONS = [ + "count", + "sum", + "mean", + "min", + "max", + "median", +] as const; +export type Aggregation = (typeof AGGREGATIONS)[number]; + +export type FilterOperator = + | "eq" + | "neq" + | "gt" + | "gte" + | "lt" + | "lte" + | "in" + | "isNull" + | "notNull"; + +/** A filter as parsed from the query string, before schema-aware coercion. */ +export interface RawFilter { + column: string; + op: FilterOperator; + /** Raw string value for eq/neq/gt/gte/lt/lte. */ + value?: string; + /** Raw string values for `in`. */ + values?: string[]; +} + +export interface ParsedQuery { + /** null means "negotiate via the Accept header" */ + format: OutputFormat | null; + groupBy: string[]; + ops: Aggregation[]; + /** Column to aggregate. Required for every op except count. */ + column: string | null; + limit: number | null; + offset: number; + orderBy: { key: string; direction: "asc" | "desc" } | null; + filters: RawFilter[]; +} + +/** Upper bound for `limit` and `offset`; keeps responses and sort buffers + * within Workers memory limits. */ +export const MAX_LIMIT = 100000; + +export class QueryError extends Error { + status: number; + details?: Record; + constructor( + message: string, + status = 400, + details?: Record + ) { + super(message); + this.status = status; + this.details = details; + } +} + + +const COMPARISON_PREFIX = /^(eq|neq|gt|gte|lt|lte)\.([\s\S]*)$/; + +/** + * Parses the inner portion of an `in.(...)` list. Items may be double-quoted + * to include commas or leading/trailing whitespace, e.g. + * `in.("Smith, John",UCSB)`. + */ +export function parseInList(inner: string): string[] { + const values: string[] = []; + let i = 0; + while (i < inner.length) { + // skip whitespace between items + while (i < inner.length && inner[i] === " ") i++; + if (inner[i] === '"') { + // quoted item; "" escapes a literal quote + let value = ""; + i++; + while (i < inner.length) { + if (inner[i] === '"' && inner[i + 1] === '"') { + value += '"'; + i += 2; + } else if (inner[i] === '"') { + i++; + break; + } else { + value += inner[i]; + i++; + } + } + values.push(value); + // skip to next comma + while (i < inner.length && inner[i] !== ",") i++; + i++; + } else { + let end = inner.indexOf(",", i); + if (end === -1) end = inner.length; + values.push(inner.slice(i, end).trim()); + i = end + 1; + } + } + return values.filter((v) => v.length > 0); +} + +function parseFilterParam(column: string, raw: string): RawFilter { + if (raw === "is.null") { + return { column, op: "isNull" }; + } + if (raw === "not.null") { + return { column, op: "notNull" }; + } + if (raw.startsWith("in.")) { + if (!raw.startsWith("in.(") || !raw.endsWith(")")) { + throw new QueryError( + `Malformed in list for column "${column}". Use in.(a,b,"c,d").` + ); + } + const values = parseInList(raw.slice(4, -1)); + if (values.length === 0) { + throw new QueryError( + `Empty in.() list for column "${column}". Provide at least one value.` + ); + } + return { column, op: "in", values }; + } + const match = COMPARISON_PREFIX.exec(raw); + if (match) { + return { + column, + op: match[1] as FilterOperator, + value: match[2], + }; + } + // bare value = equality + return { column, op: "eq", value: raw }; +} + +export function parseQueryParams(searchParams: URLSearchParams): ParsedQuery { + const filters: RawFilter[] = []; + for (const [key, value] of searchParams.entries()) { + if (key.startsWith("q.")) { + const column = key.slice(2); + if (!column) { + throw new QueryError(`Invalid filter parameter "${key}".`); + } + filters.push(parseFilterParam(column, value)); + } + } + + let format: OutputFormat | null = null; + const f = searchParams.get("f"); + if (f !== null) { + if (f !== "json" && f !== "html") { + throw new QueryError(`Invalid format "${f}". Use f=json or f=html.`); + } + format = f; + } + + const groupBy = (searchParams.get("groupBy") || "") + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + + const ops: Aggregation[] = []; + const opParam = searchParams.get("op"); + if (opParam !== null) { + for (const op of opParam.split(",").map((s) => s.trim())) { + if (!op) continue; + if (!(AGGREGATIONS as readonly string[]).includes(op)) { + throw new QueryError( + `Unknown aggregation "${op}". Supported: ${AGGREGATIONS.join(", ")}.` + ); + } + if (!ops.includes(op as Aggregation)) { + ops.push(op as Aggregation); + } + } + } + + const column = searchParams.get("column"); + const needsColumn = ops.filter((op) => op !== "count"); + if (needsColumn.length > 0 && !column) { + throw new QueryError( + `The "column" parameter is required for op=${needsColumn.join(",")}.` + ); + } + if (groupBy.length > 0 && ops.length === 0) { + throw new QueryError( + `groupBy requires at least one aggregation via the "op" parameter.` + ); + } + + let limit: number | null = null; + const limitParam = searchParams.get("limit"); + if (limitParam !== null && limitParam.trim() !== "") { + if (!/^\d+$/.test(limitParam.trim())) { + throw new QueryError( + `Invalid limit "${limitParam}". Must be a positive integer, or omit for no limit.` + ); + } + limit = parseInt(limitParam, 10); + if (limit < 1 || limit > MAX_LIMIT) { + throw new QueryError( + `Invalid limit "${limitParam}". Must be between 1 and ${MAX_LIMIT}.` + ); + } + } + + let offset = 0; + const offsetParam = searchParams.get("offset"); + if (offsetParam !== null) { + if (!/^\d+$/.test(offsetParam.trim())) { + throw new QueryError( + `Invalid offset "${offsetParam}". Must be a non-negative integer.` + ); + } + offset = parseInt(offsetParam, 10); + if (offset > MAX_LIMIT) { + throw new QueryError( + `Invalid offset "${offsetParam}". Must be at most ${MAX_LIMIT}.` + ); + } + } + + let orderBy: ParsedQuery["orderBy"] = null; + const orderByParam = searchParams.get("orderBy"); + if (orderByParam !== null && orderByParam.trim().length > 0) { + // Split on the last colon so keys containing ":" still parse. + const raw = orderByParam.trim(); + const colon = raw.lastIndexOf(":"); + let key = raw; + let direction: "asc" | "desc" = "asc"; + if (colon !== -1) { + const suffix = raw.slice(colon + 1); + if (suffix !== "asc" && suffix !== "desc") { + throw new QueryError( + `Invalid orderBy direction "${suffix}". Use :asc or :desc.` + ); + } + key = raw.slice(0, colon); + direction = suffix; + } + key = key.trim(); + if (!key) { + throw new QueryError(`orderBy requires a key, e.g. orderBy=mean:desc.`); + } + orderBy = { key, direction }; + } + + return { format, groupBy, ops, column, limit, offset, orderBy, filters }; +} + +/** + * Produces a canonical query string so that logically identical requests + * (parameter order, whitespace) share a single cache entry. + */ +export function canonicalQueryString(searchParams: URLSearchParams): string { + const entries: [string, string][] = []; + for (const [key, value] of searchParams.entries()) { + if (key === "f") continue; // format normalized separately + entries.push([key, value]); + } + entries.sort((a, b) => + a[0] === b[0] ? a[1].localeCompare(b[1]) : a[0].localeCompare(b[0]) + ); + const params = new URLSearchParams(); + for (const [key, value] of entries) { + params.append(key, value); + } + return params.toString(); +} diff --git a/packages/pmtiles-server/src/dataTables/ui/html.ts b/packages/pmtiles-server/src/dataTables/ui/html.ts new file mode 100644 index 000000000..f2d1d38e8 --- /dev/null +++ b/packages/pmtiles-server/src/dataTables/ui/html.ts @@ -0,0 +1,522 @@ +/** + * A self-contained demo UI for exercising the query engine. Served when the + * query endpoint is requested with `f=html`. The page loads sibling + * column-stats.json and runs queries against the same endpoint with `f=json`, + * forwarding `access_token` / `ns` from the page URL so ACL-protected tables + * work in the browser. + */ + +function escapeHtml(text: string): string { + return text + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + +export function queryUiHtml(tablePath: string): string { + const safePath = escapeHtml(tablePath); + // Escape "<" so the embedded string can never close the + +`; +} diff --git a/packages/pmtiles-server/src/dataTablesBackend.ts b/packages/pmtiles-server/src/dataTablesBackend.ts new file mode 100644 index 000000000..319d9c0ce --- /dev/null +++ b/packages/pmtiles-server/src/dataTablesBackend.ts @@ -0,0 +1,233 @@ +import { WorkerEntrypoint } from "cloudflare:workers"; +import { FileMetaData, parquetMetadataAsync } from "hyparquet"; +import { openR2File } from "./dataTables/engine/asyncBuffer"; +import { executeQuery } from "./dataTables/engine/execute"; +import { planQuery } from "./dataTables/engine/plan"; +import { + canonicalQueryString, + parseQueryParams, + QueryError, +} from "./dataTables/params"; +import { queryUiHtml } from "./dataTables/ui/html"; + +/** Browser cache lifetime for query JSON responses. */ +const BROWSER_MAX_AGE = 86400; +/** CDN/edge cache lifetime. Table paths are versioned per uploadId. */ +const EDGE_MAX_AGE = 604800; +/** Default page size for raw-row queries when the client omits `limit`; + * prevents accidentally materializing an entire table as JSON. */ +const RAW_DEFAULT_LIMIT = 10000; + +/** Parsed parquet footers are small and immutable per etag. */ +const MAX_METADATA_ENTRIES = 100; +const metadataCache = new Map(); + +/** + * Cached entrypoint: runs hyparquet aggregations over + * `projects/{slug}/public/{uuid}/dataTables/{uploadId}/data.parquet`. + * + * Auth is enforced by the default gateway before this entrypoint is invoked; + * data-table paths classify as `published` under the parent layer UUID. + */ +export class DataTablesBackend extends WorkerEntrypoint { + async fetch(request: Request): Promise { + return handleDataTableQuery(request, this.env); + } +} + +export async function handleDataTableQuery( + request: Request, + env: Env, +): Promise { + if (request.method !== "GET") { + return new Response("Method Not Allowed", { + status: 405, + headers: { Allow: "GET" }, + }); + } + + const url = new URL(request.url); + const pathname = url.pathname; + if (!pathname.endsWith("/query")) { + return jsonError("Not found. Query endpoint is {tablePath}/query", 404); + } + const tablePath = pathname.replace(/^\/+/, "").slice(0, -"/query".length); + if (!tablePath) { + return jsonError("Missing table path", 404); + } + + const requestStart = Date.now(); + const timer = new Timer(requestStart); + + let query; + try { + query = parseQueryParams(url.searchParams); + } catch (error) { + return errorResponse(error); + } + if (query.ops.length === 0 && query.limit === null) { + query = { ...query, limit: RAW_DEFAULT_LIMIT }; + } + timer.mark("parse"); + + // Format depends only on the URL (`f=html`), never the Accept header: the + // Workers cache key is path+query, so Accept-based negotiation would let a + // cached HTML response be served to JSON clients and vice versa. + if (query.format === "html") { + return new Response(queryUiHtml(tablePath), { + headers: { + "Content-Type": "text/html; charset=utf-8", + "Access-Control-Allow-Origin": "*", + "Cache-Control": "public, max-age=300", + }, + }); + } + + const canonicalQuery = canonicalQueryString(url.searchParams); + const started = Date.now(); + try { + const source = await openR2File({ + bucket: env.TILES_BUCKET, + key: `${tablePath}/data.parquet`, + }); + timer.mark("open"); + if (!source) { + throw new QueryError( + `No data table found at "${tablePath}/data.parquet".`, + 404, + ); + } + + const metadata = await getParquetMetadata(source); + timer.mark("metadata"); + const planned = Date.now(); + const plan = await planQuery(metadata, query, source.buffer); + timer.mark("plan"); + const result = await executeQuery({ + file: source.buffer, + metadata, + query, + plan, + cacheKey: `${tablePath}@${source.etag}`, + }); + timer.mark("execute"); + const finished = Date.now(); + + const body = { + table: tablePath, + totalRows: plan.totalRows, + rowsScanned: result.rowsScanned, + rowsMatched: result.rowsMatched, + rowGroups: { + total: plan.rowGroupsTotal, + scanned: plan.rowGroupsScanned, + }, + timing: { + metadataMs: planned - started, + executeMs: finished - planned, + totalMs: finished - started, + }, + ...(result.groups !== undefined + ? { groups: result.groups } + : { rows: result.rows }), + }; + timer.mark("serialize"); + + const etag = await makeETag(source.etag, canonicalQuery); + const headers = { + "Content-Type": "application/json; charset=utf-8", + ETag: etag, + "Server-Timing": timer.header(), + "Access-Control-Allow-Origin": "*", + "Timing-Allow-Origin": "*", + "Cache-Control": `public, max-age=${BROWSER_MAX_AGE}, s-maxage=${EDGE_MAX_AGE}, immutable`, + }; + if (request.headers.get("if-none-match") === etag) { + return new Response(null, { status: 304, headers }); + } + return new Response(JSON.stringify(body), { headers }); + } catch (error) { + return errorResponse(error); + } +} + +async function getParquetMetadata(source: { + buffer: Parameters[0]; + etag: string; +}): Promise { + const cached = metadataCache.get(source.etag); + if (cached) { + return cached; + } + const metadata = await parquetMetadataAsync(source.buffer); + if (metadataCache.size >= MAX_METADATA_ENTRIES) { + const oldest = metadataCache.keys().next().value; + if (oldest !== undefined) { + metadataCache.delete(oldest); + } + } + metadataCache.set(source.etag, metadata); + return metadata; +} + +async function makeETag(objectEtag: string, canonicalQuery: string) { + const data = new TextEncoder().encode(`${objectEtag}|${canonicalQuery}`); + const digest = await crypto.subtle.digest("SHA-1", data); + const hex = [...new Uint8Array(digest)] + .map((b) => b.toString(16).padStart(2, "0")) + .join(""); + return `"${hex}"`; +} + +class Timer { + private marks: Array<[string, number]> = []; + private last: number; + + constructor(start: number) { + this.last = start; + } + + mark(name: string): void { + const now = Date.now(); + this.marks.push([name, now - this.last]); + this.last = now; + } + + header(): string { + return this.marks.map(([name, dur]) => `${name};dur=${dur}`).join(", "); + } +} + +function errorResponse(error: unknown): Response { + if (error instanceof QueryError) { + return Response.json( + { error: error.message, ...error.details }, + { + status: error.status, + headers: { + "Access-Control-Allow-Origin": "*", + "Cache-Control": "no-store", + }, + }, + ); + } + console.error( + JSON.stringify({ + message: "data table query failed", + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + }), + ); + return jsonError("Internal server error", 500); +} + +function jsonError(message: string, status: number): Response { + return new Response(JSON.stringify({ error: message }), { + status, + headers: { + "Content-Type": "application/json; charset=utf-8", + "Access-Control-Allow-Origin": "*", + "Cache-Control": "no-store", + }, + }); +} diff --git a/packages/pmtiles-server/src/index.ts b/packages/pmtiles-server/src/index.ts index 386cbdcc1..0b4b6f924 100644 --- a/packages/pmtiles-server/src/index.ts +++ b/packages/pmtiles-server/src/index.ts @@ -1,5 +1,6 @@ import { WorkerEntrypoint } from "cloudflare:workers"; import { corsPreflightResponse } from "./auth/cors"; +import { DataTablesBackend } from "./dataTablesBackend"; import { handleClassifiedRequest } from "./gateway"; import { ObjectBackend } from "./objectBackend"; import { PropertiesBackend } from "./propertiesBackend"; @@ -10,18 +11,22 @@ import { } from "./resource"; import { TilesBackend } from "./tilesBackend"; -export { ObjectBackend, PropertiesBackend, TilesBackend }; +export { DataTablesBackend, ObjectBackend, PropertiesBackend, TilesBackend }; /** * Default entrypoint: authorize (using `?ns=` / `?access_token=`), then route - * to TilesBackend, ObjectBackend, or PropertiesBackend. + * to TilesBackend, ObjectBackend, PropertiesBackend, or DataTablesBackend. */ export default class extends WorkerEntrypoint { async fetch(request: Request): Promise { const url = new URL(request.url); if (request.method === "OPTIONS") { - if (url.pathname === "/properties" || url.pathname === "/properties/") { + if ( + url.pathname === "/properties" || + url.pathname === "/properties/" || + isDataTableQueryPath(url.pathname) + ) { return new Response(null, { status: 204, headers: { @@ -39,6 +44,10 @@ export default class extends WorkerEntrypoint { return this.routeProperties(request); } + if (isDataTableQueryPath(url.pathname)) { + return this.routeDataTableQuery(request); + } + const resource = classifyResource(url.pathname); if (!resource) return new Response("Invalid object path", { status: 400 }); // uploads host is always opaque R2; tiles/overlay hosts prefer PMTiles @@ -62,6 +71,41 @@ export default class extends WorkerEntrypoint { ); } + /** + * Overlay data-table aggregations. Paths classify as `published` under the + * parent layer UUID (`…/public/{uuid}/dataTables/{uploadId}/query`), so the + * existing tiles ACL applies. Query string is part of the cache key. + */ + private async routeDataTableQuery(request: Request): Promise { + const url = new URL(request.url); + const resource = classifyResource(url.pathname); + if (!resource) { + return new Response(JSON.stringify({ error: "Invalid object path" }), { + status: 400, + headers: { + "Content-Type": "application/json; charset=utf-8", + "Access-Control-Allow-Origin": "*", + "Cache-Control": "no-store", + }, + }); + } + return handleClassifiedRequest( + request, + this.env, + { + fetch: (req, options) => + this.ctx.exports.DataTablesBackend.fetch(req, options), + }, + resource, + { + ns: aclNamespaceFromRequest(request), + enforce: resourceAclEnabled(this.env, resource), + includeQueryInCacheKey: true, + waitUntil: (p) => this.ctx.waitUntil(p), + }, + ); + } + private async routeProperties(request: Request): Promise { const url = new URL(request.url); const dataset = url.searchParams.get("dataset"); @@ -103,6 +147,14 @@ export default class extends WorkerEntrypoint { } } +/** True for overlay data-table query endpoints (hyparquet aggregation). */ +function isDataTableQueryPath(pathname: string): boolean { + return ( + pathname.endsWith("/query") && + pathname.includes("/dataTables/") + ); +} + const TILE_EXT = "(?:mvt|pbf|png|webp|jpg|jpeg)"; // Same archive-name charset as TilesBackend TILE_ROUTE. const ARCHIVE_NAME = "[0-9a-zA-Z/!\\-_.*'()]+"; diff --git a/packages/pmtiles-server/src/resource.ts b/packages/pmtiles-server/src/resource.ts index b45cf672d..5c90e3497 100644 --- a/packages/pmtiles-server/src/resource.ts +++ b/packages/pmtiles-server/src/resource.ts @@ -38,6 +38,11 @@ export function normalizeObjectKey(pathOrKey: string): string | null { ) { return null; } + // ACL docs live in the same R2 bucket for Worker-side auth, but must never + // be HTTP-readable — they enumerate layer UUIDs and access rules. + if (key.split("/")[0].toLowerCase() === "acl") { + return null; + } return key; } diff --git a/packages/pmtiles-server/test/dataTables/blockReader.test.ts b/packages/pmtiles-server/test/dataTables/blockReader.test.ts new file mode 100644 index 000000000..4d98ebea0 --- /dev/null +++ b/packages/pmtiles-server/test/dataTables/blockReader.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; +import { + ByteBudgetCache, + coalesceRuns, + createBlockReader, +} from "../../src/dataTables/engine/blockReader"; + +function makeFile(byteLength: number): Uint8Array { + const data = new Uint8Array(byteLength); + for (let i = 0; i < byteLength; i++) { + data[i] = i % 251; + } + return data; +} + +interface Harness { + read: (start: number, end?: number) => Promise; + fetches: Array<[number, number]>; + memory: ByteBudgetCache; +} + +function makeReader( + file: Uint8Array, + { blockSize = 16, memoryBudget = 1024 * 1024 } = {} +): Harness { + const fetches: Array<[number, number]> = []; + const memory = new ByteBudgetCache(memoryBudget); + const read = createBlockReader({ + byteLength: file.byteLength, + blockSize, + cacheId: "test@etag", + memory, + fetchRange: async (start, end) => { + fetches.push([start, end]); + return file.slice(start, end).buffer; + }, + }); + return { read, fetches, memory }; +} + +describe("coalesceRuns", () => { + it("groups contiguous indices into runs", () => { + expect(coalesceRuns([0, 1, 2, 5, 6, 9])).toEqual([ + [0, 2], + [5, 6], + [9, 9], + ]); + expect(coalesceRuns([])).toEqual([]); + expect(coalesceRuns([3])).toEqual([[3, 3]]); + }); +}); + +describe("ByteBudgetCache", () => { + it("evicts least recently used entries when over budget", () => { + const cache = new ByteBudgetCache(30); + cache.set("a", new ArrayBuffer(10)); + cache.set("b", new ArrayBuffer(10)); + cache.set("c", new ArrayBuffer(10)); + // Touch "a" so "b" becomes the eviction candidate. + cache.get("a"); + cache.set("d", new ArrayBuffer(10)); + expect(cache.get("a")).toBeDefined(); + expect(cache.get("b")).toBeUndefined(); + expect(cache.get("c")).toBeDefined(); + expect(cache.get("d")).toBeDefined(); + expect(cache.totalBytes).toBe(30); + }); +}); + +describe("createBlockReader", () => { + it("returns exact bytes for unaligned ranges", async () => { + const file = makeFile(100); + const { read } = makeReader(file); + const result = new Uint8Array(await read(5, 37)); + expect(result).toEqual(file.slice(5, 37)); + }); + + it("clamps to end of file and handles empty ranges", async () => { + const file = makeFile(50); + const { read } = makeReader(file); + expect(new Uint8Array(await read(40))).toEqual(file.slice(40)); + expect((await read(10, 10)).byteLength).toBe(0); + expect((await read(60, 70)).byteLength).toBe(0); + }); + + it("fetches contiguous missing blocks in a single ranged request", async () => { + const file = makeFile(100); + const { read, fetches } = makeReader(file); + await read(0, 60); // blocks 0-3 + expect(fetches).toEqual([[0, 64]]); + }); + + it("serves overlapping reads from memory without refetching", async () => { + const file = makeFile(100); + const { read, fetches } = makeReader(file); + await read(0, 40); // blocks 0-2 + await read(10, 30); // fully covered + expect(fetches).toHaveLength(1); + await read(20, 90); // needs blocks 1-5; 3-5 missing + expect(fetches).toEqual([ + [0, 48], + [48, 96], + ]); + const result = new Uint8Array(await read(20, 90)); + expect(result).toEqual(file.slice(20, 90)); + }); +}); diff --git a/packages/pmtiles-server/test/dataTables/calculations.test.ts b/packages/pmtiles-server/test/dataTables/calculations.test.ts new file mode 100644 index 000000000..801294654 --- /dev/null +++ b/packages/pmtiles-server/test/dataTables/calculations.test.ts @@ -0,0 +1,486 @@ +import { beforeAll, describe, expect, it } from "vitest"; +import { + AsyncBuffer, + FileMetaData, + asyncBufferFromFile, + parquetMetadataAsync, +} from "hyparquet"; +import { parseQueryParams } from "../../src/dataTables/params"; +import { planQuery } from "../../src/dataTables/engine/plan"; +import { executeQuery, QueryResult } from "../../src/dataTables/engine/execute"; + +/** + * Aggregation-correctness tests for the fixture PISCO kelp-forest swath survey + * (353,253 rows, 3 row groups). These calculations drive thematic maps and + * time-series charts used in a contentious public policy process, so every + * expected value here was computed *independently* with DuckDB against the same + * data.parquet fixture (see the queries in the comment above each test), not by + * trusting the engine's own output. + * + * The two primary product use cases are exercised heavily: + * 1. Thematic map -> groupBy=site, aggregating `count` across surveys. + * 2. Site time series -> fixed site, groupBy=year, one or more classcodes. + * + * Semantics being pinned (verified to match DuckDB / SQL): + * - sum/mean/min/max/median ignore NULL `count` values. + * - `count` WITH an aggregation column = COUNT(col) (non-null values); + * WITHOUT a column = COUNT(*) (matching rows). + * - A group whose values are all NULL yields sum/mean/min/max = null and + * count(col) = 0, while still reporting rowsMatched > 0. + * - count = 0 is a real value (sum stays 0, not null). + */ + +const FIXTURE = "test/dataTables/fixtures/data.parquet"; + +let file: AsyncBuffer; +let metadata: FileMetaData; + +beforeAll(async () => { + file = await asyncBufferFromFile(FIXTURE); + metadata = await parquetMetadataAsync(file); +}); + +async function run(qs: string): Promise { + const query = parseQueryParams(new URLSearchParams(qs)); + const plan = await planQuery(metadata, query, file); + return executeQuery({ file, metadata, query, plan }); +} + +/** Index aggregated groups by the value of a single groupBy column. */ +function byKey(result: QueryResult, key: string): Record { + return Object.fromEntries(result.groups!.map((g: any) => [g[key], g])); +} + +const MEAN_PRECISION = 6; // digits after the decimal point for toBeCloseTo + +// --------------------------------------------------------------------------- +// Whole-table aggregates (no filters, no groupBy) +// --------------------------------------------------------------------------- +describe("whole-table aggregates", () => { + // DuckDB: sum=11283244, min=0, max=13450, count(count)=353234, avg=31.9426895… + it("sum/min/max/mean/count over the entire count column", async () => { + const r = await run("op=sum,min,max,mean,count&column=count"); + const g = r.groups![0] as any; + expect(g.sum).toBe(11283244); + expect(g.min).toBe(0); + expect(g.max).toBe(13450); + expect(g.count).toBe(353234); // non-null values (19 rows have null count) + expect(g.mean).toBeCloseTo(31.942689548571202, MEAN_PRECISION); + expect(r.rowsScanned).toBe(353253); + expect(r.rowsMatched).toBe(353253); + }); + + // COUNT(*) counts matching rows; the 19 null-count rows are included. + it("count without a column is COUNT(*), including null-count rows", async () => { + const r = await run("op=count"); + expect(r.groups).toEqual([{ count: 353253 }]); + }); +}); + +// --------------------------------------------------------------------------- +// NULL handling edge cases +// --------------------------------------------------------------------------- +describe("null handling", () => { + // STRPURAD has 21,014 rows but 10 have a NULL count. + // DuckDB: count(count)=21004, sum=5403907, min=1, max=13450, + // avg=257.2798990668444, median=41. + it("excludes null values from sum/mean/min/max/median/count(col)", async () => { + const r = await run( + "op=sum,min,max,mean,median,count&column=count&q.classcode=STRPURAD" + ); + const g = r.groups![0] as any; + expect(r.rowsMatched).toBe(21014); // rows matched includes the 10 nulls + expect(g.count).toBe(21004); // but count(col) excludes them + expect(g.sum).toBe(5403907); + expect(g.min).toBe(1); + expect(g.max).toBe(13450); + expect(g.median).toBe(41); + expect(g.mean).toBeCloseTo(257.2798990668444, MEAN_PRECISION); + }); + + // NO_ORG is a single row with count = 0 (a real zero, not null). + // DuckDB: count=1, sum=0, min=0, max=0, avg=0. + it("treats count=0 as a real value, not null", async () => { + const r = await run( + "op=sum,min,max,mean,count&column=count&q.classcode=NO_ORG" + ); + expect(r.rowsMatched).toBe(1); + expect(r.groups).toEqual([{ sum: 0, min: 0, max: 0, mean: 0, count: 1 }]); + }); + + // STRPURAD at CASPAR_3 in 2015: 6 rows, every count NULL. + // Aggregates collapse to null; count(col)=0 while rowsMatched=6. + it("all-null group yields null aggregates but non-zero rowsMatched", async () => { + const r = await run( + "op=sum,min,max,mean,count&column=count" + + "&q.classcode=STRPURAD&q.year=2015&q.site=CASPAR_3" + ); + expect(r.rowsMatched).toBe(6); + expect(r.groups).toEqual([ + { sum: null, min: null, max: null, mean: null, count: 0 }, + ]); + }); + + // Same group, COUNT(*) still sees all 6 rows. + it("COUNT(*) counts rows even when every value is null", async () => { + const r = await run( + "op=count&q.classcode=STRPURAD&q.year=2015&q.site=CASPAR_3" + ); + expect(r.groups).toEqual([{ count: 6 }]); + }); +}); + +// --------------------------------------------------------------------------- +// Thematic map: groupBy=site (PRIMARY use case) +// --------------------------------------------------------------------------- +describe("thematic map (groupBy=site)", () => { + // MEGUND in 2004, per site. DuckDB reports 33 sites; spot-checked below. + it("aggregates count per site for one species/year", async () => { + const r = await run( + "groupBy=site&op=sum,mean,min,max,count&column=count" + + "&q.classcode=MEGUND&q.year=2004" + ); + expect(r.groups).toHaveLength(33); + expect(r.rowsMatched).toBe(126); + const s = byKey(r, "site"); + + expect(s["ROCKY_POINT_N"]).toMatchObject({ + sum: 490, + min: 3, + max: 105, + count: 11, + }); + expect(s["ROCKY_POINT_N"].mean).toBeCloseTo(44.54545454545455, MEAN_PRECISION); + + expect(s["SCI_VALLEY_CEN"]).toMatchObject({ sum: 332, count: 3 }); + expect(s["SCI_VALLEY_CEN"].mean).toBeCloseTo(110.66666666666667, MEAN_PRECISION); + + expect(s["ANACAPA_EAST_ISLE_CEN"]).toMatchObject({ + sum: 51, + min: 3, + max: 37, + mean: 17, + count: 3, + }); + + // Grand total across all site sums must equal the ungrouped sum (2031). + const total = r.groups!.reduce((acc: number, g: any) => acc + g.sum, 0); + expect(total).toBe(2031); + }); + + // STRPURAD in 2015, per site: 115 sites, mixing fully-null, partially-null, + // and single-row groups. DuckDB: rows=1139, count(count)=1129. + it("per-site aggregation with null and single-row groups", async () => { + const r = await run( + "groupBy=site&op=sum,mean,min,max,count&column=count" + + "&q.classcode=STRPURAD&q.year=2015" + ); + expect(r.groups).toHaveLength(115); + expect(r.rowsMatched).toBe(1139); + const s = byKey(r, "site"); + + // Fully-null site: aggregates null, count(col)=0. + expect(s["CASPAR_3"]).toEqual({ + site: "CASPAR_3", + sum: null, + mean: null, + min: null, + max: null, + count: 0, + }); + + // Partially-null site: 6 rows, 2 non-null counts (510, 643). + expect(s["TEN_MILE_2"]).toMatchObject({ + sum: 1153, + min: 510, + max: 643, + mean: 576.5, + count: 2, + }); + + // Single-row site. + expect(s["PYRAMID_POINT_1"]).toEqual({ + site: "PYRAMID_POINT_1", + sum: 1, + mean: 1, + min: 1, + max: 1, + count: 1, + }); + + expect(s["ANACAPA_LIGHTHOUSE_REEF_CEN"]).toMatchObject({ + sum: 6490, + min: 820, + max: 3100, + mean: 1622.5, + count: 4, + }); + + // count(col) totals across sites should equal DuckDB's 1129 non-null values. + const nonNull = r.groups!.reduce((acc: number, g: any) => acc + g.count, 0); + expect(nonNull).toBe(1129); + }); + + // Thematic maps often show the top-N sites. PISGIG across all years, + // ordered by total count desc. DuckDB top 5 verified below. + it("orders sites by summed count and paginates (top-N)", async () => { + const r = await run( + "groupBy=site&op=sum,count&column=count&q.classcode=PISGIG" + + "&orderBy=sum:desc&limit=5" + ); + expect(r.groups!.map((g: any) => [g.site, g.sum])).toEqual([ + ["SMI_HARRIS_PT_RESERVE_E", 1884], + ["HOPKINS_DC", 1882], + ["SCI_HAZARDS_CEN", 1454], + ["SCI_PAINTED_CAVE_E", 1438], + ["SCI_HAZARDS_W", 1245], + ]); + }); + + // Top-N by mean (density) rather than total, for the same MEGUND 2004 slice. + it("orders sites by mean count and paginates", async () => { + const r = await run( + "groupBy=site&op=mean,sum,count&column=count" + + "&q.classcode=MEGUND&q.year=2004&orderBy=mean:desc&limit=3" + ); + const sites = r.groups!.map((g: any) => g.site); + expect(sites).toEqual([ + "SCI_VALLEY_CEN", + "ANACAPA_EAST_ISLE_W", + "ROCKY_POINT_N", + ]); + expect((r.groups![0] as any).mean).toBeCloseTo( + 110.66666666666667, + MEAN_PRECISION + ); + expect((r.groups![1] as any).mean).toBeCloseTo(52.5, MEAN_PRECISION); + }); +}); + +// --------------------------------------------------------------------------- +// Site time series: groupBy=year at a fixed site (SECONDARY use case) +// --------------------------------------------------------------------------- +describe("site time series (groupBy=year)", () => { + // MACPYRAD (giant kelp) at HOPKINS_UC, every survey year. DuckDB: 26 years. + it("single-species yearly series at one site", async () => { + const r = await run( + "groupBy=year&op=sum,mean,min,max,count&column=count" + + "&q.classcode=MACPYRAD&q.site=HOPKINS_UC" + ); + expect(r.groups).toHaveLength(26); + const y = byKey(r, "year"); + + expect(y[1999]).toMatchObject({ sum: 74, min: 1, max: 2, count: 63 }); + expect(y[1999].mean).toBeCloseTo(1.1746031746031746, MEAN_PRECISION); + + // 2004: every observation is 1. + expect(y[2004]).toMatchObject({ sum: 27, min: 1, max: 1, mean: 1, count: 27 }); + + expect(y[2024]).toMatchObject({ sum: 84, min: 1, max: 11, count: 32 }); + expect(y[2024].mean).toBeCloseTo(2.625, MEAN_PRECISION); + }); + + // Multi-species series (in-list): purple urchin adults + recruits at + // HOPKINS_UC, 2018-2020..2022. Both classcodes fold into each year's totals. + it("multi-species yearly series at one site", async () => { + const r = await run( + "groupBy=year&op=sum,mean,min,max,count&column=count" + + "&q.classcode=in.(STRPURAD,STRPURREC)&q.site=HOPKINS_UC" + + "&q.year=gte.2018&q.year=lte.2022" + ); + expect(r.groups).toHaveLength(5); + const y = byKey(r, "year"); + + expect(y[2018]).toMatchObject({ sum: 2642, min: 1, max: 471, count: 33 }); + expect(y[2018].mean).toBeCloseTo(80.06060606060606, MEAN_PRECISION); + + // 2020 urchin bloom: STRPURAD 6260 + STRPURREC 819 = 7079. + expect(y[2020]).toMatchObject({ sum: 7079, min: 1, max: 733, count: 52 }); + expect(y[2020].mean).toBeCloseTo(136.1346153846154, MEAN_PRECISION); + + expect(y[2022]).toMatchObject({ sum: 5150, min: 3, max: 617, count: 39 }); + expect(y[2022].mean).toBeCloseTo(132.05128205128204, MEAN_PRECISION); + }); + + // Time series constrained to a temporal window via ANDed range filters. + it("yearly series within an inclusive temporal range", async () => { + const r = await run( + "groupBy=year&op=sum,mean,count&column=count" + + "&q.classcode=KELKEL&q.year=gte.2010&q.year=lte.2015" + ); + expect(r.groups).toHaveLength(6); + const y = byKey(r, "year"); + expect(y[2010]).toMatchObject({ sum: 1709, count: 248 }); + expect(y[2010].mean).toBeCloseTo(6.891129032258065, MEAN_PRECISION); + expect(y[2011]).toMatchObject({ sum: 1595, count: 290, mean: 5.5 }); + expect(y[2015]).toMatchObject({ sum: 889, count: 172 }); + expect(y[2015].mean).toBeCloseTo(5.1686046511627906, MEAN_PRECISION); + }); +}); + +// --------------------------------------------------------------------------- +// Filter operators +// --------------------------------------------------------------------------- +describe("filter operators", () => { + // in-list on the grouped column itself, aggregating per classcode. + it("in-list species, aggregated per classcode", async () => { + const r = await run( + "groupBy=classcode&op=sum,mean,count&column=count" + + "&q.classcode=in.(PYCHEL,PISGIG,PISBRE)" + ); + const c = byKey(r, "classcode"); + expect(c["PISBRE"]).toMatchObject({ sum: 13504, count: 2328 }); + expect(c["PISBRE"].mean).toBeCloseTo(5.8006872852233675, MEAN_PRECISION); + expect(c["PISGIG"]).toMatchObject({ sum: 73413, count: 10623 }); + expect(c["PISGIG"].mean).toBeCloseTo(6.910759672408924, MEAN_PRECISION); + expect(c["PYCHEL"]).toMatchObject({ sum: 9664, count: 3674 }); + expect(c["PYCHEL"].mean).toBeCloseTo(2.6303756124115405, MEAN_PRECISION); + }); + + // neq excludes both the value and nulls; campus has no nulls here. + it("neq filter", async () => { + const r = await run( + "op=sum,count&column=count&q.classcode=HALRUF&q.campus=neq.UCSC" + ); + expect(r.groups).toEqual([{ sum: 5284, count: 1317 }]); + expect(r.rowsMatched).toBe(1317); + }); + + // Exclusive gt/lt bounds: year>2010 & year<2013 selects 2011-2012 only. + it("exclusive numeric bounds (gt/lt)", async () => { + const r = await run( + "op=sum,mean,count&column=count&q.classcode=CRAGIG&q.year=gt.2010&q.year=lt.2013" + ); + const g = r.groups![0] as any; + expect(g.sum).toBe(1294); + expect(g.count).toBe(496); + expect(g.mean).toBeCloseTo(2.6088709677419355, MEAN_PRECISION); + expect(r.rowsMatched).toBe(496); + }); + + // Inclusive range rollup (no groupBy) over a temporal window. + it("inclusive range rollup (gte/lte)", async () => { + const r = await run( + "op=sum,mean,min,max,count&column=count" + + "&q.classcode=KELKEL&q.year=gte.2010&q.year=lte.2015" + ); + const g = r.groups![0] as any; + expect(g).toMatchObject({ sum: 7357, min: 1, max: 113, count: 1299 }); + expect(g.mean).toBeCloseTo(5.663587374903772, MEAN_PRECISION); + expect(r.rowsMatched).toBe(1299); + }); + + // Filter on a floating-point column (depth). Null depths are excluded by + // the comparison, so rowsMatched already reflects non-null depths. + it("filters on a double column (depth)", async () => { + const r = await run( + "op=sum,mean,count&column=count&q.classcode=PANINT&q.depth=gte.10" + ); + const g = r.groups![0] as any; + expect(g.sum).toBe(2688); + expect(g.count).toBe(1211); + expect(g.mean).toBeCloseTo(2.2196531791907512, MEAN_PRECISION); + expect(r.rowsMatched).toBe(1211); + }); +}); + +// --------------------------------------------------------------------------- +// Median +// --------------------------------------------------------------------------- +describe("median", () => { + // Even value count -> average of the two middle values. SARMUT n=80 -> 4.5. + it("interpolates the median for even value counts", async () => { + const r = await run( + "op=median,count,sum,mean,min,max&column=count&q.classcode=SARMUT" + ); + expect(r.groups![0]).toMatchObject({ + median: 4.5, + count: 80, + sum: 3466, + min: 1, + max: 643, + }); + expect((r.groups![0] as any).mean).toBeCloseTo(43.325, MEAN_PRECISION); + }); + + // ACTBRA n=16 -> median 22.5. + it("median for a small even-count group", async () => { + const r = await run( + "op=median,count,sum,max&column=count&q.classcode=ACTBRA" + ); + expect(r.groups![0]).toMatchObject({ + median: 22.5, + count: 16, + sum: 686, + max: 150, + }); + }); + + // Odd value count -> the middle value. PTETES = [1,1,1] -> 1. + it("median for an odd-count group", async () => { + const r = await run("op=median,count&column=count&q.classcode=PTETES"); + expect(r.groups![0]).toEqual({ median: 1, count: 3 }); + }); + + // Median must ignore null values just like the other aggregates. + // STRPURAD has 10 nulls; DuckDB median over the 21004 non-null values = 41. + it("median ignores null values", async () => { + const r = await run("op=median,count&column=count&q.classcode=STRPURAD"); + expect(r.groups![0]).toEqual({ median: 41, count: 21004 }); + }); +}); + +// --------------------------------------------------------------------------- +// Multi-key grouping and empty results +// --------------------------------------------------------------------------- +describe("multi-key grouping and empty results", () => { + // Group by two dimensions at once. DuckDB: 14 distinct campus/zone pairs; + // top 6 by row count verified below. + it("groups by two columns and orders by count", async () => { + const r = await run( + "groupBy=campus,zone&op=count&orderBy=count:desc&limit=6" + ); + expect( + r.groups!.map((g: any) => [g.campus, g.zone, g.count]) + ).toEqual([ + ["UCSC", "MID", 55133], + ["UCSB", "INNER", 52143], + ["UCSC", "OUTER", 51828], + ["UCSB", "OUTER", 51353], + ["UCSC", "INNER", 42793], + ["UCSB", "MID", 30255], + ]); + }); + + it("counts all distinct campus/zone pairs", async () => { + const r = await run("groupBy=campus,zone&op=count"); + expect(r.groups).toHaveLength(14); + }); + + // A species that does not exist matches nothing: no groups, even for a + // grouped thematic-map query. + it("returns no groups for a non-existent species (groupBy=site)", async () => { + const r = await run( + "groupBy=site&op=sum,mean,count&column=count&q.classcode=NOTREAL" + ); + expect(r.groups).toEqual([]); + expect(r.rowsMatched).toBe(0); + }); + + // Same, without groupBy: a global aggregate over zero matches produces no + // group at all (rather than a row of nulls). + it("returns no groups for a global aggregate with zero matches", async () => { + const r = await run("op=sum,mean,count&column=count&q.classcode=NOTREAL"); + expect(r.groups).toEqual([]); + expect(r.rowsMatched).toBe(0); + }); + + // Impossible temporal window. + it("returns no groups when a range filter excludes everything", async () => { + const r = await run( + "groupBy=year&op=sum,count&column=count&q.classcode=KELKEL&q.year=gt.3000" + ); + expect(r.groups).toEqual([]); + expect(r.rowsMatched).toBe(0); + }); +}); diff --git a/packages/pmtiles-server/test/dataTables/engine.test.ts b/packages/pmtiles-server/test/dataTables/engine.test.ts new file mode 100644 index 000000000..74709ea1d --- /dev/null +++ b/packages/pmtiles-server/test/dataTables/engine.test.ts @@ -0,0 +1,255 @@ +import { beforeAll, describe, expect, it } from "vitest"; +import { + AsyncBuffer, + FileMetaData, + asyncBufferFromFile, + parquetMetadataAsync, +} from "hyparquet"; +import { parseQueryParams, QueryError } from "../../src/dataTables/params"; +import { planQuery } from "../../src/dataTables/engine/plan"; +import { executeQuery } from "../../src/dataTables/engine/execute"; + +/** + * Engine tests run directly in Node against a real data table produced by + * the ingestion pipeline (PISCO kelp forest swath surveys, 353,253 rows, + * 3 row groups). Expected values were computed independently with DuckDB / + * a full scan of the file. + */ + +// vitest runs with cwd at the package root +const FIXTURE = "test/dataTables/fixtures/data.parquet"; + +let file: AsyncBuffer; +let metadata: FileMetaData; + +beforeAll(async () => { + file = await asyncBufferFromFile(FIXTURE); + metadata = await parquetMetadataAsync(file); +}); + +async function run(qs: string, useBloomFilters = true) { + const query = parseQueryParams(new URLSearchParams(qs)); + const plan = await planQuery( + metadata, + query, + useBloomFilters ? file : undefined + ); + const result = await executeQuery({ file, metadata, query, plan }); + return { query, plan, result }; +} + +describe("example query", () => { + it("groupBy=site&op=mean,count&column=count&q.year=2018&q.classcode=PYCHEL", async () => { + const { result } = await run( + "groupBy=site&op=mean,count&column=count&q.year=2018&q.classcode=PYCHEL" + ); + expect(result.groups).toHaveLength(3); + const bySite = Object.fromEntries( + result.groups!.map((g: any) => [g.site, g]) + ); + expect(Object.keys(bySite).sort()).toEqual([ + "LITTLE_IRISH_CEN", + "PESCADERO_DC", + "PINOS", + ]); + for (const g of result.groups!) { + expect(g).toMatchObject({ mean: 1, count: 1 }); + } + expect(result.rowsMatched).toBe(3); + }); +}); + +describe("aggregations", () => { + it("computes global sum/min/max/median/mean without groupBy", async () => { + const { result } = await run( + "op=sum,min,max,median,mean,count&column=count&q.year=2018&q.classcode=PYCHEL" + ); + expect(result.groups).toEqual([ + { sum: 3, min: 1, max: 1, median: 1, mean: 1, count: 3 }, + ]); + }); + + it("count without a column counts matching rows", async () => { + const { result } = await run("op=count&q.campus=HSU"); + expect(result.groups).toEqual([{ count: 11515 }]); + }); + + it("orders groups by aggregate value and paginates", async () => { + const { result } = await run( + "groupBy=campus&op=count&orderBy=count:desc&limit=2" + ); + expect(result.groups!.map((g: any) => g.campus)).toEqual(["UCSC", "UCSB"]); + expect(result.groups![0].count).toBe(149754); + }); +}); + +describe("row group pruning", () => { + // Fixture row group stats: + // rg0: 122,880 rows, year [1999,2020], campus [UCSC,UCSC], site_name_old all null + // rg1: 122,880 rows, year [1999,2024], campus [UCSB,UCSC] + // rg2: 107,493 rows, year [2004,2024], campus [HSU,VRG] + it("prunes row groups via string equality statistics", async () => { + const { plan, result } = await run("op=count&q.campus=HSU"); + expect(plan.rowGroupsTotal).toBe(3); + expect(plan.rowGroupsScanned).toBe(1); + expect(result.rowsScanned).toBe(107493); + expect(result.rowsMatched).toBe(11515); + }); + + it("prunes row groups via numeric comparison statistics", async () => { + const { plan } = await run("op=count&q.year=lte.2003"); + expect(plan.rowGroupsScanned).toBe(2); // rg2 min year is 2004 + }); + + it("prunes all-null row groups for not.null filters", async () => { + const { plan } = await run("op=count&q.site_name_old=not.null"); + expect(plan.rowGroupsScanned).toBe(2); // rg0 is 100% null + }); + + it("scans everything when statistics cannot exclude (blooms disabled)", async () => { + const { plan } = await run("op=count&q.classcode=PYCHEL", false); + expect(plan.rowGroupsScanned).toBe(3); + }); + + // 72 of the fixture's 182 species codes occur in only one row group; + // min/max statistics can't prune them (codes span the alphabet in every + // row group) but bloom filters prove absence. + it("prunes row groups via bloom filters for eq on high-cardinality strings", async () => { + // SOLSPP only occurs in rg0 + const { plan, result } = await run("op=count&q.classcode=SOLSPP"); + expect(plan.rowGroupsScanned).toBe(1); + expect(result.rowsScanned).toBe(122880); + expect(result.groups![0].count).toBeGreaterThan(0); + }); + + it("prunes row groups via bloom filters for in lists", async () => { + // SOLSPP is rg0-only, MACPYRJUV is rg1-only; rg2 can be skipped + const { plan } = await run( + "op=count&q.classcode=in.(SOLSPP,MACPYRJUV)" + ); + expect(plan.rowGroupsScanned).toBe(2); + }); + + it("merges contiguous surviving row groups into one read span", async () => { + const { plan } = await run("op=count&q.year=lte.2003"); + expect(plan.spans).toEqual([{ rowStart: 0, rowEnd: 245760 }]); + }); +}); + +describe("filters", () => { + it("equality against strings", async () => { + const { result } = await run("op=count&q.observer=CHAD BURT"); + expect(result.groups).toEqual([{ count: 100 }]); + }); + + it("in lists", async () => { + const { result } = await run( + "op=count&q.observer=in.(CHAD BURT,LYALL BELLQUIST)" + ); + expect(result.groups).toEqual([{ count: 650 }]); // 100 + 550 + }); + + it("numeric comparisons", async () => { + const { result } = await run("q.count=gte.4000"); + expect(result.rowsMatched).toBe(134); + }); + + it("null tests", async () => { + const { result } = await run("op=count&q.size=is.null"); + expect(result.groups).toEqual([{ count: 217323 }]); + }); + + it("ANDed range filters on the same column", async () => { + const { result } = await run( + "op=count&q.year=gte.2010&q.year=lte.2012" + ); + expect(result.groups).toEqual([{ count: 57005 }]); + }); + + it("only reads the columns a query needs", async () => { + const { plan } = await run( + "groupBy=site&op=mean&column=count&q.year=2018" + ); + expect(plan.neededColumns!.sort()).toEqual(["count", "site", "year"]); + }); +}); + +describe("decoded column cache", () => { + it("returns identical results when served from the cache", async () => { + const query = parseQueryParams( + new URLSearchParams( + "groupBy=site&op=count,mean&column=count&q.classcode=PISBRE&q.year=2017" + ) + ); + const plan = await planQuery(metadata, query, file); + const cold = await executeQuery({ + file, + metadata, + query, + plan, + cacheKey: "test-etag", + }); + const warm = await executeQuery({ + file, + metadata, + query, + plan, + cacheKey: "test-etag", + }); + expect(warm).toEqual(cold); + expect(warm.rowsMatched).toBe(62); + }); +}); + +describe("raw row output", () => { + it("returns filtered rows with limit/offset/orderBy", async () => { + const { result } = await run( + "q.count=gte.4000&orderBy=count:desc&limit=3" + ); + expect(result.rows).toHaveLength(3); + expect(result.rows![0].count).toBe(13450); // max count in the table + const counts = result.rows!.map((r: any) => r.count); + expect([...counts].sort((a, b) => b - a)).toEqual(counts); + }); + + it("serializes values as JSON-safe types (no BigInt)", async () => { + const { result } = await run("limit=1"); + const row = result.rows![0]; + expect(typeof row.year).toBe("number"); + expect(typeof row.site).toBe("string"); + expect(() => JSON.stringify(row)).not.toThrow(); + }); +}); + +describe("validation", () => { + it("rejects unknown columns, listing valid ones", async () => { + await expect(run("q.bogus=1")).rejects.toThrow(QueryError); + try { + await run("q.bogus=1"); + } catch (e: any) { + expect(e.details.validColumns.map((c: any) => c.name)).toContain("site"); + } + }); + + it("rejects comparison operators on string columns", async () => { + await expect(run("q.site=gte.A")).rejects.toThrow( + /only supported for numeric/ + ); + }); + + it("rejects non-numeric values for numeric columns", async () => { + await expect(run("q.year=abc")).rejects.toThrow(/Invalid numeric value/); + }); + + it("rejects numeric aggregations of string columns", async () => { + await expect( + run("groupBy=site&op=mean&column=observer") + ).rejects.toThrow(/require a numeric column/); + }); + + it("rejects orderBy keys missing from the output", async () => { + await expect( + run("groupBy=site&op=count&orderBy=depth") + ).rejects.toThrow(/orderBy/); + }); +}); diff --git a/packages/pmtiles-server/test/dataTables/fixtures/data.parquet b/packages/pmtiles-server/test/dataTables/fixtures/data.parquet new file mode 100644 index 000000000..32353861d Binary files /dev/null and b/packages/pmtiles-server/test/dataTables/fixtures/data.parquet differ diff --git a/packages/pmtiles-server/test/dataTables/params.test.ts b/packages/pmtiles-server/test/dataTables/params.test.ts new file mode 100644 index 000000000..a8c23f2ea --- /dev/null +++ b/packages/pmtiles-server/test/dataTables/params.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from "vitest"; +import { + canonicalQueryString, + parseInList, + parseQueryParams, + QueryError, +} from "../../src/dataTables/params"; + +function parse(qs: string) { + return parseQueryParams(new URLSearchParams(qs)); +} + +describe("parseInList", () => { + it("parses simple lists", () => { + expect(parseInList("a,b,c")).toEqual(["a", "b", "c"]); + }); + + it("trims unquoted items", () => { + expect(parseInList("a , b")).toEqual(["a", "b"]); + }); + + it("supports quoted items containing commas", () => { + expect(parseInList('"Smith, John",UCSB')).toEqual(["Smith, John", "UCSB"]); + }); + + it("supports escaped quotes", () => { + expect(parseInList('"say ""hi""",x')).toEqual(['say "hi"', "x"]); + }); +}); + +describe("parseQueryParams", () => { + it("parses bare values as equality filters", () => { + const q = parse("q.year=2018&q.classcode=PYCHEL"); + expect(q.filters).toEqual([ + { column: "year", op: "eq", value: "2018" }, + { column: "classcode", op: "eq", value: "PYCHEL" }, + ]); + }); + + it("parses comparison operators", () => { + const q = parse("q.count=gte.5&q.depth=lt.10.5"); + expect(q.filters).toEqual([ + { column: "count", op: "gte", value: "5" }, + { column: "depth", op: "lt", value: "10.5" }, + ]); + }); + + it("parses in lists", () => { + const q = parse("q.observer=in.(Chad Burt,Lyal Belquist)"); + expect(q.filters).toEqual([ + { column: "observer", op: "in", values: ["Chad Burt", "Lyal Belquist"] }, + ]); + }); + + it("parses null tests", () => { + const q = parse("q.size=is.null&q.depth=not.null"); + expect(q.filters).toEqual([ + { column: "size", op: "isNull" }, + { column: "depth", op: "notNull" }, + ]); + }); + + it("ANDs repeated filters on the same column", () => { + const q = parse("q.year=gte.2010&q.year=lte.2018"); + expect(q.filters).toHaveLength(2); + }); + + it("parses groupBy, op and column", () => { + const q = parse("groupBy=site,zone&op=mean,count&column=count"); + expect(q.groupBy).toEqual(["site", "zone"]); + expect(q.ops).toEqual(["mean", "count"]); + expect(q.column).toBe("count"); + }); + + it("requires column for non-count aggregations", () => { + expect(() => parse("groupBy=site&op=mean")).toThrow(QueryError); + }); + + it("allows op=count without a column", () => { + const q = parse("groupBy=site&op=count"); + expect(q.column).toBeNull(); + }); + + it("requires op when groupBy is present", () => { + expect(() => parse("groupBy=site")).toThrow(QueryError); + }); + + it("rejects unknown aggregations", () => { + expect(() => parse("op=stddev&column=count")).toThrow(QueryError); + }); + + it("defaults to no limit and accepts explicit limits", () => { + expect(parse("groupBy=site&op=count").limit).toBeNull(); + expect(parse("limit=500").limit).toBe(500); + expect(parse("limit=100000").limit).toBe(100000); + }); + + it("rejects invalid formats, limits and offsets", () => { + expect(() => parse("f=xml")).toThrow(QueryError); + expect(() => parse("limit=0")).toThrow(QueryError); + expect(() => parse("limit=abc")).toThrow(QueryError); + expect(() => parse("offset=-1")).toThrow(QueryError); + }); + + it("parses orderBy with direction", () => { + expect(parse("orderBy=mean:desc").orderBy).toEqual({ + key: "mean", + direction: "desc", + }); + expect(parse("orderBy=site").orderBy).toEqual({ + key: "site", + direction: "asc", + }); + expect(() => parse("orderBy=site:sideways")).toThrow(QueryError); + }); +}); + +describe("canonicalQueryString", () => { + it("orders parameters deterministically and drops f", () => { + const a = canonicalQueryString( + new URLSearchParams("f=json&q.b=2&q.a=1&groupBy=site&op=mean&column=c") + ); + const b = canonicalQueryString( + new URLSearchParams("op=mean&q.a=1&column=c&groupBy=site&q.b=2&f=html") + ); + expect(a).toBe(b); + expect(a).not.toContain("f="); + }); +}); diff --git a/packages/pmtiles-server/test/objectDownload.spec.ts b/packages/pmtiles-server/test/objectDownload.spec.ts index e57f4b954..3e2712ed6 100644 --- a/packages/pmtiles-server/test/objectDownload.spec.ts +++ b/packages/pmtiles-server/test/objectDownload.spec.ts @@ -3,6 +3,7 @@ import { generateKeyPair, SignJWT, type KeyLike } from "jose"; import { beforeAll, describe, expect, it } from "vitest"; import type { ProjectAclDoc } from "../src/auth/types"; import { handleClassifiedRequest } from "../src/gateway"; +import { handleObjectRequest } from "../src/objectBackend"; import { aclNamespaceFromRequest, classifyResource } from "../src/resource"; import { handleTilesBackendRequest } from "../src/tilesBackend"; @@ -79,6 +80,29 @@ describe("object downloads", () => { expect(await response.text()).toContain("FeatureCollection"); }); + it("does not serve ACL documents from ObjectBackend", async () => { + const ns = `acl-download-${crypto.randomUUID()}`; + const aclKey = `acl/${ns}/projects/${SLUG}.json`; + await env.TILES_BUCKET.put( + aclKey, + JSON.stringify({ + v: 1, + slug: SLUG, + public: [UUID], + rules: [], + protected: {}, + }), + ); + + const response = await handleObjectRequest( + new Request(`https://tiles.seasketch.org/${aclKey}`), + env, + ); + + expect(response.status).toBe(400); + expect(await response.text()).toBe("Invalid object path"); + }); + it("does not treat TileJSON paths as raw object keys", async () => { // No matching .pmtiles archive → KeyNotFoundError → 404 tileset, not object const response = await handleTilesBackendRequest( diff --git a/packages/pmtiles-server/test/resource.spec.ts b/packages/pmtiles-server/test/resource.spec.ts index a55b99cc1..f7c82f818 100644 --- a/packages/pmtiles-server/test/resource.spec.ts +++ b/packages/pmtiles-server/test/resource.spec.ts @@ -22,6 +22,16 @@ describe("overlay resource classification", () => { "/projects/example/public/11111111-1111-1111-1111-111111111111.fgb", ), ).toMatchObject({ kind: "published", slug: "example" }); + // Data-table assets under the parent layer UUID inherit published ACL. + expect( + classifyResource( + "/projects/example/public/11111111-1111-1111-1111-111111111111/dataTables/22222222-2222-2222-2222-222222222222/query", + ), + ).toMatchObject({ + kind: "published", + slug: "example", + uuid: "11111111-1111-1111-1111-111111111111", + }); expect( classifyResource("/projects/example/subdivided/42-output.fgb"), ).toMatchObject({ kind: "subdivided", slug: "example" }); @@ -33,6 +43,13 @@ describe("overlay resource classification", () => { expect(normalizeObjectKey("/projects%2F..%2Fsecret")).toBeNull(); }); + it("rejects ACL document keys so they are not HTTP-downloadable", () => { + expect(normalizeObjectKey("/acl/prod/projects/example.json")).toBeNull(); + expect(normalizeObjectKey("acl/dev-cburt/projects/example.json")).toBeNull(); + expect(normalizeObjectKey("/ACL/prod/projects/example.json")).toBeNull(); + expect(classifyResource("/acl/prod/projects/example.json")).toBeNull(); + }); + it("defaults ACL enforcement off and missing ACL docs public", () => { expect(aclEnabled({} as Env)).toBe(false); expect(aclEnabled({ AUTH_ACL_ENABLED: "true" } as Env)).toBe(true); diff --git a/packages/pmtiles-server/vitest.config.mts b/packages/pmtiles-server/vitest.config.mts index b6c748a17..2090387f6 100644 --- a/packages/pmtiles-server/vitest.config.mts +++ b/packages/pmtiles-server/vitest.config.mts @@ -1,10 +1,37 @@ import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; import { defineConfig } from "vitest/config"; +// Pure unit tests for the data-table query engine run in plain node (no +// workerd), which allows fixture file access and faster startup. +const NODE_TESTS = [ + "test/dataTables/engine.test.ts", + "test/dataTables/calculations.test.ts", + "test/dataTables/params.test.ts", + "test/dataTables/blockReader.test.ts", +]; + export default defineConfig({ - plugins: [ - cloudflareTest({ - wrangler: { configPath: "./wrangler.toml" }, - }), - ], + test: { + projects: [ + { + test: { + name: "workers", + include: ["test/**/*.spec.ts", "test/**/*.test.ts"], + exclude: NODE_TESTS, + }, + plugins: [ + cloudflareTest({ + wrangler: { configPath: "./wrangler.toml" }, + }), + ], + }, + { + test: { + name: "node-engine", + environment: "node", + include: NODE_TESTS, + }, + }, + ], + }, }); diff --git a/packages/pmtiles-server/wrangler.toml b/packages/pmtiles-server/wrangler.toml index bd07d6c4d..d90674da4 100644 --- a/packages/pmtiles-server/wrangler.toml +++ b/packages/pmtiles-server/wrangler.toml @@ -2,7 +2,8 @@ name = "overlay-data-server" main = "src/index.ts" compatibility_date = "2026-07-14" # Request-scoped timing context for Server-Timing (see src/timing.ts). -compatibility_flags = ["nodejs_als"] +# nodejs_compat covers ALS (timing) and hyparquet's node-lean deps. +compatibility_flags = ["nodejs_compat"] # Workers Caching: serves cache hits without invoking the worker, collapses # concurrent misses, and participates in tiered caching. @@ -33,6 +34,12 @@ type = "worker" [exports.PropertiesBackend.cache] enabled = true +# Data-table /query aggregations are deterministic per path+query string. +[exports.DataTablesBackend] +type = "worker" +[exports.DataTablesBackend.cache] +enabled = true + [vars] # Rollout knobs. Defaults are lax for cutover. AUTH_ACL_ENABLED = "false" diff --git a/packages/spatial-uploads-handler/dist/bin/geostats.js b/packages/spatial-uploads-handler/dist/bin/geostats.js index b16687846..46277879b 100644 --- a/packages/spatial-uploads-handler/dist/bin/geostats.js +++ b/packages/spatial-uploads-handler/dist/bin/geostats.js @@ -15,23 +15,13 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? ( }) : function(o, v) { o["default"] = v; }); -var __importStar = (this && this.__importStar) || (function () { - var ownKeys = function(o) { - ownKeys = Object.getOwnPropertyNames || function (o) { - var ar = []; - for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k; - return ar; - }; - return ownKeys(o); - }; - return function (mod) { - if (mod && mod.__esModule) return mod; - var result = {}; - if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]); - __setModuleDefault(result, mod); - return result; - }; -})(); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; Object.defineProperty(exports, "__esModule", { value: true }); /** * Calculates geostats or rasterInfo for a given dataset. For debugging diff --git a/packages/spatial-uploads-handler/dist/bin/geostats.js.map b/packages/spatial-uploads-handler/dist/bin/geostats.js.map index cc83d0a77..19cf3320e 100644 --- a/packages/spatial-uploads-handler/dist/bin/geostats.js.map +++ b/packages/spatial-uploads-handler/dist/bin/geostats.js.map @@ -1 +1 @@ -{"version":3,"file":"geostats.js","sourceRoot":"","sources":["../../bin/geostats.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;GAKG;AACH,kEAA+D;AAC/D,0EAAwE;AACxE,uCAAyB;AAEzB,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjC,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAEnC,IAAI,CAAC,QAAQ,EAAE,CAAC;IACd,OAAO,CAAC,KAAK,CACX,qEAAqE,CACtE,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,IAAI,KAAK,CAAC;IACV,IACE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QACzB,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC;QAC7B,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC1B,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EACzB,CAAC;QACD,KAAK,GAAG,MAAM,IAAA,gDAAuB,EAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;SAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACjE,KAAK,GAAG,MAAM,IAAA,uCAAkB,EAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACjD,IAAI,UAAU,EAAE,CAAC;QACf,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAC;IAChD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACzB,CAAC;AACH,CAAC;AAED,GAAG,EAAE,CAAC"} \ No newline at end of file +{"version":3,"file":"geostats.js","sourceRoot":"","sources":["../../bin/geostats.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;GAKG;AACH,kEAA+D;AAC/D,0EAAwE;AACxE,uCAAyB;AAEzB,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjC,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAEnC,IAAI,CAAC,QAAQ,EAAE,CAAC;IACd,OAAO,CAAC,KAAK,CACX,qEAAqE,CACtE,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,IAAI,KAAK,CAAC;IACV,IACE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QACzB,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC;QAC7B,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC1B,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EACzB,CAAC;QACD,KAAK,GAAG,MAAM,IAAA,gDAAuB,EAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;SAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACjE,KAAK,GAAG,MAAM,IAAA,uCAAkB,EAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACjD,IAAI,UAAU,EAAE,CAAC;QACf,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAC;IAChD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACzB,CAAC;AACH,CAAC;AAED,GAAG,EAAE,CAAC"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/handler.d.ts b/packages/spatial-uploads-handler/dist/handler.d.ts index 71ddc5e36..ed1fa4ba5 100644 --- a/packages/spatial-uploads-handler/dist/handler.d.ts +++ b/packages/spatial-uploads-handler/dist/handler.d.ts @@ -1,5 +1,5 @@ import type { SpatialUploadsHandlerRequest } from "./src/spatialUploadsHandlerTypes"; -export type { SpatialUploadsHandlerRequest } from "./src/spatialUploadsHandlerTypes"; +export type { SpatialUploadsHandlerRequest, DelimitedUploadProcessingOptions, } from "./src/spatialUploadsHandlerTypes"; export declare const processUpload: (event: SpatialUploadsHandlerRequest) => Promise<{ log: string; logfile: string; diff --git a/packages/spatial-uploads-handler/dist/handler.d.ts.map b/packages/spatial-uploads-handler/dist/handler.d.ts.map index a78c6ec1d..3f9880933 100644 --- a/packages/spatial-uploads-handler/dist/handler.d.ts.map +++ b/packages/spatial-uploads-handler/dist/handler.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../handler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,kCAAkC,CAAC;AAErF,YAAY,EAAE,4BAA4B,EAAE,MAAM,kCAAkC,CAAC;AAErF,eAAO,MAAM,aAAa,GAAU,OAAO,4BAA4B;;;;;;;;EAqBtE,CAAC"} \ No newline at end of file +{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../handler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,kCAAkC,CAAC;AAErF,YAAY,EACV,4BAA4B,EAC5B,gCAAgC,GACjC,MAAM,kCAAkC,CAAC;AAE1C,eAAO,MAAM,aAAa,GAAU,OAAO,4BAA4B;;;;;;;;EAsBtE,CAAC"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/handler.js b/packages/spatial-uploads-handler/dist/handler.js index d7855b0a3..d5e8dd30f 100644 --- a/packages/spatial-uploads-handler/dist/handler.js +++ b/packages/spatial-uploads-handler/dist/handler.js @@ -8,7 +8,7 @@ const handleUpload_1 = __importDefault(require("./src/handleUpload")); const processUpload = async (event) => { const s3LogPath = `s3://${process.env.BUCKET}/${event.taskId}.log.txt`; try { - const outputs = await (0, handleUpload_1.default)(event.taskId, event.objectKey, event.suffix, event.requestingUser, event.skipLoggingProgress, event.enableAiDataAnalyst); + const outputs = await (0, handleUpload_1.default)(event.taskId, event.objectKey, event.suffix, event.requestingUser, event.skipLoggingProgress, event.enableAiDataAnalyst, event.processingOptions); return { ...outputs, log: s3LogPath, diff --git a/packages/spatial-uploads-handler/dist/handler.js.map b/packages/spatial-uploads-handler/dist/handler.js.map index ffc13db06..57fcb110c 100644 --- a/packages/spatial-uploads-handler/dist/handler.js.map +++ b/packages/spatial-uploads-handler/dist/handler.js.map @@ -1 +1 @@ -{"version":3,"file":"handler.js","sourceRoot":"","sources":["../handler.ts"],"names":[],"mappings":";;;;;;AAAA,sEAA8C;AAKvC,MAAM,aAAa,GAAG,KAAK,EAAE,KAAmC,EAAE,EAAE;IACzE,MAAM,SAAS,GAAG,QAAQ,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,UAAU,CAAC;IACvE,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,IAAA,sBAAY,EAChC,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,cAAc,EACpB,KAAK,CAAC,mBAAmB,EACzB,KAAK,CAAC,mBAAmB,CAC1B,CAAC;QACF,OAAO;YACL,GAAG,OAAO;YACV,GAAG,EAAE,SAAS;SACf,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,KAAK,EAAG,CAAW,CAAC,OAAO;YAC3B,GAAG,EAAE,SAAS;SACf,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AArBW,QAAA,aAAa,iBAqBxB"} \ No newline at end of file +{"version":3,"file":"handler.js","sourceRoot":"","sources":["../handler.ts"],"names":[],"mappings":";;;;;;AAAA,sEAA8C;AAQvC,MAAM,aAAa,GAAG,KAAK,EAAE,KAAmC,EAAE,EAAE;IACzE,MAAM,SAAS,GAAG,QAAQ,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,UAAU,CAAC;IACvE,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,IAAA,sBAAY,EAChC,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,cAAc,EACpB,KAAK,CAAC,mBAAmB,EACzB,KAAK,CAAC,mBAAmB,EACzB,KAAK,CAAC,iBAAiB,CACxB,CAAC;QACF,OAAO;YACL,GAAG,OAAO;YACV,GAAG,EAAE,SAAS;SACf,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,KAAK,EAAG,CAAW,CAAC,OAAO;YAC3B,GAAG,EAAE,SAAS;SACf,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AAtBW,QAAA,aAAa,iBAsBxB"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/formats/delimited.d.ts b/packages/spatial-uploads-handler/dist/src/formats/delimited.d.ts new file mode 100644 index 000000000..302efc6ce --- /dev/null +++ b/packages/spatial-uploads-handler/dist/src/formats/delimited.d.ts @@ -0,0 +1,13 @@ +import { Logger } from "../logger"; +import type { DelimitedUploadProcessingOptions } from "../spatialUploadsHandlerTypes"; +/** + * Converts a delimited text file (CSV/TSV/TXT) to GeoJSON using GDAL's CSV + * driver, based on the column mapping and CRS chosen by the user (or + * auto-detected client-side). The resulting file feeds into the same + * normalize-to-FlatGeobuf/tiling pipeline used for other vector formats. + * + * See https://gdal.org/en/stable/drivers/vector/csv.html for open option + * reference. + */ +export declare function convertDelimitedToGeoJSON(logger: Logger, inputPath: string, outputPath: string, options: DelimitedUploadProcessingOptions): Promise; +//# sourceMappingURL=delimited.d.ts.map \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/formats/delimited.d.ts.map b/packages/spatial-uploads-handler/dist/src/formats/delimited.d.ts.map new file mode 100644 index 000000000..eccb51222 --- /dev/null +++ b/packages/spatial-uploads-handler/dist/src/formats/delimited.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"delimited.d.ts","sourceRoot":"","sources":["../../../src/formats/delimited.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACnC,OAAO,KAAK,EAAE,gCAAgC,EAAE,MAAM,+BAA+B,CAAC;AAYtF;;;;;;;;GAQG;AACH,wBAAsB,yBAAyB,CAC7C,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,gCAAgC,GACxC,OAAO,CAAC,IAAI,CAAC,CA0Df"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/formats/delimited.js b/packages/spatial-uploads-handler/dist/src/formats/delimited.js new file mode 100644 index 000000000..e889e01fe --- /dev/null +++ b/packages/spatial-uploads-handler/dist/src/formats/delimited.js @@ -0,0 +1,61 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.convertDelimitedToGeoJSON = convertDelimitedToGeoJSON; +const SEPARATOR_BY_DELIMITER = { + ",": "COMMA", + "\t": "TAB", + ";": "SEMICOLON", + "|": "PIPE", +}; +/** + * Converts a delimited text file (CSV/TSV/TXT) to GeoJSON using GDAL's CSV + * driver, based on the column mapping and CRS chosen by the user (or + * auto-detected client-side). The resulting file feeds into the same + * normalize-to-FlatGeobuf/tiling pipeline used for other vector formats. + * + * See https://gdal.org/en/stable/drivers/vector/csv.html for open option + * reference. + */ +async function convertDelimitedToGeoJSON(logger, inputPath, outputPath, options) { + if (options.crs && options.crs !== "EPSG:4326") { + throw new Error("Only WGS 84 (EPSG:4326) decimal degree coordinates are supported for delimited uploads."); + } + const openOptions = [ + "-oo", + `SEPARATOR=${SEPARATOR_BY_DELIMITER[options.delimiter]}`, + "-oo", + `HEADERS=${options.hasHeaderRow ? "YES" : "NO"}`, + // Geometry columns are already represented as the output geometry; don't + // duplicate them as regular attributes. + "-oo", + "KEEP_GEOM_COLUMNS=NO", + ]; + if (options.geometryMode === "wkt") { + if (!options.geometryField) { + throw new Error("processingOptions.geometryField is required when geometryMode is 'wkt'"); + } + openOptions.push("-oo", `GEOM_POSSIBLE_NAMES=${options.geometryField}`); + } + else { + if (!options.xField || !options.yField) { + throw new Error("processingOptions.xField and yField are required when geometryMode is 'point_xy'"); + } + openOptions.push("-oo", `X_POSSIBLE_NAMES=${options.xField}`, "-oo", `Y_POSSIBLE_NAMES=${options.yField}`); + } + await logger.exec([ + "ogr2ogr", + [ + "-f", + "GeoJSON", + "-s_srs", + "EPSG:4326", + "-t_srs", + "EPSG:4326", + "-skipfailures", + ...openOptions, + outputPath, + inputPath, + ], + ], "Problem converting delimited text file to spatial data. Double check the selected columns and coordinate reference system.", 2 / 30); +} +//# sourceMappingURL=delimited.js.map \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/formats/delimited.js.map b/packages/spatial-uploads-handler/dist/src/formats/delimited.js.map new file mode 100644 index 000000000..d3e4361fd --- /dev/null +++ b/packages/spatial-uploads-handler/dist/src/formats/delimited.js.map @@ -0,0 +1 @@ +{"version":3,"file":"delimited.js","sourceRoot":"","sources":["../../../src/formats/delimited.ts"],"names":[],"mappings":";;AAsBA,8DA+DC;AAlFD,MAAM,sBAAsB,GAGxB;IACF,GAAG,EAAE,OAAO;IACZ,IAAI,EAAE,KAAK;IACX,GAAG,EAAE,WAAW;IAChB,GAAG,EAAE,MAAM;CACZ,CAAC;AAEF;;;;;;;;GAQG;AACI,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,SAAiB,EACjB,UAAkB,EAClB,OAAyC;IAEzC,IAAI,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,KAAK,WAAW,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG;QAClB,KAAK;QACL,aAAa,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;QACxD,KAAK;QACL,WAAW,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE;QAChD,yEAAyE;QACzE,wCAAwC;QACxC,KAAK;QACL,sBAAsB;KACvB,CAAC;IAEF,IAAI,OAAO,CAAC,YAAY,KAAK,KAAK,EAAE,CAAC;QACnC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CACb,wEAAwE,CACzE,CAAC;QACJ,CAAC;QACD,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,uBAAuB,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1E,CAAC;SAAM,CAAC;QACN,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CACb,kFAAkF,CACnF,CAAC;QACJ,CAAC;QACD,WAAW,CAAC,IAAI,CACd,KAAK,EACL,oBAAoB,OAAO,CAAC,MAAM,EAAE,EACpC,KAAK,EACL,oBAAoB,OAAO,CAAC,MAAM,EAAE,CACrC,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,CAAC,IAAI,CACf;QACE,SAAS;QACT;YACE,IAAI;YACJ,SAAS;YACT,QAAQ;YACR,WAAW;YACX,QAAQ;YACR,WAAW;YACX,eAAe;YACf,GAAG,WAAW;YACd,UAAU;YACV,SAAS;SACV;KACF,EACD,4HAA4H,EAC5H,CAAC,GAAG,EAAE,CACP,CAAC;AACJ,CAAC"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/handleUpload.d.ts b/packages/spatial-uploads-handler/dist/src/handleUpload.d.ts index 801629256..1c39884ed 100644 --- a/packages/spatial-uploads-handler/dist/src/handleUpload.d.ts +++ b/packages/spatial-uploads-handler/dist/src/handleUpload.d.ts @@ -1,5 +1,6 @@ import { type ProcessedUploadResponse } from "./uploadPipelineShared"; -export type { SpatialUploadsHandlerRequest } from "./spatialUploadsHandlerTypes"; +import type { DelimitedUploadProcessingOptions } from "./spatialUploadsHandlerTypes"; +export type { SpatialUploadsHandlerRequest, DelimitedUploadProcessingOptions, } from "./spatialUploadsHandlerTypes"; export { MAX_OUTPUT_SIZE, MVT_THRESHOLD, } from "./uploadPipelineShared"; export type { ProcessedUploadLayer, ProcessedUploadResponse, ProgressUpdater, ResponseOutput, SupportedTypes, } from "./uploadPipelineShared"; export default function handleUpload( @@ -14,5 +15,7 @@ slug: string, */ requestingUser: string, skipLoggingProgress?: boolean, /** When true, run column intelligence / title / attribution LLMs (requires CF_AIG_* env). */ -enableAiDataAnalyst?: boolean): Promise; +enableAiDataAnalyst?: boolean, +/** Column mapping / CRS for delimited (CSV/TSV/TXT) uploads. */ +processingOptions?: DelimitedUploadProcessingOptions): Promise; //# sourceMappingURL=handleUpload.d.ts.map \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/handleUpload.d.ts.map b/packages/spatial-uploads-handler/dist/src/handleUpload.d.ts.map index 1858957e4..a9a5baf15 100644 --- a/packages/spatial-uploads-handler/dist/src/handleUpload.d.ts.map +++ b/packages/spatial-uploads-handler/dist/src/handleUpload.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"handleUpload.d.ts","sourceRoot":"","sources":["../../src/handleUpload.ts"],"names":[],"mappings":"AAoBA,OAAO,EAGL,KAAK,uBAAuB,EAG7B,MAAM,wBAAwB,CAAC;AAEhC,YAAY,EAAE,4BAA4B,EAAE,MAAM,8BAA8B,CAAC;AAEjF,OAAO,EACL,eAAe,EACf,aAAa,GACd,MAAM,wBAAwB,CAAC;AAChC,YAAY,EACV,oBAAoB,EACpB,uBAAuB,EACvB,eAAe,EACf,cAAc,EACd,cAAc,GACf,MAAM,wBAAwB,CAAC;AAIhC,wBAA8B,YAAY;AACxC,mCAAmC;AACnC,KAAK,EAAE,MAAM;AACb,yBAAyB;AACzB,SAAS,EAAE,MAAM;AACjB,yBAAyB;AACzB,IAAI,EAAE,MAAM;AACZ;;GAEG;AACH,cAAc,EAAE,MAAM,EACtB,mBAAmB,CAAC,EAAE,OAAO;AAC7B,6FAA6F;AAC7F,mBAAmB,CAAC,EAAE,OAAO,GAC5B,OAAO,CAAC,uBAAuB,CAAC,CAgRlC"} \ No newline at end of file +{"version":3,"file":"handleUpload.d.ts","sourceRoot":"","sources":["../../src/handleUpload.ts"],"names":[],"mappings":"AAoBA,OAAO,EAGL,KAAK,uBAAuB,EAG7B,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,gCAAgC,EAAE,MAAM,8BAA8B,CAAC;AAErF,YAAY,EACV,4BAA4B,EAC5B,gCAAgC,GACjC,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,eAAe,EACf,aAAa,GACd,MAAM,wBAAwB,CAAC;AAChC,YAAY,EACV,oBAAoB,EACpB,uBAAuB,EACvB,eAAe,EACf,cAAc,EACd,cAAc,GACf,MAAM,wBAAwB,CAAC;AAIhC,wBAA8B,YAAY;AACxC,mCAAmC;AACnC,KAAK,EAAE,MAAM;AACb,yBAAyB;AACzB,SAAS,EAAE,MAAM;AACjB,yBAAyB;AACzB,IAAI,EAAE,MAAM;AACZ;;GAEG;AACH,cAAc,EAAE,MAAM,EACtB,mBAAmB,CAAC,EAAE,OAAO;AAC7B,6FAA6F;AAC7F,mBAAmB,CAAC,EAAE,OAAO;AAC7B,gEAAgE;AAChE,iBAAiB,CAAC,EAAE,gCAAgC,GACnD,OAAO,CAAC,uBAAuB,CAAC,CAiRlC"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/handleUpload.js b/packages/spatial-uploads-handler/dist/src/handleUpload.js index f1c2d474e..1432abb5c 100644 --- a/packages/spatial-uploads-handler/dist/src/handleUpload.js +++ b/packages/spatial-uploads-handler/dist/src/handleUpload.js @@ -66,7 +66,9 @@ slug, */ requestingUser, skipLoggingProgress, /** When true, run column intelligence / title / attribution LLMs (requires CF_AIG_* env). */ -enableAiDataAnalyst) { +enableAiDataAnalyst, +/** Column mapping / CRS for delimited (CSV/TSV/TXT) uploads. */ +processingOptions) { if (DEBUG) { console.log("DEBUG MODE ENABLED"); } @@ -154,6 +156,7 @@ enableAiDataAnalyst) { uploadFilename, workingDirectory: dist, enableAiDataAnalyst, + processingOptions, }); stats = vectorResult.layers; aiDataAnalystNotesPromise = vectorResult.aiDataAnalystNotesPromise; diff --git a/packages/spatial-uploads-handler/dist/src/handleUpload.js.map b/packages/spatial-uploads-handler/dist/src/handleUpload.js.map index 2313fd446..5552a91bb 100644 --- a/packages/spatial-uploads-handler/dist/src/handleUpload.js.map +++ b/packages/spatial-uploads-handler/dist/src/handleUpload.js.map @@ -1 +1 @@ -{"version":3,"file":"handleUpload.js","sourceRoot":"","sources":["../../src/handleUpload.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4CA,+BA8RC;AA1UD,yDAA+C;AAC/C,6BAA8B;AAC9B,2BAAmC;AACnC,2CAA6B;AAC7B,8DAImC;AACnC,kDAAgE;AAChE,MAAM,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,+BAA+B,CAAC,CAAC;AAClE,+DAA4D;AAC5D,+DAA4D;AAC5D,6DAA0D;AAC1D,uCAAiD;AACjD,qCAAkC;AAGlC,+BAAgC;AAChC,8CAA+C;AAC/C,iEAMgC;AAIhC,+DAGgC;AAF9B,uHAAA,eAAe,OAAA;AACf,qHAAA,aAAa,OAAA;AAUf,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,MAAM,CAAC;AAE5B,KAAK,UAAU,YAAY;AACxC,mCAAmC;AACnC,KAAa;AACb,yBAAyB;AACzB,SAAiB;AACjB,yBAAyB;AACzB,IAAY;AACZ;;GAEG;AACH,cAAsB,EACtB,mBAA6B;AAC7B,6FAA6F;AAC7F,mBAA6B;IAE7B,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAA,4BAAS,GAAE,CAAC;IACnC,MAAM,OAAO,GAA2C,EAAE,CAAC;IAC3D,MAAM,OAAO,GAAG,YAAY,IAAI,SAAS,CAAC;IAE1C;;;;OAIG;IACH,MAAM,cAAc,GAAoB,KAAK,EAC3C,KAAwC,EACxC,eAAuB,EACvB,QAAiB,EACjB,EAAE;QACF,IAAI,mBAAmB,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,QAAQ,CAAC,KAAK,CAClB,gIAAgI,EAChI,CAAC,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,CAAC,CAC1C,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,CAAC,KAAK,CAClB,uGAAuG,EACvG,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,CAChC,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;IAEF,qEAAqE;IACrE,+CAA+C;IAC/C,MAAM,MAAM,GAAG,IAAA,aAAO,EAAC;QACrB,aAAa,EAAE,IAAI;QACnB,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,UAAU;KACnB,CAAC,CAAC;IAEH,6EAA6E;IAC7E,sBAAsB;IACtB,MAAM,MAAM,GAAG,IAAI,eAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;QACnD,IAAI,mBAAmB,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,QAAQ,CAAC,KAAK,CACnC,+FAA+F,EAC/F,CAAC,QAAQ,EAAE,KAAK,CAAC,CAClB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5C,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,4BAA4B,EAAE,CAAC,CAAC,CAAC;IAEtE,MAAM,SAAS,GAAG,QAAQ,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,KAAK,UAAU,CAAC;IAChE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAEhD,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IACtB,MAAM,YAAY,GAAG,IAAI,CAAC;IAC1B,IAAI,GAAG,GAAG,KAAK,EAAE,CAAC;IAClB,MAAM,KAAK,GAAG,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,OAAO,CAAC;IAEhD,0CAA0C;IAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,eAAe,GAAG,GAAG,IAAI,CAAC,IAAI,CAChC,MAAM,CAAC,IAAI,EACX,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAC9B,EAAE,CAAC;IACJ,MAAM,cAAc,CAAC,SAAS,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,IAAA,mBAAS,EACb,eAAe,EACf,QAAQ,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAO,EAAE,SAAS,CAAC,EAAE,EACnD,MAAM,EACN,CAAC,GAAG,EAAE,CACP,CAAC;IAEF,IAAI,KAAmC,CAAC;IACxC,IAAI,yBAAkE,CAAC;IAEvE,MAAM,cAAc,GAAG,YAAY,GAAG,GAAG,CAAC;IAE1C,8EAA8E;IAC9E,cAAc;IACd,IAAI,CAAC;QACH,IAAI,KAAK,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YAC3B,MAAM,YAAY,GAAG,MAAM,IAAA,yCAAmB,EAAC;gBAC7C,MAAM;gBACN,IAAI,EAAE,eAAe;gBACrB,OAAO;gBACP,cAAc;gBACd,OAAO;gBACP,KAAK;gBACL,YAAY;gBACZ,cAAc;gBACd,gBAAgB,EAAE,IAAI;gBACtB,mBAAmB;aACpB,CAAC,CAAC;YACH,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;YAChC,yBAAyB,GAAG,YAAY,CAAC,yBAAyB,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,MAAM,YAAY,GAAG,MAAM,IAAA,yCAAmB,EAAC;gBAC7C,MAAM;gBACN,IAAI,EAAE,eAAe;gBACrB,OAAO;gBACP,cAAc;gBACd,OAAO;gBACP,KAAK;gBACL,YAAY;gBACZ,cAAc;gBACd,gBAAgB,EAAE,IAAI;gBACtB,mBAAmB;aACpB,CAAC,CAAC;YACH,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC;YAC5B,yBAAyB,GAAG,YAAY,CAAC,yBAAyB,CAAC;QACrE,CAAC;QAED,iCAAiC;QACjC,IAAI,MAA4B,CAAC;QACjC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACjC,CAAC;QAED,qDAAqD;QACrD,MAAM,cAAc,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;QACtD,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,sCAAe,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CACb,8BAA8B,KAAK,CACjC,sCAAe,CAChB,eAAe,KAAK,CACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,sCAAe,CAAE,CAAC,IAAI,CACpD,EAAE,CACJ,CAAC;QACJ,CAAC;QACD,kCAAkC;QAClC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,IAAA,mBAAS,EAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,cAAc,GAAG,CAAC,KAAK,IAAI,GAAG,KAAK,KAAK,CAAC;QAC/C,IAAI,mBAAmB,EAAE,CAAC;YACxB,MAAM,cAAc,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;QACrD,CAAC;aAAM,IAAI,cAAc,EAAE,CAAC;YAC1B,MAAM,cAAc,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;QACrD,CAAC;aAAM,CAAC;YACN,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAChD,CAAC;QACD,MAAM,kBAAkB,GAAG,MAAM,yBAAyB,CAAC;QAE3D,MAAM,cAAc,CAAC,SAAS,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC;QAEtD,wEAAwE;QACxE,IAAI,SAA6B,CAAC;QAClC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC9B,SAAS,GAAG,MAAM,CAAC,GAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC;iBAAM,IACL,MAAM,CAAC,IAAI,KAAK,SAAS;gBACzB,CAAC,MAAM,CAAC,UAAU;gBAClB,CAAC,SAAS,EACV,CAAC;gBACD,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC;YACzB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QAED,uCAAuC;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAClD,IAAA,kBAAa,EAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,IAAA,mBAAS,EAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAEzD,MAAM,QAAQ,GAAwD;YACpE,MAAM,EAAE;gBACN;oBACE,QAAQ,EAAE,cAAc;oBACxB,IAAI,EAAE,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK;oBACzD,QAAQ;oBACR,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAC3B,GAAG,CAAC;wBACJ,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;qBACrB,CAAC,CAAC;oBACH,MAAM,EAAE,MAAM,IAAI,SAAS;oBAC3B,GAAG,EAAE,SAAS;oBACd,kBAAkB,EAChB,KAAK;wBACJ,KAAoB,CAAC,YAAY;4BAChC,4CAA2B,CAAC,UAAU;oBAC1C,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACtD;aACF;YACD,OAAO,EAAE,SAAS;SACnB,CAAC;QACF,0CAA0C;QAC1C,MAAM,QAAQ,CAAC,KAAK,CAClB,sEAAsE,EACtE;YACE,IAAI,CAAC,SAAS,CAAC;gBACb,KAAK,EAAE,KAAK;gBACZ,IAAI,EAAE,QAAQ;aACf,CAAC;SACH,CACF,CAAC;QACF,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,KAAK,GAAG,CAAU,CAAC;QACzB,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,CAC5B,gIAAgI,EAChI,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CACrC,CAAC;QACJ,CAAC;QACD,IACE,OAAO,CAAC,GAAG,CAAC,WAAW;YACvB,OAAO,CAAC,GAAG,CAAC,aAAa;YACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EACrC,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,4BAAgB,CAAC;gBACnC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM;gBAC1B,GAAG,EAAE,SAAS;aACf,CAAC,CAAC;YACH,MAAM,oBAAoB,GAAG,MAAM,YAAY,CAC7C,OAAO,CAAC,GAAG,CAAC,2BAA2B;gBACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B;gBAC3C,CAAC,CAAC,IAAI,oBAAQ,CAAC;oBACX,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,UAAW;oBAC/B,WAAW,EAAE;wBACX,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,2BAA2B;wBACpD,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,+BAA+B;qBAC7D;iBACF,CAAC;gBACJ,CAAC,CAAC,QAAQ,EACZ,OAAO,EACP;gBACE,SAAS,EAAE,MAAM;aAClB,CACF,CAAC;YAEF,MAAM,IAAA,uCAAkB,EACtB,YAAY,GAAG,GAAG,EAClB,oBAAoB,EACpB,MAAM,CAAC,MAAM,EACb,OAAO,CAAC,GAAG,CAAC,MAAO,EACnB,SAAS,EACT,cAAc,EACd,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,CAC5B,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAClD,IAAA,kBAAa,EAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,IAAA,mBAAS,EAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,QAAQ,GAAG,IAAI,oBAAQ,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,UAAW,EAAE,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"handleUpload.js","sourceRoot":"","sources":["../../src/handleUpload.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgDA,+BAiSC;AAjVD,yDAA+C;AAC/C,6BAA8B;AAC9B,2BAAmC;AACnC,2CAA6B;AAC7B,8DAImC;AACnC,kDAAgE;AAChE,MAAM,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,+BAA+B,CAAC,CAAC;AAClE,+DAA4D;AAC5D,+DAA4D;AAC5D,6DAA0D;AAC1D,uCAAiD;AACjD,qCAAkC;AAGlC,+BAAgC;AAChC,8CAA+C;AAC/C,iEAMgC;AAQhC,+DAGgC;AAF9B,uHAAA,eAAe,OAAA;AACf,qHAAA,aAAa,OAAA;AAUf,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,MAAM,CAAC;AAE5B,KAAK,UAAU,YAAY;AACxC,mCAAmC;AACnC,KAAa;AACb,yBAAyB;AACzB,SAAiB;AACjB,yBAAyB;AACzB,IAAY;AACZ;;GAEG;AACH,cAAsB,EACtB,mBAA6B;AAC7B,6FAA6F;AAC7F,mBAA6B;AAC7B,gEAAgE;AAChE,iBAAoD;IAEpD,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAA,4BAAS,GAAE,CAAC;IACnC,MAAM,OAAO,GAA2C,EAAE,CAAC;IAC3D,MAAM,OAAO,GAAG,YAAY,IAAI,SAAS,CAAC;IAE1C;;;;OAIG;IACH,MAAM,cAAc,GAAoB,KAAK,EAC3C,KAAwC,EACxC,eAAuB,EACvB,QAAiB,EACjB,EAAE;QACF,IAAI,mBAAmB,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,QAAQ,CAAC,KAAK,CAClB,gIAAgI,EAChI,CAAC,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,CAAC,CAC1C,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,CAAC,KAAK,CAClB,uGAAuG,EACvG,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,CAChC,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;IAEF,qEAAqE;IACrE,+CAA+C;IAC/C,MAAM,MAAM,GAAG,IAAA,aAAO,EAAC;QACrB,aAAa,EAAE,IAAI;QACnB,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,UAAU;KACnB,CAAC,CAAC;IAEH,6EAA6E;IAC7E,sBAAsB;IACtB,MAAM,MAAM,GAAG,IAAI,eAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;QACnD,IAAI,mBAAmB,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,QAAQ,CAAC,KAAK,CACnC,+FAA+F,EAC/F,CAAC,QAAQ,EAAE,KAAK,CAAC,CAClB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5C,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,4BAA4B,EAAE,CAAC,CAAC,CAAC;IAEtE,MAAM,SAAS,GAAG,QAAQ,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,KAAK,UAAU,CAAC;IAChE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAEhD,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IACtB,MAAM,YAAY,GAAG,IAAI,CAAC;IAC1B,IAAI,GAAG,GAAG,KAAK,EAAE,CAAC;IAClB,MAAM,KAAK,GAAG,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,OAAO,CAAC;IAEhD,0CAA0C;IAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,eAAe,GAAG,GAAG,IAAI,CAAC,IAAI,CAChC,MAAM,CAAC,IAAI,EACX,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAC9B,EAAE,CAAC;IACJ,MAAM,cAAc,CAAC,SAAS,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,IAAA,mBAAS,EACb,eAAe,EACf,QAAQ,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAO,EAAE,SAAS,CAAC,EAAE,EACnD,MAAM,EACN,CAAC,GAAG,EAAE,CACP,CAAC;IAEF,IAAI,KAAmC,CAAC;IACxC,IAAI,yBAAkE,CAAC;IAEvE,MAAM,cAAc,GAAG,YAAY,GAAG,GAAG,CAAC;IAE1C,8EAA8E;IAC9E,cAAc;IACd,IAAI,CAAC;QACH,IAAI,KAAK,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YAC3B,MAAM,YAAY,GAAG,MAAM,IAAA,yCAAmB,EAAC;gBAC7C,MAAM;gBACN,IAAI,EAAE,eAAe;gBACrB,OAAO;gBACP,cAAc;gBACd,OAAO;gBACP,KAAK;gBACL,YAAY;gBACZ,cAAc;gBACd,gBAAgB,EAAE,IAAI;gBACtB,mBAAmB;aACpB,CAAC,CAAC;YACH,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;YAChC,yBAAyB,GAAG,YAAY,CAAC,yBAAyB,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,MAAM,YAAY,GAAG,MAAM,IAAA,yCAAmB,EAAC;gBAC7C,MAAM;gBACN,IAAI,EAAE,eAAe;gBACrB,OAAO;gBACP,cAAc;gBACd,OAAO;gBACP,KAAK;gBACL,YAAY;gBACZ,cAAc;gBACd,gBAAgB,EAAE,IAAI;gBACtB,mBAAmB;gBACnB,iBAAiB;aAClB,CAAC,CAAC;YACH,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC;YAC5B,yBAAyB,GAAG,YAAY,CAAC,yBAAyB,CAAC;QACrE,CAAC;QAED,iCAAiC;QACjC,IAAI,MAA4B,CAAC;QACjC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACjC,CAAC;QAED,qDAAqD;QACrD,MAAM,cAAc,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;QACtD,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,sCAAe,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CACb,8BAA8B,KAAK,CACjC,sCAAe,CAChB,eAAe,KAAK,CACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,sCAAe,CAAE,CAAC,IAAI,CACpD,EAAE,CACJ,CAAC;QACJ,CAAC;QACD,kCAAkC;QAClC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,IAAA,mBAAS,EAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,cAAc,GAAG,CAAC,KAAK,IAAI,GAAG,KAAK,KAAK,CAAC;QAC/C,IAAI,mBAAmB,EAAE,CAAC;YACxB,MAAM,cAAc,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;QACrD,CAAC;aAAM,IAAI,cAAc,EAAE,CAAC;YAC1B,MAAM,cAAc,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;QACrD,CAAC;aAAM,CAAC;YACN,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAChD,CAAC;QACD,MAAM,kBAAkB,GAAG,MAAM,yBAAyB,CAAC;QAE3D,MAAM,cAAc,CAAC,SAAS,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC;QAEtD,wEAAwE;QACxE,IAAI,SAA6B,CAAC;QAClC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC9B,SAAS,GAAG,MAAM,CAAC,GAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC;iBAAM,IACL,MAAM,CAAC,IAAI,KAAK,SAAS;gBACzB,CAAC,MAAM,CAAC,UAAU;gBAClB,CAAC,SAAS,EACV,CAAC;gBACD,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC;YACzB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QAED,uCAAuC;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAClD,IAAA,kBAAa,EAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,IAAA,mBAAS,EAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAEzD,MAAM,QAAQ,GAAwD;YACpE,MAAM,EAAE;gBACN;oBACE,QAAQ,EAAE,cAAc;oBACxB,IAAI,EAAE,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK;oBACzD,QAAQ;oBACR,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAC3B,GAAG,CAAC;wBACJ,KAAK,EAAE,SAAS;wBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;qBACrB,CAAC,CAAC;oBACH,MAAM,EAAE,MAAM,IAAI,SAAS;oBAC3B,GAAG,EAAE,SAAS;oBACd,kBAAkB,EAChB,KAAK;wBACJ,KAAoB,CAAC,YAAY;4BAChC,4CAA2B,CAAC,UAAU;oBAC1C,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACtD;aACF;YACD,OAAO,EAAE,SAAS;SACnB,CAAC;QACF,0CAA0C;QAC1C,MAAM,QAAQ,CAAC,KAAK,CAClB,sEAAsE,EACtE;YACE,IAAI,CAAC,SAAS,CAAC;gBACb,KAAK,EAAE,KAAK;gBACZ,IAAI,EAAE,QAAQ;aACf,CAAC;SACH,CACF,CAAC;QACF,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjB,MAAM,KAAK,GAAG,CAAU,CAAC;QACzB,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,CAC5B,gIAAgI,EAChI,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CACrC,CAAC;QACJ,CAAC;QACD,IACE,OAAO,CAAC,GAAG,CAAC,WAAW;YACvB,OAAO,CAAC,GAAG,CAAC,aAAa;YACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EACrC,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,4BAAgB,CAAC;gBACnC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM;gBAC1B,GAAG,EAAE,SAAS;aACf,CAAC,CAAC;YACH,MAAM,oBAAoB,GAAG,MAAM,YAAY,CAC7C,OAAO,CAAC,GAAG,CAAC,2BAA2B;gBACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B;gBAC3C,CAAC,CAAC,IAAI,oBAAQ,CAAC;oBACX,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,UAAW;oBAC/B,WAAW,EAAE;wBACX,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,2BAA2B;wBACpD,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,+BAA+B;qBAC7D;iBACF,CAAC;gBACJ,CAAC,CAAC,QAAQ,EACZ,OAAO,EACP;gBACE,SAAS,EAAE,MAAM;aAClB,CACF,CAAC;YAEF,MAAM,IAAA,uCAAkB,EACtB,YAAY,GAAG,GAAG,EAClB,oBAAoB,EACpB,MAAM,CAAC,MAAM,EACb,OAAO,CAAC,GAAG,CAAC,MAAO,EACnB,SAAS,EACT,cAAc,EACd,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,CAC5B,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAClD,IAAA,kBAAa,EAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,IAAA,mBAAS,EAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,QAAQ,GAAG,IAAI,oBAAQ,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,UAAW,EAAE,CAAC,CAAC"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/index.d.ts b/packages/spatial-uploads-handler/dist/src/index.d.ts index d0a4f4771..df4d00173 100644 --- a/packages/spatial-uploads-handler/dist/src/index.d.ts +++ b/packages/spatial-uploads-handler/dist/src/index.d.ts @@ -1,4 +1,4 @@ -export { ProcessedUploadLayer, SpatialUploadsHandlerRequest, ProcessedUploadResponse, } from "./handleUpload"; +export { ProcessedUploadLayer, SpatialUploadsHandlerRequest, ProcessedUploadResponse, DelimitedUploadProcessingOptions, } from "./handleUpload"; /** Invoke the geostats PII classifier Lambda (e.g. backfill tasks). */ export { classifyGeostatsPii } from "./aiUploadNotes"; //# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/index.d.ts.map b/packages/spatial-uploads-handler/dist/src/index.d.ts.map index a8b102f8e..79c65f464 100644 --- a/packages/spatial-uploads-handler/dist/src/index.d.ts.map +++ b/packages/spatial-uploads-handler/dist/src/index.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,oBAAoB,EACpB,4BAA4B,EAC5B,uBAAuB,GACxB,MAAM,gBAAgB,CAAC;AAExB,uEAAuE;AACvE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC"} \ No newline at end of file +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,oBAAoB,EACpB,4BAA4B,EAC5B,uBAAuB,EACvB,gCAAgC,GACjC,MAAM,gBAAgB,CAAC;AAExB,uEAAuE;AACvE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/index.js.map b/packages/spatial-uploads-handler/dist/src/index.js.map index fb77a08aa..d37ca0271 100644 --- a/packages/spatial-uploads-handler/dist/src/index.js.map +++ b/packages/spatial-uploads-handler/dist/src/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;AAMA,uEAAuE;AACvE,iDAAsD;AAA7C,oHAAA,mBAAmB,OAAA"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;AAOA,uEAAuE;AACvE,iDAAsD;AAA7C,oHAAA,mBAAmB,OAAA"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/processVectorUpload.d.ts b/packages/spatial-uploads-handler/dist/src/processVectorUpload.d.ts index 91b44558b..84b6abad6 100644 --- a/packages/spatial-uploads-handler/dist/src/processVectorUpload.d.ts +++ b/packages/spatial-uploads-handler/dist/src/processVectorUpload.d.ts @@ -2,6 +2,7 @@ import { GeostatsLayer } from "@seasketch/geostats-types"; import { type ProgressUpdater, type ResponseOutput } from "./uploadPipelineShared"; import { Logger } from "./logger"; import { type AiDataAnalystNotes } from "ai-data-analyst"; +import type { DelimitedUploadProcessingOptions } from "./spatialUploadsHandlerTypes"; export default function fromMarkdown(md: string): any; /** * Process a vector upload, converting it to a normalized FlatGeobuf file and @@ -30,6 +31,8 @@ export declare function processVectorUpload(options: { uploadFilename: string; /** When true, run title / attribution / column intelligence LLMs (requires CF_AIG_* env). */ enableAiDataAnalyst?: boolean; + /** Column mapping / CRS for delimited (CSV/TSV/TXT) uploads. Required if `path` has a delimited extension. */ + processingOptions?: DelimitedUploadProcessingOptions; }): Promise<{ layers: GeostatsLayer[]; /** Resolve after local processing; caller should await after uploads so LLMs overlap I/O. */ diff --git a/packages/spatial-uploads-handler/dist/src/processVectorUpload.d.ts.map b/packages/spatial-uploads-handler/dist/src/processVectorUpload.d.ts.map index 4afcef27b..1b0cac095 100644 --- a/packages/spatial-uploads-handler/dist/src/processVectorUpload.d.ts.map +++ b/packages/spatial-uploads-handler/dist/src/processVectorUpload.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"processVectorUpload.d.ts","sourceRoot":"","sources":["../../src/processVectorUpload.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAoB,MAAM,2BAA2B,CAAC;AAC5E,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,cAAc,EAEpB,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAGlC,OAAO,EAIL,KAAK,kBAAkB,EAGxB,MAAM,iBAAiB,CAAC;AAQzB,MAAM,CAAC,OAAO,UAAU,YAAY,CAAC,EAAE,EAAE,MAAM,OAE9C;AAkFD;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,EAAE;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,8BAA8B;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,sDAAsD;IACtD,OAAO,EAAE,CAAC,cAAc,GAAG;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,EAAE,CAAC;IAChD,cAAc,EAAE,eAAe,CAAC;IAEhC,OAAO,EAAE,MAAM,CAAC;IAChB,6DAA6D;IAC7D,gBAAgB,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,YAAY,EAAE,MAAM,CAAC;IACrB,yEAAyE;IACzE,cAAc,EAAE,MAAM,CAAC;IACvB,6FAA6F;IAC7F,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,6FAA6F;IAC7F,yBAAyB,EAAE,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAAC;CACpE,CAAC,CAicD"} \ No newline at end of file +{"version":3,"file":"processVectorUpload.d.ts","sourceRoot":"","sources":["../../src/processVectorUpload.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAoB,MAAM,2BAA2B,CAAC;AAC5E,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,cAAc,EAEpB,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAGlC,OAAO,EAIL,KAAK,kBAAkB,EAGxB,MAAM,iBAAiB,CAAC;AAQzB,OAAO,KAAK,EAAE,gCAAgC,EAAE,MAAM,8BAA8B,CAAC;AAIrF,MAAM,CAAC,OAAO,UAAU,YAAY,CAAC,EAAE,EAAE,MAAM,OAE9C;AAkFD;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,EAAE;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,8BAA8B;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,sDAAsD;IACtD,OAAO,EAAE,CAAC,cAAc,GAAG;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,EAAE,CAAC;IAChD,cAAc,EAAE,eAAe,CAAC;IAEhC,OAAO,EAAE,MAAM,CAAC;IAChB,6DAA6D;IAC7D,gBAAgB,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,YAAY,EAAE,MAAM,CAAC;IACrB,yEAAyE;IACzE,cAAc,EAAE,MAAM,CAAC;IACvB,6FAA6F;IAC7F,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,8GAA8G;IAC9G,iBAAiB,CAAC,EAAE,gCAAgC,CAAC;CACtD,GAAG,OAAO,CAAC;IACV,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,6FAA6F;IAC7F,yBAAyB,EAAE,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAAC;CACpE,CAAC,CA+dD"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/processVectorUpload.js b/packages/spatial-uploads-handler/dist/src/processVectorUpload.js index 6fe4e611e..f8e1f3184 100644 --- a/packages/spatial-uploads-handler/dist/src/processVectorUpload.js +++ b/packages/spatial-uploads-handler/dist/src/processVectorUpload.js @@ -10,6 +10,8 @@ const prosemirror_markdown_1 = require("prosemirror-markdown"); const metadata_parser_1 = require("@seasketch/metadata-parser"); const ai_data_analyst_1 = require("ai-data-analyst"); const aiUploadNotes_1 = require("./aiUploadNotes"); +const delimited_1 = require("./formats/delimited"); +const DELIMITED_EXTENSIONS = new Set([".csv", ".tsv", ".txt"]); function fromMarkdown(md) { var _a; return (_a = prosemirror_markdown_1.defaultMarkdownParser.parse(md)) === null || _a === void 0 ? void 0 : _a.toJSON(); @@ -94,13 +96,23 @@ async function collectAttributionSidecarContents(logger, workingDirectory, worki */ async function processVectorUpload(options) { var _a; - const { logger, path, outputs, updateProgress, baseKey, workingDirectory, jobId, originalName, uploadFilename, enableAiDataAnalyst, } = options; + const { logger, path, outputs, updateProgress, baseKey, workingDirectory, jobId, originalName, uploadFilename, enableAiDataAnalyst, processingOptions, } = options; if (enableAiDataAnalyst) { (0, aiUploadNotes_1.assertAiDataAnalystEnvVarsPresent)(); } const originalFilePath = path; let workingFilePath = path; await updateProgress("running", "validating"); + const isDelimited = DELIMITED_EXTENSIONS.has((0, path_1.parse)(path).ext); + if (isDelimited) { + if (!processingOptions) { + throw new Error("This file requires column mapping configuration before it can be uploaded. Please configure coordinate columns and try again."); + } + const convertedPath = (0, path_1.join)(workingDirectory, jobId + ".geojson"); + await updateProgress("running", "converting delimited text"); + await (0, delimited_1.convertDelimitedToGeoJSON)(logger, workingFilePath, convertedPath, processingOptions); + workingFilePath = convertedPath; + } let titleP = enableAiDataAnalyst ? (0, aiUploadNotes_1.asNeverReject)((0, ai_data_analyst_1.generateTitle)(uploadFilename), "generateTitle") : null; @@ -240,6 +252,14 @@ async function processVectorUpload(options) { else { throw new Error("Not a recognized file type"); } + if (isDelimited) { + if (/Feature Count: 0\b/.test(ogrInfo)) { + throw new Error("No spatial features could be extracted from this file using the selected columns. Double check the column mapping and try again."); + } + // The original output should reflect the file as uploaded (delimited + // text), not the GeoJSON it was converted to above. + type = "CSV"; + } const sidecarFiles = await collectAttributionSidecarContents(logger, workingDirectory, workingFilePath, isZip || isRar); const attributionInputs = []; for (const s of sidecarFiles) { diff --git a/packages/spatial-uploads-handler/dist/src/processVectorUpload.js.map b/packages/spatial-uploads-handler/dist/src/processVectorUpload.js.map index cb5118961..16c0ce56d 100644 --- a/packages/spatial-uploads-handler/dist/src/processVectorUpload.js.map +++ b/packages/spatial-uploads-handler/dist/src/processVectorUpload.js.map @@ -1 +1 @@ -{"version":3,"file":"processVectorUpload.js","sourceRoot":"","sources":["../../src/processVectorUpload.ts"],"names":[],"mappings":";;AA4BA,+BAEC;AAyFD,kDAwdC;AA9kBD,iEAKgC;AAChC,+BAA4D;AAC5D,2BAAwD;AACxD,qEAAmE;AAEnE,+DAA6D;AAC7D,gEAAmE;AACnE,qDAOyB;AACzB,mDAKyB;AAEzB,SAAwB,YAAY,CAAC,EAAU;;IAC7C,OAAO,MAAA,4CAAqB,CAAC,KAAK,CAAC,EAAE,CAAC,0CAAE,MAAM,EAAE,CAAC;AACnD,CAAC;AAID,KAAK,UAAU,iCAAiC,CAC9C,MAAc,EACd,gBAAwB,EACxB,eAAuB,EACvB,WAAoB;IAEpB,MAAM,OAAO,GAA6C,EAAE,CAAC;IAC7D,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,IAAI,CACpC;YACE,MAAM;YACN;gBACE,gBAAgB;gBAChB,OAAO;gBACP,GAAG;gBACH,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,GAAG;gBACH,OAAO;gBACP,OAAO;gBACP,IAAI;gBACJ,OAAO;gBACP,QAAQ;gBACR,GAAG;aACJ;SACF,EACD,yDAAyD,EACzD,CAAC,CACF,CAAC;QACF,IAAI,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,IAAI,EAAE,EAAE,CAAC;YACzB,KAAK,MAAM,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBACtB,IAAI,CAAC,CAAC,IAAI,CAAC,IAAA,eAAU,EAAC,CAAC,CAAC,EAAE,CAAC;oBACzB,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,GAAG,IAAA,YAAS,EAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;gBAC3C,MAAM,IAAI,GAAgB,GAAG,KAAK,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBAC3D,IAAI,CAAC;oBACH,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAA,iBAAY,EAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC3D,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,IAAA,YAAS,EAAC,eAAe,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG;YACZ,IAAI;YACJ,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/D,CAAC;QACF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;gBAC3C,MAAM,SAAS,GAAG,IAAA,WAAQ,EAAC,GAAG,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC;gBAClD,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;oBACxB,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACpB,IAAI,IAAA,eAAU,EAAC,SAAS,CAAC,EAAE,CAAC;oBAC1B,IAAI,CAAC;wBACH,OAAO,CAAC,IAAI,CAAC;4BACX,IAAI,EAAE,GAAG;4BACT,OAAO,EAAE,IAAA,iBAAY,EAAC,SAAS,EAAE,MAAM,CAAC;yBACzC,CAAC,CAAC;oBACL,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;oBACjE,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,mBAAmB,CAAC,OAmBzC;;IAKC,MAAM,EACJ,MAAM,EACN,IAAI,EACJ,OAAO,EACP,cAAc,EACd,OAAO,EACP,gBAAgB,EAChB,KAAK,EACL,YAAY,EACZ,cAAc,EACd,mBAAmB,GACpB,GAAG,OAAO,CAAC;IACZ,IAAI,mBAAmB,EAAE,CAAC;QACxB,IAAA,iDAAiC,GAAE,CAAC;IACtC,CAAC;IACD,MAAM,gBAAgB,GAAG,IAAI,CAAC;IAC9B,IAAI,eAAe,GAAG,IAAI,CAAC;IAC3B,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAE9C,IAAI,MAAM,GAAG,mBAAmB;QAC9B,CAAC,CAAC,IAAA,6BAAa,EAAC,IAAA,+BAAa,EAAC,cAAc,CAAC,EAAE,eAAe,CAAC;QAC/D,CAAC,CAAC,IAAI,CAAC;IACT,IAAI,YAAY,GAEL,IAAI,CAAC;IAChB,IAAI,OAAO,GAEA,IAAI,CAAC;IAEhB,IAAI,IAAoB,CAAC;IACzB,IAAI,EAAE,GAAG,EAAE,GAAG,IAAA,YAAS,EAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,GAAG,KAAK,MAAM,CAAC;IAC7B,MAAM,KAAK,GAAG,GAAG,KAAK,MAAM,CAAC;IAC7B,IAAI,QAAQ,GAA4B,IAAI,CAAC;IAE7C,2DAA2D;IAC3D,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;QACnB,MAAM,cAAc,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC7C,oBAAoB;QACpB,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,MAAM,CAAC,IAAI,CACf,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC,EAC1D,wBAAwB,EACxB,CAAC,GAAG,EAAE,CACP,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,CAAC,IAAI,CACf,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,GAAG,gBAAgB,EAAE,eAAe,CAAC,CAAC,EAC3D,8BAA8B,EAC9B,CAAC,GAAG,EAAE,CACP,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;QACpD,qDAAqD;QACrD,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,IAAI,CACjC;YACE,MAAM;YACN;gBACE,gBAAgB;gBAChB,OAAO;gBACP,GAAG;gBACH,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,OAAO;gBACP,OAAO;aACR;SACF,EACD,0CAA0C,EAC1C,CAAC,GAAG,EAAE,CACP,CAAC;QAEF,iDAAiD;QACjD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC;YACE,MAAM;YACN;gBACE,gBAAgB;gBAChB,OAAO;gBACP,GAAG;gBACH,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,OAAO;gBACP,OAAO;aACR;SACF,EACD,uDAAuD,EACvD,CAAC,GAAG,EAAE,CACP,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;iBAAM,IAAI,KAAK,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAED,oEAAoE;QACpE,2BAA2B;QAC3B,eAAe,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,MAAM,EAAE,GAAG,EAAE,GAAG,IAAA,YAAS,EAAC,eAAe,CAAC,CAAC;QAC3C,iDAAiD;QACjD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC;YACE,MAAM;YACN;gBACE,gBAAgB;gBAChB,OAAO;gBACP,GAAG;gBACH,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,OAAO;gBACP,OAAO;aACR;SACF,EACD,+CAA+C,EAC/C,CAAC,GAAG,EAAE,CACP,CAAC;QACF,IAAI,CAAC;YACH,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC1C,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;oBAC5B,IAAI,CAAC;wBACH,MAAM,IAAI,GAAG,IAAA,iBAAY,EAAC,OAAO,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;wBAClD,MAAM,cAAc,GAAG,MAAM,IAAA,uCAAqB,EAAC,IAAI,CAAC,CAAC;wBACzD,IAAI,cAAc,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC7D,QAAQ,GAAG,cAAc,CAAC;4BAC1B,OAAO,CAAC,IAAI,CAAC;gCACX,IAAI,EAAE,aAAa;gCACnB,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAG;gCACnC,MAAM,EAAE,GACN,OAAO,CAAC,GAAG,CAAC,gBACd,IAAI,OAAO,IAAI,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAG,EAAE;gCACnD,KAAK,EAAE,OAAO;gCACd,IAAI,EAAE,IAAA,aAAQ,EAAC,OAAO,CAAC,CAAC,IAAI;gCAC5B,GAAG,EAAE,GACH,OAAO,CAAC,GAAG,CAAC,gBACd,IAAI,OAAO,IAAI,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAG,EAAE;6BACpD,CAAC,CAAC;4BACH,MAAM;wBACR,CAAC;oBACH,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;wBAC9C,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,oBAAoB;QACtB,CAAC;IACH,CAAC;IAED,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAE9C,2EAA2E;IAC3E,qBAAqB;IACrB,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,IAAI,CAC/B,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC,EAC5C,GAAG,KAAK,MAAM;QACZ,CAAC,CAAC,gGAAgG;QAClG,CAAC,CAAC,+BAA+B,EACnC,CAAC,GAAG,EAAE,CACP,CAAC;IACF,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,IAAI,GAAG,SAAS,CAAC;IACnB,CAAC;SAAM,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,IAAI,GAAG,YAAY,CAAC;IACtB,CAAC;SAAM,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,IAAI,GAAG,iBAAiB,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,iCAAiC,CAC1D,MAAM,EACN,gBAAgB,EAChB,eAAe,EACf,KAAK,IAAI,KAAK,CACf,CAAC;IACF,MAAM,iBAAiB,GAAa,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,mBAAmB,CAAC;QAC5E,iBAAiB,CAAC,IAAI,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjD,iBAAiB,CAAC,IAAI,CACpB,uDAAuD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAClF,CAAC;IACJ,CAAC;IACD,IAAI,mBAAmB,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxD,YAAY,GAAG,IAAA,6BAAa,EAC1B,IAAA,qCAAmB,EAAC,iBAAiB,CAAC,EACtC,qBAAqB,CACtB,CAAC;IACJ,CAAC;IAED,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAChC,IAAI,4BAA4B,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/C,mBAAmB,GAAG,IAAI,CAAC;IAC7B,CAAC;IAED,sEAAsE;IACtE,MAAM,cAAc,GAAG;QACrB,IAAI;QACJ,QAAQ,EAAE,KAAK,GAAG,GAAG;QACrB,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,IAAI,KAAK,GAAG,GAAG,EAAE;QACnE,KAAK,EAAE,gBAAgB;QACvB,IAAI,EAAE,IAAA,aAAQ,EAAC,gBAAgB,CAAC,CAAC,IAAI;QACrC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,IAAI,KAAK,GAAG,GAAG,EAAE;QAChE,UAAU,EAAE,IAAI;QAChB,kBAAkB,EAAE,mBAAmB,IAAI,IAAI,KAAK,YAAY;KACjE,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAE7B,IAAI,oBAAoB,GAAG,eAAe,CAAC;IAC3C,IAAI,wBAAwB,GAAG,IAAA,aAAQ,EAAC,eAAe,CAAC,CAAC,IAAI,CAAC;IAC9D,IAAI,CAAC,mBAAmB,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;QAClD,yEAAyE;QACzE,kBAAkB;QAClB,oBAAoB,GAAG,IAAA,WAAQ,EAAC,gBAAgB,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC;QAClE,MAAM,cAAc,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;QACrD,MAAM,MAAM,CAAC,IAAI,CACf;YACE,SAAS;YACT;gBACE,eAAe;gBACf,QAAQ;gBACR,WAAW;gBACX,MAAM;gBACN,kBAAkB;gBAClB,KAAK;gBACL,+BAA+B;gBAC/B,kBAAkB;gBAClB,oBAAoB;gBACpB,eAAe;gBACf,MAAM;gBACN,YAAY;aACb;SACF,EACD,kCAAkC,EAClC,CAAC,GAAG,EAAE,CACP,CAAC;QAEF,gDAAgD;QAChD,wBAAwB,GAAG,IAAA,aAAQ,EAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC;QAC/D,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,YAAY;YAClB,QAAQ,EAAE,GAAG,KAAK,MAAM;YACxB,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,IAAI,KAAK,MAAM;YACjE,KAAK,EAAE,oBAAoB;YAC3B,IAAI,EAAE,wBAAwB;YAC9B,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,IAAI,KAAK,MAAM;YAC9D,kBAAkB,EAAE,IAAI;SACzB,CAAC,CAAC;IACL,CAAC;IAED,4DAA4D;IAC5D,MAAM,KAAK,GAAG,MAAM,IAAA,gDAAuB,EAAC,oBAAoB,CAAC,CAAC;IAElE,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC/B,CAAC;IAED,iFAAiF;IACjF,wFAAwF;IACxF,IAAI,cAAc,GAAyB,IAAI,CAAC;IAChD,IAAI,mBAAmB,EAAE,CAAC;QACxB,iFAAiF;QACjF,kFAAkF;QAClF,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE;YACpB,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,IAAA,mCAAmB,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/C,OAAO,IAAA,6BAAa,EAClB,IAAA,4CAA0B,EAAC,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EACpD,4BAA4B,CAC7B,CAAC;QACJ,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;SAAM,CAAC;QACN,cAAc,GAAG,CAAC,KAAK,IAAI,EAAE;YAC3B,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,IAAA,mCAAmB,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IACD,4EAA4E;IAC5E,yEAAyE;IACzE,wBAAwB;IACxB,MAAM,cAAc,GAClB,wBAAwB,IAAI,oCAAa;QACzC,CAAC,cAAc,CAAC,IAAI,KAAK,SAAS;YAChC,cAAc,CAAC,IAAI,IAAI,oCAAa,GAAG,EAAE,CAAC,CAAC;IAE/C,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,WAAW,GAAG,IAAA,WAAQ,EAAC,gBAAgB,EAAE,KAAK,GAAG,eAAe,CAAC,CAAC;QACxE,MAAM,MAAM,CAAC,IAAI,CACf;YACE,SAAS;YACT;gBACE,eAAe;gBACf,QAAQ;gBACR,WAAW;gBACX,IAAI;gBACJ,SAAS;gBACT,MAAM;gBACN,kBAAkB;gBAClB,WAAW;gBACX,oBAAoB;aACrB;SACF,EACD,+BAA+B,EAC/B,EAAE,GAAG,EAAE,CACR,CAAC;QACF,sCAAsC;QACtC,sEAAsE;QACtE,2EAA2E;QAC3E,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,IAAI,KAAK,eAAe;YAC1E,KAAK,EAAE,WAAW;YAClB,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,IAAI,KAAK,eAAe;YACvE,IAAI,EAAE,IAAA,aAAQ,EAAC,WAAW,CAAC,CAAC,IAAI;YAChC,QAAQ,EAAE,GAAG,KAAK,eAAe;SAClC,CAAC,CAAC;IACL,CAAC;IAED,8CAA8C;IAC9C,IAAI,wBAAwB,IAAI,oCAAa,EAAE,CAAC;QAC9C,qEAAqE;QACrE,qEAAqE;QACrE,gEAAgE;QAChE,8BAA8B;QAC9B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,IAAI,CAC/B,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC,EACvC,mDAAmD,EACnD,CAAC,GAAG,EAAE,CACP,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,CAAA,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,0CAAE,UAAU,KAAI,EAAE,CAAC;QACjD,MAAM,iBAAiB,GAIjB,EAAE,CAAC;QACT,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtB,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QACD;;;;;;;;;;WAUG;QACH,MAAM,OAAO,GAAG,IAAA,WAAQ,EAAC,gBAAgB,EAAE,KAAK,GAAG,UAAU,CAAC,CAAC;QAC/D,MAAM,WAAW,GAAG,IAAA,WAAQ,EAAC,gBAAgB,EAAE,KAAK,GAAG,UAAU,CAAC,CAAC;QACnE,MAAM,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC1C,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACvC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ;YACnB,CAAC,CAAE,KAAa,CAAC,QAAQ,CAAC;QAC5B,MAAM,MAAM,CAAC,IAAI,CACf;YACE,YAAY;YACZ;gBACE,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;gBACrE,IAAI;gBACJ,IAAI,YAAY,GAAG;gBACnB,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC;oBAC7B,CAAC,CAAC;wBACE,mDAAmD;wBACnD,4GAA4G;wBAC5G,IAAI;wBACJ,GAAG;qBACJ;oBACH,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,wBAAwB,GAAG,OAAO;oBACnE,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC;oBACd,CAAC,CAAC;wBACE,KAAK;wBACL,GAAG,CAAC,wBAAwB,GAAG,OAAO;4BACpC,CAAC,CAAC,CAAC,+BAA+B,EAAE,GAAG,CAAC;4BACxC,CAAC,CAAC,EAAE,CAAC;qBACR,CAAC;gBACN,gBAAgB;gBAChB,0BAA0B;gBAC1B,IAAI;gBACJ,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE;gBACpD,IAAI;gBACJ,OAAO;gBACP,oBAAoB;aACrB;SACF,EACD,mBAAmB,EACnB,EAAE,GAAG,EAAE,CACR,CAAC;QACF,yEAAyE;QACzE,UAAU;QACV,MAAM,MAAM,CAAC,IAAI,CACf,CAAC,SAAS,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,EAC9C,2BAA2B,EAC3B,CAAC,GAAG,EAAE,CACP,CAAC;QACF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,OAAO,IAAI,KAAK,UAAU;YACjE,KAAK,EAAE,WAAW;YAClB,IAAI,EAAE,IAAA,aAAQ,EAAC,WAAW,CAAC,CAAC,IAAI;YAChC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,IAAI,KAAK,UAAU;YAChE,QAAQ,EAAE,GAAG,KAAK,UAAU;SAC7B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,yBAAyB,GAAG,mBAAmB;QACnD,CAAC,CAAC,IAAA,qDAAqC,EAAC;YACpC,cAAc;YACd,MAAM;YACN,YAAY;YACZ,OAAO;SACR,CAAC;QACJ,CAAC,CAAC,cAAe,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAE1C,OAAO;QACL,MAAM,EAAE,KAAK;QACb,yBAAyB;KAC1B,CAAC;AACJ,CAAC"} \ No newline at end of file +{"version":3,"file":"processVectorUpload.js","sourceRoot":"","sources":["../../src/processVectorUpload.ts"],"names":[],"mappings":";;AAgCA,+BAEC;AAyFD,kDAwfC;AAlnBD,iEAKgC;AAChC,+BAA4D;AAC5D,2BAAwD;AACxD,qEAAmE;AAEnE,+DAA6D;AAC7D,gEAAmE;AACnE,qDAOyB;AACzB,mDAKyB;AACzB,mDAAgE;AAGhE,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAE/D,SAAwB,YAAY,CAAC,EAAU;;IAC7C,OAAO,MAAA,4CAAqB,CAAC,KAAK,CAAC,EAAE,CAAC,0CAAE,MAAM,EAAE,CAAC;AACnD,CAAC;AAID,KAAK,UAAU,iCAAiC,CAC9C,MAAc,EACd,gBAAwB,EACxB,eAAuB,EACvB,WAAoB;IAEpB,MAAM,OAAO,GAA6C,EAAE,CAAC;IAC7D,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,IAAI,CACpC;YACE,MAAM;YACN;gBACE,gBAAgB;gBAChB,OAAO;gBACP,GAAG;gBACH,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,GAAG;gBACH,OAAO;gBACP,OAAO;gBACP,IAAI;gBACJ,OAAO;gBACP,QAAQ;gBACR,GAAG;aACJ;SACF,EACD,yDAAyD,EACzD,CAAC,CACF,CAAC;QACF,IAAI,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,IAAI,EAAE,EAAE,CAAC;YACzB,KAAK,MAAM,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBACtB,IAAI,CAAC,CAAC,IAAI,CAAC,IAAA,eAAU,EAAC,CAAC,CAAC,EAAE,CAAC;oBACzB,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,GAAG,IAAA,YAAS,EAAC,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;gBAC3C,MAAM,IAAI,GAAgB,GAAG,KAAK,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBAC3D,IAAI,CAAC;oBACH,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAA,iBAAY,EAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC3D,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,IAAA,YAAS,EAAC,eAAe,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG;YACZ,IAAI;YACJ,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/D,CAAC;QACF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;gBAC3C,MAAM,SAAS,GAAG,IAAA,WAAQ,EAAC,GAAG,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC;gBAClD,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;oBACxB,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACpB,IAAI,IAAA,eAAU,EAAC,SAAS,CAAC,EAAE,CAAC;oBAC1B,IAAI,CAAC;wBACH,OAAO,CAAC,IAAI,CAAC;4BACX,IAAI,EAAE,GAAG;4BACT,OAAO,EAAE,IAAA,iBAAY,EAAC,SAAS,EAAE,MAAM,CAAC;yBACzC,CAAC,CAAC;oBACL,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;oBACjE,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,mBAAmB,CAAC,OAqBzC;;IAKC,MAAM,EACJ,MAAM,EACN,IAAI,EACJ,OAAO,EACP,cAAc,EACd,OAAO,EACP,gBAAgB,EAChB,KAAK,EACL,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,iBAAiB,GAClB,GAAG,OAAO,CAAC;IACZ,IAAI,mBAAmB,EAAE,CAAC;QACxB,IAAA,iDAAiC,GAAE,CAAC;IACtC,CAAC;IACD,MAAM,gBAAgB,GAAG,IAAI,CAAC;IAC9B,IAAI,eAAe,GAAG,IAAI,CAAC;IAC3B,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAE9C,MAAM,WAAW,GAAG,oBAAoB,CAAC,GAAG,CAAC,IAAA,YAAS,EAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAClE,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CACb,+HAA+H,CAChI,CAAC;QACJ,CAAC;QACD,MAAM,aAAa,GAAG,IAAA,WAAQ,EAAC,gBAAgB,EAAE,KAAK,GAAG,UAAU,CAAC,CAAC;QACrE,MAAM,cAAc,CAAC,SAAS,EAAE,2BAA2B,CAAC,CAAC;QAC7D,MAAM,IAAA,qCAAyB,EAC7B,MAAM,EACN,eAAe,EACf,aAAa,EACb,iBAAiB,CAClB,CAAC;QACF,eAAe,GAAG,aAAa,CAAC;IAClC,CAAC;IAED,IAAI,MAAM,GAAG,mBAAmB;QAC9B,CAAC,CAAC,IAAA,6BAAa,EAAC,IAAA,+BAAa,EAAC,cAAc,CAAC,EAAE,eAAe,CAAC;QAC/D,CAAC,CAAC,IAAI,CAAC;IACT,IAAI,YAAY,GAEL,IAAI,CAAC;IAChB,IAAI,OAAO,GAEA,IAAI,CAAC;IAEhB,IAAI,IAAoB,CAAC;IACzB,IAAI,EAAE,GAAG,EAAE,GAAG,IAAA,YAAS,EAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,GAAG,KAAK,MAAM,CAAC;IAC7B,MAAM,KAAK,GAAG,GAAG,KAAK,MAAM,CAAC;IAC7B,IAAI,QAAQ,GAA4B,IAAI,CAAC;IAE7C,2DAA2D;IAC3D,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;QACnB,MAAM,cAAc,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC7C,oBAAoB;QACpB,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,MAAM,CAAC,IAAI,CACf,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC,EAC1D,wBAAwB,EACxB,CAAC,GAAG,EAAE,CACP,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,CAAC,IAAI,CACf,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,GAAG,gBAAgB,EAAE,eAAe,CAAC,CAAC,EAC3D,8BAA8B,EAC9B,CAAC,GAAG,EAAE,CACP,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;QACpD,qDAAqD;QACrD,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,IAAI,CACjC;YACE,MAAM;YACN;gBACE,gBAAgB;gBAChB,OAAO;gBACP,GAAG;gBACH,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,OAAO;gBACP,OAAO;aACR;SACF,EACD,0CAA0C,EAC1C,CAAC,GAAG,EAAE,CACP,CAAC;QAEF,iDAAiD;QACjD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC;YACE,MAAM;YACN;gBACE,gBAAgB;gBAChB,OAAO;gBACP,GAAG;gBACH,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,OAAO;gBACP,OAAO;aACR;SACF,EACD,uDAAuD,EACvD,CAAC,GAAG,EAAE,CACP,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;iBAAM,IAAI,KAAK,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAED,oEAAoE;QACpE,2BAA2B;QAC3B,eAAe,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,MAAM,EAAE,GAAG,EAAE,GAAG,IAAA,YAAS,EAAC,eAAe,CAAC,CAAC;QAC3C,iDAAiD;QACjD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC;YACE,MAAM;YACN;gBACE,gBAAgB;gBAChB,OAAO;gBACP,GAAG;gBACH,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,OAAO;gBACP,OAAO;aACR;SACF,EACD,+CAA+C,EAC/C,CAAC,GAAG,EAAE,CACP,CAAC;QACF,IAAI,CAAC;YACH,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC1C,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;oBAC5B,IAAI,CAAC;wBACH,MAAM,IAAI,GAAG,IAAA,iBAAY,EAAC,OAAO,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;wBAClD,MAAM,cAAc,GAAG,MAAM,IAAA,uCAAqB,EAAC,IAAI,CAAC,CAAC;wBACzD,IAAI,cAAc,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC7D,QAAQ,GAAG,cAAc,CAAC;4BAC1B,OAAO,CAAC,IAAI,CAAC;gCACX,IAAI,EAAE,aAAa;gCACnB,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAG;gCACnC,MAAM,EAAE,GACN,OAAO,CAAC,GAAG,CAAC,gBACd,IAAI,OAAO,IAAI,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAG,EAAE;gCACnD,KAAK,EAAE,OAAO;gCACd,IAAI,EAAE,IAAA,aAAQ,EAAC,OAAO,CAAC,CAAC,IAAI;gCAC5B,GAAG,EAAE,GACH,OAAO,CAAC,GAAG,CAAC,gBACd,IAAI,OAAO,IAAI,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAG,EAAE;6BACpD,CAAC,CAAC;4BACH,MAAM;wBACR,CAAC;oBACH,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;wBAC9C,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,oBAAoB;QACtB,CAAC;IACH,CAAC;IAED,MAAM,cAAc,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAE9C,2EAA2E;IAC3E,qBAAqB;IACrB,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,IAAI,CAC/B,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC,EAC5C,GAAG,KAAK,MAAM;QACZ,CAAC,CAAC,gGAAgG;QAClG,CAAC,CAAC,+BAA+B,EACnC,CAAC,GAAG,EAAE,CACP,CAAC;IACF,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,IAAI,GAAG,SAAS,CAAC;IACnB,CAAC;SAAM,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,IAAI,GAAG,YAAY,CAAC;IACtB,CAAC;SAAM,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,IAAI,GAAG,iBAAiB,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAChD,CAAC;IAED,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CACb,kIAAkI,CACnI,CAAC;QACJ,CAAC;QACD,qEAAqE;QACrE,oDAAoD;QACpD,IAAI,GAAG,KAAK,CAAC;IACf,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,iCAAiC,CAC1D,MAAM,EACN,gBAAgB,EAChB,eAAe,EACf,KAAK,IAAI,KAAK,CACf,CAAC;IACF,MAAM,iBAAiB,GAAa,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,mBAAmB,CAAC;QAC5E,iBAAiB,CAAC,IAAI,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjD,iBAAiB,CAAC,IAAI,CACpB,uDAAuD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAClF,CAAC;IACJ,CAAC;IACD,IAAI,mBAAmB,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxD,YAAY,GAAG,IAAA,6BAAa,EAC1B,IAAA,qCAAmB,EAAC,iBAAiB,CAAC,EACtC,qBAAqB,CACtB,CAAC;IACJ,CAAC;IAED,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAChC,IAAI,4BAA4B,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/C,mBAAmB,GAAG,IAAI,CAAC;IAC7B,CAAC;IAED,sEAAsE;IACtE,MAAM,cAAc,GAAG;QACrB,IAAI;QACJ,QAAQ,EAAE,KAAK,GAAG,GAAG;QACrB,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,IAAI,KAAK,GAAG,GAAG,EAAE;QACnE,KAAK,EAAE,gBAAgB;QACvB,IAAI,EAAE,IAAA,aAAQ,EAAC,gBAAgB,CAAC,CAAC,IAAI;QACrC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,IAAI,KAAK,GAAG,GAAG,EAAE;QAChE,UAAU,EAAE,IAAI;QAChB,kBAAkB,EAAE,mBAAmB,IAAI,IAAI,KAAK,YAAY;KACjE,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAE7B,IAAI,oBAAoB,GAAG,eAAe,CAAC;IAC3C,IAAI,wBAAwB,GAAG,IAAA,aAAQ,EAAC,eAAe,CAAC,CAAC,IAAI,CAAC;IAC9D,IAAI,CAAC,mBAAmB,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;QAClD,yEAAyE;QACzE,kBAAkB;QAClB,oBAAoB,GAAG,IAAA,WAAQ,EAAC,gBAAgB,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC;QAClE,MAAM,cAAc,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;QACrD,MAAM,MAAM,CAAC,IAAI,CACf;YACE,SAAS;YACT;gBACE,eAAe;gBACf,QAAQ;gBACR,WAAW;gBACX,MAAM;gBACN,kBAAkB;gBAClB,KAAK;gBACL,+BAA+B;gBAC/B,kBAAkB;gBAClB,oBAAoB;gBACpB,eAAe;gBACf,MAAM;gBACN,YAAY;aACb;SACF,EACD,kCAAkC,EAClC,CAAC,GAAG,EAAE,CACP,CAAC;QAEF,gDAAgD;QAChD,wBAAwB,GAAG,IAAA,aAAQ,EAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC;QAC/D,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,YAAY;YAClB,QAAQ,EAAE,GAAG,KAAK,MAAM;YACxB,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,IAAI,KAAK,MAAM;YACjE,KAAK,EAAE,oBAAoB;YAC3B,IAAI,EAAE,wBAAwB;YAC9B,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,IAAI,KAAK,MAAM;YAC9D,kBAAkB,EAAE,IAAI;SACzB,CAAC,CAAC;IACL,CAAC;IAED,4DAA4D;IAC5D,MAAM,KAAK,GAAG,MAAM,IAAA,gDAAuB,EAAC,oBAAoB,CAAC,CAAC;IAElE,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC/B,CAAC;IAED,iFAAiF;IACjF,wFAAwF;IACxF,IAAI,cAAc,GAAyB,IAAI,CAAC;IAChD,IAAI,mBAAmB,EAAE,CAAC;QACxB,iFAAiF;QACjF,kFAAkF;QAClF,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE;YACpB,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,IAAA,mCAAmB,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/C,OAAO,IAAA,6BAAa,EAClB,IAAA,4CAA0B,EAAC,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EACpD,4BAA4B,CAC7B,CAAC;QACJ,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;SAAM,CAAC;QACN,cAAc,GAAG,CAAC,KAAK,IAAI,EAAE;YAC3B,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,IAAA,mCAAmB,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IACD,4EAA4E;IAC5E,yEAAyE;IACzE,wBAAwB;IACxB,MAAM,cAAc,GAClB,wBAAwB,IAAI,oCAAa;QACzC,CAAC,cAAc,CAAC,IAAI,KAAK,SAAS;YAChC,cAAc,CAAC,IAAI,IAAI,oCAAa,GAAG,EAAE,CAAC,CAAC;IAE/C,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,WAAW,GAAG,IAAA,WAAQ,EAAC,gBAAgB,EAAE,KAAK,GAAG,eAAe,CAAC,CAAC;QACxE,MAAM,MAAM,CAAC,IAAI,CACf;YACE,SAAS;YACT;gBACE,eAAe;gBACf,QAAQ;gBACR,WAAW;gBACX,IAAI;gBACJ,SAAS;gBACT,MAAM;gBACN,kBAAkB;gBAClB,WAAW;gBACX,oBAAoB;aACrB;SACF,EACD,+BAA+B,EAC/B,EAAE,GAAG,EAAE,CACR,CAAC;QACF,sCAAsC;QACtC,sEAAsE;QACtE,2EAA2E;QAC3E,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,IAAI,KAAK,eAAe;YAC1E,KAAK,EAAE,WAAW;YAClB,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,IAAI,KAAK,eAAe;YACvE,IAAI,EAAE,IAAA,aAAQ,EAAC,WAAW,CAAC,CAAC,IAAI;YAChC,QAAQ,EAAE,GAAG,KAAK,eAAe;SAClC,CAAC,CAAC;IACL,CAAC;IAED,8CAA8C;IAC9C,IAAI,wBAAwB,IAAI,oCAAa,EAAE,CAAC;QAC9C,qEAAqE;QACrE,qEAAqE;QACrE,gEAAgE;QAChE,8BAA8B;QAC9B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,IAAI,CAC/B,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC,EACvC,mDAAmD,EACnD,CAAC,GAAG,EAAE,CACP,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,CAAA,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,0CAAE,UAAU,KAAI,EAAE,CAAC;QACjD,MAAM,iBAAiB,GAIjB,EAAE,CAAC;QACT,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtB,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QACD;;;;;;;;;;WAUG;QACH,MAAM,OAAO,GAAG,IAAA,WAAQ,EAAC,gBAAgB,EAAE,KAAK,GAAG,UAAU,CAAC,CAAC;QAC/D,MAAM,WAAW,GAAG,IAAA,WAAQ,EAAC,gBAAgB,EAAE,KAAK,GAAG,UAAU,CAAC,CAAC;QACnE,MAAM,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC1C,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACvC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ;YACnB,CAAC,CAAE,KAAa,CAAC,QAAQ,CAAC;QAC5B,MAAM,MAAM,CAAC,IAAI,CACf;YACE,YAAY;YACZ;gBACE,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;gBACrE,IAAI;gBACJ,IAAI,YAAY,GAAG;gBACnB,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC;oBAC7B,CAAC,CAAC;wBACE,mDAAmD;wBACnD,4GAA4G;wBAC5G,IAAI;wBACJ,GAAG;qBACJ;oBACH,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,wBAAwB,GAAG,OAAO;oBACnE,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC;oBACd,CAAC,CAAC;wBACE,KAAK;wBACL,GAAG,CAAC,wBAAwB,GAAG,OAAO;4BACpC,CAAC,CAAC,CAAC,+BAA+B,EAAE,GAAG,CAAC;4BACxC,CAAC,CAAC,EAAE,CAAC;qBACR,CAAC;gBACN,gBAAgB;gBAChB,0BAA0B;gBAC1B,IAAI;gBACJ,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE;gBACpD,IAAI;gBACJ,OAAO;gBACP,oBAAoB;aACrB;SACF,EACD,mBAAmB,EACnB,EAAE,GAAG,EAAE,CACR,CAAC;QACF,yEAAyE;QACzE,UAAU;QACV,MAAM,MAAM,CAAC,IAAI,CACf,CAAC,SAAS,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,EAC9C,2BAA2B,EAC3B,CAAC,GAAG,EAAE,CACP,CAAC;QACF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,OAAO,IAAI,KAAK,UAAU;YACjE,KAAK,EAAE,WAAW;YAClB,IAAI,EAAE,IAAA,aAAQ,EAAC,WAAW,CAAC,CAAC,IAAI;YAChC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,IAAI,KAAK,UAAU;YAChE,QAAQ,EAAE,GAAG,KAAK,UAAU;SAC7B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,yBAAyB,GAAG,mBAAmB;QACnD,CAAC,CAAC,IAAA,qDAAqC,EAAC;YACpC,cAAc;YACd,MAAM;YACN,YAAY;YACZ,OAAO;SACR,CAAC;QACJ,CAAC,CAAC,cAAe,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAE1C,OAAO;QACL,MAAM,EAAE,KAAK;QACb,yBAAyB;KAC1B,CAAC;AACJ,CAAC"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/spatialUploadsHandlerTypes.d.ts b/packages/spatial-uploads-handler/dist/src/spatialUploadsHandlerTypes.d.ts index f1e4b043f..d7f86d1f1 100644 --- a/packages/spatial-uploads-handler/dist/src/spatialUploadsHandlerTypes.d.ts +++ b/packages/spatial-uploads-handler/dist/src/spatialUploadsHandlerTypes.d.ts @@ -10,5 +10,22 @@ export interface SpatialUploadsHandlerRequest { skipLoggingProgress?: boolean; requestingUser: string; enableAiDataAnalyst?: boolean; + processingOptions?: DelimitedUploadProcessingOptions; } +/** + * Column mapping and coordinate reference system chosen (or auto-detected) + * for a delimited text (CSV/TSV/TXT) spatial upload. Mirrored on the client + * in `packages/client/src/admin/uploads/delimitedSpatial/types.ts` — keep + * both in sync when making changes. + */ +export type DelimitedUploadProcessingOptions = { + kind: "delimited"; + geometryMode: "point_xy" | "wkt"; + xField?: string; + yField?: string; + geometryField?: string; + crs: string; + delimiter: "," | "\t" | ";" | "|"; + hasHeaderRow: boolean; +}; //# sourceMappingURL=spatialUploadsHandlerTypes.d.ts.map \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/spatialUploadsHandlerTypes.d.ts.map b/packages/spatial-uploads-handler/dist/src/spatialUploadsHandlerTypes.d.ts.map index 94e669812..3f2172cf4 100644 --- a/packages/spatial-uploads-handler/dist/src/spatialUploadsHandlerTypes.d.ts.map +++ b/packages/spatial-uploads-handler/dist/src/spatialUploadsHandlerTypes.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"spatialUploadsHandlerTypes.d.ts","sourceRoot":"","sources":["../../src/spatialUploadsHandlerTypes.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,WAAW,4BAA4B;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAGlB,MAAM,EAAE,MAAM,CAAC;IAGf,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAE9B,cAAc,EAAE,MAAM,CAAC;IAGvB,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B"} \ No newline at end of file +{"version":3,"file":"spatialUploadsHandlerTypes.d.ts","sourceRoot":"","sources":["../../src/spatialUploadsHandlerTypes.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,WAAW,4BAA4B;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAGlB,MAAM,EAAE,MAAM,CAAC;IAGf,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAE9B,cAAc,EAAE,MAAM,CAAC;IAGvB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAG9B,iBAAiB,CAAC,EAAE,gCAAgC,CAAC;CACtD;AAED;;;;;GAKG;AACH,MAAM,MAAM,gCAAgC,GAAG;IAC7C,IAAI,EAAE,WAAW,CAAC;IAClB,YAAY,EAAE,UAAU,GAAG,KAAK,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,GAAG,GAAG,IAAI,GAAG,GAAG,GAAG,GAAG,CAAC;IAClC,YAAY,EAAE,OAAO,CAAC;CACvB,CAAC"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/uploadPipelineShared.d.ts b/packages/spatial-uploads-handler/dist/src/uploadPipelineShared.d.ts index 2e288dc53..1a6b64e19 100644 --- a/packages/spatial-uploads-handler/dist/src/uploadPipelineShared.d.ts +++ b/packages/spatial-uploads-handler/dist/src/uploadPipelineShared.d.ts @@ -7,7 +7,7 @@ import type { AiDataAnalystNotes } from "ai-data-analyst"; * `processVectorUpload` do not import `handleUpload` (which imports them), * which would create a circular dependency and break Node 22+ / tsx. */ -export type SupportedTypes = "GeoJSON" | "FlatGeobuf" | "ZippedShapefile" | "GeoTIFF" | "NetCDF"; +export type SupportedTypes = "GeoJSON" | "FlatGeobuf" | "ZippedShapefile" | "GeoTIFF" | "NetCDF" | "CSV"; export interface ResponseOutput { /** Remote location string as used in rclone */ remote: string; diff --git a/packages/spatial-uploads-handler/dist/src/uploadPipelineShared.d.ts.map b/packages/spatial-uploads-handler/dist/src/uploadPipelineShared.d.ts.map index b1f66464f..df41279ce 100644 --- a/packages/spatial-uploads-handler/dist/src/uploadPipelineShared.d.ts.map +++ b/packages/spatial-uploads-handler/dist/src/uploadPipelineShared.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"uploadPipelineShared.d.ts","sourceRoot":"","sources":["../../src/uploadPipelineShared.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAC3E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAE1D;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GACtB,SAAS,GACT,YAAY,GACZ,iBAAiB,GACjB,SAAS,GACT,QAAQ,CAAC;AAEb,MAAM,WAAW,cAAc;IAC7B,+CAA+C;IAC/C,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,IAAI,EACA,cAAc,GACd,SAAS,GAET,KAAK,GACL,aAAa,CAAC;IAClB,2DAA2D;IAC3D,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,eAAe;IACf,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;wDACoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,qCAAqC;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,aAAa,GAAG,UAAU,GAAG,IAAI,CAAC;IAC5C,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAGD,eAAO,MAAM,aAAa,SAAU,CAAC;AACrC,0DAA0D;AAC1D,eAAO,MAAM,eAAe,aAAa,CAAC;AAE1C,MAAM,MAAM,eAAe,GAAG,CAC5B,KAAK,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,EACxC,eAAe,EAAE,MAAM,EACvB,QAAQ,CAAC,EAAE,MAAM,KACd,OAAO,CAAC,IAAI,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"uploadPipelineShared.d.ts","sourceRoot":"","sources":["../../src/uploadPipelineShared.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAC3E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAE1D;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GACtB,SAAS,GACT,YAAY,GACZ,iBAAiB,GACjB,SAAS,GACT,QAAQ,GACR,KAAK,CAAC;AAEV,MAAM,WAAW,cAAc;IAC7B,+CAA+C;IAC/C,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,IAAI,EACA,cAAc,GACd,SAAS,GAET,KAAK,GACL,aAAa,CAAC;IAClB,2DAA2D;IAC3D,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,eAAe;IACf,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;wDACoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,qCAAqC;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,aAAa,GAAG,UAAU,GAAG,IAAI,CAAC;IAC5C,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAGD,eAAO,MAAM,aAAa,SAAU,CAAC;AACrC,0DAA0D;AAC1D,eAAO,MAAM,eAAe,aAAa,CAAC;AAE1C,MAAM,MAAM,eAAe,GAAG,CAC5B,KAAK,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,EACxC,eAAe,EAAE,MAAM,EACvB,QAAQ,CAAC,EAAE,MAAM,KACd,OAAO,CAAC,IAAI,CAAC,CAAC"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/dist/src/uploadPipelineShared.js.map b/packages/spatial-uploads-handler/dist/src/uploadPipelineShared.js.map index 1c2fd14c9..75f41706b 100644 --- a/packages/spatial-uploads-handler/dist/src/uploadPipelineShared.js.map +++ b/packages/spatial-uploads-handler/dist/src/uploadPipelineShared.js.map @@ -1 +1 @@ -{"version":3,"file":"uploadPipelineShared.js","sourceRoot":"","sources":["../../src/uploadPipelineShared.ts"],"names":[],"mappings":";;;AA4DA,2DAA2D;AAC9C,QAAA,aAAa,GAAG,MAAO,CAAC;AACrC,0DAA0D;AAC7C,QAAA,eAAe,GAAG,UAAU,CAAC"} \ No newline at end of file +{"version":3,"file":"uploadPipelineShared.js","sourceRoot":"","sources":["../../src/uploadPipelineShared.ts"],"names":[],"mappings":";;;AA6DA,2DAA2D;AAC9C,QAAA,aAAa,GAAG,MAAO,CAAC;AACrC,0DAA0D;AAC7C,QAAA,eAAe,GAAG,UAAU,CAAC"} \ No newline at end of file diff --git a/packages/spatial-uploads-handler/handler.ts b/packages/spatial-uploads-handler/handler.ts index 037c7d8e9..985a94f9f 100644 --- a/packages/spatial-uploads-handler/handler.ts +++ b/packages/spatial-uploads-handler/handler.ts @@ -1,7 +1,10 @@ import handleUpload from "./src/handleUpload"; import type { SpatialUploadsHandlerRequest } from "./src/spatialUploadsHandlerTypes"; -export type { SpatialUploadsHandlerRequest } from "./src/spatialUploadsHandlerTypes"; +export type { + SpatialUploadsHandlerRequest, + DelimitedUploadProcessingOptions, +} from "./src/spatialUploadsHandlerTypes"; export const processUpload = async (event: SpatialUploadsHandlerRequest) => { const s3LogPath = `s3://${process.env.BUCKET}/${event.taskId}.log.txt`; @@ -13,6 +16,7 @@ export const processUpload = async (event: SpatialUploadsHandlerRequest) => { event.requestingUser, event.skipLoggingProgress, event.enableAiDataAnalyst, + event.processingOptions, ); return { ...outputs, diff --git a/packages/spatial-uploads-handler/server.ts b/packages/spatial-uploads-handler/server.ts index 33fddf927..e0136f21d 100755 --- a/packages/spatial-uploads-handler/server.ts +++ b/packages/spatial-uploads-handler/server.ts @@ -23,6 +23,7 @@ const server = createServer(function (req, res) { data.requestingUser, data.skipLoggingProgress, data.enableAiDataAnalyst, + data.processingOptions, ); for (const layer of outputs.layers) { console.log(`outputted - ${layer.name} ${layer.url}`); diff --git a/packages/spatial-uploads-handler/src/formats/delimited.ts b/packages/spatial-uploads-handler/src/formats/delimited.ts new file mode 100644 index 000000000..923b0ff26 --- /dev/null +++ b/packages/spatial-uploads-handler/src/formats/delimited.ts @@ -0,0 +1,86 @@ +import { Logger } from "../logger"; +import type { DelimitedUploadProcessingOptions } from "../spatialUploadsHandlerTypes"; + +const SEPARATOR_BY_DELIMITER: Record< + DelimitedUploadProcessingOptions["delimiter"], + string +> = { + ",": "COMMA", + "\t": "TAB", + ";": "SEMICOLON", + "|": "PIPE", +}; + +/** + * Converts a delimited text file (CSV/TSV/TXT) to GeoJSON using GDAL's CSV + * driver, based on the column mapping and CRS chosen by the user (or + * auto-detected client-side). The resulting file feeds into the same + * normalize-to-FlatGeobuf/tiling pipeline used for other vector formats. + * + * See https://gdal.org/en/stable/drivers/vector/csv.html for open option + * reference. + */ +export async function convertDelimitedToGeoJSON( + logger: Logger, + inputPath: string, + outputPath: string, + options: DelimitedUploadProcessingOptions, +): Promise { + if (options.crs && options.crs !== "EPSG:4326") { + throw new Error( + "Only WGS 84 (EPSG:4326) decimal degree coordinates are supported for delimited uploads.", + ); + } + + const openOptions = [ + "-oo", + `SEPARATOR=${SEPARATOR_BY_DELIMITER[options.delimiter]}`, + "-oo", + `HEADERS=${options.hasHeaderRow ? "YES" : "NO"}`, + // Geometry columns are already represented as the output geometry; don't + // duplicate them as regular attributes. + "-oo", + "KEEP_GEOM_COLUMNS=NO", + ]; + + if (options.geometryMode === "wkt") { + if (!options.geometryField) { + throw new Error( + "processingOptions.geometryField is required when geometryMode is 'wkt'", + ); + } + openOptions.push("-oo", `GEOM_POSSIBLE_NAMES=${options.geometryField}`); + } else { + if (!options.xField || !options.yField) { + throw new Error( + "processingOptions.xField and yField are required when geometryMode is 'point_xy'", + ); + } + openOptions.push( + "-oo", + `X_POSSIBLE_NAMES=${options.xField}`, + "-oo", + `Y_POSSIBLE_NAMES=${options.yField}`, + ); + } + + await logger.exec( + [ + "ogr2ogr", + [ + "-f", + "GeoJSON", + "-s_srs", + "EPSG:4326", + "-t_srs", + "EPSG:4326", + "-skipfailures", + ...openOptions, + outputPath, + inputPath, + ], + ], + "Problem converting delimited text file to spatial data. Double check the selected columns and coordinate reference system.", + 2 / 30, + ); +} diff --git a/packages/spatial-uploads-handler/src/handleUpload.ts b/packages/spatial-uploads-handler/src/handleUpload.ts index 17b4910f3..2bf218c72 100644 --- a/packages/spatial-uploads-handler/src/handleUpload.ts +++ b/packages/spatial-uploads-handler/src/handleUpload.ts @@ -25,8 +25,12 @@ import { type ProgressUpdater, type ResponseOutput, } from "./uploadPipelineShared"; +import type { DelimitedUploadProcessingOptions } from "./spatialUploadsHandlerTypes"; -export type { SpatialUploadsHandlerRequest } from "./spatialUploadsHandlerTypes"; +export type { + SpatialUploadsHandlerRequest, + DelimitedUploadProcessingOptions, +} from "./spatialUploadsHandlerTypes"; export { MAX_OUTPUT_SIZE, @@ -56,6 +60,8 @@ export default async function handleUpload( skipLoggingProgress?: boolean, /** When true, run column intelligence / title / attribution LLMs (requires CF_AIG_* env). */ enableAiDataAnalyst?: boolean, + /** Column mapping / CRS for delimited (CSV/TSV/TXT) uploads. */ + processingOptions?: DelimitedUploadProcessingOptions, ): Promise { if (DEBUG) { console.log("DEBUG MODE ENABLED"); @@ -173,6 +179,7 @@ export default async function handleUpload( uploadFilename, workingDirectory: dist, enableAiDataAnalyst, + processingOptions, }); stats = vectorResult.layers; aiDataAnalystNotesPromise = vectorResult.aiDataAnalystNotesPromise; diff --git a/packages/spatial-uploads-handler/src/index.ts b/packages/spatial-uploads-handler/src/index.ts index acbb0cf1a..b993b5dcf 100644 --- a/packages/spatial-uploads-handler/src/index.ts +++ b/packages/spatial-uploads-handler/src/index.ts @@ -2,6 +2,7 @@ export { ProcessedUploadLayer, SpatialUploadsHandlerRequest, ProcessedUploadResponse, + DelimitedUploadProcessingOptions, } from "./handleUpload"; /** Invoke the geostats PII classifier Lambda (e.g. backfill tasks). */ diff --git a/packages/spatial-uploads-handler/src/processVectorUpload.ts b/packages/spatial-uploads-handler/src/processVectorUpload.ts index 101bb431a..d887e5a1d 100644 --- a/packages/spatial-uploads-handler/src/processVectorUpload.ts +++ b/packages/spatial-uploads-handler/src/processVectorUpload.ts @@ -25,6 +25,10 @@ import { classifyGeostatsPii, composeAiDataAnalystNotesFromPromises, } from "./aiUploadNotes"; +import { convertDelimitedToGeoJSON } from "./formats/delimited"; +import type { DelimitedUploadProcessingOptions } from "./spatialUploadsHandlerTypes"; + +const DELIMITED_EXTENSIONS = new Set([".csv", ".tsv", ".txt"]); export default function fromMarkdown(md: string) { return defaultMarkdownParser.parse(md)?.toJSON(); @@ -136,6 +140,8 @@ export async function processVectorUpload(options: { uploadFilename: string; /** When true, run title / attribution / column intelligence LLMs (requires CF_AIG_* env). */ enableAiDataAnalyst?: boolean; + /** Column mapping / CRS for delimited (CSV/TSV/TXT) uploads. Required if `path` has a delimited extension. */ + processingOptions?: DelimitedUploadProcessingOptions; }): Promise<{ layers: GeostatsLayer[]; /** Resolve after local processing; caller should await after uploads so LLMs overlap I/O. */ @@ -152,6 +158,7 @@ export async function processVectorUpload(options: { originalName, uploadFilename, enableAiDataAnalyst, + processingOptions, } = options; if (enableAiDataAnalyst) { assertAiDataAnalystEnvVarsPresent(); @@ -160,6 +167,24 @@ export async function processVectorUpload(options: { let workingFilePath = path; await updateProgress("running", "validating"); + const isDelimited = DELIMITED_EXTENSIONS.has(parsePath(path).ext); + if (isDelimited) { + if (!processingOptions) { + throw new Error( + "This file requires column mapping configuration before it can be uploaded. Please configure coordinate columns and try again.", + ); + } + const convertedPath = pathJoin(workingDirectory, jobId + ".geojson"); + await updateProgress("running", "converting delimited text"); + await convertDelimitedToGeoJSON( + logger, + workingFilePath, + convertedPath, + processingOptions, + ); + workingFilePath = convertedPath; + } + let titleP = enableAiDataAnalyst ? asNeverReject(generateTitle(uploadFilename), "generateTitle") : null; @@ -332,6 +357,17 @@ export async function processVectorUpload(options: { throw new Error("Not a recognized file type"); } + if (isDelimited) { + if (/Feature Count: 0\b/.test(ogrInfo)) { + throw new Error( + "No spatial features could be extracted from this file using the selected columns. Double check the column mapping and try again.", + ); + } + // The original output should reflect the file as uploaded (delimited + // text), not the GeoJSON it was converted to above. + type = "CSV"; + } + const sidecarFiles = await collectAttributionSidecarContents( logger, workingDirectory, diff --git a/packages/spatial-uploads-handler/src/spatialUploadsHandlerTypes.ts b/packages/spatial-uploads-handler/src/spatialUploadsHandlerTypes.ts index 08ecc8e52..77b44ea8f 100644 --- a/packages/spatial-uploads-handler/src/spatialUploadsHandlerTypes.ts +++ b/packages/spatial-uploads-handler/src/spatialUploadsHandlerTypes.ts @@ -17,4 +17,24 @@ export interface SpatialUploadsHandlerRequest { // Whether to enable AI data analyst. If not set or false, will skip sending // data to openai. *Will* still run classification of PII using local models. enableAiDataAnalyst?: boolean; + // Column mapping / CRS configuration for delimited (CSV/TSV/TXT) uploads. + // Required if the upload is a delimited text file. + processingOptions?: DelimitedUploadProcessingOptions; } + +/** + * Column mapping and coordinate reference system chosen (or auto-detected) + * for a delimited text (CSV/TSV/TXT) spatial upload. Mirrored on the client + * in `packages/client/src/admin/uploads/delimitedSpatial/types.ts` — keep + * both in sync when making changes. + */ +export type DelimitedUploadProcessingOptions = { + kind: "delimited"; + geometryMode: "point_xy" | "wkt"; + xField?: string; + yField?: string; + geometryField?: string; + crs: string; + delimiter: "," | "\t" | ";" | "|"; + hasHeaderRow: boolean; +}; diff --git a/packages/spatial-uploads-handler/src/uploadPipelineShared.ts b/packages/spatial-uploads-handler/src/uploadPipelineShared.ts index 01111703d..a9ee43153 100644 --- a/packages/spatial-uploads-handler/src/uploadPipelineShared.ts +++ b/packages/spatial-uploads-handler/src/uploadPipelineShared.ts @@ -13,7 +13,8 @@ export type SupportedTypes = | "FlatGeobuf" | "ZippedShapefile" | "GeoTIFF" - | "NetCDF"; + | "NetCDF" + | "CSV"; export interface ResponseOutput { /** Remote location string as used in rclone */