From c3bfec37b14b1dbf8bfd8a0e587099beb98c9a61 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Wed, 8 Jul 2026 13:16:06 -0400 Subject: [PATCH 01/14] Add Auto Align (deep-feature matching) to the Camera Registration panel New 'Auto Align (current frame)' button computes the camera-pair alignment automatically using the MINIMA-LoFTR matcher hosted by the VIAME interactive service (companion VIAME branch dev/auto-align), a LoFTR fine-tuned for cross-modality matching (EO<->IR). Flow: the panel asks the platform bridge for an alignment of the selected pair on the current frame; desktop ViewerLoader resolves each camera's image path (same per-camera getters stereo uses) and sends both to the interactive service, which returns a homography plus ~24 spatially-spread inlier correspondences in native pixels. Those are injected as ordinary point pairs (CameraRegistrationStore. applyAutoAlignment): they appear in the correspondences table, are hand-refinable like picked points, fit through the normal homography path, and persist through the existing save and Export-menu downloads. Provenance (autoAlignModel, autoAlignInlierRatio) is stamped into the registration source. Desktop-only, mirroring the text-query wiring end to end: interactive.ts autoAlign -> ipcService alignment-auto-align / alignment-available (matcher weights present in the VIAME install) -> frontend api -> Viewer's provide/inject auto-align bridge (dive-common/use/useAutoAlign) -> RegistrationTools. Web (and installs without the weights) never see the button. Unmatchable scenes surface the service's actionable error (e.g. 'try a frame with more structure') instead of applying a bad warp. Co-Authored-By: Claude Fable 5 --- client/dive-common/apispec.ts | 44 +++++++++ .../CameraRegistration/RegistrationTools.vue | 97 +++++++++++++++++++ client/dive-common/components/Viewer.vue | 30 +++++- client/dive-common/use/useAutoAlign.ts | 31 ++++++ client/platform/desktop/backend/ipcService.ts | 37 +++++++ .../desktop/backend/native/interactive.ts | 51 ++++++++++ client/platform/desktop/frontend/api.ts | 18 ++++ .../frontend/components/ViewerLoader.vue | 43 ++++++++ .../CameraRegistrationStore.spec.ts | 54 +++++++++++ .../alignedView/CameraRegistrationStore.ts | 32 ++++++ 10 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 client/dive-common/use/useAutoAlign.ts diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index bed5b63b8..bf0517928 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -519,6 +519,50 @@ export interface TextQueryResponse { fallback?: boolean; } +export interface AutoAlignRequest { + /** Path to camera A's image for the chosen frame */ + imagePathA: string; + /** Path to camera B's image for the chosen frame */ + imagePathB: string; + options?: { + /** RANSAC reprojection threshold in matcher-resolution pixels (default 2.0) */ + ransacThreshold?: number; + /** Minimum raw matches before RANSAC (default 100) */ + minMatches?: number; + /** Minimum RANSAC inliers (default 30) */ + minInliers?: number; + /** Minimum inlier ratio (default 0.15) */ + minInlierRatio?: number; + /** How many spatially-spread inlier correspondences to return (default 24) */ + topK?: number; + /** Matcher coarse confidence threshold (default 0.2) */ + matchThreshold?: number; + }; +} + +export interface AutoAlignResponse { + /** Whether alignment succeeded (quality gate passed) */ + success: boolean; + /** Machine-readable failure code: + * insufficient_matches | low_confidence | degenerate_homography */ + code?: string; + /** Human-readable error, actionable for the user */ + error?: string; + /** 3x3 homography mapping camera A native pixels -> camera B native pixels */ + homography?: number[][]; + /** Spatially-spread inlier correspondences [ax, ay, bx, by][] in native px */ + inliers?: [number, number, number, number][]; + numMatches?: number; + numInliers?: number; + inlierRatio?: number; + /** Native [width, height] of each input image */ + imageSizeA?: [number, number]; + imageSizeB?: [number, number]; + /** Matcher identifier (e.g. 'minima_loftr') */ + model?: string; + elapsedMs?: number; +} + export interface RefineDetectionsRequest { /** Path to the image file */ imagePath: string; diff --git a/client/dive-common/components/CameraRegistration/RegistrationTools.vue b/client/dive-common/components/CameraRegistration/RegistrationTools.vue index 58197c0df..7882984b4 100644 --- a/client/dive-common/components/CameraRegistration/RegistrationTools.vue +++ b/client/dive-common/components/CameraRegistration/RegistrationTools.vue @@ -16,6 +16,7 @@ import { buildPerCameraRegistrationFiles } from 'vue-media-annotator/alignedView import TooltipBtn from 'vue-media-annotator/components/TooltipButton.vue'; import { useApi } from 'dive-common/apispec'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; +import { useAutoAlign } from 'dive-common/use/useAutoAlign'; export default defineComponent({ name: 'CameraRegistration', @@ -369,6 +370,65 @@ export default defineComponent({ } } + /** + * Auto Align: ask the platform's deep matcher (interactive service) for + * correspondences between the selected pair on the current frame, then + * inject them as ordinary point pairs and fit a homography. The service + * is null / unavailable on platforms without the capability (web), which + * hides the button entirely. + */ + const autoAlignService = useAutoAlign(); + const autoAlignAvailable = computed(() => !!autoAlignService?.available.value); + const autoAligning = ref(false); + const autoAlignError = ref(null); + const autoAlignSummary = ref(null); + + async function runAutoAlign() { + if (!autoAlignService || !camLeft.value || !camRight.value) { + return; + } + if (correspondences.value.length > 0 || hasLoadedTransform.value) { + const confirmed = await prompt({ + title: 'Auto Align', + text: `This will replace the existing points/transform for ${camLeft.value} → ` + + `${camRight.value} with automatically matched points. Continue?`, + confirm: true, + }); + if (!confirmed) { + return; + } + } + autoAligning.value = true; + autoAlignError.value = null; + autoAlignSummary.value = null; + try { + const result = await autoAlignService.run(camLeft.value, camRight.value); + if (!result.success || !result.inliers || result.inliers.length === 0) { + autoAlignError.value = result.error + || 'Auto-align could not compute an alignment for this frame.'; + return; + } + registration.applyAutoAlignment( + camLeft.value, + camRight.value, + result.inliers, + { + autoAlignModel: result.model, + autoAlignInlierRatio: result.inlierRatio, + }, + ); + const consensus = result.inlierRatio !== undefined + ? ` (${Math.round(result.inlierRatio * 100)}% match consensus)` : ''; + autoAlignSummary.value = `Aligned with ${result.inliers.length} matched ` + + `points${consensus}. Review the points and overlay warp, refine if ` + + 'needed, then save.'; + } catch (err) { + autoAlignError.value = err instanceof Error ? err.message : String(err); + } finally { + autoAligning.value = false; + } + } + return { cameras, cameraAlignmentStatuses, @@ -404,6 +464,11 @@ export default defineComponent({ setTransformType, setAlignmentMode, save, + autoAlignAvailable, + autoAligning, + autoAlignError, + autoAlignSummary, + runAutoAlign, }; }, }); @@ -522,6 +587,38 @@ export default defineComponent({ points is optional: fitting {{ minPoints }} or more pairs replaces it. + + + mdi-auto-fix + + Auto Align (current frame) + + + {{ autoAlignError }} + + + {{ autoAlignSummary }} + + Promise) | null>, + default: null, + }, }, setup(props, { emit }) { const { prompt, visible } = usePrompt(); @@ -573,6 +587,20 @@ export default defineComponent({ const lassoMode = useLassoMode(); provide(LassoModeSymbol, lassoMode); + // Auto-align bridge for the Camera Registration panel: injects the current + // frame into the platform handler (mirrors onTextQuerySubmit). Provided + // unconditionally; `available` tracks the handler prop, which the desktop + // platform sets once its availability probe resolves (never on web). + provideAutoAlign({ + available: computed(() => !!props.autoAlignHandler), + run: (cameraA: string, cameraB: string) => { + if (!props.autoAlignHandler) { + return Promise.reject(new Error('Auto-align is not available on this platform')); + } + return props.autoAlignHandler(cameraA, cameraB, aggregateController.value.frame.value) as Promise; + }, + }); + // Provides wrappers for actions to integrate with settings const { linkingTrack, diff --git a/client/dive-common/use/useAutoAlign.ts b/client/dive-common/use/useAutoAlign.ts new file mode 100644 index 000000000..62306503b --- /dev/null +++ b/client/dive-common/use/useAutoAlign.ts @@ -0,0 +1,31 @@ +import { provide, inject, Ref } from 'vue'; +import type { AutoAlignResponse } from 'dive-common/apispec'; + +/** + * Auto-align service bridge for the Camera Registration panel. + * + * The platform layer (desktop ViewerLoader) implements the actual call — it + * resolves each camera's image path for the current frame and invokes the + * interactive service. Viewer.vue provides it (wrapping in the current frame + * number); RegistrationTools.vue injects it. On platforms without the + * capability (web) nothing is provided and the panel hides the button. + */ +export interface AutoAlignService { + /** + * Whether auto-align is usable right now. Reactive: the platform's + * availability probe (matcher weights installed?) resolves after mount. + */ + available: Readonly>; + /** Compute an alignment from camera A to camera B on the current frame. */ + run: (cameraA: string, cameraB: string) => Promise; +} + +const AutoAlignSymbol = Symbol('autoAlign'); + +export function provideAutoAlign(service: AutoAlignService) { + provide(AutoAlignSymbol, service); +} + +export function useAutoAlign(): AutoAlignService | null { + return inject(AutoAlignSymbol, null); +} diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index b546ac1cc..710dcf303 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -57,6 +57,16 @@ function isSam3Installed(viamePath: string): boolean { path.join(pipelinesDir, configName), )); } + +// Auto-align (Camera Registration panel) needs the MINIMA-LoFTR matcher weights +// in the VIAME install; without them the button is hidden. +function isAutoAlignInstalled(viamePath: string): boolean { + if (process.env.VIAME_ALIGNMENT_WEIGHTS + && fs.existsSync(process.env.VIAME_ALIGNMENT_WEIGHTS)) { + return true; + } + return fs.existsSync(path.join(viamePath, 'configs', 'pipelines', 'models', 'minima_loftr.ckpt')); +} if (OS.platform() === 'win32') { win32.initialize(); } @@ -424,6 +434,33 @@ export default function register() { return { installed: isSam3Installed(currentSettings.viamePath) }; }); + ipcMain.handle('alignment-available', () => { + const currentSettings = settings.get(); + return { installed: isAutoAlignInstalled(currentSettings.viamePath) }; + }); + + ipcMain.handle('alignment-auto-align', async (_, args: { + imagePathA: string; + imagePathB: string; + options?: { + ransacThreshold?: number; + minMatches?: number; + minInliers?: number; + minInlierRatio?: number; + topK?: number; + matchThreshold?: number; + }; + }) => { + const segService = getInteractiveServiceManager(); + + // Auto-align only needs the service process running -- its matcher model + // loads lazily inside the auto_align request (and stays resident). + await segService.ensureStarted(settings.get()); + + const response = await segService.autoAlign(args); + return response; + }); + ipcMain.handle('segmentation-text-query', async (_, args: { imagePath: string; frameTime?: number; diff --git a/client/platform/desktop/backend/native/interactive.ts b/client/platform/desktop/backend/native/interactive.ts index 5ea1ac74e..e142e1ab5 100644 --- a/client/platform/desktop/backend/native/interactive.ts +++ b/client/platform/desktop/backend/native/interactive.ts @@ -477,6 +477,57 @@ export class InteractiveServiceManager extends EventEmitter { }, 'Text query'); } + /** + * Auto-align: compute a cross-modality homography between two camera + * frames (image A -> image B, native pixel coordinates) with the deep + * matcher hosted by the alignment backend. The matcher model loads lazily + * on the first call and stays resident in the service process. + */ + async autoAlign(request: { + imagePathA: string; + imagePathB: string; + options?: { + ransacThreshold?: number; + minMatches?: number; + minInliers?: number; + minInlierRatio?: number; + topK?: number; + matchThreshold?: number; + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + }): Promise { + const response = await this.sendRequest({ + command: 'auto_align', + image_path_a: request.imagePathA, + image_path_b: request.imagePathB, + options: request.options && { + ransac_threshold: request.options.ransacThreshold, + min_matches: request.options.minMatches, + min_inliers: request.options.minInliers, + min_inlier_ratio: request.options.minInlierRatio, + top_k: request.options.topK, + match_threshold: request.options.matchThreshold, + }, + }, 'Auto align'); + // Map the service's snake_case payload to the AutoAlignResponse shape. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const raw = response as any; + return { + success: raw.success, + code: raw.code, + error: raw.error, + homography: raw.homography, + inliers: raw.inliers, + numMatches: raw.num_matches, + numInliers: raw.num_inliers, + inlierRatio: raw.inlier_ratio, + imageSizeA: raw.image_size_a, + imageSizeB: raw.image_size_b, + model: raw.model, + elapsedMs: raw.elapsed_ms, + }; + } + async refineDetections(request: { imagePath: string; detections: { diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 451320312..5008a88f2 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -8,6 +8,7 @@ import type { SegmentationPredictRequest, SegmentationPredictResponse, SegmentationStatusResponse, SegmentationStereoSegmentRequest, SegmentationStereoSegmentResponse, TextQueryRequest, TextQueryResponse, RefineDetectionsRequest, RefineDetectionsResponse, + AutoAlignRequest, AutoAlignResponse, } from 'dive-common/apispec'; import { @@ -369,6 +370,20 @@ async function textQuery(request: TextQueryRequest): Promise return window.diveDesktop.invoke('segmentation-text-query', request); } +/** + * Auto Align API (Camera Registration panel) + * Computes a cross-modality homography between two camera frames using the + * deep matcher hosted by the interactive service. + */ + +async function autoAlignAvailable(): Promise<{ installed: boolean }> { + return window.diveDesktop.invoke('alignment-available'); +} + +async function autoAlign(request: AutoAlignRequest): Promise { + return window.diveDesktop.invoke('alignment-auto-align', request); +} + async function refineDetections(request: RefineDetectionsRequest): Promise { return window.diveDesktop.invoke('segmentation-refine', request); } @@ -761,6 +776,9 @@ export { textQuery, refineDetections, runTextQueryPipeline, + /* Auto Align APIs */ + autoAlignAvailable, + autoAlign, /* Stereo APIs */ stereoEnable, stereoDisable, diff --git a/client/platform/desktop/frontend/components/ViewerLoader.vue b/client/platform/desktop/frontend/components/ViewerLoader.vue index edc887870..92df467b6 100644 --- a/client/platform/desktop/frontend/components/ViewerLoader.vue +++ b/client/platform/desktop/frontend/components/ViewerLoader.vue @@ -29,6 +29,7 @@ import { segmentationSam3Installed, loadMetadata, textQuery, runTextQueryPipeline, + autoAlign, autoAlignAvailable, stereoEnable, stereoDisable, stereoSetFrame, stereoTransferLine, stereoTransferPoints, stereoMeasureLine, stereoAggregateLengths, onStereoDisparityReady, onStereoDisparityError, @@ -151,6 +152,7 @@ export default defineComponent({ const readOnlyMode = computed(() => settings.value?.readonlyMode || false); const timeFilter: Ref<[number, number] | null> = ref(null); const textQueryAvailable = ref(false); + const autoAlignReady = ref(false); async function refreshTextQueryAvailability() { try { @@ -161,8 +163,18 @@ export default defineComponent({ } } + async function refreshAutoAlignAvailability() { + try { + const result = await autoAlignAvailable(); + autoAlignReady.value = result.installed; + } catch { + autoAlignReady.value = false; + } + } + watch(() => settings.value?.viamePath, () => { refreshTextQueryAvailability(); + refreshAutoAlignAvailability(); }); watch( @@ -547,6 +559,7 @@ export default defineComponent({ onMounted(() => { initializeSegmentation(); refreshTextQueryAvailability(); + refreshAutoAlignAvailability(); }); /** @@ -728,6 +741,33 @@ export default defineComponent({ } } + /** + * Auto-align handler for the Camera Registration panel (passed down through + * Viewer's auto-align bridge). Resolves each camera's image path for the + * requested frame via the same per-camera getters stereo uses, then asks + * the interactive service for a deep-matched homography. The returned + * correspondences/homography map camera A native pixels -> camera B. + */ + async function handleAutoAlign(cameraA: string, cameraB: string, frameNum: number) { + // Populates stereoImagePathGetters from multicam metadata (no-op when + // already loaded); false means this is a single-camera dataset. + const isMulticam = await loadStereoMetadata(); + if (!isMulticam) { + throw new Error('Auto-align requires a multi-camera dataset'); + } + const getterA = stereoImagePathGetters.value[cameraA]; + const getterB = stereoImagePathGetters.value[cameraB]; + if (!getterA || !getterB) { + throw new Error(`Could not resolve media for cameras "${cameraA}" and "${cameraB}"`); + } + const imagePathA = getterA(frameNum); + const imagePathB = getterB(frameNum); + if (!imagePathA || !imagePathB) { + throw new Error(`No image found for frame ${frameNum} on both cameras`); + } + return autoAlign({ imagePathA, imagePathB }); + } + // The backend stereo service is needed whenever either stereo feature is on // (length-on-modify or cross-camera auto-compute). Track the combined state // so toggling one feature while the other is already on does not restart it. @@ -1817,6 +1857,8 @@ export default defineComponent({ handleTextQueryInit, handleTextQueryAllFrames, textQueryAvailable, + autoAlignReady, + handleAutoAlign, openLink, /* Stereo */ stereoLoadingDialog, @@ -1847,6 +1889,7 @@ export default defineComponent({ :read-only-mode="readOnlyMode || runningPipelines.length > 0" :text-query-enabled="true" :text-query-available="textQueryAvailable" + :auto-align-handler="autoAlignReady ? handleAutoAlign : null" @change-camera="changeCamera" @large-image-warning="largeImageWarning()" @text-query-submit="handleTextQuerySubmit" diff --git a/client/src/alignedView/CameraRegistrationStore.spec.ts b/client/src/alignedView/CameraRegistrationStore.spec.ts index 6d0ba7814..dffaea6ac 100644 --- a/client/src/alignedView/CameraRegistrationStore.spec.ts +++ b/client/src/alignedView/CameraRegistrationStore.spec.ts @@ -734,6 +734,60 @@ describe('CameraRegistrationStore', () => { }); }); + describe('applyAutoAlignment', () => { + // A pure translation by (5, -3), as [ax, ay, bx, by] matcher-style rows. + const inliers: [number, number, number, number][] = [ + [0, 0, 5, -3], [10, 0, 15, -3], [10, 10, 15, 7], [0, 10, 5, 7], + [5, 5, 10, 2], + ]; + + it('injects inliers as correspondences and fits a homography', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('rgb', 'ir'); + const key = store.pairKey('rgb', 'ir'); + store.applyAutoAlignment('rgb', 'ir', inliers, { autoAlignModel: 'minima_loftr' }); + expect(store.correspondences.value[key]).toHaveLength(5); + expect(store.transformTypes.value[key]).toBe('homography'); + const { AtoB } = store.homographies.value[key]; + expect(AtoB[0][2]).toBeCloseTo(5, 5); + expect(AtoB[1][2]).toBeCloseTo(-3, 5); + expect(store.source.value).toMatchObject({ autoAlignModel: 'minima_loftr' }); + expect(store.fitError.value).toBeNull(); + expect(store.dirty.value).toBe(true); + }); + + it('replaces existing points and clears authoring state', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('rgb', 'ir'); + const key = store.pairKey('rgb', 'ir'); + store.addPoint('rgb', [1, 1]); + store.addPoint('ir', [2, 2]); + store.selectCorrespondence(store.correspondences.value[key][0].id); + store.addPoint('rgb', [3, 3]); // pending + store.applyAutoAlignment('rgb', 'ir', inliers); + expect(store.correspondences.value[key]).toHaveLength(5); + expect(store.pendingPoint.value).toBeNull(); + expect(store.selectedCorrespondenceId.value).toBeNull(); + // Injected points are ordinary correspondences: editable like picked ones. + const first = store.correspondences.value[key][0]; + store.updateCorrespondencePoint(first.id, 'rgb', [100, 100]); + expect(store.correspondences.value[key][0].a).toEqual([100, 100]); + }); + + it('surfaces a fitError instead of throwing on degenerate inliers', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('rgb', 'ir'); + const key = store.pairKey('rgb', 'ir'); + // All points collinear: enough rows for a homography but unsolvable. + const degenerate: [number, number, number, number][] = [ + [0, 0, 0, 0], [1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], + ]; + store.applyAutoAlignment('rgb', 'ir', degenerate); + expect(store.correspondences.value[key]).toHaveLength(4); + expect(store.fitError.value).not.toBeNull(); + }); + }); + describe('registration file round trip', () => { it('serializes and reloads all pairs', () => { const store = new CameraRegistrationStore(); diff --git a/client/src/alignedView/CameraRegistrationStore.ts b/client/src/alignedView/CameraRegistrationStore.ts index 48ac51d18..d07d4db9e 100644 --- a/client/src/alignedView/CameraRegistrationStore.ts +++ b/client/src/alignedView/CameraRegistrationStore.ts @@ -637,6 +637,38 @@ export default class CameraRegistrationStore { return { AtoB, BtoA }; } + /** + * Apply an automatically computed alignment for `camA` -> `camB`: replace the + * pair's correspondences with the matcher's inlier points ([ax, ay, bx, by] + * rows in native pixels), switch the pair to a homography fit, and fit from + * those points. The points land in the normal correspondence table so the + * user can inspect, delete, or drag-refine them exactly like hand-picked + * ones. Provenance (`meta`, e.g. { autoAlign: { model, inlierRatio } }) is + * merged into {@link source} so it travels with saves/exports. + */ + applyAutoAlignment( + camA: string, + camB: string, + inliers: [number, number, number, number][], + meta?: Record, + ) { + const key = this.pairKey(camA, camB); + const list: Correspondence[] = inliers.map(([ax, ay, bx, by]) => ({ + // eslint-disable-next-line no-plusplus + id: this.nextId++, + a: [ax, ay] as Point, + b: [bx, by] as Point, + })); + this.correspondences.value = { ...this.correspondences.value, [key]: list }; + this.pendingPoint.value = null; + this.selectedCorrespondenceId.value = null; + this.transformTypes.value = { ...this.transformTypes.value, [key]: 'homography' }; + if (meta) { + this.source.value = { ...(this.source.value || {}), ...meta }; + } + this.maybeFitPair(key); + } + /** * Parse and load a registration JSON file (the per-camera * _to__registration.json format written by From 9a16a307725174e771020b3275a60d6dec3ddc4e Mon Sep 17 00:00:00 2001 From: romleiaj Date: Wed, 8 Jul 2026 13:17:24 -0400 Subject: [PATCH 02/14] Document Auto Align feature plan and Phase 0 benchmark results Co-Authored-By: Claude Fable 5 --- AUTO_ALIGN_FEATURE_PLAN.md | 211 +++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 AUTO_ALIGN_FEATURE_PLAN.md diff --git a/AUTO_ALIGN_FEATURE_PLAN.md b/AUTO_ALIGN_FEATURE_PLAN.md new file mode 100644 index 000000000..cd20046bd --- /dev/null +++ b/AUTO_ALIGN_FEATURE_PLAN.md @@ -0,0 +1,211 @@ +# Auto Align (deep-feature alignment) — implementation plan + +Adds an **"Auto Align" button to the Manual Alignment tab** (branch `dev/keypoint-gui`) +that computes a homography between two cameras of a multicam rig (e.g. EO ↔ IR) using a +deep dense feature matcher, and injects the result into the existing calibration +workflow for review, refinement, and save. + +**Decisions made (2026-07-08):** desktop (Electron) first · benchmark matchers before +picking a default · model loads lazily on first use and stays resident for the session. +**Phase 0 result: MINIMA-LoFTR is the default matcher** (see Phase 0 section). + +**Status (2026-07-08): Phases 0–3 implemented** on branch `dev/auto-align` in both +repos (dive: `444ce81e`; VIAME: `eb8ce809f` vendored matcher + `381829d86` backend). +Python backend verified against the 9 real kamera EO/IR pairs (standalone AND through +interactive_service routing). Client: lint clean, store unit tests added (71 pass). +Remaining: run the desktop app against a real VIAME install with +`minima_loftr.ckpt` placed in `configs/pipelines/models/` (or +`$VIAME_ALIGNMENT_WEIGHTS`), and eventually Phase 4 (web). + +--- + +## Why no VIAME pipeline + +VIAME's existing EO/IR registration is classical only (`register_multimodal_unsync_ocv.pipe` += SIFT+RANSAC; `register_multimodal_*_itk.pipe` = ITK point-set registration). No deep +matcher (LoFTR/SuperGlue/RoMa) exists anywhere in VIAME, kamera, or seal-tk. Running a +kwiver pipeline for a one-shot two-image job is heavyweight: image-list files, full +runner spin-up, transcode checks, output plumbing, and the GPU batch-job queue. + +Instead we extend the **interactive service** — the persistent Python process +(`viame.core.interactive_service`) added for the text-query / SAM feature (#1708). It +already: + +- runs inside the VIAME Python environment (torch available), spawned lazily by + `InteractiveServiceManager` (`client/platform/desktop/backend/native/interactive.ts`); +- speaks JSON-over-stdin/stdout with request ids and a 300 s timeout; +- lazy-loads models on first relevant request and keeps them resident (SAM precedent); +- already has a two-image request shape (`stereo setFrame` takes + `left_image_path` + `right_image_path`). + +Auto-align is a new command on this service, not a pipeline. GPU impact on a shared +server: one auto-align at a time per user process (the service is single-threaded +request/response), ~5 GB VRAM while the model is resident, sub-second–seconds per pair. + +## Model choice (Phase 0 decides) + +| Candidate | Cross-modal (RGB↔IR) | License | Notes | +|---|---|---|---| +| **RoMaV2** (requested) | ⚠️ Regressed vs v1 on WxBS IR↔RGB (55.4 vs 60.8 mAA@10px); authors acknowledge it "struggles with the IR-to-RGB multi-modal subset". No thermal training data. | Code MIT, but frozen **DINOv3** backbone = custom Meta license (attribution + redistribution terms) | Single 1.1 GB checkpoint, presets `precise/base/fast/turbo`, ~5 GB VRAM | +| **MINIMA-RoMa** | ✅ Purpose-trained on ~480M synthetic multimodal pairs; benchmarked on real RGB↔thermal homography (METU-VisTIR) | Apache-2.0 code; RoMa-v1 stack (MIT + Apache DINOv2) | Same runtime profile as RoMa v1. Likely winner. | +| **XoFTR** | ✅ Thermal↔visible-specific (semi-dense, LoFTR-style) | Apache-2.0 | Much lighter than RoMa-class; fallback if VRAM/speed matters | + +All three share the integration contract: images in → matched keypoints out → +`cv2.findHomography(..., cv2.USAC_MAGSAC)`. The service treats the matcher as a +pluggable backend selected by config, so the default can change without touching DIVE. + +--- + +## Architecture + +``` +CalibrationTools.vue ── "Auto Align" button (camLeft, camRight, current frame) + │ emit / injected handler +ViewerLoader.vue ── resolve paths: stereoImagePathGetters[camLeft/Right](frame), frameTime + │ window.ipcRenderer.invoke('interactive-auto-align', req) +ipcService.ts ──► InteractiveServiceManager.autoAlign() (interactive.ts) + │ {"command":"auto_align", "left_image_path", "right_image_path", ...} +viame.core.interactive_service ── auto_align handler: + load matcher (lazy, resident) → preprocess IR → dense match + → sample ~5000 pts → MAGSAC homography (thresh ~2 px) + → {homography: 3x3, inliers: [[ax,ay,bx,by]…] (top ~24, spatially spread), + inlier_ratio, model, elapsed_ms} + │ +CameraCalibrationStore ── inject inliers as correspondences → fitTransform + → user inspects table + ghost overlay, refines, hits Save (existing path) +``` + +### Result injection: correspondences, not a raw matrix + +The Python side runs MAGSAC and returns only **inliers**, so the client's plain DLT +(`estimateTransform('homography', …)` — no RANSAC exists client-side, and none is +needed) fits them safely. Injecting as correspondences (rather than a "loaded" matrix) +means the result: + +- appears in the existing correspondences table — inspectable, deletable, draggable + ("auto then hand-refine" for free); +- round-trips through the existing save/load/export (`cameraHomographies`, + `cameraCorrespondences`, `cameraTransformTypes` metadata) with zero persistence work; +- flips `dirty` so the Save button lights up as usual. + +Stamp `cameraCalibrationSource` with `{ method: 'auto', model, inlierRatio }` for +provenance. + +--- + +## Phases + +### Phase 0 — Benchmark ✅ (done 2026-07-08, bench code in `~/kitware/noaa/autoalign-bench`) + +Ran RoMaV2 (fast/base/precise), MINIMA-RoMa, and MINIMA-LoFTR on the 9 synchronized +EO/IR pairs in `~/kitware/noaa/data/kamera_datasets` (EO 12768×9564 8-bit JPEG, IR +640×512 **16-bit** TIFF; EO downscaled to 1600 for matching — "work" coords below). +No ground-truth calibration existed, so accuracy = cross-frame consistency (same rig → +H ≈ constant across all 9 frames; corner-position std in EO-work px) + cross-model +consensus + visual warp overlays. Hardware: Apple MPS (server CUDA will be faster). + +| model | ok | med inl% | worst inl% | consist mean/max px | med s/pair | +|---|---|---|---|---|---| +| **MINIMA-LoFTR** | 9/9 | 57.5 | 48.8 | **2.81 / 5.78** | **1.9** | +| MINIMA-RoMa | 9/9 | 60.9 | 47.1 | 2.81 / 5.66 | 21.2 | +| RoMaV2 base | 9/9* | 73.7 | *2.7 (garbage)* | 5.05 / 38.3 | 6.9 | +| RoMaV2 fast | 9/9* | 78.4 | *1.3 (garbage)* | 10.1 / 84.6 | 4.1 | +| RoMaV2 precise | 8/9 | 68.3 | *0.4 (garbage)* | 20.6 / 116.7 | 229 (+1 OOM) | + +**Decision: MINIMA-LoFTR is the default matcher.** Equal accuracy to MINIMA-RoMa, +10× faster, 11.5M params / 44 MB weights / <1 GB VRAM (vs ~1.5 GB weights + multi-GB +VRAM for the RoMa-class models), Apache-2.0 end-to-end. Ideal for a shared GPU server. +Keep the backend pluggable; MINIMA-RoMa is the fallback/high-quality option. + +Findings that shape the implementation: + +- **RoMaV2's published IR↔RGB regression reproduced**: it collapsed (≤2.7% inliers, + wrong H) on a low-contrast homogeneous forest-canopy frame that both MINIMA models + handled at ~50–63% inliers with a visually correct warp. Multimodal fine-tuning is + what matters, not architecture generation. RoMaV2 is out. +- On frames with structure, all models agree to ≲1.6 px (consensus check), so the + matcher choice is about robustness on hard frames, not best-case accuracy. +- 2.81 px consistency at 1600-wide work res ≈ ~1.2 IR px — comfortably below manual + click accuracy. +- **Quality gate needed**: reject results with inlier ratio < ~20% or a degenerate H + (condition/det sanity) and tell the user to pick a frame with more scene structure — + the failure mode is a real scene property (featureless canopy), not model noise. +- 16-bit IR preprocessing validated: 2–98% percentile normalize → 8-bit → 3-channel. + IR dynamic range varies 5× between frames, so per-frame normalization is required. +- MINIMA's `DataIOWrapper` is CUDA-hardcoded (`torch.cuda.synchronize`, `np.float`) — + drive the underlying LoFTR module directly (kornia ≥0.7 needs a one-line + `create_meshgrid` import shim; grayscale input, long side 640, df=8). + +### Phase 1 — Python: `auto_align` command (VIAME checkout) + +In `viame.core.interactive_service` (same module the text-query/stereo commands live in): + +- `{"command": "auto_align", "left_image_path", "right_image_path", "frame_time"?, + "options": {"model"?, "max_dim"?, "num_samples"?, "ransac_threshold"?}}`. +- **IR preprocessing** (matters more than model choice): load 16-bit single-channel IR → + percentile normalize (e.g. 2–98%) → 8-bit → replicate to 3 channels. +- Match → sample ~5000 correspondences → `cv2.findHomography(kptsA, kptsB, + cv2.USAC_MAGSAC, ransacReprojThreshold≈2.0, confidence=0.999999, maxIters=10000)` + (looser than the README's 0.2 px — appropriate for cross-modal noise). +- Select top ~24 inliers spatially spread (grid-bucket by position) for the client. +- Failure modes are structured errors: too few matches, degenerate H (checked via + condition/determinant sanity), model not installed. +- Model backend: small registry keyed by name; **lazy load on first request, stays + resident for the process lifetime** (mirrors SAM). `torch.manual_seed` + + `cv2.setRNGSeed` for reproducibility. +- Weights: pre-fetch into the torch-hub cache at install time if possible; otherwise + first-use download with a clear progress/error message. + +### Phase 2 — Electron plumbing (mirror text-query end-to-end) + +- `client/platform/desktop/backend/native/interactive.ts`: `autoAlign(request)` method + modeled on `setFrame`/`textQuery`; availability probe (is the matcher package + importable) modeled on `segmentationSam3Installed`. +- `client/platform/desktop/backend/ipcService.ts`: `interactive-auto-align` (+ + `interactive-auto-align-available`) handlers. +- `client/platform/desktop/frontend/api.ts`: `autoAlign()` + probe. +- `client/platform/desktop/frontend/components/ViewerLoader.vue`: handler resolving both + camera image paths via `stereoImagePathGetters.value[cam]` (per-camera getters already + exist for multicam; pass `frameTime` for video), then calling the API. Provide the + handler down to the calibration panel (same pattern as `segmentationGetImagePath`). + +### Phase 3 — Store + UI + +- `client/src/CameraCalibrationStore.ts`: `applyAutoAlignment(camA, camB, inliers, + meta)` — replaces (with confirm) the pair's correspondences, sets transform type to + `homography`, calls the existing fit path, records source metadata. +- `client/dive-common/components/CameraCalibration/CalibrationTools.vue`: + - "Auto Align" button next to the Transform Type block; enabled when a valid distinct + pair is selected **and** the desktop probe says available (web renders nothing — + same gating as `textQueryEnabled`). + - Spinner during the request (`save()`/`saving` pattern), errors surfaced like + `fitError`, confirm-overwrite prompt when correspondences already exist + (`usePrompt`). + - After success the existing ghost overlay + correspondence table give immediate + visual verification; user refines and Saves normally. + +### Phase 4 (later, optional) — Web support + +Celery task on the GPU `pipelines` queue + a `dive_rpc` endpoint returning the same JSON +(queue already serializes GPU work per worker via `worker_prefetch_multiplier=1`). +Phase 3's store/UI code is platform-agnostic; only the transport differs. Out of scope +for now. + +--- + +## Risks / notes + +- **Cross-modal quality is the main risk** — hence Phase 0 before integration. If all + matchers underperform on the actual imagery, the button can still ship as + "auto-propose, human verifies", which the correspondence-injection design supports. +- **Licensing:** if RoMaV2 wins the benchmark, its DINOv3 backbone's Meta license + (attribution, redistribution terms) needs a sign-off for this project; MINIMA/XoFTR + are clean Apache-2.0. +- **VRAM residency:** resident-per-session was chosen for desktop; if multiple desktop + sessions share one GPU box and contention appears, add an idle-TTL unload behind a + setting (one-line policy change in the service). +- **Large-image/tiled datasets:** `getCameraImage` returns null for tiled sources, but + the desktop path resolver (`stereoImagePathGetters`) is unaffected; only a future web + port needs to care. +- **16-bit IR** (`dev/read-16bit-ir` branch exists): normalization in Phase 1 must match + however DIVE renders those frames, so the picked points visually correspond. From 644634521815e389cccc9589ff706fadfa5db7b4 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Wed, 8 Jul 2026 17:18:53 -0400 Subject: [PATCH 03/14] Let Auto Align accept any multi-camera dataset, not just stereoscopic handleAutoAlign gated on loadStereoMetadata(), which rejects anything that isn't a stereoscopic (subType === 'stereo') dataset. A plain multicam rig -- e.g. an EO/IR pair (subType === 'multicam') -- therefore failed with "Auto-align requires a multi-camera dataset" even though deep-feature alignment needs only two cameras' images for the current frame, not a calibrated stereo pair. Factor the (stereo-agnostic) per-camera image-path-getter population out of loadStereoMetadata into populateMultiCamImagePathGetters, and add loadMultiCamMetadata that gates on isMultiCamDatasetMeta (stereo OR multicam). handleAutoAlign now uses the latter. loadStereoMetadata keeps its isStereoscopicDatasetMeta gate unchanged, so the stereo service still never loads on a non-stereo dataset; the getter-building code is moved verbatim. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../frontend/components/ViewerLoader.vue | 108 +++++++++++------- 1 file changed, 68 insertions(+), 40 deletions(-) diff --git a/client/platform/desktop/frontend/components/ViewerLoader.vue b/client/platform/desktop/frontend/components/ViewerLoader.vue index 92df467b6..db72e2bdf 100644 --- a/client/platform/desktop/frontend/components/ViewerLoader.vue +++ b/client/platform/desktop/frontend/components/ViewerLoader.vue @@ -11,7 +11,7 @@ import context from 'dive-common/store/context'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import { SegmentationPredictRequest } from 'dive-common/apispec'; import { clientSettings } from 'dive-common/store/settings'; -import { isStereoscopicDatasetMeta } from 'dive-common/multicamDisplay'; +import { isStereoscopicDatasetMeta, isMultiCamDatasetMeta } from 'dive-common/multicamDisplay'; import type { StereoAnnotationCompleteParams, StereoAnnotationResetParams, @@ -652,56 +652,83 @@ export default defineComponent({ let stereoDatasetFps: number | undefined; /** - * Load multicam metadata for both cameras to build image path getters + * Populate stereoImagePathGetters (per-camera frame -> image path) from a + * dataset's already-loaded multicam metadata. This part is not stereo + * specific: both the stereo features and Auto Align resolve per-camera + * image paths the same way. Callers gate on the dataset kind they support + * (stereo requires a stereoscopic dataset; auto-align accepts any multicam) + * before calling this. `meta.multiCamMedia` must already be truthy. */ + async function populateMultiCamImagePathGetters( + meta: Awaited>, + ): Promise { + // Extract calibration file path from multiCam metadata + stereoCalibrationFile = meta.multiCam?.calibration || undefined; + // Capture the dataset-level fps as a fallback for per-camera frame times. + stereoDatasetFps = meta.fps || meta.originalFps || stereoDatasetFps; + + // Skip per-camera metadata loading if already populated (e.g. by initializeSegmentation) + if (Object.keys(stereoImagePathGetters.value).length > 0) return true; + + const { cameras } = meta.multiCamMedia; + const cameraNames = Object.keys(cameras); + + for (let i = 0; i < cameraNames.length; i += 1) { + const cam = cameraNames[i]; + const cameraId = `${props.id}/${cam}`; + // eslint-disable-next-line no-await-in-loop + const camMeta = await loadMetadata(cameraId); + const { + originalBasePath, originalImageFiles, type, originalVideoFile, + } = camMeta; + + stereoImagePathGetters.value[cam] = (frameNum: number): string => { + if (type === 'video') { + return joinPath(originalBasePath, originalVideoFile || ''); + } + if (originalImageFiles && originalImageFiles[frameNum]) { + const imagePath = originalImageFiles[frameNum]; + if (isAbsolutePath(imagePath)) { + return imagePath; + } + return joinPath(originalBasePath, imagePath); + } + return ''; + }; + } + return true; + } + async function loadStereoMetadata(): Promise { try { const meta = await loadMetadata(props.id); // Plain multicam and single-camera datasets have no stereo pair: report // no stereo so the caller does not load the stereo service. if (!meta.multiCamMedia || !isStereoscopicDatasetMeta(meta)) return false; - - // Extract calibration file path from multiCam metadata - stereoCalibrationFile = meta.multiCam?.calibration || undefined; - // Capture the dataset-level fps as a fallback for per-camera frame times. - stereoDatasetFps = meta.fps || meta.originalFps || stereoDatasetFps; - - // Skip per-camera metadata loading if already populated (e.g. by initializeSegmentation) - if (Object.keys(stereoImagePathGetters.value).length > 0) return true; - - const { cameras } = meta.multiCamMedia; - const cameraNames = Object.keys(cameras); - - for (let i = 0; i < cameraNames.length; i += 1) { - const cam = cameraNames[i]; - const cameraId = `${props.id}/${cam}`; - // eslint-disable-next-line no-await-in-loop - const camMeta = await loadMetadata(cameraId); - const { - originalBasePath, originalImageFiles, type, originalVideoFile, - } = camMeta; - - stereoImagePathGetters.value[cam] = (frameNum: number): string => { - if (type === 'video') { - return joinPath(originalBasePath, originalVideoFile || ''); - } - if (originalImageFiles && originalImageFiles[frameNum]) { - const imagePath = originalImageFiles[frameNum]; - if (isAbsolutePath(imagePath)) { - return imagePath; - } - return joinPath(originalBasePath, imagePath); - } - return ''; - }; - } - return true; + return await populateMultiCamImagePathGetters(meta); } catch (err) { console.error('[Stereo] Failed to load multicam metadata:', err); return false; } } + /** + * Multicam metadata loader for Auto Align. Unlike loadStereoMetadata this + * accepts ANY multi-camera dataset (stereoscopic or plain 'multicam', e.g. + * an EO/IR rig), since deep-feature alignment does not require a calibrated + * stereo pair -- only two cameras' images for the current frame. + */ + async function loadMultiCamMetadata(): Promise { + try { + const meta = await loadMetadata(props.id); + if (!meta.multiCamMedia || !isMultiCamDatasetMeta(meta)) return false; + return await populateMultiCamImagePathGetters(meta); + } catch (err) { + console.error('[MultiCam] Failed to load multicam metadata:', err); + return false; + } + } + // Track last stereo frame to avoid redundant set_frame calls let lastStereoFrame = -1; @@ -750,8 +777,9 @@ export default defineComponent({ */ async function handleAutoAlign(cameraA: string, cameraB: string, frameNum: number) { // Populates stereoImagePathGetters from multicam metadata (no-op when - // already loaded); false means this is a single-camera dataset. - const isMulticam = await loadStereoMetadata(); + // already loaded). Accepts any multi-camera dataset (stereoscopic or + // plain multicam); false means this is a single-camera dataset. + const isMulticam = await loadMultiCamMetadata(); if (!isMulticam) { throw new Error('Auto-align requires a multi-camera dataset'); } From f9dfb79e2ff85fb58dfef1cf41ab83ef2cb41517 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Sun, 12 Jul 2026 12:21:11 -0400 Subject: [PATCH 04/14] Rename Auto Align to Auto Register The button creates a camera-pair REGISTRATION -- it automates exactly what the Camera Registration panel does by hand -- so name it with the registration vocabulary (registration = creating the transform, Align View = consuming it). Renames the UI strings, the store method (applyAutoRegistration), the provide/inject bridge (dive-common/use/useAutoRegister), the apispec types (AutoRegisterRequest/Response), the desktop API and IPC channels (auto-register / auto-register-available), the provenance keys stamped into the registration source (autoRegisterModel, autoRegisterInlierRatio), and the feature-plan doc. The interactive service's wire name (command: 'auto_align', VIAME branch dev/auto-align) is deliberately unchanged -- it renames in step with the service side. Co-Authored-By: Claude Fable 5 --- ...E_PLAN.md => AUTO_REGISTER_FEATURE_PLAN.md | 22 +++--- client/dive-common/apispec.ts | 4 +- .../CameraRegistration/RegistrationTools.vue | 74 +++++++++---------- client/dive-common/components/Viewer.vue | 22 +++--- .../{useAutoAlign.ts => useAutoRegister.ts} | 20 ++--- client/platform/desktop/backend/ipcService.ts | 14 ++-- .../desktop/backend/native/interactive.ts | 10 ++- client/platform/desktop/frontend/api.ts | 18 ++--- .../frontend/components/ViewerLoader.vue | 38 +++++----- .../CameraRegistrationStore.spec.ts | 10 +-- .../alignedView/CameraRegistrationStore.ts | 4 +- 11 files changed, 119 insertions(+), 117 deletions(-) rename AUTO_ALIGN_FEATURE_PLAN.md => AUTO_REGISTER_FEATURE_PLAN.md (93%) rename client/dive-common/use/{useAutoAlign.ts => useAutoRegister.ts} (51%) diff --git a/AUTO_ALIGN_FEATURE_PLAN.md b/AUTO_REGISTER_FEATURE_PLAN.md similarity index 93% rename from AUTO_ALIGN_FEATURE_PLAN.md rename to AUTO_REGISTER_FEATURE_PLAN.md index cd20046bd..333ea9401 100644 --- a/AUTO_ALIGN_FEATURE_PLAN.md +++ b/AUTO_REGISTER_FEATURE_PLAN.md @@ -1,6 +1,6 @@ -# Auto Align (deep-feature alignment) — implementation plan +# Auto Register (deep-feature alignment) — implementation plan -Adds an **"Auto Align" button to the Manual Alignment tab** (branch `dev/keypoint-gui`) +Adds an **"Auto Register" button to the Camera Registration tab** (branch `dev/keypoint-gui`) that computes a homography between two cameras of a multicam rig (e.g. EO ↔ IR) using a deep dense feature matcher, and injects the result into the existing calibration workflow for review, refinement, and save. @@ -38,8 +38,8 @@ already: - already has a two-image request shape (`stereo setFrame` takes `left_image_path` + `right_image_path`). -Auto-align is a new command on this service, not a pipeline. GPU impact on a shared -server: one auto-align at a time per user process (the service is single-threaded +Auto-register is a new command on this service, not a pipeline. GPU impact on a shared +server: one auto-register at a time per user process (the service is single-threaded request/response), ~5 GB VRAM while the model is resident, sub-second–seconds per pair. ## Model choice (Phase 0 decides) @@ -59,7 +59,7 @@ pluggable backend selected by config, so the default can change without touching ## Architecture ``` -CalibrationTools.vue ── "Auto Align" button (camLeft, camRight, current frame) +RegistrationTools.vue ── "Auto Register" button (camLeft, camRight, current frame) │ emit / injected handler ViewerLoader.vue ── resolve paths: stereoImagePathGetters[camLeft/Right](frame), frameTime │ window.ipcRenderer.invoke('interactive-auto-align', req) @@ -71,7 +71,7 @@ viame.core.interactive_service ── auto_align handler: → {homography: 3x3, inliers: [[ax,ay,bx,by]…] (top ~24, spatially spread), inlier_ratio, model, elapsed_ms} │ -CameraCalibrationStore ── inject inliers as correspondences → fitTransform +CameraRegistrationStore ── inject inliers as correspondences → fitTransform → user inspects table + ghost overlay, refines, hits Save (existing path) ``` @@ -161,8 +161,8 @@ In `viame.core.interactive_service` (same module the text-query/stereo commands - `client/platform/desktop/backend/native/interactive.ts`: `autoAlign(request)` method modeled on `setFrame`/`textQuery`; availability probe (is the matcher package importable) modeled on `segmentationSam3Installed`. -- `client/platform/desktop/backend/ipcService.ts`: `interactive-auto-align` (+ - `interactive-auto-align-available`) handlers. +- `client/platform/desktop/backend/ipcService.ts`: `interactive-auto-register` (+ + `interactive-auto-register-available`) handlers. - `client/platform/desktop/frontend/api.ts`: `autoAlign()` + probe. - `client/platform/desktop/frontend/components/ViewerLoader.vue`: handler resolving both camera image paths via `stereoImagePathGetters.value[cam]` (per-camera getters already @@ -171,11 +171,11 @@ In `viame.core.interactive_service` (same module the text-query/stereo commands ### Phase 3 — Store + UI -- `client/src/CameraCalibrationStore.ts`: `applyAutoAlignment(camA, camB, inliers, +- `client/src/CameraRegistrationStore.ts`: `applyAutoAlignment(camA, camB, inliers, meta)` — replaces (with confirm) the pair's correspondences, sets transform type to `homography`, calls the existing fit path, records source metadata. -- `client/dive-common/components/CameraCalibration/CalibrationTools.vue`: - - "Auto Align" button next to the Transform Type block; enabled when a valid distinct +- `client/dive-common/components/CameraCalibration/RegistrationTools.vue`: + - "Auto Register" button next to the Transform Type block; enabled when a valid distinct pair is selected **and** the desktop probe says available (web renders nothing — same gating as `textQueryEnabled`). - Spinner during the request (`save()`/`saving` pattern), errors surfaced like diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index bf0517928..c682a82e0 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -519,7 +519,7 @@ export interface TextQueryResponse { fallback?: boolean; } -export interface AutoAlignRequest { +export interface AutoRegisterRequest { /** Path to camera A's image for the chosen frame */ imagePathA: string; /** Path to camera B's image for the chosen frame */ @@ -540,7 +540,7 @@ export interface AutoAlignRequest { }; } -export interface AutoAlignResponse { +export interface AutoRegisterResponse { /** Whether alignment succeeded (quality gate passed) */ success: boolean; /** Machine-readable failure code: diff --git a/client/dive-common/components/CameraRegistration/RegistrationTools.vue b/client/dive-common/components/CameraRegistration/RegistrationTools.vue index 7882984b4..ecdb6fff2 100644 --- a/client/dive-common/components/CameraRegistration/RegistrationTools.vue +++ b/client/dive-common/components/CameraRegistration/RegistrationTools.vue @@ -16,7 +16,7 @@ import { buildPerCameraRegistrationFiles } from 'vue-media-annotator/alignedView import TooltipBtn from 'vue-media-annotator/components/TooltipButton.vue'; import { useApi } from 'dive-common/apispec'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; -import { useAutoAlign } from 'dive-common/use/useAutoAlign'; +import { useAutoRegister } from 'dive-common/use/useAutoRegister'; export default defineComponent({ name: 'CameraRegistration', @@ -371,25 +371,25 @@ export default defineComponent({ } /** - * Auto Align: ask the platform's deep matcher (interactive service) for + * Auto Register: ask the platform's deep matcher (interactive service) for * correspondences between the selected pair on the current frame, then * inject them as ordinary point pairs and fit a homography. The service * is null / unavailable on platforms without the capability (web), which * hides the button entirely. */ - const autoAlignService = useAutoAlign(); - const autoAlignAvailable = computed(() => !!autoAlignService?.available.value); - const autoAligning = ref(false); - const autoAlignError = ref(null); - const autoAlignSummary = ref(null); - - async function runAutoAlign() { - if (!autoAlignService || !camLeft.value || !camRight.value) { + const autoRegisterService = useAutoRegister(); + const autoRegisterAvailable = computed(() => !!autoRegisterService?.available.value); + const autoRegistering = ref(false); + const autoRegisterError = ref(null); + const autoRegisterSummary = ref(null); + + async function runAutoRegister() { + if (!autoRegisterService || !camLeft.value || !camRight.value) { return; } if (correspondences.value.length > 0 || hasLoadedTransform.value) { const confirmed = await prompt({ - title: 'Auto Align', + title: 'Auto Register', text: `This will replace the existing points/transform for ${camLeft.value} → ` + `${camRight.value} with automatically matched points. Continue?`, confirm: true, @@ -398,34 +398,34 @@ export default defineComponent({ return; } } - autoAligning.value = true; - autoAlignError.value = null; - autoAlignSummary.value = null; + autoRegistering.value = true; + autoRegisterError.value = null; + autoRegisterSummary.value = null; try { - const result = await autoAlignService.run(camLeft.value, camRight.value); + const result = await autoRegisterService.run(camLeft.value, camRight.value); if (!result.success || !result.inliers || result.inliers.length === 0) { - autoAlignError.value = result.error - || 'Auto-align could not compute an alignment for this frame.'; + autoRegisterError.value = result.error + || 'Auto-register could not compute an alignment for this frame.'; return; } - registration.applyAutoAlignment( + registration.applyAutoRegistration( camLeft.value, camRight.value, result.inliers, { - autoAlignModel: result.model, - autoAlignInlierRatio: result.inlierRatio, + autoRegisterModel: result.model, + autoRegisterInlierRatio: result.inlierRatio, }, ); const consensus = result.inlierRatio !== undefined ? ` (${Math.round(result.inlierRatio * 100)}% match consensus)` : ''; - autoAlignSummary.value = `Aligned with ${result.inliers.length} matched ` + autoRegisterSummary.value = `Aligned with ${result.inliers.length} matched ` + `points${consensus}. Review the points and overlay warp, refine if ` + 'needed, then save.'; } catch (err) { - autoAlignError.value = err instanceof Error ? err.message : String(err); + autoRegisterError.value = err instanceof Error ? err.message : String(err); } finally { - autoAligning.value = false; + autoRegistering.value = false; } } @@ -464,11 +464,11 @@ export default defineComponent({ setTransformType, setAlignmentMode, save, - autoAlignAvailable, - autoAligning, - autoAlignError, - autoAlignSummary, - runAutoAlign, + autoRegisterAvailable, + autoRegistering, + autoRegisterError, + autoRegisterSummary, + runAutoRegister, }; }, }); @@ -588,15 +588,15 @@ export default defineComponent({ mdi-auto-fix - Auto Align (current frame) + Auto Register (current frame) - {{ autoAlignError }} + {{ autoRegisterError }} - {{ autoAlignSummary }} + {{ autoRegisterSummary }} Promise) | null>, default: null, @@ -587,17 +587,17 @@ export default defineComponent({ const lassoMode = useLassoMode(); provide(LassoModeSymbol, lassoMode); - // Auto-align bridge for the Camera Registration panel: injects the current + // Auto-register bridge for the Camera Registration panel: injects the current // frame into the platform handler (mirrors onTextQuerySubmit). Provided // unconditionally; `available` tracks the handler prop, which the desktop // platform sets once its availability probe resolves (never on web). - provideAutoAlign({ - available: computed(() => !!props.autoAlignHandler), + provideAutoRegister({ + available: computed(() => !!props.autoRegisterHandler), run: (cameraA: string, cameraB: string) => { - if (!props.autoAlignHandler) { - return Promise.reject(new Error('Auto-align is not available on this platform')); + if (!props.autoRegisterHandler) { + return Promise.reject(new Error('Auto-register is not available on this platform')); } - return props.autoAlignHandler(cameraA, cameraB, aggregateController.value.frame.value) as Promise; + return props.autoRegisterHandler(cameraA, cameraB, aggregateController.value.frame.value) as Promise; }, }); diff --git a/client/dive-common/use/useAutoAlign.ts b/client/dive-common/use/useAutoRegister.ts similarity index 51% rename from client/dive-common/use/useAutoAlign.ts rename to client/dive-common/use/useAutoRegister.ts index 62306503b..5c9dceeac 100644 --- a/client/dive-common/use/useAutoAlign.ts +++ b/client/dive-common/use/useAutoRegister.ts @@ -1,8 +1,8 @@ import { provide, inject, Ref } from 'vue'; -import type { AutoAlignResponse } from 'dive-common/apispec'; +import type { AutoRegisterResponse } from 'dive-common/apispec'; /** - * Auto-align service bridge for the Camera Registration panel. + * Auto-register service bridge for the Camera Registration panel. * * The platform layer (desktop ViewerLoader) implements the actual call — it * resolves each camera's image path for the current frame and invokes the @@ -10,22 +10,22 @@ import type { AutoAlignResponse } from 'dive-common/apispec'; * number); RegistrationTools.vue injects it. On platforms without the * capability (web) nothing is provided and the panel hides the button. */ -export interface AutoAlignService { +export interface AutoRegisterService { /** - * Whether auto-align is usable right now. Reactive: the platform's + * Whether auto-register is usable right now. Reactive: the platform's * availability probe (matcher weights installed?) resolves after mount. */ available: Readonly>; /** Compute an alignment from camera A to camera B on the current frame. */ - run: (cameraA: string, cameraB: string) => Promise; + run: (cameraA: string, cameraB: string) => Promise; } -const AutoAlignSymbol = Symbol('autoAlign'); +const AutoRegisterSymbol = Symbol('autoRegister'); -export function provideAutoAlign(service: AutoAlignService) { - provide(AutoAlignSymbol, service); +export function provideAutoRegister(service: AutoRegisterService) { + provide(AutoRegisterSymbol, service); } -export function useAutoAlign(): AutoAlignService | null { - return inject(AutoAlignSymbol, null); +export function useAutoRegister(): AutoRegisterService | null { + return inject(AutoRegisterSymbol, null); } diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index 710dcf303..cd8e71acf 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -58,9 +58,9 @@ function isSam3Installed(viamePath: string): boolean { )); } -// Auto-align (Camera Registration panel) needs the MINIMA-LoFTR matcher weights +// Auto-register (Camera Registration panel) needs the MINIMA-LoFTR matcher weights // in the VIAME install; without them the button is hidden. -function isAutoAlignInstalled(viamePath: string): boolean { +function isAutoRegisterInstalled(viamePath: string): boolean { if (process.env.VIAME_ALIGNMENT_WEIGHTS && fs.existsSync(process.env.VIAME_ALIGNMENT_WEIGHTS)) { return true; @@ -434,12 +434,12 @@ export default function register() { return { installed: isSam3Installed(currentSettings.viamePath) }; }); - ipcMain.handle('alignment-available', () => { + ipcMain.handle('auto-register-available', () => { const currentSettings = settings.get(); - return { installed: isAutoAlignInstalled(currentSettings.viamePath) }; + return { installed: isAutoRegisterInstalled(currentSettings.viamePath) }; }); - ipcMain.handle('alignment-auto-align', async (_, args: { + ipcMain.handle('auto-register', async (_, args: { imagePathA: string; imagePathB: string; options?: { @@ -453,11 +453,11 @@ export default function register() { }) => { const segService = getInteractiveServiceManager(); - // Auto-align only needs the service process running -- its matcher model + // Auto-register only needs the service process running -- its matcher model // loads lazily inside the auto_align request (and stays resident). await segService.ensureStarted(settings.get()); - const response = await segService.autoAlign(args); + const response = await segService.autoRegister(args); return response; }); diff --git a/client/platform/desktop/backend/native/interactive.ts b/client/platform/desktop/backend/native/interactive.ts index e142e1ab5..15dc961ab 100644 --- a/client/platform/desktop/backend/native/interactive.ts +++ b/client/platform/desktop/backend/native/interactive.ts @@ -478,12 +478,12 @@ export class InteractiveServiceManager extends EventEmitter { } /** - * Auto-align: compute a cross-modality homography between two camera + * Auto-register: compute a cross-modality homography between two camera * frames (image A -> image B, native pixel coordinates) with the deep * matcher hosted by the alignment backend. The matcher model loads lazily * on the first call and stays resident in the service process. */ - async autoAlign(request: { + async autoRegister(request: { imagePathA: string; imagePathB: string; options?: { @@ -497,6 +497,8 @@ export class InteractiveServiceManager extends EventEmitter { // eslint-disable-next-line @typescript-eslint/no-explicit-any }): Promise { const response = await this.sendRequest({ + // The interactive service's wire name (VIAME branch dev/auto-align) + // still says "align"; rename here in step with the service side. command: 'auto_align', image_path_a: request.imagePathA, image_path_b: request.imagePathB, @@ -508,8 +510,8 @@ export class InteractiveServiceManager extends EventEmitter { top_k: request.options.topK, match_threshold: request.options.matchThreshold, }, - }, 'Auto align'); - // Map the service's snake_case payload to the AutoAlignResponse shape. + }, 'Auto register'); + // Map the service's snake_case payload to the AutoRegisterResponse shape. // eslint-disable-next-line @typescript-eslint/no-explicit-any const raw = response as any; return { diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 5008a88f2..2cb98d6e9 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -8,7 +8,7 @@ import type { SegmentationPredictRequest, SegmentationPredictResponse, SegmentationStatusResponse, SegmentationStereoSegmentRequest, SegmentationStereoSegmentResponse, TextQueryRequest, TextQueryResponse, RefineDetectionsRequest, RefineDetectionsResponse, - AutoAlignRequest, AutoAlignResponse, + AutoRegisterRequest, AutoRegisterResponse, } from 'dive-common/apispec'; import { @@ -371,17 +371,17 @@ async function textQuery(request: TextQueryRequest): Promise } /** - * Auto Align API (Camera Registration panel) + * Auto Register API (Camera Registration panel) * Computes a cross-modality homography between two camera frames using the * deep matcher hosted by the interactive service. */ -async function autoAlignAvailable(): Promise<{ installed: boolean }> { - return window.diveDesktop.invoke('alignment-available'); +async function autoRegisterAvailable(): Promise<{ installed: boolean }> { + return window.diveDesktop.invoke('auto-register-available'); } -async function autoAlign(request: AutoAlignRequest): Promise { - return window.diveDesktop.invoke('alignment-auto-align', request); +async function autoRegister(request: AutoRegisterRequest): Promise { + return window.diveDesktop.invoke('auto-register', request); } async function refineDetections(request: RefineDetectionsRequest): Promise { @@ -776,9 +776,9 @@ export { textQuery, refineDetections, runTextQueryPipeline, - /* Auto Align APIs */ - autoAlignAvailable, - autoAlign, + /* Auto Register APIs */ + autoRegisterAvailable, + autoRegister, /* Stereo APIs */ stereoEnable, stereoDisable, diff --git a/client/platform/desktop/frontend/components/ViewerLoader.vue b/client/platform/desktop/frontend/components/ViewerLoader.vue index db72e2bdf..86c6ca688 100644 --- a/client/platform/desktop/frontend/components/ViewerLoader.vue +++ b/client/platform/desktop/frontend/components/ViewerLoader.vue @@ -29,7 +29,7 @@ import { segmentationSam3Installed, loadMetadata, textQuery, runTextQueryPipeline, - autoAlign, autoAlignAvailable, + autoRegister, autoRegisterAvailable, stereoEnable, stereoDisable, stereoSetFrame, stereoTransferLine, stereoTransferPoints, stereoMeasureLine, stereoAggregateLengths, onStereoDisparityReady, onStereoDisparityError, @@ -152,7 +152,7 @@ export default defineComponent({ const readOnlyMode = computed(() => settings.value?.readonlyMode || false); const timeFilter: Ref<[number, number] | null> = ref(null); const textQueryAvailable = ref(false); - const autoAlignReady = ref(false); + const autoRegisterReady = ref(false); async function refreshTextQueryAvailability() { try { @@ -163,18 +163,18 @@ export default defineComponent({ } } - async function refreshAutoAlignAvailability() { + async function refreshAutoRegisterAvailability() { try { - const result = await autoAlignAvailable(); - autoAlignReady.value = result.installed; + const result = await autoRegisterAvailable(); + autoRegisterReady.value = result.installed; } catch { - autoAlignReady.value = false; + autoRegisterReady.value = false; } } watch(() => settings.value?.viamePath, () => { refreshTextQueryAvailability(); - refreshAutoAlignAvailability(); + refreshAutoRegisterAvailability(); }); watch( @@ -559,7 +559,7 @@ export default defineComponent({ onMounted(() => { initializeSegmentation(); refreshTextQueryAvailability(); - refreshAutoAlignAvailability(); + refreshAutoRegisterAvailability(); }); /** @@ -654,9 +654,9 @@ export default defineComponent({ /** * Populate stereoImagePathGetters (per-camera frame -> image path) from a * dataset's already-loaded multicam metadata. This part is not stereo - * specific: both the stereo features and Auto Align resolve per-camera + * specific: both the stereo features and Auto Register resolve per-camera * image paths the same way. Callers gate on the dataset kind they support - * (stereo requires a stereoscopic dataset; auto-align accepts any multicam) + * (stereo requires a stereoscopic dataset; auto-register accepts any multicam) * before calling this. `meta.multiCamMedia` must already be truthy. */ async function populateMultiCamImagePathGetters( @@ -713,7 +713,7 @@ export default defineComponent({ } /** - * Multicam metadata loader for Auto Align. Unlike loadStereoMetadata this + * Multicam metadata loader for Auto Register. Unlike loadStereoMetadata this * accepts ANY multi-camera dataset (stereoscopic or plain 'multicam', e.g. * an EO/IR rig), since deep-feature alignment does not require a calibrated * stereo pair -- only two cameras' images for the current frame. @@ -769,19 +769,19 @@ export default defineComponent({ } /** - * Auto-align handler for the Camera Registration panel (passed down through - * Viewer's auto-align bridge). Resolves each camera's image path for the + * Auto-register handler for the Camera Registration panel (passed down through + * Viewer's auto-register bridge). Resolves each camera's image path for the * requested frame via the same per-camera getters stereo uses, then asks * the interactive service for a deep-matched homography. The returned * correspondences/homography map camera A native pixels -> camera B. */ - async function handleAutoAlign(cameraA: string, cameraB: string, frameNum: number) { + async function handleAutoRegister(cameraA: string, cameraB: string, frameNum: number) { // Populates stereoImagePathGetters from multicam metadata (no-op when // already loaded). Accepts any multi-camera dataset (stereoscopic or // plain multicam); false means this is a single-camera dataset. const isMulticam = await loadMultiCamMetadata(); if (!isMulticam) { - throw new Error('Auto-align requires a multi-camera dataset'); + throw new Error('Auto-register requires a multi-camera dataset'); } const getterA = stereoImagePathGetters.value[cameraA]; const getterB = stereoImagePathGetters.value[cameraB]; @@ -793,7 +793,7 @@ export default defineComponent({ if (!imagePathA || !imagePathB) { throw new Error(`No image found for frame ${frameNum} on both cameras`); } - return autoAlign({ imagePathA, imagePathB }); + return autoRegister({ imagePathA, imagePathB }); } // The backend stereo service is needed whenever either stereo feature is on @@ -1885,8 +1885,8 @@ export default defineComponent({ handleTextQueryInit, handleTextQueryAllFrames, textQueryAvailable, - autoAlignReady, - handleAutoAlign, + autoRegisterReady, + handleAutoRegister, openLink, /* Stereo */ stereoLoadingDialog, @@ -1917,7 +1917,7 @@ export default defineComponent({ :read-only-mode="readOnlyMode || runningPipelines.length > 0" :text-query-enabled="true" :text-query-available="textQueryAvailable" - :auto-align-handler="autoAlignReady ? handleAutoAlign : null" + :auto-register-handler="autoRegisterReady ? handleAutoRegister : null" @change-camera="changeCamera" @large-image-warning="largeImageWarning()" @text-query-submit="handleTextQuerySubmit" diff --git a/client/src/alignedView/CameraRegistrationStore.spec.ts b/client/src/alignedView/CameraRegistrationStore.spec.ts index dffaea6ac..875e044a2 100644 --- a/client/src/alignedView/CameraRegistrationStore.spec.ts +++ b/client/src/alignedView/CameraRegistrationStore.spec.ts @@ -734,7 +734,7 @@ describe('CameraRegistrationStore', () => { }); }); - describe('applyAutoAlignment', () => { + describe('applyAutoRegistration', () => { // A pure translation by (5, -3), as [ax, ay, bx, by] matcher-style rows. const inliers: [number, number, number, number][] = [ [0, 0, 5, -3], [10, 0, 15, -3], [10, 10, 15, 7], [0, 10, 5, 7], @@ -745,13 +745,13 @@ describe('CameraRegistrationStore', () => { const store = new CameraRegistrationStore(); store.setActivePair('rgb', 'ir'); const key = store.pairKey('rgb', 'ir'); - store.applyAutoAlignment('rgb', 'ir', inliers, { autoAlignModel: 'minima_loftr' }); + store.applyAutoRegistration('rgb', 'ir', inliers, { autoRegisterModel: 'minima_loftr' }); expect(store.correspondences.value[key]).toHaveLength(5); expect(store.transformTypes.value[key]).toBe('homography'); const { AtoB } = store.homographies.value[key]; expect(AtoB[0][2]).toBeCloseTo(5, 5); expect(AtoB[1][2]).toBeCloseTo(-3, 5); - expect(store.source.value).toMatchObject({ autoAlignModel: 'minima_loftr' }); + expect(store.source.value).toMatchObject({ autoRegisterModel: 'minima_loftr' }); expect(store.fitError.value).toBeNull(); expect(store.dirty.value).toBe(true); }); @@ -764,7 +764,7 @@ describe('CameraRegistrationStore', () => { store.addPoint('ir', [2, 2]); store.selectCorrespondence(store.correspondences.value[key][0].id); store.addPoint('rgb', [3, 3]); // pending - store.applyAutoAlignment('rgb', 'ir', inliers); + store.applyAutoRegistration('rgb', 'ir', inliers); expect(store.correspondences.value[key]).toHaveLength(5); expect(store.pendingPoint.value).toBeNull(); expect(store.selectedCorrespondenceId.value).toBeNull(); @@ -782,7 +782,7 @@ describe('CameraRegistrationStore', () => { const degenerate: [number, number, number, number][] = [ [0, 0, 0, 0], [1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], ]; - store.applyAutoAlignment('rgb', 'ir', degenerate); + store.applyAutoRegistration('rgb', 'ir', degenerate); expect(store.correspondences.value[key]).toHaveLength(4); expect(store.fitError.value).not.toBeNull(); }); diff --git a/client/src/alignedView/CameraRegistrationStore.ts b/client/src/alignedView/CameraRegistrationStore.ts index d07d4db9e..4e99e783d 100644 --- a/client/src/alignedView/CameraRegistrationStore.ts +++ b/client/src/alignedView/CameraRegistrationStore.ts @@ -643,10 +643,10 @@ export default class CameraRegistrationStore { * rows in native pixels), switch the pair to a homography fit, and fit from * those points. The points land in the normal correspondence table so the * user can inspect, delete, or drag-refine them exactly like hand-picked - * ones. Provenance (`meta`, e.g. { autoAlign: { model, inlierRatio } }) is + * ones. Provenance (`meta`, e.g. { autoRegister: { model, inlierRatio } }) is * merged into {@link source} so it travels with saves/exports. */ - applyAutoAlignment( + applyAutoRegistration( camA: string, camB: string, inliers: [number, number, number, number][], From 3d6885d58fc50cc48765a508774beb81628bc2f5 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Tue, 14 Jul 2026 16:25:01 -0400 Subject: [PATCH 05/14] Move from 'auto_align' to 'register_images' naming --- client/platform/desktop/backend/ipcService.ts | 2 +- client/platform/desktop/backend/native/interactive.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index cd8e71acf..3487e6783 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -454,7 +454,7 @@ export default function register() { const segService = getInteractiveServiceManager(); // Auto-register only needs the service process running -- its matcher model - // loads lazily inside the auto_align request (and stays resident). + // loads lazily inside the register_images request (and stays resident). await segService.ensureStarted(settings.get()); const response = await segService.autoRegister(args); diff --git a/client/platform/desktop/backend/native/interactive.ts b/client/platform/desktop/backend/native/interactive.ts index 15dc961ab..207a6b1b2 100644 --- a/client/platform/desktop/backend/native/interactive.ts +++ b/client/platform/desktop/backend/native/interactive.ts @@ -497,9 +497,9 @@ export class InteractiveServiceManager extends EventEmitter { // eslint-disable-next-line @typescript-eslint/no-explicit-any }): Promise { const response = await this.sendRequest({ - // The interactive service's wire name (VIAME branch dev/auto-align) - // still says "align"; rename here in step with the service side. - command: 'auto_align', + // Wire name is verb-object style, unlike the autoRegister() method + // above, which matches the UI naming. + command: 'register_images', image_path_a: request.imagePathA, image_path_b: request.imagePathB, options: request.options && { From e8f89dac02f8088c8e2ca37437534e81fe566194 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Tue, 14 Jul 2026 16:29:19 -0400 Subject: [PATCH 06/14] Scrub alignment language, move to register elsewhere --- client/platform/desktop/backend/ipcService.ts | 4 ++-- client/platform/desktop/frontend/api.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index 3487e6783..25386a742 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -434,12 +434,12 @@ export default function register() { return { installed: isSam3Installed(currentSettings.viamePath) }; }); - ipcMain.handle('auto-register-available', () => { + ipcMain.handle('register-images-available', () => { const currentSettings = settings.get(); return { installed: isAutoRegisterInstalled(currentSettings.viamePath) }; }); - ipcMain.handle('auto-register', async (_, args: { + ipcMain.handle('register-images', async (_, args: { imagePathA: string; imagePathB: string; options?: { diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 2cb98d6e9..f7f8b9da3 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -377,11 +377,11 @@ async function textQuery(request: TextQueryRequest): Promise */ async function autoRegisterAvailable(): Promise<{ installed: boolean }> { - return window.diveDesktop.invoke('auto-register-available'); + return window.diveDesktop.invoke('register-images-available'); } async function autoRegister(request: AutoRegisterRequest): Promise { - return window.diveDesktop.invoke('auto-register', request); + return window.diveDesktop.invoke('register-images', request); } async function refineDetections(request: RefineDetectionsRequest): Promise { From 46f77f9a1bda036a954198a7f89bd1b1e1d98795 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 20 Jul 2026 16:06:40 -0400 Subject: [PATCH 07/14] Turn on point picking when Auto Register injects its matches The injected correspondences were invisible after a successful Auto Register unless the user separately flipped the "Pick points" toggle, because the keypoint layer draws nothing while picking is off. Enable picking in applyAutoRegistration so the points appear immediately for review, matching the summary text's "Review the points" instruction. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01J4dW5stcXQBj89d5Kc35bw --- client/src/alignedView/CameraRegistrationStore.spec.ts | 3 +++ client/src/alignedView/CameraRegistrationStore.ts | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/client/src/alignedView/CameraRegistrationStore.spec.ts b/client/src/alignedView/CameraRegistrationStore.spec.ts index 875e044a2..84c27e66f 100644 --- a/client/src/alignedView/CameraRegistrationStore.spec.ts +++ b/client/src/alignedView/CameraRegistrationStore.spec.ts @@ -745,7 +745,10 @@ describe('CameraRegistrationStore', () => { const store = new CameraRegistrationStore(); store.setActivePair('rgb', 'ir'); const key = store.pairKey('rgb', 'ir'); + expect(store.pickingEnabled.value).toBe(false); store.applyAutoRegistration('rgb', 'ir', inliers, { autoRegisterModel: 'minima_loftr' }); + // Picking turns on so the injected points are visible for review. + expect(store.pickingEnabled.value).toBe(true); expect(store.correspondences.value[key]).toHaveLength(5); expect(store.transformTypes.value[key]).toBe('homography'); const { AtoB } = store.homographies.value[key]; diff --git a/client/src/alignedView/CameraRegistrationStore.ts b/client/src/alignedView/CameraRegistrationStore.ts index 4e99e783d..0677cfeb2 100644 --- a/client/src/alignedView/CameraRegistrationStore.ts +++ b/client/src/alignedView/CameraRegistrationStore.ts @@ -644,7 +644,9 @@ export default class CameraRegistrationStore { * those points. The points land in the normal correspondence table so the * user can inspect, delete, or drag-refine them exactly like hand-picked * ones. Provenance (`meta`, e.g. { autoRegister: { model, inlierRatio } }) is - * merged into {@link source} so it travels with saves/exports. + * merged into {@link source} so it travels with saves/exports. Picking is + * switched on so the injected points are immediately visible for that + * review -- the keypoint layer draws nothing while picking is off. */ applyAutoRegistration( camA: string, @@ -666,6 +668,7 @@ export default class CameraRegistrationStore { if (meta) { this.source.value = { ...(this.source.value || {}), ...meta }; } + this.pickingEnabled.value = true; this.maybeFitPair(key); } From ee79a3d90e08281bcf7a5b64bf2b05e2e403275a Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 20 Jul 2026 16:09:32 -0400 Subject: [PATCH 08/14] Rename the "Pick points" toggle to "Edit points" Since Auto Register and registration files can both populate the correspondence table, the toggle's job is broader than placing new points: it gates seeing, dragging, and deleting them too. Comments that referenced the toggle by name follow suit. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01J4dW5stcXQBj89d5Kc35bw --- .../components/CameraRegistration/RegistrationTools.vue | 4 ++-- client/dive-common/components/Viewer.vue | 2 +- client/src/alignedView/CameraRegistrationStore.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/dive-common/components/CameraRegistration/RegistrationTools.vue b/client/dive-common/components/CameraRegistration/RegistrationTools.vue index ecdb6fff2..ef9b29612 100644 --- a/client/dive-common/components/CameraRegistration/RegistrationTools.vue +++ b/client/dive-common/components/CameraRegistration/RegistrationTools.vue @@ -80,7 +80,7 @@ export default defineComponent({ /** * Author-vs-review posture: picking defaults on for a pair that still * needs points, and off for one whose transform came from a registration - * file (review it; the "Pick points" toggle opts back in to refine). + * file (review it; the "Edit points" toggle opts back in to refine). * Re-applied whenever the active pair changes identity, so it overrides a * manual toggle on pair switch -- each pair opens in its own posture. */ @@ -634,7 +634,7 @@ export default defineComponent({