diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index bed5b63b8..c682a82e0 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -519,6 +519,50 @@ export interface TextQueryResponse { fallback?: boolean; } +export interface AutoRegisterRequest { + /** Path to camera A's image for the chosen frame */ + imagePathA: string; + /** Path to camera B's image for the chosen frame */ + imagePathB: string; + options?: { + /** RANSAC reprojection threshold in matcher-resolution pixels (default 2.0) */ + ransacThreshold?: number; + /** Minimum raw matches before RANSAC (default 100) */ + minMatches?: number; + /** Minimum RANSAC inliers (default 30) */ + minInliers?: number; + /** Minimum inlier ratio (default 0.15) */ + minInlierRatio?: number; + /** How many spatially-spread inlier correspondences to return (default 24) */ + topK?: number; + /** Matcher coarse confidence threshold (default 0.2) */ + matchThreshold?: number; + }; +} + +export interface AutoRegisterResponse { + /** Whether alignment succeeded (quality gate passed) */ + success: boolean; + /** Machine-readable failure code: + * insufficient_matches | low_confidence | degenerate_homography */ + code?: string; + /** Human-readable error, actionable for the user */ + error?: string; + /** 3x3 homography mapping camera A native pixels -> camera B native pixels */ + homography?: number[][]; + /** Spatially-spread inlier correspondences [ax, ay, bx, by][] in native px */ + inliers?: [number, number, number, number][]; + numMatches?: number; + numInliers?: number; + inlierRatio?: number; + /** Native [width, height] of each input image */ + imageSizeA?: [number, number]; + imageSizeB?: [number, number]; + /** Matcher identifier (e.g. 'minima_loftr') */ + model?: string; + elapsedMs?: number; +} + export interface RefineDetectionsRequest { /** Path to the image file */ imagePath: string; diff --git a/client/dive-common/components/CameraRegistration/RegistrationTools.vue b/client/dive-common/components/CameraRegistration/RegistrationTools.vue index 58197c0df..94da0e4be 100644 --- a/client/dive-common/components/CameraRegistration/RegistrationTools.vue +++ b/client/dive-common/components/CameraRegistration/RegistrationTools.vue @@ -16,6 +16,7 @@ import { buildPerCameraRegistrationFiles } from 'vue-media-annotator/alignedView import TooltipBtn from 'vue-media-annotator/components/TooltipButton.vue'; import { useApi } from 'dive-common/apispec'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; +import { useAutoRegister } from 'dive-common/use/useAutoRegister'; export default defineComponent({ name: 'CameraRegistration', @@ -64,7 +65,7 @@ export default defineComponent({ return { icon: complete ? 'mdi-check-circle' : 'mdi-alert', color: complete ? 'success' : 'warning', - text: `${total - unresolvedCount}/${total} cameras registered`, + text: `${total - unresolvedCount}/${total} cameras ready`, }; }); const camLeft = ref(null); @@ -79,7 +80,7 @@ export default defineComponent({ /** * Author-vs-review posture: picking defaults on for a pair that still * needs points, and off for one whose transform came from a registration - * file (review it; the "Pick points" toggle opts back in to refine). + * file (review it; the "Edit points" toggle opts back in to refine). * Re-applied whenever the active pair changes identity, so it overrides a * manual toggle on pair switch -- each pair opens in its own posture. */ @@ -369,6 +370,57 @@ export default defineComponent({ } } + /** + * Auto Register: ask the platform's deep matcher (interactive service) for + * correspondences between the selected pair on the current frame, then + * inject them as ordinary point pairs and fit a homography. The service + * is null / unavailable on platforms without the capability (web), which + * hides the button entirely. + */ + const autoRegisterService = useAutoRegister(); + const autoRegisterAvailable = computed(() => !!autoRegisterService?.available.value); + const autoRegistering = ref(false); + const autoRegisterError = ref(null); + const autoRegisterSummary = ref(null); + + async function runAutoRegister() { + if (!autoRegisterService || !camLeft.value || !camRight.value) { + return; + } + if (correspondences.value.length > 0 || hasLoadedTransform.value) { + const confirmed = await prompt({ + title: 'Auto Register', + text: `This will replace the existing points/transform for ${camLeft.value} → ` + + `${camRight.value} with automatically matched points. Continue?`, + confirm: true, + }); + if (!confirmed) { + return; + } + } + autoRegistering.value = true; + autoRegisterError.value = null; + autoRegisterSummary.value = null; + try { + const result = await autoRegisterService.run(camLeft.value, camRight.value); + if (!result.success || !result.inliers || result.inliers.length === 0) { + autoRegisterError.value = result.error + || 'Auto-register could not compute an alignment for this frame.'; + return; + } + registration.applyAutoRegistration(camLeft.value, camRight.value, result.inliers); + const consensus = result.inlierRatio !== undefined + ? ` (${Math.round(result.inlierRatio * 100)}% match consensus)` : ''; + autoRegisterSummary.value = `Aligned with ${result.inliers.length} matched ` + + `points${consensus}. Review the points and overlay warp, refine if ` + + 'needed, then save.'; + } catch (err) { + autoRegisterError.value = err instanceof Error ? err.message : String(err); + } finally { + autoRegistering.value = false; + } + } + return { cameras, cameraAlignmentStatuses, @@ -404,6 +456,11 @@ export default defineComponent({ setTransformType, setAlignmentMode, save, + autoRegisterAvailable, + autoRegistering, + autoRegisterError, + autoRegisterSummary, + runAutoRegister, }; }, }); @@ -429,14 +486,23 @@ export default defineComponent({ > Source: {{ sourceReadout }} + This pair has been refined in-app since the source registration was - produced. Save, then download the camera's registration from the - Export menu to hand the refinement (and its points) back to the - producer. + produced. + +
+ + + Automatically match points between the two cameras on the current frame + + + {{ autoRegisterError }} + + + {{ autoRegisterSummary }} + + { const progress = alignedView.registrationProgress.value; return progress - ? `${progress.registered}/${progress.total} cameras registered` + ? `${progress.registered}/${progress.total} cameras ready` : ''; }); const activeCameraName = computed(() => { diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 72140f9cc..bff15b030 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -60,13 +60,16 @@ import ControlsContainer from 'dive-common/components/ControlsContainer.vue'; import Sidebar from 'dive-common/components/Sidebar.vue'; import BottomPanel from 'dive-common/components/BottomPanel.vue'; import { useModeManager, useSave, useLassoMode } from 'dive-common/use'; +import { provideAutoRegister } from 'dive-common/use/useAutoRegister'; import type { StereoAnnotationCompleteParams, StereoAnnotationResetParams, StereoSegmentationFinalizeParams, } from 'dive-common/use/useModeManager'; import clientSettingsSetup, { clientSettings, isStereoInteractiveModeEnabled } from 'dive-common/store/settings'; -import { useApi, FrameImage, DatasetType } from 'dive-common/apispec'; +import { + useApi, FrameImage, DatasetType, AutoRegisterResponse, +} from 'dive-common/apispec'; import { orderedMultiCamCameraNames } from 'dive-common/multicamDisplay'; import { buildAlignedTimeline, buildInverseAlignedIndex, computeGapSlots, TimelineResult, @@ -159,6 +162,17 @@ export default defineComponent({ type: Boolean, default: false, }, + /** + * Platform-supplied auto-register runner for the Camera Registration panel: + * resolves each camera's image for a frame and computes an alignment via + * the interactive service. Desktop-only; when null (web) the panel hides + * its Auto Register button. + */ + autoRegisterHandler: { + type: Function as PropType< + ((cameraA: string, cameraB: string, frameNum: number) => Promise) | null>, + default: null, + }, }, setup(props, { emit }) { const { prompt, visible } = usePrompt(); @@ -478,7 +492,7 @@ export default defineComponent({ }); // Publish how much of the rig resolves so UI outside the viewer core // (e.g. the import menu's "Import to all cameras" checkbox) shows the - // same "N/M cameras registered" status as the Align View toggle. + // same "N/M cameras ready" status as the Align View toggle. const registrationProgress = computed(() => { if (!isMultiCameraDataset.value) { return null; @@ -510,7 +524,7 @@ export default defineComponent({ * Camera panes currently displayed. While the Camera Registration panel is * open with an active pair on a 3+ camera dataset, only the pair's two * panes show, so the left/right alignment flow reads without unrelated - * panes in between (regardless of whether Pick points is toggled on). + * panes in between (regardless of whether Edit points is toggled on). * Panes are hidden (v-show), not unmounted, so their viewers keep state. */ const displayedCameras = computed(() => { @@ -573,6 +587,20 @@ export default defineComponent({ const lassoMode = useLassoMode(); provide(LassoModeSymbol, lassoMode); + // Auto-register bridge for the Camera Registration panel: injects the current + // frame into the platform handler (mirrors onTextQuerySubmit). Provided + // unconditionally; `available` tracks the handler prop, which the desktop + // platform sets once its availability probe resolves (never on web). + provideAutoRegister({ + available: computed(() => !!props.autoRegisterHandler), + run: (cameraA: string, cameraB: string) => { + if (!props.autoRegisterHandler) { + return Promise.reject(new Error('Auto-register is not available on this platform')); + } + return props.autoRegisterHandler(cameraA, cameraB, aggregateController.value.frame.value) as Promise; + }, + }); + // Provides wrappers for actions to integrate with settings const { linkingTrack, diff --git a/client/dive-common/components/alignedViewTooltip.spec.ts b/client/dive-common/components/alignedViewTooltip.spec.ts index cba2be334..e0d89f99a 100644 --- a/client/dive-common/components/alignedViewTooltip.spec.ts +++ b/client/dive-common/components/alignedViewTooltip.spec.ts @@ -25,7 +25,7 @@ describe('alignedViewTooltipText', () => { enabled: false, sourceIsMixed: false, progress: { registered: 1, total: 3 }, - })).toBe('Align View — 1/3 cameras registered'); + })).toBe('Align View — 1/3 cameras ready'); }); it('appends a mixed-calibration warning', () => { diff --git a/client/dive-common/components/alignedViewTooltip.ts b/client/dive-common/components/alignedViewTooltip.ts index 3a3ae4541..c0cc3cec4 100644 --- a/client/dive-common/components/alignedViewTooltip.ts +++ b/client/dive-common/components/alignedViewTooltip.ts @@ -26,7 +26,7 @@ export default function alignedViewTooltipText(options: { return `Align View on (draw/edit on any camera)${mixedNote}`; } if (progress && progress.registered < progress.total) { - return `Align View — ${progress.registered}/${progress.total} cameras registered${mixedNote}`; + return `Align View — ${progress.registered}/${progress.total} cameras ready${mixedNote}`; } return `Align View${mixedNote}`; } diff --git a/client/dive-common/use/useAutoRegister.ts b/client/dive-common/use/useAutoRegister.ts new file mode 100644 index 000000000..5c9dceeac --- /dev/null +++ b/client/dive-common/use/useAutoRegister.ts @@ -0,0 +1,31 @@ +import { provide, inject, Ref } from 'vue'; +import type { AutoRegisterResponse } from 'dive-common/apispec'; + +/** + * Auto-register service bridge for the Camera Registration panel. + * + * The platform layer (desktop ViewerLoader) implements the actual call — it + * resolves each camera's image path for the current frame and invokes the + * interactive service. Viewer.vue provides it (wrapping in the current frame + * number); RegistrationTools.vue injects it. On platforms without the + * capability (web) nothing is provided and the panel hides the button. + */ +export interface AutoRegisterService { + /** + * Whether auto-register is usable right now. Reactive: the platform's + * availability probe (matcher weights installed?) resolves after mount. + */ + available: Readonly>; + /** Compute an alignment from camera A to camera B on the current frame. */ + run: (cameraA: string, cameraB: string) => Promise; +} + +const AutoRegisterSymbol = Symbol('autoRegister'); + +export function provideAutoRegister(service: AutoRegisterService) { + provide(AutoRegisterSymbol, service); +} + +export function useAutoRegister(): AutoRegisterService | null { + return inject(AutoRegisterSymbol, null); +} diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index b546ac1cc..acec71b11 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -6,7 +6,7 @@ import { app, ipcMain, dialog, BrowserWindow, } from 'electron'; import { MultiCamImportArgs } from 'dive-common/apispec'; -import type { Pipe } from 'dive-common/apispec'; +import type { AutoRegisterRequest, Pipe } from 'dive-common/apispec'; import { DesktopJobUpdate, RunPipeline, RunTraining, Settings, ExportDatasetArgs, ExportMulticamEverythingArgs, @@ -57,6 +57,16 @@ function isSam3Installed(viamePath: string): boolean { path.join(pipelinesDir, configName), )); } + +// Auto-register (Camera Registration panel) needs the MINIMA-LoFTR matcher weights +// in the VIAME install; without them the button is hidden. +function isAutoRegisterInstalled(viamePath: string): boolean { + if (process.env.VIAME_ALIGNMENT_WEIGHTS + && fs.existsSync(process.env.VIAME_ALIGNMENT_WEIGHTS)) { + return true; + } + return fs.existsSync(path.join(viamePath, 'configs', 'pipelines', 'models', 'minima_loftr.ckpt')); +} if (OS.platform() === 'win32') { win32.initialize(); } @@ -424,6 +434,22 @@ export default function register() { return { installed: isSam3Installed(currentSettings.viamePath) }; }); + ipcMain.handle('register-images-available', () => { + const currentSettings = settings.get(); + return { installed: isAutoRegisterInstalled(currentSettings.viamePath) }; + }); + + ipcMain.handle('register-images', async (_, args: AutoRegisterRequest) => { + const segService = getInteractiveServiceManager(); + + // Auto-register only needs the service process running -- its matcher model + // loads lazily inside the register_images request (and stays resident). + await segService.ensureStarted(settings.get()); + + const response = await segService.autoRegister(args); + return response; + }); + ipcMain.handle('segmentation-text-query', async (_, args: { imagePath: string; frameTime?: number; diff --git a/client/platform/desktop/backend/native/interactive.ts b/client/platform/desktop/backend/native/interactive.ts index 5ea1ac74e..814172087 100644 --- a/client/platform/desktop/backend/native/interactive.ts +++ b/client/platform/desktop/backend/native/interactive.ts @@ -19,6 +19,7 @@ import { spawn, ChildProcess } from 'child_process'; import npath from 'path'; import readline from 'readline'; import { EventEmitter } from 'events'; +import type { AutoRegisterRequest, AutoRegisterResponse } from 'dive-common/apispec'; import { Settings } from 'platform/desktop/constants'; import { observeChild } from './processManager'; import linux from './linux'; @@ -114,6 +115,25 @@ interface PendingRequest { timeout: NodeJS.Timeout; } +/** + * Snake_case payload of the service's register_images response; mapped to the + * camelCase AutoRegisterResponse before it leaves this module. + */ +interface RegisterImagesWireResponse { + success: boolean; + code?: string; + error?: string; + homography?: number[][]; + inliers?: [number, number, number, number][]; + num_matches?: number; + num_inliers?: number; + inlier_ratio?: number; + image_size_a?: [number, number]; + image_size_b?: [number, number]; + model?: string; + elapsed_ms?: number; +} + export class InteractiveServiceManager extends EventEmitter { private process: ChildProcess | null = null; @@ -477,6 +497,45 @@ export class InteractiveServiceManager extends EventEmitter { }, 'Text query'); } + /** + * Auto-register: compute a cross-modality homography between two camera + * frames (image A -> image B, native pixel coordinates) with the deep + * matcher hosted by the alignment backend. The matcher model loads lazily + * on the first call and stays resident in the service process. + */ + async autoRegister(request: AutoRegisterRequest): Promise { + const raw = await this.sendRequest({ + // Wire name is verb-object style, unlike the autoRegister() method + // above, which matches the UI naming. + command: 'register_images', + image_path_a: request.imagePathA, + image_path_b: request.imagePathB, + options: request.options && { + ransac_threshold: request.options.ransacThreshold, + min_matches: request.options.minMatches, + min_inliers: request.options.minInliers, + min_inlier_ratio: request.options.minInlierRatio, + top_k: request.options.topK, + match_threshold: request.options.matchThreshold, + }, + }, 'Auto register') as RegisterImagesWireResponse; + // Map the service's snake_case payload to the AutoRegisterResponse shape. + return { + success: raw.success, + code: raw.code, + error: raw.error, + homography: raw.homography, + inliers: raw.inliers, + numMatches: raw.num_matches, + numInliers: raw.num_inliers, + inlierRatio: raw.inlier_ratio, + imageSizeA: raw.image_size_a, + imageSizeB: raw.image_size_b, + model: raw.model, + elapsedMs: raw.elapsed_ms, + }; + } + async refineDetections(request: { imagePath: string; detections: { diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 451320312..f7f8b9da3 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -8,6 +8,7 @@ import type { SegmentationPredictRequest, SegmentationPredictResponse, SegmentationStatusResponse, SegmentationStereoSegmentRequest, SegmentationStereoSegmentResponse, TextQueryRequest, TextQueryResponse, RefineDetectionsRequest, RefineDetectionsResponse, + AutoRegisterRequest, AutoRegisterResponse, } from 'dive-common/apispec'; import { @@ -369,6 +370,20 @@ async function textQuery(request: TextQueryRequest): Promise return window.diveDesktop.invoke('segmentation-text-query', request); } +/** + * Auto Register API (Camera Registration panel) + * Computes a cross-modality homography between two camera frames using the + * deep matcher hosted by the interactive service. + */ + +async function autoRegisterAvailable(): Promise<{ installed: boolean }> { + return window.diveDesktop.invoke('register-images-available'); +} + +async function autoRegister(request: AutoRegisterRequest): Promise { + return window.diveDesktop.invoke('register-images', request); +} + async function refineDetections(request: RefineDetectionsRequest): Promise { return window.diveDesktop.invoke('segmentation-refine', request); } @@ -761,6 +776,9 @@ export { textQuery, refineDetections, runTextQueryPipeline, + /* Auto Register APIs */ + autoRegisterAvailable, + autoRegister, /* Stereo APIs */ stereoEnable, stereoDisable, diff --git a/client/platform/desktop/frontend/components/ViewerLoader.vue b/client/platform/desktop/frontend/components/ViewerLoader.vue index edc887870..86c6ca688 100644 --- a/client/platform/desktop/frontend/components/ViewerLoader.vue +++ b/client/platform/desktop/frontend/components/ViewerLoader.vue @@ -11,7 +11,7 @@ import context from 'dive-common/store/context'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import { SegmentationPredictRequest } from 'dive-common/apispec'; import { clientSettings } from 'dive-common/store/settings'; -import { isStereoscopicDatasetMeta } from 'dive-common/multicamDisplay'; +import { isStereoscopicDatasetMeta, isMultiCamDatasetMeta } from 'dive-common/multicamDisplay'; import type { StereoAnnotationCompleteParams, StereoAnnotationResetParams, @@ -29,6 +29,7 @@ import { segmentationSam3Installed, loadMetadata, textQuery, runTextQueryPipeline, + autoRegister, autoRegisterAvailable, stereoEnable, stereoDisable, stereoSetFrame, stereoTransferLine, stereoTransferPoints, stereoMeasureLine, stereoAggregateLengths, onStereoDisparityReady, onStereoDisparityError, @@ -151,6 +152,7 @@ export default defineComponent({ const readOnlyMode = computed(() => settings.value?.readonlyMode || false); const timeFilter: Ref<[number, number] | null> = ref(null); const textQueryAvailable = ref(false); + const autoRegisterReady = ref(false); async function refreshTextQueryAvailability() { try { @@ -161,8 +163,18 @@ export default defineComponent({ } } + async function refreshAutoRegisterAvailability() { + try { + const result = await autoRegisterAvailable(); + autoRegisterReady.value = result.installed; + } catch { + autoRegisterReady.value = false; + } + } + watch(() => settings.value?.viamePath, () => { refreshTextQueryAvailability(); + refreshAutoRegisterAvailability(); }); watch( @@ -547,6 +559,7 @@ export default defineComponent({ onMounted(() => { initializeSegmentation(); refreshTextQueryAvailability(); + refreshAutoRegisterAvailability(); }); /** @@ -639,56 +652,83 @@ export default defineComponent({ let stereoDatasetFps: number | undefined; /** - * Load multicam metadata for both cameras to build image path getters + * Populate stereoImagePathGetters (per-camera frame -> image path) from a + * dataset's already-loaded multicam metadata. This part is not stereo + * specific: both the stereo features and Auto Register resolve per-camera + * image paths the same way. Callers gate on the dataset kind they support + * (stereo requires a stereoscopic dataset; auto-register accepts any multicam) + * before calling this. `meta.multiCamMedia` must already be truthy. */ + async function populateMultiCamImagePathGetters( + meta: Awaited>, + ): Promise { + // Extract calibration file path from multiCam metadata + stereoCalibrationFile = meta.multiCam?.calibration || undefined; + // Capture the dataset-level fps as a fallback for per-camera frame times. + stereoDatasetFps = meta.fps || meta.originalFps || stereoDatasetFps; + + // Skip per-camera metadata loading if already populated (e.g. by initializeSegmentation) + if (Object.keys(stereoImagePathGetters.value).length > 0) return true; + + const { cameras } = meta.multiCamMedia; + const cameraNames = Object.keys(cameras); + + for (let i = 0; i < cameraNames.length; i += 1) { + const cam = cameraNames[i]; + const cameraId = `${props.id}/${cam}`; + // eslint-disable-next-line no-await-in-loop + const camMeta = await loadMetadata(cameraId); + const { + originalBasePath, originalImageFiles, type, originalVideoFile, + } = camMeta; + + stereoImagePathGetters.value[cam] = (frameNum: number): string => { + if (type === 'video') { + return joinPath(originalBasePath, originalVideoFile || ''); + } + if (originalImageFiles && originalImageFiles[frameNum]) { + const imagePath = originalImageFiles[frameNum]; + if (isAbsolutePath(imagePath)) { + return imagePath; + } + return joinPath(originalBasePath, imagePath); + } + return ''; + }; + } + return true; + } + async function loadStereoMetadata(): Promise { try { const meta = await loadMetadata(props.id); // Plain multicam and single-camera datasets have no stereo pair: report // no stereo so the caller does not load the stereo service. if (!meta.multiCamMedia || !isStereoscopicDatasetMeta(meta)) return false; - - // Extract calibration file path from multiCam metadata - stereoCalibrationFile = meta.multiCam?.calibration || undefined; - // Capture the dataset-level fps as a fallback for per-camera frame times. - stereoDatasetFps = meta.fps || meta.originalFps || stereoDatasetFps; - - // Skip per-camera metadata loading if already populated (e.g. by initializeSegmentation) - if (Object.keys(stereoImagePathGetters.value).length > 0) return true; - - const { cameras } = meta.multiCamMedia; - const cameraNames = Object.keys(cameras); - - for (let i = 0; i < cameraNames.length; i += 1) { - const cam = cameraNames[i]; - const cameraId = `${props.id}/${cam}`; - // eslint-disable-next-line no-await-in-loop - const camMeta = await loadMetadata(cameraId); - const { - originalBasePath, originalImageFiles, type, originalVideoFile, - } = camMeta; - - stereoImagePathGetters.value[cam] = (frameNum: number): string => { - if (type === 'video') { - return joinPath(originalBasePath, originalVideoFile || ''); - } - if (originalImageFiles && originalImageFiles[frameNum]) { - const imagePath = originalImageFiles[frameNum]; - if (isAbsolutePath(imagePath)) { - return imagePath; - } - return joinPath(originalBasePath, imagePath); - } - return ''; - }; - } - return true; + return await populateMultiCamImagePathGetters(meta); } catch (err) { console.error('[Stereo] Failed to load multicam metadata:', err); return false; } } + /** + * Multicam metadata loader for Auto Register. Unlike loadStereoMetadata this + * accepts ANY multi-camera dataset (stereoscopic or plain 'multicam', e.g. + * an EO/IR rig), since deep-feature alignment does not require a calibrated + * stereo pair -- only two cameras' images for the current frame. + */ + async function loadMultiCamMetadata(): Promise { + try { + const meta = await loadMetadata(props.id); + if (!meta.multiCamMedia || !isMultiCamDatasetMeta(meta)) return false; + return await populateMultiCamImagePathGetters(meta); + } catch (err) { + console.error('[MultiCam] Failed to load multicam metadata:', err); + return false; + } + } + // Track last stereo frame to avoid redundant set_frame calls let lastStereoFrame = -1; @@ -728,6 +768,34 @@ export default defineComponent({ } } + /** + * Auto-register handler for the Camera Registration panel (passed down through + * Viewer's auto-register bridge). Resolves each camera's image path for the + * requested frame via the same per-camera getters stereo uses, then asks + * the interactive service for a deep-matched homography. The returned + * correspondences/homography map camera A native pixels -> camera B. + */ + async function handleAutoRegister(cameraA: string, cameraB: string, frameNum: number) { + // Populates stereoImagePathGetters from multicam metadata (no-op when + // already loaded). Accepts any multi-camera dataset (stereoscopic or + // plain multicam); false means this is a single-camera dataset. + const isMulticam = await loadMultiCamMetadata(); + if (!isMulticam) { + throw new Error('Auto-register requires a multi-camera dataset'); + } + const getterA = stereoImagePathGetters.value[cameraA]; + const getterB = stereoImagePathGetters.value[cameraB]; + if (!getterA || !getterB) { + throw new Error(`Could not resolve media for cameras "${cameraA}" and "${cameraB}"`); + } + const imagePathA = getterA(frameNum); + const imagePathB = getterB(frameNum); + if (!imagePathA || !imagePathB) { + throw new Error(`No image found for frame ${frameNum} on both cameras`); + } + return autoRegister({ imagePathA, imagePathB }); + } + // The backend stereo service is needed whenever either stereo feature is on // (length-on-modify or cross-camera auto-compute). Track the combined state // so toggling one feature while the other is already on does not restart it. @@ -1817,6 +1885,8 @@ export default defineComponent({ handleTextQueryInit, handleTextQueryAllFrames, textQueryAvailable, + autoRegisterReady, + handleAutoRegister, openLink, /* Stereo */ stereoLoadingDialog, @@ -1847,6 +1917,7 @@ export default defineComponent({ :read-only-mode="readOnlyMode || runningPipelines.length > 0" :text-query-enabled="true" :text-query-available="textQueryAvailable" + :auto-register-handler="autoRegisterReady ? handleAutoRegister : null" @change-camera="changeCamera" @large-image-warning="largeImageWarning()" @text-query-submit="handleTextQuerySubmit" diff --git a/client/src/alignedView/AlignedViewStore.ts b/client/src/alignedView/AlignedViewStore.ts index 0053e0ab2..9d1348ec8 100644 --- a/client/src/alignedView/AlignedViewStore.ts +++ b/client/src/alignedView/AlignedViewStore.ts @@ -47,7 +47,7 @@ export default class AlignedViewStore { * reference space, out of all loaded cameras. Maintained by the viewer * alongside {@link setTransforms}; null for single-camera datasets. Lets * UI outside the viewer core (e.g. the import menu) show the same - * "N/M cameras registered" status as the Align View toggle without + * "N/M cameras ready" status as the Align View toggle without * re-deriving the pair graph. */ registrationProgress: Ref<{ registered: number; total: number } | null>; diff --git a/client/src/alignedView/CameraRegistrationStore.spec.ts b/client/src/alignedView/CameraRegistrationStore.spec.ts index 6d0ba7814..9f157bad4 100644 --- a/client/src/alignedView/CameraRegistrationStore.spec.ts +++ b/client/src/alignedView/CameraRegistrationStore.spec.ts @@ -734,6 +734,79 @@ describe('CameraRegistrationStore', () => { }); }); + describe('applyAutoRegistration', () => { + // A pure translation by (5, -3), as [ax, ay, bx, by] matcher-style rows. + const inliers: [number, number, number, number][] = [ + [0, 0, 5, -3], [10, 0, 15, -3], [10, 10, 15, 7], [0, 10, 5, 7], + [5, 5, 10, 2], + ]; + + it('injects inliers as correspondences and fits a homography', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('rgb', 'ir'); + const key = store.pairKey('rgb', 'ir'); + expect(store.pickingEnabled.value).toBe(false); + store.applyAutoRegistration('rgb', 'ir', inliers); + // Picking turns on so the injected points are visible for review. + expect(store.pickingEnabled.value).toBe(true); + expect(store.correspondences.value[key]).toHaveLength(5); + expect(store.transformTypes.value[key]).toBe('homography'); + const { AtoB } = store.homographies.value[key]; + expect(AtoB[0][2]).toBeCloseTo(5, 5); + expect(AtoB[1][2]).toBeCloseTo(-3, 5); + expect(store.fitError.value).toBeNull(); + expect(store.dirty.value).toBe(true); + }); + + it('leaves the rig-global source stamp untouched', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('rgb', 'ir'); + // A producer stamp from a loaded registration: rig-global, so a + // single pair's auto-register must not restamp it (that would rewrite + // every camera's file on the next save, not just this pair's). + const producer = { model: 'N56RF', flight: 'dataset_1' }; + store.source.value = { ...producer }; + store.applyAutoRegistration('rgb', 'ir', inliers); + expect(store.source.value).toEqual(producer); + // And with no loaded registration, none appears. + const fresh = new CameraRegistrationStore(); + fresh.setActivePair('rgb', 'ir'); + fresh.applyAutoRegistration('rgb', 'ir', inliers); + expect(fresh.source.value).toBeNull(); + }); + + it('replaces existing points and clears authoring state', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('rgb', 'ir'); + const key = store.pairKey('rgb', 'ir'); + store.addPoint('rgb', [1, 1]); + store.addPoint('ir', [2, 2]); + store.selectCorrespondence(store.correspondences.value[key][0].id); + store.addPoint('rgb', [3, 3]); // pending + store.applyAutoRegistration('rgb', 'ir', inliers); + expect(store.correspondences.value[key]).toHaveLength(5); + expect(store.pendingPoint.value).toBeNull(); + expect(store.selectedCorrespondenceId.value).toBeNull(); + // Injected points are ordinary correspondences: editable like picked ones. + const first = store.correspondences.value[key][0]; + store.updateCorrespondencePoint(first.id, 'rgb', [100, 100]); + expect(store.correspondences.value[key][0].a).toEqual([100, 100]); + }); + + it('surfaces a fitError instead of throwing on degenerate inliers', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('rgb', 'ir'); + const key = store.pairKey('rgb', 'ir'); + // All points collinear: enough rows for a homography but unsolvable. + const degenerate: [number, number, number, number][] = [ + [0, 0, 0, 0], [1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], + ]; + store.applyAutoRegistration('rgb', 'ir', degenerate); + expect(store.correspondences.value[key]).toHaveLength(4); + expect(store.fitError.value).not.toBeNull(); + }); + }); + describe('registration file round trip', () => { it('serializes and reloads all pairs', () => { const store = new CameraRegistrationStore(); diff --git a/client/src/alignedView/CameraRegistrationStore.ts b/client/src/alignedView/CameraRegistrationStore.ts index 48ac51d18..40c3f70bf 100644 --- a/client/src/alignedView/CameraRegistrationStore.ts +++ b/client/src/alignedView/CameraRegistrationStore.ts @@ -463,7 +463,7 @@ export default class CameraRegistrationStore { * unless the pair already has a file-loaded transform, in which case it * opens in a review posture (false) so stray clicks don't start placing * points on top of a registration that needs no points. The user can still - * opt back in via the panel's "Pick points" toggle to refine. + * opt back in via the panel's "Edit points" toggle to refine. */ pickingDefaultFor(key: string | null): boolean { return !(key && this.homographies.value[key] && this.isLoadedHomography(key)); @@ -637,6 +637,44 @@ export default class CameraRegistrationStore { return { AtoB, BtoA }; } + /** + * Apply an automatically computed alignment for `camA` -> `camB`: replace the + * pair's correspondences with the matcher's inlier points ([ax, ay, bx, by] + * rows in native pixels), switch the pair to a homography fit, and fit from + * those points. The points land in the normal correspondence table so the + * user can inspect, delete, or drag-refine them exactly like hand-picked + * ones. Picking is switched on so the injected points are immediately + * visible for that review -- the keypoint layer draws nothing while picking + * is off. + * + * Deliberately does NOT touch {@link source}: that stamp is rig-global + * (written into EVERY per-camera registration file), so recording this one + * pair's matcher provenance there would falsely restamp -- and therefore + * rewrite -- the other cameras' files on the next save. The pair's + * divergence from a loaded producer registration still surfaces through + * {@link isRefinedFromSource}, like any in-app refit. Persisted per-pair + * matcher provenance needs a pair-level source in the file format first. + */ + applyAutoRegistration( + camA: string, + camB: string, + inliers: [number, number, number, number][], + ) { + const key = this.pairKey(camA, camB); + const list: Correspondence[] = inliers.map(([ax, ay, bx, by]) => ({ + // eslint-disable-next-line no-plusplus + id: this.nextId++, + a: [ax, ay] as Point, + b: [bx, by] as Point, + })); + this.correspondences.value = { ...this.correspondences.value, [key]: list }; + this.pendingPoint.value = null; + this.selectedCorrespondenceId.value = null; + this.transformTypes.value = { ...this.transformTypes.value, [key]: 'homography' }; + this.pickingEnabled.value = true; + this.maybeFitPair(key); + } + /** * Parse and load a registration JSON file (the per-camera * _to__registration.json format written by