diff --git a/client/dive-common/multicamDisplay.ts b/client/dive-common/multicamDisplay.ts index ad54c052e..e42a308c1 100644 --- a/client/dive-common/multicamDisplay.ts +++ b/client/dive-common/multicamDisplay.ts @@ -57,6 +57,19 @@ export function referenceCameraName(multiCamMedia: MultiCamMediaLike | null | un return defaultDisplay && ordered.includes(defaultDisplay) ? defaultDisplay : ordered[0]; } +/** + * Camera order for 2-cam/3-cam VIAME pipelines: the registration reference + * camera feeds input1 (the per-camera registrations all map onto the + * reference, and the pipes warp everything onto camera 1's frame), remaining + * cameras keep display order. Which detector a pipe runs on which input is + * the pipe's documented contract, not something DIVE infers. + */ +export function pipelineOrderedCameraNames(multiCamMedia: MultiCamMediaLike | null | undefined): string[] { + const ordered = orderedMultiCamCameraNames(multiCamMedia); + const reference = referenceCameraName(multiCamMedia); + return reference ? [reference, ...ordered.filter((name) => name !== reference)] : ordered; +} + export function isMultiCamSubType(subType: SubType | string | null | undefined): subType is MultiCamSubType { return subType === 'stereo' || subType === 'multicam'; } diff --git a/client/platform/desktop/backend/native/cameraRegistration.spec.ts b/client/platform/desktop/backend/native/cameraRegistration.spec.ts new file mode 100644 index 000000000..3b151bb73 --- /dev/null +++ b/client/platform/desktop/backend/native/cameraRegistration.spec.ts @@ -0,0 +1,192 @@ +import mockfs from 'mock-fs'; +import npath from 'path'; +import fs from 'fs-extra'; +import { + it, expect, describe, afterAll, vi, +} from 'vitest'; + +import { Settings, JsonMeta } from 'platform/desktop/constants'; +import { buildRegistrationPipelineArgs } from './cameraRegistration'; + +// mock-fs no longer intercepts fs-extra's exists checks on newer Node; +// route them through statSync like common.spec.ts does. +vi.mock('fs-extra', async () => { + const actual = await vi.importActual('fs-extra'); + const fsNode = await import('node:fs'); + const existsByStat = (targetPath: fsNode.PathLike) => { + try { + fsNode.statSync(targetPath); + return true; + } catch { + return false; + } + }; + + const patchedDefault = { + ...actual.default, + existsSync: existsByStat, + pathExistsSync: existsByStat, + }; + + return { + ...actual, + default: patchedDefault, + existsSync: existsByStat, + pathExistsSync: existsByStat, + }; +}); + +const settings: Settings = { + version: 1, + dataPath: '/home/user/viamedata', + viamePath: '/opt/viame', + readonlyMode: false, + overrides: {}, +}; + +const identity = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; +const irToRgb = [[1, 0, 5], [0, 1, -3], [0, 0, 1]]; +const uvToRgb = [[1, 0, -8], [0, 1, 2], [0, 0, 1]]; + +function registrationFile(pairs: unknown[]) { + return JSON.stringify({ type: 'dive-camera-registration', version: 1, pairs }); +} + +const projectFiles = { 'meta.json': '{}', 'result_1.json': '{}' }; + +mockfs({ + [npath.join(settings.dataPath, 'DIVE_Projects')]: { + withreg: { + ...projectFiles, + 'ir_to_rgb_registration.json': registrationFile([ + { + left: 'ir', right: 'rgb', points: [], leftToRight: irToRgb, rightToLeft: identity, + }, + ]), + 'uv_to_rgb_registration.json': registrationFile([ + { + left: 'uv', right: 'rgb', points: [], leftToRight: uvToRgb, rightToLeft: identity, + }, + ]), + }, + partial: { + ...projectFiles, + 'ir_to_rgb_registration.json': registrationFile([ + { + left: 'ir', right: 'rgb', points: [], leftToRight: irToRgb, rightToLeft: identity, + }, + ]), + // Non-star pair (two non-reference cameras): explicitly unsupported + // for pipelines even though fitted. + 'uv_to_ir_registration.json': registrationFile([ + { + left: 'uv', right: 'ir', points: [], leftToRight: uvToRgb, rightToLeft: identity, + }, + ]), + }, + noreg: { ...projectFiles }, + seeded: { ...projectFiles }, + }, + '/home/user/job': {}, +}); + +afterAll(() => mockfs.restore()); + +function multiCamMeta(id: string, cameras: string[], defaultDisplay: string): JsonMeta { + return { + version: 1, + type: 'multi', + id, + fps: 1, + originalBasePath: '/home/user/data', + originalImageFiles: [], + originalVideoFile: '', + transcodedVideoFile: '', + transcodedImageFiles: [], + name: id, + createdAt: 'now', + subType: 'multicam', + multiCam: { + cameras: Object.fromEntries(cameras.map((name) => [name, { + type: 'image-sequence', + originalBasePath: `/home/user/data/${name}`, + originalImageFiles: [], + originalVideoFile: '', + transcodedVideoFile: '', + transcodedImageFiles: [], + }])), + defaultDisplay, + }, + } as unknown as JsonMeta; +} + +describe('buildRegistrationPipelineArgs', () => { + it('writes one file per camera pair and pins each warp pair/direction', async () => { + // Display order intentionally scrambles the input: the pipeline order is + // reference-first then display order (rgb, uv, ir -- IR displays last), + // so uv lands on input2 / warp2 and ir on input3 / warp3. + const meta = multiCamMeta('withreg', ['ir', 'rgb', 'uv'], 'rgb'); + const jobWorkDir = '/home/user/job/full'; + await fs.ensureDir(jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + + const irPath = npath.join(jobWorkDir, 'ir_to_rgb_registration.json'); + const uvPath = npath.join(jobWorkDir, 'uv_to_rgb_registration.json'); + expect(args).toStrictEqual({ + 'warp2:transformation_file': uvPath, + 'warp2:transform_reader:type': 'dive', + 'warp2:transform_reader:dive:from_camera': 'uv', + 'warp2:transform_reader:dive:to_camera': 'rgb', + 'warp3:transformation_file': irPath, + 'warp3:transform_reader:type': 'dive', + 'warp3:transform_reader:dive:from_camera': 'ir', + 'warp3:transform_reader:dive:to_camera': 'rgb', + }); + const written = await fs.readJSON(irPath); + expect(written.type).toBe('dive-camera-registration'); + expect(written.pairs).toHaveLength(1); + expect(written.pairs[0].left).toBe('ir'); + expect(written.pairs[0].right).toBe('rgb'); + expect(written.pairs[0].leftToRight).toStrictEqual(irToRgb); + expect((await fs.readJSON(uvPath)).pairs[0].left).toBe('uv'); + }); + + it('skips cameras without a fitted reference pair; non-star pairs never reach the job', async () => { + const meta = multiCamMeta('partial', ['rgb', 'ir', 'uv'], 'rgb'); + const jobWorkDir = '/home/user/job/partial'; + await fs.ensureDir(jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + // Pipeline order is rgb, uv, ir: uv (warp2) has only the unsupported + // uv-to-ir pair, so it gets nothing; ir (warp3) has a reference pair. + expect(Object.keys(args).some((key) => key.startsWith('warp2'))).toBe(false); + expect(args['warp3:transform_reader:dive:from_camera']).toBe('ir'); + const irPath = npath.join(jobWorkDir, 'ir_to_rgb_registration.json'); + expect(args['warp3:transformation_file']).toBe(irPath); + // The uv-to-ir pair is dropped from ir's job file too. + const written = await fs.readJSON(irPath); + expect(written.pairs).toHaveLength(1); + expect(written.pairs[0].right).toBe('rgb'); + expect(await fs.readdir(jobWorkDir)).toStrictEqual(['ir_to_rgb_registration.json']); + }); + + it('returns no args when the dataset has no registration', async () => { + const meta = multiCamMeta('noreg', ['rgb', 'ir'], 'rgb'); + const jobWorkDir = '/home/user/job/noreg'; + await fs.ensureDir(jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + expect(args).toStrictEqual({}); + expect(await fs.readdir(jobWorkDir)).toStrictEqual([]); + }); + + it('falls back to the import-time seed in dataset meta when no files exist', async () => { + const meta = multiCamMeta('seeded', ['rgb', 'ir'], 'rgb'); + meta.cameraHomographies = { + 'ir::rgb': { AtoB: irToRgb, BtoA: identity }, + }; + const jobWorkDir = '/home/user/job/seeded'; + await fs.ensureDir(jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + expect(args['warp2:transform_reader:dive:from_camera']).toBe('ir'); + expect(args['warp2:transform_reader:dive:to_camera']).toBe('rgb'); + }); +}); diff --git a/client/platform/desktop/backend/native/cameraRegistration.ts b/client/platform/desktop/backend/native/cameraRegistration.ts index 9ffcff38d..994efdc87 100644 --- a/client/platform/desktop/backend/native/cameraRegistration.ts +++ b/client/platform/desktop/backend/native/cameraRegistration.ts @@ -12,12 +12,16 @@ import fs from 'fs-extra'; import { TransformType, DEFAULT_TRANSFORM_TYPE } from 'vue-media-annotator/alignedView/transform'; import { buildPerCameraRegistrationFiles, registrationValuesSummary, filterRegistrationValues, - mergeRegistrationValues, mergeRegistrationSources, CameraRegistrationValues, + mergeRegistrationValues, mergeRegistrationSources, registrationFileName, + CameraRegistrationValues, } from 'vue-media-annotator/alignedView/cameraRegistrationFiles'; import { readTransformMatrix } from 'vue-media-annotator/alignedView/alignedView'; import { invert3, Matrix3 } from 'vue-media-annotator/alignedView/homography'; import { DatasetMetaMutable } from 'dive-common/apispec'; -import { referenceCameraName as multicamReferenceCameraName } from 'dive-common/multicamDisplay'; +import { + referenceCameraName as multicamReferenceCameraName, + pipelineOrderedCameraNames, +} from 'dive-common/multicamDisplay'; import { RegistrationFileNamePattern, compareRegistrationCandidates, @@ -192,6 +196,73 @@ export async function loadEffectiveRegistration( }; } +/** + * Build the kwiver -s settings that hand a dataset's camera registration to + * a 2-cam/3-cam pipeline. One standard _to__registration + * file per non-reference camera is written into the job work dir; each + * camera's warp process (warp2, warp3, ... matching the pipeline's + * reference-first camera order) receives its own single-pair file. The pair + * and direction are still pinned through the reader's from_camera/to_camera + * config, since a pair may be stored in either orientation. + * + * Only pairs registering a camera directly onto the reference are + * supported: pairs between two non-reference cameras are explicitly + * unsupported here (there is no transform composition) and never reach the + * pipeline. Cameras without a fitted reference pair get no settings; a + * pipeline that needs one then fails at configure time with the file name + * it was missing. + */ +export async function buildRegistrationPipelineArgs( + settings: Settings, + meta: JsonMeta, + jobWorkDir: string, +): Promise> { + const args: Record = {}; + if (!meta.multiCam) { + return args; + } + const projectDirInfo = await getValidatedProjectDir(settings, meta.id); + const values = await loadEffectiveRegistration(projectDirInfo.basePath, meta); + const reference = multicamReferenceCameraName(meta.multiCam); + if (!reference) { + return args; + } + const files = buildPerCameraRegistrationFiles(values, reference); + const writes: Promise[] = []; + pipelineOrderedCameraNames(meta.multiCam).forEach((camera, index) => { + if (index === 0 || camera === reference) { + return; + } + // Points-only pairs have no matrix the warp could apply. + const fitted = values.homographies[`${camera}::${reference}`] + || values.homographies[`${reference}::${camera}`]; + if (!fitted) { + return; + } + const file = files.find((candidate) => candidate.camera === camera); + if (!file) { + return; + } + // Unsupported non-reference pairs are dropped from the file body too, so + // the job dir only ever holds camera-to-reference registrations. + const referencePairs = file.body.pairs.filter( + (pair) => pair.left === reference || pair.right === reference, + ); + if (!referencePairs.length) { + return; + } + const registrationPath = npath.join(jobWorkDir, registrationFileName(camera, reference)); + writes.push(writeJsonFile(registrationPath, { ...file.body, pairs: referencePairs })); + const warp = `warp${index + 1}`; + args[`${warp}:transformation_file`] = registrationPath; + args[`${warp}:transform_reader:type`] = 'dive'; + args[`${warp}:transform_reader:dive:from_camera`] = camera; + args[`${warp}:transform_reader:dive:to_camera`] = reference; + }); + await Promise.all(writes); + return args; +} + /** * Persist camera registration to standalone per-camera files in a dataset * directory, merging partial updates with whatever is already on disk. diff --git a/client/platform/desktop/backend/native/multiCamUtils.ts b/client/platform/desktop/backend/native/multiCamUtils.ts index f6ccf7b5d..0cf4e1632 100644 --- a/client/platform/desktop/backend/native/multiCamUtils.ts +++ b/client/platform/desktop/backend/native/multiCamUtils.ts @@ -11,6 +11,7 @@ import { JsonMeta, Settings } from 'platform/desktop/constants'; import { loadAnnotationFile, loadJsonMetadata, getValidatedProjectDir } from 'platform/desktop/backend/native/common'; import { serialize } from 'platform/desktop/backend/serializers/viame'; import { parseFrameTimestamp } from 'dive-common/frameTimestamp'; +import { pipelineOrderedCameraNames } from 'dive-common/multicamDisplay'; /** * Figure out the destination location @@ -76,11 +77,25 @@ function getTranscodedMultiCamType(imageListFile: string, jsonMeta: JsonMeta) { throw new Error(`No associate type for ${imageListFile} in multiCam data`); } -async function writeMultiCamStereoPipelineArgs(jobWorkDir: string, meta: JsonMeta, settings: Settings, utility = false, forceTranscoded = false) { +async function writeMultiCamStereoPipelineArgs( + jobWorkDir: string, + meta: JsonMeta, + settings: Settings, + utility = false, + forceTranscoded = false, + // 2-cam/3-cam pipes treat camera 1 as the reference frame the other + // cameras register onto, so their inputs go reference-first; stereo + // measurement keeps the stored left/right order. + referenceFirst = false, +) { const argFilePair: Record = {}; const outFiles: Record = {}; if (meta.multiCam && meta.multiCam.cameras) { - const cameraList = Object.entries(meta.multiCam.cameras); + const { cameras } = meta.multiCam; + const cameraNames = referenceFirst + ? pipelineOrderedCameraNames(meta.multiCam).filter((name) => name in cameras) + : Object.keys(cameras); + const cameraList = cameraNames.map((name) => [name, cameras[name]] as const); for (let i = 0; i < cameraList.length; i += 1) { const [key, list] = cameraList[i]; const { originalBasePath } = list; @@ -152,6 +167,7 @@ function getMultiCamUrls( } const multiCamMedia: MultiCamMedia = { cameras: {}, + cameraOrder: projectMetaData.multiCam.cameraOrder, defaultDisplay: projectMetaData.multiCam.defaultDisplay, }; diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index 2ab5f8dd0..e4c96a382 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -23,6 +23,7 @@ import { } from 'dive-common/constants'; import * as common from './common'; import { jobFileEchoMiddleware, createWorkingDirectory, createCustomWorkingDirectory } from './utils'; +import { buildRegistrationPipelineArgs } from './cameraRegistration'; import { getMultiCamImageFiles, getMultiCamVideoPath, writeMultiCamStereoPipelineArgs, @@ -304,8 +305,9 @@ async function runPipeline( let multiOutFiles: Record; if (meta.multiCam && stereoOrMultiCam) { + const isMultiCamPipeline = multiCamPipelineMarkers.includes(pipeline.type); // eslint-disable-next-line max-len - const { argFilePair, outFiles } = await writeMultiCamStereoPipelineArgs(jobWorkDir, meta, settings, requiresInput); + const { argFilePair, outFiles } = await writeMultiCamStereoPipelineArgs(jobWorkDir, meta, settings, requiresInput, false, isMultiCamPipeline); Object.entries(argFilePair).forEach(([arg, file]) => { command.push(`-s ${arg}="${file}"`); }); @@ -327,6 +329,14 @@ async function runPipeline( command.push(`-s measurer:calibration_file="${meta.multiCam.calibration}"`); command.push(`-s calibration_reader:file="${meta.multiCam.calibration}"`); } + if (isMultiCamPipeline) { + // Hand the camera registration (Aligned View homographies) to the + // pipeline's per-camera warp processes. + const registrationArgs = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + Object.entries(registrationArgs).forEach(([arg, value]) => { + command.push(`-s ${arg}="${value}"`); + }); + } } else if (pipeline.type === stereoPipelineMarker) { throw new Error('Attempting to run a multicam pipeline on non multicam data'); } diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py index 7dd64b0a1..e23caa052 100644 --- a/server/dive_server/crud_rpc.py +++ b/server/dive_server/crud_rpc.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta import json -from typing import Dict, List, Optional, Tuple, TypedDict from pathlib import Path +from typing import Dict, List, Optional, Tuple, TypedDict from girder.constants import AccessType from girder.exceptions import RestException @@ -19,7 +19,12 @@ from dive_server import crud, crud_annotation, crud_dataset from dive_tasks import tasks from dive_tasks.utils import choose_annotation_fps -from dive_tasks.multicam_pipeline import is_stereo_or_multicam_pipeline, pipeline_requires_input +from dive_tasks.multicam_pipeline import ( + build_registration_pairs, + is_stereo_or_multicam_pipeline, + pipeline_camera_order, + pipeline_requires_input, +) from dive_utils import TRUTHY_META_VALUES, asbool, constants, fromMeta, models, types from dive_utils.constants import TrainingModelExtensions from dive_utils.serializers import dive, kpf, kwcoco, viame @@ -287,12 +292,24 @@ def run_pipeline( multicam_cameras: List[types.MulticamCameraJob] = [] multicam_default_display = '' calibration_item_id: Optional[str] = None + is_warp_pipeline = False + reference_camera = '' if dataset_type == constants.MultiType: multi_cam = fromMeta(folder, constants.MultiCamMarker, required=True) multicam_default_display = multi_cam['defaultDisplay'] camera_order = crud_dataset._multicam_camera_order(multi_cam) cameras_meta = multi_cam.get('cameras') or {} + is_warp_pipeline = pipeline['type'] in constants.MultiCamPipelineMarkers + if is_warp_pipeline and camera_order: + # 2-cam/3-cam pipes treat camera 1 as the reference frame the + # other cameras register onto; feed them reference-first. + reference_camera = ( + multicam_default_display + if multicam_default_display in cameras_meta + else camera_order[0] + ) + camera_order = pipeline_camera_order(camera_order, reference_camera) for name in camera_order: cam_info = cameras_meta[name] folder_id = cam_info.get('folderId') @@ -352,6 +369,15 @@ def run_pipeline( params['multicam_requires_input'] = multicam_requires_input if calibration_item_id: params['calibration_item_id'] = calibration_item_id + if is_warp_pipeline and reference_camera: + registration_pairs = build_registration_pairs(folder.get('meta') or {}) + if any( + pair.get('leftToRight') or pair.get('rightToLeft') for pair in registration_pairs + ): + params['multicam_registration'] = { + 'reference': reference_camera, + 'pairs': registration_pairs, + } if metadata_file_key and metadata_file_item_id: params['metadata_file_key'] = metadata_file_key params['metadata_file_item_id'] = metadata_file_item_id diff --git a/server/dive_tasks/multicam_pipeline.py b/server/dive_tasks/multicam_pipeline.py index 9a7f4bd75..435411cd1 100644 --- a/server/dive_tasks/multicam_pipeline.py +++ b/server/dive_tasks/multicam_pipeline.py @@ -2,13 +2,14 @@ from __future__ import annotations +import json from pathlib import Path import re import shlex from typing import Dict, List, Optional, Tuple from dive_utils import constants -from dive_utils.types import MulticamCameraJob, PipelineDescription +from dive_utils.types import MulticamCameraJob, MulticamRegistrationJob, PipelineDescription _PIPELINE_INPUT_PATTERN = re.compile(r'utility_|filter_|transcode_|measurement_') @@ -67,6 +68,111 @@ def append_metadata_file_kwiver_settings( command.append(f'-s {shlex.quote(kwiver_key)}={shlex.quote(str(metadata_path))}') +def pipeline_camera_order(camera_names: List[str], reference: str) -> List[str]: + """ + Camera order for 2-cam/3-cam pipelines, matching the desktop client: the + registration reference camera feeds input1 (the per-camera registrations + all map onto the reference, and the pipes warp everything onto camera 1's + frame), remaining cameras keep display order. Which detector a pipe runs + on which input is the pipe's documented contract, not something DIVE + infers. + """ + if reference not in camera_names: + return camera_names + return [reference] + [name for name in camera_names if name != reference] + + +def build_registration_pairs(folder_meta: dict) -> List[dict]: + """ + Convert a dataset folder's camera registration meta (cameraHomographies / + cameraCorrespondences / cameraTransformTypes, keyed by directional + "left::right") into dive-camera-registration file pairs. + """ + homographies = folder_meta.get('cameraHomographies') or {} + correspondences = folder_meta.get('cameraCorrespondences') or {} + transform_types = folder_meta.get('cameraTransformTypes') or {} + keys = set(homographies) | set(correspondences) | set(transform_types) + pairs: List[dict] = [] + for key in sorted(keys): + left, _, right = key.partition('::') + homography = homographies.get(key) + pairs.append( + { + 'left': left, + 'right': right, + 'points': [ + [c['a'][0], c['a'][1], c['b'][0], c['b'][1]] + for c in correspondences.get(key) or [] + ], + 'leftToRight': homography.get('AtoB') if homography else None, + 'rightToLeft': homography.get('BtoA') if homography else None, + 'transformType': transform_types.get(key, 'similarity'), + } + ) + return pairs + + +def build_registration_kwiver_settings( + work_dir: Path, + cameras: List[MulticamCameraJob], + registration: MulticamRegistrationJob, +) -> Dict[str, str]: + """ + Build the -s settings handing the camera registration to a 2-cam/3-cam + pipeline. One standard _to__registration.json per + non-reference camera is written into the work dir; each camera's warp + process (warp2, warp3, ... matching the job camera order) gets its own + single-pair file. The pair and direction are still pinned via the + reader's from_camera/to_camera config, since a pair may be stored in + either orientation. + + Only pairs registering a camera directly onto the reference are + supported: pairs between two non-reference cameras are explicitly + unsupported here (there is no transform composition) and never reach the + pipeline. Cameras without a fitted reference pair get no settings. + """ + reference = registration.get('reference') + if not reference: + return {} + reference_pairs = [ + pair + for pair in registration.get('pairs') or [] + if reference in (pair['left'], pair['right']) and pair['left'] != pair['right'] + ] + pairs_by_camera: Dict[str, List[dict]] = {} + for pair in reference_pairs: + camera = pair['left'] if pair['right'] == reference else pair['right'] + pairs_by_camera.setdefault(camera, []).append(pair) + fitted = { + (pair['left'], pair['right']) + for pair in reference_pairs + if pair.get('leftToRight') or pair.get('rightToLeft') + } + settings: Dict[str, str] = {} + for index, camera in enumerate(cameras): + name = camera['name'] + if index == 0 or name == reference: + continue + if (name, reference) not in fitted and (reference, name) not in fitted: + continue + camera_pairs = pairs_by_camera.get(name) + if not camera_pairs: + continue + registration_path = work_dir / f'{name}_to_{reference}_registration.json' + with open(registration_path, 'w', encoding='utf-8') as registration_file: + json.dump( + {'type': 'dive-camera-registration', 'version': 1, 'pairs': camera_pairs}, + registration_file, + indent=2, + ) + warp = f'warp{index + 1}' + settings[f'{warp}:transformation_file'] = str(registration_path) + settings[f'{warp}:transform_reader:type'] = 'dive' + settings[f'{warp}:transform_reader:dive:from_camera'] = name + settings[f'{warp}:transform_reader:dive:to_camera'] = reference + return settings + + def build_multicam_kwiver_settings( work_dir: Path, cameras: List[MulticamCameraJob], diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py index 22d810aad..cf54d0d17 100644 --- a/server/dive_tasks/tasks.py +++ b/server/dive_tasks/tasks.py @@ -24,6 +24,7 @@ append_metadata_file_kwiver_settings, append_stereo_calibration_kwiver_settings, build_multicam_kwiver_settings, + build_registration_kwiver_settings, find_downloaded_calibration_file, is_stereo_measurement_pipeline, ) @@ -433,6 +434,14 @@ def run_pipeline(self: Task, params: PipelineJob): for arg, file_name in arg_file_pair.items(): command.append(f"-s {shlex.quote(arg)}={shlex.quote(file_name)}") + multicam_registration = multicam_params.get('multicam_registration') + if multicam_registration: + registration_settings = build_registration_kwiver_settings( + _working_directory_path, multicam_cameras, multicam_registration + ) + for arg, value in registration_settings.items(): + command.append(f'-s {shlex.quote(arg)}={shlex.quote(value)}') + calibration_item_id = multicam_params.get('calibration_item_id') if calibration_item_id and is_stereo_measurement_pipeline(pipeline): cal_item = gc.getItem(calibration_item_id) @@ -902,7 +911,9 @@ def convert_calibration(self: Task, itemId: str): folder = gc.getFolder(folder_id) multi_cam = (folder.get('meta') or {}).get(constants.MultiCamMarker) or {} if str(multi_cam.get(constants.CalibrationItemIdMarker)) != str(itemId): - manager.write('Calibration source was replaced before linking JSON; discarding output.\n') + manager.write( + 'Calibration source was replaced before linking JSON; discarding output.\n' + ) gc.delete(f'item/{json_item_id}') return diff --git a/server/dive_utils/types.py b/server/dive_utils/types.py index a70377014..b3cf34c66 100644 --- a/server/dive_utils/types.py +++ b/server/dive_utils/types.py @@ -138,6 +138,17 @@ class PipelineJob(TypedDict): metadata_file_key: NotRequired[Optional[str]] +class MulticamRegistrationJob(TypedDict): + """Camera registration handed to a 2-cam/3-cam pipeline's warp processes. + + Pairs use the dive-camera-registration file layout: left/right camera + names, correspondence points, and leftToRight/rightToLeft 3x3 matrices. + """ + + reference: str + pairs: List[dict] + + class MulticamPipelineJob(PipelineJob, total=False): """Pipeline job fields set when running stereo/multicam pipelines on a multi dataset.""" @@ -145,6 +156,7 @@ class MulticamPipelineJob(PipelineJob, total=False): multicam_default_display: str calibration_item_id: Optional[str] multicam_requires_input: bool + multicam_registration: Optional[MulticamRegistrationJob] class TrainingJob(TypedDict): diff --git a/server/tests/test_multicam_pipeline.py b/server/tests/test_multicam_pipeline.py index 1c4641b84..6ed90f5fb 100644 --- a/server/tests/test_multicam_pipeline.py +++ b/server/tests/test_multicam_pipeline.py @@ -1,11 +1,15 @@ +import json from pathlib import Path from dive_tasks.multicam_pipeline import ( append_stereo_calibration_kwiver_settings, build_multicam_kwiver_settings, + build_registration_kwiver_settings, + build_registration_pairs, find_downloaded_calibration_file, is_stereo_measurement_pipeline, is_stereo_or_multicam_pipeline, + pipeline_camera_order, pipeline_requires_input, ) from dive_utils import constants @@ -73,6 +77,113 @@ def test_build_multicam_kwiver_settings_image_sequence(tmp_path: Path): ) +def test_pipeline_camera_order(): + # Reference first, remaining display order preserved. + assert pipeline_camera_order(['ir', 'rgb', 'uv'], 'rgb') == ['rgb', 'ir', 'uv'] + assert pipeline_camera_order(['rgb', 'uv', 'ir'], 'rgb') == ['rgb', 'uv', 'ir'] + assert pipeline_camera_order(['CENT_IR', 'CENT_EO'], 'CENT_EO') == ['CENT_EO', 'CENT_IR'] + # Unknown reference leaves the order alone. + assert pipeline_camera_order(['a', 'b'], 'missing') == ['a', 'b'] + + +IR_TO_RGB = [[1, 0, 5], [0, 1, -3], [0, 0, 1]] +RGB_TO_IR = [[1, 0, -5], [0, 1, 3], [0, 0, 1]] + + +def test_build_registration_pairs(): + folder_meta = { + 'cameraHomographies': {'ir::rgb': {'AtoB': IR_TO_RGB, 'BtoA': RGB_TO_IR}}, + 'cameraCorrespondences': { + 'ir::rgb': [{'id': 1, 'a': [1, 2], 'b': [3, 4]}], + 'uv::rgb': [{'id': 1, 'a': [5, 6], 'b': [7, 8]}], + }, + 'cameraTransformTypes': {'ir::rgb': 'affine'}, + } + pairs = build_registration_pairs(folder_meta) + assert pairs == [ + { + 'left': 'ir', + 'right': 'rgb', + 'points': [[1, 2, 3, 4]], + 'leftToRight': IR_TO_RGB, + 'rightToLeft': RGB_TO_IR, + 'transformType': 'affine', + }, + { + 'left': 'uv', + 'right': 'rgb', + 'points': [[5, 6, 7, 8]], + 'leftToRight': None, + 'rightToLeft': None, + 'transformType': 'similarity', + }, + ] + assert build_registration_pairs({}) == [] + + +def test_build_registration_kwiver_settings(tmp_path: Path): + cameras = [ + {'name': 'rgb', 'folder_id': '1', 'media_type': constants.ImageSequenceType}, + {'name': 'ir', 'folder_id': '2', 'media_type': constants.ImageSequenceType}, + {'name': 'uv', 'folder_id': '3', 'media_type': constants.ImageSequenceType}, + ] + registration = { + 'reference': 'rgb', + 'pairs': [ + { + 'left': 'ir', + 'right': 'rgb', + 'points': [], + 'leftToRight': IR_TO_RGB, + 'rightToLeft': RGB_TO_IR, + 'transformType': 'similarity', + }, + # Points-only pair: uv has nothing fitted, so no warp3 settings. + { + 'left': 'uv', + 'right': 'rgb', + 'points': [[1, 2, 3, 4]], + 'leftToRight': None, + 'rightToLeft': None, + 'transformType': 'similarity', + }, + # Non-star pair (two non-reference cameras): explicitly + # unsupported, never reaches the pipeline even though fitted. + { + 'left': 'uv', + 'right': 'ir', + 'points': [], + 'leftToRight': IR_TO_RGB, + 'rightToLeft': RGB_TO_IR, + 'transformType': 'similarity', + }, + ], + } + settings = build_registration_kwiver_settings(tmp_path, cameras, registration) + # One file per camera pair; uv is points-only so it gets no file or settings. + registration_path = str(tmp_path / 'ir_to_rgb_registration.json') + assert settings == { + 'warp2:transformation_file': registration_path, + 'warp2:transform_reader:type': 'dive', + 'warp2:transform_reader:dive:from_camera': 'ir', + 'warp2:transform_reader:dive:to_camera': 'rgb', + } + written = json.loads((tmp_path / 'ir_to_rgb_registration.json').read_text(encoding='utf-8')) + assert written['type'] == 'dive-camera-registration' + assert len(written['pairs']) == 1 + assert written['pairs'][0]['left'] == 'ir' + # uv produced no file: its only fitted pair skips the reference. + assert list(tmp_path.iterdir()) == [tmp_path / 'ir_to_rgb_registration.json'] + + +def test_build_registration_kwiver_settings_empty(tmp_path: Path): + cameras = [{'name': 'rgb', 'folder_id': '1', 'media_type': constants.ImageSequenceType}] + assert ( + build_registration_kwiver_settings(tmp_path, cameras, {'reference': '', 'pairs': []}) == {} + ) + assert list(tmp_path.iterdir()) == [] + + def test_build_multicam_kwiver_settings_video(tmp_path: Path): cameras = [ {'name': 'left', 'folder_id': 'l', 'media_type': constants.VideoType},