Kitware · romleiaj · Jul 15, 2026
diff --git a/client/dive-common/multicamDisplay.ts b/client/dive-common/multicamDisplay.ts
@@ -57,6 +57,19 @@ export function referenceCameraName(multiCamMedia: MultiCamMediaLike | null | un
   return defaultDisplay && ordered.includes(defaultDisplay) ? defaultDisplay : ordered[0];
 }
 
+/**
+ * Camera order for 2-cam/3-cam VIAME pipelines: the registration reference
+ * camera feeds input1 (the per-camera registrations all map onto the
+ * reference, and the pipes warp everything onto camera 1's frame), remaining
+ * cameras keep display order. Which detector a pipe runs on which input is
+ * the pipe's documented contract, not something DIVE infers.
+ */
+export function pipelineOrderedCameraNames(multiCamMedia: MultiCamMediaLike | null | undefined): string[] {
+  const ordered = orderedMultiCamCameraNames(multiCamMedia);
+  const reference = referenceCameraName(multiCamMedia);
+  return reference ? [reference, ...ordered.filter((name) => name !== reference)] : ordered;
+}
+
 export function isMultiCamSubType(subType: SubType | string | null | undefined): subType is MultiCamSubType {
   return subType === 'stereo' || subType === 'multicam';
 }

diff --git a/client/platform/desktop/backend/native/cameraRegistration.spec.ts b/client/platform/desktop/backend/native/cameraRegistration.spec.ts
@@ -0,0 +1,192 @@
+import mockfs from 'mock-fs';
+import npath from 'path';
+import fs from 'fs-extra';
+import {
+  it, expect, describe, afterAll, vi,
+} from 'vitest';
+
+import { Settings, JsonMeta } from 'platform/desktop/constants';
+import { buildRegistrationPipelineArgs } from './cameraRegistration';
+
+// mock-fs no longer intercepts fs-extra's exists checks on newer Node;
+// route them through statSync like common.spec.ts does.
+vi.mock('fs-extra', async () => {
+  const actual = await vi.importActual<typeof import('fs-extra')>('fs-extra');
+  const fsNode = await import('node:fs');
+  const existsByStat = (targetPath: fsNode.PathLike) => {
+    try {
+      fsNode.statSync(targetPath);
+      return true;
+    } catch {
+      return false;
+    }
+  };
+
+  const patchedDefault = {
+    ...actual.default,
+    existsSync: existsByStat,
+    pathExistsSync: existsByStat,
+  };
+
+  return {
+    ...actual,
+    default: patchedDefault,
+    existsSync: existsByStat,
+    pathExistsSync: existsByStat,
+  };
+});
+
+const settings: Settings = {
+  version: 1,
+  dataPath: '/home/user/viamedata',
+  viamePath: '/opt/viame',
+  readonlyMode: false,
+  overrides: {},
+};
+
+const identity = [[1, 0, 0], [0, 1, 0], [0, 0, 1]];
+const irToRgb = [[1, 0, 5], [0, 1, -3], [0, 0, 1]];
+const uvToRgb = [[1, 0, -8], [0, 1, 2], [0, 0, 1]];
+
+function registrationFile(pairs: unknown[]) {
+  return JSON.stringify({ type: 'dive-camera-registration', version: 1, pairs });
+}
+
+const projectFiles = { 'meta.json': '{}', 'result_1.json': '{}' };
+
+mockfs({
+  [npath.join(settings.dataPath, 'DIVE_Projects')]: {
+    withreg: {
+      ...projectFiles,
+      'ir_to_rgb_registration.json': registrationFile([
+        {
+          left: 'ir', right: 'rgb', points: [], leftToRight: irToRgb, rightToLeft: identity,
+        },
+      ]),
+      'uv_to_rgb_registration.json': registrationFile([
+        {
+          left: 'uv', right: 'rgb', points: [], leftToRight: uvToRgb, rightToLeft: identity,
+        },
+      ]),
+    },
+    partial: {
+      ...projectFiles,
+      'ir_to_rgb_registration.json': registrationFile([
+        {
+          left: 'ir', right: 'rgb', points: [], leftToRight: irToRgb, rightToLeft: identity,
+        },
+      ]),
+      // Non-star pair (two non-reference cameras): explicitly unsupported
+      // for pipelines even though fitted.
+      'uv_to_ir_registration.json': registrationFile([
+        {
+          left: 'uv', right: 'ir', points: [], leftToRight: uvToRgb, rightToLeft: identity,
+        },
+      ]),
+    },
+    noreg: { ...projectFiles },
+    seeded: { ...projectFiles },
+  },
+  '/home/user/job': {},
+});
+
+afterAll(() => mockfs.restore());
+
+function multiCamMeta(id: string, cameras: string[], defaultDisplay: string): JsonMeta {
+  return {
+    version: 1,
+    type: 'multi',
+    id,
+    fps: 1,
+    originalBasePath: '/home/user/data',
+    originalImageFiles: [],
+    originalVideoFile: '',
+    transcodedVideoFile: '',
+    transcodedImageFiles: [],
+    name: id,
+    createdAt: 'now',
+    subType: 'multicam',
+    multiCam: {
+      cameras: Object.fromEntries(cameras.map((name) => [name, {
+        type: 'image-sequence',
+        originalBasePath: `/home/user/data/${name}`,
+        originalImageFiles: [],
+        originalVideoFile: '',
+        transcodedVideoFile: '',
+        transcodedImageFiles: [],
+      }])),
+      defaultDisplay,
+    },
+  } as unknown as JsonMeta;
+}
+
+describe('buildRegistrationPipelineArgs', () => {
+  it('writes one file per camera pair and pins each warp pair/direction', async () => {
+    // Display order intentionally scrambles the input: the pipeline order is
+    // reference-first then display order (rgb, uv, ir -- IR displays last),
+    // so uv lands on input2 / warp2 and ir on input3 / warp3.
+    const meta = multiCamMeta('withreg', ['ir', 'rgb', 'uv'], 'rgb');
+    const jobWorkDir = '/home/user/job/full';
+    await fs.ensureDir(jobWorkDir);
+    const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir);
+
+    const irPath = npath.join(jobWorkDir, 'ir_to_rgb_registration.json');
+    const uvPath = npath.join(jobWorkDir, 'uv_to_rgb_registration.json');
+    expect(args).toStrictEqual({
+      'warp2:transformation_file': uvPath,
+      'warp2:transform_reader:type': 'dive',
+      'warp2:transform_reader:dive:from_camera': 'uv',
+      'warp2:transform_reader:dive:to_camera': 'rgb',
+      'warp3:transformation_file': irPath,
+      'warp3:transform_reader:type': 'dive',
+      'warp3:transform_reader:dive:from_camera': 'ir',
+      'warp3:transform_reader:dive:to_camera': 'rgb',
+    });
+    const written = await fs.readJSON(irPath);
+    expect(written.type).toBe('dive-camera-registration');
+    expect(written.pairs).toHaveLength(1);
+    expect(written.pairs[0].left).toBe('ir');
+    expect(written.pairs[0].right).toBe('rgb');
+    expect(written.pairs[0].leftToRight).toStrictEqual(irToRgb);
+    expect((await fs.readJSON(uvPath)).pairs[0].left).toBe('uv');
+  });
+
+  it('skips cameras without a fitted reference pair; non-star pairs never reach the job', async () => {
+    const meta = multiCamMeta('partial', ['rgb', 'ir', 'uv'], 'rgb');
+    const jobWorkDir = '/home/user/job/partial';
+    await fs.ensureDir(jobWorkDir);
+    const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir);
+    // Pipeline order is rgb, uv, ir: uv (warp2) has only the unsupported
+    // uv-to-ir pair, so it gets nothing; ir (warp3) has a reference pair.
+    expect(Object.keys(args).some((key) => key.startsWith('warp2'))).toBe(false);
+    expect(args['warp3:transform_reader:dive:from_camera']).toBe('ir');
+    const irPath = npath.join(jobWorkDir, 'ir_to_rgb_registration.json');
+    expect(args['warp3:transformation_file']).toBe(irPath);
+    // The uv-to-ir pair is dropped from ir's job file too.
+    const written = await fs.readJSON(irPath);
+    expect(written.pairs).toHaveLength(1);
+    expect(written.pairs[0].right).toBe('rgb');
+    expect(await fs.readdir(jobWorkDir)).toStrictEqual(['ir_to_rgb_registration.json']);
+  });
+
+  it('returns no args when the dataset has no registration', async () => {
+    const meta = multiCamMeta('noreg', ['rgb', 'ir'], 'rgb');
+    const jobWorkDir = '/home/user/job/noreg';
+    await fs.ensureDir(jobWorkDir);
+    const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir);
+    expect(args).toStrictEqual({});
+    expect(await fs.readdir(jobWorkDir)).toStrictEqual([]);
+  });
+
+  it('falls back to the import-time seed in dataset meta when no files exist', async () => {
+    const meta = multiCamMeta('seeded', ['rgb', 'ir'], 'rgb');
+    meta.cameraHomographies = {
+      'ir::rgb': { AtoB: irToRgb, BtoA: identity },
+    };
+    const jobWorkDir = '/home/user/job/seeded';
+    await fs.ensureDir(jobWorkDir);
+    const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir);
+    expect(args['warp2:transform_reader:dive:from_camera']).toBe('ir');
+    expect(args['warp2:transform_reader:dive:to_camera']).toBe('rgb');
+  });
+});
diff --git a/client/platform/desktop/backend/native/cameraRegistration.ts b/client/platform/desktop/backend/native/cameraRegistration.ts
@@ -12,12 +12,16 @@ import fs from 'fs-extra';
 import { TransformType, DEFAULT_TRANSFORM_TYPE } from 'vue-media-annotator/alignedView/transform';
 import {
   buildPerCameraRegistrationFiles, registrationValuesSummary, filterRegistrationValues,
-  mergeRegistrationValues, mergeRegistrationSources, CameraRegistrationValues,
+  mergeRegistrationValues, mergeRegistrationSources, registrationFileName,
+  CameraRegistrationValues,
 } from 'vue-media-annotator/alignedView/cameraRegistrationFiles';
 import { readTransformMatrix } from 'vue-media-annotator/alignedView/alignedView';
 import { invert3, Matrix3 } from 'vue-media-annotator/alignedView/homography';
 import { DatasetMetaMutable } from 'dive-common/apispec';
-import { referenceCameraName as multicamReferenceCameraName } from 'dive-common/multicamDisplay';
+import {
+  referenceCameraName as multicamReferenceCameraName,
+  pipelineOrderedCameraNames,
+} from 'dive-common/multicamDisplay';
 import {
   RegistrationFileNamePattern,
   compareRegistrationCandidates,
@@ -192,6 +196,73 @@ export async function loadEffectiveRegistration(
   };
 }
 
+/**
+ * Build the kwiver -s settings that hand a dataset's camera registration to
+ * a 2-cam/3-cam pipeline. One standard <camera>_to_<reference>_registration
+ * file per non-reference camera is written into the job work dir; each
+ * camera's warp process (warp2, warp3, ... matching the pipeline's
+ * reference-first camera order) receives its own single-pair file. The pair
+ * and direction are still pinned through the reader's from_camera/to_camera
+ * config, since a pair may be stored in either orientation.
+ *
+ * Only pairs registering a camera directly onto the reference are
+ * supported: pairs between two non-reference cameras are explicitly
+ * unsupported here (there is no transform composition) and never reach the
+ * pipeline. Cameras without a fitted reference pair get no settings; a
+ * pipeline that needs one then fails at configure time with the file name
+ * it was missing.
+ */
+export async function buildRegistrationPipelineArgs(
+  settings: Settings,
+  meta: JsonMeta,
+  jobWorkDir: string,
+): Promise<Record<string, string>> {
+  const args: Record<string, string> = {};
+  if (!meta.multiCam) {
+    return args;
+  }
+  const projectDirInfo = await getValidatedProjectDir(settings, meta.id);
+  const values = await loadEffectiveRegistration(projectDirInfo.basePath, meta);
+  const reference = multicamReferenceCameraName(meta.multiCam);
+  if (!reference) {
+    return args;
+  }
+  const files = buildPerCameraRegistrationFiles(values, reference);
+  const writes: Promise<void>[] = [];
+  pipelineOrderedCameraNames(meta.multiCam).forEach((camera, index) => {
+    if (index === 0 || camera === reference) {
+      return;
+    }
+    // Points-only pairs have no matrix the warp could apply.
+    const fitted = values.homographies[`${camera}::${reference}`]
+      || values.homographies[`${reference}::${camera}`];
+    if (!fitted) {
+      return;
+    }
+    const file = files.find((candidate) => candidate.camera === camera);
+    if (!file) {
+      return;
+    }
+    // Unsupported non-reference pairs are dropped from the file body too, so
+    // the job dir only ever holds camera-to-reference registrations.
+    const referencePairs = file.body.pairs.filter(
+      (pair) => pair.left === reference || pair.right === reference,
+    );
+    if (!referencePairs.length) {
+      return;
+    }
+    const registrationPath = npath.join(jobWorkDir, registrationFileName(camera, reference));
+    writes.push(writeJsonFile(registrationPath, { ...file.body, pairs: referencePairs }));
+    const warp = `warp${index + 1}`;
+    args[`${warp}:transformation_file`] = registrationPath;
+    args[`${warp}:transform_reader:type`] = 'dive';
+    args[`${warp}:transform_reader:dive:from_camera`] = camera;
+    args[`${warp}:transform_reader:dive:to_camera`] = reference;
+  });
+  await Promise.all(writes);
+  return args;
+}
+
 /**
  * Persist camera registration to standalone per-camera files in a dataset
  * directory, merging partial updates with whatever is already on disk.

diff --git a/client/platform/desktop/backend/native/multiCamUtils.ts b/client/platform/desktop/backend/native/multiCamUtils.ts
@@ -11,6 +11,7 @@ import { JsonMeta, Settings } from 'platform/desktop/constants';
 import { loadAnnotationFile, loadJsonMetadata, getValidatedProjectDir } from 'platform/desktop/backend/native/common';
 import { serialize } from 'platform/desktop/backend/serializers/viame';
 import { parseFrameTimestamp } from 'dive-common/frameTimestamp';
+import { pipelineOrderedCameraNames } from 'dive-common/multicamDisplay';
 
 /**
  * Figure out the destination location
@@ -76,11 +77,25 @@ function getTranscodedMultiCamType(imageListFile: string, jsonMeta: JsonMeta) {
   throw new Error(`No associate type for ${imageListFile} in multiCam data`);
 }
 
-async function writeMultiCamStereoPipelineArgs(jobWorkDir: string, meta: JsonMeta, settings: Settings, utility = false, forceTranscoded = false) {
+async function writeMultiCamStereoPipelineArgs(
+  jobWorkDir: string,
+  meta: JsonMeta,
+  settings: Settings,
+  utility = false,
+  forceTranscoded = false,
+  // 2-cam/3-cam pipes treat camera 1 as the reference frame the other
+  // cameras register onto, so their inputs go reference-first; stereo
+  // measurement keeps the stored left/right order.
+  referenceFirst = false,
+) {
   const argFilePair: Record<string, string> = {};
   const outFiles: Record<string, string> = {};
   if (meta.multiCam && meta.multiCam.cameras) {
-    const cameraList = Object.entries(meta.multiCam.cameras);
+    const { cameras } = meta.multiCam;
+    const cameraNames = referenceFirst
+      ? pipelineOrderedCameraNames(meta.multiCam).filter((name) => name in cameras)
+      : Object.keys(cameras);
+    const cameraList = cameraNames.map((name) => [name, cameras[name]] as const);
     for (let i = 0; i < cameraList.length; i += 1) {
       const [key, list] = cameraList[i];
       const { originalBasePath } = list;
@@ -152,6 +167,7 @@ function getMultiCamUrls(
     }
     const multiCamMedia: MultiCamMedia = {
       cameras: {},
+      cameraOrder: projectMetaData.multiCam.cameraOrder,
       defaultDisplay: projectMetaData.multiCam.defaultDisplay,
     };
 

diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts
@@ -23,6 +23,7 @@ import {
 } from 'dive-common/constants';
 import * as common from './common';
 import { jobFileEchoMiddleware, createWorkingDirectory, createCustomWorkingDirectory } from './utils';
+import { buildRegistrationPipelineArgs } from './cameraRegistration';
 import {
   getMultiCamImageFiles, getMultiCamVideoPath,
   writeMultiCamStereoPipelineArgs,
@@ -304,8 +305,9 @@ async function runPipeline(
 
   let multiOutFiles: Record<string, string>;
   if (meta.multiCam && stereoOrMultiCam) {
+    const isMultiCamPipeline = multiCamPipelineMarkers.includes(pipeline.type);
     // eslint-disable-next-line max-len
-    const { argFilePair, outFiles } = await writeMultiCamStereoPipelineArgs(jobWorkDir, meta, settings, requiresInput);
+    const { argFilePair, outFiles } = await writeMultiCamStereoPipelineArgs(jobWorkDir, meta, settings, requiresInput, false, isMultiCamPipeline);
     Object.entries(argFilePair).forEach(([arg, file]) => {
       command.push(`-s ${arg}="${file}"`);
     });
@@ -327,6 +329,14 @@ async function runPipeline(
       command.push(`-s measurer:calibration_file="${meta.multiCam.calibration}"`);
       command.push(`-s calibration_reader:file="${meta.multiCam.calibration}"`);
     }
+    if (isMultiCamPipeline) {
+      // Hand the camera registration (Aligned View homographies) to the
+      // pipeline's per-camera warp processes.
+      const registrationArgs = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir);
+      Object.entries(registrationArgs).forEach(([arg, value]) => {
+        command.push(`-s ${arg}="${value}"`);
+      });
+    }
   } else if (pipeline.type === stereoPipelineMarker) {
     throw new Error('Attempting to run a multicam pipeline on non multicam data');
   }