diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index e1fd84a66..2ce9256b5 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -8,6 +8,9 @@ import { Attribute } from 'vue-media-annotator/use/AttributeTypes'; import { CustomStyle } from 'vue-media-annotator/StyleManager'; import { AttributeTrackFilter } from 'vue-media-annotator/AttributeTrackFilterControls'; import { ImageEnhancements } from 'vue-media-annotator/use/useImageEnhancements'; +import { + CameraHomographies, CameraCorrespondences, CameraTransformTypes, RegistrationSource, +} from 'vue-media-annotator/alignedView/CameraRegistrationStore'; import type { PercentileStretch } from 'vue-media-annotator/use/useImageEnhancements'; type DatasetType = 'image-sequence' | 'video' | 'multi' | 'large-image'; @@ -136,6 +139,12 @@ export interface MultiCamImportFolderArgs { sourceList: Record; // path/track file per camera @@ -186,9 +195,14 @@ interface DatasetMetaMutable { attributes?: Readonly>; attributeTrackFilters?: Readonly>; datasetInfo?: Record; + cameraHomographies?: CameraHomographies; + cameraCorrespondences?: CameraCorrespondences; + cameraTransformTypes?: CameraTransformTypes; + /** Producer provenance of the camera registration (see RegistrationSource). */ + cameraRegistrationSource?: RegistrationSource | null; error?: string; } -const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo']; +const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes', 'cameraRegistrationSource']; interface DatasetMeta extends DatasetMetaMutable { id: Readonly; @@ -278,7 +292,7 @@ interface Api { saveAttributeTrackFilters(datasetId: string, args: SaveAttributeTrackFilterArgs): Promise; // Non-Endpoint shared functions - openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip', directory?: boolean): + openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip' | 'transform', directory?: boolean): Promise<{canceled?: boolean; filePaths: string[]; fileList?: File[]; root?: string}>; /** Desktop: immediate child directory names under a parent folder (multicam subfolder import). */ listImmediateSubfolders?(parentPath: string): Promise; @@ -294,10 +308,18 @@ interface Api { ): Promise; /** Desktop: stereoscopic calibration file in a parent folder root. */ findParentFolderCalibrationFile?(parentPath: string): Promise; + /** + * Desktop: every DIVE camera-calibration .json (alignment transforms) in a + * parent folder root: per-camera *_registration.json files first, then + * other self-identified candidates. + */ + findParentFolderTransformFiles?(parentPath: string): Promise; /** True when the dataset folder has an attached stereoscopic calibration file. */ hasCalibrationFile?(datasetId: string): Promise; /** Web: stash a calibration File for multicam upload lookup. */ stashCalibrationFile?(key: string, file: File): void; + /** Web: stash a per-camera registration transform File for multicam upload lookup. */ + stashTransformFile?(key: string, file: File): void; getTiles?(itemId: string, projection?: string): Promise; // eslint-disable-next-line @typescript-eslint/no-explicit-any getTileURL?(itemId: string, x: number, y: number, level: number, query: Record): @@ -315,6 +337,16 @@ interface Api { saveCalibration?(path: string): Promise<{ savedPath: string; updatedDatasetIds: string[] }>; /** Desktop: set the stereo camera/calibration file for a single dataset. */ importCalibrationFile?(datasetId: string, path: string): Promise<{ calibration: string }>; + /** + * Merge a DIVE registration .json into an existing multicam dataset's + * saved camera registration. Web reads the provided File; desktop reads + * the path. options.camera keeps only the file's pairs naming that + * camera, replacing that camera's current pairs while other cameras' + * pairs are kept. + */ + importCameraRegistration?(datasetId: string, path: string, file?: File, + options?: { camera?: string }): + Promise<{ cameras: string[]; pairCount: number }>; /** Desktop: copy the dataset's current camera/calibration file out to destPath. */ exportCalibrationFile?(datasetId: string, destPath: string): Promise<{ exportedPath: string }>; /** Download/export the dataset's current calibration file (platform-specific). */ diff --git a/client/dive-common/components/ImportAnnotations.vue b/client/dive-common/components/ImportAnnotations.vue index ec67c4f9c..2694ef839 100644 --- a/client/dive-common/components/ImportAnnotations.vue +++ b/client/dive-common/components/ImportAnnotations.vue @@ -6,11 +6,14 @@ import { useApi } from 'dive-common/apispec'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import { clientSettings } from 'dive-common/store/settings'; import clearLengthAttributes from 'dive-common/utils/clearLengthAttributes'; +import warpAnnotationsAcrossCameras from 'dive-common/utils/warpAnnotationsAcrossCameras'; import { cloneDeep } from 'lodash'; import { useAnnotationSets, useAnnotationSet, useHandler, useCameraStore, useSelectedCamera, + useAlignedView, useCameraRegistration, } from 'vue-media-annotator/provides'; import { getResponseError } from 'vue-media-annotator/utils'; +import { parentDatasetId } from 'dive-common/compositeDatasetId'; export default defineComponent({ name: 'ImportAnnotations', @@ -52,7 +55,20 @@ export default defineComponent({ const { reloadAnnotations, save } = useHandler(); const cameraStore = useCameraStore(); const selectedCamera = useSelectedCamera(); + const alignedView = useAlignedView(); const isMulticamDataset = computed(() => cameraStore.camMap.value.size > 1); + // Warping detections onto other cameras requires the whole rig to be + // registered (a native->reference transform for every camera). + const canWarpToAllCameras = computed( + () => isMulticamDataset.value && alignedView.available.value, + ); + const warpToAllCameras = ref(false); + const warpToAllCamerasHint = computed(() => { + const progress = alignedView.registrationProgress.value; + return progress + ? `${progress.registered}/${progress.total} cameras registered` + : ''; + }); const activeCameraName = computed(() => { if (!isMulticamDataset.value) { return null; @@ -68,6 +84,35 @@ export default defineComponent({ const cameraFileSupported = computed( () => !!api.importCalibrationFile && props.subType === 'stereo', ); + // Camera registration import (per-camera transform files) is meaningful + // for any multicam dataset. + const cameraRegistration = useCameraRegistration(); + const registrationSupported = computed( + () => !!api.importCameraRegistration && isMulticamDataset.value, + ); + // One import button per non-reference camera pair, labeled in the + // direction of the mapping -- "Import ir → eo" registers ir onto the + // reference camera (the import dialog's Reference Camera choice, + // published by the viewer), matching the + // _to__registration.json file names -- and colored by + // whether that camera already has a registration (importing onto an + // existing one replaces it, after confirmation). + const registrationImportTargets = computed(() => { + if (!registrationSupported.value) { + return []; + } + const pairKeys = [ + ...Object.keys(cameraRegistration.homographies.value), + ...Object.keys(cameraRegistration.correspondences.value), + ]; + const cams = [...cameraStore.camMap.value.keys()]; + const reference = alignedView.reference.value ?? cams[0]; + return cams.filter((camera) => camera !== reference).map((camera) => ({ + camera, + label: `Import ${camera} → ${reference}`, + registered: pairKeys.some((key) => key.split('::').includes(camera)), + })); + }); const currentCalibrationName = computed(() => { if (!props.calibrationFile) return ''; return props.calibrationFile.replace(/^.*[\\/]/, ''); @@ -141,9 +186,24 @@ export default defineComponent({ } if (importFile) { - processing.value = false; await reloadAnnotations(); + if ( + warpToAllCameras.value + && canWarpToAllCameras.value + && activeCameraName.value + && alignedView.toReference.value + ) { + const warped = warpAnnotationsAcrossCameras( + cameraStore, + alignedView.toReference.value, + activeCameraName.value, + ); + if (warped.tracks > 0) { + await save(); + } + } } + processing.value = false; } } catch (error) { const text = [getResponseError(error)]; @@ -182,6 +242,61 @@ export default defineComponent({ }); } }; + const openRegistrationUpload = async (camera: string) => { + if (!api.importCameraRegistration) return; + const target = registrationImportTargets.value.find((entry) => entry.camera === camera); + if (target?.registered) { + const confirmed = await prompt({ + title: 'Replace Registration?', + text: `Camera "${camera}" already has a registration. Importing will replace it.`, + positiveButton: 'Replace', + negativeButton: 'Cancel', + confirm: true, + }); + if (!confirmed) return; + } + try { + const ret = await openFromDisk('transform'); + if (ret.canceled || !ret.filePaths.length) return; + menuOpen.value = false; + processing.value = true; + const result = await api.importCameraRegistration( + props.datasetId, + ret.filePaths[0], + ret.fileList?.[0], + { camera }, + ); + // Rehydrate the store from the freshly persisted meta so the Align + // View and mirroring pick up the new transforms immediately. + const meta = await api.loadMetadata(parentDatasetId(props.datasetId)); + cameraRegistration.hydrate( + meta.cameraHomographies, + meta.cameraCorrespondences, + meta.cameraTransformTypes, + meta.cameraRegistrationSource, + ); + processing.value = false; + const unknown = result.cameras.filter((name) => !cameraStore.camMap.value.has(name)); + if (unknown.length) { + await prompt({ + title: 'Registration Imported', + text: [ + `Imported ${result.pairCount} pair(s), but the file names camera(s) not in this dataset:`, + unknown.join(', '), + 'Pair bodies name their own cameras, so these pairs will not resolve until matching cameras exist.', + ], + positiveButton: 'OK', + }); + } + } catch (error) { + processing.value = false; + prompt({ + title: 'Registration Import Failed', + text: [getResponseError(error)], + positiveButton: 'OK', + }); + } + }; const applyLastCalibration = async () => { if (!api.importCalibrationFile || !lastCalibrationPath.value) return; try { @@ -211,10 +326,13 @@ export default defineComponent({ return { openUpload, openCalibrationUpload, + openRegistrationUpload, applyLastCalibration, showLastCalibrationSuggestion, lastCalibrationFileName, cameraFileSupported, + registrationSupported, + registrationImportTargets, currentCalibrationName, processing, menuOpen, @@ -225,6 +343,9 @@ export default defineComponent({ currentSet, isMulticamDataset, activeCameraName, + canWarpToAllCameras, + warpToAllCameras, + warpToAllCamerasHint, }; }, }); @@ -260,7 +381,7 @@ export default defineComponent({ - Import Annotation Data + Import Supplementary Data - Export Annotation Data + Export Supplementary Data + + + Zip all cameras: media, annotations, calibration, and dataset metadata diff --git a/client/platform/web-girder/views/Upload.vue b/client/platform/web-girder/views/Upload.vue index 01bbe0e15..f70e2a066 100644 --- a/client/platform/web-girder/views/Upload.vue +++ b/client/platform/web-girder/views/Upload.vue @@ -25,6 +25,7 @@ import { createGirderFolder, createMulticamDataset, deleteResources, + saveMetadata, uploadCalibrationItem, validateUploadGroup, waitForFolderDatasetReady, @@ -34,11 +35,14 @@ import { getAnnotationFile, getCalibrationFile, getFilesForSourceKey, + getTransformFile, flattenUploadFiles, removeCameraFolderFiles, renameCameraFolderFiles, stashCameraFolderFiles, + stashTransformFile, } from 'platform/web-girder/multicamFileRegistry'; +import { parseRegistrationSeed } from 'platform/web-girder/multicamRegistrationSeed'; import { isAllowedStereoCalibrationFilename, stereoCalibrationAllowedExtensionsLabel, @@ -379,17 +383,20 @@ export default defineComponent({ closeUpload?: boolean; showProgressOverlay?: boolean; progressLabel?: string; + /** Prompt registration warnings inline; batch collects surface them per row instead. */ + promptRegistrationWarnings?: boolean; } const runMultiCamFolderImport = async ( args: MultiCamImportFolderArgs, options: MultiCamFolderImportOptions = {}, - ): Promise => { + ): Promise<{ id: string; registrationWarnings: string[] }> => { const { openViewer = true, closeUpload = true, showProgressOverlay = true, progressLabel = '', + promptRegistrationWarnings = true, } = options; const labelPrefix = progressLabel ? `${progressLabel} — ` : ''; if (!props.location?._id || props.location._modelType !== 'folder') { @@ -412,6 +419,18 @@ export default defineComponent({ if (!datasetName) { throw new Error('Dataset name is required'); } + // Parse attached transform/registration files up front so a bad file + // fails the import before anything is created (desktop parity). + const transformEntries = Object.entries(args.sourceList) + .filter(([, source]) => source.transformFile) + .map(([cameraName, source]) => ({ + cameraName, + fileName: source.transformFile as string, + file: getTransformFile(source.transformFile as string), + })); + const registrationSeed = transformEntries.length + ? await parseRegistrationSeed(transformEntries, Object.keys(args.sourceList)) + : null; const fps = args.type === VideoType ? DefaultVideoFPS : (clientSettings.annotationFPS || 1); @@ -526,6 +545,28 @@ export default defineComponent({ }); multicamLinked = true; + if (registrationSeed?.values) { + // Seed the dataset's saved camera registration (the same + // registration the in-app panel edits and the Align button + // applies); the camera* fields are allowlisted in the meta PATCH. + setMulticamImportProgress(98, `${labelPrefix}Saving camera registration…`); + await saveMetadata(parentFolder._id, { + cameraHomographies: registrationSeed.values.homographies, + cameraCorrespondences: registrationSeed.values.correspondences, + cameraTransformTypes: registrationSeed.values.transformTypes, + ...(registrationSeed.values.source + ? { cameraRegistrationSource: registrationSeed.values.source } + : {}), + }); + } + if (promptRegistrationWarnings && registrationSeed?.warnings.length) { + await prompt({ + title: 'Registration Warnings', + text: registrationSeed.warnings, + positiveButton: 'OK', + }); + } + if (openViewer) { setMulticamImportProgress(100, `${labelPrefix}Opening viewer…`); clearMulticamFileRegistry(); @@ -534,7 +575,10 @@ export default defineComponent({ close(); } } - return parentFolder._id; + return { + id: parentFolder._id, + registrationWarnings: registrationSeed?.warnings ?? [], + }; } catch (err) { if (datasetFolderId && !multicamLinked) { try { @@ -583,7 +627,7 @@ export default defineComponent({ const importBatchCollect = async (collect: MultiCamBatchCollect, datasetName: string) => { if (!collect.importArgs) { - return; + return undefined; } clearMulticamFileRegistry(); collect.cameras.forEach((camera) => { @@ -592,16 +636,31 @@ export default defineComponent({ filesForCameraSource(camera.sourcePath, batchImportFiles.value), ); }); - await runMultiCamFolderImport( + // Re-stash the collect's registration Files by their scan-time paths so + // the seeding lookup finds them (the registry is cleared per collect). + Object.values(collect.importArgs.sourceList).forEach((source) => { + if (!source.transformFile) { + return; + } + const file = batchImportFiles.value.find( + (f) => (f.webkitRelativePath || f.name).replace(/\\/g, '/') === source.transformFile, + ); + if (file) { + stashTransformFile(source.transformFile, file); + } + }); + const { registrationWarnings } = await runMultiCamFolderImport( { ...collect.importArgs, datasetName }, { openViewer: false, closeUpload: false, showProgressOverlay: false, progressLabel: collect.name, + promptRegistrationWarnings: false, }, ); clearMulticamFileRegistry(); + return registrationWarnings; }; // Filter to show how many files are left to upload const filesNotUploaded = (item: PendingUpload) => item.files.filter( @@ -760,6 +819,7 @@ export default defineComponent({ v-else-if="importMultiCamDialog" :stereo="stereo" :data-type="multiCamOpenType" + :enable-transform-import="true" :enable-subfolder-import="true" :register-subfolder-cameras="registerSubfolderCameras" :unregister-subfolder-camera="unregisterSubfolderCamera" diff --git a/client/src/alignedView/AlignedViewStore.ts b/client/src/alignedView/AlignedViewStore.ts new file mode 100644 index 000000000..0053e0ab2 --- /dev/null +++ b/client/src/alignedView/AlignedViewStore.ts @@ -0,0 +1,130 @@ +import { + computed, ref, ComputedRef, Ref, +} from 'vue'; +import type { Matrix3, Point } from './homography'; +import { + cameraPairTransform, isIdentityMatrix3, mapPoint, +} from './alignedView'; + +/** + * Shared reactive state for the multicam "aligned view" toggle: when every + * non-reference camera has a stored/fitted transform into the reference + * camera's space, the user may warp each camera's display into that shared + * space during normal annotation review and link pan/zoom across all cameras + * (SEAL-TK features 2 + 3). + * + * Stored annotation geometry ALWAYS remains in native image space (decision + * D3); the transforms exposed here are applied at draw time only. + */ +export default class AlignedViewStore { + /** User toggle: whether the aligned view is requested. */ + enabled: Ref; + + /** + * Reference camera name that registration maps onto (the import dialog's + * Reference Camera choice, falling back to first in display order). Set + * whenever the dataset is multicam -- even before the rig resolves -- so + * UI can name the reference; null for single-camera datasets. + */ + reference: Ref; + + /** + * Per-camera native->reference matrices covering EVERY loaded camera, or + * null when at least one camera lacks a usable transform (see + * {@link resolveToReferenceTransforms}). + */ + toReference: Ref | null>; + + /** + * Externally suspended (e.g. while registration point picking is active, + * which records raw native-space clicks and manages its own aligned + * preview). Suspension un-warps the display without losing the toggle. + */ + suspended: Ref; + + /** + * How much of the rig resolves: cameras with a usable transform into the + * reference space, out of all loaded cameras. Maintained by the viewer + * alongside {@link setTransforms}; null for single-camera datasets. Lets + * UI outside the viewer core (e.g. the import menu) show the same + * "N/M cameras registered" status as the Align View toggle without + * re-deriving the pair graph. + */ + registrationProgress: Ref<{ registered: number; total: number } | null>; + + /** A usable transform exists for every camera, so the toggle may be shown. */ + available: ComputedRef; + + /** The aligned view is currently applied to rendering and navigation. */ + active: ComputedRef; + + constructor() { + this.enabled = ref(false); + this.reference = ref(null); + this.toReference = ref(null); + this.suspended = ref(false); + this.registrationProgress = ref(null); + this.available = computed(() => this.reference.value !== null + && this.toReference.value !== null + && Object.keys(this.toReference.value).length > 1); + this.active = computed(() => this.enabled.value + && !this.suspended.value + && this.available.value); + } + + /** Replace the resolved transform set (null when unavailable). */ + setTransforms(reference: string | null, toReference: Record | null) { + this.reference.value = reference; + this.toReference.value = toReference; + } + + setRegistrationProgress(progress: { registered: number; total: number } | null) { + this.registrationProgress.value = progress; + } + + setEnabled(enabled: boolean) { + this.enabled.value = enabled; + } + + setSuspended(suspended: boolean) { + this.suspended.value = suspended; + } + + /** + * Display transform for `camera` (native -> aligned/reference space), or + * null when the aligned view is inactive or the camera renders unwarped + * (the reference camera / identity). Callers treat null as "draw exactly + * as today", keeping the off state byte-identical to current behavior. + */ + cameraTransform(camera: string): Matrix3 | null { + if (!this.active.value || !this.toReference.value) { + return null; + } + const matrix = this.toReference.value[camera]; + if (!matrix || isIdentityMatrix3(matrix)) { + return null; + } + return matrix; + } + + /** + * Matrix mapping `from`-camera native pixels onto `to`-camera native + * pixels, composed through the reference space. Null when inactive or + * either camera is unresolved. + */ + cameraToCamera(from: string, to: string): Matrix3 | null { + if (!this.active.value || !this.toReference.value) { + return null; + } + return cameraPairTransform(this.toReference.value, from, to); + } + + /** Map a native-space point of `from` onto `to`'s native space (null when unavailable). */ + mapCameraPoint(from: string, to: string, point: Point): Point | null { + const matrix = this.cameraToCamera(from, to); + if (!matrix) { + return null; + } + return mapPoint(matrix, point); + } +} diff --git a/client/src/alignedView/CameraRegistrationStore.spec.ts b/client/src/alignedView/CameraRegistrationStore.spec.ts new file mode 100644 index 000000000..8b3e52834 --- /dev/null +++ b/client/src/alignedView/CameraRegistrationStore.spec.ts @@ -0,0 +1,339 @@ +/// +import CameraRegistrationStore from './CameraRegistrationStore'; + +describe('CameraRegistrationStore', () => { + // Pure translation by (+5, -3): trivially invertible. + const translate = [[1, 0, 5], [0, 1, -3], [0, 0, 1]]; + // Four correspondence rows consistent with `translate`: right = left + (5, -3). + const translationPointRows = [ + [0, 0, 5, -3], + [10, 0, 15, -3], + [10, 10, 15, 7], + [0, 10, 5, 7], + ]; + + it('produces a directional pairKey that preserves left/right order', () => { + const store = new CameraRegistrationStore(); + expect(store.pairKey('rgb', 'ir')).toEqual('rgb::ir'); + expect(store.pairKey('rgb', 'ir')).not.toEqual(store.pairKey('ir', 'rgb')); + }); + + it('hydrates homographies and resets prior calibration state', () => { + const store = new CameraRegistrationStore(); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: [[1, 1, 6, -2]], transformType: 'translation', + }], + })); + const saved = { 'a::b': { AtoB: [[1, 0, 0], [0, 1, 0], [0, 0, 1]], BtoA: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] } }; + store.hydrate(saved); + expect(store.homographies.value).toEqual(saved); + expect(store.correspondences.value).toEqual({}); + expect(store.transformTypes.value).toEqual({}); + }); + + it('hydrates transform types alongside homographies', () => { + const store = new CameraRegistrationStore(); + store.hydrate({}, {}, { 'a::b': 'rigid' }); + expect(store.transformTypeForPair('a::b')).toBe('rigid'); + expect(store.transformTypeForPair('unset::pair')).toBe('similarity'); + }); + + it('hydrates correspondences and resumes id allocation', () => { + const store = new CameraRegistrationStore(); + const correspondences = { + 'rgb::ir': [ + { id: 1, a: [1, 2], b: [3, 4] }, + { id: 2, a: [5, 6], b: [7, 8] }, + ], + }; + store.hydrate({}, correspondences); + expect(store.correspondences.value).toEqual(correspondences); + // Points loaded afterwards pick up ids after the highest restored id. + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ left: 'rgb', right: 'ir', points: [[9, 9, 10, 10]] }], + })); + expect(store.correspondences.value['rgb::ir'][0].id).toBe(3); + }); + + describe('dirty / markSaved', () => { + it('tracks unsaved changes and resets on markSaved and hydrate', () => { + const store = new CameraRegistrationStore(); + expect(store.dirty.value).toBe(false); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: translate, + }], + })); + expect(store.dirty.value).toBe(true); + store.markSaved(); + expect(store.dirty.value).toBe(false); + store.hydrate({ 'a::b': { AtoB: translate, BtoA: translate } }); + // Freshly hydrated state is the saved baseline. + expect(store.dirty.value).toBe(false); + }); + }); + + describe('sourceIsMixed', () => { + it('flags only the mixed composite stamp the file merger produces', () => { + const store = new CameraRegistrationStore(); + expect(store.sourceIsMixed()).toBe(false); + store.hydrate(undefined, undefined, undefined, { producer: 'kamera', run: 'fl07' }); + expect(store.sourceIsMixed()).toBe(false); + store.hydrate(undefined, undefined, undefined, { + mixed: true, + files: { 'calibration_ir.json': { run: 'fl07' }, 'calibration_uv.json': { run: 'fl09' } }, + }); + expect(store.sourceIsMixed()).toBe(true); + }); + }); + + describe('loaded (file-sourced) homographies', () => { + /** Load a matrix-only (point-less) pair from calibration JSON, marking it 'loaded'. */ + function loadMatrixOnlyPair(store: CameraRegistrationStore, left: string, right: string, rightToLeft: number[][]) { + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left, right, points: [], leftToRight: null, rightToLeft, + }], + })); + } + + it('loads a matrix-only pair as B->A with its inverse as A->B and no points', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + loadMatrixOnlyPair(store, 'left', 'right', translate); + expect(store.correspondences.value[key]).toHaveLength(0); + expect(store.isLoadedHomography(key)).toBe(true); + const homog = store.homographies.value[key]; + expect(homog.BtoA).toEqual(translate); + expect(homog.AtoB[0][2]).toBeCloseTo(-5); + expect(homog.AtoB[1][2]).toBeCloseTo(3); + }); + + it('rejects a singular loaded matrix', () => { + const store = new CameraRegistrationStore(); + expect(() => loadMatrixOnlyPair(store, 'left', 'right', [[0, 0, 0], [0, 0, 0], [0, 0, 0]])) + .toThrow(/singular/); + }); + + it('hydrate marks an under-pointed homography as loaded', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + store.hydrate({ [key]: { AtoB: translate, BtoA: translate } }, {}, {}); + expect(store.isLoadedHomography(key)).toBe(true); + }); + }); + + describe('calibration JSON file round trip', () => { + it('serializes and reloads all pairs', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', + right: 'right', + points: translationPointRows, + leftToRight: translate, + rightToLeft: null, + transformType: 'translation', + }], + })); + const json = store.toRegistrationJson(); + + const restored = new CameraRegistrationStore(); + const result = restored.loadRegistrationText(json); + expect(result.pairCount).toBe(1); + expect(result.cameras.sort()).toEqual(['left', 'right']); + expect(restored.correspondences.value[key]).toHaveLength(4); + expect(restored.transformTypeForPair(key)).toBe('translation'); + expect(restored.homographies.value[key].AtoB[0][2]).toBeCloseTo(5); + // The missing direction was derived by inversion and round-tripped. + expect(restored.homographies.value[key].BtoA[0][2]).toBeCloseTo(-5); + // Enough points back the homography, so it is treated as fitted. + expect(restored.isLoadedHomography(key)).toBe(false); + }); + + it('includes pairs that only have a loaded homography (no points)', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('uv', 'ir'); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'uv', right: 'ir', points: [], leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], rightToLeft: null, + }], + })); + const restored = new CameraRegistrationStore(); + restored.loadRegistrationText(store.toRegistrationJson()); + expect(restored.homographies.value[key]).toBeDefined(); + expect(restored.isLoadedHomography(key)).toBe(true); + }); + + it('loads a desktop-persisted calibration.json (no "type" field, one direction only)', () => { + const store = new CameraRegistrationStore(); + const result = store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'eo', + right: 'ir', + points: [[0, 0, 5, -3]], + leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + rightToLeft: null, + transformType: 'translation', + }], + })); + expect(result.pairCount).toBe(1); + const key = store.pairKey('eo', 'ir'); + expect(store.correspondences.value[key]).toHaveLength(1); + // The missing direction is derived by inversion. + expect(store.homographies.value[key].BtoA[0][2]).toBeCloseTo(-5); + expect(store.transformTypeForPair(key)).toBe('translation'); + }); + + it('preserves the producer source stamp across a load/save round trip', () => { + const store = new CameraRegistrationStore(); + const source = { model: 'colmap-2026-07-01', swathe: 'fl07_C' }; + store.loadRegistrationText(JSON.stringify({ + version: 1, + source, + pairs: [{ + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: translate, + }], + })); + expect(store.source.value).toStrictEqual(source); + const saved = JSON.parse(store.toRegistrationJson()); + expect(saved.source).toStrictEqual(source); + }); + + it('omits the source key when no stamp was loaded', () => { + const store = new CameraRegistrationStore(); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, + }], + })); + expect('source' in JSON.parse(store.toRegistrationJson())).toBe(false); + }); + + it('clears a previous stamp when loading a file without one', () => { + const store = new CameraRegistrationStore(); + store.loadRegistrationText(JSON.stringify({ + version: 1, + source: { model: 'old' }, + pairs: [], + })); + store.loadRegistrationText(JSON.stringify({ version: 1, pairs: [] })); + expect(store.source.value).toBeNull(); + }); + + it('rejects a non-object source', () => { + const store = new CameraRegistrationStore(); + expect(() => store.loadRegistrationText(JSON.stringify({ + version: 1, source: 'colmap', pairs: [], + }))).toThrow(/"source" must be an object/); + }); + + it('hydrate restores the source stamp', () => { + const store = new CameraRegistrationStore(); + store.hydrate({}, {}, {}, { model: 'colmap-x' }); + expect(store.source.value).toStrictEqual({ model: 'colmap-x' }); + store.hydrate({}, {}, {}); + expect(store.source.value).toBeNull(); + }); + + it('flags a point-backed homography as refined when a source stamp is loaded', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + // Fresh from the producer (matrix-only): loaded, not refined. + store.loadRegistrationText(JSON.stringify({ + version: 1, + source: { model: 'colmap-x' }, + pairs: [{ + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: translate, + }], + })); + expect(store.isRefinedFromSource(key)).toBe(false); + // Point-backed under a stamp: the pair has diverged from the producer. + store.loadRegistrationText(JSON.stringify({ + version: 1, + source: { model: 'colmap-x' }, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, transformType: 'translation', + }], + })); + expect(store.isRefinedFromSource(key)).toBe(true); + }); + + it('does not flag point-backed homographies as refined when no source stamp is loaded', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, transformType: 'translation', + }], + })); + expect(store.isRefinedFromSource(key)).toBe(false); + }); + + it('keeps the refined flag across a save/load round trip', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + store.loadRegistrationText(JSON.stringify({ + version: 1, + source: { model: 'colmap-x' }, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, transformType: 'translation', + }], + })); + expect(store.isRefinedFromSource(key)).toBe(true); + + const restored = new CameraRegistrationStore(); + restored.loadRegistrationText(store.toRegistrationJson()); + // The refined pair saved with its backing points, so it re-marks as + // fitted (refined) rather than loaded. + expect(restored.isRefinedFromSource(key)).toBe(true); + }); + + it('rejects non-JSON, missing pairs, malformed pairs, and bad matrices without clobbering state', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, + }], + })); + expect(() => store.loadRegistrationText('not json')).toThrow(/valid JSON/); + expect(() => store.loadRegistrationText('{"type": "other"}')).toThrow(/pairs/); + expect(() => store.loadRegistrationText(JSON.stringify({ + version: 1, pairs: [{ left: 'a', right: 'a' }], + }))).toThrow(/distinct/); + expect(() => store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'a', + right: 'b', + points: [], + leftToRight: [[1, 0], [0, 1]], + rightToLeft: null, + }], + }))).toThrow(/3x3/); + expect(() => store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ left: 'a', right: 'b', points: [[1, 2, 3]] }], + }))).toThrow(/points row/); + expect(() => store.loadRegistrationText(JSON.stringify({ + version: 1, + pairs: [{ left: 'a', right: 'b', transformType: 'bogus' }], + }))).toThrow(/transformType/); + // Failed loads left the existing calibration alone. + expect(store.correspondences.value[key]).toHaveLength(4); + }); + }); +}); diff --git a/client/src/alignedView/CameraRegistrationStore.ts b/client/src/alignedView/CameraRegistrationStore.ts new file mode 100644 index 000000000..c6f462430 --- /dev/null +++ b/client/src/alignedView/CameraRegistrationStore.ts @@ -0,0 +1,378 @@ +import { + ref, computed, Ref, ComputedRef, +} from 'vue'; +import { + invert3, Matrix3, Point, +} from './homography'; +import { + TransformType, TRANSFORM_TYPES, DEFAULT_TRANSFORM_TYPE, minPointsForTransform, +} from './transform'; + +/** + * A single picked point pair. `a` is the point in the left camera (camA), `b` + * the point in the right camera (camB). Left/right is the order the user chose, + * which is preserved (not alphabetized) so it survives round trips through the + * calibration JSON's ordered `[leftX, leftY, rightX, rightY]` rows. + */ +export interface Correspondence { + id: number; + a: Point; + b: Point; +} + +/** Both directions of the fitted alignment transform for one camera pair. */ +export interface PairHomography { + /** Maps left (camA) image coordinates onto right (camB). */ + AtoB: Matrix3; + /** Maps right (camB) image coordinates onto left (camA). */ + BtoA: Matrix3; +} + +/** Fitted transforms keyed by {@link CameraRegistrationStore.pairKey}. */ +export type CameraHomographies = Record; + +/** + * Where a pair's homography came from: fitted in-app from picked points, or + * loaded from a calibration file (which may carry no points at all). Loaded + * homographies persist through refit checks that would otherwise clear an + * under-pointed pair, until enough points are picked to fit a replacement. + */ +type HomographySource = 'fit' | 'loaded'; + +/** + * Free-form provenance stamped into the calibration file by whatever produced + * the transforms (e.g. an external COLMAP/KAMERA model step: model version, + * swathe/flight id, generation time). DIVE never interprets it -- it is + * preserved verbatim through load/refine/save round trips so an external + * re-solver can tell which model version a returning file was refined against. + */ +export type RegistrationSource = Record; + +/** + * One camera pair in the portable registration JSON file. This is the same + * self-describing shape the desktop platform persists as the project's + * standalone per-camera _to__registration.json files + * (see desktop backend/native/common.ts), so a panel-saved file, the + * on-disk artifacts, and an import-time seed are all interchangeable: + * correspondences flattened as [leftX, leftY, rightX, rightY] rows, plus + * both fitted directions (null when unfitted). + */ +export interface RegistrationFilePair { + left: string; + right: string; + points?: number[][]; + leftToRight?: Matrix3 | null; + rightToLeft?: Matrix3 | null; + transformType?: TransformType; +} + +/** Portable calibration file: everything needed to restore all pairs. */ +export interface RegistrationFile { + /** Written by panel saves for self-identification; optional on load. */ + type?: string; + version: number; + /** Producer provenance, preserved verbatim across round trips. */ + source?: RegistrationSource | null; + pairs: RegistrationFilePair[]; +} + +/** Identifying `type` value of the registration JSON format. */ +export const REGISTRATION_FILE_TYPE = 'dive-camera-registration'; + +/** Picked correspondences keyed by {@link CameraRegistrationStore.pairKey}. */ +export type CameraCorrespondences = Record; + +/** Chosen fit model per pair, keyed by {@link CameraRegistrationStore.pairKey}. Missing entries default to 'similarity'. */ +export type CameraTransformTypes = Record; + +/** + * Shared, reactive store for camera-calibration data (correspondences, + * fitted/loaded homographies, transform-type choices, and producer + * provenance). Lives in vue-media-annotator so both the annotation layers + * (client/src/alignedView) and the dive-common side can consume it via the + * provide/inject system. Handles persistence: hydrating saved state and + * loading/saving the portable calibration JSON format. + */ +export default class CameraRegistrationStore { + correspondences: Ref; + + homographies: Ref; + + transformTypes: Ref; + + /** + * Provenance of the loaded calibration (see {@link RegistrationSource}). + * Deliberately NOT cleared by in-app edits or refits -- refinements are + * exactly what should travel back to the producer stamped with the model + * lineage they were made against. Replaced (or cleared) only when a + * calibration file is loaded or the store is re-hydrated. + */ + source: Ref; + + /** True when the calibration has unsaved changes since the last save or load. */ + dirty: ComputedRef; + + private nextId: number; + + /** Provenance per homography key; missing entries behave like 'fit'. */ + private homographySources: Record; + + /** Serialized calibration at the last save/load, the baseline for {@link dirty}. */ + private savedSnapshot: Ref; + + constructor() { + this.correspondences = ref({}); + this.homographies = ref({}); + this.transformTypes = ref({}); + this.source = ref(null); + this.nextId = 1; + this.homographySources = {}; + this.savedSnapshot = ref(this.registrationSnapshot()); + this.dirty = computed(() => this.registrationSnapshot() !== this.savedSnapshot.value); + } + + /** Serialize the saved-to-dataset calibration state (points, transforms, provenance). */ + private registrationSnapshot(): string { + return JSON.stringify({ + homographies: this.homographies.value, + correspondences: this.correspondences.value, + transformTypes: this.transformTypes.value, + source: this.source.value, + }); + } + + /** Capture the current calibration as the saved baseline, so {@link dirty} reads false. */ + markSaved() { + this.savedSnapshot.value = this.registrationSnapshot(); + } + + /** + * True when the loaded calibration was assembled from per-camera files + * whose producer stamps disagree (the loader records that as a + * `{ mixed: true, files: {...} }` composite source) -- i.e. the rig may + * mix calibration generations and deserves a visible warning rather than + * silent composition. + */ + sourceIsMixed(): boolean { + return Boolean(this.source.value + && (this.source.value as Record).mixed === true); + } + + /** + * Directional key for a camera pair: `left::right`. Order is significant and + * preserved so left/right (e.g. RGB vs IR) survives for ordered exports. + */ + // eslint-disable-next-line class-methods-use-this + pairKey(camA: string, camB: string): string { + return `${camA}::${camB}`; + } + + /** + * True when `key`'s homography came from a calibration file rather than an + * in-app fit. Not independently reactive -- always read alongside + * {@link homographies} (provenance only changes when that map does). + */ + isLoadedHomography(key: string): boolean { + return this.homographySources[key] === 'loaded'; + } + + /** + * True when `key`'s transform was fitted from in-app picked points while a + * producer-stamped calibration is loaded -- i.e. the pair has diverged from + * what the stamped {@link source} shipped (producer files carry matrix-only + * pairs, so any point-backed fit is a human refinement). Derived rather + * than stored: it survives save/reload naturally, because point-backed + * pairs re-mark as fitted on hydrate. Read alongside {@link homographies}, + * same reactivity caveat as {@link isLoadedHomography}. + */ + isRefinedFromSource(key: string): boolean { + return this.source.value !== null + && this.homographies.value[key] !== undefined + && this.homographySources[key] === 'fit'; + } + + /** The chosen fit model for `key`, defaulting to {@link DEFAULT_TRANSFORM_TYPE} when unset. */ + transformTypeForPair(key: string): TransformType { + return this.transformTypes.value[key] || DEFAULT_TRANSFORM_TYPE; + } + + /** + * Serialize every pair with content (points and/or a homography) as the + * portable calibration JSON file (see {@link RegistrationFile}). Pairs whose + * only state is a transform-type choice are omitted. + */ + toRegistrationJson(): string { + const keys = new Set([ + ...Object.keys(this.correspondences.value).filter( + (key) => this.correspondences.value[key].length > 0, + ), + ...Object.keys(this.homographies.value), + ]); + const pairs: RegistrationFilePair[] = [...keys].sort().map((key) => { + const [left, right] = key.split('::'); + const homography = this.homographies.value[key] || null; + return { + left, + right, + points: (this.correspondences.value[key] || []).map((c) => [c.a[0], c.a[1], c.b[0], c.b[1]]), + leftToRight: homography ? homography.AtoB : null, + rightToLeft: homography ? homography.BtoA : null, + transformType: this.transformTypeForPair(key), + }; + }); + const file: RegistrationFile = { + type: REGISTRATION_FILE_TYPE, + version: 1, + ...(this.source.value ? { source: this.source.value } : {}), + pairs, + }; + return JSON.stringify(file, null, 2); + } + + /** + * Parse and load a calibration JSON file (the format written by + * {@link toRegistrationJson}), REPLACING all pairs' correspondences, + * homographies, and transform types. Throws a descriptive Error on + * malformed input without touching current state. Returns the camera names + * referenced by the file so callers can warn about ones missing from the + * loaded dataset. + */ + loadRegistrationText(text: string): { cameras: string[]; pairCount: number } { + let data: unknown; + try { + data = JSON.parse(text); + } catch { + throw new Error('File is not valid JSON'); + } + const file = data as Partial; + if (!Array.isArray(file?.pairs)) { + throw new Error('Not a DIVE camera registration file (expected a "pairs" list)'); + } + const source = CameraRegistrationStore.readSource(file.source); + const correspondences: CameraCorrespondences = {}; + const homographies: CameraHomographies = {}; + const transformTypes: CameraTransformTypes = {}; + const cameras = new Set(); + file.pairs.forEach((pair, i) => { + const context = `Pair ${i + 1}`; + if (typeof pair?.left !== 'string' || typeof pair?.right !== 'string' + || !pair.left || !pair.right || pair.left === pair.right) { + throw new Error(`${context}: "left" and "right" must be two distinct camera names`); + } + const key = this.pairKey(pair.left, pair.right); + cameras.add(pair.left); + cameras.add(pair.right); + if (pair.transformType !== undefined) { + if (!TRANSFORM_TYPES.some((t) => t.value === pair.transformType)) { + throw new Error( + `${context}: unknown transformType "${pair.transformType}" (expected one of ${TRANSFORM_TYPES.map((t) => t.value).join(', ')})`, + ); + } + transformTypes[key] = pair.transformType; + } + correspondences[key] = (pair.points || []).map((row, j) => { + const [ax, ay, bx, by] = CameraRegistrationStore.readPointsRow(row, `${context}, points row ${j + 1}`); + // eslint-disable-next-line no-plusplus + return { id: this.nextId++, a: [ax, ay] as Point, b: [bx, by] as Point }; + }); + const leftToRight = (pair.leftToRight === null || pair.leftToRight === undefined) + ? null + : CameraRegistrationStore.readMatrix(pair.leftToRight, `${context}, leftToRight`); + const rightToLeft = (pair.rightToLeft === null || pair.rightToLeft === undefined) + ? null + : CameraRegistrationStore.readMatrix(pair.rightToLeft, `${context}, rightToLeft`); + if (leftToRight || rightToLeft) { + // If only one direction is present, derive the other by inversion + // (readMatrix guarantees invertibility). + homographies[key] = { + AtoB: leftToRight ?? invert3(rightToLeft as Matrix3), + BtoA: rightToLeft ?? invert3(leftToRight as Matrix3), + }; + } + }); + this.correspondences.value = correspondences; + this.homographies.value = homographies; + this.transformTypes.value = transformTypes; + this.source.value = source; + this.markHomographySources(); + return { cameras: [...cameras], pairCount: file.pairs.length }; + } + + /** Validate an untrusted `source` value: a plain object, or absent (-> null). */ + private static readSource(raw: unknown): RegistrationSource | null { + if (raw === undefined || raw === null) { + return null; + } + if (typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error('"source" must be an object when present'); + } + return raw as RegistrationSource; + } + + /** Validate an untrusted value as a 4-element finite [leftX, leftY, rightX, rightY] row. */ + private static readPointsRow(raw: unknown, context: string): [number, number, number, number] { + if (!Array.isArray(raw) || raw.length !== 4) { + throw new Error(`${context}: expected [leftX, leftY, rightX, rightY]`); + } + const nums = raw.map(Number); + if (nums.some((n) => !Number.isFinite(n))) { + throw new Error(`${context}: expected [leftX, leftY, rightX, rightY] with finite numbers`); + } + return [nums[0], nums[1], nums[2], nums[3]]; + } + + /** Validate an untrusted value as an invertible row-major 3x3 matrix. */ + private static readMatrix(raw: unknown, context: string): Matrix3 { + if (!Array.isArray(raw) || raw.length !== 3 + || raw.some((row) => !Array.isArray(row) || row.length !== 3)) { + throw new Error(`${context}: expected a 3x3 matrix`); + } + const m = (raw as unknown[][]).map((row) => row.map(Number)); + if (m.some((row) => row.some((n) => !Number.isFinite(n)))) { + throw new Error(`${context}: matrix entries must be finite numbers`); + } + try { + invert3(m); + } catch { + throw new Error(`${context}: matrix is singular (not invertible)`); + } + return m; + } + + /** + * Reset homography provenance after bulk-loading state: a homography whose + * pair lacks enough points for its transform type can only have come from a + * file ('loaded', so refit checks preserve it); one with enough points is + * treated as fitted from them. + */ + private markHomographySources() { + this.homographySources = {}; + Object.keys(this.homographies.value).forEach((key) => { + const count = (this.correspondences.value[key] || []).length; + const required = minPointsForTransform(this.transformTypeForPair(key)); + this.homographySources[key] = count >= required ? 'fit' : 'loaded'; + }); + } + + /** Reset state and load saved homographies, correspondences, transform type choices, and provenance. */ + hydrate( + homographies?: CameraHomographies, + correspondences?: CameraCorrespondences, + transformTypes?: CameraTransformTypes, + source?: RegistrationSource | null, + ) { + this.homographies.value = homographies ? { ...homographies } : {}; + this.correspondences.value = correspondences ? { ...correspondences } : {}; + this.transformTypes.value = transformTypes ? { ...transformTypes } : {}; + this.source.value = source ?? null; + this.markHomographySources(); + // Resume id allocation past any restored correspondences. + let maxId = 0; + Object.values(this.correspondences.value).forEach((list) => { + list.forEach((c) => { maxId = Math.max(maxId, c.id); }); + }); + this.nextId = maxId + 1; + // The freshly loaded state is the saved baseline. + this.markSaved(); + } +} diff --git a/client/src/alignedView/README.md b/client/src/alignedView/README.md new file mode 100644 index 000000000..a0ee2367c --- /dev/null +++ b/client/src/alignedView/README.md @@ -0,0 +1,61 @@ +# aligned view and camera registration + +Multicam datasets can register each camera onto a shared reference space so the user can review annotations in an **aligned view** (SEAL-TK features 2 + 3). This folder holds the math, persistence format, reactive stores, and pure helpers that power that workflow. + +Stored annotation geometry always remains in each camera's native image space; transforms here are applied at draw time and for linked pan/zoom only. + +## Modules + +| File | Role | +| --- | --- | +| `homography.ts` | Low-level 3×3 matrix primitives: multiply, invert, apply, solve homographies (normalized DLT), linear-system helpers, and GeoJS warp-grid utilities (`subdivideWarpQuads`, `warpGridSize`, `localLinkedScale`). | +| `transform.ts` | Higher-level alignment models (translation, rigid, similarity, affine, homography) that all return a `Matrix3`. Defines `TransformType`, UI labels, minimum point counts, and `DEFAULT_TRANSFORM_TYPE` (`similarity`). | +| `CameraRegistrationStore.ts` | Reactive Vue store for in-app calibration: picked point correspondences, fitted/loaded pair transforms, per-pair transform-type choices, and producer provenance. Loads/saves the portable registration JSON and tracks dirty state. | +| `cameraRegistrationFiles.ts` | Serialization helpers for the per-camera `_to__registration.json` file format. Converts between the store's keyed state and self-describing file pairs; builds export bundles and filters imports to a single camera. | +| `alignedView.ts` | Pure helpers for the aligned view: compose pair homographies into per-camera native→reference matrices (`resolveToReferenceTransforms`), map geometry between cameras and reference space, and validate untrusted matrices from dataset meta. | +| `AlignedViewStore.ts` | Reactive Vue store for the aligned-view toggle: enabled/suspended state, resolved `toReference` matrices, registration progress, and `cameraTransform` / `cameraToCamera` accessors used by layers and linked navigation. | + +Each module has a matching `*.spec.ts` unit test alongside it. + +## How the pieces connect + +``` +picked points + transform type + │ + ▼ + CameraRegistrationStore ──fit──► homography / transform + │ │ + │ save/load │ pair matrices + ▼ ▼ + cameraRegistrationFiles alignedView.ts + │ │ + │ export/import │ resolveToReferenceTransforms + ▼ ▼ + desktop + web persistence AlignedViewStore + │ + ▼ + layers, linked viewers, annotation warping +``` + +- **Calibration** flows through `CameraRegistrationStore` and `cameraRegistrationFiles` (panel, desktop on-disk files, web Girder uploads, multicam import seed). +- **Display** reads fitted pair homographies from the store, resolves every camera into the reference camera's space via `alignedView.ts`, and exposes the result through `AlignedViewStore`. +- **Math** is layered: `homography.ts` for matrices and full homographies; `transform.ts` for constrained models that need fewer point pairs on near-rigid rigs. + +## Conventions + +- The **reference camera** is the first camera in display order (`multiCamList[0]`). Its transform is the identity. +- Pair keys use directional `camA::camB` notation. `AtoB` maps camA pixels onto camB. +- A camera reaches the reference by breadth-first composition through the pair graph (e.g. UV→IR→EO on a three-camera rig). +- The aligned view is all-or-none: if any loaded camera lacks a path to the reference, `resolveToReferenceTransforms` returns null and the toggle stays unavailable. + +## Import paths + +Within `vue-media-annotator`, import from this subfolder, for example: + +```ts +import AlignedViewStore from 'vue-media-annotator/alignedView/AlignedViewStore'; +import { resolveToReferenceTransforms } from 'vue-media-annotator/alignedView/alignedView'; +import { applyHomography } from 'vue-media-annotator/alignedView/homography'; +``` + +Both stores are also re-exported from `vue-media-annotator/index` and wired through `provides.ts` for inject/use helpers. diff --git a/client/src/alignedView/alignedView.spec.ts b/client/src/alignedView/alignedView.spec.ts new file mode 100644 index 000000000..278b52f28 --- /dev/null +++ b/client/src/alignedView/alignedView.spec.ts @@ -0,0 +1,301 @@ +/// +import { + IDENTITY3, + isIdentityMatrix3, + readTransformMatrix, + composeThroughPairs, + resolveToReferenceTransforms, + unresolvedCameras, + cameraPairTransform, + mapPoint, + mapBounds, + mapRotatedBounds, + mapGeoJSONFeatures, +} from './alignedView'; +import { applyHomography, Matrix3, Point } from './homography'; +import type { CameraHomographies } from './CameraRegistrationStore'; +import AlignedViewStore from './AlignedViewStore'; + +/** Simple affine helpers for readable fixtures. */ +function translation(tx: number, ty: number): Matrix3 { + return [[1, 0, tx], [0, 1, ty], [0, 0, 1]]; +} +function scale(s: number): Matrix3 { + return [[s, 0, 0], [0, s, 0], [0, 0, 1]]; +} + +function expectPointClose(actual: Point, expected: Point) { + expect(actual[0]).toBeCloseTo(expected[0], 6); + expect(actual[1]).toBeCloseTo(expected[1], 6); +} + +describe('isIdentityMatrix3', () => { + it('accepts the identity and rejects non-identity', () => { + expect(isIdentityMatrix3(IDENTITY3)).toBe(true); + expect(isIdentityMatrix3(translation(0, 0))).toBe(true); + expect(isIdentityMatrix3(translation(1, 0))).toBe(false); + expect(isIdentityMatrix3(scale(2))).toBe(false); + }); +}); + +describe('readTransformMatrix', () => { + it('accepts a valid row-major 3x3', () => { + const m = readTransformMatrix([[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + expect(m).toEqual([[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); + it('rejects malformed shapes', () => { + expect(readTransformMatrix(undefined)).toBeNull(); + expect(readTransformMatrix(null)).toBeNull(); + expect(readTransformMatrix('matrix')).toBeNull(); + expect(readTransformMatrix([[1, 0], [0, 1]])).toBeNull(); + expect(readTransformMatrix([[1, 0, 0], [0, 1, 0]])).toBeNull(); + expect(readTransformMatrix([[1, 0, 0], [0, 1, 0], [0, 0]])).toBeNull(); + }); + it('rejects non-finite values', () => { + expect(readTransformMatrix([[1, 0, NaN], [0, 1, 0], [0, 0, 1]])).toBeNull(); + expect(readTransformMatrix([[1, 0, Infinity], [0, 1, 0], [0, 0, 1]])).toBeNull(); + expect(readTransformMatrix([[1, 0, '5'], [0, 1, 'abc'], [0, 0, 1]])).toBeNull(); + }); + it('rejects singular matrices', () => { + expect(readTransformMatrix([[1, 1, 0], [1, 1, 0], [0, 0, 1]])).toBeNull(); + expect(readTransformMatrix([[0, 0, 0], [0, 0, 0], [0, 0, 0]])).toBeNull(); + }); +}); + +describe('composeThroughPairs', () => { + const irToEo = translation(100, 50); + const uvToIr = scale(2); + const homographies: CameraHomographies = { + // eo::ir stored with AtoB = eo->ir, so BtoA = ir->eo. + 'eo::ir': { AtoB: [[1, 0, -100], [0, 1, -50], [0, 0, 1]], BtoA: irToEo }, + // uv::ir stored with AtoB = uv->ir. + 'uv::ir': { AtoB: uvToIr, BtoA: [[0.5, 0, 0], [0, 0.5, 0], [0, 0, 1]] }, + }; + it('returns identity for the reference itself', () => { + expect(composeThroughPairs('eo', 'eo', homographies)).toEqual(IDENTITY3); + }); + it('uses a direct pair edge in either direction', () => { + const m = composeThroughPairs('ir', 'eo', homographies); + expect(m).not.toBeNull(); + expectPointClose(applyHomography(m as Matrix3, [10, 20]), [110, 70]); + }); + it('composes multi-hop paths (uv -> ir -> eo)', () => { + const m = composeThroughPairs('uv', 'eo', homographies); + expect(m).not.toBeNull(); + // uv (x, y) -> ir (2x, 2y) -> eo (2x + 100, 2y + 50). + expectPointClose(applyHomography(m as Matrix3, [10, 20]), [120, 90]); + }); + it('returns null when no path exists', () => { + expect(composeThroughPairs('flir', 'eo', homographies)).toBeNull(); + expect(composeThroughPairs('ir', 'eo', {})).toBeNull(); + }); +}); + +describe('resolveToReferenceTransforms', () => { + const cameras = ['eo', 'ir', 'uv']; + const homographies: CameraHomographies = { + 'eo::ir': { AtoB: translation(-100, 0), BtoA: translation(100, 0) }, + 'uv::ir': { AtoB: scale(2), BtoA: scale(0.5) }, + }; + it('resolves every camera through calibration pairs', () => { + const result = resolveToReferenceTransforms(cameras, 'eo', homographies); + expect(result).not.toBeNull(); + const transforms = result as Record; + expect(transforms.eo).toEqual(IDENTITY3); + expectPointClose(applyHomography(transforms.ir, [5, 5]), [105, 5]); + expectPointClose(applyHomography(transforms.uv, [5, 5]), [110, 10]); + }); + it('is all-or-none: any unresolved camera fails the whole set', () => { + expect(resolveToReferenceTransforms(['eo', 'ir', 'flir'], 'eo', homographies)).toBeNull(); + expect(resolveToReferenceTransforms(cameras, 'eo', {})).toBeNull(); + }); + it('fails when the reference is not among the cameras or the list is empty', () => { + expect(resolveToReferenceTransforms([], 'eo', homographies)).toBeNull(); + expect(resolveToReferenceTransforms(['ir', 'uv'], 'eo', homographies)).toBeNull(); + }); +}); + +describe('unresolvedCameras', () => { + const homographies: CameraHomographies = { + 'eo::ir': { AtoB: translation(-100, 0), BtoA: translation(100, 0) }, + }; + it('names cameras with no path to the reference', () => { + expect(unresolvedCameras(['eo', 'ir', 'uv'], 'eo', homographies)).toEqual(['uv']); + expect(unresolvedCameras(['eo', 'ir', 'uv'], 'eo', {})).toEqual(['ir', 'uv']); + }); + it('is empty when every camera resolves', () => { + expect(unresolvedCameras(['eo', 'ir'], 'eo', homographies)).toEqual([]); + }); +}); + +describe('cameraPairTransform', () => { + it('composes from -> reference -> to', () => { + const toReference = { + eo: IDENTITY3, + ir: translation(100, 0), + uv: scale(2), + }; + // ir (x, y) -> ref (x + 100, y) -> uv ((x + 100) / 2, y / 2). + const m = cameraPairTransform(toReference, 'ir', 'uv'); + expect(m).not.toBeNull(); + expectPointClose(applyHomography(m as Matrix3, [10, 20]), [55, 10]); + }); + it('returns null for unresolved cameras', () => { + expect(cameraPairTransform({ eo: IDENTITY3 }, 'eo', 'ir')).toBeNull(); + expect(cameraPairTransform({ eo: IDENTITY3 }, 'ir', 'eo')).toBeNull(); + }); +}); + +describe('mapPoint', () => { + it('is identity for a null matrix', () => { + expect(mapPoint(null, [3, 4])).toEqual([3, 4]); + }); + it('applies the matrix otherwise', () => { + expectPointClose(mapPoint(translation(1, 2), [3, 4]), [4, 6]); + }); +}); + +describe('mapBounds', () => { + it('translates and scales an axis-aligned box', () => { + expect(mapBounds(translation(10, -5), [0, 0, 4, 6])).toEqual([10, -5, 14, 1]); + expect(mapBounds(scale(2), [1, 2, 3, 4])).toEqual([2, 4, 6, 8]); + }); + it('returns the AABB of the mapped corners under rotation', () => { + // 90-degree rotation about the origin: (x, y) -> (-y, x). + const rot90: Matrix3 = [[0, -1, 0], [1, 0, 0], [0, 0, 1]]; + const [x1, y1, x2, y2] = mapBounds(rot90, [0, 0, 4, 2]); + expect(x1).toBeCloseTo(-2, 6); + expect(y1).toBeCloseTo(0, 6); + expect(x2).toBeCloseTo(0, 6); + expect(y2).toBeCloseTo(4, 6); + }); +}); + +describe('mapRotatedBounds', () => { + const rot = (angle: number): Matrix3 => [ + [Math.cos(angle), -Math.sin(angle), 0], + [Math.sin(angle), Math.cos(angle), 0], + [0, 0, 1], + ]; + it('adds the transform rotation and keeps the rect size under similarity', () => { + const theta = Math.PI / 6; + const result = mapRotatedBounds(rot(theta), [10, 20, 50, 40], Math.PI / 8); + expect(result.rotation).toBeCloseTo(Math.PI / 8 + theta, 6); + // The rect's own dimensions (40 x 20) are preserved by a pure rotation. + expect(result.bounds[2] - result.bounds[0]).toBeCloseTo(40, 6); + expect(result.bounds[3] - result.bounds[1]).toBeCloseTo(20, 6); + }); + it('is a passthrough for the identity', () => { + const result = mapRotatedBounds(IDENTITY3, [10, 20, 50, 40], 0.3); + expect(result.rotation).toBeCloseTo(0.3, 6); + expect(result.bounds[0]).toBeCloseTo(10, 6); + expect(result.bounds[1]).toBeCloseTo(20, 6); + expect(result.bounds[2]).toBeCloseTo(50, 6); + expect(result.bounds[3]).toBeCloseTo(40, 6); + }); + it('scales the rect and centers under translation + scale', () => { + const matrix = [[2, 0, 100], [0, 2, -10], [0, 0, 1]]; + const result = mapRotatedBounds(matrix, [0, 0, 10, 20], 0.5); + expect(result.rotation).toBeCloseTo(0.5, 6); + expect(result.bounds[2] - result.bounds[0]).toBeCloseTo(20, 6); + expect(result.bounds[3] - result.bounds[1]).toBeCloseTo(40, 6); + // The center (5, 10) maps to (110, 10). + expect((result.bounds[0] + result.bounds[2]) / 2).toBeCloseTo(110, 6); + expect((result.bounds[1] + result.bounds[3]) / 2).toBeCloseTo(10, 6); + }); +}); + +describe('mapGeoJSONFeatures', () => { + it('maps points, lines, and polygon rings without mutating the source', () => { + const source: GeoJSON.Feature[] = [ + { + type: 'Feature', + geometry: { type: 'Point', coordinates: [1, 2] }, + properties: { key: 'head' }, + }, + { + type: 'Feature', + geometry: { type: 'LineString', coordinates: [[0, 0], [3, 4]] }, + properties: { key: 'HeadTails' }, + }, + { + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [[[0, 0], [4, 0], [4, 4], [0, 0]], [[1, 1], [2, 1], [2, 2], [1, 1]]], + }, + properties: { key: '' }, + }, + ]; + const mapped = mapGeoJSONFeatures(translation(10, 20), source); + expect(mapped[0].geometry).toEqual({ type: 'Point', coordinates: [11, 22] }); + expect(mapped[1].geometry).toEqual({ type: 'LineString', coordinates: [[10, 20], [13, 24]] }); + expect(mapped[2].geometry.type).toBe('Polygon'); + expect((mapped[2].geometry as GeoJSON.Polygon).coordinates[1][2]).toEqual([12, 22]); + expect(mapped[0].properties).toEqual({ key: 'head' }); + // Deep copy: the source geometry/properties are untouched and unshared. + expect(source[0].geometry).toEqual({ type: 'Point', coordinates: [1, 2] }); + expect(mapped[0].properties).not.toBe(source[0].properties); + expect(mapped[2].geometry).not.toBe(source[2].geometry); + }); +}); + +describe('AlignedViewStore', () => { + function makeResolvedStore() { + const store = new AlignedViewStore(); + store.setTransforms('eo', { + eo: IDENTITY3, + ir: translation(100, 0), + }); + return store; + } + + it('is unavailable and inactive by default', () => { + const store = new AlignedViewStore(); + expect(store.available.value).toBe(false); + store.setEnabled(true); + expect(store.active.value).toBe(false); + expect(store.cameraTransform('ir')).toBeNull(); + expect(store.cameraToCamera('eo', 'ir')).toBeNull(); + }); + + it('activates only when enabled, available, and not suspended', () => { + const store = makeResolvedStore(); + expect(store.available.value).toBe(true); + expect(store.active.value).toBe(false); + store.setEnabled(true); + expect(store.active.value).toBe(true); + store.setSuspended(true); + expect(store.active.value).toBe(false); + expect(store.cameraTransform('ir')).toBeNull(); + store.setSuspended(false); + expect(store.active.value).toBe(true); + }); + + it('exposes a display transform for warped cameras and null for the reference', () => { + const store = makeResolvedStore(); + store.setEnabled(true); + expect(store.cameraTransform('eo')).toBeNull(); + const m = store.cameraTransform('ir'); + expect(m).not.toBeNull(); + expectPointClose(applyHomography(m as Matrix3, [1, 1]), [101, 1]); + // Unknown camera also renders unwarped. + expect(store.cameraTransform('uv')).toBeNull(); + }); + + it('maps cross-camera points through the reference', () => { + const store = makeResolvedStore(); + store.setEnabled(true); + expectPointClose(store.mapCameraPoint('ir', 'eo', [0, 0]) as Point, [100, 0]); + expectPointClose(store.mapCameraPoint('eo', 'ir', [100, 0]) as Point, [0, 0]); + expect(store.mapCameraPoint('eo', 'flir', [0, 0])).toBeNull(); + }); + + it('becomes unavailable when a camera set collapses to one entry', () => { + const store = new AlignedViewStore(); + store.setTransforms('eo', { eo: IDENTITY3 }); + store.setEnabled(true); + expect(store.available.value).toBe(false); + expect(store.active.value).toBe(false); + }); +}); diff --git a/client/src/alignedView/alignedView.ts b/client/src/alignedView/alignedView.ts new file mode 100644 index 000000000..e462ff273 --- /dev/null +++ b/client/src/alignedView/alignedView.ts @@ -0,0 +1,305 @@ +/** + * Pure helpers for the multicam "aligned view" (SEAL-TK features 2 + 3): + * resolving a per-camera native->reference-space transform for every loaded + * camera from the available sources, and composing camera-to-camera mappings + * through the reference camera. + * + * Conventions (documented assumptions, see migration plan Q3): + * - The REFERENCE camera is the first camera in display order + * (`meta.multiCamMedia.cameraOrder[0]`, i.e. `multiCamList[0]`). Its + * transform is the identity. + * - Calibration-tool homographies are stored per directional pair key + * `camA::camB` with `AtoB` mapping camA pixels onto camB (and `BtoA` the + * inverse); a camera's path to the reference is found by composing pair + * edges (breadth-first, so up-to-3-camera rigs may chain e.g. UV->IR->EO). + * - The calibration store is the SINGLE source the viewer resolves from: + * whatever the calibration panel shows/saves is what the Align button + * applies. External calibration .json files enter that same store, either + * through the panel's Load calibration button or by seeding the saved + * calibration at multicam import time. + */ +import { + Matrix3, matMul3, invert3, applyHomography, Point, +} from './homography'; +import type { CameraHomographies } from './CameraRegistrationStore'; +import type { RectBounds } from '../utils'; + +export const IDENTITY3: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + +/** True when `m` is (numerically) the identity transform. */ +export function isIdentityMatrix3(m: Matrix3, eps = 1e-9): boolean { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + if (Math.abs(m[i][j] - IDENTITY3[i][j]) > eps) { + return false; + } + } + } + return true; +} + +/** + * Defensively validate an untrusted value (e.g. read from dataset meta JSON) + * as a row-major 3x3 matrix usable as a display transform. Returns null for + * anything malformed, non-finite, or singular (non-invertible) rather than + * throwing, so callers can treat "bad matrix" as "no matrix". + */ +export function readTransformMatrix(raw: unknown): Matrix3 | null { + if (!Array.isArray(raw) || raw.length !== 3) { + return null; + } + const m: number[][] = []; + for (let i = 0; i < 3; i += 1) { + const row = raw[i]; + if (!Array.isArray(row) || row.length !== 3) { + return null; + } + const nums = row.map(Number); + if (nums.some((v) => !Number.isFinite(v))) { + return null; + } + m.push(nums); + } + try { + invert3(m); + } catch { + return null; + } + return m; +} + +/** One directed edge of the calibration-pair graph: `matrix` maps `from` pixels onto `to`. */ +interface PairEdge { + to: string; + matrix: Matrix3; +} + +/** Build the bidirectional camera adjacency from calibration pair homographies. */ +function buildPairGraph(homographies: CameraHomographies): Record { + const graph: Record = {}; + const addEdge = (from: string, to: string, matrix: Matrix3) => { + if (!graph[from]) { + graph[from] = []; + } + graph[from].push({ to, matrix }); + }; + Object.entries(homographies).forEach(([key, pair]) => { + const [camA, camB] = key.split('::'); + if (!camA || !camB || camA === camB || !pair) { + return; + } + addEdge(camA, camB, pair.AtoB); + addEdge(camB, camA, pair.BtoA); + }); + return graph; +} + +/** + * Compose a `camera` -> `reference` matrix by walking the calibration pair + * graph breadth-first (shortest hop count). Returns null when no path exists. + */ +export function composeThroughPairs( + camera: string, + reference: string, + homographies: CameraHomographies, +): Matrix3 | null { + if (camera === reference) { + return IDENTITY3; + } + const graph = buildPairGraph(homographies); + // BFS queue of (camera, accumulated camera->node matrix). + const queue: { node: string; matrix: Matrix3 }[] = [{ node: camera, matrix: IDENTITY3 }]; + const visited = new Set([camera]); + while (queue.length) { + const { node, matrix } = queue.shift() as { node: string; matrix: Matrix3 }; + const edges = graph[node] || []; + for (let i = 0; i < edges.length; i += 1) { + const edge = edges[i]; + if (!visited.has(edge.to)) { + // p_to = edge.matrix * p_node and p_node = matrix * p_camera. + const composed = matMul3(edge.matrix, matrix); + if (edge.to === reference) { + return composed; + } + visited.add(edge.to); + queue.push({ node: edge.to, matrix: composed }); + } + } + } + return null; +} + +/** + * Resolve a native->reference matrix for EVERY camera in `cameras` from the + * calibration store's pair homographies, or null when any camera lacks a + * usable transform (the aligned view is all-or-none: a partially-aligned + * display would be misleading). The reference camera always maps by the + * identity. + */ +export function resolveToReferenceTransforms( + cameras: string[], + reference: string, + homographies: CameraHomographies, +): Record | null { + if (!cameras.length || !cameras.includes(reference)) { + return null; + } + const result: Record = {}; + for (let i = 0; i < cameras.length; i += 1) { + const camera = cameras[i]; + if (camera === reference) { + result[camera] = IDENTITY3; + } else { + const matrix = composeThroughPairs(camera, reference, homographies); + if (!matrix) { + return null; + } + result[camera] = matrix; + } + } + return result; +} + +/** + * The cameras in `cameras` with no composed transform path to `reference` -- + * the reason {@link resolveToReferenceTransforms} returned null, named so the + * UI can say exactly which pair(s) still need calibrating instead of silently + * hiding the aligned view. + */ +export function unresolvedCameras( + cameras: string[], + reference: string, + homographies: CameraHomographies, +): string[] { + return cameras.filter( + (camera) => camera !== reference + && composeThroughPairs(camera, reference, homographies) === null, + ); +} + +/** + * Matrix mapping `from`-camera native pixels onto `to`-camera native pixels, + * composed through the shared reference space: + * p_to = inv(T_to->ref) * T_from->ref * p_from. + * Returns null when either camera has no resolved transform. + */ +export function cameraPairTransform( + toReference: Record, + from: string, + to: string, +): Matrix3 | null { + const fromRef = toReference[from]; + const toRef = toReference[to]; + if (!fromRef || !toRef) { + return null; + } + try { + return matMul3(invert3(toRef), fromRef); + } catch { + return null; + } +} + +/** Map a point through a nullable transform (identity when null). */ +export function mapPoint(matrix: Matrix3 | null, point: Point): Point { + if (!matrix) { + return point; + } + return applyHomography(matrix, point); +} + +/** Axis-aligned bounding box of a set of points. */ +function pointsToBounds(points: Point[]): RectBounds { + const xs = points.map((p) => p[0]); + const ys = points.map((p) => p[1]); + return [Math.min(...xs), Math.min(...ys), Math.max(...xs), Math.max(...ys)]; +} + +/** The four corners of a RectBounds, clockwise from (x1, y1). */ +function boundsCorners(bounds: RectBounds): Point[] { + return [ + [bounds[0], bounds[1]], + [bounds[2], bounds[1]], + [bounds[2], bounds[3]], + [bounds[0], bounds[3]], + ]; +} + +/** + * Map an axis-aligned RectBounds through a homography. A non-affine transform + * turns a rectangle into a general quad, so the result is the axis-aligned + * bounding box of the four mapped corners. + */ +export function mapBounds(matrix: Matrix3, bounds: RectBounds): RectBounds { + return pointsToBounds(boundsCorners(bounds).map((c) => applyHomography(matrix, c))); +} + +/** + * Map a rotated rectangle (RectBounds + rotation in radians about the bounds + * center, the convention of utils.rotateGeoJSONCoordinates) through a + * homography, returning a best-fit rotated rectangle in the target space: + * the mapped direction of the rect's top edge gives the new rotation, and the + * mapped corners un-rotated about their centroid give the new bounds. Exact + * for similarity transforms; a least-surprise approximation for the rest. + */ +export function mapRotatedBounds( + matrix: Matrix3, + bounds: RectBounds, + rotation: number, +): { bounds: RectBounds; rotation: number } { + const cx = (bounds[0] + bounds[2]) / 2; + const cy = (bounds[1] + bounds[3]) / 2; + const rotate = (p: Point, angle: number, ox: number, oy: number): Point => { + const cos = Math.cos(angle); + const sin = Math.sin(angle); + const dx = p[0] - ox; + const dy = p[1] - oy; + return [ox + dx * cos - dy * sin, oy + dx * sin + dy * cos]; + }; + const mapped = boundsCorners(bounds) + .map((c) => applyHomography(matrix, rotate(c, rotation, cx, cy))); + // The top edge (corner 0 -> corner 1) points along +x at zero rotation, so + // its mapped direction is the combined rotation in the target space. + const newRotation = Math.atan2( + mapped[1][1] - mapped[0][1], + mapped[1][0] - mapped[0][0], + ); + const mcx = mapped.reduce((sum, p) => sum + p[0], 0) / mapped.length; + const mcy = mapped.reduce((sum, p) => sum + p[1], 0) / mapped.length; + const unrotated = mapped.map((p) => rotate(p, -newRotation, mcx, mcy)); + return { bounds: pointsToBounds(unrotated), rotation: newRotation }; +} + +type MappableGeometry = GeoJSON.Point | GeoJSON.LineString | GeoJSON.Polygon; + +/** + * Deep-copy GeoJSON features (the shapes tracks store per-frame: Point, + * LineString, Polygon) with every coordinate mapped through a homography. + * Unsupported geometry types are copied through unmapped. + */ +export function mapGeoJSONFeatures( + matrix: Matrix3, + features: GeoJSON.Feature[], +): GeoJSON.Feature[] { + const mapPosition = (p: GeoJSON.Position): GeoJSON.Position => ( + applyHomography(matrix, [p[0], p[1]]) + ); + return features.map((feature) => { + let geometry: MappableGeometry; + if (feature.geometry.type === 'Point') { + geometry = { type: 'Point', coordinates: mapPosition(feature.geometry.coordinates) }; + } else if (feature.geometry.type === 'LineString') { + geometry = { type: 'LineString', coordinates: feature.geometry.coordinates.map(mapPosition) }; + } else { + geometry = { + type: 'Polygon', + coordinates: feature.geometry.coordinates.map((ring) => ring.map(mapPosition)), + }; + } + return { + ...feature, + geometry: geometry as T, + properties: feature.properties ? { ...feature.properties } : feature.properties, + }; + }); +} diff --git a/client/src/alignedView/cameraRegistrationFiles.spec.ts b/client/src/alignedView/cameraRegistrationFiles.spec.ts new file mode 100644 index 000000000..f832cea5f --- /dev/null +++ b/client/src/alignedView/cameraRegistrationFiles.spec.ts @@ -0,0 +1,165 @@ +// eslint-disable-next-line import/no-extraneous-dependencies -- Vitest is only used in tests +import { describe, expect, it } from 'vitest'; + +import { + buildPerCameraRegistrationFiles, + registrationValuesSummary, + filterRegistrationValues, + mergeRegistrationValues, + CameraRegistrationValues, +} from './cameraRegistrationFiles'; + +const IDENTITY = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; +const SHIFT = [[1, 0, 5], [0, 1, -3], [0, 0, 1]]; +const UNSHIFT = [[1, 0, -5], [0, 1, 3], [0, 0, 1]]; + +function values(partial: Partial): CameraRegistrationValues { + return { + homographies: {}, + correspondences: {}, + transformTypes: {}, + source: null, + ...partial, + }; +} + +describe('buildPerCameraRegistrationFiles', () => { + it('groups each pair under its non-reference camera, sorted', () => { + const files = buildPerCameraRegistrationFiles(values({ + homographies: { + 'rgb::uv': { AtoB: IDENTITY, BtoA: IDENTITY }, + 'ir::rgb': { AtoB: SHIFT, BtoA: UNSHIFT }, + }, + }), 'rgb'); + expect(files.map((f) => f.camera)).toStrictEqual(['ir', 'uv']); + expect(files.map((f) => f.name)).toStrictEqual(['ir_to_rgb_registration.json', 'uv_to_rgb_registration.json']); + // The pair whose RIGHT camera is the reference files under its left. + expect(files[0].body.pairs).toStrictEqual([{ + left: 'ir', + right: 'rgb', + points: [], + leftToRight: SHIFT, + rightToLeft: UNSHIFT, + transformType: 'similarity', + }]); + }); + + it('falls back to right-camera grouping without a reference', () => { + const files = buildPerCameraRegistrationFiles(values({ + homographies: { 'rgb::ir': { AtoB: SHIFT, BtoA: UNSHIFT } }, + }), null); + expect(files.map((f) => f.camera)).toStrictEqual(['ir']); + }); + + it('self-identifies files and carries a plain source stamp', () => { + const source = { producer: 'kamera', run: 'fl07' }; + const [file] = buildPerCameraRegistrationFiles(values({ + homographies: { 'rgb::ir': { AtoB: SHIFT, BtoA: UNSHIFT } }, + source, + }), 'rgb'); + expect(file.body.type).toBe('dive-camera-registration'); + expect(file.body.source).toStrictEqual(source); + }); + + it('never stamps files with a mixed composite source', () => { + const [file] = buildPerCameraRegistrationFiles(values({ + homographies: { 'rgb::ir': { AtoB: SHIFT, BtoA: UNSHIFT } }, + source: { mixed: true, files: {} }, + }), 'rgb'); + expect('source' in file.body).toBe(false); + }); + + it('serializes correspondences as leftX leftY rightX rightY rows', () => { + const [file] = buildPerCameraRegistrationFiles(values({ + correspondences: { + 'rgb::ir': [{ id: 1, a: [10, 20], b: [12, 22] }], + }, + }), 'rgb'); + expect(file.body.pairs[0].points).toStrictEqual([[10, 20, 12, 22]]); + expect(file.body.pairs[0].leftToRight).toBeNull(); + }); +}); + +describe('filterRegistrationValues', () => { + const multi = values({ + homographies: { + 'rgb::ir': { AtoB: SHIFT, BtoA: UNSHIFT }, + 'rgb::uv': { AtoB: IDENTITY, BtoA: IDENTITY }, + }, + transformTypes: { 'rgb::ir': 'rigid', 'rgb::uv': 'affine' }, + source: { producer: 'kamera' }, + }); + + it('keeps only pairs naming the camera, on either side', () => { + const filtered = filterRegistrationValues(multi, 'ir'); + expect(Object.keys(filtered.homographies)).toStrictEqual(['rgb::ir']); + expect(Object.keys(filtered.transformTypes)).toStrictEqual(['rgb::ir']); + expect(filtered.source).toStrictEqual({ producer: 'kamera' }); + }); + + it('yields an empty calibration for an unknown camera', () => { + expect(registrationValuesSummary(filterRegistrationValues(multi, 'zz')).pairCount).toBe(0); + }); +}); + +describe('registrationValuesSummary', () => { + it('counts distinct pairs and names their cameras', () => { + const summary = registrationValuesSummary(values({ + homographies: { 'rgb::ir': { AtoB: SHIFT, BtoA: UNSHIFT } }, + correspondences: { 'rgb::uv': [{ id: 1, a: [1, 2], b: [3, 4] }] }, + })); + expect(summary.pairCount).toBe(2); + expect(summary.cameras.sort()).toStrictEqual(['ir', 'rgb', 'uv']); + }); +}); + +describe('mergeRegistrationValues', () => { + const existing = values({ + homographies: { 'rgb::ir': { AtoB: SHIFT, BtoA: UNSHIFT } }, + correspondences: { 'rgb::ir': [{ id: 1, a: [1, 2], b: [3, 4] }] }, + transformTypes: { 'rgb::ir': 'rigid' }, + source: { producer: 'kamera', run: 'fl07' }, + }); + + it('keeps pairs the import does not name', () => { + const merged = mergeRegistrationValues(existing, values({ + homographies: { 'rgb::uv': { AtoB: IDENTITY, BtoA: IDENTITY } }, + }), 'calibration_uv.json'); + expect(Object.keys(merged.homographies).sort()).toStrictEqual(['rgb::ir', 'rgb::uv']); + expect(merged.transformTypes['rgb::ir']).toBe('rigid'); + }); + + it('replaces a named pair wholly, dropping stale points and model choice', () => { + const merged = mergeRegistrationValues(existing, values({ + homographies: { 'rgb::ir': { AtoB: IDENTITY, BtoA: IDENTITY } }, + }), 'calibration_ir.json'); + expect(merged.homographies['rgb::ir'].AtoB).toStrictEqual(IDENTITY); + expect(merged.correspondences['rgb::ir']).toBeUndefined(); + expect(merged.transformTypes['rgb::ir']).toBeUndefined(); + }); + + it('keeps the existing stamp when the import carries none', () => { + const merged = mergeRegistrationValues(existing, values({ + homographies: { 'rgb::uv': { AtoB: IDENTITY, BtoA: IDENTITY } }, + }), 'calibration_uv.json'); + expect(merged.source).toStrictEqual({ producer: 'kamera', run: 'fl07' }); + }); + + it('keeps a single stamp when both agree, mixes when they disagree', () => { + const agreeing = mergeRegistrationValues(existing, values({ + source: { producer: 'kamera', run: 'fl07' }, + }), 'calibration_uv.json'); + expect(agreeing.source).toStrictEqual({ producer: 'kamera', run: 'fl07' }); + + const disagreeing = mergeRegistrationValues(existing, values({ + source: { producer: 'kamera', run: 'fl09' }, + }), 'calibration_uv.json'); + expect(disagreeing.source).toStrictEqual({ + mixed: true, + files: { + previous: { producer: 'kamera', run: 'fl07' }, + 'calibration_uv.json': { producer: 'kamera', run: 'fl09' }, + }, + }); + }); +}); diff --git a/client/src/alignedView/cameraRegistrationFiles.ts b/client/src/alignedView/cameraRegistrationFiles.ts new file mode 100644 index 000000000..afeb86461 --- /dev/null +++ b/client/src/alignedView/cameraRegistrationFiles.ts @@ -0,0 +1,251 @@ +/** + * The portable per-camera image-registration file format, shared by every + * producer and consumer of _to__registration.json files: + * the desktop backend (persistence + export), the web client (export + * downloads + import uploads), and the multicam import seed. One file per + * non-reference camera, named for the mapping it carries (camera registers + * onto the reference) and self-identified with + * `type: 'dive-camera-registration'`; pair bodies name their own cameras, + * so file names are discovery/provenance only. + */ +import { + RegistrationFile, RegistrationFilePair, RegistrationSource, + CameraCorrespondences, CameraHomographies, CameraTransformTypes, + REGISTRATION_FILE_TYPE, +} from './CameraRegistrationStore'; +import { DEFAULT_TRANSFORM_TYPE } from './transform'; + +/** The complete in-app calibration state for one dataset. */ +export interface CameraRegistrationValues { + homographies: CameraHomographies; + correspondences: CameraCorrespondences; + transformTypes: CameraTransformTypes; + source: RegistrationSource | null; +} + +/** + * File name for one camera's registration. The destination (the camera it + * registers onto -- normally the rig reference) is part of the name so the + * file states its own direction: ir_to_eo_registration.json registers ir + * onto eo. Omitted when the camera's pairs have no single partner. + */ +export function registrationFileName(camera: string, destination: string | null): string { + return destination + ? `${camera}_to_${destination}_registration.json` + : `${camera}_registration.json`; +} + +/** + * Restrict a calibration to the pairs naming `camera` (either side). Used by + * the per-camera import buttons so a multi-pair file only contributes the + * chosen camera's pair(s). + */ +export function filterRegistrationValues( + values: CameraRegistrationValues, + camera: string, +): CameraRegistrationValues { + const keep = (key: string) => key.split('::').includes(camera); + const filterRecord = (record: Record): Record => Object.fromEntries( + Object.entries(record).filter(([key]) => keep(key)), + ); + return { + homographies: filterRecord(values.homographies), + correspondences: filterRecord(values.correspondences), + transformTypes: filterRecord(values.transformTypes), + source: values.source, + }; +} + +/** The distinct camera names and pair count a calibration holds. */ +export function registrationValuesSummary( + values: CameraRegistrationValues, +): { cameras: string[]; pairCount: number } { + const keys = new Set([ + ...Object.keys(values.homographies), + ...Object.keys(values.correspondences), + ...Object.keys(values.transformTypes), + ]); + const cameras = new Set(); + keys.forEach((key) => key.split('::').forEach((name) => cameras.add(name))); + return { cameras: [...cameras], pairCount: keys.size }; +} + +/** + * Convert the in-app calibration state (keyed by directional "left::right") + * into the self-describing list of file pairs. + */ +function toRegistrationFilePairs(values: CameraRegistrationValues): RegistrationFilePair[] { + const keys = new Set([ + ...Object.keys(values.homographies), + ...Object.keys(values.correspondences), + ...Object.keys(values.transformTypes), + ]); + return [...keys].map((key) => { + const [left, right] = key.split('::'); + const homography = values.homographies[key]; + return { + left, + right, + points: (values.correspondences[key] || []).map((c) => [c.a[0], c.a[1], c.b[0], c.b[1]]), + leftToRight: homography ? homography.AtoB : null, + rightToLeft: homography ? homography.BtoA : null, + transformType: values.transformTypes[key] || DEFAULT_TRANSFORM_TYPE, + }; + }); +} + +/** + * Group a calibration into its per-camera file bodies: one self-identified + * _to__registration.json per non-reference camera + * (reference = first camera in display order). A pair not touching the + * reference files under its right camera, named for that pair's other side; + * grouping is cosmetic either way since pair bodies are authoritative on + * load. + */ +export function buildPerCameraRegistrationFiles( + values: CameraRegistrationValues, + referenceCamera: string | null, +): { camera: string; destination: string | null; name: string; body: RegistrationFile }[] { + const pairsByCamera = new Map(); + toRegistrationFilePairs(values).forEach((pair) => { + let camera = pair.right; + if (referenceCamera !== null && pair.right === referenceCamera + && pair.left !== referenceCamera) { + camera = pair.left; + } + pairsByCamera.set(camera, [...(pairsByCamera.get(camera) ?? []), pair]); + }); + // A mixed composite stamp describes the assembled SET, not any single file; + // stamping every per-camera file with it would present a unanimous + // (therefore "consistent") rig on the next load, hiding the very mismatch + // it records. Per-file stamps resume with the next externally produced file. + const fileSource = values.source + && (values.source as Record).mixed === true + ? null + : values.source; + return [...pairsByCamera.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([camera, pairs]) => { + // The camera this file's content warps onto: the single partner named + // across its pairs (normally the rig reference). Multiple partners + // leave the name destination-free rather than picking one arbitrarily. + const partners = new Set(pairs.flatMap( + (pair) => [pair.left, pair.right].filter((name) => name !== camera), + )); + const destination = partners.size === 1 ? [...partners][0] : null; + return { + camera, + destination, + name: registrationFileName(camera, destination), + body: { + type: REGISTRATION_FILE_TYPE, + version: 1, + ...(fileSource ? { source: fileSource } : {}), + pairs, + }, + }; + }); +} + +/** + * Merge the per-file producer stamps of a calibration file set. All stamped + * files agreeing (deep-equal) yields that stamp; disagreement yields a + * composite `{ mixed: true, files: {...} }` so the client can surface a + * mixed-generation warning instead of composing silently -- the failure mode + * per-camera files invite is a rig assembled from files regenerated at + * different times. + */ +export function mergeRegistrationSources( + stamps: { file: string; source: RegistrationSource | null }[], +): RegistrationSource | null { + const stamped = stamps.filter((entry) => entry.source !== null); + if (stamped.length === 0) { + return null; + } + const first = JSON.stringify(stamped[0].source); + if (stamped.every((entry) => JSON.stringify(entry.source) === first)) { + return stamped[0].source; + } + return { + mixed: true, + files: Object.fromEntries(stamps.map((entry) => [entry.file, entry.source])), + }; +} + +/** + * Warning for a registration file naming cameras a dataset doesn't have, + * shared by the desktop and web import seeding paths. Pair bodies are + * authoritative on load, so such pairs import fine but never resolve in the + * Aligned View. A warning rather than a failure: the mismatch may be + * intentional (a rig file shared across datasets) or fixable later. Null + * when every named camera is known. + */ +export function unknownCameraWarning( + fileName: string, + namedCameras: string[], + datasetCameras: string[], +): string | null { + const unknown = [...new Set(namedCameras)] + .filter((name) => !datasetCameras.includes(name)) + .sort(); + if (!unknown.length) { + return null; + } + return `Registration file "${fileName}" names camera(s) not in this dataset: ` + + `${unknown.join(', ')}. Pairs bind by the camera names in the file, so these ` + + 'transforms will not take effect unless camera names match.'; +} + +/** + * Merge a newly imported calibration into a dataset's existing one. Every + * pair the import names replaces that pair wholly (points, transforms, and + * model choice together -- a pair is one artifact); pairs it doesn't name + * are kept, so per-camera files can be imported one at a time to assemble a + * rig. Producer stamps follow the same policy as multi-file loading: + * agreement keeps the stamp, disagreement is recorded as a + * `{ mixed: true, files: {...} }` composite so the client can warn about a + * rig assembled from different calibration generations. + */ +export function mergeRegistrationValues( + existing: CameraRegistrationValues, + incoming: CameraRegistrationValues, + incomingLabel: string, +): CameraRegistrationValues { + const homographies = { ...existing.homographies }; + const correspondences = { ...existing.correspondences }; + const transformTypes = { ...existing.transformTypes }; + const incomingKeys = new Set([ + ...Object.keys(incoming.homographies), + ...Object.keys(incoming.correspondences), + ...Object.keys(incoming.transformTypes), + ]); + incomingKeys.forEach((key) => { + delete homographies[key]; + delete correspondences[key]; + delete transformTypes[key]; + if (incoming.homographies[key]) { + homographies[key] = incoming.homographies[key]; + } + if (incoming.correspondences[key]?.length) { + correspondences[key] = incoming.correspondences[key]; + } + if (incoming.transformTypes[key]) { + transformTypes[key] = incoming.transformTypes[key]; + } + }); + let source: RegistrationSource | null; + if (incoming.source === null) { + source = existing.source; + } else if (existing.source === null + || JSON.stringify(existing.source) === JSON.stringify(incoming.source)) { + source = incoming.source; + } else { + source = { + mixed: true, + files: { previous: existing.source, [incomingLabel]: incoming.source }, + }; + } + return { + homographies, correspondences, transformTypes, source, + }; +} diff --git a/client/src/alignedView/homography.spec.ts b/client/src/alignedView/homography.spec.ts new file mode 100644 index 000000000..9facf0512 --- /dev/null +++ b/client/src/alignedView/homography.spec.ts @@ -0,0 +1,258 @@ +/// +import { + solveHomography, + applyHomography, + invert3, + matMul3, + subdivideWarpQuads, + warpGridSize, + localLinkedScale, + Point, + Matrix3, +} from './homography'; + +function expectMatrixClose(actual: Matrix3, expected: Matrix3, tol = 1e-6) { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + expect(actual[i][j]).toBeCloseTo(expected[i][j], 5); + // tol referenced to keep signature explicit + expect(Math.abs(actual[i][j] - expected[i][j])).toBeLessThan(tol * 10); + } + } +} + +const unitSquare: Point[] = [[0, 0], [1, 0], [1, 1], [0, 1]]; + +describe('homography', () => { + it('recovers the identity from identical correspondences', () => { + const H = solveHomography(unitSquare, unitSquare); + expectMatrixClose(H, [[1, 0, 0], [0, 1, 0], [0, 0, 1]]); + }); + + it('recovers a pure translation', () => { + const dst = unitSquare.map(([x, y]): Point => [x + 5, y - 3]); + const H = solveHomography(unitSquare, dst); + expectMatrixClose(H, [[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); + + it('recovers a scale + translation', () => { + const H = solveHomography(unitSquare, [[10, 10], [30, 10], [30, 30], [10, 30]]); + expectMatrixClose(H, [[20, 0, 10], [0, 20, 10], [0, 0, 1]]); + }); + + it('maps source points onto destination points (least-squares, >4 pts)', () => { + // A known projective transform applied to 5 points; solver should recover it. + const truth: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + const src: Point[] = [[0, 0], [100, 0], [100, 80], [0, 80], [50, 40]]; + const dst = src.map((p) => applyHomography(truth, p)); + const H = solveHomography(src, dst); + src.forEach((p) => { + const [u, v] = applyHomography(H, p); + const [eu, ev] = applyHomography(truth, p); + expect(u).toBeCloseTo(eu, 3); + expect(v).toBeCloseTo(ev, 3); + }); + }); + + it('round-trips through its inverse (H * H^-1 ~= I)', () => { + const H = solveHomography(unitSquare, [[10, 5], [40, 8], [38, 35], [9, 33]]); + const product = matMul3(H, invert3(H)); + const scale = 1 / product[2][2]; + const normalized = product.map((r) => r.map((c) => c * scale)) as Matrix3; + expectMatrixClose(normalized, [[1, 0, 0], [0, 1, 0], [0, 0, 1]]); + }); + + it('throws with fewer than 4 correspondences', () => { + expect(() => solveHomography(unitSquare.slice(0, 3), unitSquare.slice(0, 3))).toThrow(); + }); + + it('throws on a degenerate (collinear) point configuration despite having 4 points', () => { + const collinear: Point[] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + expect(() => solveHomography(collinear, collinear)).toThrow(/degenerate/i); + }); +}); + +describe('warpGridSize', () => { + const affine: Matrix3 = [[1.5, 0.2, 30], [-0.1, 0.9, -12], [0, 0, 1]]; + const projective: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + + it('returns 1 for a pure affine transform', () => { + expect(warpGridSize(affine, 640, 480)).toBe(1); + }); + + it('returns maxN when perspective terms are non-negligible', () => { + expect(warpGridSize(projective, 640, 480)).toBe(8); + expect(warpGridSize(projective, 640, 480, 12)).toBe(12); + }); + + it('returns 1 for negligible perspective terms', () => { + // Perspective terms exist but vary w by only ~0.006% over the extent. + const nearAffine: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [1e-7, 0, 1]]; + expect(warpGridSize(nearAffine, 640, 480)).toBe(1); + }); + + it('returns maxN when the horizon crosses the image (w changes sign)', () => { + const extreme: Matrix3 = [[1, 0, 0], [0, 1, 0], [-0.01, 0, 1]]; + // w at x=0 is 1, at x=640 is -5.4. + expect(warpGridSize(extreme, 640, 480)).toBe(8); + }); +}); + +describe('subdivideWarpQuads', () => { + const projective: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + + it('with n=1 produces a single quad matching the full image corners', () => { + const [quad, ...rest] = subdivideWarpQuads(projective, 640, 480, 1); + expect(rest).toHaveLength(0); + expect(quad.crop).toEqual({ + left: 0, top: 0, right: 640, bottom: 480, + }); + expect(quad.ul).toEqual(applyHomography(projective, [0, 0])); + expect(quad.ur).toEqual(applyHomography(projective, [640, 0])); + expect(quad.lr).toEqual(applyHomography(projective, [640, 480])); + expect(quad.ll).toEqual(applyHomography(projective, [0, 480])); + }); + + it('maps every sub-quad corner through the exact homography', () => { + const n = 8; + const quads = subdivideWarpQuads(projective, 640, 480, n); + expect(quads).toHaveLength(n * n); + quads.forEach((q) => { + expect(q.ul).toEqual(applyHomography(projective, [q.crop.left, q.crop.top])); + expect(q.ur).toEqual(applyHomography(projective, [q.crop.right, q.crop.top])); + expect(q.lr).toEqual(applyHomography(projective, [q.crop.right, q.crop.bottom])); + expect(q.ll).toEqual(applyHomography(projective, [q.crop.left, q.crop.bottom])); + }); + }); + + it('expands cells by the overlap (clamped to the image), corners still exact', () => { + const n = 4; + const overlap = 2; + const plain = subdivideWarpQuads(projective, 640, 480, n); + const padded = subdivideWarpQuads(projective, 640, 480, n, overlap); + expect(padded).toHaveLength(plain.length); + padded.forEach((q, i) => { + const base = plain[i].crop; + expect(q.crop.left).toBe(Math.max(0, base.left - overlap)); + expect(q.crop.right).toBe(Math.min(640, base.right + overlap)); + expect(q.crop.top).toBe(Math.max(0, base.top - overlap)); + expect(q.crop.bottom).toBe(Math.min(480, base.bottom + overlap)); + // Corners remain the exact projective mapping of the (expanded) crop, + // so overlapping regions of adjacent cells render identical content. + expect(q.ul).toEqual(applyHomography(projective, [q.crop.left, q.crop.top])); + expect(q.lr).toEqual(applyHomography(projective, [q.crop.right, q.crop.bottom])); + }); + // Interior edges overlap: cell 0's right crop passes cell 1's left crop. + expect(padded[0].crop.right).toBeGreaterThan(padded[1].crop.left); + }); + + it('tiles the source image exactly, with integer grid lines and no gaps', () => { + const n = 8; + const width = 641; // not divisible by n + const height = 479; + const quads = subdivideWarpQuads(projective, width, height, n); + const lefts = new Set(quads.map((q) => q.crop.left)); + const tops = new Set(quads.map((q) => q.crop.top)); + expect(lefts.size).toBe(n); + expect(tops.size).toBe(n); + quads.forEach((q) => { + [q.crop.left, q.crop.top, q.crop.right, q.crop.bottom].forEach((v) => { + expect(Number.isInteger(v)).toBe(true); + }); + expect(q.crop.right).toBeGreaterThan(q.crop.left); + expect(q.crop.bottom).toBeGreaterThan(q.crop.top); + // Each cell's right/bottom edge is another cell's left/top edge or the border. + expect(q.crop.right === width || lefts.has(q.crop.right)).toBe(true); + expect(q.crop.bottom === height || tops.has(q.crop.bottom)).toBe(true); + }); + // Crop areas sum to the full image area (exact tiling, no overlap/gap). + const area = quads.reduce( + (sum, q) => sum + (q.crop.right - q.crop.left) * (q.crop.bottom - q.crop.top), + 0, + ); + expect(area).toBe(width * height); + }); + + it('skips degenerate zero-area cells for images smaller than the grid', () => { + const quads = subdivideWarpQuads(projective, 3, 3, 8); + expect(quads.length).toBeGreaterThan(0); + quads.forEach((q) => { + expect(q.crop.right).toBeGreaterThan(q.crop.left); + expect(q.crop.bottom).toBeGreaterThan(q.crop.top); + }); + const area = quads.reduce( + (sum, q) => sum + (q.crop.right - q.crop.left) * (q.crop.bottom - q.crop.top), + 0, + ); + expect(area).toBe(9); + }); + + it('sub-quad centers stay close to the true projective warp (approximation quality)', () => { + // The deviation between the piecewise-affine render and the true warp is + // largest in cell interiors; measure it at the center of every cell as the + // distance between the true projective warp of the cell's source center + // and the average of the four warped corners (what an affine-ish renderer + // produces there). + const width = 640; + const height = 480; + const centerError = (quads: ReturnType) => Math.max( + ...quads.map((q) => { + const midSrc: Point = [ + (q.crop.left + q.crop.right) / 2, + (q.crop.top + q.crop.bottom) / 2, + ]; + const truth = applyHomography(projective, midSrc); + const approx: Point = [ + (q.ul[0] + q.ur[0] + q.lr[0] + q.ll[0]) / 4, + (q.ul[1] + q.ur[1] + q.lr[1] + q.ll[1]) / 4, + ]; + return Math.hypot(approx[0] - truth[0], approx[1] - truth[1]); + }), + ); + const singleQuadError = centerError(subdivideWarpQuads(projective, width, height, 1)); + const gridError = centerError(subdivideWarpQuads(projective, width, height, 8)); + // This matrix has strong perspective: a single (parallelogram-rendered) + // quad is tens of pixels off at the image center. + expect(singleQuadError).toBeGreaterThan(10); + // Error shrinks roughly with 1/n^2; at n=8 even this extreme perspective + // (w varies ~2x across the image) is down to a couple of pixels. + expect(gridError).toBeLessThan(3); + expect(gridError).toBeLessThan(singleQuadError / 20); + }); +}); + +describe('localLinkedScale', () => { + const mapper = (matrix: Matrix3) => (p: Point) => applyHomography(matrix, p); + + it('returns 1 for the identity mapping', () => { + const identity: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + expect(localLinkedScale(mapper(identity), [100, 200])).toBeCloseTo(1); + }); + + it('recovers a uniform similarity scale regardless of rotation', () => { + const s = 2.5; + const cos = Math.cos(Math.PI / 6) * s; + const sin = Math.sin(Math.PI / 6) * s; + const similarity: Matrix3 = [[cos, -sin, 10], [sin, cos, -4], [0, 0, 1]]; + expect(localLinkedScale(mapper(similarity), [50, 75])).toBeCloseTo(s); + }); + + it('samples the local scale of a projective transform at the given point', () => { + const homography: Matrix3 = [[1, 0, 0], [0, 1, 0], [0.001, 0, 1]]; + const nearOrigin = localLinkedScale(mapper(homography), [0, 0], 1); + const farRight = localLinkedScale(mapper(homography), [500, 0], 1); + expect(nearOrigin).toBeCloseTo(1, 1); + // At x=500 the perspective divide (w = 1.5) has shrunk the local scale + // well below 1; exact value differs per axis, so just assert the shrink. + expect(farRight).not.toBeNull(); + expect(farRight as number).toBeLessThan(0.7); + }); + + it('returns null when the mapping is unavailable', () => { + expect(localLinkedScale(() => null, [10, 10])).toBeNull(); + }); + + it('returns null for a degenerate (collapsing) mapping', () => { + expect(localLinkedScale(() => [3, 3], [10, 10])).toBeNull(); + }); +}); diff --git a/client/src/alignedView/homography.ts b/client/src/alignedView/homography.ts new file mode 100644 index 000000000..ff09188a7 --- /dev/null +++ b/client/src/alignedView/homography.ts @@ -0,0 +1,309 @@ +/** + * Self-contained homography estimation via the normalized Direct Linear Transform. + * + * Given >= 4 point correspondences src[i] -> dst[i] (both [x, y] in image + * coordinates), {@link solveHomography} returns the 3x3 matrix H such that, in + * homogeneous coordinates, dst ~= H * src. Exact for 4 points, least-squares for + * more. This is the client-side analogue of OpenCV's cv2.findHomography used by + * the keypointgui reference app; the warp itself is done by geojs (quadFeature). + */ + +export type Point = [number, number]; +export type Matrix3 = number[][]; + +/** Multiply two 3x3 matrices. */ +export function matMul3(a: Matrix3, b: Matrix3): Matrix3 { + const out: Matrix3 = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + let sum = 0; + for (let k = 0; k < 3; k += 1) { + sum += a[i][k] * b[k][j]; + } + out[i][j] = sum; + } + } + return out; +} + +/** Invert a 3x3 matrix. Throws if the matrix is singular. */ +export function invert3(m: Matrix3): Matrix3 { + const [a, b, c] = m[0]; + const [d, e, f] = m[1]; + const [g, h, i] = m[2]; + const A = e * i - f * h; + const B = -(d * i - f * g); + const C = d * h - e * g; + const det = a * A + b * B + c * C; + if (Math.abs(det) < 1e-12) { + throw new Error('Cannot invert singular matrix'); + } + const invDet = 1 / det; + return [ + [A * invDet, -(b * i - c * h) * invDet, (b * f - c * e) * invDet], + [B * invDet, (a * i - c * g) * invDet, -(a * f - c * d) * invDet], + [C * invDet, -(a * h - b * g) * invDet, (a * e - b * d) * invDet], + ]; +} + +/** Apply a 3x3 homography to a single point (perspective divide). */ +export function applyHomography(h: Matrix3, p: Point): Point { + const x = h[0][0] * p[0] + h[0][1] * p[1] + h[0][2]; + const y = h[1][0] * p[0] + h[1][1] * p[1] + h[1][2]; + const w = h[2][0] * p[0] + h[2][1] * p[1] + h[2][2]; + return [x / w, y / w]; +} + +/** + * Local scale factor of a point mapping around `center`: how many target-image + * pixels one source-image pixel spans there, estimated by probing `delta` + * pixels along each axis and averaging. For similarity/affine transforms this + * is constant; for homographies it varies with position, which is why it's + * sampled at a specific point (e.g. the current view center for linked + * pan/zoom). Returns null when the mapping is unavailable or degenerate at + * that point. + */ +export function localLinkedScale( + mapPoint: (p: Point) => Point | null, + center: Point, + delta = 10, +): number | null { + const mapped = mapPoint(center); + const mappedX = mapPoint([center[0] + delta, center[1]]); + const mappedY = mapPoint([center[0], center[1] + delta]); + if (!mapped || !mappedX || !mappedY) { + return null; + } + const scaleX = Math.hypot(mappedX[0] - mapped[0], mappedX[1] - mapped[1]) / delta; + const scaleY = Math.hypot(mappedY[0] - mapped[0], mappedY[1] - mapped[1]) / delta; + const scale = (scaleX + scaleY) / 2; + if (!Number.isFinite(scale) || scale <= 0) { + return null; + } + return scale; +} + +/** + * One cell of a subdivided image warp: the axis-aligned source-image rectangle + * `crop` (in source pixels) and the four destination corners it maps to under + * the exact projective transform. See {@link subdivideWarpQuads}. + */ +export interface WarpQuad { + ul: Point; + ur: Point; + lr: Point; + ll: Point; + crop: { left: number; top: number; right: number; bottom: number }; +} + +/** + * Choose a subdivision grid size for rendering the warp of a `width` x `height` + * image through `h` with an affine-only quad renderer (e.g. geojs' canvas + * renderer, which draws each quad from only three of its corners). A pure + * affine matrix (zero perspective row terms) warps exactly as a single + * parallelogram, so 1 is returned; when the perspective terms are + * non-negligible over the image extent, `maxN` is returned so each sub-quad is + * approximately affine. + */ +export function warpGridSize(h: Matrix3, width: number, height: number, maxN = 8): number { + // Homogeneous w at each image corner: constant w <=> affine transform. + const wAt = (x: number, y: number) => h[2][0] * x + h[2][1] * y + h[2][2]; + const ws = [wAt(0, 0), wAt(width, 0), wAt(width, height), wAt(0, height)]; + if (ws.some((w) => !Number.isFinite(w) || w === 0)) { + return maxN; + } + const absW = ws.map((w) => Math.abs(w)); + const maxAbs = Math.max(...absW); + const minAbs = Math.min(...absW); + // Sign change means the horizon line crosses the image: definitely projective. + if (ws.some((w) => w * ws[0] < 0)) { + return maxN; + } + // Relative variation of w across the quad; below ~0.1% the affine + // approximation is visually indistinguishable (sub-pixel for typical sizes). + return (maxAbs - minAbs) / maxAbs < 1e-3 ? 1 : maxN; +} + +/** + * Subdivide the warp of a `width` x `height` image through `h` into an n x n + * grid of {@link WarpQuad}s. Every sub-quad corner is mapped through the exact + * projective transform, so rendering each cell as an (approximately affine) + * textured quad converges to the true projective warp as n grows -- unlike + * rendering the whole image as a single quad, which an affine canvas renderer + * collapses to a parallelogram. Grid lines land on integer source pixels; + * degenerate (zero-area) cells from tiny images are skipped. + * + * `overlap` expands each cell by that many source pixels (clamped to the + * image). The canvas renderer antialiases every quad's border against the + * transparent background, so abutting cells meet as two half-transparent + * edges and show as dark seam lines along the grid; overlapping cells paint + * over each other's seams with pixel-identical content (corners still map + * through the same exact homography). Only for opaque drawing -- translucent + * quads would double-blend in the overlap, so semi-transparent consumers + * must apply opacity at the layer level and draw quads opaque. + */ +export function subdivideWarpQuads( + h: Matrix3, + width: number, + height: number, + n: number, + overlap = 0, +): WarpQuad[] { + const cells = Math.max(1, Math.floor(n)); + const xs: number[] = []; + const ys: number[] = []; + for (let i = 0; i <= cells; i += 1) { + xs.push(Math.round((i * width) / cells)); + ys.push(Math.round((i * height) / cells)); + } + const quads: WarpQuad[] = []; + for (let row = 0; row < cells; row += 1) { + for (let col = 0; col < cells; col += 1) { + const left = Math.max(0, xs[col] - overlap); + const right = Math.min(width, xs[col + 1] + overlap); + const top = Math.max(0, ys[row] - overlap); + const bottom = Math.min(height, ys[row + 1] + overlap); + if (right <= left || bottom <= top) { + // eslint-disable-next-line no-continue + continue; + } + quads.push({ + ul: applyHomography(h, [left, top]), + ur: applyHomography(h, [right, top]), + lr: applyHomography(h, [right, bottom]), + ll: applyHomography(h, [left, bottom]), + crop: { + left, top, right, bottom, + }, + }); + } + } + return quads; +} + +/** + * Hartley normalization: translate points to the centroid and scale so the + * mean distance from the origin is sqrt(2). Returns the normalized points and + * the 3x3 transform T such that normalized = T * original. + */ +function normalizePoints(pts: Point[]): { normalized: Point[]; transform: Matrix3 } { + const n = pts.length; + let cx = 0; + let cy = 0; + pts.forEach(([x, y]) => { cx += x; cy += y; }); + cx /= n; + cy /= n; + let meanDist = 0; + pts.forEach(([x, y]) => { meanDist += Math.hypot(x - cx, y - cy); }); + meanDist /= n; + const scale = meanDist > 1e-12 ? Math.SQRT2 / meanDist : 1; + const transform: Matrix3 = [ + [scale, 0, -scale * cx], + [0, scale, -scale * cy], + [0, 0, 1], + ]; + const normalized = pts.map(([x, y]): Point => [(x - cx) * scale, (y - cy) * scale]); + return { normalized, transform }; +} + +/** + * Solve the linear system A x = b for x using Gaussian elimination with partial + * pivoting. A is square (n x n), modified in place. + */ +/* eslint-disable no-param-reassign */ +export function solveLinearSystem(A: number[][], b: number[]): number[] { + const n = b.length; + for (let col = 0; col < n; col += 1) { + // Partial pivot: find the row with the largest magnitude in this column. + let pivot = col; + for (let row = col + 1; row < n; row += 1) { + if (Math.abs(A[row][col]) > Math.abs(A[pivot][col])) { + pivot = row; + } + } + if (Math.abs(A[pivot][col]) < 1e-12) { + throw new Error('Degenerate point configuration; cannot solve linear system'); + } + if (pivot !== col) { + [A[col], A[pivot]] = [A[pivot], A[col]]; + [b[col], b[pivot]] = [b[pivot], b[col]]; + } + // Eliminate below. + for (let row = col + 1; row < n; row += 1) { + const factor = A[row][col] / A[col][col]; + for (let k = col; k < n; k += 1) { + A[row][k] -= factor * A[col][k]; + } + b[row] -= factor * b[col]; + } + } + // Back-substitution. + const x = new Array(n).fill(0); + for (let row = n - 1; row >= 0; row -= 1) { + let sum = b[row]; + for (let k = row + 1; k < n; k += 1) { + sum -= A[row][k] * x[k]; + } + x[row] = sum / A[row][row]; + } + return x; +} +/* eslint-enable no-param-reassign */ + +/** + * Estimate the homography H (dst ~= H * src) from >= 4 correspondences. + * + * Uses the h33 = 1 formulation in Hartley-normalized coordinates, solved via the + * normal equations (least-squares for > 4 points, exact for 4). Normalization + * keeps the system well-conditioned for typical image-pixel magnitudes. + */ +export function solveHomography(src: Point[], dst: Point[]): Matrix3 { + if (src.length !== dst.length) { + throw new Error('src and dst must have the same number of points'); + } + if (src.length < 4) { + throw new Error('At least 4 point correspondences are required'); + } + + const { normalized: srcN, transform: T1 } = normalizePoints(src); + const { normalized: dstN, transform: T2 } = normalizePoints(dst); + + // Build the 2N x 8 design matrix for the unknowns [h11..h32] (h33 fixed to 1). + const rows: number[][] = []; + const rhs: number[] = []; + for (let i = 0; i < srcN.length; i += 1) { + const [x, y] = srcN[i]; + const [u, v] = dstN[i]; + rows.push([x, y, 1, 0, 0, 0, -x * u, -y * u]); + rhs.push(u); + rows.push([0, 0, 0, x, y, 1, -x * v, -y * v]); + rhs.push(v); + } + + // Normal equations: (A^T A) h = A^T b -> an 8x8 system. + const ata: number[][] = Array.from({ length: 8 }, () => new Array(8).fill(0)); + const atb: number[] = new Array(8).fill(0); + for (let r = 0; r < rows.length; r += 1) { + const row = rows[r]; + for (let i = 0; i < 8; i += 1) { + atb[i] += row[i] * rhs[r]; + for (let j = 0; j < 8; j += 1) { + ata[i][j] += row[i] * row[j]; + } + } + } + + const h = solveLinearSystem(ata, atb); + const hNorm: Matrix3 = [ + [h[0], h[1], h[2]], + [h[3], h[4], h[5]], + [h[6], h[7], 1], + ]; + + // Denormalize: H = inv(T2) * Hnorm * T1. + const denorm = matMul3(matMul3(invert3(T2), hNorm), T1); + + // Scale so H[2][2] == 1 for a canonical form. + const scale = Math.abs(denorm[2][2]) > 1e-12 ? 1 / denorm[2][2] : 1; + return denorm.map((row) => row.map((value) => value * scale)); +} diff --git a/client/src/alignedView/transform.spec.ts b/client/src/alignedView/transform.spec.ts new file mode 100644 index 000000000..ba8d0ab2c --- /dev/null +++ b/client/src/alignedView/transform.spec.ts @@ -0,0 +1,163 @@ +/// +import { applyHomography, solveHomography, Matrix3 } from './homography'; +import { + TransformType, + minPointsForTransform, + estimateTranslation, + estimateRigid, + estimateSimilarity, + estimateAffine, + estimateTransform, + Point, +} from './transform'; + +function expectMatrixClose(actual: Matrix3, expected: Matrix3, precision = 4) { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + expect(actual[i][j]).toBeCloseTo(expected[i][j], precision); + } + } +} + +function expectRoundTrip(h: Matrix3, src: Point[], dst: Point[], precision = 4) { + src.forEach((p, i) => { + const [x, y] = applyHomography(h, p); + expect(x).toBeCloseTo(dst[i][0], precision); + expect(y).toBeCloseTo(dst[i][1], precision); + }); +} + +const fixtureSrc: Point[] = [[0, 0], [10, 0], [10, 10], [0, 10], [5, 3]]; + +describe('minPointsForTransform', () => { + it('returns the keypointgui-matching minimum for each type', () => { + const expected: Record = { + translation: 1, rigid: 2, similarity: 2, affine: 3, homography: 4, + }; + (Object.keys(expected) as TransformType[]).forEach((type) => { + expect(minPointsForTransform(type)).toBe(expected[type]); + }); + }); +}); + +describe('estimateTranslation', () => { + it('recovers an exact translation from a single point', () => { + const H = estimateTranslation([[3, 4]], [[8, 1]]); + expectMatrixClose(H, [[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); + + it('recovers the mean translation from multiple points', () => { + const dst = fixtureSrc.map(([x, y]): Point => [x + 5, y - 3]); + const H = estimateTranslation(fixtureSrc, dst); + expectMatrixClose(H, [[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); +}); + +describe('estimateRigid', () => { + it('recovers a known rotation + translation from 2 points', () => { + const theta = Math.PI / 6; + const cos = Math.cos(theta); + const sin = Math.sin(theta); + const truth: Matrix3 = [[cos, -sin, 5], [sin, cos, -3], [0, 0, 1]]; + const src = fixtureSrc.slice(0, 2); + const dst = src.map((p) => applyHomography(truth, p)); + const H = estimateRigid(src, dst); + expectMatrixClose(H, truth); + // 2x2 block is a proper rotation (unit determinant, no scale/reflection). + const det = H[0][0] * H[1][1] - H[0][1] * H[1][0]; + expect(det).toBeCloseTo(1, 5); + }); + + it('least-squares fits a rigid transform from more than 2 noisy points', () => { + const theta = Math.PI / 4; + const cos = Math.cos(theta); + const sin = Math.sin(theta); + const truth: Matrix3 = [[cos, -sin, 2], [sin, cos, 7], [0, 0, 1]]; + const dst = fixtureSrc.map((p) => applyHomography(truth, p)); + const H = estimateRigid(fixtureSrc, dst); + expectRoundTrip(H, fixtureSrc, dst); + }); +}); + +describe('estimateSimilarity', () => { + it('recovers a known rotation + uniform scale + translation from 2 points', () => { + const theta = Math.PI / 3; + const scale = 2.5; + const cos = Math.cos(theta) * scale; + const sin = Math.sin(theta) * scale; + const truth: Matrix3 = [[cos, -sin, 4], [sin, cos, -6], [0, 0, 1]]; + const src = fixtureSrc.slice(0, 2); + const dst = src.map((p) => applyHomography(truth, p)); + const H = estimateSimilarity(src, dst); + expectMatrixClose(H, truth); + }); + + it('least-squares fits a similarity transform from more points', () => { + const theta = -Math.PI / 5; + const scale = 0.75; + const cos = Math.cos(theta) * scale; + const sin = Math.sin(theta) * scale; + const truth: Matrix3 = [[cos, -sin, -1], [sin, cos, 3], [0, 0, 1]]; + const dst = fixtureSrc.map((p) => applyHomography(truth, p)); + const H = estimateSimilarity(fixtureSrc, dst); + expectRoundTrip(H, fixtureSrc, dst); + }); +}); + +describe('estimateAffine', () => { + const truth: Matrix3 = [[1.2, 0.3, 5], [0.1, 0.9, -4], [0, 0, 1]]; + + it('recovers a known affine transform exactly from 3 non-collinear points', () => { + const src = fixtureSrc.slice(0, 3); + const dst = src.map((p) => applyHomography(truth, p)); + const H = estimateAffine(src, dst); + expectMatrixClose(H, truth); + }); + + it('recovers the same affine transform from more than 3 points', () => { + const dst = fixtureSrc.map((p) => applyHomography(truth, p)); + const H = estimateAffine(fixtureSrc, dst); + expectMatrixClose(H, truth); + }); +}); + +describe('estimateTransform', () => { + it('delegates to solveHomography for the homography type', () => { + const truth: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + const src: Point[] = [[0, 0], [100, 0], [100, 80], [0, 80], [50, 40]]; + const dst = src.map((p) => applyHomography(truth, p)); + expectMatrixClose(estimateTransform('homography', src, dst), solveHomography(src, dst)); + }); + + it('throws below the minimum point count for each type', () => { + const one: Point[] = [[0, 0]]; + const two: Point[] = [[0, 0], [1, 0]]; + expect(() => estimateTransform('translation', [], [])).toThrow(); + expect(() => estimateTransform('rigid', one, one)).toThrow(); + expect(() => estimateTransform('similarity', one, one)).toThrow(); + expect(() => estimateTransform('affine', two, two)).toThrow(); + expect(() => estimateTransform('homography', fixtureSrc.slice(0, 3), fixtureSrc.slice(0, 3))).toThrow(); + }); + + it('throws when src and dst lengths differ', () => { + expect(() => estimateTransform('translation', [[0, 0]], [])).toThrow(); + }); + + it('round-trips src -> dst for every transform type', () => { + const cases: { type: TransformType; truth: Matrix3 }[] = [ + { type: 'translation', truth: [[1, 0, 5], [0, 1, -3], [0, 0, 1]] }, + { type: 'rigid', truth: [[Math.cos(0.4), -Math.sin(0.4), 1], [Math.sin(0.4), Math.cos(0.4), 2], [0, 0, 1]] }, + { + type: 'similarity', + truth: [[1.5 * Math.cos(0.2), -1.5 * Math.sin(0.2), 3], [1.5 * Math.sin(0.2), 1.5 * Math.cos(0.2), 4], [0, 0, 1]], + }, + { type: 'affine', truth: [[1.1, 0.2, 2], [-0.1, 0.95, 6], [0, 0, 1]] }, + { type: 'homography', truth: [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]] }, + ]; + cases.forEach(({ type, truth }) => { + const dst = fixtureSrc.map((p) => applyHomography(truth, p)); + const H = estimateTransform(type, fixtureSrc, dst); + expectRoundTrip(H, fixtureSrc, dst, 3); + }); + }); +}); diff --git a/client/src/alignedView/transform.ts b/client/src/alignedView/transform.ts new file mode 100644 index 000000000..34461b272 --- /dev/null +++ b/client/src/alignedView/transform.ts @@ -0,0 +1,180 @@ +/** + * Alignment transform models for camera calibration, layered on top of the matrix + * primitives in {@link "./homography"} (`Matrix3`, `applyHomography`, `invert3`, + * `solveLinearSystem`). Mirrors keypointgui's `fit_homography` transform types + * (translation / rigid / similarity / affine / homography) so near-rigid EO/IR + * rigs can be fit with fewer, more stable point pairs than a full homography + * requires. Every estimator returns a plain `Matrix3`, so callers (warping, + * inverse-mapping) never need to special-case the transform type. + */ + +import { + Point, Matrix3, solveLinearSystem, solveHomography, +} from './homography'; + +export type TransformType = 'translation' | 'rigid' | 'similarity' | 'affine' | 'homography'; + +/** UI-friendly ordered list of transform types, for dropdowns. */ +export const TRANSFORM_TYPES: { value: TransformType; text: string }[] = [ + { value: 'translation', text: 'Translation' }, + { value: 'rigid', text: 'Rigid' }, + { value: 'similarity', text: 'Similarity' }, + { value: 'affine', text: 'Affine' }, + { value: 'homography', text: 'Homography' }, +]; + +/** + * The transform type assumed for a pair that has no explicit choice yet. Near- + * rigid EO/IR rigs fit stably from few points with a similarity model, so it is + * the sensible default the UI shows. MUST be the single source of this default + * everywhere a pair's type is resolved (the store, the panel, and both platform + * persistence layers), so a pair fitted at the default resolves to the same + * model after a save/reload on every platform. + */ +export const DEFAULT_TRANSFORM_TYPE: TransformType = 'similarity'; + +const MIN_POINTS: Record = { + translation: 1, + rigid: 2, + similarity: 2, + affine: 3, + homography: 4, +}; + +/** Minimum correspondence count to fit `type`, matching keypointgui's fit_homography. */ +export function minPointsForTransform(type: TransformType): number { + return MIN_POINTS[type]; +} + +/** Pure translation: H = identity with translation = mean(dst - src). Exact for 1+ points. */ +export function estimateTranslation(src: Point[], dst: Point[]): Matrix3 { + const n = src.length; + let dx = 0; + let dy = 0; + for (let i = 0; i < n; i += 1) { + dx += dst[i][0] - src[i][0]; + dy += dst[i][1] - src[i][1]; + } + dx /= n; + dy /= n; + return [[1, 0, dx], [0, 1, dy], [0, 0, 1]]; +} + +/** + * Closed-form 2D Procrustes/Umeyama fit: rotation (plus optional uniform scale) + * and translation minimizing sum |R*src + t - dst|^2. SVD-free because a 2D + * rotation is a single angle: `theta = atan2(B, A)` is the angle that maximizes + * the projected alignment sum, and (for similarity) the optimal uniform scale is + * the ratio of that alignment's magnitude to the source points' spread. + */ +function estimateRotationScaleTranslation( + src: Point[], + dst: Point[], + allowScale: boolean, +): Matrix3 { + const n = src.length; + let csx = 0; + let csy = 0; + let cdx = 0; + let cdy = 0; + for (let i = 0; i < n; i += 1) { + csx += src[i][0]; + csy += src[i][1]; + cdx += dst[i][0]; + cdy += dst[i][1]; + } + csx /= n; + csy /= n; + cdx /= n; + cdy /= n; + + let a = 0; + let b = 0; + let srcVar = 0; + for (let i = 0; i < n; i += 1) { + const sx = src[i][0] - csx; + const sy = src[i][1] - csy; + const dx = dst[i][0] - cdx; + const dy = dst[i][1] - cdy; + a += sx * dx + sy * dy; + b += sx * dy - sy * dx; + srcVar += sx * sx + sy * sy; + } + + const theta = Math.atan2(b, a); + const scale = allowScale && srcVar > 1e-12 ? Math.hypot(a, b) / srcVar : 1; + const r00 = Math.cos(theta) * scale; + const r01 = -Math.sin(theta) * scale; + const r10 = Math.sin(theta) * scale; + const r11 = Math.cos(theta) * scale; + const tx = cdx - (r00 * csx + r01 * csy); + const ty = cdy - (r10 * csx + r11 * csy); + return [[r00, r01, tx], [r10, r11, ty], [0, 0, 1]]; +} + +/** Rotation + translation only (no scale, no shear). Matches keypointgui's rigid fit. */ +export function estimateRigid(src: Point[], dst: Point[]): Matrix3 { + return estimateRotationScaleTranslation(src, dst, false); +} + +/** Rotation + uniform scale + translation. Matches keypointgui's similarity fit. */ +export function estimateSimilarity(src: Point[], dst: Point[]): Matrix3 { + return estimateRotationScaleTranslation(src, dst, true); +} + +/** + * Full 6-DOF affine fit via two independent 3-unknown least-squares solves (one + * output row at a time): dst_x = a*src_x + b*src_y + tx, dst_y = c*src_x + d*src_y + * + ty. Exact for 3 non-collinear points, least-squares for more. + */ +export function estimateAffine(src: Point[], dst: Point[]): Matrix3 { + const n = src.length; + const ata: number[][] = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; + const atbX: number[] = [0, 0, 0]; + const atbY: number[] = [0, 0, 0]; + for (let i = 0; i < n; i += 1) { + const row = [src[i][0], src[i][1], 1]; + const dx = dst[i][0]; + const dy = dst[i][1]; + for (let r = 0; r < 3; r += 1) { + atbX[r] += row[r] * dx; + atbY[r] += row[r] * dy; + for (let c = 0; c < 3; c += 1) { + ata[r][c] += row[r] * row[c]; + } + } + } + // solveLinearSystem mutates its arguments in place, so each solve gets its own copy. + const rowX = solveLinearSystem(ata.map((row) => [...row]), [...atbX]); + const rowY = solveLinearSystem(ata.map((row) => [...row]), [...atbY]); + return [ + [rowX[0], rowX[1], rowX[2]], + [rowY[0], rowY[1], rowY[2]], + [0, 0, 1], + ]; +} + +/** Estimate a `Matrix3` mapping src -> dst using `type`, enforcing its minimum point count. */ +export function estimateTransform(type: TransformType, src: Point[], dst: Point[]): Matrix3 { + if (src.length !== dst.length) { + throw new Error('src and dst must have the same number of points'); + } + const required = minPointsForTransform(type); + if (src.length < required) { + throw new Error(`At least ${required} point pair(s) are required for a ${type} transform`); + } + switch (type) { + case 'translation': + return estimateTranslation(src, dst); + case 'rigid': + return estimateRigid(src, dst); + case 'similarity': + return estimateSimilarity(src, dst); + case 'affine': + return estimateAffine(src, dst); + case 'homography': + return solveHomography(src, dst); + default: + throw new Error(`Unknown transform type: ${type}`); + } +} diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index a50ab44df..834709ba4 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -20,9 +20,6 @@ import TextLayer, { FormatTextRow } from '../layers/AnnotationLayers/TextLayer'; import AttributeLayer from '../layers/AnnotationLayers/AttributeLayer'; import AttributeBoxLayer from '../layers/AnnotationLayers/AttributeBoxLayer'; import type { AnnotationId } from '../BaseAnnotation'; -import { - geojsonToBound, isRotationValue, ROTATION_ATTRIBUTE_NAME, featureHasSegmentationPolygon, -} from '../utils'; import { VisibleAnnotationTypes } from '../layers'; import UILayer from '../layers/UILayers/UILayer'; import ToolTipWidget from '../layers/UILayers/ToolTipWidget.vue'; @@ -39,6 +36,7 @@ import { useAnnotatorPreferences, useGroupStyleManager, useCameraStore, + useAlignedView, useSelectedCamera, useAttributes, useComparisonSets, @@ -46,6 +44,11 @@ import { useSegmentationPoints, } from '../provides'; import SegmentationPointsLayer from '../layers/AnnotationLayers/SegmentationPointsLayer'; +import useLayerManagerAlignedView from './layerManager/useLayerManagerAlignedView'; +import useLayerRefresh from './layerManager/useLayerRefresh'; +import useSegmentationPointsLayer from './layerManager/useSegmentationPointsLayer'; +import useAnnotationClickHandling from './layerManager/useAnnotationClickHandling'; +import { cameraAwaitingGeometry, isCreatingNewDetection } from './layerManager/multicamCreation'; /** LayerManager is a component intended to be used as a child of an Annotator. * It provides logic for switching which layers are visible, but more importantly @@ -77,6 +80,12 @@ export default defineComponent({ // Viewer may not provide lasso context in tests or minimal embeds. } const cameraStore = useCameraStore(); + let alignedView: ReturnType | undefined; + try { + alignedView = useAlignedView(); + } catch { + // aligned view store may not be provided in tests or minimal embeds. + } const selectedCamera = useSelectedCamera(); const comparison = useComparisonSets(); const trackStore = cameraStore.camMap.value.get(props.camera)?.trackStore; @@ -107,6 +116,20 @@ export default defineComponent({ const flickNumberRef = annotator.flick; const hasFrameRef = annotator.hasFrame; + const { + alignedDisplayTransform, + mapDisplayPoint, + featureToDisplay, + setupDisplayTransformWatches, + ...alignedViewHelpers + } = useLayerManagerAlignedView({ + camera: props.camera, + annotator, + aggregateController, + alignedView, + editingModeRef, + }); + const rectAnnotationLayer = new RectangleLayer({ annotator, stateStyling: trackStyleManager.stateStyles, @@ -179,20 +202,6 @@ export default defineComponent({ const segmentationPointsRef = useSegmentationPoints(); const segmentationPointsLayer = new SegmentationPointsLayer(annotator); - // Watch for segmentation points updates - only show points for the current - // frame, and only on the selected camera. The prompt points belong to the - // image being segmented; without the camera check every camera rendered - // them at the same pixel coordinates, which looked like an (unwarped) - // point being created on the other camera. - watch([segmentationPointsRef, frameNumberRef, selectedCamera], ([newPoints, currentFrame, currentCamera]) => { - if (newPoints.points.length > 0 && newPoints.frameNum === currentFrame - && props.camera === currentCamera) { - segmentationPointsLayer.updatePoints(newPoints.points, newPoints.labels); - } else { - segmentationPointsLayer.clear(); - } - }, { deep: true }); - const updateAttributes = () => { const newList = attributes.value.filter((item) => item.render).sort((a, b) => { if (a.render && b.render) { @@ -215,45 +224,14 @@ export default defineComponent({ }; uiLayer.addDOMWidget('customToolTip', ToolTipWidget, toolTipWidgetProps, { x: 10, y: 10 }); - // True when the selected track is a brand-new detection with no geometry yet - // on this camera at this frame (the user is mid-creation). Used to mirror the - // creation cursor onto non-selected cameras so a new detection can be drawn - // on any camera. Must look up the track on props.camera (same as - // cameraAwaitingGeometry): `frame` is this layer's local frame, and under an - // aligned timeline that diverges from selectedCamera's local frame. - function isCreatingNewDetection(frame: number, trackId: AnnotationId | null): boolean { - if (trackId === null) return false; - const t = cameraStore.getPossibleTrack(trackId, props.camera); - if (!t) return false; - return t.getFeature(frame)[0] == null; - } - - // True when, while editing, the selected track has no geometry on THIS - // camera at this frame (the track may not exist on this camera at all). - // For Point mode (point-click segmentation) and Polygon mode, "no geometry" - // means no polygon at the selected key here yet, so a detection that only - // has a box still accepts a draw. - // Complements isCreatingNewDetection: after the detection is drawn on one - // camera, the creation cursor stays live on the cameras still missing it, - // so it can be drawn on each in turn (same track id) without selecting the - // camera first. - function cameraAwaitingGeometry( - frame: number, - trackId: AnnotationId | null, - editingTrack: false | EditAnnotationTypes, - selectedKey: string, - ): boolean { - if (trackId === null || !editingTrack) return false; - if (!cameraStore.getAnyPossibleTrack(trackId)) return false; - const t = cameraStore.getPossibleTrack(trackId, props.camera); - if (!t) return true; - const [feature] = t.getFeature(frame); - if (feature == null) return true; - if (editingTrack === 'Point' || editingTrack === 'Polygon') { - return !featureHasSegmentationPolygon(feature, selectedKey); - } - return false; - } + useSegmentationPointsLayer({ + camera: props.camera, + segmentationPointsRef, + frameNumberRef, + selectedCamera, + segmentationPointsLayer, + mapDisplayPoint, + }); function updateLayers( frame: number, @@ -265,6 +243,11 @@ export default defineComponent({ selectedKey: string, colorBy: string, ) { + // Drawing and editing work on every camera while the aligned view is + // on: the edit layer operates in display (warped) space -- it is fed + // display-space copies of the geometry (featureToDisplay below) and its + // draws/edits are mapped back to native through alignedDisplayInverse + // in the update:geojson handler before committing to track storage. const currentFrameIds: AnnotationId[] | undefined = trackStore?.intervalTree .search([frame, frame]) .map((str) => parseInt(str, 10)); @@ -323,9 +306,13 @@ export default defineComponent({ } if (clientSettings.annotatorPreferences.lockedCamera.enabled) { if (trackFrame.features?.bounds) { + // Under the aligned view the display is warped, so center + // on the displayed (warped) location, not the native one. const coords = { - x: (trackFrame.features.bounds[0] + trackFrame.features.bounds[2]) / 2.0, - y: (trackFrame.features.bounds[1] + trackFrame.features.bounds[3]) / 2.0, + ...mapDisplayPoint( + (trackFrame.features.bounds[0] + trackFrame.features.bounds[2]) / 2.0, + (trackFrame.features.bounds[1] + trackFrame.features.bounds[3]) / 2.0, + ), z: 0, }; const [x0, y0, x1, y1] = trackFrame.features.bounds; @@ -344,10 +331,14 @@ export default defineComponent({ const halfWidth = (width * multiplyBoundsVal) / 2.0; const halfHeight = (height * multiplyBoundsVal) / 2.0; - const left = centerX - halfWidth; - const right = centerX + halfWidth; - const top = centerY - halfHeight; - const bottom = centerY + halfHeight; + // Map the zoom-target corners into display space too + // (identity when the aligned view is off). + const ulMapped = mapDisplayPoint(centerX - halfWidth, centerY - halfHeight); + const lrMapped = mapDisplayPoint(centerX + halfWidth, centerY + halfHeight); + const left = Math.min(ulMapped.x, lrMapped.x); + const right = Math.max(ulMapped.x, lrMapped.x); + const top = Math.min(ulMapped.y, lrMapped.y); + const bottom = Math.max(ulMapped.y, lrMapped.y); const zoomAndCenter = annotator.geoViewerRef.value.zoomAndCenterFromBounds({ left, top, right, bottom, @@ -387,7 +378,11 @@ export default defineComponent({ } else { lineLayer.disable(); } - if (visibleModes.includes('TrackTail')) { + // Track tails read multi-frame geometry straight from the trackStore + // (not FrameDataTrack) and are not routed through the display + // transform, so they are hidden for warped cameras while the aligned + // view is on rather than rendered in the wrong (native) space. + if (visibleModes.includes('TrackTail') && !alignedDisplayTransform.value) { tailLayer.updateSettings( frame, annotatorPrefs.value.trackTails.before, @@ -435,11 +430,29 @@ export default defineComponent({ if (editingTrack) { editAnnotationLayer.setType(editingTrack); editAnnotationLayer.setKey(selectedKey); - editAnnotationLayer.changeData(editingTracks); + // The edit layer works in display space: hand it display-space + // copies of the feature so its handles land on warped imagery + // (identity when this camera renders unwarped). + editAnnotationLayer.changeData(editingTracks.map((trackFrame) => ({ + ...trackFrame, + features: featureToDisplay(trackFrame.features), + }))); } } else if (editingTrack && props.camera !== selectedCamera.value - && (isCreatingNewDetection(frame, selectedTrackId) - || cameraAwaitingGeometry(frame, selectedTrackId, editingTrack, selectedKey))) { + && (isCreatingNewDetection( + cameraStore, + props.camera, + frame, + selectedTrackId, + ) + || cameraAwaitingGeometry( + cameraStore, + props.camera, + frame, + selectedTrackId, + editingTrack, + selectedKey, + ))) { // Seamless multicam creation: keep the creation cursor live on every // camera (not just the selected one) so a brand-new detection can be // drawn on whichever camera the user starts on -- and, once drawn on @@ -457,57 +470,47 @@ export default defineComponent({ } } - /** - * Disables every annotation/edit layer without touching stored track - * data. Used when this camera has no frame at the current aligned- - * timeline slot (hasFrameRef false) -- frameNumberRef still holds the - * last real local frame, so leaving the layers as-is would draw stale - * boxes over a blank pane. - */ - function disableAllLayers() { - rectAnnotationLayer.disable(); - overlapLayer.disable(); - polyAnnotationLayer.disable(); - lineLayer.disable(); - pointLayer.disable(); - tailLayer.disable(); - textLayer.disable(); - attributeLayer.disable(); - attributeBoxLayer.disable(); - editAnnotationLayer.disable(); - segmentationPointsLayer.clear(); - hoverOvered.value = []; - uiLayer.setToolTipWidget('customToolTip', false); - } - - /** Re-runs updateLayers with current ref values, or blanks the layers when this camera has no frame at the current aligned-timeline slot. */ - function refreshLayers() { - if (!hasFrameRef.value) { - disableAllLayers(); - return; - } - updateLayers( - frameNumberRef.value, - editingModeRef.value, - selectedTrackIdRef.value, - multiSeletListRef.value, - enabledTracksRef.value, - visibleModesRef.value, - selectedKeyRef.value, - props.colorBy, - ); - } + const { refreshLayers } = useLayerRefresh({ + hasFrameRef, + frameNumberRef, + editingModeRef, + selectedTrackIdRef, + multiSelectListRef: multiSeletListRef, + enabledTracksRef, + visibleModesRef, + selectedKeyRef, + colorBy: toRef(props, 'colorBy'), + layers: { + rectAnnotationLayer, + overlapLayer, + polyAnnotationLayer, + lineLayer, + pointLayer, + tailLayer, + textLayer, + attributeLayer, + attributeBoxLayer, + editAnnotationLayer, + segmentationPointsLayer, + uiLayer, + }, + hoverOvered, + updateLayers, + }); - watch(hasFrameRef, () => refreshLayers()); + /** Layers whose stored-geometry rendering follows the aligned-view warp. */ + const displayTransformedLayers = [ + rectAnnotationLayer, + overlapLayer, + polyAnnotationLayer, + lineLayer, + pointLayer, + textLayer, + attributeBoxLayer, + attributeLayer, + ]; - /** - * TODO: for some reason, GeoJS requires us to initialize - * by calling the render function twice. This is a bug. - * https://github.com/Kitware/dive/issues/365 - */ - [1, 2].forEach(() => { - refreshLayers(); - }); + setupDisplayTransformWatches(displayTransformedLayers, refreshLayers, frameNumberRef); /** Shallow watch */ watch( @@ -564,255 +567,26 @@ export default defineComponent({ }, ); - // Set briefly when a draw finalizes so the same click — e.g. the 2nd line - // vertex / rectangle corner landing on an overlapping existing detection — - // doesn't also select that detection. Cleared on the next macrotask, so only - // the click that completed the draw is suppressed. - let justFinalizedCreation = false; - - // Guards against a single physical click being handled more than once when - // it lands on overlapping features on different layers. A skinny box hugging - // its head/tail line is the common case: the click hits both the rectangle - // polygon and the line, so both layers emit annotation-(right-)clicked in - // the same tick. Because trackEdit toggles edit mode, the second emit would - // immediately undo the first (right-click enters edit, then leaves it, - // ending up merely selected). Cleared on the next macrotask, so distinct - // user clicks (separate ticks) are unaffected. - let clickHandledThisTick = false; - - const Clicked = (trackId: number, editing: boolean, modifiers?: {ctrl: boolean}) => { - // The click that just finalized a new detection should not also select an - // existing detection underneath the final vertex. - if (justFinalizedCreation) { - return; - } - if (clickHandledThisTick) { - return; - } - clickHandledThisTick = true; - window.setTimeout(() => { clickHandledThisTick = false; }, 0); - // Clicking a detection in a camera that isn't selected yet: switch to that - // camera AND act on the detection in the same click — left-click selects, - // right-click edits — instead of requiring a separate click to switch first. - if (selectedCamera.value !== props.camera) { - // Mid new-detection creation, a left-click on this camera is the start of - // a draw (handled by the creation layer + update routing) — don't let it - // select an existing detection under the first corner. Right-click still - // switches to editing the clicked detection. - if (editAnnotationLayer.getMode() === 'creation' && !editing) { - return; - } - if (trackId !== null) { - handler.selectCamera(props.camera, false); - // selectCamera switches synchronously in the normal path; bail if a mode - // (e.g. linking) blocked the switch so we don't act on the wrong camera. - if (selectedCamera.value === props.camera) { - handler.trackSelect(trackId, false, modifiers); - if (editing) { - handler.trackEdit(trackId); - } - } - } - return; - } - //So we only want to pass the click whjen not in creation mode or editing mode for features - if (editAnnotationLayer.getMode() !== 'creation') { - editAnnotationLayer.disable(); - // When entering editing mode (right-click), use trackEdit so the - // geometry type is auto-detected (e.g. LineString vs rectangle). - if (editing && trackId !== null) { - handler.trackEdit(trackId); - } else { - handler.trackSelect(trackId, editing, modifiers); - } - } else if (editing && trackId !== null) { - // Right-click on another detection while in creation mode: - // cancel creation and switch to editing the clicked detection - editAnnotationLayer.disable(); - handler.trackEdit(trackId); - } - }; - - //Sync of internal geoJS state with the application - editAnnotationLayer.bus.$on('editing-annotation-sync', (editing: boolean, deselect?: boolean) => { - if (deselect) { - handler.trackSelect(null, false); - } else { - handler.trackSelect(selectedTrackIdRef.value, editing); - } - }); - // Handle right-click to confirm/lock annotation in Point mode (segmentation) - editAnnotationLayer.bus.$on('confirm-annotation', () => { - handler.confirmRecipe(); - }); - // Register callback so pressing 'n' (new detection) finalizes in-progress shapes - handler.registerFinalizeCreation(() => { - editAnnotationLayer.finalizeInProgress(); - }); - rectAnnotationLayer.bus.$on('annotation-clicked', Clicked); - rectAnnotationLayer.bus.$on('annotation-right-clicked', Clicked); - rectAnnotationLayer.bus.$on('annotation-ctrl-clicked', Clicked); - polyAnnotationLayer.bus.$on('annotation-clicked', Clicked); - polyAnnotationLayer.bus.$on('annotation-right-clicked', Clicked); - // Handle right-click polygon selection for multi-polygon support - polyAnnotationLayer.bus.$on('polygon-right-clicked', (_trackId: number, polygonKey: string) => { - // If in creation mode, cancel it first so we can select the polygon - if (editAnnotationLayer.getMode() === 'creation') { - handler.cancelCreation(); - } - // Set the polygon key for the right-clicked polygon - handler.selectFeatureHandle(-1, polygonKey); - // Force layer update to load the selected polygon - // This is especially important when already editing the same track - // since annotation-right-clicked won't be emitted in that case - window.setTimeout(() => { - refreshLayers(); - }, 0); - }); - polyAnnotationLayer.bus.$on('annotation-ctrl-clicked', Clicked); - lineLayer.bus.$on('annotation-clicked', Clicked); - lineLayer.bus.$on('annotation-right-clicked', Clicked); - // Handle polygon selection for multi-polygon support - polyAnnotationLayer.bus.$on('polygon-clicked', (_trackId: number, polygonKey: string) => { - // If in creation mode, don't interrupt - let the edit layer handle clicks for placing points - // This is important for hole drawing where left-clicks place hole vertices - if (editAnnotationLayer.getMode() === 'creation') { - return; - } - handler.selectFeatureHandle(-1, polygonKey); - // Force layer update to load the newly selected polygon - // Use nextTick to ensure the selectedKey ref has been updated - window.setTimeout(() => { - refreshLayers(); - }, 0); - }); - // Handle right-click outside polygons to finalize/cancel creation - polyAnnotationLayer.bus.$on('polygon-right-clicked-outside', () => { - if (editAnnotationLayer.getMode() === 'creation') { - // Cancel creation and go back to editing the default polygon - handler.cancelCreation(); - handler.selectFeatureHandle(-1, ''); - window.setTimeout(() => { - refreshLayers(); - }, 0); - } - }); - editAnnotationLayer.bus.$on('update:geojson', ( - mode: 'in-progress' | 'editing', - geometryCompleteEvent: boolean, - data: GeoJSON.Feature, - type: string, - key = '', - cb: () => void = () => (undefined), - ) => { - // Seamless multicam creation: a draw that lands on a camera that isn't the - // selected one must commit on THIS camera. - if (props.camera !== selectedCamera.value - && (cameraAwaitingGeometry(frameNumberRef.value, selectedTrackIdRef.value, editingModeRef.value, selectedKeyRef.value) - || isCreatingNewDetection(frameNumberRef.value, selectedTrackIdRef.value))) { - // Draw landed on a camera that is not selected. Create the same-id - // track here when needed, then switch so the update below commits on - // this camera under the current track id. Check cameraAwaitingGeometry - // before isCreatingNewDetection: once an empty track exists on this - // camera the latter is also true, but trackAdd() would deselect and - // start a new id. For Point mode, selectCamera finalizes the source - // camera's pending mask and clears recipe prompt points. - const trackId = selectedTrackIdRef.value as number; - if (!cameraStore.getPossibleTrack(trackId, props.camera)) { - const anyTrack = cameraStore.getAnyPossibleTrack(trackId); - const trackType = anyTrack?.confidencePairs?.[0]?.[0] || 'unknown'; - trackStore.add(frameNumberRef.value, trackType, undefined, trackId); - } - // preserveSelection: the source camera's track may have no geometry yet - // (the draw is happening here), so selectCamera must not abort it and - // null selectedTrackId — the in-progress draw commits under this id. - handler.selectCamera(props.camera, false, true); - if (editingModeRef.value === 'Point' && selectedCamera.value !== props.camera) { - // The switch was blocked (e.g. linking mode): a segmentation point - // clicked on this camera must never be added to the selected - // camera's prompt points, so drop the click. - return; - } - } else if (props.camera !== selectedCamera.value - && editingModeRef.value && selectedTrackIdRef.value !== null - && cameraStore.getPossibleTrack(selectedTrackIdRef.value, props.camera)) { - // Editing the selected track's existing geometry on this camera - // (edit handles are live on every camera showing the track): switch - // so the edit commits to THIS camera's track. Normally the mousedown - // that grabbed the handle already switched via Viewer.changeCamera; - // this is the fallback when the update lands first. - handler.selectCamera(props.camera, false); - if (selectedCamera.value !== props.camera) { - // Switch blocked: never commit this camera's edit to the selected - // camera's track. - return; - } - } - if (type === 'rectangle') { - const bounds = geojsonToBound(data as GeoJSON.Feature); - // Extract rotation from properties if it exists - const rotation = data.properties && isRotationValue(data.properties?.[ROTATION_ATTRIBUTE_NAME]) - ? data.properties[ROTATION_ATTRIBUTE_NAME] as number - : undefined; - cb(); - handler.updateRectBounds(frameNumberRef.value, flickNumberRef.value, bounds, rotation); - } else { - handler.updateGeoJSON(mode, frameNumberRef.value, flickNumberRef.value, data, key, cb); - } - // Jump into edit mode if we completed a new shape - if (geometryCompleteEvent) { - // Suppress the select that rides along with this same finalizing click. - justFinalizedCreation = true; - window.setTimeout(() => { justFinalizedCreation = false; }, 0); - refreshLayers(); - } + const { wireHandlers } = useAnnotationClickHandling({ + camera: props.camera, + handler, + selectedCamera, + selectedTrackIdRef, + selectedKeyRef, + frameNumberRef, + flickNumberRef, + editingModeRef, + cameraStore, + trackStore, + alignedView: alignedViewHelpers, + editAnnotationLayer, + rectAnnotationLayer, + polyAnnotationLayer, + lineLayer, + refreshLayers, }); - editAnnotationLayer.bus.$on( - 'update:selectedIndex', - (index: number, _type: EditAnnotationTypes, key?: string) => { - // When deselecting (index -1), don't change the key - it may have been - // set by polygon-right-clicked/polygon-clicked for multi-polygon selection - if (index >= 0 && key !== undefined) { - handler.selectFeatureHandle(index, key); - } else { - // Just update the handle index, preserve the current key - handler.selectFeatureHandle(index, selectedKeyRef.value); - } - }, - ); - // Handle clicks outside the edit polygon to allow selecting other polygons - editAnnotationLayer.bus.$on('click-outside-edit', (geo: { x: number; y: number }) => { - // Check which polygon was clicked by iterating through formatted data - const point: [number, number] = [geo.x, geo.y]; - const polygonData = polyAnnotationLayer.formattedData; - - // Find the polygon that contains the click point - const clickedPolygon = polygonData.find((data) => { - const coords = data.polygon.coordinates[0] as [number, number][]; - // Ray casting algorithm - let inside = false; - for (let i = 0, j = coords.length - 1; i < coords.length; j = i, i += 1) { - const xi = coords[i][0]; - const yi = coords[i][1]; - const xj = coords[j][0]; - const yj = coords[j][1]; - const intersect = ((yi > point[1]) !== (yj > point[1])) - && (point[0] < ((xj - xi) * (point[1] - yi)) / (yj - yi) + xi); - if (intersect) inside = !inside; - } - return inside; - }); + wireHandlers(); - if (clickedPolygon) { - const polygonKey = clickedPolygon.polygonKey || ''; - // Select the clicked polygon - handler.selectFeatureHandle(-1, polygonKey); - // Force layer update to load the newly selected polygon - window.setTimeout(() => { - refreshLayers(); - }, 0); - } - }); const annotationHoverTooltip = ( found: { styleType: [string, number]; diff --git a/client/src/components/annotators/ImageAnnotator.vue b/client/src/components/annotators/ImageAnnotator.vue index 2d8f414a9..7ff1bfd48 100644 --- a/client/src/components/annotators/ImageAnnotator.vue +++ b/client/src/components/annotators/ImageAnnotator.vue @@ -157,6 +157,7 @@ export default defineComponent({ }, ]) .draw(); + data.imageRevision += 1; } /** * Adds a single frame to the pendingImgs array for loading and assigns it to the main @@ -358,6 +359,7 @@ export default defineComponent({ } else { local.quadFeature.layer().node().css('filter', `url(#${props.filterId})`); } + data.imageRevision += 1; } }, { deep: true }, diff --git a/client/src/components/annotators/README.md b/client/src/components/annotators/README.md index 7ba8d12d3..68952b6d4 100644 --- a/client/src/components/annotators/README.md +++ b/client/src/components/annotators/README.md @@ -19,3 +19,11 @@ export default defineComponent({ }, }); ``` + +## Linked viewer navigation + +Multicam pan/zoom linking lives in `./linkedViewers/`. See that folder's [README](./linkedViewers/README.md) for how `useLinkedViewers` and `useAlignedNavigation` work, and how they relate to the aligned view and raw camera sync. + +```typescript +import { useAlignedNavigation } from 'vue-media-annotator/components'; +``` diff --git a/client/src/components/annotators/VideoAnnotator.vue b/client/src/components/annotators/VideoAnnotator.vue index aea42d6e3..9e2b58282 100644 --- a/client/src/components/annotators/VideoAnnotator.vue +++ b/client/src/components/annotators/VideoAnnotator.vue @@ -243,6 +243,7 @@ export default defineComponent({ } else { quadFeatureLayer.node().css('filter', `url(#${props.filterId})`); } + data.imageRevision += 1; } }, { deep: true }, @@ -284,6 +285,9 @@ export default defineComponent({ }, ]) .draw(); + // The