From 7db36670f59249bc90c6ef3f44aca1e9ce00ba57 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Thu, 9 Jul 2026 16:17:20 -0400 Subject: [PATCH 01/13] Add interactive calibration state, keypoint markers, and pair link Extend CameraCalibrationStore with the Manual Alignment creation state and operations: active pair, point picking (add/pick/move/drag-refine, select/delete, clear last/pair), transform fitting (fitTransform / maybeFitPair via the transform estimators), the ghost-overlay alignment mode/opacity, linked-nav toggle, and cursor/recenter plumbing. CalibrationKeypointLayer renders the picked correspondences (locked blue / unlocked yellow, exact-pixel markers, click-select and drag) and the cross-pane ghost warp preview; CameraImage moves here with it. useCalibrationNavigation links the active pair's panes through the fitted homography while picking (standing down under the Align View). Co-Authored-By: Claude Fable 5 --- .../CameraRegistrationStore.spec.ts | 707 +++++++++++++++--- .../alignedView/CameraRegistrationStore.ts | 427 ++++++++++- .../useCalibrationNavigation.spec.ts | 140 ++++ .../linkedViewers/useCalibrationNavigation.ts | 121 +++ client/src/components/index.ts | 1 + .../CalibrationKeypointLayer.ts | 513 +++++++++++++ 6 files changed, 1816 insertions(+), 93 deletions(-) create mode 100644 client/src/components/annotators/linkedViewers/useCalibrationNavigation.spec.ts create mode 100644 client/src/components/annotators/linkedViewers/useCalibrationNavigation.ts create mode 100644 client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts diff --git a/client/src/alignedView/CameraRegistrationStore.spec.ts b/client/src/alignedView/CameraRegistrationStore.spec.ts index 8b3e52834..5fc6b746d 100644 --- a/client/src/alignedView/CameraRegistrationStore.spec.ts +++ b/client/src/alignedView/CameraRegistrationStore.spec.ts @@ -2,35 +2,286 @@ import CameraRegistrationStore from './CameraRegistrationStore'; describe('CameraRegistrationStore', () => { - // Pure translation by (+5, -3): trivially invertible. - const translate = [[1, 0, 5], [0, 1, -3], [0, 0, 1]]; - // Four correspondence rows consistent with `translate`: right = left + (5, -3). - const translationPointRows = [ - [0, 0, 5, -3], - [10, 0, 15, -3], - [10, 10, 15, 7], - [0, 10, 5, 7], - ]; - it('produces a directional pairKey that preserves left/right order', () => { const store = new CameraRegistrationStore(); expect(store.pairKey('rgb', 'ir')).toEqual('rgb::ir'); expect(store.pairKey('rgb', 'ir')).not.toEqual(store.pairKey('ir', 'rgb')); }); - it('hydrates homographies and resets prior calibration state', () => { + it('preserves the chosen left/right order on the active pair', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('rgb', 'ir'); + expect(store.activePair.value).toEqual({ camA: 'rgb', camB: 'ir' }); + }); + + it('clears the active pair for identical or empty cameras', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('cam', 'cam'); + expect(store.activePair.value).toBeNull(); + }); + + it('resets alignment to original when switching pairs', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('AtoB'); + store.setActivePair('left', 'other'); + expect(store.alignment.value).toMatchObject({ mode: 'original' }); + }); + + it('forms one correspondence from a blue->red two-click sequence', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [10, 20]); // pending (blue) + expect(store.pendingPoint.value).not.toBeNull(); + expect(store.correspondences.value[key]).toBeUndefined(); + store.addPoint('right', [30, 40]); // completes (red) + expect(store.pendingPoint.value).toBeNull(); + expect(store.correspondences.value[key]).toHaveLength(1); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [10, 20], b: [30, 40] }); + }); + + it('maps points to a/b by camera regardless of click order', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('right', [30, 40]); // click right first + store.addPoint('left', [10, 20]); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [10, 20], b: [30, 40] }); + }); + + it('replaces the pending point when the same camera is clicked twice', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('left', [2, 2]); + expect(store.pendingPoint.value).toMatchObject({ camera: 'left', coord: [2, 2] }); + }); + + it('ignores points for cameras outside the active pair', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('other', [1, 1]); + expect(store.pendingPoint.value).toBeNull(); + }); + + it('removes a correspondence by id', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + const { id } = store.correspondences.value[key][0]; + store.removeCorrespondence(id); + expect(store.correspondences.value[key]).toHaveLength(0); + }); + + it('moves one side of a correspondence via updateCorrespondencePoint', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + const { id } = store.correspondences.value[key][0]; + store.updateCorrespondencePoint(id, 'left', [5, 6]); + expect(store.correspondences.value[key][0].a).toEqual([5, 6]); + expect(store.correspondences.value[key][0].b).toEqual([2, 2]); + store.updateCorrespondencePoint(id, 'right', [7, 8]); + expect(store.correspondences.value[key][0].b).toEqual([7, 8]); + }); + + it('ignores updateCorrespondencePoint for unknown ids or cameras outside the pair', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + const { id } = store.correspondences.value[key][0]; + store.updateCorrespondencePoint(id + 99, 'left', [5, 6]); + store.updateCorrespondencePoint(id, 'other', [5, 6]); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [1, 1], b: [2, 2] }); + }); + + it('refits the pair homography when a point is drag-refined while alignment is active', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + const before = store.homographies.value[key].AtoB[0][2]; + const { id } = store.correspondences.value[key][0]; + store.updateCorrespondencePoint(id, 'right', [40, 40]); + expect(store.homographies.value[key].AtoB[0][2]).not.toBeCloseTo(before, 5); + }); + + it('moves the pending point only for its own camera', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); + store.movePendingPoint('right', [9, 9]); + expect(store.pendingPoint.value).toMatchObject({ camera: 'left', coord: [1, 1] }); + store.movePendingPoint('left', [9, 9]); + expect(store.pendingPoint.value).toMatchObject({ camera: 'left', coord: [9, 9] }); + }); + + it('fits a homography from >= 4 pairs and stores both directions', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + // A pure translation by (5, -3). + const pts: [number, number][] = [[0, 0], [10, 0], [10, 10], [0, 10]]; + pts.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', [p[0] + 5, p[1] - 3]); + }); + const { AtoB, BtoA } = store.fitTransform(key); + expect(AtoB[0][2]).toBeCloseTo(5, 5); + expect(AtoB[1][2]).toBeCloseTo(-3, 5); + expect(BtoA[0][2]).toBeCloseTo(-5, 5); + expect(store.homographies.value[key]).toBeDefined(); + }); + + it('throws when fitting with fewer than 4 pairs (default homography type)', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + expect(() => store.fitTransform(key)).toThrow(); + }); + + it('surfaces a fitError instead of throwing when maybeFitPair hits a degenerate configuration', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.setTransformType(key, 'homography'); + // 4 collinear points satisfy the homography minimum count but are degenerate. + const pts: [number, number][] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + pts.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', p); + }); + expect(() => store.maybeFitPair(key)).not.toThrow(); + expect(store.fitError.value).toMatch(/degenerate/i); + expect(store.homographies.value[key]).toBeUndefined(); + }); + + it('clears a stale fitError once the active pair fits successfully', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.setTransformType(key, 'homography'); + const collinear: [number, number][] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + collinear.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', p); + }); + store.maybeFitPair(key); + expect(store.fitError.value).not.toBeNull(); + store.clearPair(); + addFourTranslationPairs(store); + store.maybeFitPair(key); + expect(store.fitError.value).toBeNull(); + expect(store.homographies.value[key]).toBeDefined(); + }); + + it('clears fitError when switching to a different active pair', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.setTransformType(key, 'homography'); + const collinear: [number, number][] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + collinear.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', p); + }); + store.maybeFitPair(key); + expect(store.fitError.value).not.toBeNull(); + store.setActivePair('left', 'other'); + expect(store.fitError.value).toBeNull(); + }); + + function addFourTranslationPairs(store: CameraRegistrationStore) { + const pts: [number, number][] = [[0, 0], [10, 0], [10, 10], [0, 10]]; + pts.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', [p[0] + 5, p[1] - 3]); + }); + } + + it('fits when enabling alignment mode with >= 4 pairs', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('AtoB'); + expect(store.homographies.value[key]).toBeDefined(); + expect(store.homographies.value[key].AtoB[0][2]).toBeCloseTo(5, 5); + }); + + it('does not enable alignment mode with fewer than 4 pairs (default homography type)', () => { const store = new CameraRegistrationStore(); - store.loadRegistrationText(JSON.stringify({ - version: 1, - pairs: [{ - left: 'left', right: 'right', points: [[1, 1, 6, -2]], transformType: 'translation', - }], - })); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('original'); + expect(store.homographies.value).toEqual({}); + }); + + it('refits when correspondences change while alignment mode is active', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + const before = store.homographies.value[key].AtoB[0][2]; + store.addPoint('left', [20, 20]); + store.addPoint('right', [30, 14]); + expect(store.homographies.value[key].AtoB[0][2]).not.toBeCloseTo(before, 5); + }); + + it('reverts alignment to original when correspondences drop below the transform minimum', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + const { id } = store.correspondences.value[key][0]; + store.removeCorrespondence(id); + store.removeCorrespondence(store.correspondences.value[key][0].id); + store.removeCorrespondence(store.correspondences.value[key][0].id); + store.removeCorrespondence(store.correspondences.value[key][0].id); + expect(store.correspondences.value[key]).toHaveLength(0); + expect(store.alignment.value.mode).toBe('original'); + expect(store.homographies.value[key]).toBeUndefined(); + }); + + it('maybeFitActivePair fits without enabling alignment mode', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.maybeFitActivePair(); + expect(store.alignment.value.mode).toBe('original'); + expect(store.homographies.value[key]).toBeDefined(); + }); + + it('hydrates homographies and resets transient state', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); const saved = { 'a::b': { AtoB: [[1, 0, 0], [0, 1, 0], [0, 0, 1]], BtoA: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] } }; store.hydrate(saved); expect(store.homographies.value).toEqual(saved); + expect(store.activePair.value).toBeNull(); + expect(store.pendingPoint.value).toBeNull(); expect(store.correspondences.value).toEqual({}); expect(store.transformTypes.value).toEqual({}); + expect(store.alignment.value).toEqual({ mode: 'original', opacity: 0.5 }); }); it('hydrates transform types alongside homographies', () => { @@ -50,30 +301,267 @@ describe('CameraRegistrationStore', () => { }; store.hydrate({}, correspondences); expect(store.correspondences.value).toEqual(correspondences); - // Points loaded afterwards pick up ids after the highest restored id. - store.loadRegistrationText(JSON.stringify({ - version: 1, - pairs: [{ left: 'rgb', right: 'ir', points: [[9, 9, 10, 10]] }], - })); - expect(store.correspondences.value['rgb::ir'][0].id).toBe(3); + // New points pick up after the highest restored id. + store.setActivePair('rgb', 'ir'); + store.addPoint('rgb', [9, 9]); + store.addPoint('ir', [10, 10]); + expect(store.correspondences.value['rgb::ir'][2].id).toBe(3); }); - describe('dirty / markSaved', () => { - it('tracks unsaved changes and resets on markSaved and hydrate', () => { + describe('transform type selection', () => { + it('fits a rigid transform from 2 pairs where a homography would throw', () => { const store = new CameraRegistrationStore(); - expect(store.dirty.value).toBe(false); - store.loadRegistrationText(JSON.stringify({ - version: 1, - pairs: [{ - left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: translate, - }], - })); - expect(store.dirty.value).toBe(true); - store.markSaved(); - expect(store.dirty.value).toBe(false); - store.hydrate({ 'a::b': { AtoB: translate, BtoA: translate } }); - // Freshly hydrated state is the saved baseline. - expect(store.dirty.value).toBe(false); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [0, 0]); + store.addPoint('right', [5, -3]); + store.addPoint('left', [10, 0]); + store.addPoint('right', [15, -3]); + store.setTransformType(key, 'homography'); + expect(() => store.fitTransform(key)).toThrow(); + store.setTransformType(key, 'rigid'); + expect(store.homographies.value[key]).toBeDefined(); + expect(store.homographies.value[key].AtoB[0][2]).toBeCloseTo(5, 4); + }); + + it('clears the fit and reverts alignment when switching to a type needing more points than are picked', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [0, 0]); + store.addPoint('right', [5, -3]); + store.addPoint('left', [10, 0]); + store.addPoint('right', [15, -3]); + store.setTransformType(key, 'rigid'); + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('AtoB'); + + store.setTransformType(key, 'homography'); // needs 4, only 2 picked + expect(store.homographies.value[key]).toBeUndefined(); + expect(store.alignment.value.mode).toBe('original'); + }); + }); + + describe('setAlignmentMode guards', () => { + it('setAlignmentMode leaves mode original when the pair lacks enough points', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.setAlignmentMode('BtoA'); + expect(store.alignment.value.mode).toBe('original'); + }); + }); + + describe('pickPoint', () => { + it('records a native pick like addPoint in the Picking (original) mode', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.pickPoint('right', [15, 7]); + expect(store.pendingPoint.value).toMatchObject({ camera: 'right', coord: [15, 7] }); + }); + + it('is a no-op while an overlay warp is active', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + store.pickPoint('right', [15, 7]); + // The warp mode blocks new picks; nothing is pending and the pairs are unchanged. + expect(store.pendingPoint.value).toBeNull(); + expect(store.correspondences.value[store.pairKey('left', 'right')]).toHaveLength(4); + }); + }); + + describe('correspondence selection', () => { + function storeWithOnePair() { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + const key = store.pairKey('left', 'right'); + const { id } = store.correspondences.value[key][0]; + return { store, key, id }; + } + + it('selects an active-pair correspondence and clears via null or unknown ids', () => { + const { store, id } = storeWithOnePair(); + store.selectCorrespondence(id); + expect(store.selectedCorrespondenceId.value).toBe(id); + store.selectCorrespondence(id + 99); + expect(store.selectedCorrespondenceId.value).toBeNull(); + store.selectCorrespondence(id); + store.selectCorrespondence(null); + expect(store.selectedCorrespondenceId.value).toBeNull(); + }); + + it('removeSelectedCorrespondence removes both cameras\' points and clears the selection', () => { + const { store, key, id } = storeWithOnePair(); + store.selectCorrespondence(id); + store.removeSelectedCorrespondence(); + expect(store.correspondences.value[key]).toHaveLength(0); + expect(store.selectedCorrespondenceId.value).toBeNull(); + // No selection: a further call is a no-op. + store.removeSelectedCorrespondence(); + expect(store.correspondences.value[key]).toHaveLength(0); + }); + + it('clears the selection when the selected pair is removed, undone, or the pair switches', () => { + const first = storeWithOnePair(); + first.store.selectCorrespondence(first.id); + first.store.removeCorrespondence(first.id); + expect(first.store.selectedCorrespondenceId.value).toBeNull(); + + const second = storeWithOnePair(); + second.store.selectCorrespondence(second.id); + second.store.clearLast(); + expect(second.store.selectedCorrespondenceId.value).toBeNull(); + + const third = storeWithOnePair(); + third.store.selectCorrespondence(third.id); + third.store.setActivePair('left', 'other'); + expect(third.store.selectedCorrespondenceId.value).toBeNull(); + }); + + it('clears the selection on clearPair, load, and hydrate', () => { + const { store, id } = storeWithOnePair(); + store.selectCorrespondence(id); + store.clearPair(); + expect(store.selectedCorrespondenceId.value).toBeNull(); + + const loaded = storeWithOnePair(); + loaded.store.selectCorrespondence(loaded.id); + loaded.store.loadRegistrationText(JSON.stringify({ version: 1, pairs: [] })); + expect(loaded.store.selectedCorrespondenceId.value).toBeNull(); + + const hydrated = storeWithOnePair(); + hydrated.store.selectCorrespondence(hydrated.id); + hydrated.store.hydrate(); + expect(hydrated.store.selectedCorrespondenceId.value).toBeNull(); + }); + }); + + describe('linked navigation', () => { + it('linkedPoint maps a point from camA to camB and back, via the fitted homography', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); // left -> right is +5, -3 + store.maybeFitActivePair(); + const fromLeft = store.linkedPoint('left', [1, 1]); + expect(fromLeft).toMatchObject({ camera: 'right', coord: [6, -2] }); + const fromRight = store.linkedPoint('right', [6, -2]); + expect(fromRight).toMatchObject({ camera: 'left', coord: [1, 1] }); + }); + + it('linkedPoint returns null when the pair has no fitted homography yet', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + expect(store.linkedPoint('left', [1, 1])).toBeNull(); + }); + + it('linkedPoint returns null for a camera outside the active pair', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + expect(store.linkedPoint('other', [1, 1])).toBeNull(); + }); + }); + + describe('cursor coordinate readout', () => { + it('records and clears the cursor coordinate', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.setCursorCoord('left', [12, 34]); + expect(store.cursorCoord.value).toEqual({ camera: 'left', coord: [12, 34] }); + store.clearCursorCoord(); + expect(store.cursorCoord.value).toBeNull(); + }); + + it('clears the cursor coordinate when switching pairs', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.setCursorCoord('left', [12, 34]); + store.setActivePair('left', 'other'); + expect(store.cursorCoord.value).toBeNull(); + }); + }); + + describe('clearLast', () => { + it('drops the pending point without touching completed correspondences', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + store.addPoint('left', [3, 3]); // pending + store.clearLast(); + expect(store.pendingPoint.value).toBeNull(); + expect(store.correspondences.value[key]).toHaveLength(1); + }); + + it('removes the last completed correspondence when there is no pending point', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + store.addPoint('left', [3, 3]); + store.addPoint('right', [4, 4]); + store.clearLast(); + expect(store.correspondences.value[key]).toHaveLength(1); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [1, 1], b: [2, 2] }); + }); + + it('is a no-op with nothing to undo', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + expect(() => store.clearLast()).not.toThrow(); + store.clearLast(); + expect(store.pendingPoint.value).toBeNull(); + }); + + it('refits when clearing last while alignment mode is active', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.setTransformType(key, 'homography'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + store.clearLast(); + expect(store.correspondences.value[key]).toHaveLength(3); + expect(store.alignment.value.mode).toBe('original'); + expect(store.homographies.value[key]).toBeUndefined(); + }); + }); + + describe('requestRecenter', () => { + it('records a recenter request for a camera in the active pair', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.requestRecenter('right', [7, 8]); + expect(store.recenterRequest.value).toMatchObject({ camera: 'right', coord: [7, 8] }); + }); + + it('ignores a recenter request for a camera outside the active pair', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.requestRecenter('other', [7, 8]); + expect(store.recenterRequest.value).toBeNull(); + }); + + it('assigns a new id to each request so repeated identical requests still change', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.requestRecenter('left', [1, 1]); + const firstId = store.recenterRequest.value?.id; + store.requestRecenter('left', [1, 1]); + expect(store.recenterRequest.value?.id).not.toBe(firstId); + }); + + it('clears the recenter request when switching pairs', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + store.requestRecenter('left', [1, 1]); + store.setActivePair('left', 'other'); + expect(store.recenterRequest.value).toBeNull(); }); }); @@ -92,6 +580,9 @@ describe('CameraRegistrationStore', () => { }); describe('loaded (file-sourced) homographies', () => { + // Pure translation by (+5, -3): trivially invertible. + const translate = [[1, 0, 5], [0, 1, -3], [0, 0, 1]]; + /** Load a matrix-only (point-less) pair from calibration JSON, marking it 'loaded'. */ function loadMatrixOnlyPair(store: CameraRegistrationStore, left: string, right: string, rightToLeft: number[][]) { store.loadRegistrationText(JSON.stringify({ @@ -104,6 +595,7 @@ describe('CameraRegistrationStore', () => { it('loads a matrix-only pair as B->A with its inverse as A->B and no points', () => { const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); const key = store.pairKey('left', 'right'); loadMatrixOnlyPair(store, 'left', 'right', translate); expect(store.correspondences.value[key]).toHaveLength(0); @@ -114,46 +606,85 @@ describe('CameraRegistrationStore', () => { expect(homog.AtoB[1][2]).toBeCloseTo(3); }); + it('keeps a loaded homography through refit checks with too few points', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + loadMatrixOnlyPair(store, 'left', 'right', translate); + store.maybeFitPair(key); + expect(store.homographies.value[key]).toBeDefined(); + // Alignment can activate directly off the loaded transform. + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('AtoB'); + }); + + it('replaces a loaded homography once enough points are picked and fitted', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + loadMatrixOnlyPair(store, 'left', 'right', [[1, 0, 100], [0, 1, 100], [0, 0, 1]]); + addFourTranslationPairs(store); + store.maybeFitPair(key); + expect(store.isLoadedHomography(key)).toBe(false); + // Fitted from points: right = left + (5, -3), so AtoB translates by (5, -3). + expect(store.homographies.value[key].AtoB[0][2]).toBeCloseTo(5); + expect(store.homographies.value[key].AtoB[1][2]).toBeCloseTo(-3); + }); + + it('clearPair removes a loaded homography', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + loadMatrixOnlyPair(store, 'left', 'right', translate); + store.clearPair(); + expect(store.homographies.value[key]).toBeUndefined(); + expect(store.isLoadedHomography(key)).toBe(false); + }); + + it('clearPair also removes a stale fitted homography immediately', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.fitTransform(key); + store.clearPair(); + expect(store.homographies.value[key]).toBeUndefined(); + }); + it('rejects a singular loaded matrix', () => { const store = new CameraRegistrationStore(); expect(() => loadMatrixOnlyPair(store, 'left', 'right', [[0, 0, 0], [0, 0, 0], [0, 0, 0]])) .toThrow(/singular/); }); - it('hydrate marks an under-pointed homography as loaded', () => { + it('hydrate marks an under-pointed homography as loaded so it survives refit checks', () => { const store = new CameraRegistrationStore(); const key = store.pairKey('left', 'right'); store.hydrate({ [key]: { AtoB: translate, BtoA: translate } }, {}, {}); expect(store.isLoadedHomography(key)).toBe(true); + store.maybeFitPair(key); + expect(store.homographies.value[key]).toBeDefined(); }); }); describe('calibration JSON file round trip', () => { it('serializes and reloads all pairs', () => { const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); const key = store.pairKey('left', 'right'); - store.loadRegistrationText(JSON.stringify({ - version: 1, - pairs: [{ - left: 'left', - right: 'right', - points: translationPointRows, - leftToRight: translate, - rightToLeft: null, - transformType: 'translation', - }], - })); + addFourTranslationPairs(store); + store.setTransformType(key, 'translation'); + store.fitTransform(key); const json = store.toRegistrationJson(); const restored = new CameraRegistrationStore(); + restored.setActivePair('left', 'right'); const result = restored.loadRegistrationText(json); expect(result.pairCount).toBe(1); expect(result.cameras.sort()).toEqual(['left', 'right']); expect(restored.correspondences.value[key]).toHaveLength(4); expect(restored.transformTypeForPair(key)).toBe('translation'); expect(restored.homographies.value[key].AtoB[0][2]).toBeCloseTo(5); - // The missing direction was derived by inversion and round-tripped. - expect(restored.homographies.value[key].BtoA[0][2]).toBeCloseTo(-5); // Enough points back the homography, so it is treated as fitted. expect(restored.isLoadedHomography(key)).toBe(false); }); @@ -173,6 +704,17 @@ describe('CameraRegistrationStore', () => { expect(restored.isLoadedHomography(key)).toBe(true); }); + it('reverts alignment to original and keeps the active pair on load', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + const json = store.toRegistrationJson(); + store.loadRegistrationText(json); + expect(store.alignment.value.mode).toBe('original'); + expect(store.activePair.value).toEqual({ camA: 'left', camB: 'right' }); + }); + it('loads a desktop-persisted calibration.json (no "type" field, one direction only)', () => { const store = new CameraRegistrationStore(); const result = store.loadRegistrationText(JSON.stringify({ @@ -194,29 +736,30 @@ describe('CameraRegistrationStore', () => { expect(store.transformTypeForPair(key)).toBe('translation'); }); - it('preserves the producer source stamp across a load/save round trip', () => { + it('preserves the producer source stamp across load, refinement, and save', () => { const store = new CameraRegistrationStore(); const source = { model: 'colmap-2026-07-01', swathe: 'fl07_C' }; + store.setActivePair('left', 'right'); store.loadRegistrationText(JSON.stringify({ version: 1, source, pairs: [{ - left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: translate, + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], }], })); expect(store.source.value).toStrictEqual(source); + // In-app refinement replaces the transform but keeps the lineage stamp. + addFourTranslationPairs(store); + store.maybeFitPair(store.pairKey('left', 'right')); + expect(store.source.value).toStrictEqual(source); const saved = JSON.parse(store.toRegistrationJson()); expect(saved.source).toStrictEqual(source); }); it('omits the source key when no stamp was loaded', () => { const store = new CameraRegistrationStore(); - store.loadRegistrationText(JSON.stringify({ - version: 1, - pairs: [{ - left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, - }], - })); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); expect('source' in JSON.parse(store.toRegistrationJson())).toBe(false); }); @@ -246,69 +789,59 @@ describe('CameraRegistrationStore', () => { expect(store.source.value).toBeNull(); }); - it('flags a point-backed homography as refined when a source stamp is loaded', () => { + it('flags a pair as refined once an in-app fit replaces a stamped matrix', () => { const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); const key = store.pairKey('left', 'right'); - // Fresh from the producer (matrix-only): loaded, not refined. store.loadRegistrationText(JSON.stringify({ version: 1, source: { model: 'colmap-x' }, pairs: [{ - left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: translate, + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], }], })); + // Fresh from the producer: loaded, not refined. expect(store.isRefinedFromSource(key)).toBe(false); - // Point-backed under a stamp: the pair has diverged from the producer. - store.loadRegistrationText(JSON.stringify({ - version: 1, - source: { model: 'colmap-x' }, - pairs: [{ - left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, transformType: 'translation', - }], - })); + addFourTranslationPairs(store); + store.maybeFitPair(key); expect(store.isRefinedFromSource(key)).toBe(true); }); - it('does not flag point-backed homographies as refined when no source stamp is loaded', () => { + it('does not flag fits as refined when no source stamp is loaded', () => { const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); const key = store.pairKey('left', 'right'); - store.loadRegistrationText(JSON.stringify({ - version: 1, - pairs: [{ - left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, transformType: 'translation', - }], - })); + addFourTranslationPairs(store); + store.fitTransform(key); expect(store.isRefinedFromSource(key)).toBe(false); }); it('keeps the refined flag across a save/load round trip', () => { const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); const key = store.pairKey('left', 'right'); store.loadRegistrationText(JSON.stringify({ version: 1, source: { model: 'colmap-x' }, pairs: [{ - left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, transformType: 'translation', + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: [[1, 0, 100], [0, 1, 100], [0, 0, 1]], }], })); - expect(store.isRefinedFromSource(key)).toBe(true); + addFourTranslationPairs(store); + store.maybeFitPair(key); const restored = new CameraRegistrationStore(); restored.loadRegistrationText(store.toRegistrationJson()); - // The refined pair saved with its backing points, so it re-marks as + // The refit pair saved with its backing points, so it re-marks as // fitted (refined) rather than loaded. expect(restored.isRefinedFromSource(key)).toBe(true); }); it('rejects non-JSON, missing pairs, malformed pairs, and bad matrices without clobbering state', () => { const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); const key = store.pairKey('left', 'right'); - store.loadRegistrationText(JSON.stringify({ - version: 1, - pairs: [{ - left: 'left', right: 'right', points: translationPointRows, leftToRight: translate, rightToLeft: null, - }], - })); + addFourTranslationPairs(store); expect(() => store.loadRegistrationText('not json')).toThrow(/valid JSON/); expect(() => store.loadRegistrationText('{"type": "other"}')).toThrow(/pairs/); expect(() => store.loadRegistrationText(JSON.stringify({ diff --git a/client/src/alignedView/CameraRegistrationStore.ts b/client/src/alignedView/CameraRegistrationStore.ts index c6f462430..c0a748a64 100644 --- a/client/src/alignedView/CameraRegistrationStore.ts +++ b/client/src/alignedView/CameraRegistrationStore.ts @@ -2,10 +2,10 @@ import { ref, computed, Ref, ComputedRef, } from 'vue'; import { - invert3, Matrix3, Point, + invert3, applyHomography, Matrix3, Point, } from './homography'; import { - TransformType, TRANSFORM_TYPES, DEFAULT_TRANSFORM_TYPE, minPointsForTransform, + TransformType, TRANSFORM_TYPES, DEFAULT_TRANSFORM_TYPE, minPointsForTransform, estimateTransform, } from './transform'; /** @@ -85,21 +85,80 @@ export type CameraCorrespondences = Record; /** Chosen fit model per pair, keyed by {@link CameraRegistrationStore.pairKey}. Missing entries default to 'similarity'. */ export type CameraTransformTypes = Record; +/** Which image is warped onto which for the in-app aligned-picking preview. */ +export type AlignmentMode = 'original' | 'AtoB' | 'BtoA'; + +export interface AlignmentState { + mode: AlignmentMode; + opacity: number; +} + +/** Active pair. `camA` is the left camera, `camB` the right (user-chosen order). */ +export interface ActivePair { + camA: string; + camB: string; +} + /** - * Shared, reactive store for camera-calibration data (correspondences, + * Shared, reactive store for camera-registration data (correspondences, * fitted/loaded homographies, transform-type choices, and producer * provenance). Lives in vue-media-annotator so both the annotation layers * (client/src/alignedView) and the dive-common side can consume it via the * provide/inject system. Handles persistence: hydrating saved state and - * loading/saving the portable calibration JSON format. + * loading/saving the portable registration JSON format. + * + * Also holds the interactive creation state, implementing the keypointgui + * blue->red pairing flow: the first click in one camera sets a pending + * point; the next click in the *other* camera completes a pair. */ export default class CameraRegistrationStore { + activePair: Ref; + + pickingEnabled: Ref; + + pendingPoint: Ref<{ camera: string; coord: Point } | null>; + correspondences: Ref; homographies: Ref; transformTypes: Ref; + alignment: Ref; + + /** + * Whether pan/zoom is linked between the active pair's two cameras through + * the fitted transform (see {@link useCalibrationNavigation}). Only has an + * effect once a transform is fitted; toggled from the panel's "Fit pan/zoom". + */ + linkedNav: Ref; + + /** + * Correspondence currently selected in the picking UI (grabbed marker / + * clicked table row), highlighted in BOTH cameras' panes and deletable via + * the panel or the Delete key. Authoring state only -- never persisted. + */ + selectedCorrespondenceId: Ref; + + /** Native-pixel coordinate under the cursor, for the calibration panel's live readout. */ + cursorCoord: Ref<{ camera: string; coord: Point } | null>; + + /** + * A one-shot "recenter here" request (e.g. from a right-click), keyed by an + * incrementing id so repeated requests at the same coordinate still trigger + * watchers. See {@link requestRecenter}. + */ + recenterRequest: Ref<{ camera: string; coord: Point; id: number } | null>; + + /** + * Message from the most recent failed fit attempt (e.g. collinear/degenerate + * points that satisfy the minimum count but can't be solved), or null if the + * active pair's last fit attempt (if any) succeeded. Surfaced by the + * calibration panel instead of letting the estimator's exception escape a + * geojs click handler. + */ + fitError: Ref; + /** * Provenance of the loaded calibration (see {@link RegistrationSource}). * Deliberately NOT cleared by in-app edits or refits -- refinements are @@ -114,6 +173,8 @@ export default class CameraRegistrationStore { private nextId: number; + private nextRecenterId: number; + /** Provenance per homography key; missing entries behave like 'fit'. */ private homographySources: Record; @@ -121,11 +182,21 @@ export default class CameraRegistrationStore { private savedSnapshot: Ref; constructor() { + this.activePair = ref(null); + this.pickingEnabled = ref(false); + this.pendingPoint = ref(null); this.correspondences = ref({}); this.homographies = ref({}); this.transformTypes = ref({}); + this.alignment = ref({ mode: 'original', opacity: 0.5 }); + this.linkedNav = ref(true); + this.selectedCorrespondenceId = ref(null); + this.cursorCoord = ref(null); + this.recenterRequest = ref(null); + this.fitError = ref(null); this.source = ref(null); this.nextId = 1; + this.nextRecenterId = 1; this.homographySources = {}; this.savedSnapshot = ref(this.registrationSnapshot()); this.dirty = computed(() => this.registrationSnapshot() !== this.savedSnapshot.value); @@ -167,6 +238,188 @@ export default class CameraRegistrationStore { return `${camA}::${camB}`; } + /** Key of the currently active pair, or null if none selected. */ + activePairKey(): string | null { + const pair = this.activePair.value; + return pair ? this.pairKey(pair.camA, pair.camB) : null; + } + + /** Select a camera pair. `left` becomes camA, `right` becomes camB. */ + setActivePair(left: string | null, right: string | null) { + if (!left || !right || left === right) { + this.activePair.value = null; + } else { + this.activePair.value = { camA: left, camB: right }; + } + this.pendingPoint.value = null; + // Switching pairs invalidates any active overlay warp: drop back to the + // unwarped Picking mode so the new pair starts from its own native views. + this.alignment.value = { mode: 'original', opacity: this.alignment.value.opacity }; + this.selectedCorrespondenceId.value = null; + this.cursorCoord.value = null; + this.recenterRequest.value = null; + this.fitError.value = null; + } + + /** + * Add a clicked image point for `camera`. The first click sets a pending point; + * a subsequent click in the *other* camera of the active pair completes a pair. + * Clicking the same camera again replaces the pending point. + */ + addPoint(camera: string, coord: Point) { + const pair = this.activePair.value; + if (!pair || (camera !== pair.camA && camera !== pair.camB)) { + return; + } + const pending = this.pendingPoint.value; + if (!pending || pending.camera === camera) { + this.pendingPoint.value = { camera, coord }; + return; + } + const key = this.pairKey(pair.camA, pair.camB); + const a = pending.camera === pair.camA ? pending.coord : coord; + const b = pending.camera === pair.camB ? pending.coord : coord; + const list = this.correspondences.value[key] + ? [...this.correspondences.value[key]] + : []; + // eslint-disable-next-line no-plusplus + list.push({ id: this.nextId++, a, b }); + this.correspondences.value = { ...this.correspondences.value, [key]: list }; + this.pendingPoint.value = null; + this.syncAlignmentHomography(); + } + + /** + * Record a click at `coord` (native pixel coords of `camera`'s own pane). + * New points are only picked in the unwarped 'original' (Picking) mode: while + * an overlay warp is active the panes show a warped preview rather than native + * coordinates, so clicks there are ignored. + */ + pickPoint(camera: string, coord: Point) { + if (this.alignment.value.mode !== 'original') { + return; + } + this.addPoint(camera, coord); + } + + /** + * Move one side of an existing correspondence (drag-to-refine). `camera` + * selects which side (a for camA, b for camB); the pair is refit live so + * the alignment ghost and linked navigation track the drag. + */ + updateCorrespondencePoint(id: number, camera: string, coord: Point) { + const pair = this.activePair.value; + if (!pair || (camera !== pair.camA && camera !== pair.camB)) { + return; + } + const key = this.pairKey(pair.camA, pair.camB); + const list = this.correspondences.value[key]; + if (!list || !list.some((c) => c.id === id)) { + return; + } + const side = camera === pair.camA ? 'a' : 'b'; + this.correspondences.value = { + ...this.correspondences.value, + [key]: list.map((c) => (c.id === id ? { ...c, [side]: coord } : c)), + }; + this.syncAlignmentHomography(); + } + + /** Move the pending (blue) point while it is being drag-refined. */ + movePendingPoint(camera: string, coord: Point) { + const pending = this.pendingPoint.value; + if (!pending || pending.camera !== camera) { + return; + } + this.pendingPoint.value = { camera, coord }; + } + + /** Remove a correspondence (by id) from the active pair -- both cameras' points at once. */ + removeCorrespondence(id: number) { + const key = this.activePairKey(); + if (!key) { + return; + } + const list = this.correspondences.value[key]; + if (!list) { + return; + } + this.correspondences.value = { + ...this.correspondences.value, + [key]: list.filter((c) => c.id !== id), + }; + if (this.selectedCorrespondenceId.value === id) { + this.selectedCorrespondenceId.value = null; + } + this.syncAlignmentHomography(); + } + + /** + * Select a correspondence marker for inspection/deletion (null clears). + * Only ids belonging to the active pair are selectable; anything else + * clears the selection. + */ + selectCorrespondence(id: number | null) { + if (id === null) { + this.selectedCorrespondenceId.value = null; + return; + } + const key = this.activePairKey(); + const list = key ? this.correspondences.value[key] : undefined; + this.selectedCorrespondenceId.value = (list && list.some((c) => c.id === id)) ? id : null; + } + + /** Remove the selected correspondence (both cameras' points). No-op without a selection. */ + removeSelectedCorrespondence() { + const id = this.selectedCorrespondenceId.value; + if (id !== null) { + this.removeCorrespondence(id); + } + } + + /** + * Drop all correspondences, the pending point, and any homography + * (fitted or file-loaded) for the active pair. + */ + clearPair() { + const key = this.activePairKey(); + this.pendingPoint.value = null; + this.selectedCorrespondenceId.value = null; + if (!key) { + return; + } + this.correspondences.value = { ...this.correspondences.value, [key]: [] }; + // Clearing is explicit: a file-loaded homography goes too. Dropping the + // 'loaded' mark lets maybeFitPair remove it through the normal path. + delete this.homographySources[key]; + this.maybeFitPair(key); + } + + /** + * Undo one step, mirroring keypointgui's Clear Last button: if there's a + * pending (blue) point, drop it; otherwise remove the most recently + * completed correspondence for the active pair. + */ + clearLast() { + if (this.pendingPoint.value) { + this.pendingPoint.value = null; + return; + } + const key = this.activePairKey(); + if (!key) { + return; + } + const list = this.correspondences.value[key]; + if (!list || list.length === 0) { + return; + } + if (this.selectedCorrespondenceId.value === list[list.length - 1].id) { + this.selectedCorrespondenceId.value = null; + } + this.correspondences.value = { ...this.correspondences.value, [key]: list.slice(0, -1) }; + this.syncAlignmentHomography(); + } + /** * True when `key`'s homography came from a calibration file rather than an * in-app fit. Not independently reactive -- always read alongside @@ -196,6 +449,154 @@ export default class CameraRegistrationStore { return this.transformTypes.value[key] || DEFAULT_TRANSFORM_TYPE; } + /** Choose the fit model for `key` and immediately (re)fit or clear as needed. */ + setTransformType(key: string, type: TransformType) { + this.transformTypes.value = { ...this.transformTypes.value, [key]: type }; + this.maybeFitPair(key); + } + + /** + * Fit `key` when it has enough points for its chosen transform type; otherwise + * clear its homography and, if it's the active (aligned) pair, revert + * alignment to 'original'. A fit can still fail past the minimum-count check + * (e.g. collinear/near-duplicate points make the system unsolvable); that's + * caught here and surfaced via {@link fitError} instead of throwing out of a + * geojs click handler, keeping any previously fitted homography in place. + */ + maybeFitPair(key: string) { + const list = this.correspondences.value[key]; + const required = minPointsForTransform(this.transformTypeForPair(key)); + if (!list || list.length < required) { + // A file-loaded homography has no backing points; it stays in place + // until enough points are picked to fit a replacement (or the pair is + // explicitly cleared, which drops its 'loaded' mark first). + if (this.homographySources[key] !== 'loaded') { + const rest = { ...this.homographies.value }; + delete rest[key]; + this.homographies.value = rest; + delete this.homographySources[key]; + if (this.activePairKey() === key && this.alignment.value.mode !== 'original') { + this.alignment.value = { ...this.alignment.value, mode: 'original' }; + } + } + if (this.activePairKey() === key) { + this.fitError.value = null; + } + return; + } + try { + this.fitTransform(key); + if (this.activePairKey() === key) { + this.fitError.value = null; + } + } catch (err) { + if (this.activePairKey() === key) { + this.fitError.value = err instanceof Error ? err.message : String(err); + } + } + } + + /** Fit the active pair when it has enough points; otherwise clear/revert as in {@link maybeFitPair}. */ + maybeFitActivePair() { + const key = this.activePairKey(); + if (!key) { + return; + } + this.maybeFitPair(key); + } + + /** Enable or change the alignment (ghost overlay) mode, fitting the pair first if needed. */ + setAlignmentMode(mode: AlignmentMode) { + if (mode !== 'original') { + this.maybeFitActivePair(); + const key = this.activePairKey(); + if (!key || !this.homographies.value[key]) { + // Not enough points for the active pair's transform type; stay original. + return; + } + } + this.alignment.value = { ...this.alignment.value, mode }; + } + + /** Ghost overlay opacity, independent of alignment mode. */ + setAlignmentOpacity(opacity: number) { + this.alignment.value = { ...this.alignment.value, opacity }; + } + + /** + * Map `coord` (native pixel space of `camera`) to the corresponding point in + * the *other* camera of the active pair, via the fitted homography. Returns + * `null` when `camera` isn't part of the active pair or the pair has no + * fitted homography yet (not enough correspondences) -- callers should treat + * that as "nothing to link to" rather than an error. + */ + linkedPoint(camera: string, coord: Point): { camera: string; coord: Point } | null { + const pair = this.activePair.value; + if (!pair || (camera !== pair.camA && camera !== pair.camB)) { + return null; + } + const homog = this.homographies.value[this.pairKey(pair.camA, pair.camB)]; + if (!homog) { + return null; + } + const other = camera === pair.camA ? pair.camB : pair.camA; + const matrix = camera === pair.camA ? homog.AtoB : homog.BtoA; + return { camera: other, coord: applyHomography(matrix, coord) }; + } + + /** Record the native-pixel coordinate under the cursor for `camera` (calibration panel readout). */ + setCursorCoord(camera: string, coord: Point) { + this.cursorCoord.value = { camera, coord }; + } + + /** Clear the cursor coordinate readout (e.g. on mouse leave). */ + clearCursorCoord() { + this.cursorCoord.value = null; + } + + /** + * Request that `camera` (native pixel coords `coord`) and, when the pair has + * a fitted homography, the other camera of the active pair (via + * {@link linkedPoint}) recenter their views on this location. Consumed by + * {@link useCalibrationNavigation}; a no-op if `camera` isn't part of the + * active pair. A one-shot "snap to this feature" action, distinct from the + * continuous pan/zoom link that is active while picking. + */ + requestRecenter(camera: string, coord: Point) { + const pair = this.activePair.value; + if (!pair || (camera !== pair.camA && camera !== pair.camB)) { + return; + } + // eslint-disable-next-line no-plusplus + this.recenterRequest.value = { camera, coord, id: this.nextRecenterId++ }; + } + + /** Re-fit the active pair while alignment is active (mode != 'original'). */ + private syncAlignmentHomography() { + if (this.alignment.value.mode !== 'original') { + this.maybeFitActivePair(); + } + } + + /** + * Fit `key`'s chosen transform type from its correspondences (see + * {@link minPointsForTransform} for the required count). Computes both + * directions and stores them. Returns the fitted pair. + */ + fitTransform(key: string): PairHomography { + const list = this.correspondences.value[key]; + const type = this.transformTypeForPair(key); + const required = minPointsForTransform(type); + if (!list || list.length < required) { + throw new Error(`At least ${required} point pair(s) are required to fit a ${type} transform`); + } + const AtoB = estimateTransform(type, list.map((c) => c.a), list.map((c) => c.b)); + const BtoA = invert3(AtoB); + this.homographies.value = { ...this.homographies.value, [key]: { AtoB, BtoA } }; + this.homographySources[key] = 'fit'; + return { AtoB, BtoA }; + } + /** * Serialize every pair with content (points and/or a homography) as the * portable calibration JSON file (see {@link RegistrationFile}). Pairs whose @@ -230,9 +631,11 @@ export default class CameraRegistrationStore { } /** - * Parse and load a calibration JSON file (the format written by + * Parse and load a registration JSON file (the format written by * {@link toRegistrationJson}), REPLACING all pairs' correspondences, - * homographies, and transform types. Throws a descriptive Error on + * homographies, and transform types. The active pair selection and picking + * toggle are left alone; the alignment ghost reverts to 'original' since + * the transform under it changed wholesale. Throws a descriptive Error on * malformed input without touching current state. Returns the camera names * referenced by the file so callers can warn about ones missing from the * loaded dataset. @@ -295,6 +698,10 @@ export default class CameraRegistrationStore { this.transformTypes.value = transformTypes; this.source.value = source; this.markHomographySources(); + this.pendingPoint.value = null; + this.selectedCorrespondenceId.value = null; + this.fitError.value = null; + this.alignment.value = { ...this.alignment.value, mode: 'original' }; return { cameras: [...cameras], pairCount: file.pairs.length }; } @@ -366,6 +773,14 @@ export default class CameraRegistrationStore { this.transformTypes.value = transformTypes ? { ...transformTypes } : {}; this.source.value = source ?? null; this.markHomographySources(); + this.activePair.value = null; + this.pendingPoint.value = null; + this.pickingEnabled.value = false; + this.alignment.value = { mode: 'original', opacity: 0.5 }; + this.selectedCorrespondenceId.value = null; + this.cursorCoord.value = null; + this.recenterRequest.value = null; + this.fitError.value = null; // Resume id allocation past any restored correspondences. let maxId = 0; Object.values(this.correspondences.value).forEach((list) => { diff --git a/client/src/components/annotators/linkedViewers/useCalibrationNavigation.spec.ts b/client/src/components/annotators/linkedViewers/useCalibrationNavigation.spec.ts new file mode 100644 index 000000000..06f20b893 --- /dev/null +++ b/client/src/components/annotators/linkedViewers/useCalibrationNavigation.spec.ts @@ -0,0 +1,140 @@ +/// +import { + ref, shallowRef, nextTick, Ref, +} from 'vue'; +import useCalibrationNavigation from './useCalibrationNavigation'; +import type CameraRegistrationStore from '../../../alignedView/CameraRegistrationStore'; +import type { AggregateMediaController } from '../mediaControllerType'; +import type { Point } from '../../../alignedView/homography'; + +vi.mock('geojs', () => ({ default: { event: { pan: 'geo_pan', zoom: 'geo_zoom' } } })); + +/** Minimal stand-in for a geojs viewer: center/zoom state + geoOn events. */ +function fakeViewer(baseUnitsPerPixel: number) { + const state = { center: { x: 0, y: 0 }, zoom: 0 }; + const handlers: Record void>> = {}; + return { + geoOn(evt: string, handler: () => void) { + handlers[evt] = handlers[evt] || []; + handlers[evt].push(handler); + }, + geoOff(evt: string, handler: () => void) { + handlers[evt] = (handlers[evt] || []).filter((h) => h !== handler); + }, + center(c?: { x: number; y: number }) { + if (c) { + state.center = { ...c }; + } + return state.center; + }, + zoom(z?: number) { + if (z !== undefined) { + state.zoom = z; + } + return state.zoom; + }, + unitsPerPixel(z: number) { + return baseUnitsPerPixel / 2 ** z; + }, + trigger(evt: string) { + (handlers[evt] || []).forEach((h) => h()); + }, + }; +} + +function makeHarness() { + const eo = fakeViewer(1); + const ir = fakeViewer(8); + const controllers: Record }> = { + eo: { geoViewerRef: ref(eo) }, + ir: { geoViewerRef: ref(ir) }, + }; + const resizing = ref(false); + const resizeTrigger = ref(0); + // shallowRef: a plain ref would deep-unwrap the nested resizing / + // resizeTrigger refs, unlike the real aggregate controller object. + const aggregate = shallowRef({ + resizing, + resizeTrigger, + getController: (name: string) => controllers[name], + }) as unknown as Ref; + + const pickingEnabled = ref(false); + const linkedNav = ref(false); + const homographies = ref>({}); + const fitted = ref(true); + // eo -> ir is a pure +100 x-translation (and ir -> eo its inverse), so the + // linked scale is 1 and centers map by simple offset. + const calibration = { + pickingEnabled, + linkedNav, + activePair: ref({ camA: 'eo', camB: 'ir' }), + homographies, + recenterRequest: ref(null), + linkedPoint(camera: string, coord: Point) { + if (!fitted.value) { + return null; + } + if (camera === 'eo') { + return { camera: 'ir', coord: [coord[0] + 100, coord[1]] as Point }; + } + return { camera: 'eo', coord: [coord[0] - 100, coord[1]] as Point }; + }, + } as unknown as CameraRegistrationStore; + + useCalibrationNavigation(aggregate, calibration); + return { + eo, ir, pickingEnabled, linkedNav, homographies, fitted, + }; +} + +describe('useCalibrationNavigation', () => { + it('snaps the pair immediately when "Fit pan/zoom" turns on', async () => { + const { + eo, ir, pickingEnabled, linkedNav, + } = makeHarness(); + pickingEnabled.value = true; + await nextTick(); + eo.center({ x: 250, y: 150 }); + eo.zoom(1); // units-per-pixel = 0.5 + + // Off: nothing linked yet. + expect(ir.center()).toEqual({ x: 0, y: 0 }); + + linkedNav.value = true; + await nextTick(); + + // No pan/zoom event fired: the toggle itself lined the pair up. + expect(ir.center()).toEqual({ x: 350, y: 150 }); + // Matching extent (scale 1) through ir's own zoom-0 baseline: log2(8 / 0.5). + expect(ir.zoom()).toBeCloseTo(4, 6); + }); + + it('re-snaps when the fitted homography changes under the link', async () => { + const { + eo, ir, pickingEnabled, linkedNav, homographies, + } = makeHarness(); + pickingEnabled.value = true; + linkedNav.value = true; + await nextTick(); + + eo.center({ x: 10, y: 20 }); + homographies.value = { 'eo|ir': 'refit' }; + await nextTick(); + + expect(ir.center()).toEqual({ x: 110, y: 20 }); + }); + + it('does not snap while no fit exists yet', async () => { + const { + ir, pickingEnabled, linkedNav, fitted, + } = makeHarness(); + fitted.value = false; + pickingEnabled.value = true; + linkedNav.value = true; + await nextTick(); + + expect(ir.center()).toEqual({ x: 0, y: 0 }); + expect(ir.zoom()).toBe(0); + }); +}); diff --git a/client/src/components/annotators/linkedViewers/useCalibrationNavigation.ts b/client/src/components/annotators/linkedViewers/useCalibrationNavigation.ts new file mode 100644 index 000000000..e5da709e8 --- /dev/null +++ b/client/src/components/annotators/linkedViewers/useCalibrationNavigation.ts @@ -0,0 +1,121 @@ +import { Ref, watch } from 'vue'; +import type { AggregateMediaController } from '../mediaControllerType'; +import type CameraRegistrationStore from '../../../alignedView/CameraRegistrationStore'; +import type AlignedViewStore from '../../../alignedView/AlignedViewStore'; +import { localLinkedScale } from '../../../alignedView/homography'; +import type { Point } from '../../../alignedView/homography'; +import useLinkedViewers from './useLinkedViewers'; + +/** + * Links pan/zoom recentering between the two cameras of the active calibration + * pair: panning or zooming one recenters the other on the same point, mapped + * through the pair's fitted homography ({@link CameraRegistrationStore.linkedPoint}). + * The mapping assumes UNWARPED panes showing native coordinates, so this is + * active only while point picking is (the aligned-view link, + * {@link useAlignedNavigation}, owns navigation otherwise) and stands down if + * the aligned view is somehow active. Distinct from the general "sync cameras" + * toggle (Controls.vue), which assumes identical pixel scale between panes. + */ +export default function useCalibrationNavigation( + aggregateController: Ref, + calibration: CameraRegistrationStore, + alignedView?: AlignedViewStore, +) { + const { + viewer, teardown, attach, guarded, applyView, + } = useLinkedViewers(aggregateController); + + function link(camera: string, otherCamera: string) { + return () => guarded(() => { + // The homography mapping assumes unwarped panes; the aligned-view link + // owns navigation while it is active. + if (alignedView?.active.value) { + return; + } + // Ignore the pan/zoom events onResize emits while resetting each pane to + // its own native bounds, so one pane's reset isn't mapped onto its pair. + if (aggregateController.value.resizing.value) { + return; + } + const source = viewer(camera); + const target = viewer(otherCamera); + if (!source || !target) { + return; + } + const center = source.center(); + const linked = calibration.linkedPoint(camera, [center.x, center.y]); + if (!linked || linked.camera !== otherCamera) { + return; + } + // Match the visible extent: one source pixel spans `scale` target pixels + // here (position-dependent for non-similarity fits, so sampled at center; + // null when unavailable -- leave the target's zoom alone). + const scale = localLinkedScale( + (p) => calibration.linkedPoint(camera, p)?.coord ?? null, + [center.x, center.y], + ); + applyView(target, { + center: { x: linked.coord[0], y: linked.coord[1] }, + unitsPerPixel: scale === null ? null : source.unitsPerPixel(source.zoom()) * scale, + }); + }); + } + + function setup() { + teardown(); + const pair = calibration.activePair.value; + // Authoring UI: active while picking and "Fit pan/zoom" is on (once a fit + // exists, linkedPoint returns matches). attach() no-ops for a not-yet-ready + // pane; the resizeTrigger watch re-runs setup once both viewers exist. + if (calibration.pickingEnabled.value && calibration.linkedNav.value && pair) { + attach(pair.camA, link(pair.camA, pair.camB)); + attach(pair.camB, link(pair.camB, pair.camA)); + // Snap immediately from camA so toggling "Fit pan/zoom" on (or a refit + // under it) lines the pair up right away instead of waiting for the + // first pan/zoom event. No-ops harmlessly while no fit exists yet + // (linkedPoint returns null). + link(pair.camA, pair.camB)(); + } + } + + watch( + [ + calibration.pickingEnabled, + calibration.linkedNav, + calibration.activePair, + calibration.homographies, + aggregateController.value.resizeTrigger, + ], + setup, + { deep: true }, + ); + + /** + * One-shot recenter (right-click while picking): center the clicked camera on + * the clicked point and, when the pair has a fitted homography, the other + * camera on the corresponding point. Guarded so it doesn't loop back through + * the continuous link above. + */ + function handleRecenterRequest( + request: { camera: string; coord: Point; id: number } | null, + ) { + if (!request || alignedView?.active.value) { + return; + } + const pair = calibration.activePair.value; + if (!pair || (request.camera !== pair.camA && request.camera !== pair.camB)) { + return; + } + const source = viewer(request.camera); + if (source) { + guarded(() => source.center({ x: request.coord[0], y: request.coord[1] })); + } + const linked = calibration.linkedPoint(request.camera, request.coord); + const target = linked && viewer(linked.camera); + if (linked && target) { + guarded(() => target.center({ x: linked.coord[0], y: linked.coord[1] })); + } + } + + watch(() => calibration.recenterRequest.value, handleRecenterRequest); +} diff --git a/client/src/components/index.ts b/client/src/components/index.ts index 908416aa7..2eaec7028 100644 --- a/client/src/components/index.ts +++ b/client/src/components/index.ts @@ -28,6 +28,7 @@ import TypePicker from './TypePicker.vue'; export * from './annotators/useMediaController'; export { default as useAnnotatorImageCursor } from './annotators/useAnnotatorImageCursor'; +export { default as useCalibrationNavigation } from './annotators/linkedViewers/useCalibrationNavigation'; export { default as useAlignedNavigation } from './annotators/linkedViewers/useAlignedNavigation'; export { /* Annotators */ diff --git a/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts b/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts new file mode 100644 index 000000000..01743e600 --- /dev/null +++ b/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts @@ -0,0 +1,513 @@ +import geo, { GeoEvent } from 'geojs'; +import BaseLayer, { BaseLayerParams, LayerStyle } from '../BaseLayer'; +import { FrameDataTrack } from '../LayerTypes'; +import CameraRegistrationStore from '../../alignedView/CameraRegistrationStore'; +import { subdivideWarpQuads, warpGridSize } from '../../alignedView/homography'; +import type { CameraImage } from '../AlignedImageLayer'; + +export interface CalibrationPointData { + x: number; + y: number; + label: string; + pending: boolean; + /** Marker belongs to the selected correspondence (highlighted in both panes). */ + selected: boolean; + /** Owning correspondence id; undefined for the pending (in-progress) point. */ + correspondenceId?: number; +} + +export type { CameraImage }; + +interface CalibrationLayerParams { + calibration: CameraRegistrationStore; + /** Resolve another camera's currently displayed frame image (for the overlay). */ + getCameraImage?: (camera: string) => CameraImage | null; +} + +/** + * How many animation frames to keep re-checking whether the ghost source + * camera's displayed image element has changed after an update trigger (image + * sequences swap their quad datum asynchronously once the new frame loads). + * ~60 frames is about one second at typical refresh rates. + */ +const GHOST_REFRESH_MAX_ATTEMPTS = 60; + +/** Display-pixel radius within which a mousedown grabs an existing marker to drag-refine it. */ +const DRAG_HIT_RADIUS_PX = 10; + +/** + * Renders this camera's picked calibration points (numbered markers, the pending + * "blue" point highlighted) and, when alignment mode is active, a ghost overlay + * of the other camera's frame warped through the fitted homography (geojs + * quadFeature). One instance is created per camera in LayerManager. + */ +export default class CalibrationKeypointLayer extends BaseLayer { + calibration: CameraRegistrationStore; + + getCameraImage?: (camera: string) => CameraImage | null; + + /** The source element currently rendered as the ghost overlay, if any. */ + private ghostSource: HTMLImageElement | HTMLVideoElement | null = null; + + /** Pending requestAnimationFrame handle for the ghost staleness re-check loop. */ + private ghostRetryHandle: number | null = null; + + private ghostRetryAttempts = 0; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + textFeature: any; + + /** Small bright dot drawn at each marker's exact pixel (inside the ring). */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + centerFeature: any; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + quadFeature: any; + + /** The ghost quads' own feature layer: opacity is applied here (layer-level) + * so the seam-hiding cell overlap doesn't double-blend (see updateGhost). */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + quadLayer: any; + + /** + * Marker currently being drag-refined, or null/undefined. NOTE: these two + * fields deliberately have no initializers -- BaseLayer's constructor calls + * initialize() (which assigns mapNode) before subclass field initializers + * would run, so an `= null` here would wipe the assignment afterward (same + * reason textFeature/quadFeature above are declared bare). + */ + private dragTarget?: { correspondenceId?: number; pending: boolean } | null; + + private mapNode?: HTMLElement; + + private boundDragMove = (evt: MouseEvent) => this.handleDragMove(evt); + + private boundDragEnd = () => this.handleDragEnd(); + + constructor(params: BaseLayerParams & CalibrationLayerParams) { + super(params); + this.calibration = params.calibration; + this.getCameraImage = params.getCameraImage; + // Listen on the map, which is what emits geo.event.mouseclick. The event + // exposes button state and image (gcs) coordinates at the top level. + this.annotator.geoViewerRef.value.geoOn( + geo.event.mouseclick, + (e: GeoEvent) => this.handleClick(e), + ); + this.annotator.geoViewerRef.value.geoOn( + geo.event.mousemove, + (e: GeoEvent) => this.handleMouseMove(e), + ); + this.update(); + } + + initialize() { + const geoViewer = this.annotator.geoViewerRef.value; + // quad, point, and text features require different geojs renderers, so each + // must live in its own feature layer. Create the quad first so the overlay + // image renders beneath the picked points and labels. + const quadLayer = geoViewer.createLayer('feature', { + features: ['quad'], + autoshareRenderer: false, + renderer: 'canvas', + }); + this.quadLayer = quadLayer; + this.quadFeature = quadLayer.createFeature('quad'); + + const pointLayer = geoViewer.createLayer('feature', { features: ['point'] }); + this.featureLayer = pointLayer.createFeature('point'); + // A second point feature on the same layer, created after the ring so it + // renders on top: a small bright dot marking each marker's exact pixel. + // Unlocked points (the pending point being placed, or a selected one) are + // yellow; locked (committed) points are blue. + this.centerFeature = pointLayer.createFeature('point').style({ + fill: true, + fillColor: (data: CalibrationPointData) => ( + (data.selected || data.pending) ? 'yellow' : 'cyan'), + fillOpacity: 1, + radius: (data: CalibrationPointData) => (data.selected ? 3.5 : 2.5), + strokeColor: (data: CalibrationPointData) => ( + (data.selected || data.pending) ? 'orange' : 'blue'), + strokeWidth: 1, + }); + + // Drag-to-refine: a capture-phase mousedown on the map node runs before + // geojs' own interactor sees the event, so grabbing a marker can suppress + // the pan gesture (and the click that would otherwise place a new point). + [this.mapNode] = geoViewer.node(); + if (this.mapNode) { + this.mapNode.addEventListener('mousedown', (evt: MouseEvent) => this.handleDragStart(evt), true); + } + + const textLayer = geoViewer.createLayer('feature', { features: ['text'] }); + this.textFeature = textLayer + .createFeature('text') + .text((data: CalibrationPointData) => data.label) + .position((data: CalibrationPointData) => ({ x: data.x, y: data.y })) + .style({ + color: 'white', + fontSize: '14px', + textAlign: 'center', + textBaseline: 'bottom', + offset: { x: 0, y: -10 }, + }); + super.initialize(); + } + + /** + * Map-level click handler: left click records a point when picking is + * active for this camera; right click requests a recenter of both cameras + * in the active pair on the clicked location (see + * {@link CameraRegistrationStore.requestRecenter}) instead of picking. + */ + handleClick(e: GeoEvent) { + if (!this.calibration || !this.calibration.pickingEnabled.value) { + return; + } + // Map-level mouseclick exposes buttonsDown at the top level; feature-level + // events nest it under `mouse`. + const buttonsDown = e.buttonsDown || (e.mouse && e.mouse.buttonsDown); + const pair = this.calibration.activePair.value; + const cam = this.annotator.cameraName.value; + if (!pair || (cam !== pair.camA && cam !== pair.camB)) { + return; + } + if (!e.geo) { + return; + } + // e.geo is already in image (gcs) coordinates. + if (buttonsDown && buttonsDown.right) { + this.calibration.requestRecenter(cam, [e.geo.x, e.geo.y]); + return; + } + if (buttonsDown && !buttonsDown.left) { + return; + } + // Clicking empty space places a new point; it also clears any marker + // selection (a press ON a marker never reaches here -- handleDragStart + // captures it). + this.calibration.selectCorrespondence(null); + this.calibration.pickPoint(cam, [e.geo.x, e.geo.y]); + } + + /** + * Map-level mousemove handler: updates the live coordinate readout while + * picking is active, and shows a grab cursor over draggable markers. + */ + handleMouseMove(e: GeoEvent) { + if (!this.calibration || !this.calibration.pickingEnabled.value || !e.geo) { + return; + } + const pair = this.calibration.activePair.value; + const cam = this.annotator.cameraName.value; + if (!pair || (cam !== pair.camA && cam !== pair.camB)) { + return; + } + this.calibration.setCursorCoord(cam, [e.geo.x, e.geo.y]); + if (this.mapNode && !this.dragTarget && e.map) { + const hit = this.findMarkerAtDisplay(e.map); + this.mapNode.style.cursor = hit ? 'grab' : ''; + } + } + + /** Nearest rendered marker within DRAG_HIT_RADIUS_PX of a display-space point, or null. */ + private findMarkerAtDisplay(display: { x: number; y: number }): CalibrationPointData | null { + const map = this.annotator.geoViewerRef.value; + if (!map) { + return null; + } + let best: CalibrationPointData | null = null; + let bestDist = DRAG_HIT_RADIUS_PX; + this.formattedData.forEach((d) => { + const disp = map.gcsToDisplay({ x: d.x, y: d.y }); + const dist = Math.hypot(disp.x - display.x, disp.y - display.y); + if (dist <= bestDist) { + best = d; + bestDist = dist; + } + }); + return best; + } + + /** Display-space coordinates of a DOM mouse event relative to the map node. */ + private eventDisplayCoords(evt: MouseEvent): { x: number; y: number } | null { + if (!this.mapNode) { + return null; + } + const rect = this.mapNode.getBoundingClientRect(); + return { x: evt.clientX - rect.left, y: evt.clientY - rect.top }; + } + + /** + * Capture-phase mousedown: when picking is active and the press lands on an + * existing marker, start dragging it instead of letting geojs pan the map + * (or synthesize the click that would place a new point on top of it). + */ + private handleDragStart(evt: MouseEvent) { + if (evt.button !== 0 || !this.calibration || !this.calibration.pickingEnabled.value) { + return; + } + const pair = this.calibration.activePair.value; + const cam = this.annotator.cameraName.value; + if (!pair || (cam !== pair.camA && cam !== pair.camB)) { + return; + } + const display = this.eventDisplayCoords(evt); + if (!display) { + return; + } + const hit = this.findMarkerAtDisplay(display); + if (!hit) { + return; + } + evt.preventDefault(); + evt.stopImmediatePropagation(); + // Grabbing a marker selects its correspondence (highlighted in both + // panes, deletable via the panel or the Delete key); grabbing the + // pending point clears the selection. + this.calibration.selectCorrespondence(hit.correspondenceId ?? null); + this.dragTarget = { correspondenceId: hit.correspondenceId, pending: hit.pending }; + if (this.mapNode) { + this.mapNode.style.cursor = 'grabbing'; + } + window.addEventListener('mousemove', this.boundDragMove); + window.addEventListener('mouseup', this.boundDragEnd); + } + + private handleDragMove(evt: MouseEvent) { + if (!this.dragTarget || !this.calibration) { + return; + } + const map = this.annotator.geoViewerRef.value; + const display = this.eventDisplayCoords(evt); + if (!map || !display) { + return; + } + const gcs = map.displayToGcs(display); + const cam = this.annotator.cameraName.value; + if (this.dragTarget.pending) { + this.calibration.movePendingPoint(cam, [gcs.x, gcs.y]); + } else if (this.dragTarget.correspondenceId !== undefined) { + this.calibration.updateCorrespondencePoint(this.dragTarget.correspondenceId, cam, [gcs.x, gcs.y]); + } + } + + private handleDragEnd() { + this.dragTarget = null; + if (this.mapNode) { + this.mapNode.style.cursor = ''; + } + window.removeEventListener('mousemove', this.boundDragMove); + window.removeEventListener('mouseup', this.boundDragEnd); + } + + // eslint-disable-next-line class-methods-use-this, @typescript-eslint/no-unused-vars + formatData(_frameData: FrameDataTrack[]): CalibrationPointData[] { + const result: CalibrationPointData[] = []; + if (!this.calibration) { + return result; + } + // Point markers are authoring UI: they show only while picking is on + // (the correspondences themselves stay in the store either way). + if (!this.calibration.pickingEnabled.value) { + return result; + } + const pair = this.calibration.activePair.value; + const cam = this.annotator.cameraName.value; + if (!pair || (cam !== pair.camA && cam !== pair.camB)) { + return result; + } + const key = this.calibration.pairKey(pair.camA, pair.camB); + const list = this.calibration.correspondences.value[key] || []; + const selectedId = this.calibration.selectedCorrespondenceId.value; + list.forEach((c, i) => { + const coord = cam === pair.camA ? c.a : c.b; + result.push({ + x: coord[0], + y: coord[1], + label: `${i + 1}`, + pending: false, + selected: c.id === selectedId, + correspondenceId: c.id, + }); + }); + const pending = this.calibration.pendingPoint.value; + if (pending && pending.camera === cam) { + result.push({ + x: pending.coord[0], + y: pending.coord[1], + label: `${list.length + 1}`, + pending: true, + selected: false, + }); + } + return result; + } + + /** + * Render (or clear) the aligned ghost overlay. The ghost is drawn in the + * destination camera's pane: the source camera's image is warped through the + * fitted `homog[mode]` matrix -- the same matrix that + * {@link CameraRegistrationStore.pickPoint} inverts to attribute a ghost-pane + * click back to the source camera, so rendering and click attribution use the + * same exact projective mapping. Because geojs' canvas quad renderer is + * affine-only (a single quad is drawn as a parallelogram from three of its + * corners), a transform with non-negligible perspective terms is rendered as + * an n x n grid of sub-quads whose corners are each mapped through the exact + * homography (see {@link subdivideWarpQuads}); each sub-quad is approximately + * affine, so the rendered warp matches the projective mapping to sub-pixel + * accuracy and ghost-targeted picks land where the user visually aligned. + */ + updateGhost() { + if (!this.quadFeature) { + return; + } + const clear = () => { + this.ghostSource = null; + this.cancelGhostRefresh(); + this.quadFeature.data([]).draw(); + }; + const alignment = this.calibration?.alignment.value; + const pair = this.calibration?.activePair.value; + if (!alignment || alignment.mode === 'original' || !pair || !this.getCameraImage) { + clear(); + return; + } + const key = this.calibration.pairKey(pair.camA, pair.camB); + const homog = this.calibration.homographies.value[key]; + if (!homog) { + clear(); + return; + } + const { mode } = alignment; + const srcCam = mode === 'BtoA' ? pair.camB : pair.camA; + const dstCam = mode === 'BtoA' ? pair.camA : pair.camB; + if (this.annotator.cameraName.value !== dstCam) { + clear(); + return; + } + const src = this.getCameraImage(srcCam); + if (!src || !src.width || !src.height) { + clear(); + // The source pane's frame may simply not have finished loading yet; + // keep re-checking briefly (see scheduleGhostRefresh). + this.scheduleGhostRefresh(srcCam); + return; + } + const h = homog[mode]; + const { width: w, height: hgt } = src; + const grid = warpGridSize(h, w, hgt); + // 2px cell overlap hides the canvas antialiasing seams between abutting + // sub-quads (dark grid lines). Overlapped quads must draw opaque -- the + // ghost's transparency is applied once at the layer level below, so the + // overlap doesn't double-blend into brighter seams. + const quads = subdivideWarpQuads(h, w, hgt, grid, 2).map((q) => ({ + ul: { x: q.ul[0], y: q.ul[1] }, + ur: { x: q.ur[0], y: q.ur[1] }, + lr: { x: q.lr[0], y: q.lr[1] }, + ll: { x: q.ll[0], y: q.ll[1] }, + // geojs crop: left/top/right/bottom select the source-pixel region; + // x/y (the "size after crop") are set to the full source size so that + // region stretches across the whole sub-quad. + crop: { + ...q.crop, x: w, y: hgt, + }, + [src.kind]: src.source, + })); + this.ghostSource = src.source; + this.quadLayer.opacity(alignment.opacity); + this.quadFeature + .data(quads) + .style('opacity', 1) + .draw(); + if (src.kind === 'image') { + // Image sequences swap the source pane's asynchronously after the + // frame finishes loading, with no event reaching this layer; poll + // briefly so the ghost catches up (video elements update in place). + this.scheduleGhostRefresh(srcCam); + } else { + this.cancelGhostRefresh(); + } + } + + /** Cancel any pending ghost staleness re-check. */ + private cancelGhostRefresh() { + if (this.ghostRetryHandle !== null) { + cancelAnimationFrame(this.ghostRetryHandle); + this.ghostRetryHandle = null; + } + } + + /** + * Bounded requestAnimationFrame loop that re-checks whether `srcCam`'s + * displayed image element differs from the one the ghost was rendered from, + * and re-renders the ghost when it does. This covers the gap between a frame + * change (which triggers {@link update} immediately) and the moment + * ImageAnnotator actually swaps the loaded into its quad datum. + */ + private scheduleGhostRefresh(srcCam: string) { + this.cancelGhostRefresh(); + this.ghostRetryAttempts = 0; + if (typeof requestAnimationFrame !== 'function') { + return; + } + const tick = () => { + this.ghostRetryHandle = null; + const src = this.getCameraImage ? this.getCameraImage(srcCam) : null; + if (src && src.source && src.width && src.height && src.source !== this.ghostSource) { + // Re-render with the new element; updateGhost restarts this loop. + this.updateGhost(); + return; + } + this.ghostRetryAttempts += 1; + if (this.ghostRetryAttempts < GHOST_REFRESH_MAX_ATTEMPTS) { + this.ghostRetryHandle = requestAnimationFrame(tick); + } + }; + this.ghostRetryHandle = requestAnimationFrame(tick); + } + + /** Recompute points and the ghost overlay from the store and redraw. */ + update() { + this.formattedData = this.formatData([]); + this.redraw(); + this.updateGhost(); + } + + redraw(): null { + this.featureLayer.data(this.formattedData).draw(); + this.centerFeature.data(this.formattedData).draw(); + this.textFeature.data(this.formattedData).draw(); + return null; + } + + disable() { + this.featureLayer.data([]).draw(); + this.centerFeature.data([]).draw(); + this.textFeature.data([]).draw(); + this.ghostSource = null; + this.cancelGhostRefresh(); + if (this.quadFeature) { + this.quadFeature.data([]).draw(); + } + } + + // eslint-disable-next-line class-methods-use-this + createStyle(): LayerStyle { + return { + ...super.createStyle(), + // Hollow ring: leave the center transparent so the exact pixel being + // picked stays visible through the marker (a solid disc hides the very + // target it marks). A bright center dot (see centerFeature) marks the + // exact pixel; the dark ring + bright dot stay legible on both light and + // dark imagery. Unlocked points (pending or selected) are yellow; locked + // (committed) points are blue. The selected point gets a larger ring. + fill: false, + fillOpacity: 0, + radius: (data: CalibrationPointData) => (data.selected ? 9 : 7), + strokeColor: (data: CalibrationPointData) => ( + (data.selected || data.pending) ? 'orange' : 'blue'), + strokeWidth: (data: CalibrationPointData) => (data.selected ? 3 : 2), + }; + } +} From c129620b17d44f4b458c5c491166d75bfb80ae55 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Thu, 9 Jul 2026 16:17:33 -0400 Subject: [PATCH 02/13] Add the Manual Alignment panel for in-app calibration creation CalibrationTools.vue (modeled on VIAME's keypointgui): pick point pairs across a camera pair, fit the chosen transform model with a fit-readiness countdown, preview the fit as a ghost overlay with warp opacity, review rig alignment status (N/M cameras calibrated), and save/load the calibration -- to dataset meta and the portable calibration.json. Wired into the Viewer as a context panel: workspace chrome minimizes while calibrating, only the active pair's panes show on 3+ camera rigs, the general Align warp suspends while picking so picks are always native-space, and LayerManager mounts the keypoint marker/ghost layer per pane. Co-Authored-By: Claude Fable 5 --- .../components/AlignedViewToggle.vue | 7 +- .../CameraCalibration/CalibrationTools.vue | 678 ++++++++++++++++++ client/dive-common/components/Viewer.vue | 83 ++- client/dive-common/store/context.ts | 5 + client/src/components/LayerManager.vue | 62 ++ 5 files changed, 831 insertions(+), 4 deletions(-) create mode 100644 client/dive-common/components/CameraCalibration/CalibrationTools.vue diff --git a/client/dive-common/components/AlignedViewToggle.vue b/client/dive-common/components/AlignedViewToggle.vue index a28230f93..47d308d09 100644 --- a/client/dive-common/components/AlignedViewToggle.vue +++ b/client/dive-common/components/AlignedViewToggle.vue @@ -37,9 +37,14 @@ export default defineComponent({ alignedView.setEnabled(!alignedView.enabled.value); }; + // The aligned view is suspended while picking, so the button reads as + // unavailable rather than accepting a toggle that has no visible effect. + const pickingEnabled = computed(() => cameraRegistration.pickingEnabled.value); + return { available, enabled, + pickingEnabled, tooltip, toggle, }; @@ -57,7 +62,7 @@ export default defineComponent({ small class="mx-1" :color="enabled ? 'primary' : 'default'" - :disabled="!available" + :disabled="!available || pickingEnabled" @click="toggle" > diff --git a/client/dive-common/components/CameraCalibration/CalibrationTools.vue b/client/dive-common/components/CameraCalibration/CalibrationTools.vue new file mode 100644 index 000000000..379cb4a0c --- /dev/null +++ b/client/dive-common/components/CameraCalibration/CalibrationTools.vue @@ -0,0 +1,678 @@ + + + diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index ca49ebd5b..3cfb73553 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -31,6 +31,7 @@ import { LargeImageAnnotator, LayerManager, useMediaController, + useCalibrationNavigation, useAlignedNavigation, TrackList, FilterList, @@ -87,6 +88,7 @@ import context from 'dive-common/store/context'; import { MarkChangesPendingFilter } from 'vue-media-annotator/BaseFilterControls'; import GroupSidebarVue from './GroupSidebar.vue'; import MultiCamToolsVue from './MultiCamTools.vue'; +import CalibrationToolsVue from './CameraCalibration/CalibrationTools.vue'; import MultiCamToolbar from './MultiCamToolbar.vue'; import AlignedViewToggle from './AlignedViewToggle.vue'; import PrimaryAttributeTrackFilter from './PrimaryAttributeTrackFilter.vue'; @@ -323,6 +325,29 @@ export default defineComponent({ }); const showUserSettingsDialog = ref(false); + // When the Manual Alignment panel opens, minimize the workspace chrome to + // give the picking view more room: collapse the left type-filter sidebar and + // the bottom detections graph. This is a soft default -- the normal sidebar + // and timeline toggles still work while calibrating, so the user can bring + // either back -- and whatever layout they had before is restored on close. + const calibrationActive = computed(() => context.state.active === CalibrationToolsVue.name); + let preCalibrationSidebarMode: 'left' | 'bottom' | 'collapsed' | null = null; + let preCalibrationControlsCollapsed = false; + watch(calibrationActive, (active) => { + if (active) { + preCalibrationSidebarMode = sidebarMode.value; + preCalibrationControlsCollapsed = controlsCollapsed.value; + if (sidebarMode.value === 'left') { + sidebarMode.value = 'collapsed'; + } + controlsCollapsed.value = true; + } else if (preCalibrationSidebarMode !== null) { + sidebarMode.value = preCalibrationSidebarMode; + controlsCollapsed.value = preCalibrationControlsCollapsed; + preCalibrationSidebarMode = null; + } + }); + watch(sidebarMode, (mode) => { if (mode === 'left' || mode === 'bottom') { clientSettings.layoutSettings.sidebarPosition = mode; @@ -411,8 +436,10 @@ export default defineComponent({ * review. Reference camera = the Reference Camera chosen at import * (stored as defaultDisplay), falling back to the first camera in * display order. Transforms come from the registration store's pair - * homographies (loaded from a registration file or the dataset's saved - * meta), composed through the pair graph. + * homographies (picked in-app via the Manual Alignment panel, or loaded + * from a registration file or the dataset's saved meta), composed + * through the pair graph -- the single registration the panel edits and + * saves is exactly what the Align button applies. * * Store instances are always created (provideAnnotator runs before * loadData resolves), but watches, aligned navigation, and metadata @@ -448,6 +475,16 @@ export default defineComponent({ selectedCamera, setResetZoomOverride, }); + // The Manual Alignment pair link maps through the homography for + // UNWARPED panes, so it needs the aligned view state to stand down + // while displays are warped into reference space. + useCalibrationNavigation(aggregateController, cameraRegistration, alignedView); + // Registration point picking records raw native-space clicks and + // renders its own aligned preview; suspend the general warp while it + // is active so picks are never taken against a warped display. + watch(cameraRegistration.pickingEnabled, (picking) => { + alignedView.setSuspended(picking); + }, { immediate: true }); // Publish the reference even while unresolved so UI outside the viewer // core (e.g. the import menu's per-pair buttons) can name it. watch([alignedResolution, referenceCamera], ([resolution, reference]) => { @@ -497,6 +534,31 @@ export default defineComponent({ const unresolved = unresolvedCameras(cams, reference, cameraRegistration.homographies.value); return { registered: cams.length - unresolved.length, total: cams.length }; }); + /** + * Camera panes currently displayed. While the Manual Alignment panel is + * open with an active pair on a 3+ camera dataset, only the pair's two + * panes show, so the left/right alignment flow reads without unrelated + * panes in between (regardless of whether Pick points is toggled on). + * Panes are hidden (v-show), not unmounted, so their viewers keep state. + */ + const displayedCameras = computed(() => { + const pair = cameraRegistration.activePair.value; + if (calibrationActive.value && pair) { + const pairCameras = multiCamList.value.filter( + (camera) => camera === pair.camA || camera === pair.camB, + ); + if (pairCameras.length === 2) { + return pairCameras; + } + } + return multiCamList.value; + }); + watch(displayedCameras, async () => { + // Hidden/shown siblings change the remaining panes' sizes; resize the + // geojs maps once the DOM has settled. + await nextTick(); + handleResize(); + }); // This context for removal const removeGroups = (id: AnnotationId) => { cameraStore.removeGroups(id); @@ -1523,11 +1585,19 @@ export default defineComponent({ component: MultiCamToolsVue, description: 'Multi Camera Tools', }); + context.register({ + component: CalibrationToolsVue, + description: 'Manual Alignment', + }); } else { context.unregister({ component: MultiCamToolsVue, description: 'Multi Camera Tools', }); + context.unregister({ + component: CalibrationToolsVue, + description: 'Manual Alignment', + }); context.register({ description: 'Group Manager', component: GroupSidebarVue, @@ -1938,6 +2008,7 @@ export default defineComponent({ deleteAttributeHandler, saveTooltipText, showMultiCamToolbar, + displayedCameras, seekToFrame, resetAggregateZoom, /* large image methods */ @@ -2251,10 +2322,16 @@ export default defineComponent({ class="d-flex flex-column grow" >
+
= { description: 'Multi Camera Tools', component: MultiCamTools, }, + [CalibrationTools.name]: { + description: 'Manual Alignment', + component: CalibrationTools, + }, [AttributesSideBar.name]: { description: 'Attribute Details', component: AttributesSideBar, diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 834709ba4..cbde0fcd0 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -12,6 +12,7 @@ import PointLayer from '../layers/AnnotationLayers/PointLayer'; import LineLayer from '../layers/AnnotationLayers/LineLayer'; import TailLayer from '../layers/AnnotationLayers/TailLayer'; import OverlapLayer from '../layers/AnnotationLayers/OverlapLayer'; +import CalibrationKeypointLayer from '../layers/AnnotationLayers/CalibrationKeypointLayer'; import EditAnnotationLayer, { EditAnnotationTypes } from '../layers/EditAnnotationLayer'; import LassoSelectionLayer from '../layers/LassoSelectionLayer'; @@ -36,6 +37,7 @@ import { useAnnotatorPreferences, useGroupStyleManager, useCameraStore, + useCameraRegistration, useAlignedView, useSelectedCamera, useAttributes, @@ -45,6 +47,7 @@ import { } from '../provides'; import SegmentationPointsLayer from '../layers/AnnotationLayers/SegmentationPointsLayer'; import useLayerManagerAlignedView from './layerManager/useLayerManagerAlignedView'; +import { getCameraQuadMedia } from './layerManager/quadMediaSource'; import useLayerRefresh from './layerManager/useLayerRefresh'; import useSegmentationPointsLayer from './layerManager/useSegmentationPointsLayer'; import useAnnotationClickHandling from './layerManager/useAnnotationClickHandling'; @@ -80,6 +83,12 @@ export default defineComponent({ // Viewer may not provide lasso context in tests or minimal embeds. } const cameraStore = useCameraStore(); + let cameraCalibration: ReturnType | undefined; + try { + cameraCalibration = useCameraRegistration(); + } catch { + // calibration store may not be provided in tests or minimal embeds. + } let alignedView: ReturnType | undefined; try { alignedView = useAlignedView(); @@ -202,6 +211,59 @@ export default defineComponent({ const segmentationPointsRef = useSegmentationPoints(); const segmentationPointsLayer = new SegmentationPointsLayer(annotator); + const calibrationLayer = cameraCalibration + ? new CalibrationKeypointLayer({ + annotator, + stateStyling: trackStyleManager.stateStyles, + typeStyling: typeStylingRef, + calibration: cameraCalibration, + getCameraImage: (cam: string) => getCameraQuadMedia( + (c) => aggregateController.value.getController(c), + cam, + ), + }) + : undefined; + + if (cameraCalibration && calibrationLayer) { + const calibration = cameraCalibration; + /** + * Frame number of the camera whose image is being ghosted into another + * pane, or null when no ghost is active. Watched so the ghost re-renders + * when the *source* pane scrubs, not just this pane -- this pane's own + * frameNumberRef can update before (or without) the source's, and the + * source image element itself only swaps after its frame finishes + * loading (see CalibrationKeypointLayer.scheduleGhostRefresh). + */ + const ghostSourceFrame = computed(() => { + const { mode } = calibration.alignment.value; + const pair = calibration.activePair.value; + if (mode === 'original' || !pair) { + return null; + } + const srcCam = mode === 'BtoA' ? pair.camB : pair.camA; + try { + return aggregateController.value.getController(srcCam).frame.value; + } catch { + return null; + } + }); + watch( + [ + cameraCalibration.activePair, + cameraCalibration.pickingEnabled, + cameraCalibration.correspondences, + cameraCalibration.pendingPoint, + cameraCalibration.selectedCorrespondenceId, + cameraCalibration.homographies, + cameraCalibration.alignment, + frameNumberRef, + ghostSourceFrame, + ], + () => calibrationLayer.update(), + { deep: true }, + ); + } + const updateAttributes = () => { const newList = attributes.value.filter((item) => item.render).sort((a, b) => { if (a.render && b.render) { From f2091ccef0f13fe9b5d779997b9e4223ffd6a3e1 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Sat, 11 Jul 2026 23:55:38 -0400 Subject: [PATCH 03/13] Adopt image-registration naming in the Manual Alignment feature Rename the feature's own files and identifiers to the registration vocabulary the rest of the branch adopted: CalibrationTools -> RegistrationTools (CameraRegistration/), CalibrationKeypointLayer -> RegistrationKeypointLayer, useCalibrationNavigation -> useRegistrationNavigation, and the calibration-named locals, props, and UI strings that referenced them. Stereo calibration (a genuinely different artifact) keeps its name. Reuse shared helpers where the feature had its own copies: the panel's unknown-camera notice now comes from unknownCameraWarning, and the geojs warp-quad mapping duplicated between AlignedImageLayer and the keypoint ghost is extracted to geojsWarpQuads in alignedView/homography. The linkedViewers README's "planned" registration-pair-link section now describes the shipped composable. Co-Authored-By: Claude Fable 5 --- .../RegistrationTools.vue} | 189 +++++++++--------- client/dive-common/components/Viewer.vue | 34 ++-- client/dive-common/store/context.ts | 2 +- .../alignedView/CameraRegistrationStore.ts | 10 +- client/src/alignedView/README.md | 2 +- client/src/alignedView/homography.spec.ts | 29 +++ client/src/alignedView/homography.ts | 43 ++++ client/src/components/LayerManager.vue | 40 ++-- .../annotators/linkedViewers/README.md | 14 +- .../linkedViewers/useAlignedNavigation.ts | 4 +- .../linkedViewers/useLinkedViewers.ts | 4 +- ...c.ts => useRegistrationNavigation.spec.ts} | 8 +- ...gation.ts => useRegistrationNavigation.ts} | 28 +-- .../annotators/mediaControllerType.ts | 2 +- client/src/components/index.ts | 2 +- client/src/layers/AlignedImageLayer.ts | 22 +- ...tLayer.ts => RegistrationKeypointLayer.ts} | 112 +++++------ 17 files changed, 296 insertions(+), 249 deletions(-) rename client/dive-common/components/{CameraCalibration/CalibrationTools.vue => CameraRegistration/RegistrationTools.vue} (76%) rename client/src/components/annotators/linkedViewers/{useCalibrationNavigation.spec.ts => useRegistrationNavigation.spec.ts} (95%) rename client/src/components/annotators/linkedViewers/{useCalibrationNavigation.ts => useRegistrationNavigation.ts} (85%) rename client/src/layers/AnnotationLayers/{CalibrationKeypointLayer.ts => RegistrationKeypointLayer.ts} (82%) diff --git a/client/dive-common/components/CameraCalibration/CalibrationTools.vue b/client/dive-common/components/CameraRegistration/RegistrationTools.vue similarity index 76% rename from client/dive-common/components/CameraCalibration/CalibrationTools.vue rename to client/dive-common/components/CameraRegistration/RegistrationTools.vue index 379cb4a0c..b52f5f5fb 100644 --- a/client/dive-common/components/CameraCalibration/CalibrationTools.vue +++ b/client/dive-common/components/CameraRegistration/RegistrationTools.vue @@ -11,17 +11,18 @@ import { TransformType, TRANSFORM_TYPES, DEFAULT_TRANSFORM_TYPE, minPointsForTransform, } from 'vue-media-annotator/alignedView/transform'; import { unresolvedCameras } from 'vue-media-annotator/alignedView/alignedView'; +import { unknownCameraWarning } from 'vue-media-annotator/alignedView/cameraRegistrationFiles'; import TooltipBtn from 'vue-media-annotator/components/TooltipButton.vue'; import { useApi } from 'dive-common/apispec'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; export default defineComponent({ - name: 'CameraCalibration', + name: 'CameraRegistration', description: 'Manual Alignment', components: { TooltipBtn }, setup() { const cameraStore = useCameraStore(); - const calibration = useCameraRegistration(); + const registration = useCameraRegistration(); const datasetId = useDatasetId(); const { saveMetadata } = useApi(); const { prompt } = usePrompt(); @@ -31,7 +32,7 @@ export default defineComponent({ * Per-camera alignment status for the whole rig, driving the status block: * the first camera (display order) is the reference (identity); every other * camera is 'resolved' when it has a fitted path to the reference, else - * 'unresolved' (still needs calibration to satisfy the Align button). + * 'unresolved' (still needs registration to satisfy the Align button). */ const cameraAlignmentStatuses = computed(() => { const list = cameras.value; @@ -40,7 +41,7 @@ export default defineComponent({ return [] as { name: string; status: 'reference' | 'resolved' | 'unresolved' }[]; } const unresolved = new Set( - unresolvedCameras(list, reference, calibration.homographies.value), + unresolvedCameras(list, reference, registration.homographies.value), ); return list.map((name) => { let status: 'reference' | 'resolved' | 'unresolved' = 'resolved'; @@ -61,7 +62,7 @@ export default defineComponent({ return { icon: complete ? 'mdi-check-circle' : 'mdi-alert', color: complete ? 'success' : 'warning', - text: `${total - unresolvedCount}/${total} cameras calibrated`, + text: `${total - unresolvedCount}/${total} cameras registered`, }; }); const camLeft = ref(null); @@ -74,20 +75,20 @@ export default defineComponent({ } watch([camLeft, camRight], () => { - calibration.setActivePair(camLeft.value, camRight.value); + registration.setActivePair(camLeft.value, camRight.value); }, { immediate: true }); // Picking is scoped to this panel: it is always on while the Manual // Alignment panel is open, and turns off when the panel closes (unmounts), // so the viewer can't be left in picking mode -- pair-only panes, suspended // aligned view -- with no visible control to get back out. - calibration.pickingEnabled.value = true; + registration.pickingEnabled.value = true; onBeforeUnmount(() => { - calibration.pickingEnabled.value = false; + registration.pickingEnabled.value = false; }); // Switching datasets while this panel stays mounted re-runs the viewer's - // loadDataset -> calibration.hydrate(), which clears picking and the active + // loadDataset -> registration.hydrate(), which clears picking and the active // pair. This panel establishes those only at mount, so without this it // would be left visibly open but inert (dead markers, stale selectors). // Re-establish them whenever the camera set changes. @@ -102,38 +103,38 @@ export default defineComponent({ } else { // Same camera names as before: the selectors don't change, so // re-establish the pair that hydrate() nulled. - calibration.setActivePair(camLeft.value, camRight.value); + registration.setActivePair(camLeft.value, camRight.value); } - calibration.pickingEnabled.value = true; + registration.pickingEnabled.value = true; }); - const activeKey = computed(() => calibration.activePairKey()); + const activeKey = computed(() => registration.activePairKey()); const correspondences = computed(() => { const key = activeKey.value; - return key ? (calibration.correspondences.value[key] || []) : []; + return key ? (registration.correspondences.value[key] || []) : []; }); const transformType = computed( () => (activeKey.value - ? calibration.transformTypeForPair(activeKey.value) + ? registration.transformTypeForPair(activeKey.value) : DEFAULT_TRANSFORM_TYPE), ); const minPoints = computed(() => minPointsForTransform(transformType.value)); const canFit = computed(() => correspondences.value.length >= minPoints.value); - const selectedCorrespondenceId = computed(() => calibration.selectedCorrespondenceId.value); + const selectedCorrespondenceId = computed(() => registration.selectedCorrespondenceId.value); /** * Delete the unlocked point on Del/Backspace: the selected correspondence * (both cameras' points) if one is selected, otherwise the pending point * that is mid-placement. */ function deleteSelectedCorrespondence() { - if (calibration.selectedCorrespondenceId.value !== null) { - calibration.removeSelectedCorrespondence(); - } else if (calibration.pendingPoint.value !== null) { - calibration.clearLast(); + if (registration.selectedCorrespondenceId.value !== null) { + registration.removeSelectedCorrespondence(); + } else if (registration.pendingPoint.value !== null) { + registration.clearLast(); } } const canClearLast = computed( - () => calibration.pendingPoint.value !== null || correspondences.value.length > 0, + () => registration.pendingPoint.value !== null || correspondences.value.length > 0, ); /** How many more correspondence pairs are needed before the transform can be fit. */ const remainingPoints = computed(() => Math.max(0, minPoints.value - correspondences.value.length)); @@ -149,22 +150,22 @@ export default defineComponent({ }); /** The active pair has a usable transform: enough points to fit one, or one loaded from a file. */ const hasTransform = computed(() => canFit.value - || Boolean(activeKey.value && calibration.homographies.value[activeKey.value])); - /** The active pair's transform came from a calibration file (no in-app fit backing it). */ + || Boolean(activeKey.value && registration.homographies.value[activeKey.value])); + /** The active pair's transform came from a registration file (no in-app fit backing it). */ const hasLoadedTransform = computed(() => { const key = activeKey.value; - return Boolean(key && calibration.homographies.value[key] && calibration.isLoadedHomography(key)); + return Boolean(key && registration.homographies.value[key] && registration.isLoadedHomography(key)); }); /** - * The active pair was refit in-app while a producer-stamped calibration is + * The active pair was refit in-app while a producer-stamped registration is * loaded, so its transform has diverged from what the stamped source * shipped -- worth saving and sending back to the producer. */ const refinedFromSource = computed(() => { const key = activeKey.value; // Touch homographies so provenance changes recompute this (see store docs). - return Boolean(key && calibration.homographies.value[key] - && calibration.isRefinedFromSource(key)); + return Boolean(key && registration.homographies.value[key] + && registration.isRefinedFromSource(key)); }); const canClearPair = computed( () => correspondences.value.length > 0 || hasLoadedTransform.value, @@ -179,21 +180,21 @@ export default defineComponent({ function setTransformType(type: TransformType) { const key = activeKey.value; if (key) { - calibration.setTransformType(key, type); + registration.setTransformType(key, type); } } function setAlignmentMode(mode: 'original' | 'AtoB' | 'BtoA') { - calibration.setAlignmentMode(mode); + registration.setAlignmentMode(mode); } /** - * Human-readable summary of the loaded calibration's provenance stamp + * Human-readable summary of the loaded registration's provenance stamp * (scalar entries only -- nested structures are preserved in the file but * not displayed). */ const sourceReadout = computed(() => { - const source = calibration.source.value; + const source = registration.source.value; if (!source) { return null; } @@ -205,12 +206,12 @@ export default defineComponent({ /** Live cursor readout text: this camera's coord, and its linked point in the other camera. */ const cursorReadout = computed(() => { - const cursor = calibration.cursorCoord.value; + const cursor = registration.cursorCoord.value; if (!cursor) { return null; } const [x, y] = cursor.coord; - const other = calibration.linkedPoint(cursor.camera, cursor.coord); + const other = registration.linkedPoint(cursor.camera, cursor.coord); const here = `${cursor.camera}: (${x.toFixed(1)}, ${y.toFixed(1)})`; if (!other) { return here; @@ -220,39 +221,39 @@ export default defineComponent({ }); /** - * Persist the calibration (all pairs) with the dataset: it is written as - * the project's per-camera calibration_.json files and restored + * Persist the registration (all pairs) with the dataset: it is written as + * the project's per-camera _to__registration.json files and restored * on every dataset load (so the Align button works across sessions). * Deliberately not gated on the * active pair having correspondences: saving must also be able to persist - * a cleared state (so stale saved calibration doesn't survive Clear All / + * a cleared state (so stale saved registration doesn't survive Clear All / * per-row deletes) and state belonging to non-active pairs. Use - * {@link exportCalibration} to get a portable copy for sharing. + * {@link exportRegistration} to get a portable copy for sharing. */ async function save() { saving.value = true; try { - calibration.maybeFitActivePair(); + registration.maybeFitActivePair(); await saveMetadata(datasetId.value, { - cameraHomographies: calibration.homographies.value, - cameraCorrespondences: calibration.correspondences.value, - cameraTransformTypes: calibration.transformTypes.value, - cameraRegistrationSource: calibration.source.value, + cameraHomographies: registration.homographies.value, + cameraCorrespondences: registration.correspondences.value, + cameraTransformTypes: registration.transformTypes.value, + cameraRegistrationSource: registration.source.value, }); - calibration.markSaved(); + registration.markSaved(); } finally { saving.value = false; } } /** - * Download the calibration as a portable .json -- for handing refinements + * Download the registration as a portable .json -- for handing refinements * (points included) back to an external producer, or reusing on another * dataset. Saving is separate: see {@link save}. */ - function exportCalibration() { - calibration.maybeFitActivePair(); - downloadText(calibration.toRegistrationJson(), 'camera-registration.json'); + function exportRegistration() { + registration.maybeFitActivePair(); + downloadText(registration.toRegistrationJson(), 'camera-registration.json'); } /** Trigger a browser download of `text` as `filename`. */ @@ -268,47 +269,43 @@ export default defineComponent({ URL.revokeObjectURL(url); } - const calibrationFileInput = ref(null); - const loadCalibrationError = ref(null); - const loadCalibrationWarning = ref(null); + const registrationFileInput = ref(null); + const loadRegistrationError = ref(null); + const loadRegistrationWarning = ref(null); - const hasAnyCalibration = computed( - () => Object.values(calibration.correspondences.value).some((list) => list.length > 0) - || Object.keys(calibration.homographies.value).length > 0, + const hasAnyRegistration = computed( + () => Object.values(registration.correspondences.value).some((list) => list.length > 0) + || Object.keys(registration.homographies.value).length > 0, ); - /** Load a calibration .json, replacing every pair's state. */ - async function loadJsonCalibration(file: File) { + /** Load a registration .json, replacing every pair's state. */ + async function loadJsonRegistration(file: File) { const text = await file.text(); - if (hasAnyCalibration.value) { + if (hasAnyRegistration.value) { const confirmed = await prompt({ - title: 'Load calibration', - text: 'This will replace the current calibration for ALL camera pairs. Continue?', + title: 'Load registration', + text: 'This will replace the current registration for ALL camera pairs. Continue?', confirm: true, }); if (!confirmed) { return; } } - const { cameras: fileCameras } = calibration.loadRegistrationText(text); - const known = new Set(cameras.value); - const unknown = fileCameras.filter((name) => !known.has(name)); - if (unknown.length) { - loadCalibrationWarning.value = `Loaded, but these cameras are not in this dataset: ${unknown.join(', ')}`; - } + const { cameras: fileCameras } = registration.loadRegistrationText(text); + loadRegistrationWarning.value = unknownCameraWarning(file.name, fileCameras, cameras.value); } - function onCalibrationFileSelected(event: Event) { + function onRegistrationFileSelected(event: Event) { const input = event.target as HTMLInputElement; const file = input.files?.[0]; input.value = ''; if (!file) { return; } - loadCalibrationError.value = null; - loadCalibrationWarning.value = null; - loadJsonCalibration(file).catch((err) => { - loadCalibrationError.value = err instanceof Error ? err.message : String(err); + loadRegistrationError.value = null; + loadRegistrationWarning.value = null; + loadJsonRegistration(file).catch((err) => { + loadRegistrationError.value = err instanceof Error ? err.message : String(err); }); } @@ -318,10 +315,10 @@ export default defineComponent({ alignmentSummary, camLeft, camRight, - calibration, - pickingEnabled: calibration.pickingEnabled, - alignment: calibration.alignment, - fitError: calibration.fitError, + registration, + pickingEnabled: registration.pickingEnabled, + alignment: registration.alignment, + fitError: registration.fitError, selectedCorrespondenceId, deleteSelectedCorrespondence, cursorReadout, @@ -338,18 +335,18 @@ export default defineComponent({ canClearLast, canFit, fitQualityColor, - linkedNav: calibration.linkedNav, - dirty: calibration.dirty, + linkedNav: registration.linkedNav, + dirty: registration.dirty, saving, sourceReadout, - calibrationFileInput, - loadCalibrationError, - loadCalibrationWarning, + registrationFileInput, + loadRegistrationError, + loadRegistrationWarning, setTransformType, setAlignmentMode, save, - exportCalibration, - onCalibrationFileSelected, + exportRegistration, + onRegistrationFileSelected, }; }, }); @@ -369,32 +366,32 @@ export default defineComponent({ - Load calibration + Load registration - {{ loadCalibrationError }} + {{ loadRegistrationError }} - {{ loadCalibrationWarning }} + {{ loadRegistrationWarning }} - This pair has been refined in-app since the source calibration was - produced. Export the calibration to hand the refinement (and its + This pair has been refined in-app since the source registration was + produced. Export the registration to hand the refinement (and its points) back to the producer. @@ -498,14 +495,14 @@ export default defineComponent({ icon="mdi-undo" :disabled="!canClearLast" tooltip-text="Undo the pending point, or the last completed pair" - @click="calibration.clearLast()" + @click="registration.clearLast()" />
{{ i + 1 }} {{ c.a[0].toFixed(1) }}, {{ c.a[1].toFixed(1) }} @@ -539,7 +536,7 @@ export default defineComponent({ color="error" icon="mdi-delete" tooltip-text="Remove this pair" - @click="calibration.removeCorrespondence(c.id)" + @click="registration.removeCorrespondence(c.id)" /> @@ -663,16 +660,16 @@ export default defineComponent({ class="mb-2" @click="save" > - {{ dirty ? 'Save calibration' : 'Calibration saved' }} + {{ dirty ? 'Save registration' : 'Registration saved' }} - Export calibration (.json) + Export registration (.json)
diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 3cfb73553..63e8a2955 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -31,7 +31,7 @@ import { LargeImageAnnotator, LayerManager, useMediaController, - useCalibrationNavigation, + useRegistrationNavigation, useAlignedNavigation, TrackList, FilterList, @@ -88,7 +88,7 @@ import context from 'dive-common/store/context'; import { MarkChangesPendingFilter } from 'vue-media-annotator/BaseFilterControls'; import GroupSidebarVue from './GroupSidebar.vue'; import MultiCamToolsVue from './MultiCamTools.vue'; -import CalibrationToolsVue from './CameraCalibration/CalibrationTools.vue'; +import RegistrationToolsVue from './CameraRegistration/RegistrationTools.vue'; import MultiCamToolbar from './MultiCamToolbar.vue'; import AlignedViewToggle from './AlignedViewToggle.vue'; import PrimaryAttributeTrackFilter from './PrimaryAttributeTrackFilter.vue'; @@ -328,23 +328,23 @@ export default defineComponent({ // When the Manual Alignment panel opens, minimize the workspace chrome to // give the picking view more room: collapse the left type-filter sidebar and // the bottom detections graph. This is a soft default -- the normal sidebar - // and timeline toggles still work while calibrating, so the user can bring + // and timeline toggles still work while registering, so the user can bring // either back -- and whatever layout they had before is restored on close. - const calibrationActive = computed(() => context.state.active === CalibrationToolsVue.name); - let preCalibrationSidebarMode: 'left' | 'bottom' | 'collapsed' | null = null; - let preCalibrationControlsCollapsed = false; - watch(calibrationActive, (active) => { + const registrationActive = computed(() => context.state.active === RegistrationToolsVue.name); + let preRegistrationSidebarMode: 'left' | 'bottom' | 'collapsed' | null = null; + let preRegistrationControlsCollapsed = false; + watch(registrationActive, (active) => { if (active) { - preCalibrationSidebarMode = sidebarMode.value; - preCalibrationControlsCollapsed = controlsCollapsed.value; + preRegistrationSidebarMode = sidebarMode.value; + preRegistrationControlsCollapsed = controlsCollapsed.value; if (sidebarMode.value === 'left') { sidebarMode.value = 'collapsed'; } controlsCollapsed.value = true; - } else if (preCalibrationSidebarMode !== null) { - sidebarMode.value = preCalibrationSidebarMode; - controlsCollapsed.value = preCalibrationControlsCollapsed; - preCalibrationSidebarMode = null; + } else if (preRegistrationSidebarMode !== null) { + sidebarMode.value = preRegistrationSidebarMode; + controlsCollapsed.value = preRegistrationControlsCollapsed; + preRegistrationSidebarMode = null; } }); @@ -478,7 +478,7 @@ export default defineComponent({ // The Manual Alignment pair link maps through the homography for // UNWARPED panes, so it needs the aligned view state to stand down // while displays are warped into reference space. - useCalibrationNavigation(aggregateController, cameraRegistration, alignedView); + useRegistrationNavigation(aggregateController, cameraRegistration, alignedView); // Registration point picking records raw native-space clicks and // renders its own aligned preview; suspend the general warp while it // is active so picks are never taken against a warped display. @@ -543,7 +543,7 @@ export default defineComponent({ */ const displayedCameras = computed(() => { const pair = cameraRegistration.activePair.value; - if (calibrationActive.value && pair) { + if (registrationActive.value && pair) { const pairCameras = multiCamList.value.filter( (camera) => camera === pair.camA || camera === pair.camB, ); @@ -1586,7 +1586,7 @@ export default defineComponent({ description: 'Multi Camera Tools', }); context.register({ - component: CalibrationToolsVue, + component: RegistrationToolsVue, description: 'Manual Alignment', }); } else { @@ -1595,7 +1595,7 @@ export default defineComponent({ description: 'Multi Camera Tools', }); context.unregister({ - component: CalibrationToolsVue, + component: RegistrationToolsVue, description: 'Manual Alignment', }); context.register({ diff --git a/client/dive-common/store/context.ts b/client/dive-common/store/context.ts index 65de367a2..220ff1d0c 100644 --- a/client/dive-common/store/context.ts +++ b/client/dive-common/store/context.ts @@ -5,7 +5,7 @@ import ImageEnhancements from 'vue-media-annotator/components/ImageEnhancements. import GroupSidebar from 'dive-common/components/GroupSidebar.vue'; import AttributesSideBar from 'dive-common/components/Attributes/AttributesSideBar.vue'; import MultiCamTools from 'dive-common/components/MultiCamTools.vue'; -import CalibrationTools from 'dive-common/components/CameraCalibration/CalibrationTools.vue'; +import CalibrationTools from 'dive-common/components/CameraRegistration/RegistrationTools.vue'; import AttributeTrackFilters from 'vue-media-annotator/components/AttributeTrackFilters.vue'; import DatasetInfo from 'dive-common/components/DatasetInfo.vue'; diff --git a/client/src/alignedView/CameraRegistrationStore.ts b/client/src/alignedView/CameraRegistrationStore.ts index c0a748a64..3ad29f999 100644 --- a/client/src/alignedView/CameraRegistrationStore.ts +++ b/client/src/alignedView/CameraRegistrationStore.ts @@ -128,7 +128,7 @@ export default class CameraRegistrationStore { /** * Whether pan/zoom is linked between the active pair's two cameras through - * the fitted transform (see {@link useCalibrationNavigation}). Only has an + * the fitted transform (see {@link useRegistrationNavigation}). Only has an * effect once a transform is fitted; toggled from the panel's "Fit pan/zoom". */ linkedNav: Ref; @@ -140,7 +140,7 @@ export default class CameraRegistrationStore { */ selectedCorrespondenceId: Ref; - /** Native-pixel coordinate under the cursor, for the calibration panel's live readout. */ + /** Native-pixel coordinate under the cursor, for the registration panel's live readout. */ cursorCoord: Ref<{ camera: string; coord: Point } | null>; /** @@ -154,7 +154,7 @@ export default class CameraRegistrationStore { * Message from the most recent failed fit attempt (e.g. collinear/degenerate * points that satisfy the minimum count but can't be solved), or null if the * active pair's last fit attempt (if any) succeeded. Surfaced by the - * calibration panel instead of letting the estimator's exception escape a + * registration panel instead of letting the estimator's exception escape a * geojs click handler. */ fitError: Ref; @@ -544,7 +544,7 @@ export default class CameraRegistrationStore { return { camera: other, coord: applyHomography(matrix, coord) }; } - /** Record the native-pixel coordinate under the cursor for `camera` (calibration panel readout). */ + /** Record the native-pixel coordinate under the cursor for `camera` (registration panel readout). */ setCursorCoord(camera: string, coord: Point) { this.cursorCoord.value = { camera, coord }; } @@ -558,7 +558,7 @@ export default class CameraRegistrationStore { * Request that `camera` (native pixel coords `coord`) and, when the pair has * a fitted homography, the other camera of the active pair (via * {@link linkedPoint}) recenter their views on this location. Consumed by - * {@link useCalibrationNavigation}; a no-op if `camera` isn't part of the + * {@link useRegistrationNavigation}; a no-op if `camera` isn't part of the * active pair. A one-shot "snap to this feature" action, distinct from the * continuous pan/zoom link that is active while picking. */ diff --git a/client/src/alignedView/README.md b/client/src/alignedView/README.md index a0ee2367c..824a5c9f6 100644 --- a/client/src/alignedView/README.md +++ b/client/src/alignedView/README.md @@ -8,7 +8,7 @@ Stored annotation geometry always remains in each camera's native image space; t | File | Role | | --- | --- | -| `homography.ts` | Low-level 3×3 matrix primitives: multiply, invert, apply, solve homographies (normalized DLT), linear-system helpers, and GeoJS warp-grid utilities (`subdivideWarpQuads`, `warpGridSize`, `localLinkedScale`). | +| `homography.ts` | Low-level 3×3 matrix primitives: multiply, invert, apply, solve homographies (normalized DLT), linear-system helpers, and GeoJS warp-grid utilities (`subdivideWarpQuads`, `warpGridSize`, `geojsWarpQuads`, `localLinkedScale`). | | `transform.ts` | Higher-level alignment models (translation, rigid, similarity, affine, homography) that all return a `Matrix3`. Defines `TransformType`, UI labels, minimum point counts, and `DEFAULT_TRANSFORM_TYPE` (`similarity`). | | `CameraRegistrationStore.ts` | Reactive Vue store for in-app calibration: picked point correspondences, fitted/loaded pair transforms, per-pair transform-type choices, and producer provenance. Loads/saves the portable registration JSON and tracks dirty state. | | `cameraRegistrationFiles.ts` | Serialization helpers for the per-camera `_to__registration.json` file format. Converts between the store's keyed state and self-describing file pairs; builds export bundles and filters imports to a single camera. | diff --git a/client/src/alignedView/homography.spec.ts b/client/src/alignedView/homography.spec.ts index 9facf0512..180dc4a73 100644 --- a/client/src/alignedView/homography.spec.ts +++ b/client/src/alignedView/homography.spec.ts @@ -6,6 +6,7 @@ import { matMul3, subdivideWarpQuads, warpGridSize, + geojsWarpQuads, localLinkedScale, Point, Matrix3, @@ -256,3 +257,31 @@ describe('localLinkedScale', () => { expect(localLinkedScale(() => [3, 3], [10, 10])).toBeNull(); }); }); + +describe('geojsWarpQuads', () => { + it('maps an affine warp to a single geojs quad with a full-size crop stretch', () => { + const translate: Matrix3 = [[1, 0, 5], [0, 1, -3], [0, 0, 1]]; + const [quad, ...rest] = geojsWarpQuads(translate, 640, 480); + expect(rest).toHaveLength(0); + expect(quad.ul).toEqual({ x: 5, y: -3 }); + expect(quad.lr).toEqual({ x: 645, y: 477 }); + // left/top/right/bottom select the cell's source region; x/y are the full + // source size so that region stretches across the whole sub-quad. + expect(quad.crop).toEqual({ + left: 0, top: 0, right: 640, bottom: 480, x: 640, y: 480, + }); + }); + + it('subdivides a projective warp and maps each corner through the homography', () => { + const projective: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + const grid = warpGridSize(projective, 640, 480); + const quads = geojsWarpQuads(projective, 640, 480); + expect(quads).toHaveLength(grid * grid); + quads.forEach((q) => { + const [x, y] = applyHomography(projective, [q.crop.left, q.crop.top]); + expect(q.ul).toEqual({ x, y }); + expect(q.crop.x).toBe(640); + expect(q.crop.y).toBe(480); + }); + }); +}); diff --git a/client/src/alignedView/homography.ts b/client/src/alignedView/homography.ts index ff09188a7..518297efd 100644 --- a/client/src/alignedView/homography.ts +++ b/client/src/alignedView/homography.ts @@ -181,6 +181,49 @@ export function subdivideWarpQuads( return quads; } +/** + * A subdivided warp cell as geojs quad-feature data: destination corners as + * {x, y} points plus a `crop` whose left/top/right/bottom select the cell's + * source-pixel region and whose x/y (the "size after crop") are the full + * source size, so that region stretches across the whole sub-quad. Callers + * attach the texture source under geojs' `image`/`video` datum key. + */ +export interface GeojsWarpQuad { + ul: { x: number; y: number }; + ur: { x: number; y: number }; + lr: { x: number; y: number }; + ll: { x: number; y: number }; + crop: { + left: number; top: number; right: number; bottom: number; x: number; y: number; + }; +} + +/** + * Build geojs quad-feature data for warping a `width` x `height` image + * through `h`: pick the subdivision grid ({@link warpGridSize}), subdivide + * ({@link subdivideWarpQuads}), and map each cell into geojs' quad shape. + * `overlap` (source pixels) hides the canvas antialiasing seams between + * abutting cells; overlapped quads must draw opaque (see + * {@link subdivideWarpQuads}). + */ +export function geojsWarpQuads( + h: Matrix3, + width: number, + height: number, + overlap = 0, +): GeojsWarpQuad[] { + const grid = warpGridSize(h, width, height); + return subdivideWarpQuads(h, width, height, grid, overlap).map((q) => ({ + ul: { x: q.ul[0], y: q.ul[1] }, + ur: { x: q.ur[0], y: q.ur[1] }, + lr: { x: q.lr[0], y: q.lr[1] }, + ll: { x: q.ll[0], y: q.ll[1] }, + crop: { + ...q.crop, x: width, y: height, + }, + })); +} + /** * Hartley normalization: translate points to the centroid and scale so the * mean distance from the origin is sqrt(2). Returns the normalized points and diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index cbde0fcd0..4104b1943 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -12,7 +12,7 @@ import PointLayer from '../layers/AnnotationLayers/PointLayer'; import LineLayer from '../layers/AnnotationLayers/LineLayer'; import TailLayer from '../layers/AnnotationLayers/TailLayer'; import OverlapLayer from '../layers/AnnotationLayers/OverlapLayer'; -import CalibrationKeypointLayer from '../layers/AnnotationLayers/CalibrationKeypointLayer'; +import RegistrationKeypointLayer from '../layers/AnnotationLayers/RegistrationKeypointLayer'; import EditAnnotationLayer, { EditAnnotationTypes } from '../layers/EditAnnotationLayer'; import LassoSelectionLayer from '../layers/LassoSelectionLayer'; @@ -83,11 +83,11 @@ export default defineComponent({ // Viewer may not provide lasso context in tests or minimal embeds. } const cameraStore = useCameraStore(); - let cameraCalibration: ReturnType | undefined; + let cameraRegistration: ReturnType | undefined; try { - cameraCalibration = useCameraRegistration(); + cameraRegistration = useCameraRegistration(); } catch { - // calibration store may not be provided in tests or minimal embeds. + // registration store may not be provided in tests or minimal embeds. } let alignedView: ReturnType | undefined; try { @@ -211,12 +211,12 @@ export default defineComponent({ const segmentationPointsRef = useSegmentationPoints(); const segmentationPointsLayer = new SegmentationPointsLayer(annotator); - const calibrationLayer = cameraCalibration - ? new CalibrationKeypointLayer({ + const registrationLayer = cameraRegistration + ? new RegistrationKeypointLayer({ annotator, stateStyling: trackStyleManager.stateStyles, typeStyling: typeStylingRef, - calibration: cameraCalibration, + registration: cameraRegistration, getCameraImage: (cam: string) => getCameraQuadMedia( (c) => aggregateController.value.getController(c), cam, @@ -224,19 +224,19 @@ export default defineComponent({ }) : undefined; - if (cameraCalibration && calibrationLayer) { - const calibration = cameraCalibration; + if (cameraRegistration && registrationLayer) { + const registration = cameraRegistration; /** * Frame number of the camera whose image is being ghosted into another * pane, or null when no ghost is active. Watched so the ghost re-renders * when the *source* pane scrubs, not just this pane -- this pane's own * frameNumberRef can update before (or without) the source's, and the * source image element itself only swaps after its frame finishes - * loading (see CalibrationKeypointLayer.scheduleGhostRefresh). + * loading (see RegistrationKeypointLayer.scheduleGhostRefresh). */ const ghostSourceFrame = computed(() => { - const { mode } = calibration.alignment.value; - const pair = calibration.activePair.value; + const { mode } = registration.alignment.value; + const pair = registration.activePair.value; if (mode === 'original' || !pair) { return null; } @@ -249,17 +249,17 @@ export default defineComponent({ }); watch( [ - cameraCalibration.activePair, - cameraCalibration.pickingEnabled, - cameraCalibration.correspondences, - cameraCalibration.pendingPoint, - cameraCalibration.selectedCorrespondenceId, - cameraCalibration.homographies, - cameraCalibration.alignment, + cameraRegistration.activePair, + cameraRegistration.pickingEnabled, + cameraRegistration.correspondences, + cameraRegistration.pendingPoint, + cameraRegistration.selectedCorrespondenceId, + cameraRegistration.homographies, + cameraRegistration.alignment, frameNumberRef, ghostSourceFrame, ], - () => calibrationLayer.update(), + () => registrationLayer.update(), { deep: true }, ); } diff --git a/client/src/components/annotators/linkedViewers/README.md b/client/src/components/annotators/linkedViewers/README.md index bb59f1278..4e7d0e275 100644 --- a/client/src/components/annotators/linkedViewers/README.md +++ b/client/src/components/annotators/linkedViewers/README.md @@ -2,7 +2,7 @@ Multicam datasets show one GeoJS pane per camera. When the user pans or zooms one pane, linked-viewer navigation keeps the other panes showing the same region of the scene. -This folder holds the shared plumbing and the aligned-view link. A separate calibration-pair link (`useCalibrationNavigation`, not yet in this folder) reuses the same plumbing with a different coordinate mapping. +This folder holds the shared plumbing and the two links that reuse it with different coordinate mappings: the aligned-view link and the registration-pair link. ## Modules @@ -10,6 +10,7 @@ This folder holds the shared plumbing and the aligned-view link. A separate cali | --- | --- | | `useLinkedViewers.ts` | Shared GeoJS listener wiring, re-entrancy guard, and zoom-baseline conversion. | | `useAlignedNavigation.ts` | Pan/zoom link for the **aligned view** (SEAL-TK feature 3). | +| `useRegistrationNavigation.ts` | Pan/zoom link for the **active registration pair** while picking points in the Manual Alignment panel. | ## How linking works @@ -49,22 +50,23 @@ Do **not** map centers through camera-to-camera homographies here — rendering - raw camera sync is enabled (`cameraSync`, the un-warped screen-delta sync in Controls.vue); - `aggregateController.resizing` is true (programmatic `onResize` resets emit pan/zoom in native space — ignore them; `resizeTrigger` re-snaps from the reference once resize settles). -### Calibration pair link (future) +### Registration pair link (`useRegistrationNavigation`) -`useCalibrationNavigation` (planned) maps through the homography between a calibrated pair and stands down while the aligned view is active. It will call the same `useLinkedViewers` helpers with a different `LinkedView` mapping. +`useRegistrationNavigation` maps through the active pair's fitted homography (`CameraRegistrationStore.linkedPoint`) so the two panes track each other while picking points — panes render UNWARPED here, so the link must apply the transform itself. It calls the same `useLinkedViewers` helpers with a different `LinkedView` mapping, and stands down while the aligned view is active (which links all panes through the identity instead), while `linkedNav` is toggled off, or while no transform is fitted. ## Wiring -`Viewer.vue` installs aligned navigation at the multicam root: +`Viewer.vue` installs both links at the multicam root: ```typescript -import { useAlignedNavigation } from 'vue-media-annotator/components'; +import { useAlignedNavigation, useRegistrationNavigation } from 'vue-media-annotator/components'; useAlignedNavigation(aggregateController, alignedView, multiCamList); +useRegistrationNavigation(aggregateController, cameraRegistration, alignedView); ``` `AggregateMediaController.resizing` and `resizeTrigger` exist specifically so linked navigation can distinguish user moves from programmatic resize resets — see `mediaControllerType.ts`. ## Tests -`useAlignedNavigation.spec.ts` exercises identity linking, immediate snap on activation, deactivation reset, resize guards, and stand-down while inactive / suspended / camera-sync on. +`useAlignedNavigation.spec.ts` exercises identity linking, immediate snap on activation, deactivation reset, resize guards, and stand-down while inactive / suspended / camera-sync on. `useRegistrationNavigation.spec.ts` exercises the immediate snap when the pair link turns on, re-snap when the fitted homography changes, and stand-down while no fit exists. diff --git a/client/src/components/annotators/linkedViewers/useAlignedNavigation.ts b/client/src/components/annotators/linkedViewers/useAlignedNavigation.ts index 0965a6222..3a5ffda43 100644 --- a/client/src/components/annotators/linkedViewers/useAlignedNavigation.ts +++ b/client/src/components/annotators/linkedViewers/useAlignedNavigation.ts @@ -14,8 +14,8 @@ import useLinkedViewers from './useLinkedViewers'; * camera-to-camera transforms here would re-apply a transform the rendering has * already applied.) Distinct from the raw "sync cameras" toggle (Controls.vue), * which forwards raw screen deltas for UNWARPED panes and is hidden whenever the - * aligned view is available, and from the calibration pair link - * ({@link useCalibrationNavigation}), which maps through the homography and + * aligned view is available, and from the registration pair link + * ({@link useRegistrationNavigation}), which maps through the homography and * stands down while the aligned view is active. */ export default function useAlignedNavigation( diff --git a/client/src/components/annotators/linkedViewers/useLinkedViewers.ts b/client/src/components/annotators/linkedViewers/useLinkedViewers.ts index cdba47928..016166271 100644 --- a/client/src/components/annotators/linkedViewers/useLinkedViewers.ts +++ b/client/src/components/annotators/linkedViewers/useLinkedViewers.ts @@ -13,8 +13,8 @@ export interface LinkedView { } /** - * Shared plumbing for the two multicam pan/zoom links -- the calibration pair - * link ({@link useCalibrationNavigation}) and the aligned-view link + * Shared plumbing for the two multicam pan/zoom links -- the registration pair + * link ({@link useRegistrationNavigation}) and the aligned-view link * ({@link useAlignedNavigation}). Both attach geojs pan/zoom listeners to a set * of panes and, when one moves, drive the others to a matching view; they * differ only in how a source pane's view maps onto a target pane. This owns diff --git a/client/src/components/annotators/linkedViewers/useCalibrationNavigation.spec.ts b/client/src/components/annotators/linkedViewers/useRegistrationNavigation.spec.ts similarity index 95% rename from client/src/components/annotators/linkedViewers/useCalibrationNavigation.spec.ts rename to client/src/components/annotators/linkedViewers/useRegistrationNavigation.spec.ts index 06f20b893..dcc2e4db8 100644 --- a/client/src/components/annotators/linkedViewers/useCalibrationNavigation.spec.ts +++ b/client/src/components/annotators/linkedViewers/useRegistrationNavigation.spec.ts @@ -2,7 +2,7 @@ import { ref, shallowRef, nextTick, Ref, } from 'vue'; -import useCalibrationNavigation from './useCalibrationNavigation'; +import useRegistrationNavigation from './useRegistrationNavigation'; import type CameraRegistrationStore from '../../../alignedView/CameraRegistrationStore'; import type { AggregateMediaController } from '../mediaControllerType'; import type { Point } from '../../../alignedView/homography'; @@ -65,7 +65,7 @@ function makeHarness() { const fitted = ref(true); // eo -> ir is a pure +100 x-translation (and ir -> eo its inverse), so the // linked scale is 1 and centers map by simple offset. - const calibration = { + const registration = { pickingEnabled, linkedNav, activePair: ref({ camA: 'eo', camB: 'ir' }), @@ -82,13 +82,13 @@ function makeHarness() { }, } as unknown as CameraRegistrationStore; - useCalibrationNavigation(aggregate, calibration); + useRegistrationNavigation(aggregate, registration); return { eo, ir, pickingEnabled, linkedNav, homographies, fitted, }; } -describe('useCalibrationNavigation', () => { +describe('useRegistrationNavigation', () => { it('snaps the pair immediately when "Fit pan/zoom" turns on', async () => { const { eo, ir, pickingEnabled, linkedNav, diff --git a/client/src/components/annotators/linkedViewers/useCalibrationNavigation.ts b/client/src/components/annotators/linkedViewers/useRegistrationNavigation.ts similarity index 85% rename from client/src/components/annotators/linkedViewers/useCalibrationNavigation.ts rename to client/src/components/annotators/linkedViewers/useRegistrationNavigation.ts index e5da709e8..1a2ac8bae 100644 --- a/client/src/components/annotators/linkedViewers/useCalibrationNavigation.ts +++ b/client/src/components/annotators/linkedViewers/useRegistrationNavigation.ts @@ -7,7 +7,7 @@ import type { Point } from '../../../alignedView/homography'; import useLinkedViewers from './useLinkedViewers'; /** - * Links pan/zoom recentering between the two cameras of the active calibration + * Links pan/zoom recentering between the two cameras of the active registration * pair: panning or zooming one recenters the other on the same point, mapped * through the pair's fitted homography ({@link CameraRegistrationStore.linkedPoint}). * The mapping assumes UNWARPED panes showing native coordinates, so this is @@ -16,9 +16,9 @@ import useLinkedViewers from './useLinkedViewers'; * the aligned view is somehow active. Distinct from the general "sync cameras" * toggle (Controls.vue), which assumes identical pixel scale between panes. */ -export default function useCalibrationNavigation( +export default function useRegistrationNavigation( aggregateController: Ref, - calibration: CameraRegistrationStore, + registration: CameraRegistrationStore, alignedView?: AlignedViewStore, ) { const { @@ -43,7 +43,7 @@ export default function useCalibrationNavigation( return; } const center = source.center(); - const linked = calibration.linkedPoint(camera, [center.x, center.y]); + const linked = registration.linkedPoint(camera, [center.x, center.y]); if (!linked || linked.camera !== otherCamera) { return; } @@ -51,7 +51,7 @@ export default function useCalibrationNavigation( // here (position-dependent for non-similarity fits, so sampled at center; // null when unavailable -- leave the target's zoom alone). const scale = localLinkedScale( - (p) => calibration.linkedPoint(camera, p)?.coord ?? null, + (p) => registration.linkedPoint(camera, p)?.coord ?? null, [center.x, center.y], ); applyView(target, { @@ -63,11 +63,11 @@ export default function useCalibrationNavigation( function setup() { teardown(); - const pair = calibration.activePair.value; + const pair = registration.activePair.value; // Authoring UI: active while picking and "Fit pan/zoom" is on (once a fit // exists, linkedPoint returns matches). attach() no-ops for a not-yet-ready // pane; the resizeTrigger watch re-runs setup once both viewers exist. - if (calibration.pickingEnabled.value && calibration.linkedNav.value && pair) { + if (registration.pickingEnabled.value && registration.linkedNav.value && pair) { attach(pair.camA, link(pair.camA, pair.camB)); attach(pair.camB, link(pair.camB, pair.camA)); // Snap immediately from camA so toggling "Fit pan/zoom" on (or a refit @@ -80,10 +80,10 @@ export default function useCalibrationNavigation( watch( [ - calibration.pickingEnabled, - calibration.linkedNav, - calibration.activePair, - calibration.homographies, + registration.pickingEnabled, + registration.linkedNav, + registration.activePair, + registration.homographies, aggregateController.value.resizeTrigger, ], setup, @@ -102,7 +102,7 @@ export default function useCalibrationNavigation( if (!request || alignedView?.active.value) { return; } - const pair = calibration.activePair.value; + const pair = registration.activePair.value; if (!pair || (request.camera !== pair.camA && request.camera !== pair.camB)) { return; } @@ -110,12 +110,12 @@ export default function useCalibrationNavigation( if (source) { guarded(() => source.center({ x: request.coord[0], y: request.coord[1] })); } - const linked = calibration.linkedPoint(request.camera, request.coord); + const linked = registration.linkedPoint(request.camera, request.coord); const target = linked && viewer(linked.camera); if (linked && target) { guarded(() => target.center({ x: linked.coord[0], y: linked.coord[1] })); } } - watch(() => calibration.recenterRequest.value, handleRecenterRequest); + watch(() => registration.recenterRequest.value, handleRecenterRequest); } diff --git a/client/src/components/annotators/mediaControllerType.ts b/client/src/components/annotators/mediaControllerType.ts index bdfdc5df9..8f99b4331 100644 --- a/client/src/components/annotators/mediaControllerType.ts +++ b/client/src/components/annotators/mediaControllerType.ts @@ -48,7 +48,7 @@ export interface AggregateMediaController { /** * True only while onResize is applying its programmatic size()/resetZoom() * to the panes. resetZoom emits GeoJS pan/zoom events synchronously; the - * linked-viewer navigation (useAlignedNavigation / useCalibrationNavigation) + * linked-viewer navigation (useAlignedNavigation / useRegistrationNavigation) * must ignore those so one pane's native-space reset isn't broadcast to the * others as if it were a shared-space move (which parks warped panes on an * empty corner). The resizeTrigger bump that follows re-snaps every pane from diff --git a/client/src/components/index.ts b/client/src/components/index.ts index 2eaec7028..34a01d982 100644 --- a/client/src/components/index.ts +++ b/client/src/components/index.ts @@ -28,7 +28,7 @@ import TypePicker from './TypePicker.vue'; export * from './annotators/useMediaController'; export { default as useAnnotatorImageCursor } from './annotators/useAnnotatorImageCursor'; -export { default as useCalibrationNavigation } from './annotators/linkedViewers/useCalibrationNavigation'; +export { default as useRegistrationNavigation } from './annotators/linkedViewers/useRegistrationNavigation'; export { default as useAlignedNavigation } from './annotators/linkedViewers/useAlignedNavigation'; export { /* Annotators */ diff --git a/client/src/layers/AlignedImageLayer.ts b/client/src/layers/AlignedImageLayer.ts index 50dc82bed..ea3b6c7ef 100644 --- a/client/src/layers/AlignedImageLayer.ts +++ b/client/src/layers/AlignedImageLayer.ts @@ -1,6 +1,6 @@ import geo from 'geojs'; import type { MediaController } from '../components/annotators/mediaControllerType'; -import { Matrix3, subdivideWarpQuads, warpGridSize } from '../alignedView/homography'; +import { Matrix3, geojsWarpQuads } from '../alignedView/homography'; import { findQuadMediaLayer } from '../components/layerManager/quadMediaSource'; export interface CameraImage { @@ -32,7 +32,7 @@ interface AlignedImageLayerParams { * while the multicam aligned view is on (SEAL-TK feature 2, decision D1: the * proven quad-corner warp). The warp is drawn as an n x n grid of geojs * canvas quads whose corners are mapped through the exact projective - * transform ({@link subdivideWarpQuads}), and the annotator's own native + * transform ({@link geojsWarpQuads}), and the annotator's own native * image quad layer is hidden while the warp is shown -- so annotation layers * (whose vertices are mapped through the same matrix at draw time) land * exactly on the warped imagery. When no transform applies, everything is @@ -73,7 +73,7 @@ export default class AlignedImageLayer { // Right-click recenter: center this pane on the clicked location, in any // view mode. While the aligned view is active, the aligned pan/zoom link // then recenters every other pane on the same reference-space point; - // during calibration picking, CalibrationKeypointLayer additionally maps + // during registration picking, RegistrationKeypointLayer additionally maps // the recenter across the active pair. this.annotator.geoViewerRef.value.geoOn( geo.event.mouseclick, @@ -145,22 +145,10 @@ export default class AlignedImageLayer { return; } const { width: w, height: h } = src; - const grid = warpGridSize(transform, w, h); // 2px cell overlap hides the canvas antialiasing seams between abutting // sub-quads (dark grid lines); safe here because quads draw opaque. - const quads = subdivideWarpQuads(transform, w, h, grid, 2).map((q) => ({ - ul: { x: q.ul[0], y: q.ul[1] }, - ur: { x: q.ur[0], y: q.ur[1] }, - lr: { x: q.lr[0], y: q.lr[1] }, - ll: { x: q.ll[0], y: q.ll[1] }, - // geojs crop: left/top/right/bottom select the source-pixel region; - // x/y (the "size after crop") are set to the full source size so that - // region stretches across the whole sub-quad. - crop: { - ...q.crop, x: w, y: h, - }, - [src.kind]: src.source, - })); + const quads = geojsWarpQuads(transform, w, h, 2) + .map((q) => ({ ...q, [src.kind]: src.source })); // Mirror the native layer's CSS filter (image enhancements) so toggling // the warp doesn't change brightness/contrast rendering. const nativeLayer = this.findNativeQuadLayer(); diff --git a/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts b/client/src/layers/AnnotationLayers/RegistrationKeypointLayer.ts similarity index 82% rename from client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts rename to client/src/layers/AnnotationLayers/RegistrationKeypointLayer.ts index 01743e600..40d25921c 100644 --- a/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts +++ b/client/src/layers/AnnotationLayers/RegistrationKeypointLayer.ts @@ -2,10 +2,10 @@ import geo, { GeoEvent } from 'geojs'; import BaseLayer, { BaseLayerParams, LayerStyle } from '../BaseLayer'; import { FrameDataTrack } from '../LayerTypes'; import CameraRegistrationStore from '../../alignedView/CameraRegistrationStore'; -import { subdivideWarpQuads, warpGridSize } from '../../alignedView/homography'; +import { geojsWarpQuads } from '../../alignedView/homography'; import type { CameraImage } from '../AlignedImageLayer'; -export interface CalibrationPointData { +export interface RegistrationPointData { x: number; y: number; label: string; @@ -18,8 +18,8 @@ export interface CalibrationPointData { export type { CameraImage }; -interface CalibrationLayerParams { - calibration: CameraRegistrationStore; +interface RegistrationLayerParams { + registration: CameraRegistrationStore; /** Resolve another camera's currently displayed frame image (for the overlay). */ getCameraImage?: (camera: string) => CameraImage | null; } @@ -36,13 +36,13 @@ const GHOST_REFRESH_MAX_ATTEMPTS = 60; const DRAG_HIT_RADIUS_PX = 10; /** - * Renders this camera's picked calibration points (numbered markers, the pending + * Renders this camera's picked registration points (numbered markers, the pending * "blue" point highlighted) and, when alignment mode is active, a ghost overlay * of the other camera's frame warped through the fitted homography (geojs * quadFeature). One instance is created per camera in LayerManager. */ -export default class CalibrationKeypointLayer extends BaseLayer { - calibration: CameraRegistrationStore; +export default class RegistrationKeypointLayer extends BaseLayer { + registration: CameraRegistrationStore; getCameraImage?: (camera: string) => CameraImage | null; @@ -84,9 +84,9 @@ export default class CalibrationKeypointLayer extends BaseLayer this.handleDragEnd(); - constructor(params: BaseLayerParams & CalibrationLayerParams) { + constructor(params: BaseLayerParams & RegistrationLayerParams) { super(params); - this.calibration = params.calibration; + this.registration = params.registration; this.getCameraImage = params.getCameraImage; // Listen on the map, which is what emits geo.event.mouseclick. The event // exposes button state and image (gcs) coordinates at the top level. @@ -122,11 +122,11 @@ export default class CalibrationKeypointLayer extends BaseLayer ( + fillColor: (data: RegistrationPointData) => ( (data.selected || data.pending) ? 'yellow' : 'cyan'), fillOpacity: 1, - radius: (data: CalibrationPointData) => (data.selected ? 3.5 : 2.5), - strokeColor: (data: CalibrationPointData) => ( + radius: (data: RegistrationPointData) => (data.selected ? 3.5 : 2.5), + strokeColor: (data: RegistrationPointData) => ( (data.selected || data.pending) ? 'orange' : 'blue'), strokeWidth: 1, }); @@ -142,8 +142,8 @@ export default class CalibrationKeypointLayer extends BaseLayer data.label) - .position((data: CalibrationPointData) => ({ x: data.x, y: data.y })) + .text((data: RegistrationPointData) => data.label) + .position((data: RegistrationPointData) => ({ x: data.x, y: data.y })) .style({ color: 'white', fontSize: '14px', @@ -161,13 +161,13 @@ export default class CalibrationKeypointLayer extends BaseLayer { const disp = map.gcsToDisplay({ x: d.x, y: d.y }); @@ -244,10 +244,10 @@ export default class CalibrationKeypointLayer extends BaseLayer { const coord = cam === pair.camA ? c.a : c.b; result.push({ @@ -331,7 +331,7 @@ export default class CalibrationKeypointLayer extends BaseLayer ({ - ul: { x: q.ul[0], y: q.ul[1] }, - ur: { x: q.ur[0], y: q.ur[1] }, - lr: { x: q.lr[0], y: q.lr[1] }, - ll: { x: q.ll[0], y: q.ll[1] }, - // geojs crop: left/top/right/bottom select the source-pixel region; - // x/y (the "size after crop") are set to the full source size so that - // region stretches across the whole sub-quad. - crop: { - ...q.crop, x: w, y: hgt, - }, - [src.kind]: src.source, - })); + const quads = geojsWarpQuads(h, w, hgt, 2) + .map((q) => ({ ...q, [src.kind]: src.source })); this.ghostSource = src.source; this.quadLayer.opacity(alignment.opacity); this.quadFeature @@ -493,7 +481,7 @@ export default class CalibrationKeypointLayer extends BaseLayer { + createStyle(): LayerStyle { return { ...super.createStyle(), // Hollow ring: leave the center transparent so the exact pixel being @@ -504,10 +492,10 @@ export default class CalibrationKeypointLayer extends BaseLayer (data.selected ? 9 : 7), - strokeColor: (data: CalibrationPointData) => ( + radius: (data: RegistrationPointData) => (data.selected ? 9 : 7), + strokeColor: (data: RegistrationPointData) => ( (data.selected || data.pending) ? 'orange' : 'blue'), - strokeWidth: (data: CalibrationPointData) => (data.selected ? 3 : 2), + strokeWidth: (data: RegistrationPointData) => (data.selected ? 3 : 2), }; } } From 030b413ae9f3e29a17791275efa715bc63489aa9 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Sun, 12 Jul 2026 00:09:37 -0400 Subject: [PATCH 04/13] Drive the registration ghost from the source camera's imageRevision Same replacement as the Aligned View warp: the ghost polled with a bounded requestAnimationFrame loop to catch the source pane swapping its element after a seek. The source annotator now bumps imageRevision on that swap, so LayerManager watches the ghost source camera's revision (alongside its frame number) and re-renders the ghost from the element actually on screen -- with no ~1s give-up window on slow-loading frames. Co-Authored-By: Claude Fable 5 --- client/src/components/LayerManager.vue | 40 ++++++++--- .../annotators/mediaControllerType.ts | 2 +- .../annotators/useMediaController.ts | 2 +- .../RegistrationKeypointLayer.ts | 72 +------------------ 4 files changed, 35 insertions(+), 81 deletions(-) diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 4104b1943..83010be8e 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -227,26 +227,45 @@ export default defineComponent({ if (cameraRegistration && registrationLayer) { const registration = cameraRegistration; /** - * Frame number of the camera whose image is being ghosted into another - * pane, or null when no ghost is active. Watched so the ghost re-renders - * when the *source* pane scrubs, not just this pane -- this pane's own - * frameNumberRef can update before (or without) the source's, and the - * source image element itself only swaps after its frame finishes - * loading (see RegistrationKeypointLayer.scheduleGhostRefresh). + * The camera whose image is being ghosted into another pane, or null + * when no ghost is active. */ - const ghostSourceFrame = computed(() => { + const ghostSourceCamera = computed(() => { const { mode } = registration.alignment.value; const pair = registration.activePair.value; if (mode === 'original' || !pair) { return null; } - const srcCam = mode === 'BtoA' ? pair.camB : pair.camA; + return mode === 'BtoA' ? pair.camB : pair.camA; + }); + const ghostSourceController = (camera: string | null) => { + if (camera === null) { + return null; + } try { - return aggregateController.value.getController(srcCam).frame.value; + return aggregateController.value.getController(camera); } catch { return null; } - }); + }; + /** + * Frame number of the ghost source camera. Watched so the ghost + * re-renders when the *source* pane scrubs, not just this pane -- this + * pane's own frameNumberRef can update before (or without) the + * source's. + */ + const ghostSourceFrame = computed( + () => ghostSourceController(ghostSourceCamera.value)?.frame.value ?? null, + ); + /** + * imageRevision of the ghost source camera: its annotator swaps the + * displayed asynchronously after the frame finishes loading and + * bumps this when it does, so the ghost re-renders from the element + * actually on screen. + */ + const ghostSourceRevision = computed( + () => ghostSourceController(ghostSourceCamera.value)?.imageRevision.value ?? null, + ); watch( [ cameraRegistration.activePair, @@ -258,6 +277,7 @@ export default defineComponent({ cameraRegistration.alignment, frameNumberRef, ghostSourceFrame, + ghostSourceRevision, ], () => registrationLayer.update(), { deep: true }, diff --git a/client/src/components/annotators/mediaControllerType.ts b/client/src/components/annotators/mediaControllerType.ts index 8f99b4331..80b450c3e 100644 --- a/client/src/components/annotators/mediaControllerType.ts +++ b/client/src/components/annotators/mediaControllerType.ts @@ -107,7 +107,7 @@ export interface MediaController extends AggregateMediaController { * swap after a seek finishes loading, an image-enhancement change (the * percentile-stretch URL remap or the CSS filter toggle), or the initial * video quad. Watchers that render a snapshot of the displayed element - * (e.g. the aligned-view warp) rely on this as their + * (the aligned-view warp, the registration ghost) rely on this as their * only signal that the element changed -- every draw path must bump it. */ imageRevision: Readonly>; diff --git a/client/src/components/annotators/useMediaController.ts b/client/src/components/annotators/useMediaController.ts index 95f986153..0f297d886 100644 --- a/client/src/components/annotators/useMediaController.ts +++ b/client/src/components/annotators/useMediaController.ts @@ -36,7 +36,7 @@ interface MediaControllerReactiveData { * swap after a seek finishes loading, an image-enhancement change (the * percentile-stretch URL remap or the CSS filter toggle), or the initial * video quad. Watchers that render a snapshot of the displayed element - * (e.g. the aligned-view warp) rely on this as their + * (the aligned-view warp, the registration ghost) rely on this as their * only signal that the element changed -- every draw path must bump it. */ imageRevision: number; diff --git a/client/src/layers/AnnotationLayers/RegistrationKeypointLayer.ts b/client/src/layers/AnnotationLayers/RegistrationKeypointLayer.ts index 40d25921c..7ab0bbb8f 100644 --- a/client/src/layers/AnnotationLayers/RegistrationKeypointLayer.ts +++ b/client/src/layers/AnnotationLayers/RegistrationKeypointLayer.ts @@ -24,14 +24,6 @@ interface RegistrationLayerParams { getCameraImage?: (camera: string) => CameraImage | null; } -/** - * How many animation frames to keep re-checking whether the ghost source - * camera's displayed image element has changed after an update trigger (image - * sequences swap their quad datum asynchronously once the new frame loads). - * ~60 frames is about one second at typical refresh rates. - */ -const GHOST_REFRESH_MAX_ATTEMPTS = 60; - /** Display-pixel radius within which a mousedown grabs an existing marker to drag-refine it. */ const DRAG_HIT_RADIUS_PX = 10; @@ -46,14 +38,6 @@ export default class RegistrationKeypointLayer extends BaseLayer CameraImage | null; - /** The source element currently rendered as the ghost overlay, if any. */ - private ghostSource: HTMLImageElement | HTMLVideoElement | null = null; - - /** Pending requestAnimationFrame handle for the ghost staleness re-check loop. */ - private ghostRetryHandle: number | null = null; - - private ghostRetryAttempts = 0; - // eslint-disable-next-line @typescript-eslint/no-explicit-any textFeature: any; @@ -363,8 +347,6 @@ export default class RegistrationKeypointLayer extends BaseLayer { - this.ghostSource = null; - this.cancelGhostRefresh(); this.quadFeature.data([]).draw(); }; const alignment = this.registration?.alignment.value; @@ -388,10 +370,10 @@ export default class RegistrationKeypointLayer extends BaseLayer ({ ...q, [src.kind]: src.source })); - this.ghostSource = src.source; this.quadLayer.opacity(alignment.opacity); this.quadFeature .data(quads) .style('opacity', 1) .draw(); - if (src.kind === 'image') { - // Image sequences swap the source pane's asynchronously after the - // frame finishes loading, with no event reaching this layer; poll - // briefly so the ghost catches up (video elements update in place). - this.scheduleGhostRefresh(srcCam); - } else { - this.cancelGhostRefresh(); - } - } - - /** Cancel any pending ghost staleness re-check. */ - private cancelGhostRefresh() { - if (this.ghostRetryHandle !== null) { - cancelAnimationFrame(this.ghostRetryHandle); - this.ghostRetryHandle = null; - } - } - - /** - * Bounded requestAnimationFrame loop that re-checks whether `srcCam`'s - * displayed image element differs from the one the ghost was rendered from, - * and re-renders the ghost when it does. This covers the gap between a frame - * change (which triggers {@link update} immediately) and the moment - * ImageAnnotator actually swaps the loaded into its quad datum. - */ - private scheduleGhostRefresh(srcCam: string) { - this.cancelGhostRefresh(); - this.ghostRetryAttempts = 0; - if (typeof requestAnimationFrame !== 'function') { - return; - } - const tick = () => { - this.ghostRetryHandle = null; - const src = this.getCameraImage ? this.getCameraImage(srcCam) : null; - if (src && src.source && src.width && src.height && src.source !== this.ghostSource) { - // Re-render with the new element; updateGhost restarts this loop. - this.updateGhost(); - return; - } - this.ghostRetryAttempts += 1; - if (this.ghostRetryAttempts < GHOST_REFRESH_MAX_ATTEMPTS) { - this.ghostRetryHandle = requestAnimationFrame(tick); - } - }; - this.ghostRetryHandle = requestAnimationFrame(tick); } /** Recompute points and the ghost overlay from the store and redraw. */ @@ -473,8 +409,6 @@ export default class RegistrationKeypointLayer extends BaseLayer Date: Sun, 12 Jul 2026 11:42:31 -0400 Subject: [PATCH 05/13] Streamline the Camera Registration panel around the import/export menus Rename the Manual Alignment panel to Camera Registration and scope it to its actual job -- picking points, fitting, and saving: - Drop the panel's "Load registration" button: the Import menu's per-camera registration targets are the one import path. - Drop the panel's export button too: the Export menu's per-camera registration downloads (web and desktop) are the one export path. The refined-from-source notice now points there (save first -- the menu reads saved meta). - Remove CameraRegistrationStore.toRegistrationJson, the last whole-rig (all pairs in one file) serializer; per-camera _to__registration.json files are the only registration file format. loadRegistrationText stays as the shared parser, and its round-trip specs now serialize through the production buildPerCameraRegistrationFiles. - Protect unsaved panel edits on exit: the browser beforeunload warning, the navigate-away prompt, and the desktop close dialog now fire on registration dirt too (hasUnsavedChanges = pending annotation saves OR a dirty registration), and the desktop dialog's Save choice persists the registration via the new Viewer.saveRegistration. Co-Authored-By: Claude Fable 5 --- .../CameraRegistration/RegistrationTools.vue | 120 +----------------- client/dive-common/components/Viewer.vue | 45 +++++-- client/dive-common/store/context.ts | 8 +- .../backend/native/cameraRegistration.ts | 2 +- .../frontend/components/ViewerLoader.vue | 10 +- .../CameraRegistrationStore.spec.ts | 31 ++++- .../alignedView/CameraRegistrationStore.ts | 39 +----- .../annotators/linkedViewers/README.md | 2 +- 8 files changed, 85 insertions(+), 172 deletions(-) diff --git a/client/dive-common/components/CameraRegistration/RegistrationTools.vue b/client/dive-common/components/CameraRegistration/RegistrationTools.vue index b52f5f5fb..58c163482 100644 --- a/client/dive-common/components/CameraRegistration/RegistrationTools.vue +++ b/client/dive-common/components/CameraRegistration/RegistrationTools.vue @@ -11,21 +11,18 @@ import { TransformType, TRANSFORM_TYPES, DEFAULT_TRANSFORM_TYPE, minPointsForTransform, } from 'vue-media-annotator/alignedView/transform'; import { unresolvedCameras } from 'vue-media-annotator/alignedView/alignedView'; -import { unknownCameraWarning } from 'vue-media-annotator/alignedView/cameraRegistrationFiles'; import TooltipBtn from 'vue-media-annotator/components/TooltipButton.vue'; import { useApi } from 'dive-common/apispec'; -import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; export default defineComponent({ name: 'CameraRegistration', - description: 'Manual Alignment', + description: 'Camera Registration', components: { TooltipBtn }, setup() { const cameraStore = useCameraStore(); const registration = useCameraRegistration(); const datasetId = useDatasetId(); const { saveMetadata } = useApi(); - const { prompt } = usePrompt(); const cameras = computed(() => [...cameraStore.camMap.value.keys()]); /** @@ -227,8 +224,9 @@ export default defineComponent({ * Deliberately not gated on the * active pair having correspondences: saving must also be able to persist * a cleared state (so stale saved registration doesn't survive Clear All / - * per-row deletes) and state belonging to non-active pairs. Use - * {@link exportRegistration} to get a portable copy for sharing. + * per-row deletes) and state belonging to non-active pairs. Portable + * copies for sharing come from the Export menu's per-camera registration + * downloads, which read this saved state. */ async function save() { saving.value = true; @@ -246,69 +244,6 @@ export default defineComponent({ } } - /** - * Download the registration as a portable .json -- for handing refinements - * (points included) back to an external producer, or reusing on another - * dataset. Saving is separate: see {@link save}. - */ - function exportRegistration() { - registration.maybeFitActivePair(); - downloadText(registration.toRegistrationJson(), 'camera-registration.json'); - } - - /** Trigger a browser download of `text` as `filename`. */ - function downloadText(text: string, filename: string) { - const blob = new Blob([text], { type: 'text/plain' }); - const url = URL.createObjectURL(blob); - const anchor = document.createElement('a'); - anchor.href = url; - anchor.download = filename; - document.body.appendChild(anchor); - anchor.click(); - document.body.removeChild(anchor); - URL.revokeObjectURL(url); - } - - const registrationFileInput = ref(null); - const loadRegistrationError = ref(null); - const loadRegistrationWarning = ref(null); - - const hasAnyRegistration = computed( - () => Object.values(registration.correspondences.value).some((list) => list.length > 0) - || Object.keys(registration.homographies.value).length > 0, - ); - - /** Load a registration .json, replacing every pair's state. */ - async function loadJsonRegistration(file: File) { - const text = await file.text(); - if (hasAnyRegistration.value) { - const confirmed = await prompt({ - title: 'Load registration', - text: 'This will replace the current registration for ALL camera pairs. Continue?', - confirm: true, - }); - if (!confirmed) { - return; - } - } - const { cameras: fileCameras } = registration.loadRegistrationText(text); - loadRegistrationWarning.value = unknownCameraWarning(file.name, fileCameras, cameras.value); - } - - function onRegistrationFileSelected(event: Event) { - const input = event.target as HTMLInputElement; - const file = input.files?.[0]; - input.value = ''; - if (!file) { - return; - } - loadRegistrationError.value = null; - loadRegistrationWarning.value = null; - loadJsonRegistration(file).catch((err) => { - loadRegistrationError.value = err instanceof Error ? err.message : String(err); - }); - } - return { cameras, cameraAlignmentStatuses, @@ -339,14 +274,9 @@ export default defineComponent({ dirty: registration.dirty, saving, sourceReadout, - registrationFileInput, - loadRegistrationError, - loadRegistrationWarning, setTransformType, setAlignmentMode, save, - exportRegistration, - onRegistrationFileSelected, }; }, }); @@ -365,34 +295,6 @@ export default defineComponent({ - - - Load registration - - - {{ loadRegistrationError }} - - - {{ loadRegistrationWarning }} - This pair has been refined in-app since the source registration was - produced. Export the registration to hand the refinement (and its - points) back to the producer. + produced. Save, then download the camera's registration from the + Export menu to hand the refinement (and its points) back to the + producer.
{{ dirty ? 'Save registration' : 'Registration saved' }} - - Export registration (.json) -
diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 63e8a2955..f7127c088 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -325,7 +325,7 @@ export default defineComponent({ }); const showUserSettingsDialog = ref(false); - // When the Manual Alignment panel opens, minimize the workspace chrome to + // When the Camera Registration panel opens, minimize the workspace chrome to // give the picking view more room: collapse the left type-filter sidebar and // the bottom detections graph. This is a soft default -- the normal sidebar // and timeline toggles still work while registering, so the user can bring @@ -436,7 +436,7 @@ export default defineComponent({ * review. Reference camera = the Reference Camera chosen at import * (stored as defaultDisplay), falling back to the first camera in * display order. Transforms come from the registration store's pair - * homographies (picked in-app via the Manual Alignment panel, or loaded + * homographies (picked in-app via the Camera Registration panel, or loaded * from a registration file or the dataset's saved meta), composed * through the pair graph -- the single registration the panel edits and * saves is exactly what the Align button applies. @@ -475,7 +475,7 @@ export default defineComponent({ selectedCamera, setResetZoomOverride, }); - // The Manual Alignment pair link maps through the homography for + // The Camera Registration pair link maps through the homography for // UNWARPED panes, so it needs the aligned view state to stand down // while displays are warped into reference space. useRegistrationNavigation(aggregateController, cameraRegistration, alignedView); @@ -535,7 +535,7 @@ export default defineComponent({ return { registered: cams.length - unresolved.length, total: cams.length }; }); /** - * Camera panes currently displayed. While the Manual Alignment panel is + * Camera panes currently displayed. While the Camera Registration panel is * open with an active pair on a 3+ camera dataset, only the pair's two * panes show, so the left/right alignment flow reads without unrelated * panes in between (regardless of whether Pick points is toggled on). @@ -1158,15 +1158,42 @@ export default defineComponent({ }); // Navigation Guards used by parent component + /** + * Unsaved work the exit/navigation guards protect: pending annotation + * saves, plus Camera Registration panel edits, which track their own + * dirty state (they persist through dataset meta, outside the + * annotation save path that pendingSaveCount counts). + */ + const hasUnsavedChanges = computed( + () => pendingSaveCount.value > 0 || cameraRegistration.dirty.value, + ); + /** + * Persist unsaved Camera Registration panel edits -- the same write as + * the panel's own Save button -- so the desktop close guard's "Save" + * choice covers them too. No-op while the registration is clean. + */ + async function saveRegistration() { + if (!cameraRegistration.dirty.value) { + return; + } + cameraRegistration.maybeFitActivePair(); + await saveMetadata(datasetId.value, { + cameraHomographies: cameraRegistration.homographies.value, + cameraCorrespondences: cameraRegistration.correspondences.value, + cameraTransformTypes: cameraRegistration.transformTypes.value, + cameraRegistrationSource: cameraRegistration.source.value, + }); + cameraRegistration.markSaved(); + } async function warnBrowserExit(event: BeforeUnloadEvent) { - if (pendingSaveCount.value === 0) return; + if (!hasUnsavedChanges.value) return; event.preventDefault(); // eslint-disable-next-line no-param-reassign event.returnValue = ''; } async function navigateAwayGuard(): Promise { let result = true; - if (pendingSaveCount.value > 0) { + if (hasUnsavedChanges.value) { result = await prompt({ title: 'Save Items', text: 'There is unsaved data, would you like to continue or cancel and save?', @@ -1587,7 +1614,7 @@ export default defineComponent({ }); context.register({ component: RegistrationToolsVue, - description: 'Manual Alignment', + description: 'Camera Registration', }); } else { context.unregister({ @@ -1596,7 +1623,7 @@ export default defineComponent({ }); context.unregister({ component: RegistrationToolsVue, - description: 'Manual Alignment', + description: 'Camera Registration', }); context.register({ description: 'Group Manager', @@ -2028,6 +2055,8 @@ export default defineComponent({ // For Navigation Guarding navigateAwayGuard, warnBrowserExit, + hasUnsavedChanges, + saveRegistration, reloadAnnotations, // Annotation Sets, sets, diff --git a/client/dive-common/store/context.ts b/client/dive-common/store/context.ts index 220ff1d0c..3b5521cc4 100644 --- a/client/dive-common/store/context.ts +++ b/client/dive-common/store/context.ts @@ -5,7 +5,7 @@ import ImageEnhancements from 'vue-media-annotator/components/ImageEnhancements. import GroupSidebar from 'dive-common/components/GroupSidebar.vue'; import AttributesSideBar from 'dive-common/components/Attributes/AttributesSideBar.vue'; import MultiCamTools from 'dive-common/components/MultiCamTools.vue'; -import CalibrationTools from 'dive-common/components/CameraRegistration/RegistrationTools.vue'; +import RegistrationTools from 'dive-common/components/CameraRegistration/RegistrationTools.vue'; import AttributeTrackFilters from 'vue-media-annotator/components/AttributeTrackFilters.vue'; import DatasetInfo from 'dive-common/components/DatasetInfo.vue'; @@ -47,9 +47,9 @@ const componentMap: Record = { description: 'Multi Camera Tools', component: MultiCamTools, }, - [CalibrationTools.name]: { - description: 'Manual Alignment', - component: CalibrationTools, + [RegistrationTools.name]: { + description: 'Camera Registration', + component: RegistrationTools, }, [AttributesSideBar.name]: { description: 'Attribute Details', diff --git a/client/platform/desktop/backend/native/cameraRegistration.ts b/client/platform/desktop/backend/native/cameraRegistration.ts index 517614a6e..9ffcff38d 100644 --- a/client/platform/desktop/backend/native/cameraRegistration.ts +++ b/client/platform/desktop/backend/native/cameraRegistration.ts @@ -78,7 +78,7 @@ export function fromRegistrationPairs( const transformTypes: CameraTransformTypes = {}; pairs.forEach((pair) => { const key = `${pair.left}::${pair.right}`; - // Mirror the panel loader (CameraRegistrationStore.loadRegistrationText): + // Mirror the shared client parser (CameraRegistrationStore.loadRegistrationText): // producer files may carry only one fitted direction, so derive the // missing one by inversion. A singular matrix can't participate in the // warp either way, so such pairs contribute points only. diff --git a/client/platform/desktop/frontend/components/ViewerLoader.vue b/client/platform/desktop/frontend/components/ViewerLoader.vue index b065e459b..9cc2a596a 100644 --- a/client/platform/desktop/frontend/components/ViewerLoader.vue +++ b/client/platform/desktop/frontend/components/ViewerLoader.vue @@ -1756,13 +1756,17 @@ export default defineComponent({ // choice (save / discard / cancel) rather than the two-button navigate-away // prompt. Resolves true to allow the close, false to keep the app open. async function desktopCloseGuard(): Promise { - const count = viewerRef.value?.pendingSaveCount ?? 0; - if (!count) return true; + // Pending annotation saves OR unsaved Camera Registration panel edits. + const unsaved = viewerRef.value?.hasUnsavedChanges ?? false; + if (!unsaved) return true; const choice = await window.diveDesktop.invoke('desktop:confirm-close-unsaved'); if (choice === 'cancel') return false; if (choice === 'save') { try { - await viewerRef.value.save(); + if (viewerRef.value.pendingSaveCount > 0) { + await viewerRef.value.save(); + } + await viewerRef.value.saveRegistration(); } catch { // Save failed; the Viewer surfaces its own error, so keep the app open. return false; diff --git a/client/src/alignedView/CameraRegistrationStore.spec.ts b/client/src/alignedView/CameraRegistrationStore.spec.ts index 5fc6b746d..4f22df208 100644 --- a/client/src/alignedView/CameraRegistrationStore.spec.ts +++ b/client/src/alignedView/CameraRegistrationStore.spec.ts @@ -1,5 +1,22 @@ /// import CameraRegistrationStore from './CameraRegistrationStore'; +import { buildPerCameraRegistrationFiles } from './cameraRegistrationFiles'; + +/** + * Serialize a store's state through the production per-camera serializer + * (buildPerCameraRegistrationFiles) -- the only registration file format -- + * and return the single resulting file body as JSON text for round trips. + */ +function toSingleFileJson(store: CameraRegistrationStore): string { + const files = buildPerCameraRegistrationFiles({ + homographies: store.homographies.value, + correspondences: store.correspondences.value, + transformTypes: store.transformTypes.value, + source: store.source.value, + }, null); + expect(files).toHaveLength(1); + return JSON.stringify(files[0].body); +} describe('CameraRegistrationStore', () => { it('produces a directional pairKey that preserves left/right order', () => { @@ -667,7 +684,7 @@ describe('CameraRegistrationStore', () => { }); }); - describe('calibration JSON file round trip', () => { + describe('registration file round trip', () => { it('serializes and reloads all pairs', () => { const store = new CameraRegistrationStore(); store.setActivePair('left', 'right'); @@ -675,7 +692,7 @@ describe('CameraRegistrationStore', () => { addFourTranslationPairs(store); store.setTransformType(key, 'translation'); store.fitTransform(key); - const json = store.toRegistrationJson(); + const json = toSingleFileJson(store); const restored = new CameraRegistrationStore(); restored.setActivePair('left', 'right'); @@ -699,7 +716,7 @@ describe('CameraRegistrationStore', () => { }], })); const restored = new CameraRegistrationStore(); - restored.loadRegistrationText(store.toRegistrationJson()); + restored.loadRegistrationText(toSingleFileJson(store)); expect(restored.homographies.value[key]).toBeDefined(); expect(restored.isLoadedHomography(key)).toBe(true); }); @@ -709,7 +726,7 @@ describe('CameraRegistrationStore', () => { store.setActivePair('left', 'right'); addFourTranslationPairs(store); store.setAlignmentMode('AtoB'); - const json = store.toRegistrationJson(); + const json = toSingleFileJson(store); store.loadRegistrationText(json); expect(store.alignment.value.mode).toBe('original'); expect(store.activePair.value).toEqual({ camA: 'left', camB: 'right' }); @@ -752,7 +769,7 @@ describe('CameraRegistrationStore', () => { addFourTranslationPairs(store); store.maybeFitPair(store.pairKey('left', 'right')); expect(store.source.value).toStrictEqual(source); - const saved = JSON.parse(store.toRegistrationJson()); + const saved = JSON.parse(toSingleFileJson(store)); expect(saved.source).toStrictEqual(source); }); @@ -760,7 +777,7 @@ describe('CameraRegistrationStore', () => { const store = new CameraRegistrationStore(); store.setActivePair('left', 'right'); addFourTranslationPairs(store); - expect('source' in JSON.parse(store.toRegistrationJson())).toBe(false); + expect('source' in JSON.parse(toSingleFileJson(store))).toBe(false); }); it('clears a previous stamp when loading a file without one', () => { @@ -831,7 +848,7 @@ describe('CameraRegistrationStore', () => { store.maybeFitPair(key); const restored = new CameraRegistrationStore(); - restored.loadRegistrationText(store.toRegistrationJson()); + restored.loadRegistrationText(toSingleFileJson(store)); // The refit pair saved with its backing points, so it re-marks as // fitted (refined) rather than loaded. expect(restored.isRefinedFromSource(key)).toBe(true); diff --git a/client/src/alignedView/CameraRegistrationStore.ts b/client/src/alignedView/CameraRegistrationStore.ts index 3ad29f999..57c06e1f4 100644 --- a/client/src/alignedView/CameraRegistrationStore.ts +++ b/client/src/alignedView/CameraRegistrationStore.ts @@ -598,41 +598,10 @@ export default class CameraRegistrationStore { } /** - * Serialize every pair with content (points and/or a homography) as the - * portable calibration JSON file (see {@link RegistrationFile}). Pairs whose - * only state is a transform-type choice are omitted. - */ - toRegistrationJson(): string { - const keys = new Set([ - ...Object.keys(this.correspondences.value).filter( - (key) => this.correspondences.value[key].length > 0, - ), - ...Object.keys(this.homographies.value), - ]); - const pairs: RegistrationFilePair[] = [...keys].sort().map((key) => { - const [left, right] = key.split('::'); - const homography = this.homographies.value[key] || null; - return { - left, - right, - points: (this.correspondences.value[key] || []).map((c) => [c.a[0], c.a[1], c.b[0], c.b[1]]), - leftToRight: homography ? homography.AtoB : null, - rightToLeft: homography ? homography.BtoA : null, - transformType: this.transformTypeForPair(key), - }; - }); - const file: RegistrationFile = { - type: REGISTRATION_FILE_TYPE, - version: 1, - ...(this.source.value ? { source: this.source.value } : {}), - pairs, - }; - return JSON.stringify(file, null, 2); - } - - /** - * Parse and load a registration JSON file (the format written by - * {@link toRegistrationJson}), REPLACING all pairs' correspondences, + * Parse and load a registration JSON file (the per-camera + * _to__registration.json format written by + * cameraRegistrationFiles.ts's buildPerCameraRegistrationFiles -- + * the only registration file format), REPLACING all pairs' correspondences, * homographies, and transform types. The active pair selection and picking * toggle are left alone; the alignment ghost reverts to 'original' since * the transform under it changed wholesale. Throws a descriptive Error on diff --git a/client/src/components/annotators/linkedViewers/README.md b/client/src/components/annotators/linkedViewers/README.md index 4e7d0e275..8d28cbf1a 100644 --- a/client/src/components/annotators/linkedViewers/README.md +++ b/client/src/components/annotators/linkedViewers/README.md @@ -10,7 +10,7 @@ This folder holds the shared plumbing and the two links that reuse it with diffe | --- | --- | | `useLinkedViewers.ts` | Shared GeoJS listener wiring, re-entrancy guard, and zoom-baseline conversion. | | `useAlignedNavigation.ts` | Pan/zoom link for the **aligned view** (SEAL-TK feature 3). | -| `useRegistrationNavigation.ts` | Pan/zoom link for the **active registration pair** while picking points in the Manual Alignment panel. | +| `useRegistrationNavigation.ts` | Pan/zoom link for the **active registration pair** while picking points in the Camera Registration panel. | ## How linking works From 128d7940f42094b87f4f28d3bc792ffcb5539788 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Sun, 12 Jul 2026 13:10:00 -0400 Subject: [PATCH 06/13] Make loaded registrations first-class in the Camera Registration panel A registration imported from a producer (e.g. KAMERA) carries matrices but no picked points, and the panel treated it as incomplete: "Fit pan/zoom" stayed greyed out (it gated on point count, though the link only needs a homography), the status line claimed points were still needed, and picking was force-enabled so stray clicks placed points on a registration that needed none. - Add a three-state transform status (loaded from file / fit from N point pairs / none) and gate the pan/zoom link on having a transform at all, renaming it "Link pan/zoom"; enabling it fits the pair on demand. - Make picking an explicit "Pick points" toggle scoping the authoring UI (cursor readout, correspondences, transform type), defaulting per pair: off in review of a file-loaded transform, on when points are needed. - Decouple linked navigation from picking mode -- it now runs whenever the panel has a pair and the link is on; the panel clears the pair (and with it the warp ghost) on close to keep that scoped. - After a registration import, summarize what the loaded transform drives and, when the panel is open, re-select the imported pair in review posture (hydrate cleared it and left the panel inert). - Confirm before "Save registration" overwrites existing per-camera registration files, naming exactly the files whose content changes -- untouched pairs rewrite byte-identical and don't warn -- along with the provenance stamp being replaced. Co-Authored-By: Claude Fable 5 --- .../CameraRegistration/RegistrationTools.vue | 445 ++++++++++++------ .../components/ImportAnnotations.vue | 46 +- .../CameraRegistrationStore.spec.ts | 50 ++ .../alignedView/CameraRegistrationStore.ts | 44 +- .../useRegistrationNavigation.spec.ts | 34 +- .../useRegistrationNavigation.ts | 22 +- 6 files changed, 466 insertions(+), 175 deletions(-) diff --git a/client/dive-common/components/CameraRegistration/RegistrationTools.vue b/client/dive-common/components/CameraRegistration/RegistrationTools.vue index 58c163482..58197c0df 100644 --- a/client/dive-common/components/CameraRegistration/RegistrationTools.vue +++ b/client/dive-common/components/CameraRegistration/RegistrationTools.vue @@ -3,6 +3,7 @@ import { computed, defineComponent, onBeforeUnmount, ref, watch, } from 'vue'; import { + useAlignedView, useCameraStore, useCameraRegistration, useDatasetId, @@ -11,8 +12,10 @@ import { TransformType, TRANSFORM_TYPES, DEFAULT_TRANSFORM_TYPE, minPointsForTransform, } from 'vue-media-annotator/alignedView/transform'; import { unresolvedCameras } from 'vue-media-annotator/alignedView/alignedView'; +import { buildPerCameraRegistrationFiles } from 'vue-media-annotator/alignedView/cameraRegistrationFiles'; import TooltipBtn from 'vue-media-annotator/components/TooltipButton.vue'; import { useApi } from 'dive-common/apispec'; +import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; export default defineComponent({ name: 'CameraRegistration', @@ -22,7 +25,9 @@ export default defineComponent({ const cameraStore = useCameraStore(); const registration = useCameraRegistration(); const datasetId = useDatasetId(); + const alignedView = useAlignedView(); const { saveMetadata } = useApi(); + const { prompt } = usePrompt(); const cameras = computed(() => [...cameraStore.camMap.value.keys()]); /** @@ -71,17 +76,41 @@ export default defineComponent({ [camLeft.value, camRight.value] = cameras.value; } + /** + * Author-vs-review posture: picking defaults on for a pair that still + * needs points, and off for one whose transform came from a registration + * file (review it; the "Pick points" toggle opts back in to refine). + * Re-applied whenever the active pair changes identity, so it overrides a + * manual toggle on pair switch -- each pair opens in its own posture. + */ + function applyPickingDefault() { + registration.pickingEnabled.value = registration.pickingDefaultFor( + registration.activePairKey(), + ); + } + watch([camLeft, camRight], () => { registration.setActivePair(camLeft.value, camRight.value); + applyPickingDefault(); }, { immediate: true }); - // Picking is scoped to this panel: it is always on while the Manual - // Alignment panel is open, and turns off when the panel closes (unmounts), - // so the viewer can't be left in picking mode -- pair-only panes, suspended - // aligned view -- with no visible control to get back out. - registration.pickingEnabled.value = true; + // The pair can also be set from outside the panel (the Import menu + // re-selects a freshly imported pair); mirror such changes into the + // selectors so the panel doesn't keep showing a stale pair. + watch(registration.activePair, (pair) => { + if (pair && (pair.camA !== camLeft.value || pair.camB !== camRight.value)) { + camLeft.value = pair.camA; + camRight.value = pair.camB; + } + }); + + // Picking and the active pair are scoped to this panel: established while + // it is open, cleared when it closes (unmounts), so the viewer can't be + // left in picking mode -- or with linked pan/zoom, a warp ghost, or + // pair-only panes -- with no visible control to get back out. onBeforeUnmount(() => { registration.pickingEnabled.value = false; + registration.setActivePair(null, null); }); // Switching datasets while this panel stays mounted re-runs the viewer's @@ -102,7 +131,7 @@ export default defineComponent({ // re-establish the pair that hydrate() nulled. registration.setActivePair(camLeft.value, camRight.value); } - registration.pickingEnabled.value = true; + applyPickingDefault(); }); const activeKey = computed(() => registration.activePairKey()); @@ -135,24 +164,69 @@ export default defineComponent({ ); /** How many more correspondence pairs are needed before the transform can be fit. */ const remainingPoints = computed(() => Math.max(0, minPoints.value - correspondences.value.length)); + /** The active pair has a usable transform: enough points to fit one, or one loaded from a file. */ + const hasTransform = computed(() => canFit.value + || Boolean(activeKey.value && registration.homographies.value[activeKey.value])); + /** The active pair's transform came from a registration file (no in-app fit backing it). */ + const hasLoadedTransform = computed(() => { + const key = activeKey.value; + return Boolean(key && registration.homographies.value[key] && registration.isLoadedHomography(key)); + }); /** * Fit-robustness color: green with 12+ point pairs, yellow once the active - * transform can be fit (its own minimum, e.g. 2 for Similarity), grey below. + * transform can be fit (its own minimum, e.g. 2 for Similarity), grey + * below -- except that a file-loaded transform with no picked points is + * trusted as shipped (green). */ const fitQualityColor = computed(() => { if (correspondences.value.length >= 12) { return 'success'; } - return canFit.value ? 'warning' : 'grey'; + if (canFit.value) { + return 'warning'; + } + return hasLoadedTransform.value ? 'success' : 'grey'; }); - /** The active pair has a usable transform: enough points to fit one, or one loaded from a file. */ - const hasTransform = computed(() => canFit.value - || Boolean(activeKey.value && registration.homographies.value[activeKey.value])); - /** The active pair's transform came from a registration file (no in-app fit backing it). */ - const hasLoadedTransform = computed(() => { - const key = activeKey.value; - return Boolean(key && registration.homographies.value[key] && registration.isLoadedHomography(key)); + /** + * Three-state transform status for the active pair -- loaded from a file, + * fitted from picked points, or absent -- so a matrix-only registration + * loaded from a producer (e.g. KAMERA) reads as complete rather than + * "points still needed". + */ + const transformStatus = computed(() => { + if (hasLoadedTransform.value) { + return { + icon: 'mdi-file-check', + color: 'success', + text: 'Transform loaded from a registration file', + }; + } + if (canFit.value) { + return { + icon: 'mdi-check-circle', + color: fitQualityColor.value, + text: `Transform fit from ${correspondences.value.length} point pairs`, + }; + } + return { + icon: 'mdi-progress-clock', + color: 'grey', + text: 'No transform yet: pick points below, or import a registration (Import menu)', + }; }); + /** + * Toggle linked pan/zoom. Points are fit lazily, so enabling first fits + * the active pair when it has enough points but no transform yet + * (mirroring setAlignmentMode) -- the link engages immediately instead of + * waiting for the next fit-triggering action. + */ + function setLinkedNav(enabled: unknown) { + const on = Boolean(enabled); + if (on) { + registration.maybeFitActivePair(); + } + registration.linkedNav.value = on; + } /** * The active pair was refit in-app while a producer-stamped registration is * loaded, so its transform has diverged from what the stamped source @@ -227,11 +301,62 @@ export default defineComponent({ * per-row deletes) and state belonging to non-active pairs. Portable * copies for sharing come from the Export menu's per-camera registration * downloads, which read this saved state. + * + * Overwriting an existing saved registration (e.g. one imported from a + * producer like KAMERA) is confirmed first, naming only the per-camera + * file(s) whose content this save actually changes -- pairs the user + * didn't touch are rewritten byte-identical, which isn't an overwrite + * worth warning about. */ async function save() { + // Fit before diffing so the comparison reflects what will be written. + registration.maybeFitActivePair(); + // Group the saved baseline and the current state into per-camera files + // exactly the way the persistence layer writes them, against the same + // reference camera the backend uses (the dataset's Reference Camera + // choice, published by the viewer). + const reference = alignedView.reference.value ?? cameras.value[0] ?? null; + const savedFiles = buildPerCameraRegistrationFiles( + registration.savedRegistrationValues(), + reference, + ); + const nextFiles = new Map(buildPerCameraRegistrationFiles( + { + homographies: registration.homographies.value, + correspondences: registration.correspondences.value, + transformTypes: registration.transformTypes.value, + source: registration.source.value, + }, + reference, + ).map((file) => [file.name, file])); + // Existing files this save replaces with different content (or removes, + // for a cleared pair) -- the actual overwrites. + const overwritten = savedFiles + .filter((file) => { + const next = nextFiles.get(file.name); + return !next || JSON.stringify(next.body) !== JSON.stringify(file.body); + }) + .map((file) => file.name); + if (overwritten.length) { + const text = [ + `Saving will overwrite ${overwritten.join(', ')}.`, + ]; + if (sourceReadout.value) { + text.push(`Existing registration source: ${sourceReadout.value}`); + } + const confirmed = await prompt({ + title: 'Overwrite Saved Registration?', + text, + positiveButton: 'Overwrite', + negativeButton: 'Cancel', + confirm: true, + }); + if (!confirmed) { + return; + } + } saving.value = true; try { - registration.maybeFitActivePair(); await saveMetadata(datasetId.value, { cameraHomographies: registration.homographies.value, cameraCorrespondences: registration.correspondences.value, @@ -270,6 +395,8 @@ export default defineComponent({ canClearLast, canFit, fitQualityColor, + transformStatus, + setLinkedNav, linkedNav: registration.linkedNav, dirty: registration.dirty, saving, @@ -291,7 +418,8 @@ export default defineComponent({ class="mx-4" > - Pick corresponding points between two cameras to fit an alignment transform. + Register cameras by importing a registration file (Import menu) or by + picking corresponding points between two cameras. @@ -374,148 +502,181 @@ export default defineComponent({ class="mb-3" /> -
- {{ cursorReadout || 'Move the cursor over a camera to see its coordinates.' }} -
- - - - - - - Correspondences ({{ correspondences.length }}) - - -
- - -
- - - - - Transform loaded from a file (no picked points). Picking {{ minPoints }} or more - points and fitting will replace it. - - - No correspondences yet. At least {{ minPoints }} required for the selected transform. - - - - - - - Could not fit transform: {{ fitError }} - - - - -

Transform Type

- -
+
- {{ canFit ? 'mdi-check-circle' : 'mdi-progress-clock' }} + {{ transformStatus.icon }} - - - + + {{ transformStatus.text }}
+ + Linked pan/zoom and the overlay warp use the loaded transform. Picking + points is optional: fitting {{ minPoints }} or more pairs replaces it. + + + + Click matching features in each camera to add correspondence pairs. + + + + + + Could not fit transform: {{ fitError }} + + + +

Overlay Warp

key.split('::').includes(camera) + && key.split('::').every((name) => cameraStore.camMap.value.has(name))); + const [left, right] = importedKey + ? importedKey.split('::') + : [priorPair.camA, priorPair.camB]; + cameraRegistration.setActivePair(left, right); + cameraRegistration.pickingEnabled.value = cameraRegistration.pickingDefaultFor( + cameraRegistration.activePairKey(), + ); + } processing.value = false; const unknown = result.cameras.filter((name) => !cameraStore.camMap.value.has(name)); + const text = [ + `Imported registration for camera "${camera}".`, + 'Registration can be validated in the Camera Registration menu.', + ]; if (unknown.length) { - await prompt({ - title: 'Registration Imported', - text: [ - `Imported ${result.pairCount} pair(s), but the file names camera(s) not in this dataset:`, - unknown.join(', '), - 'Pair bodies name their own cameras, so these pairs will not resolve until matching cameras exist.', - ], - positiveButton: 'OK', - }); + text.push( + `Warning: the file also names camera(s) not in this dataset: ${unknown.join(', ')}.`, + 'Pair bodies name their own cameras, so those pairs will not resolve until matching cameras exist.', + ); } + await prompt({ + title: 'Registration Imported', + text, + positiveButton: 'OK', + }); } catch (error) { processing.value = false; prompt({ diff --git a/client/src/alignedView/CameraRegistrationStore.spec.ts b/client/src/alignedView/CameraRegistrationStore.spec.ts index 4f22df208..6d0ba7814 100644 --- a/client/src/alignedView/CameraRegistrationStore.spec.ts +++ b/client/src/alignedView/CameraRegistrationStore.spec.ts @@ -682,6 +682,56 @@ describe('CameraRegistrationStore', () => { store.maybeFitPair(key); expect(store.homographies.value[key]).toBeDefined(); }); + + it('pickingDefaultFor opens loaded pairs in review posture, others authoring', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + // No transform at all: authoring. + expect(store.pickingDefaultFor(key)).toBe(true); + expect(store.pickingDefaultFor(null)).toBe(true); + // File-loaded transform: review. + loadMatrixOnlyPair(store, 'left', 'right', translate); + expect(store.pickingDefaultFor(key)).toBe(false); + // Point-backed fit: authoring again. + addFourTranslationPairs(store); + store.maybeFitPair(key); + expect(store.pickingDefaultFor(key)).toBe(true); + }); + }); + + describe('hasSavedRegistration', () => { + const translate = [[1, 0, 5], [0, 1, -3], [0, 0, 1]]; + + it('is false for a fresh store and empty hydrate', () => { + const store = new CameraRegistrationStore(); + expect(store.hasSavedRegistration()).toBe(false); + store.hydrate({}, {}, {}); + expect(store.hasSavedRegistration()).toBe(false); + }); + + it('ignores unsaved in-progress work until markSaved', () => { + const store = new CameraRegistrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + expect(store.hasSavedRegistration()).toBe(false); + store.markSaved(); + expect(store.hasSavedRegistration()).toBe(true); + }); + + it('is true after hydrating a saved homography', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + store.hydrate({ [key]: { AtoB: translate, BtoA: translate } }, {}, {}); + expect(store.hasSavedRegistration()).toBe(true); + }); + + it('treats leftover empty correspondence lists as no registration', () => { + const store = new CameraRegistrationStore(); + const key = store.pairKey('left', 'right'); + store.hydrate({}, { [key]: [] }, {}); + expect(store.hasSavedRegistration()).toBe(false); + }); }); describe('registration file round trip', () => { diff --git a/client/src/alignedView/CameraRegistrationStore.ts b/client/src/alignedView/CameraRegistrationStore.ts index 57c06e1f4..48ac51d18 100644 --- a/client/src/alignedView/CameraRegistrationStore.ts +++ b/client/src/alignedView/CameraRegistrationStore.ts @@ -128,8 +128,9 @@ export default class CameraRegistrationStore { /** * Whether pan/zoom is linked between the active pair's two cameras through - * the fitted transform (see {@link useRegistrationNavigation}). Only has an - * effect once a transform is fitted; toggled from the panel's "Fit pan/zoom". + * the pair's transform (see {@link useRegistrationNavigation}). Only has an + * effect once a transform exists (fitted or file-loaded); toggled from the + * panel's "Link pan/zoom". */ linkedNav: Ref; @@ -217,6 +218,34 @@ export default class CameraRegistrationStore { this.savedSnapshot.value = this.registrationSnapshot(); } + /** + * The saved-baseline calibration (what the persisted registration -- the + * per-camera registration files on desktop, the dataset meta on web -- + * currently holds). Parsed fresh from the snapshot on each call. Matches + * cameraRegistrationFiles.ts's CameraRegistrationValues shape (the type + * lives there and importing it here would be circular). + */ + savedRegistrationValues(): { + homographies: CameraHomographies; + correspondences: CameraCorrespondences; + transformTypes: CameraTransformTypes; + source: RegistrationSource | null; + } { + return JSON.parse(this.savedSnapshot.value); + } + + /** + * True when the saved baseline (last hydrate/save/load) already holds + * registration data -- i.e. saving now would overwrite a persisted + * registration rather than create a fresh one. Empty correspondence lists + * don't count: clearing a pair leaves `[]` behind. + */ + hasSavedRegistration(): boolean { + const saved = this.savedRegistrationValues(); + return Object.keys(saved.homographies).length > 0 + || Object.values(saved.correspondences).some((list) => list.length > 0); + } + /** * True when the loaded calibration was assembled from per-camera files * whose producer stamps disagree (the loader records that as a @@ -429,6 +458,17 @@ export default class CameraRegistrationStore { return this.homographySources[key] === 'loaded'; } + /** + * Default picking posture when a pair becomes active: authoring (true) + * unless the pair already has a file-loaded transform, in which case it + * opens in a review posture (false) so stray clicks don't start placing + * points on top of a registration that needs no points. The user can still + * opt back in via the panel's "Pick points" toggle to refine. + */ + pickingDefaultFor(key: string | null): boolean { + return !(key && this.homographies.value[key] && this.isLoadedHomography(key)); + } + /** * True when `key`'s transform was fitted from in-app picked points while a * producer-stamped calibration is loaded -- i.e. the pair has diverged from diff --git a/client/src/components/annotators/linkedViewers/useRegistrationNavigation.spec.ts b/client/src/components/annotators/linkedViewers/useRegistrationNavigation.spec.ts index dcc2e4db8..1f3fcf487 100644 --- a/client/src/components/annotators/linkedViewers/useRegistrationNavigation.spec.ts +++ b/client/src/components/annotators/linkedViewers/useRegistrationNavigation.spec.ts @@ -59,16 +59,15 @@ function makeHarness() { getController: (name: string) => controllers[name], }) as unknown as Ref; - const pickingEnabled = ref(false); const linkedNav = ref(false); const homographies = ref>({}); const fitted = ref(true); + const activePair = ref<{ camA: string; camB: string } | null>({ camA: 'eo', camB: 'ir' }); // eo -> ir is a pure +100 x-translation (and ir -> eo its inverse), so the // linked scale is 1 and centers map by simple offset. const registration = { - pickingEnabled, linkedNav, - activePair: ref({ camA: 'eo', camB: 'ir' }), + activePair, homographies, recenterRequest: ref(null), linkedPoint(camera: string, coord: Point) { @@ -84,17 +83,15 @@ function makeHarness() { useRegistrationNavigation(aggregate, registration); return { - eo, ir, pickingEnabled, linkedNav, homographies, fitted, + eo, ir, linkedNav, activePair, homographies, fitted, }; } describe('useRegistrationNavigation', () => { - it('snaps the pair immediately when "Fit pan/zoom" turns on', async () => { + it('snaps the pair immediately when "Link pan/zoom" turns on', async () => { const { - eo, ir, pickingEnabled, linkedNav, + eo, ir, linkedNav, } = makeHarness(); - pickingEnabled.value = true; - await nextTick(); eo.center({ x: 250, y: 150 }); eo.zoom(1); // units-per-pixel = 0.5 @@ -112,9 +109,8 @@ describe('useRegistrationNavigation', () => { it('re-snaps when the fitted homography changes under the link', async () => { const { - eo, ir, pickingEnabled, linkedNav, homographies, + eo, ir, linkedNav, homographies, } = makeHarness(); - pickingEnabled.value = true; linkedNav.value = true; await nextTick(); @@ -127,14 +123,28 @@ describe('useRegistrationNavigation', () => { it('does not snap while no fit exists yet', async () => { const { - ir, pickingEnabled, linkedNav, fitted, + ir, linkedNav, fitted, } = makeHarness(); fitted.value = false; - pickingEnabled.value = true; linkedNav.value = true; await nextTick(); expect(ir.center()).toEqual({ x: 0, y: 0 }); expect(ir.zoom()).toBe(0); }); + + it('detaches when the active pair clears (panel closed)', async () => { + const { + eo, ir, linkedNav, activePair, + } = makeHarness(); + linkedNav.value = true; + await nextTick(); + + activePair.value = null; + await nextTick(); + + eo.center({ x: 250, y: 150 }); + eo.trigger('geo_pan'); + expect(ir.center()).toEqual({ x: 100, y: 0 }); + }); }); diff --git a/client/src/components/annotators/linkedViewers/useRegistrationNavigation.ts b/client/src/components/annotators/linkedViewers/useRegistrationNavigation.ts index 1a2ac8bae..d045b727f 100644 --- a/client/src/components/annotators/linkedViewers/useRegistrationNavigation.ts +++ b/client/src/components/annotators/linkedViewers/useRegistrationNavigation.ts @@ -10,10 +10,12 @@ import useLinkedViewers from './useLinkedViewers'; * Links pan/zoom recentering between the two cameras of the active registration * pair: panning or zooming one recenters the other on the same point, mapped * through the pair's fitted homography ({@link CameraRegistrationStore.linkedPoint}). - * The mapping assumes UNWARPED panes showing native coordinates, so this is - * active only while point picking is (the aligned-view link, - * {@link useAlignedNavigation}, owns navigation otherwise) and stands down if - * the aligned view is somehow active. Distinct from the general "sync cameras" + * Active whenever the Camera Registration panel has a pair selected (the + * panel clears the pair on close) and "Link pan/zoom" is on -- a file-loaded + * transform drives it just as well as picked points, so it deliberately does + * NOT require picking mode. The mapping assumes UNWARPED panes showing native + * coordinates, so it stands down while the aligned view (which owns + * navigation then, {@link useAlignedNavigation}) is active. Distinct from the general "sync cameras" * toggle (Controls.vue), which assumes identical pixel scale between panes. */ export default function useRegistrationNavigation( @@ -64,15 +66,16 @@ export default function useRegistrationNavigation( function setup() { teardown(); const pair = registration.activePair.value; - // Authoring UI: active while picking and "Fit pan/zoom" is on (once a fit - // exists, linkedPoint returns matches). attach() no-ops for a not-yet-ready + // Active while the panel has a pair and "Link pan/zoom" is on (once a + // transform exists -- fitted or file-loaded -- linkedPoint returns + // matches). attach() no-ops for a not-yet-ready // pane; the resizeTrigger watch re-runs setup once both viewers exist. - if (registration.pickingEnabled.value && registration.linkedNav.value && pair) { + if (registration.linkedNav.value && pair) { attach(pair.camA, link(pair.camA, pair.camB)); attach(pair.camB, link(pair.camB, pair.camA)); - // Snap immediately from camA so toggling "Fit pan/zoom" on (or a refit + // Snap immediately from camA so toggling "Link pan/zoom" on (or a refit // under it) lines the pair up right away instead of waiting for the - // first pan/zoom event. No-ops harmlessly while no fit exists yet + // first pan/zoom event. No-ops harmlessly while no transform exists yet // (linkedPoint returns null). link(pair.camA, pair.camB)(); } @@ -80,7 +83,6 @@ export default function useRegistrationNavigation( watch( [ - registration.pickingEnabled, registration.linkedNav, registration.activePair, registration.homographies, From f37acc4442eef58d0d4f572c8b3a94de367a09fc Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 14 Jul 2026 20:08:38 -0400 Subject: [PATCH 07/13] Remap Align View warp crops into actual texture pixels Overview canvases are smaller than native frame size; crop rects must follow the texture or ghosts/Align View sample the wrong texels. --- client/src/alignedView/homography.spec.ts | 44 +++++++++++++ client/src/alignedView/homography.ts | 62 +++++++++++++++++++ client/src/layers/AlignedImageLayer.ts | 18 ++---- .../RegistrationKeypointLayer.ts | 12 ++-- client/src/layers/cameraImage.ts | 21 +++++++ 5 files changed, 139 insertions(+), 18 deletions(-) create mode 100644 client/src/layers/cameraImage.ts diff --git a/client/src/alignedView/homography.spec.ts b/client/src/alignedView/homography.spec.ts index 180dc4a73..f611d6646 100644 --- a/client/src/alignedView/homography.spec.ts +++ b/client/src/alignedView/homography.spec.ts @@ -7,6 +7,7 @@ import { subdivideWarpQuads, warpGridSize, geojsWarpQuads, + geojsWarpQuadsForImage, localLinkedScale, Point, Matrix3, @@ -285,3 +286,46 @@ describe('geojsWarpQuads', () => { }); }); }); + +describe('geojsWarpQuadsForImage', () => { + it('leaves crops unchanged when the texture matches native dimensions', () => { + const translate: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + const img = { + naturalWidth: 640, + naturalHeight: 480, + width: 640, + height: 480, + } as HTMLImageElement; + const [quad] = geojsWarpQuadsForImage(translate, { + source: img, + kind: 'image', + width: 640, + height: 480, + }); + expect(quad.crop).toEqual({ + left: 0, top: 0, right: 640, bottom: 480, x: 640, y: 480, + }); + expect(quad.image).toBe(img); + }); + + it('remaps crops into overview texture pixels for downsampled large-image canvases', () => { + const translate: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + const canvas = { + width: 2048, + height: 1024, + } as HTMLCanvasElement; + // Native IR frame is 4x the overview texture on each axis. + const [quad] = geojsWarpQuadsForImage(translate, { + source: canvas, + kind: 'image', + width: 8192, + height: 4096, + }); + expect(quad.ul).toEqual({ x: 0, y: 0 }); + expect(quad.lr).toEqual({ x: 8192, y: 4096 }); + expect(quad.crop).toEqual({ + left: 0, top: 0, right: 2048, bottom: 1024, x: 2048, y: 1024, + }); + expect(quad.image).toBe(canvas); + }); +}); diff --git a/client/src/alignedView/homography.ts b/client/src/alignedView/homography.ts index 518297efd..410908bd3 100644 --- a/client/src/alignedView/homography.ts +++ b/client/src/alignedView/homography.ts @@ -224,6 +224,68 @@ export function geojsWarpQuads( })); } +/** + * Pixel size of a geoJS quad texture element. Canvas overviews (large-image + * frame textures) are often smaller than the logical native width/height used + * for warp math -- crop rectangles must be scaled to this size or Align View / + * ghosts sample the wrong texel range and look massively scaled. + */ +export function texturePixelSize( + source: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement, +): { width: number; height: number } { + // Duck-type so this works in node unit tests without a DOM (and across + // cross-realm element instances where instanceof can fail). + if ('videoWidth' in source && typeof (source as HTMLVideoElement).videoWidth === 'number') { + const video = source as HTMLVideoElement; + return { width: video.videoWidth, height: video.videoHeight }; + } + if ('getContext' in source && typeof (source as HTMLCanvasElement).getContext === 'function') { + const canvas = source as HTMLCanvasElement; + return { width: canvas.width, height: canvas.height }; + } + const image = source as HTMLImageElement; + return { + width: image.naturalWidth || image.width, + height: image.naturalHeight || image.height, + }; +} + +/** + * Build geojs warp quads for a {@link CameraImage}, remapping `crop` into the + * texture's actual pixel space when the texture is a downsampled overview of + * the native frame (large-image). Corner positions stay in native/map space + * so the fitted homography still lands correctly. + */ +export function geojsWarpQuadsForImage( + h: Matrix3, + image: { + source: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement; + kind: 'image' | 'video'; + width: number; + height: number; + }, + overlap = 0, +): Array { + const quads = geojsWarpQuads(h, image.width, image.height, overlap); + const tex = texturePixelSize(image.source); + const scaleX = image.width > 0 ? tex.width / image.width : 1; + const scaleY = image.height > 0 ? tex.height / image.height : 1; + const rescale = Math.abs(scaleX - 1) > 1e-9 || Math.abs(scaleY - 1) > 1e-9; + return quads.map((q) => { + const crop = rescale + ? { + left: q.crop.left * scaleX, + top: q.crop.top * scaleY, + right: q.crop.right * scaleX, + bottom: q.crop.bottom * scaleY, + x: tex.width, + y: tex.height, + } + : q.crop; + return { ...q, crop, [image.kind]: image.source }; + }); +} + /** * Hartley normalization: translate points to the centroid and scale so the * mean distance from the origin is sqrt(2). Returns the normalized points and diff --git a/client/src/layers/AlignedImageLayer.ts b/client/src/layers/AlignedImageLayer.ts index ea3b6c7ef..d2096c72b 100644 --- a/client/src/layers/AlignedImageLayer.ts +++ b/client/src/layers/AlignedImageLayer.ts @@ -1,16 +1,10 @@ import geo from 'geojs'; import type { MediaController } from '../components/annotators/mediaControllerType'; -import { Matrix3, geojsWarpQuads } from '../alignedView/homography'; +import { Matrix3, geojsWarpQuadsForImage } from '../alignedView/homography'; import { findQuadMediaLayer } from '../components/layerManager/quadMediaSource'; +import type { CameraImage } from './cameraImage'; -export interface CameraImage { - /** The texture source for the geojs quad feature: an `` for image sequences, a `