From e5904b78540f7b9e037b4fca6f6663e7aebae21c Mon Sep 17 00:00:00 2001 From: Boris Masis Date: Wed, 24 Jun 2026 09:02:26 -0700 Subject: [PATCH] Add reorientation component: face the direction of travel Reorients each spherical (360) image to face the direction of travel (the great-circle bearing toward the next image in the sequence) as the user navigates, instead of facing off to the side or behind. Can be disabled with component: { reorientation: false }. ReorientationEngine (pure, unit-tested): GPS-speed motion detection with a low-speed-turn allowance, GPS-outlier rejection, and last-confirmed-bearing fallback. Results cached and prefetched ahead. ReorientationComponent: feeds the engine from the viewer's graph and steers via a new eased state op (rotateToBasicSmooth); a manual look-around offset is preserved across the sequence. --- src/component/ComponentName.ts | 1 + .../interfaces/ReorientationConfiguration.ts | 87 ++++ .../reorientation/ReorientationComponent.ts | 330 +++++++++++++++ .../reorientation/ReorientationEngine.ts | 383 ++++++++++++++++++ src/external/component.ts | 6 + src/mapillary.ts | 3 + src/state/StateContext.ts | 13 + src/state/StateService.ts | 13 + src/state/interfaces/IStateBase.ts | 5 + src/state/interfaces/IStateContext.ts | 3 + src/state/state/InteractiveStateBase.ts | 18 + src/state/state/StateBase.ts | 20 + src/state/state/TraversingState.ts | 30 ++ src/viewer/ComponentController.ts | 1 + src/viewer/options/ComponentOptions.ts | 11 + .../reorientation/ReorientationEngine.test.ts | 158 ++++++++ test/viewer/ComponentController.test.ts | 2 + 17 files changed, 1084 insertions(+) create mode 100644 src/component/interfaces/ReorientationConfiguration.ts create mode 100644 src/component/reorientation/ReorientationComponent.ts create mode 100644 src/component/reorientation/ReorientationEngine.ts create mode 100644 test/component/reorientation/ReorientationEngine.test.ts diff --git a/src/component/ComponentName.ts b/src/component/ComponentName.ts index 4dc7b11fa..32d364c03 100644 --- a/src/component/ComponentName.ts +++ b/src/component/ComponentName.ts @@ -9,6 +9,7 @@ export type ComponentName = | "marker" | "pointer" | "popup" + | "reorientation" | "sequence" | "slider" | "spatial" diff --git a/src/component/interfaces/ReorientationConfiguration.ts b/src/component/interfaces/ReorientationConfiguration.ts new file mode 100644 index 000000000..d6cdabfb1 --- /dev/null +++ b/src/component/interfaces/ReorientationConfiguration.ts @@ -0,0 +1,87 @@ +import { ComponentConfiguration } from "./ComponentConfiguration"; + +/** + * Interface for configuration of the reorientation component. + * + * @interface + * @example + * ```js + * var viewer = new Viewer({ + * ... + * component: { + * reorientation: { + * movingSpeedMps: 1, + * }, + * }, + * ... + * }); + * ``` + */ +export interface ReorientationConfiguration extends ComponentConfiguration { + /** + * Number of images ahead in the sequence to precompute the + * reorientation for, so that a step lands on an already resolved + * bearing and the transition stays smooth. + * + * @default 3 + */ + prefetchAhead?: number; + + /** + * Minimum GPS speed, in meters per second, above which travel is + * considered sustained motion and the GPS travel bearing is trusted. + * + * @default 1 + */ + movingSpeedMps?: number; + + /** + * Below {@link movingSpeedMps}, a step is still accepted as motion + * (a slow turn rather than GPS noise) when the compass angle agrees + * with the travel bearing within {@link lowSpeedTurnMaxDeltaDeg} and + * the step is longer than this distance in meters. + * + * @default 2 + */ + lowSpeedTurnDistanceM?: number; + + /** + * Maximum angle, in degrees, between compass angle and travel bearing + * for a low-speed step to be accepted as a genuine turn. + * + * @default 30 + */ + lowSpeedTurnMaxDeltaDeg?: number; + + /** + * Maximum bearing change, in degrees, from the last confirmed moving + * bearing before a step is rejected as a GPS outlier and the previous + * bearing is held instead. + * + * @default 90 + */ + outlierMaxDeltaDeg?: number; + + /** + * Number of preceding images to scan in cache for prior motion context. + * + * @default 5 + */ + previousContextWindow?: number; + + /** + * Number of preceding images to scan when validating bearing continuity + * for a moving image. + * + * @default 10 + */ + movingHistoryWindow?: number; + + /** + * Number of preceding images to scan for the last confirmed moving + * bearing when the current step is not itself moving. + * + * @default 100 + */ + fallbackHistoryWindow?: number; +} diff --git a/src/component/reorientation/ReorientationComponent.ts b/src/component/reorientation/ReorientationComponent.ts new file mode 100644 index 000000000..2ffb87fdd --- /dev/null +++ b/src/component/reorientation/ReorientationComponent.ts @@ -0,0 +1,330 @@ +import { first } from "rxjs/operators"; + +import { Component } from "../Component"; +import { ComponentName } from "../ComponentName"; +import { ReorientationConfiguration } + from "../interfaces/ReorientationConfiguration"; +import { + bearingToBasicX, + ReorientationEngine, + ReorientationImage, + ReorientationProvider, + wrapDelta, +} from "./ReorientationEngine"; + +import { Image } from "../../graph/Image"; +import { Sequence } from "../../graph/Sequence"; +import { Container } from "../../viewer/Container"; +import { Navigator } from "../../viewer/Navigator"; + +// Skip a reorientation when it would move the current view less than this many +// degrees (on either axis) — small moves are just jitter. +const MIN_REORIENT_DEG = 15; + +/** + * @class ReorientationComponent + * + * @classdesc Reorients each spherical image to face the direction of travel + * (the great-circle bearing toward the next image in the sequence) as the + * user navigates, instead of preserving the previous look direction. If the + * user drags to look around, that manual offset is preserved across the rest + * of the sequence rather than re-facing forward on every step. Active by + * default; disable with `component: { reorientation: false }`. + * + * @example + * ```js + * var viewer = new Viewer({ + * accessToken: "", + * container: "", + * component: { reorientation: false }, + * }); + * ``` + */ +export class ReorientationComponent + extends Component { + + public static componentName: ComponentName = "reorientation"; + + private _engine: ReorientationEngine; + private _activeId: string; + private _computedBasicX: number; + + // Sequence of the last image we resolved, so a change of sequence (a fresh + // load or a deliberate jump to another capture) can be reoriented even when + // the landing image's own GPS speed reads as stationary. + private _lastSeq: string; + + // Manual horizontal look-around offset, preserved within a sequence so the + // engine doesn't yank the view back to the travel direction on every step. + private _userOffsetX: number; + + // Vertical look offset from the horizon. Seeded once per activation from the + // current view (so a shared link's y / carried pitch is kept) and updated on + // drag, then applied deterministically as 0.5 + offset. Re-reading the live + // y every image round-trips through the spherical projection and drifts, so + // it is held, not re-read. + private _userOffsetY: number; + private _ySeeded: boolean; + + constructor(name: string, container: Container, navigator: Navigator) { + super(name, container, navigator); + } + + protected _activate(): void { + const subs = this._subscriptions; + + subs.push(this._configuration$.subscribe( + (configuration: ReorientationConfiguration): void => { + this._engine = new ReorientationEngine( + this._createProvider(), configuration); + this._activeId = null; + this._computedBasicX = 0.5; + this._lastSeq = null; + this._resetOffset(); + })); + + // currentImage$ fires reliably on every navigation, including in-app + // jumps to another sequence (currentState$ did not deliver those). + subs.push(this._navigator.stateService.currentImage$.subscribe( + (image: Image): void => { + if (!image) { + return; + } + this._activeId = image.id; + this._reorient(image); + })); + + // A finished drag is a genuine user look-around (our own steering goes + // through the state, not pointer events), so capture the offset. + subs.push(this._container.mouseService.mouseDragEnd$.subscribe( + (): void => { this._captureOffset(); })); + } + + protected _deactivate(): void { + this._subscriptions.unsubscribe(); + this._engine = null; + this._activeId = null; + this._lastSeq = null; + this._resetOffset(); + } + + protected _getDefaultConfiguration(): ReorientationConfiguration { + return {}; + } + + private _reorient(image: Image): void { + const id = image.id; + const seed = this._seed(image); + const engine = this._engine; + engine.precompute(id, seed) + .then((): void => { + // Ignore stale results from navigation that has moved on. + if (this._activeId !== id || this._engine !== engine) { + return; + } + const result = engine.get(id); + // Read mesh now (after the precompute delay) so it's loaded: + // an image with SfM mesh eases to the travel direction, one + // without (disconnected) hard-cuts. The transition type is NOT + // used — an SfM image eases however you arrive (fresh URL, + // in-sequence step, or feed-click jump). + const meshV = image.mesh && image.mesh.vertices ? + image.mesh.vertices.length : -1; + const hardCut = meshV <= 0; + if (!result || !result.valid) { + return; + } + this._computedBasicX = result.basicX; + + // A fresh load or a jump to another capture lands on a new + // sequence; reorient it to travel direction even if the landing + // image's GPS speed reads as stationary. Within a sequence, + // preserve the view when not moving rather than spinning on + // jitter. + const freshSequence = + result.seq != null && result.seq !== this._lastSeq; + this._lastSeq = result.seq; + // Pitch is tracked deterministically via the offset (the live + // view y drifts ~18° through the spherical projection, so it + // can't be measured per image). On a sequence change the held + // pitch resets to horizon; the reset amount is how far the + // carried pitch must move, used to force a reorientation even + // when the horizontal change is small. + let pitchResetDeg = 0; + if (freshSequence) { + // Don't carry x/pitch/rotation offsets across sequences: + // start fresh at travel direction + horizon. A cross-sequence + // jump carries the prior image's view, which would otherwise + // persist the previous sequence's pitch (e.g. "looking down"). + pitchResetDeg = Math.abs(this._userOffsetY) * 180; + this._userOffsetX = 0; + this._userOffsetY = 0; + this._ySeeded = true; + // Drop look-ahead hints from the prior sequence so nothing + // carries over; this sequence registers its own as it goes. + this._navigator.stateService.clearReorientations(); + } + if (!result.moving && !freshSequence) { + // Low motion within the current sequence — preserve the view. + return; + } + + // Horizontal target: travel direction plus any manual offset. + const targetX = this._applyOffsetX(result.basicX); + + // Read this image's current view and reorient only if doing so + // would move it past the threshold on either axis — otherwise + // the move is just jitter. Measured per image (current view → + // target), not accumulated from a prior image. + this._navigator.stateService.getCenter().pipe(first()).subscribe( + (center: number[]): void => { + if (this._activeId !== id || this._engine !== engine) { + return; + } + // Seed the held pitch offset once from the loaded view. + if (!this._ySeeded) { + this._userOffsetY = center[1] - 0.5; + this._ySeeded = true; + } + const targetY = + Math.max(0, Math.min(1, 0.5 + this._userOffsetY)); + + // Horizontal only: the pitch is held deterministically, + // so the live y wobbles with spherical-projection round- + // trip noise (~0.1 ≈ 18°). Gating on it would just snap + // back projection drift — the very jitter we're avoiding. + const dxDeg = + Math.abs(wrapDelta(targetX - center[0])) * 360; + // Reorient if the horizontal move clears the threshold, + // or (on a sequence change) the pitch must reset by more + // than the threshold to clear a carried look up/down. + const move = dxDeg >= MIN_REORIENT_DEG || + pitchResetDeg >= MIN_REORIENT_DEG; + if (move) { + if (hardCut) { + this._navigator.stateService + .rotateToBasic([targetX, targetY]); + } else { + this._navigator.stateService + .rotateToBasicSmooth([targetX, targetY]); + } + } + + // Pre-orient the next image (hint) so a hard cut to it + // doesn't flash the carried view — but only if its + // reorientation would also clear the threshold. This is + // cross-image, so reason in absolute bearings: the view + // the next image carries in is where this one ends up. + if (result.nextId && typeof result.cca === "number") { + const endX = move ? targetX : center[0]; + const endBearing = + result.cca + (endX - 0.5) * 360; + const nid = String(result.nextId); + engine.precompute(nid) + .then((): void => { + if (this._engine !== engine) { + return; + } + const nr = engine.get(nid); + if (!nr || !nr.valid || + typeof nr.cca !== "number") { + return; + } + const nTargetX = this._applyOffsetX(nr.basicX); + const nCarriedX = + bearingToBasicX(endBearing, nr.cca); + const nDx = Math.abs( + wrapDelta(nTargetX - nCarriedX)) * 360; + if (nDx >= MIN_REORIENT_DEG) { + this._navigator.stateService + .setReorientation( + nid, [nTargetX, targetY]); + } + }) + .catch((): void => { /* skip */ }); + } + }); + }) + .catch((): void => { /* skip images we can't resolve */ }); + } + + private _applyOffsetX(basicX: number): number { + return ((basicX + this._userOffsetX) % 1 + 1) % 1; + } + + private _captureOffset(): void { + const id = this._activeId; + const engine = this._engine; + if (id == null || engine == null) { + return; + } + this._navigator.stateService.getCenter().pipe(first()).subscribe( + (center: number[]): void => { + if (this._activeId !== id || this._engine !== engine) { + return; + } + const result = engine.get(id); + const basis = result && result.valid ? + result.basicX : this._computedBasicX; + this._userOffsetX = wrapDelta(center[0] - basis); + this._userOffsetY = center[1] - 0.5; + this._ySeeded = true; + }); + } + + private _resetOffset(): void { + this._userOffsetX = 0; + this._userOffsetY = 0; + this._ySeeded = false; + } + + private _seed(image: Image): ReorientationImage { + const lngLat = image.lngLat; + return { + id: image.id, + lat: lngLat ? lngLat.lat : null, + lng: lngLat ? lngLat.lng : null, + cca: image.computedCompassAngle, + cam: image.cameraType, + seq: image.sequenceId, + ts: image.capturedAt, + }; + } + + private _createProvider(): ReorientationProvider { + const graphService = this._navigator.graphService; + const imageCache = new Map(); + return { + fetchImage: (id: string): Promise => { + id = String(id); + const cached = imageCache.get(id); + if (cached) { + return Promise.resolve(cached); + } + return new Promise((resolve, reject) => { + graphService.cacheImage$(id).pipe(first()).subscribe( + (image: Image): void => { + const o = this._seed(image); + imageCache.set(id, o); + resolve(o); + }, + (e: Error): void => reject(e)); + }); + }, + cacheImage: (image: ReorientationImage): void => { + if (image && image.id != null) { + imageCache.set(String(image.id), image); + } + }, + fetchSeqIds: (seqId: string): Promise => { + return new Promise((resolve, reject) => { + graphService.cacheSequence$(seqId).pipe(first()).subscribe( + (sequence: Sequence): void => { + resolve((sequence.imageIds || []).map(String)); + }, + (e: Error): void => reject(e)); + }); + }, + }; + } +} diff --git a/src/component/reorientation/ReorientationEngine.ts b/src/component/reorientation/ReorientationEngine.ts new file mode 100644 index 000000000..28fc9b16a --- /dev/null +++ b/src/component/reorientation/ReorientationEngine.ts @@ -0,0 +1,383 @@ +import { ReorientationConfiguration } from "../interfaces/ReorientationConfiguration"; + +/** + * Minimal image metadata the engine reasons about, decoupled from the + * graph implementation so the engine can be unit tested with a fake + * {@link ReorientationProvider}. + */ +export interface ReorientationImage { + id: string; + lat: number; + lng: number; + cca: number; + cam: string; + seq: string; + ts: number; +} + +/** + * Data source for the engine. The component implements this on top of the + * viewer's graph service; tests implement it with canned data. + */ +export interface ReorientationProvider { + fetchImage(id: string): Promise; + fetchSeqIds(seqId: string): Promise; + cacheImage?(image: ReorientationImage): void; +} + +/** + * Outcome of resolving the reorientation for a single image. + */ +export interface ReorientationResult { + valid: boolean; + reason?: string; + nextId?: string; + travel?: number; + basicX?: number; + dist?: number; + cca?: number; + speed?: number; + moving?: boolean; + seq?: string; +} + +export const DEFAULT_REORIENTATION_CONFIGURATION: + Required = { + prefetchAhead: 3, + movingSpeedMps: 1, + lowSpeedTurnDistanceM: 2, + lowSpeedTurnMaxDeltaDeg: 30, + outlierMaxDeltaDeg: 90, + previousContextWindow: 5, + movingHistoryWindow: 10, + fallbackHistoryWindow: 100, +}; + +const RAD = Math.PI / 180; +const DEG = 180 / Math.PI; +const EARTH_RADIUS_METERS = 6371000; + +function isNum(v: number): boolean { + return typeof v === "number" && Number.isFinite(v); +} + +function hasPosition(o: ReorientationImage): boolean { + return !!o && isNum(o.lat) && isNum(o.lng); +} + +/** Smallest absolute angular difference between two bearings, in degrees. */ +export function angleDelta(a: number, b: number): number { + const d = Math.abs(a - b) % 360; + return d > 180 ? 360 - d : d; +} + +/** Wrap a basic-x delta into the signed half-open range (-0.5, 0.5]. */ +export function wrapDelta(d: number): number { + return ((d % 1) + 1.5) % 1 - 0.5; +} + +export function haversineDist( + lat1: number, lon1: number, lat2: number, lon2: number): number { + const dLat = (lat2 - lat1) * RAD; + const dLon = (lon2 - lon1) * RAD; + const a = Math.sin(dLat / 2) ** 2 + + Math.cos(lat1 * RAD) * Math.cos(lat2 * RAD) * Math.sin(dLon / 2) ** 2; + return EARTH_RADIUS_METERS * 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); +} + +/** Great-circle bearing, degrees clockwise from north (compass convention). */ +export function bearing( + lat1: number, lon1: number, lat2: number, lon2: number): number { + const p1 = lat1 * RAD; + const p2 = lat2 * RAD; + const dl = (lon2 - lon1) * RAD; + const y = Math.sin(dl) * Math.cos(p2); + const x = Math.cos(p1) * Math.sin(p2) - + Math.sin(p1) * Math.cos(p2) * Math.cos(dl); + return (Math.atan2(y, x) * DEG + 360) % 360; +} + +/** + * Basic horizontal coordinate (0..1) that frames the given travel bearing. + * 0.5 is the image's compass-forward; offset by (travel - cca) / 360. + */ +export function bearingToBasicX(travelBrg: number, cca: number): number { + return (((travelBrg - cca) / 360) + 0.5 + 1) % 1; +} + +interface PrevContext { + valid: boolean; + moving: boolean; + speed: number; + travel: number; +} + +/** + * Resolves, for an image in a sequence, the basic-x that frames the + * direction of travel toward the next image. Motion is detected from GPS + * speed with a low-speed-turn allowance, GPS outliers are rejected against + * recent bearings, and stationary images fall back to the last confirmed + * moving bearing so the view is preserved rather than spun by noise. + * + * Pure with respect to the injected {@link ReorientationProvider}; holds an + * internal cache so prefetched results are reused and prior-image context is + * available without refetching. + */ +export class ReorientationEngine { + private _provider: ReorientationProvider; + private _config: Required; + private _cache: Map; + private _pending: Map>; + + constructor( + provider: ReorientationProvider, + config?: ReorientationConfiguration) { + this._provider = provider; + this._config = { + ...DEFAULT_REORIENTATION_CONFIGURATION, + ...(config || {}), + }; + this._cache = new Map(); + this._pending = new Map>(); + } + + public get(id: string): ReorientationResult | null { + return this._cache.get(String(id)) || null; + } + + public precompute( + imgId: string, + seed?: ReorientationImage, + depth?: number): Promise { + const cfg = this._config; + imgId = String(imgId); + if (depth == null) { + depth = cfg.prefetchAhead; + } + if (this._cache.has(imgId)) { + this._prefetchNext(this._cache.get(imgId), depth); + return Promise.resolve(); + } + if (this._pending.has(imgId)) { + return this._pending.get(imgId).then(() => { + this._prefetchNext(this._cache.get(imgId), depth); + }); + } + + let p: Promise; + if (seed && isNum(seed.lat) && isNum(seed.lng)) { + if (typeof this._provider.cacheImage === "function") { + this._provider.cacheImage(seed); + } + p = Promise.resolve(seed); + } else { + p = this._provider.fetchImage(imgId); + } + + // Share in-flight work so callers wait until the cache is populated. + const work = p.then((cur) => this._resolve(imgId, cur, depth)) + .catch((e: Error) => { this._invalid(imgId, e && e.message); }) + .then(() => { this._pending.delete(imgId); }); + this._pending.set(imgId, work); + return work; + } + + private _resolve( + imgId: string, + cur: ReorientationImage, + depth: number): Promise { + const reason = this._currentInvalidReason(cur); + if (reason) { + this._invalid(imgId, reason); + return Promise.resolve(); + } + return this._provider + .fetchSeqIds(cur.seq) + .then((ids) => { + const idx = ids.indexOf(String(imgId)); + if (idx < 0) { + this._invalid(imgId, "Image not in sequence"); + return; + } + if (idx >= ids.length - 1) { + this._invalid(imgId, "End of sequence"); + return; + } + const nextId = ids[idx + 1]; + return this._provider.fetchImage(nextId).then((nxt) => { + if (!hasPosition(nxt)) { + this._invalid(imgId, "Next image missing geometry"); + return; + } + return this._decide(imgId, cur, nxt, ids, idx, nextId, depth); + }); + }); + } + + private _decide( + imgId: string, + cur: ReorientationImage, + nxt: ReorientationImage, + ids: string[], + idx: number, + nextId: string, + depth: number): Promise { + const cfg = this._config; + const dist = haversineDist(cur.lat, cur.lng, nxt.lat, nxt.lng); + let tb = bearing(cur.lat, cur.lng, nxt.lat, nxt.lng); + const dt = (nxt.ts && cur.ts) ? (nxt.ts - cur.ts) / 1000 : 0; + const speed = dt > 0 ? dist / dt : 0; + let moving = false; + + let prevCtx: PrevContext = null; + for (let bi = idx - 1; + bi >= 0 && bi >= idx - cfg.previousContextWindow; bi--) { + const pd = this._cache.get(ids[bi]); + if (pd && pd.valid) { + prevCtx = { + valid: true, + moving: pd.moving, + speed: pd.speed, + travel: pd.travel, + }; + break; + } + } + + let prevPromise: Promise = Promise.resolve(prevCtx); + if (!prevCtx && idx > 0) { + prevPromise = this._provider.fetchImage(ids[idx - 1]) + .then((prev) => { + if (!hasPosition(prev)) { + return null; + } + const pDist = + haversineDist(prev.lat, prev.lng, cur.lat, cur.lng); + const pTravel = + bearing(prev.lat, prev.lng, cur.lat, cur.lng); + const pDt = (cur.ts && prev.ts) ? + (cur.ts - prev.ts) / 1000 : 0; + const pSpeed = pDt > 0 ? pDist / pDt : 0; + let pMoving = pSpeed >= cfg.movingSpeedMps; + if (!pMoving && isNum(prev.cca)) { + const pDelta = angleDelta(pTravel, prev.cca); + if (pDelta < cfg.lowSpeedTurnMaxDeltaDeg && + pDist > cfg.lowSpeedTurnDistanceM) { + pMoving = true; + } + } + return { + valid: true, + moving: pMoving, + speed: pSpeed, + travel: pTravel, + }; + }) + .catch(() => null); + } + + return prevPromise.then((prev) => { + if (speed >= cfg.movingSpeedMps) { + // On direct jumps there is no prior segment, so only the + // first image in a sequence may assume sustained motion. + moving = prev ? + (prev.moving || prev.speed >= cfg.movingSpeedMps) : + (idx === 0); + } else { + // Low speed — accept only as a genuine turn, not GPS noise, + // when the compass angle aligns with the travel bearing. + const ccaDelta = angleDelta(tb, cur.cca); + if (ccaDelta < cfg.lowSpeedTurnMaxDeltaDeg && + dist > cfg.lowSpeedTurnDistanceM) { + moving = true; + } + } + + if (moving) { + if (prev && prev.moving) { + if (angleDelta(tb, prev.travel) > cfg.outlierMaxDeltaDeg) { + moving = false; + } + } else { + for (let bi = idx - 1; + bi >= 0 && bi >= idx - cfg.movingHistoryWindow; bi--) { + const pd = this._cache.get(ids[bi]); + if (pd && pd.valid && pd.moving) { + if (angleDelta(tb, pd.travel) > + cfg.outlierMaxDeltaDeg) { + moving = false; + } + break; + } + } + } + } + + // If not moving, hold the last confirmed moving bearing so the + // view is preserved instead of spun by stationary GPS jitter. + if (!moving) { + let hasFallback = false; + for (let bi = idx - 1; + bi >= 0 && bi >= idx - cfg.fallbackHistoryWindow; bi--) { + const pd = this._cache.get(ids[bi]); + if (pd && pd.valid && pd.moving) { + tb = pd.travel; + hasFallback = true; + break; + } + } + if (!hasFallback && prev && prev.moving) { + tb = prev.travel; + hasFallback = true; + } + if (!hasFallback && idx === 0) { + moving = true; + } + } + + const result: ReorientationResult = { + valid: true, + nextId, + travel: tb, + basicX: bearingToBasicX(tb, cur.cca), + dist, + cca: cur.cca, + speed, + moving, + seq: cur.seq, + }; + this._cache.set(imgId, result); + this._prefetchNext(result, depth); + }); + } + + private _prefetchNext(d: ReorientationResult, depth: number): void { + if (depth > 0 && d && d.valid && d.nextId) { + this.precompute(d.nextId, null, depth - 1) + .catch(() => { /* ignore prefetch errors */ }); + } + } + + private _invalid(imgId: string, reason: string): void { + this._cache.set(String(imgId), { valid: false, reason }); + } + + private _currentInvalidReason(cur: ReorientationImage): string | null { + if (!cur) { + return "Missing image metadata"; + } + if (cur.cam !== "spherical" && cur.cam !== "equirectangular") { + return "Camera: " + (cur.cam || "unknown"); + } + if (!hasPosition(cur)) { + return "Missing geometry"; + } + if (!isNum(cur.cca)) { + return "Missing compass angle"; + } + if (!cur.seq) { + return "No sequence"; + } + return null; + } +} diff --git a/src/external/component.ts b/src/external/component.ts index 8323cff1a..2f67b7b43 100644 --- a/src/external/component.ts +++ b/src/external/component.ts @@ -76,6 +76,12 @@ export { PopupOptions } from "../component/popup/interfaces/PopupOptions"; export { Popup } from "../component/popup/popup/Popup"; export { PopupComponent } from "../component/popup/PopupComponent"; +// Reorientation +export { ReorientationComponent } + from "../component/reorientation/ReorientationComponent"; +export { ReorientationConfiguration } + from "../component/interfaces/ReorientationConfiguration"; + // Sequence export { SequenceConfiguration } from "../component/interfaces/SequenceConfiguration"; diff --git a/src/mapillary.ts b/src/mapillary.ts index c92eb31d1..e0150dde5 100644 --- a/src/mapillary.ts +++ b/src/mapillary.ts @@ -38,6 +38,8 @@ import { KeyboardComponent } from "./component/keyboard/KeyboardComponent"; import { MarkerComponent } from "./component/marker/MarkerComponent"; import { PointerComponent } from "./component/pointer/PointerComponent"; import { PopupComponent } from "./component/popup/PopupComponent"; +import { ReorientationComponent } + from "./component/reorientation/ReorientationComponent"; import { SequenceComponent } from "./component/sequence/SequenceComponent"; import { SpatialComponent } from "./component/spatial/SpatialComponent"; @@ -53,6 +55,7 @@ ComponentService.register(KeyboardComponent); ComponentService.register(MarkerComponent); ComponentService.register(PointerComponent); ComponentService.register(PopupComponent); +ComponentService.register(ReorientationComponent); ComponentService.register(SequenceComponent); ComponentService.register(SliderComponent); ComponentService.register(SpatialComponent); diff --git a/src/state/StateContext.ts b/src/state/StateContext.ts index d1a375a1d..f8e18aa06 100644 --- a/src/state/StateContext.ts +++ b/src/state/StateContext.ts @@ -28,6 +28,7 @@ export class StateContext implements IStateContext { currentIndex: -1, geometry, reference: { alt: 0, lat: 0, lng: 0 }, + reorientations: new Map(), trajectory: [], transitionMode: transitionMode == null ? TransitionMode.Default : transitionMode, zoom: 0, @@ -202,6 +203,18 @@ export class StateContext implements IStateContext { this._state.rotateToBasic(basic); } + public rotateToBasicSmooth(basic: number[]): void { + this._state.rotateToBasicSmooth(basic); + } + + public setReorientation(imageId: string, basic: number[]): void { + this._state.setReorientation(imageId, basic); + } + + public clearReorientations(): void { + this._state.clearReorientations(); + } + public move(delta: number): void { this._state.move(delta); } diff --git a/src/state/StateService.ts b/src/state/StateService.ts index 3b671ee54..d39b1ffd4 100644 --- a/src/state/StateService.ts +++ b/src/state/StateService.ts @@ -465,6 +465,19 @@ export class StateService { this._invokeContextOperation((context: IStateContext) => { context.rotateToBasic(basic); }); } + public rotateToBasicSmooth(basic: number[]): void { + this._inMotionOperation$.next(true); + this._invokeContextOperation((context: IStateContext) => { context.rotateToBasicSmooth(basic); }); + } + + public setReorientation(imageId: string, basic: number[]): void { + this._invokeContextOperation((context: IStateContext) => { context.setReorientation(imageId, basic); }); + } + + public clearReorientations(): void { + this._invokeContextOperation((context: IStateContext) => { context.clearReorientations(); }); + } + public move(delta: number): void { this._inMotionOperation$.next(true); this._invokeContextOperation((context: IStateContext) => { context.move(delta); }); diff --git a/src/state/interfaces/IStateBase.ts b/src/state/interfaces/IStateBase.ts index 14e9a83e5..7429790e2 100644 --- a/src/state/interfaces/IStateBase.ts +++ b/src/state/interfaces/IStateBase.ts @@ -10,6 +10,11 @@ export interface IStateBase { currentIndex: number; geometry: IGeometryProvider, reference: LngLatAlt; + // Per-image desired basic center [x, y], applied to an image as it becomes + // current on a motionless (instant) transition so it renders already + // oriented instead of flashing the carried view for one frame. Shared by + // reference across state transitions. + reorientations?: Map; trajectory: Image[]; transitionMode: TransitionMode; zoom: number; diff --git a/src/state/interfaces/IStateContext.ts b/src/state/interfaces/IStateContext.ts index 23b29803d..702b3b0c3 100644 --- a/src/state/interfaces/IStateContext.ts +++ b/src/state/interfaces/IStateContext.ts @@ -29,6 +29,9 @@ export interface IStateContext extends IAnimationState { rotateBasicUnbounded(basicRotation: number[]): void; rotateBasicWithoutInertia(basicRotation: number[]): void; rotateToBasic(basic: number[]): void; + rotateToBasicSmooth(basic: number[]): void; + setReorientation(imageId: string, basic: number[]): void; + clearReorientations(): void; move(delta: number): void; moveTo(position: number): void; zoomIn(delta: number, reference: number[]): void; diff --git a/src/state/state/InteractiveStateBase.ts b/src/state/state/InteractiveStateBase.ts index f2d70b7c4..ccf9ae34a 100644 --- a/src/state/state/InteractiveStateBase.ts +++ b/src/state/state/InteractiveStateBase.ts @@ -220,6 +220,24 @@ export abstract class InteractiveStateBase extends StateBase { this._currentCamera.lookat.fromArray(lookat); } + public rotateToBasicSmooth(basic: number[]): void { + if (this._currentImage == null) { + return; + } + + this._desiredZoom = this._zoom; + + basic[0] = this._spatial.clamp(basic[0], 0, 1); + basic[1] = this._spatial.clamp(basic[1], 0, 1); + + // Eased sibling of rotateToBasic: set _desiredLookat instead of + // snapping the current camera, so _updateLookat lerps toward the + // target each frame. Stays smooth even when applied after an image + // transition has already settled. + this._desiredLookat = new THREE.Vector3() + .fromArray(this.currentTransform.unprojectBasic(basic, this._lookatDepth)); + } + public zoomIn(delta: number, reference: number[]): void { if (this._currentImage == null) { return; diff --git a/src/state/state/StateBase.ts b/src/state/state/StateBase.ts index 4350378e8..dd4b4c14b 100644 --- a/src/state/state/StateBase.ts +++ b/src/state/state/StateBase.ts @@ -37,6 +37,8 @@ export abstract class StateBase implements IStateBase { protected _motionless: boolean; + protected _reorientations: Map; + private _referenceThreshold: number; private _referenceCellIds: Set; private _transitionThreshold: number; @@ -50,6 +52,10 @@ export abstract class StateBase implements IStateBase { this._transitionThreshold = 62.5; this._transitionMode = state.transitionMode; + // Shared by reference across transitions so a reorientation registered + // before navigating is available when the target image becomes current. + this._reorientations = state.reorientations || new Map(); + this._reference = state.reference; this._referenceCellIds = new Set( connectedComponent( @@ -165,6 +171,18 @@ export abstract class StateBase implements IStateBase { return this._transitionMode; } + public get reorientations(): Map { + return this._reorientations; + } + + public setReorientation(imageId: string, basic: number[]): void { + this._reorientations.set(imageId, basic); + } + + public clearReorientations(): void { + this._reorientations.clear(); + } + public move(delta: number): void { /*noop*/ } public moveTo(position: number): void { /*noop*/ } @@ -183,6 +201,8 @@ export abstract class StateBase implements IStateBase { public rotateToBasic(basic: number[]): void { /*noop*/ } + public rotateToBasicSmooth(basic: number[]): void { /*noop*/ } + public setSpeed(speed: number): void { /*noop*/ } public zoomIn(delta: number, reference: number[]): void { /*noop*/ } diff --git a/src/state/state/TraversingState.ts b/src/state/state/TraversingState.ts index 0c749fce2..6ef5a19a4 100644 --- a/src/state/state/TraversingState.ts +++ b/src/state/state/TraversingState.ts @@ -92,6 +92,11 @@ export class TraversingState extends InteractiveStateBase { this._zoom : 0; this._desiredLookat = null; + + // Orient the new image before it is rendered this frame so a + // motionless (instant) transition lands already facing the + // registered direction instead of flashing the carried view. + this._applyReorientation(); } let animationSpeed: number = this._animationSpeed * delta / 1e-1 * 6; @@ -151,4 +156,29 @@ export class TraversingState extends InteractiveStateBase { this._motionless = this._motionlessTransition(); } + + private _applyReorientation(): void { + // Only for instant cuts: a smooth transition should ease into the new + // direction, not start already there. + if (!this._motionless || this._currentImage == null) { + return; + } + // Only pre-orient (snap) mesh-less images. An image with SfM mesh eases + // to the travel direction, and easing must start from the carried view — + // pre-snapping it here would lose the ease. + if (this._currentImage.mesh != null && + this._currentImage.mesh.vertices.length > 0) { + return; + } + const basic = this._reorientations.get(this._currentImage.id); + if (basic == null || !isSpherical(this._currentImage.cameraType)) { + return; + } + this._currentCamera.lookat.fromArray( + this.currentTransform.unprojectBasic(basic, this._lookatDepth)); + const previousTransform = this.previousTransform != null ? + this.previousTransform : this.currentTransform; + this._previousCamera.lookat.fromArray( + previousTransform.unprojectBasic(basic, this._lookatDepth)); + } } diff --git a/src/viewer/ComponentController.ts b/src/viewer/ComponentController.ts index ce1c4fbf0..bf6f8b552 100644 --- a/src/viewer/ComponentController.ts +++ b/src/viewer/ComponentController.ts @@ -135,6 +135,7 @@ export class ComponentController { this._uTrue(options.image, "image"); this._uTrue(options.keyboard, "keyboard"); this._uTrue(options.pointer, "pointer"); + this._uTrue(options.reorientation, "reorientation"); this._uTrue(options.sequence, "sequence"); this._uTrue(options.zoom, "zoom"); } diff --git a/src/viewer/options/ComponentOptions.ts b/src/viewer/options/ComponentOptions.ts index 066230534..e53146121 100644 --- a/src/viewer/options/ComponentOptions.ts +++ b/src/viewer/options/ComponentOptions.ts @@ -4,6 +4,7 @@ import { DirectionConfiguration } from "../../component/interfaces/DirectionConf import { KeyboardConfiguration } from "../../component/interfaces/KeyboardConfiguration"; import { MarkerConfiguration } from "../../component/interfaces/MarkerConfiguration"; import { PointerConfiguration } from "../../component/interfaces/PointerConfiguration"; +import { ReorientationConfiguration } from "../../component/interfaces/ReorientationConfiguration"; import { SequenceConfiguration } from "../../component/interfaces/SequenceConfiguration"; import { SliderConfiguration } from "../../component/interfaces/SliderConfiguration"; import { SpatialConfiguration } from "../../component/interfaces/SpatialConfiguration"; @@ -108,6 +109,16 @@ export interface ComponentOptions { */ popup?: boolean; + /** + * Reorient spherical images to face the direction of travel during + * navigation. + * + * @description Requires WebGL support. + * + * @default true + */ + reorientation?: boolean | ReorientationConfiguration; + /** * Show sequence related navigation. * diff --git a/test/component/reorientation/ReorientationEngine.test.ts b/test/component/reorientation/ReorientationEngine.test.ts new file mode 100644 index 000000000..0b0fd4901 --- /dev/null +++ b/test/component/reorientation/ReorientationEngine.test.ts @@ -0,0 +1,158 @@ +import { + angleDelta, + bearing, + bearingToBasicX, + haversineDist, + ReorientationEngine, + ReorientationImage, + ReorientationProvider, + wrapDelta, +} from "../../../src/component/reorientation/ReorientationEngine"; + +describe("ReorientationEngine.math", () => { + it("computes an eastward bearing as ~90 degrees", () => { + expect(bearing(0, 0, 0, 0.001)).toBeCloseTo(90, 1); + }); + + it("computes a northward bearing as ~0 degrees", () => { + expect(bearing(0, 0, 0.001, 0)).toBeCloseTo(0, 1); + }); + + it("wraps angle deltas to the shortest arc", () => { + expect(angleDelta(10, 350)).toBeCloseTo(20, 6); + expect(angleDelta(0, 180)).toBeCloseTo(180, 6); + }); + + it("frames compass-forward at basic-x 0.5", () => { + expect(bearingToBasicX(90, 90)).toBeCloseTo(0.5, 6); + }); + + it("offsets basic-x by the travel/compass difference", () => { + // Travelling east (90) while the image faces north (0): the travel + // direction sits a quarter-turn clockwise -> basic-x 0.75. + expect(bearingToBasicX(90, 0)).toBeCloseTo(0.75, 6); + }); + + it("wraps basic-x deltas into the signed half-open range", () => { + expect(wrapDelta(0.1)).toBeCloseTo(0.1, 6); + expect(wrapDelta(0.9)).toBeCloseTo(-0.1, 6); + expect(wrapDelta(-0.9)).toBeCloseTo(0.1, 6); + }); + + it("measures distance between nearby points in meters", () => { + expect(haversineDist(0, 0, 0, 0.0001)).toBeGreaterThan(10); + expect(haversineDist(0, 0, 0, 0.0001)).toBeLessThan(13); + }); +}); + +type Fixture = { [id: string]: ReorientationImage }; + +function provider(images: Fixture, seqIds: string[]): ReorientationProvider { + return { + fetchImage: (id: string): Promise => { + const image = images[String(id)]; + return image ? + Promise.resolve(image) : + Promise.reject(new Error("missing " + id)); + }, + fetchSeqIds: (_seqId: string): Promise => + Promise.resolve(seqIds.slice()), + }; +} + +function spherical( + id: string, + lat: number, + lng: number, + cca: number, + ts: number): ReorientationImage { + return { id, lat, lng, cca, cam: "spherical", seq: "s", ts }; +} + +describe("ReorientationEngine.precompute", () => { + it("frames the travel direction for a moving image", async () => { + // Straight eastward run, ~11 m apart, 1 s apart -> ~11 m/s, facing + // north (cca 0) so travel east maps to basic-x 0.75. (capturedAt is an + // epoch in ms, so use nonzero timestamps.) + const images: Fixture = { + a: spherical("a", 0, 0.0000, 0, 1000), + b: spherical("b", 0, 0.0001, 0, 2000), + c: spherical("c", 0, 0.0002, 0, 3000), + }; + const engine = new ReorientationEngine(provider(images, ["a", "b", "c"])); + + await engine.precompute("b"); + const result = engine.get("b"); + + expect(result.valid).toBe(true); + expect(result.moving).toBe(true); + expect(result.nextId).toBe("c"); + expect(result.basicX).toBeCloseTo(0.75, 2); + }); + + it("accepts a low-speed step as a turn when compass agrees", async () => { + // ~11 m apart but 100 s apart -> ~0.11 m/s (below movingSpeedMps), + // yet compass (90) agrees with eastward travel, so it counts. + const images: Fixture = { + a: spherical("a", 0, 0.0000, 90, 1000), + b: spherical("b", 0, 0.0001, 90, 101000), + }; + const engine = new ReorientationEngine(provider(images, ["a", "b"])); + + await engine.precompute("a"); + const result = engine.get("a"); + + expect(result.valid).toBe(true); + expect(result.moving).toBe(true); + }); + + it("invalidates the last image in a sequence", async () => { + const images: Fixture = { + a: spherical("a", 0, 0.0000, 0, 0), + b: spherical("b", 0, 0.0001, 0, 1000), + }; + const engine = new ReorientationEngine(provider(images, ["a", "b"])); + + await engine.precompute("b"); + const result = engine.get("b"); + + expect(result.valid).toBe(false); + expect(result.reason).toBe("End of sequence"); + }); + + it("skips non-spherical images", async () => { + const images: Fixture = { + a: { id: "a", lat: 0, lng: 0, cca: 0, cam: "perspective", seq: "s", ts: 0 }, + b: spherical("b", 0, 0.0001, 0, 1000), + }; + const engine = new ReorientationEngine(provider(images, ["a", "b"])); + + await engine.precompute("a"); + const result = engine.get("a"); + + expect(result.valid).toBe(false); + expect(result.reason).toContain("Camera"); + }); + + it("reuses the cache instead of recomputing", async () => { + let fetches = 0; + const base = provider( + { + a: spherical("a", 0, 0.0000, 0, 0), + b: spherical("b", 0, 0.0001, 0, 1000), + c: spherical("c", 0, 0.0002, 0, 2000), + }, + ["a", "b", "c"]); + const counting: ReorientationProvider = { + fetchImage: (id: string) => { fetches++; return base.fetchImage(id); }, + fetchSeqIds: base.fetchSeqIds, + }; + const engine = new ReorientationEngine(counting); + + await engine.precompute("b"); + const before = fetches; + await engine.precompute("b"); + + expect(fetches).toBe(before); + }); +}); diff --git a/test/viewer/ComponentController.test.ts b/test/viewer/ComponentController.test.ts index 046ef174d..5cfc3d19f 100644 --- a/test/viewer/ComponentController.test.ts +++ b/test/viewer/ComponentController.test.ts @@ -59,6 +59,7 @@ class KC extends CMock { protected static _cn: ComponentName = "keyboard"; } class MaC extends CMock { protected static _cn: ComponentName = "marker"; } class PtrC extends CMock { protected static _cn: ComponentName = "pointer"; } class PC extends CMock { protected static _cn: ComponentName = "popup"; } +class ReC extends CMock { protected static _cn: ComponentName = "reorientation"; } class SeC extends CMock { protected static _cn: ComponentName = "sequence"; } class SlC extends CMock { protected static _cn: ComponentName = "slider"; } class SpC extends CMock { protected static _cn: ComponentName = "spatial"; } @@ -74,6 +75,7 @@ ComponentService.register(KC); ComponentService.register(MaC); ComponentService.register(PtrC); ComponentService.register(PC); +ComponentService.register(ReC); ComponentService.register(SeC); ComponentService.register(SlC); ComponentService.register(SpC);