From f76cb16108c4e6a2f16cd2b733db5d3fff005048 Mon Sep 17 00:00:00 2001 From: Oscar Lorentzon Date: Mon, 3 May 2021 06:16:41 -0700 Subject: [PATCH] feat: upgrade opensfm viewer to mapillary-js v4.0.0-beta.4 (#740) Summary: Pull Request resolved: https://github.com/mapillary/OpenSfM/pull/740 ## Contributions - Upgrade viewer to MapillaryJS v4.0.0-beta.4 - Fix loading using reconstruction path - Invent image buffer on client if server returns error (e.g. when no images exist for a dataset). This means the viewer will always work, even when images are missing. Errors will be logged to the browser. - Increased cell grid depth for more fine grained load - Do not hide bearing indicator in non street camera control mode - Add ENU axes visualization - Add three.js orbit controls as default camera control option using MJS custom camera control API - Earth surface hover indicator for simplified earth navigation - Add image and point count stats control - Improved spatial viz perf in MJS. Logarithmic ray tracing/camera frame intersection. Improved linear rendering logic. Should work with ~10,000 camera frames without considerable lag. Reviewed By: paulinus Differential Revision: D28031955 fbshipit-source-id: 1e44c9c5ac83df0115b9a811f118c4560007917e --- .gitignore | 2 +- viewer/node_modules.sh | 22 ++- viewer/src/control/StatsControl.js | 92 +++++++++++ viewer/src/controller/DatController.js | 27 ++-- viewer/src/controller/KeyController.js | 33 ++-- viewer/src/controller/OptionController.js | 12 +- viewer/src/opensfm.js | 1 + viewer/src/provider/DataConverter.js | 10 +- viewer/src/provider/OpensfmDataProvider.js | 56 ++++--- viewer/src/renderer/AxesRenderer.js | 146 +++++++++++++++++ viewer/src/renderer/CustomRenderer.js | 93 +++++++++++ viewer/src/renderer/EarthRenderer.js | 174 +++++++++++++++++++++ viewer/src/ui/FileLoader.js | 2 +- viewer/src/ui/OpensfmViewer.js | 124 +++++++++++---- viewer/src/ui/OrbitCameraControls.js | 139 ++++++++++++++++ viewer/src/ui/modes.js | 24 +++ viewer/src/util/coords.js | 11 ++ viewer/styles/opensfm.css | 10 ++ 18 files changed, 888 insertions(+), 90 deletions(-) create mode 100644 viewer/src/control/StatsControl.js create mode 100644 viewer/src/renderer/AxesRenderer.js create mode 100644 viewer/src/renderer/CustomRenderer.js create mode 100644 viewer/src/renderer/EarthRenderer.js create mode 100644 viewer/src/ui/OrbitCameraControls.js create mode 100644 viewer/src/ui/modes.js create mode 100644 viewer/src/util/coords.js diff --git a/.gitignore b/.gitignore index 1afcf49b1..a739a68fd 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ launch.json eval # Viewer -viewer/node_modules +node_modules # Ignore the data folder, but not the berlin example. data/* diff --git a/viewer/node_modules.sh b/viewer/node_modules.sh index e9a4d4bec..7774415a4 100755 --- a/viewer/node_modules.sh +++ b/viewer/node_modules.sh @@ -39,7 +39,7 @@ curl -fS "$UNPKG/$GLM_V/$PKG" \ # mapillary-js MJS_PKG="mapillary-js" -MJS_V="$MJS_PKG@4.0.0-beta.2" +MJS_V="$MJS_PKG@4.0.0-beta.4" MJS_DIST="dist" MJS_JS="mapillary.module.js" MJS_CSS="mapillary.css" @@ -52,3 +52,23 @@ curl -fS --create-dirs "$UNPKG/$MJS_IN/$MJS_CSS" \ -o "$MJS_OUT/$MJS_CSS" || exit 1 curl -fS "$UNPKG/$MJS_V/$PKG" \ -o "$NODE_MODULES/$MJS_PKG/$PKG" || exit 1 + +# three +THR_PKG="three" +THR_V="$THR_PKG@0.125.2" +THR_DIST="build" +THR_JS="three.module.js" + +THR_IN="$THR_V/$THR_DIST" +THR_OUT="$NODE_MODULES/$THR_PKG/$THR_DIST" +curl -fS --create-dirs "$UNPKG/$THR_IN/$THR_JS" \ + -o "$THR_OUT/$THR_JS" || exit 1 +curl -fS "$UNPKG/$THR_V/$PKG" \ + -o "$NODE_MODULES/$THR_PKG/$PKG" || exit 1 + +THR_EXM="examples/jsm" +THR_EXM_JS="controls/OrbitControls.js" +THR_EXM_IN="$THR_V/$THR_EXM" +THR_EXM_OUT="$NODE_MODULES/$THR_PKG/$THR_EXM" +curl -fS --create-dirs "$UNPKG/$THR_EXM_IN/$THR_EXM_JS" \ + -o "$THR_EXM_OUT/$THR_EXM_JS" || exit 1 diff --git a/viewer/src/control/StatsControl.js b/viewer/src/control/StatsControl.js new file mode 100644 index 000000000..ac56fb9ab --- /dev/null +++ b/viewer/src/control/StatsControl.js @@ -0,0 +1,92 @@ +/** + * @format + */ + +export class StatsControl { + constructor(options) { + const container = this._createContainer(); + this._container = container; + this._provider = options.provider; + this._shotCount = 0; + this._pointCount = 0; + this._total = this._createText( + this._makeContent('Total', this._shotCount, this._pointCount), + ); + this._container.appendChild(this._total); + this._visible = false; + if (options.visible) { + this.show(); + } + } + + get container() { + return this._container; + } + + addRawData(rawData) { + for (const data of Object.values(rawData)) { + const id = data.id; + const url = data.url; + const cluster = data.cluster; + const shotCount = Object.keys(cluster.shots).length; + const pointCount = Object.keys(cluster.points).length; + const content = this._makeContent(id, shotCount, pointCount, url); + this.addStatRow(content); + + this._shotCount += shotCount; + this._pointCount += pointCount; + } + + this._total.textContent = this._makeContent( + 'Total', + this._shotCount, + this._pointCount, + ); + } + + addStatRow(content) { + const stat = this._createText(content); + stat.classList.add('opensfm-info-text-stat'); + this._container.appendChild(stat); + } + + hide() { + if (!this._visible) { + return; + } + this._container.classList.add('opensfm-hidden'); + this._visible = false; + } + + show() { + if (this._visible) { + return; + } + this._container.classList.remove('opensfm-hidden'); + this._visible = true; + } + + _createContainer() { + const header = document.createElement('span'); + header.classList.add('opensfm-info-text', 'opensfm-info-text-header'); + header.textContent = 'Stats'; + + const container = document.createElement('div'); + container.classList.add('opensfm-control-container', 'opensfm-hidden'); + container.appendChild(header); + return container; + } + + _createText(content) { + const document = window.document; + const element = document.createElement('span'); + element.classList.add('opensfm-info-text'); + element.textContent = content; + return element; + } + + _makeContent(prefix, shotCount, pointCount, suffix) { + const append = suffix ? ` (${suffix})` : ''; + return `${prefix}: ${shotCount} images, ${pointCount} points${append}`; + } +} diff --git a/viewer/src/controller/DatController.js b/viewer/src/controller/DatController.js index fb20f52b9..3722480b9 100644 --- a/viewer/src/controller/DatController.js +++ b/viewer/src/controller/DatController.js @@ -4,11 +4,11 @@ import {GUI} from '../../node_modules/dat.gui/build/dat.gui.module.js'; import { - CameraControls, CameraVisualizationMode, OriginalPositionMode, } from '../../node_modules/mapillary-js/dist/mapillary.module.js'; import {ListController} from './ListController.js'; +import {CameraControlMode} from '../ui/modes.js'; export const FolderName = Object.freeze({ INFO: 'info', @@ -58,13 +58,13 @@ export class DatController { .onChange(v => this._onChange(name, v)); } - _addCameraControlsOption(folder) { - const cc = CameraControls; - const ccs = [cc[cc.Earth], cc[cc.Street]]; + _addCameraControlOption(folder) { + const ccm = CameraControlMode; + const ccms = [ccm.ORBIT, ccm.STREET, ccm.EARTH]; folder - .add(this._config, 'cameraControls', ccs) + .add(this._config, 'cameraControlMode', ccms) .listen() - .onChange(c => this._onChange('cameraControls', cc[c])); + .onChange(m => this._onChange('cameraControlMode', m)); } _addCameraVizualizationOption(folder) { @@ -103,6 +103,7 @@ export class DatController { folder.open(); this._addBooleanOption('commandsVisible', folder); this._addBooleanOption('thumbnailVisible', folder); + this._addBooleanOption('statsVisible', folder); this._addNumericOption('infoSize', folder); return folder; } @@ -116,7 +117,7 @@ export class DatController { _createReconstructionsController(gui) { const emitter = this._emitter; const eventType = this._eventTypes.reconstructionsSelected; - const folder = gui.addFolder('Reconstructions'); + const folder = gui.addFolder('Clusters'); folder.open(); const controller = new ListController({emitter, eventType, folder}); return controller; @@ -130,9 +131,10 @@ export class DatController { this._addCameraVizualizationOption(folder); this._addNumericOption('cameraSize', folder); this._addPositionVisualizationOption(folder); - this._addCameraControlsOption(folder); - this._addBooleanOption('tilesVisible', folder); + this._addCameraControlOption(folder); + this._addBooleanOption('cellsVisible', folder); this._addBooleanOption('imagesVisible', folder); + this._addBooleanOption('axesVisible', folder); return folder; } @@ -142,17 +144,18 @@ export class DatController { let mode = null; let type = null; switch (name) { - case 'camerasVisible': + case 'axesVisible': + case 'cellsVisible': case 'commandsVisible': case 'imagesVisible': case 'pointsVisible': + case 'statsVisible': case 'thumbnailVisible': - case 'tilesVisible': const visible = value; type = types[name]; emitter.fire(type, {type, visible}); break; - case 'cameraControls': + case 'cameraControlMode': case 'originalPositionMode': case 'cameraVisualizationMode': mode = value; diff --git a/viewer/src/controller/KeyController.js b/viewer/src/controller/KeyController.js index b55ed94fa..652651b7b 100644 --- a/viewer/src/controller/KeyController.js +++ b/viewer/src/controller/KeyController.js @@ -3,10 +3,10 @@ */ import { - CameraControls, CameraVisualizationMode, OriginalPositionMode, } from '../../node_modules/mapillary-js/dist/mapillary.module.js'; +import {CameraControlMode} from '../ui/modes.js'; export class KeyController { constructor(options) { @@ -18,17 +18,19 @@ export class KeyController { const increase = 1.1; this._commands = { // visibility + c: {value: 'axesVisible'}, + d: {value: 'cellsVisible'}, e: {value: 'commandsVisible'}, - f: {value: 'pointsVisible'}, - d: {value: 'tilesVisible'}, r: {value: 'imagesVisible'}, + f: {value: 'pointsVisible'}, + t: {value: 'statsVisible'}, v: {value: 'thumbnailVisible'}, // activity l: {value: 'datToggle'}, // mode '1': {value: 'cameraVisualizationMode'}, '2': {value: 'originalPositionMode'}, - '3': {value: 'cameraControls'}, + '3': {value: 'cameraControlMode'}, // size q: {value: 'pointSize', coeff: decrease}, w: {value: 'pointSize', coeff: increase}, @@ -79,6 +81,7 @@ export class KeyController { case 'e': case 'f': case 'r': + case 't': case 'v': const visible = this._toggle(command.value); emitter.fire(type, {type, visible}); @@ -96,8 +99,8 @@ export class KeyController { emitter.fire(type, {type, mode: opm}); break; case '3': - const cc = this._rotateCc(); - emitter.fire(type, {type, mode: cc}); + const ccm = this._rotateCcm(); + emitter.fire(type, {type, mode: ccm}); break; case 'a': case 'q': @@ -125,19 +128,21 @@ export class KeyController { return key in this._commands || key in this._customCommands; } - _rotateCc() { - const cc = CameraControls; - const earth = cc.Earth; - const street = cc.Street; + _rotateCcm() { + const ccm = CameraControlMode; + const orbit = ccm.ORBIT; + const earth = ccm.EARTH; + const street = ccm.STREET; const modeRotation = {}; + modeRotation[orbit] = earth; modeRotation[earth] = street; - modeRotation[street] = earth; + modeRotation[street] = orbit; const config = this._config; - const mode = cc[config.cameraControls]; - config.cameraControls = cc[modeRotation[mode]]; - return cc[config.cameraControls]; + const mode = config.cameraControlMode; + config.cameraControlMode = modeRotation[mode]; + return config.cameraControlMode; } _rotateCvm() { diff --git a/viewer/src/controller/OptionController.js b/viewer/src/controller/OptionController.js index 831552dc2..527611db8 100644 --- a/viewer/src/controller/OptionController.js +++ b/viewer/src/controller/OptionController.js @@ -3,28 +3,27 @@ */ import { - CameraControls, CameraVisualizationMode, OriginalPositionMode, } from '../../node_modules/mapillary-js/dist/mapillary.module.js'; import {EventEmitter} from '../util/EventEmitter.js'; -import {DatController} from './DatController.js'; +import {DatController, FolderName} from './DatController.js'; import {KeyController} from './KeyController.js'; export class OptionController { constructor(options) { - const cc = CameraControls; const cvm = CameraVisualizationMode; const opm = OriginalPositionMode; const config = Object.assign({}, options, { - cameraControls: cc[options.cameraControls], cameraVisualizationMode: cvm[options.cameraVisualizationMode], originalPositionMode: opm[options.originalPositionMode], }); const eventTypes = { - cameraControls: 'cameracontrols', + axesVisible: 'axesvisible', + cameraControlMode: 'cameracontrolmode', cameraSize: 'camerasize', cameraVisualizationMode: 'cameravisualizationmode', + cellsVisible: 'cellsvisible', commandsVisible: 'commandsvisible', datToggle: 'dattoggle', imagesVisible: 'imagesvisible', @@ -34,7 +33,7 @@ export class OptionController { pointsVisible: 'pointsvisible', reconstructionsSelected: 'reconstructionsselected', thumbnailVisible: 'thumbnailvisible', - tilesVisible: 'tilesvisible', + statsVisible: 'statsvisible', }; const emitter = new EventEmitter(); const internalOptions = {config, emitter, eventTypes}; @@ -42,6 +41,7 @@ export class OptionController { this._keyController = new KeyController(internalOptions); this._emitter = emitter; this._config = config; + this._eventTypes = eventTypes; } get config() { diff --git a/viewer/src/opensfm.js b/viewer/src/opensfm.js index 8cb5d8d94..6a0d14192 100644 --- a/viewer/src/opensfm.js +++ b/viewer/src/opensfm.js @@ -22,6 +22,7 @@ export {OpensfmViewer} from './ui/OpensfmViewer.js'; export {CancelledError} from './util/Error.js'; export {EventEmitter} from './util/EventEmitter.js'; +export * from './util/coords.js'; export * from './util/ids.js'; export * from './util/params.js'; export * from './util/types.js'; diff --git a/viewer/src/provider/DataConverter.js b/viewer/src/provider/DataConverter.js index 9e01acdac..9823ca19c 100644 --- a/viewer/src/provider/DataConverter.js +++ b/viewer/src/provider/DataConverter.js @@ -13,6 +13,14 @@ export class DataConverter { cluster(cluster, clusterId, reference) { const points = cluster.points ?? {}; + const normalize = 1 / 255; + for (const point of Object.values(points)) { + const color = point.color; + color[0] *= normalize; + color[1] *= normalize; + color[2] *= normalize; + } + return { id: clusterId, points, @@ -112,7 +120,7 @@ export class DataConverter { const cluster = {id: clusterId, url: clusterId}; const computed_rotation = shot.rotation; const height = camera.height; - const merge_id = shot.merge_cc.toString(); + const merge_id = shot.merge_cc == null ? null : shot.merge_cc.toString(); const exif_orientation = shot.orientation; const quality_score = 1; const width = camera.width; diff --git a/viewer/src/provider/OpensfmDataProvider.js b/viewer/src/provider/OpensfmDataProvider.js index ffe167121..3707354fc 100644 --- a/viewer/src/provider/OpensfmDataProvider.js +++ b/viewer/src/provider/OpensfmDataProvider.js @@ -18,7 +18,7 @@ function* generator(items, map) { export class OpensfmDataProvider extends DataProviderBase { constructor(options, geometry) { - super(geometry ?? new S2GeometryProvider(15)); + super(geometry ?? new S2GeometryProvider(16)); this._convert = new DataConverter(options); this._inventedImageBuffer = null; this._rawData = {}; @@ -32,7 +32,11 @@ export class OpensfmDataProvider extends DataProviderBase { const endpoint = options.endpoint; this._inventedImagePromise = !options.imagePath - ? this._inventImagePromise() + ? this._inventImagePromise().then(buffer => { + this._inventedImageBuffer = buffer; + this._inventedImagePromise = null; + return buffer; + }) : null; const recs = options.reconstructionPaths; @@ -100,21 +104,9 @@ export class OpensfmDataProvider extends DataProviderBase { reject(new Error('Already loading')); return; } - if (this.loaded) { - reject(new Error('Already loaded')); - return; - } - if (!isReconstructionData(file.data)) { - reject(new Error('Not a reconstruction')); - return; - } try { - this._addFile(file); - if (inventImages) { - this._inventedImagePromise = this._inventImagePromise(); - } - this.fire('load', {target: this}); + this._initialize(file, inventImages); resolve(); } catch (error) { reject(error); @@ -154,7 +146,9 @@ export class OpensfmDataProvider extends DataProviderBase { return this._getInventedImageBuffer(); } - return fetchArrayBuffer(url, cancellation); + return fetchArrayBuffer(url, cancellation).catch(() => + this._inventImagePromise(), + ); } getMesh(url) { @@ -222,7 +216,7 @@ export class OpensfmDataProvider extends DataProviderBase { meshes[uniqueId] = this._convert.mesh(shot); - const cellId = this._geometry.lngLatToCellId(image.computed_geometry); + const cellId = this._geometry.lngLatToCellId(image.geometry); if (!(cellId in cells)) { cells[cellId] = {}; } @@ -301,6 +295,28 @@ export class OpensfmDataProvider extends DataProviderBase { return imageId in this._data.images; } + _initialize(file, inventImages) { + if (this.loaded) { + reject(new Error('Already loaded')); + return; + } + + if (!isReconstructionData(file.data)) { + reject(new Error('Not a reconstruction')); + return; + } + + this._addFile(file); + if (inventImages) { + this._inventedImagePromise = this._inventImagePromise().then(buffer => { + this._inventedImageBuffer = buffer; + this._inventedImagePromise = null; + return buffer; + }); + } + this.fire('load', {target: this}); + } + _inventClusterId(index) { return `cluster_${index}`; } @@ -314,10 +330,6 @@ export class OpensfmDataProvider extends DataProviderBase { const buffer = await blob.arrayBuffer(); resolve(buffer); }, 'image/jpeg'); - }).then(buffer => { - this._inventedImageBuffer = buffer; - this._inventedImagePromise = null; - return buffer; }); } @@ -336,7 +348,7 @@ export class OpensfmDataProvider extends DataProviderBase { return this.add({data}); } - await this.load({data}); + await this._initialize({data}); this._load = null; resolve(); return Promise.resolve(); diff --git a/viewer/src/renderer/AxesRenderer.js b/viewer/src/renderer/AxesRenderer.js new file mode 100644 index 000000000..5eaf8fce0 --- /dev/null +++ b/viewer/src/renderer/AxesRenderer.js @@ -0,0 +1,146 @@ +/** + * @format + */ + +import { + AmbientLight, + ConeGeometry, + CylinderGeometry, + DirectionalLight, + Euler, + Matrix4, + Mesh, + MeshPhongMaterial, + Object3D, + Scene, + SphereGeometry, +} from '../../node_modules/three/build/three.module.js'; + +export class AxesRenderer { + constructor() { + this._scene = new Scene(); + this._scene.add(this._createAxes()); + this._scene.add(new AmbientLight(0xffffff, 0.5)); + const lightNE = new DirectionalLight(0xffffff, 0.4); + lightNE.position.set(1, 1, 1); + const lightSW = new DirectionalLight(0xffffff, 0.1); + lightSW.position.set(-1, -1, 1); + this._scene.add(lightNE, lightSW); + } + + get scene() { + return this._scene; + } + + onAdd(viewer, camera, reference) { + /* noop */ + } + + onReference(reference) { + /* noop */ + } + + onRemove() { + /* noop */ + } + + _createAxes() { + const axisLength = 10; + const coneLength = axisLength / 5; + const coneRadius = axisLength / 15; + const options = { + coneLength, + coneRadius, + cylinderLength: axisLength - coneLength, + cylinderRadius: 3e-1 * coneRadius, + }; + + const east = this._createAxis( + Object.assign({}, options, { + color: 0xff0000, + euler: [0, 0, -Math.PI / 2], + }), + ); + const north = this._createAxis( + Object.assign({}, options, { + color: 0x00ff00, + euler: [0, 0, 0], + }), + ); + const up = this._createAxis( + Object.assign({}, options, { + color: 0x0088ff, + euler: [Math.PI / 2, 0, 0], + }), + ); + + const sphereSegments = 12; + const origin = new Mesh( + new SphereGeometry( + options.cylinderRadius, + sphereSegments, + sphereSegments, + ), + new MeshPhongMaterial({ + color: 0xffffff, + flatShading: true, + }), + ); + + const axes = new Object3D(); + axes.add(east, north, up, origin); + + return axes; + } + + _createAxis(options) { + const color = options.color; + const euler = options.euler; + const coneLength = options.coneLength; + const coneRadius = options.coneRadius; + const cylinderLength = options.cylinderLength; + const cylinderRadius = options.cylinderRadius; + + const rotation = new Matrix4().multiply( + new Matrix4().makeRotationFromEuler(new Euler().fromArray(euler)), + ); + const material = new MeshPhongMaterial({ + color, + flatShading: true, + }); + + const cylinderSegments = 12; + const cylinder = new Mesh( + new CylinderGeometry( + cylinderRadius, + cylinderRadius, + cylinderLength, + cylinderSegments, + ), + material.clone(), + ); + cylinder.applyMatrix4( + rotation + .clone() + .multiply(new Matrix4().makeTranslation(0, cylinderLength / 2, 0)), + ); + + const coneSegments = 12; + const cone = new Mesh( + new ConeGeometry(coneRadius, coneLength, coneSegments), + material.clone(), + ); + cone.applyMatrix4( + rotation + .clone() + .multiply( + new Matrix4().makeTranslation(0, cylinderLength + coneLength / 2, 0), + ), + ); + + const axis = new Object3D(); + axis.add(cylinder, cone); + + return axis; + } +} diff --git a/viewer/src/renderer/CustomRenderer.js b/viewer/src/renderer/CustomRenderer.js new file mode 100644 index 000000000..86a5e6a8a --- /dev/null +++ b/viewer/src/renderer/CustomRenderer.js @@ -0,0 +1,93 @@ +/** + * @format + */ + +import { + Camera, + WebGLRenderer, +} from '../../node_modules/three/build/three.module.js'; +import {EventEmitter} from '../util/EventEmitter.js'; + +export class CustomRenderer extends EventEmitter { + constructor(viewer) { + super(); + + this._id = 'opensfm-renderer'; + this._camera = new Camera(); + this._camera.matrixAutoUpdate = false; + this._viewer = viewer; + this._reference = null; + + this._children = []; + } + + get added() { + return !!this._reference; + } + + get id() { + return this._id; + } + + get reference() { + return this._reference; + } + + add(renderer) { + if (this._children.indexOf(renderer) >= 0) { + throw new Error(`Renderer already added`); + } + this._children.push(renderer); + if (this.added) { + renderer.onAdd(this._viewer, this._camera, this._reference); + } + } + + remove(renderer) { + const index = this._children.indexOf(renderer); + if (index > -1) { + this._children.splice(index, 1); + renderer.onRemove(); + } + } + + onAdd(viewer, reference, context) { + this._reference = reference; + const canvas = viewer.getCanvas(); + const renderer = new WebGLRenderer({canvas, context}); + renderer.autoClear = false; + this._renderer = renderer; + for (const child of this._children) { + child.onAdd(this.viewer, this._camera, this._reference); + } + this.fire('add', {target: this}); + } + + onReferenceChanged(viewer, reference) { + this._reference = reference; + for (const child of this._children) { + child.onReference(this._reference); + } + this.fire('reference', {target: this}); + } + + onRemove(viewer, context) { + for (const child of this._children) { + child.onRemove(); + } + } + + render(context, viewMatrix, projectionMatrix) { + const camera = this._camera; + camera.matrix.fromArray(viewMatrix).invert(); + camera.updateMatrixWorld(true); + camera.projectionMatrix.fromArray(projectionMatrix); + camera.projectionMatrixInverse.copy(camera.projectionMatrix).invert(); + + const renderer = this._renderer; + renderer.resetState(); + for (const child of this._children) { + renderer.render(child.scene, camera); + } + } +} diff --git a/viewer/src/renderer/EarthRenderer.js b/viewer/src/renderer/EarthRenderer.js new file mode 100644 index 000000000..b1fb91aae --- /dev/null +++ b/viewer/src/renderer/EarthRenderer.js @@ -0,0 +1,174 @@ +/** + * @format + */ + +import { + BufferGeometry, + DoubleSide, + Float32BufferAttribute, + Mesh, + Raycaster, + Scene, + ShaderMaterial, + Vector2, + Vector3, +} from '../../node_modules/three/build/three.module.js'; +import {pixelToViewport} from '../util/coords.js'; + +const frag = ` +#ifdef GL_ES +precision mediump float; +#endif + +uniform float uLineWidth; +uniform float uTileSize; +uniform vec2 uMouse; +uniform bool uMouseIntersecting; +uniform float uRadius; + +varying vec4 vWorldCoords; + +void main() { + vec4 fragColor = vec4(0.0, 0.0, 0.0, 0.0); + + float mouseDist = distance(vWorldCoords.xy, uMouse); + bool showIndicator = + uMouseIntersecting && + mouseDist < uRadius; + + if (showIndicator) { + float discAlpha = (uRadius - mouseDist) / uRadius; + discAlpha = smoothstep(0.0, 1.0, discAlpha); + + vec2 pos = vWorldCoords.xy / uTileSize; + vec2 f = abs(pos - floor(pos + vec2(0.5, 0.5))); + vec2 df = fwidth(pos) * uLineWidth; + vec2 g = smoothstep(-df, df, f); + float grid = 1.0 - clamp(g.x * g.y, 0.0, 1.0); + fragColor.rgb = mix(fragColor.rgb, vec3(1.0, 1.0, 1.0), grid); + fragColor.a = discAlpha; + } + + gl_FragColor = fragColor; +} +`; + +const vert = ` +#ifdef GL_ES +precision mediump float; +#endif + +varying vec4 vWorldCoords; + +void main() { + vWorldCoords = vec4(position, 1.0); + gl_Position = projectionMatrix * viewMatrix * vec4(position, 1.0); +} +`; + +const SIZE = 10000; + +export class EarthRenderer { + constructor() { + this._earth = this._makeEarth(); + this._scene = new Scene(); + this._scene.add(this._earth); + + this._camera = null; + this._raycaster = new Raycaster(); + this._origin = new Vector3(); + this._direction = new Vector3(); + } + + get active() { + return !!this._camera; + } + + get scene() { + return this._scene; + } + + intersect(event) { + if (!this.active) { + throw new Error('Cannot intersect when inactive'); + } + + const camera = this._camera; + const raycaster = this._raycaster; + const origin = this._origin; + const direction = this._direction; + const uniforms = this._earth.material.uniforms; + + const viewport = pixelToViewport(event.pixelPoint, this._container); + origin.setFromMatrixPosition(camera.matrixWorld); + direction + .set(viewport[0], viewport[1], 0.5) + .applyMatrix4(camera.projectionMatrixInverse) + .applyMatrix4(camera.matrixWorld) + .sub(origin) + .normalize(); + raycaster.set(origin, direction); + + const intersections = raycaster.intersectObject(this._earth); + if (intersections.length) { + const intersection = intersections[0]; + uniforms.uMouseIntersecting.value = true; + uniforms.uMouse.value.x = intersection.point.x; + uniforms.uMouse.value.y = intersection.point.y; + } else { + uniforms.uMouseIntersecting.value = false; + uniforms.uMouse.value.x = 0; + uniforms.uMouse.value.y = 0; + } + } + + onAdd(viewer, camera, reference) { + this._container = viewer.getContainer(); + this._camera = camera; + } + + onReference(reference) { + /* noop */ + } + + onRemove() { + this._camera = null; + } + + _makeEarth() { + const size = SIZE; + const quad = [ + size / 2, + size / 2, + -2, + size / 2, + -size / 2, + -2, + -size / 2, + -size / 2, + -2, + -size / 2, + size / 2, + -2, + ]; + const indices = [0, 1, 2, 2, 3, 0]; + const geometry = new BufferGeometry(); + geometry.setAttribute('position', new Float32BufferAttribute(quad, 3)); + geometry.setIndex(indices); + const uniforms = { + uLineWidth: {value: 2}, + uMouse: {value: new Vector2()}, + uMouseIntersecting: {value: false}, + uRadius: {value: 10}, + uTileSize: {value: 3}, + }; + const material = new ShaderMaterial({ + fragmentShader: frag, + vertexShader: vert, + side: DoubleSide, + uniforms, + transparent: true, + }); + return new Mesh(geometry, material); + } +} diff --git a/viewer/src/ui/FileLoader.js b/viewer/src/ui/FileLoader.js index b83233745..b3f2fae22 100644 --- a/viewer/src/ui/FileLoader.js +++ b/viewer/src/ui/FileLoader.js @@ -38,7 +38,7 @@ export class FileLoader { data, name: file.name, type: null, - url: `local/${file.name}`, + url: `file:${file.name}`, })), ); } diff --git a/viewer/src/ui/OpensfmViewer.js b/viewer/src/ui/OpensfmViewer.js index 834155f2d..6a421cf25 100644 --- a/viewer/src/ui/OpensfmViewer.js +++ b/viewer/src/ui/OpensfmViewer.js @@ -3,19 +3,24 @@ */ import { - CameraControls, CameraVisualizationMode, OriginalPositionMode, RenderMode, Viewer, } from '../../node_modules/mapillary-js/dist/mapillary.module.js'; import {EventEmitter} from '../util/EventEmitter.js'; +import {AxesRenderer} from '../renderer/AxesRenderer.js'; +import {CustomRenderer} from '../renderer/CustomRenderer.js'; +import {EarthRenderer} from '../renderer/EarthRenderer.js'; import {CommandExplainerControl} from '../control/CommandExplainerControl.js'; import {InfoControl} from '../control/InfoControl.js'; +import {StatsControl} from '../control/StatsControl.js'; +import {ThumbnailControl} from '../control/ThumbnailControl.js'; import {FolderName} from '../controller/DatController.js'; import {OptionController} from '../controller/OptionController.js'; -import {ThumbnailControl} from '../control/ThumbnailControl.js'; import {FileController} from '../controller/FileController.js'; +import {OrbitCameraControls} from './OrbitCameraControls.js'; +import {convertCameraControlMode, CameraControlMode} from './modes.js'; export class OpensfmViewer extends EventEmitter { constructor(options) { @@ -34,20 +39,20 @@ export class OpensfmViewer extends EventEmitter { const spatialConfiguration = { cameraSize: 0.5, cameraVisualizationMode: cvm, + cellGridDepth: 3, + cellsVisible: false, originalPositionMode: opm, pointSize: 0.2, pointsVisible: true, - tilesVisible: false, }; + const cameraControlMode = CameraControlMode.ORBIT; const imagesVisible = false; - const cameraControls = CameraControls.Earth; - this._viewer = new Viewer({ + const viewer = new Viewer({ apiClient: this._provider, - cameraControls, + cameraControls: convertCameraControlMode(cameraControlMode), combinedPanning: false, component: { - bearing: false, cover: false, direction: false, image: imagesVisible, @@ -59,29 +64,38 @@ export class OpensfmViewer extends EventEmitter { imageTiling: false, renderMode: RenderMode.Letterbox, }); - const viewer = this._viewer; + + viewer.attachCustomCameraControls(new OrbitCameraControls()); this._spatial = viewer.getComponent('spatial'); + this._viewer = viewer; const infoSize = 0.3; - const thumbnailVisible = false; const commandsVisible = true; + const statsVisible = true; + const thumbnailVisible = false; const controllerOptions = { - cameraControls, + axesVisible: true, + cameraControlMode, commandsVisible, imagesVisible, infoSize, + statsVisible, thumbnailVisible, }; this._optionController = new OptionController( Object.assign( {}, - viewer.getComponent('spatial').defaultConfiguration, + this._spatial.defaultConfiguration, spatialConfiguration, controllerOptions, ), ); + this._statsControl = new StatsControl({ + visible: statsVisible, + provider: this._provider, + }); this._thumbnailControl = new ThumbnailControl({ visible: thumbnailVisible, provider: this._provider, @@ -96,6 +110,7 @@ export class OpensfmViewer extends EventEmitter { }); this._infoControl.addControl(this._commandExplainerControl); this._infoControl.addControl(this._thumbnailControl); + this._infoControl.addControl(this._statsControl); this._infoControl.setWidth(infoSize); this._fileController = new FileController({ @@ -107,13 +122,14 @@ export class OpensfmViewer extends EventEmitter { this._fileController, FolderName.IO, ); - const toggleCommand = { - key: 'p', - value: 'toggleFileLoader', - handler: async () => await this._fileController.toggle(), - }; - this._optionController.key.addCommand(toggleCommand); - this._commandExplainerControl.add({[toggleCommand.key]: toggleCommand}); + + this._makeCommands(); + + this._axesRenderer = new AxesRenderer(); + this._earthRenderer = new EarthRenderer(); + this._customRenderer = new CustomRenderer(this._viewer); + this._customRenderer.add(this._axesRenderer); + this._viewer.addCustomRenderer(this._customRenderer); } get commands() { @@ -144,6 +160,7 @@ export class OpensfmViewer extends EventEmitter { this._loadProvider().then(provider => { const items = Object.keys(provider.data.clusters); this._optionController.dat.addReconstructionItems(items); + this._statsControl.addRawData(provider.rawData); }); } @@ -159,8 +176,9 @@ export class OpensfmViewer extends EventEmitter { this._fileController.on('load', event => this._onFileLoad(event)); const optionController = this._optionController; - optionController.on('cameracontrols', event => - this._onCameraControls(event), + optionController.on('axesvisible', event => this._onAxesVisible(event)); + optionController.on('cameracontrolmode', event => + this._onCameraControlMode(event), ); optionController.on('camerasize', event => this._onCameraSize(event)); optionController.on('cameravisualizationmode', event => @@ -180,16 +198,18 @@ export class OpensfmViewer extends EventEmitter { optionController.on('thumbnailvisible', event => this._onThumbnailVisible(event), ); - optionController.on('tilesvisible', event => this._onTilesVisible(event)); + optionController.on('cellsvisible', event => this._onCellsVisible(event)); optionController.on('reconstructionsselected', event => this._onReconstructionsSelected(event), ); + optionController.on('statsvisible', event => this._onStatsVisible(event)); this._provider.on('opensfmdatacreate', event => this._onProviderOpensfmDataCreate(event), ); this._viewer.on('image', event => this._onViewerImage(event)); + this._viewer.on('mousemove', event => this._onViewerMouseMove(event)); } _loadProvider() { @@ -205,6 +225,16 @@ export class OpensfmViewer extends EventEmitter { }); } + _makeCommands() { + const toggleCommand = { + key: 'p', + value: 'toggleFileLoader', + handler: async () => await this._fileController.toggle(), + }; + this._optionController.key.addCommand(toggleCommand); + this._commandExplainerControl.add({[toggleCommand.key]: toggleCommand}); + } + async _move() { const params = this._params; if (!!params.img) { @@ -222,6 +252,15 @@ export class OpensfmViewer extends EventEmitter { viewer.moveTo(imageId).catch(error => console.error(error)); } + _onAxesVisible(event) { + if (event.visible) { + this._customRenderer.add(this._axesRenderer); + } else { + this._customRenderer.remove(this._axesRenderer); + } + this._viewer.triggerRerender(); + } + _onCameraSize(event) { this._configure({cameraSize: event.size}); } @@ -247,21 +286,30 @@ export class OpensfmViewer extends EventEmitter { } } - _onCameraControls(event) { + _onCameraControlMode(event) { const mode = event.mode; - const bearing = 'bearing'; const direction = 'direction'; const zoom = 'zoom'; - this._viewer.setCameraControls(mode); - if (mode === CameraControls.Earth) { - this._viewer.deactivateComponent(bearing); - this._viewer.deactivateComponent(direction); - this._viewer.deactivateComponent(zoom); - } else { - this._viewer.activateComponent(bearing); + + if (mode === CameraControlMode.STREET) { this._viewer.activateComponent(direction); this._viewer.activateComponent(zoom); + } else { + this._viewer.deactivateComponent(direction); + this._viewer.deactivateComponent(zoom); + } + + if (mode === CameraControlMode.EARTH) { + this._customRenderer.add(this._earthRenderer); + } else if (this._earthRenderer.active) { + this._customRenderer.remove(this._earthRenderer); } + + this._viewer.setCameraControls(convertCameraControlMode(mode)); + } + + _onCellsVisible(event) { + this._configure({cellsVisible: event.visible}); } async _onFileLoad(event) { @@ -284,6 +332,13 @@ export class OpensfmViewer extends EventEmitter { this._infoControl.setWidth(event.size); } + _onViewerMouseMove(event) { + if (this._earthRenderer.active) { + this._earthRenderer.intersect(event); + this._viewer.triggerRerender(); + } + } + _onOriginalPositionMode(event) { this._configure({originalPositionMode: event.mode}); } @@ -299,6 +354,7 @@ export class OpensfmViewer extends EventEmitter { _onProviderOpensfmDataCreate(event) { const clusters = Object.keys(event.data.clusters); this._optionController.dat.addReconstructionItems(clusters); + this._statsControl.addRawData(event.rawData); } _onReconstructionsSelected(event) { @@ -306,8 +362,12 @@ export class OpensfmViewer extends EventEmitter { this._viewer.setFilter(filter); } - _onTilesVisible(event) { - this._configure({tilesVisible: event.visible}); + _onStatsVisible(event) { + if (event.visible) { + this._statsControl.show(); + } else { + this._statsControl.hide(); + } } _onThumbnailVisible(event) { diff --git a/viewer/src/ui/OrbitCameraControls.js b/viewer/src/ui/OrbitCameraControls.js new file mode 100644 index 000000000..e0e5c7bae --- /dev/null +++ b/viewer/src/ui/OrbitCameraControls.js @@ -0,0 +1,139 @@ +/** + * @format + */ + +import { + Matrix4, + PerspectiveCamera, + Vector3, +} from '../../node_modules/three/build/three.module.js'; +import {OrbitControls} from '../../node_modules/three/examples/jsm/controls/OrbitControls.js'; + +const DAMPING_FACTOR = 0.1; +const FOV = 90; +const MAX_DISTANCE = 5000; +const MIN_DISTANCE = 1; + +export class OrbitCameraControls { + constructor() { + this._controls = null; + this._reference = null; + this._projectionMatrixCallback = null; + this._viewMatrixCallback = null; + this._viewMatrix = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]; + } + + onActivate(viewer, viewMatrix, projectionMatrix, reference) { + this._reference = reference; + + const container = viewer.getContainer(); + const aspect = this._calcAspect(container); + const camera = new PerspectiveCamera(FOV, aspect, 0.1, 10000); + camera.up.set(0, 0, 1); + + const matrixWorld = new Matrix4().fromArray(viewMatrix).invert(); + const me = matrixWorld.elements; + const eye = new Vector3(me[12], me[13], me[14]); + camera.position.copy(eye); + + // Set target to 10 meters distance in front of eye on activation + const forward = new Vector3(-me[8], -me[9], -me[10]); + const target = eye.clone().add(forward.normalize().multiplyScalar(10)); + + const controls = new OrbitControls(camera, container); + this._controls = controls; + controls.minDistance = MIN_DISTANCE; + controls.maxDistance = MAX_DISTANCE; + controls.enableDamping = true; + controls.dampingFactor = DAMPING_FACTOR; + controls.target.copy(target); + + camera.matrixWorld.copy(matrixWorld); + camera.matrixWorldInverse.fromArray(viewMatrix); + controls.object.updateMatrixWorld(true); + + controls.addEventListener('change', this._onControlsChange); + controls.update(); + + this._updateProjectionMatrix(); + } + + onAnimationFrame(viewer, frameId) { + // Workaround until MJS is fixed to not invoke before onActivate + if (this._controls) { + this._controls.update(); + } + } + + onAttach(viewer, viewMatrixCallback, projectionMatrixCallback) { + this._viewMatrixCallback = viewMatrixCallback; + this._projectionMatrixCallback = projectionMatrixCallback; + } + + onDeactivate(viewer) { + this._controls.removeEventListener('change', this._onControlsChange); + this._controls.dispose(); + this._controls = null; + } + + onDetach(viewer) { + this._projectionMatrixCallback = null; + this._viewMatrixCallback = null; + } + + onReference(viewer, reference) { + const oldReference = this._reference; + const camera = this._controls.object; + const target = this._controls.target; + const targetOffset = target.clone().sub(camera.position); + + const enu = camera.position; + const [lng, lat, alt] = enuToGeodetic( + enu.x, + enu.y, + enu.z, + oldReference.lng, + oldReference.lat, + oldReference.alt, + ); + const [e, n, u] = geodeticToEnu( + lng, + lat, + alt, + reference.lng, + reference.lat, + reference.alt, + ); + + const position = new Vector3(e, n, u); + this._controls.object.position.copy(position); + this._controls.object.updateMatrixWorld(true); + this._controls.target.copy(position.clone().add(targetOffset)); + + this._reference = reference; + } + + onResize(viewer) { + this._updateProjectionMatrix(); + } + + _calcAspect(element) { + const width = element.offsetWidth; + const height = element.offsetHeight; + return width === 0 || height === 0 ? 0 : width / height; + } + + _onControlsChange = () => { + this._controls.object.updateMatrixWorld(true); + this._viewMatrixCallback( + this._controls.object.matrixWorldInverse.toArray(), + ); + }; + + _updateProjectionMatrix() { + const camera = this._controls.object; + camera.aspect = this._calcAspect(this._controls.domElement); + camera.updateProjectionMatrix(); + this._projectionMatrixCallback(camera.projectionMatrix.toArray()); + } +} diff --git a/viewer/src/ui/modes.js b/viewer/src/ui/modes.js new file mode 100644 index 000000000..b4d6c67ea --- /dev/null +++ b/viewer/src/ui/modes.js @@ -0,0 +1,24 @@ +/** + * @format + */ + +import {CameraControls} from '../../node_modules/mapillary-js/dist/mapillary.module.js'; + +export const CameraControlMode = Object.freeze({ + EARTH: 'earth', + ORBIT: 'orbit', + STREET: 'street', +}); + +export function convertCameraControlMode(mode) { + switch (mode) { + case CameraControlMode.EARTH: + return CameraControls.Earth; + case CameraControlMode.ORBIT: + return CameraControls.Custom; + case CameraControlMode.STREET: + return CameraControls.Street; + default: + throw new Error('Camera control mode does not exist'); + } +} diff --git a/viewer/src/util/coords.js b/viewer/src/util/coords.js new file mode 100644 index 000000000..498cedd13 --- /dev/null +++ b/viewer/src/util/coords.js @@ -0,0 +1,11 @@ +/** + * @format + */ + +export function pixelToViewport(pixelPoint, container) { + const canvasWidth = container.offsetWidth; + const canvasHeight = container.offsetHeight; + const viewportX = (2 * pixelPoint[0]) / canvasWidth - 1; + const viewportY = 1 - (2 * pixelPoint[1]) / canvasHeight; + return [viewportX, viewportY]; +} diff --git a/viewer/styles/opensfm.css b/viewer/styles/opensfm.css index b531c3cec..c1078afa5 100644 --- a/viewer/styles/opensfm.css +++ b/viewer/styles/opensfm.css @@ -238,6 +238,16 @@ input[type='file'] { margin: 2px; } +.opensfm-info-text.opensfm-info-text-stat { + margin-bottom: 0; + margin-top: 0; + font-size: 9px; + overflow-wrap: initial; + text-overflow: ellipsis; + white-space: nowrap; + overflow: hidden; +} + .opensfm-info-text.opensfm-info-inline { display: inline-block; }