feat: upgrade opensfm viewer to mapillary-js v4.0.0-beta.4 (mapillary…

…#740)

Summary:
Pull Request resolved: mapillary#740

## Contributions
- Upgrade viewer to MapillaryJS v4.0.0-beta.4
- Fix loading using reconstruction path
- Invent image buffer on client if server returns error (e.g. when no images exist for a dataset). This means the viewer will always work, even when images are missing. Errors will be logged to the browser.
- Increased cell grid depth for more fine grained load
- Do not hide bearing indicator in non street camera control mode
- Add ENU axes visualization
- Add three.js orbit controls as default camera control option using MJS custom camera control API
- Earth surface hover indicator for simplified earth navigation
- Add image and point count stats control
- Improved spatial viz perf in MJS. Logarithmic ray tracing/camera frame intersection. Improved linear rendering logic. Should work with ~10,000 camera frames without considerable lag.

Reviewed By: paulinus

Differential Revision: D28031955

fbshipit-source-id: 1e44c9c5ac83df0115b9a811f118c4560007917e

.gitignore

@@ -18,7 +18,7 @@ launch.json
 eval
 
 # Viewer
-viewer/node_modules
+node_modules
 
 # Ignore the data folder, but not the berlin example.
 data/*

viewer/node_modules.sh

@@ -39,7 +39,7 @@ curl -fS "$UNPKG/$GLM_V/$PKG" \
 
 # mapillary-js
 MJS_PKG="mapillary-js"
-MJS_V="$MJS_PKG@4.0.0-beta.2"
+MJS_V="$MJS_PKG@4.0.0-beta.4"
 MJS_DIST="dist"
 MJS_JS="mapillary.module.js"
 MJS_CSS="mapillary.css"
@@ -52,3 +52,23 @@ curl -fS --create-dirs "$UNPKG/$MJS_IN/$MJS_CSS" \
     -o "$MJS_OUT/$MJS_CSS" || exit 1
 curl -fS "$UNPKG/$MJS_V/$PKG" \
     -o "$NODE_MODULES/$MJS_PKG/$PKG" || exit 1
+
+# three
+THR_PKG="three"
+THR_V="$THR_PKG@0.125.2"
+THR_DIST="build"
+THR_JS="three.module.js"
+
+THR_IN="$THR_V/$THR_DIST"
+THR_OUT="$NODE_MODULES/$THR_PKG/$THR_DIST"
+curl -fS --create-dirs "$UNPKG/$THR_IN/$THR_JS" \
+    -o "$THR_OUT/$THR_JS" || exit 1
+curl -fS "$UNPKG/$THR_V/$PKG" \
+    -o "$NODE_MODULES/$THR_PKG/$PKG" || exit 1
+
+THR_EXM="examples/jsm"
+THR_EXM_JS="controls/OrbitControls.js"
+THR_EXM_IN="$THR_V/$THR_EXM"
+THR_EXM_OUT="$NODE_MODULES/$THR_PKG/$THR_EXM"
+curl -fS --create-dirs "$UNPKG/$THR_EXM_IN/$THR_EXM_JS" \
+    -o "$THR_EXM_OUT/$THR_EXM_JS" || exit 1

viewer/src/control/StatsControl.js

@@ -0,0 +1,92 @@
+/**
+ * @format
+ */
+
+export class StatsControl {
+  constructor(options) {
+    const container = this._createContainer();
+    this._container = container;
+    this._provider = options.provider;
+    this._shotCount = 0;
+    this._pointCount = 0;
+    this._total = this._createText(
+      this._makeContent('Total', this._shotCount, this._pointCount),
+    );
+    this._container.appendChild(this._total);
+    this._visible = false;
+    if (options.visible) {
+      this.show();
+    }
+  }
+
+  get container() {
+    return this._container;
+  }
+
+  addRawData(rawData) {
+    for (const data of Object.values(rawData)) {
+      const id = data.id;
+      const url = data.url;
+      const cluster = data.cluster;
+      const shotCount = Object.keys(cluster.shots).length;
+      const pointCount = Object.keys(cluster.points).length;
+      const content = this._makeContent(id, shotCount, pointCount, url);
+      this.addStatRow(content);
+
+      this._shotCount += shotCount;
+      this._pointCount += pointCount;
+    }
+
+    this._total.textContent = this._makeContent(
+      'Total',
+      this._shotCount,
+      this._pointCount,
+    );
+  }
+
+  addStatRow(content) {
+    const stat = this._createText(content);
+    stat.classList.add('opensfm-info-text-stat');
+    this._container.appendChild(stat);
+  }
+
+  hide() {
+    if (!this._visible) {
+      return;
+    }
+    this._container.classList.add('opensfm-hidden');
+    this._visible = false;
+  }
+
+  show() {
+    if (this._visible) {
+      return;
+    }
+    this._container.classList.remove('opensfm-hidden');
+    this._visible = true;
+  }
+
+  _createContainer() {
+    const header = document.createElement('span');
+    header.classList.add('opensfm-info-text', 'opensfm-info-text-header');
+    header.textContent = 'Stats';
+
+    const container = document.createElement('div');
+    container.classList.add('opensfm-control-container', 'opensfm-hidden');
+    container.appendChild(header);
+    return container;
+  }
+
+  _createText(content) {
+    const document = window.document;
+    const element = document.createElement('span');
+    element.classList.add('opensfm-info-text');
+    element.textContent = content;
+    return element;
+  }
+
+  _makeContent(prefix, shotCount, pointCount, suffix) {
+    const append = suffix ? ` (${suffix})` : '';
+    return `${prefix}: ${shotCount} images, ${pointCount} points${append}`;
+  }
+}

viewer/src/controller/DatController.js

@@ -4,11 +4,11 @@
 
 import {GUI} from '../../node_modules/dat.gui/build/dat.gui.module.js';
 import {
-  CameraControls,
   CameraVisualizationMode,
   OriginalPositionMode,
 } from '../../node_modules/mapillary-js/dist/mapillary.module.js';
 import {ListController} from './ListController.js';
+import {CameraControlMode} from '../ui/modes.js';
 
 export const FolderName = Object.freeze({
   INFO: 'info',
@@ -58,13 +58,13 @@ export class DatController {
       .onChange(v => this._onChange(name, v));
   }
 
-  _addCameraControlsOption(folder) {
-    const cc = CameraControls;
-    const ccs = [cc[cc.Earth], cc[cc.Street]];
+  _addCameraControlOption(folder) {
+    const ccm = CameraControlMode;
+    const ccms = [ccm.ORBIT, ccm.STREET, ccm.EARTH];
     folder
-      .add(this._config, 'cameraControls', ccs)
+      .add(this._config, 'cameraControlMode', ccms)
       .listen()
-      .onChange(c => this._onChange('cameraControls', cc[c]));
+      .onChange(m => this._onChange('cameraControlMode', m));
   }
 
   _addCameraVizualizationOption(folder) {
@@ -103,6 +103,7 @@ export class DatController {
     folder.open();
     this._addBooleanOption('commandsVisible', folder);
     this._addBooleanOption('thumbnailVisible', folder);
+    this._addBooleanOption('statsVisible', folder);
     this._addNumericOption('infoSize', folder);
     return folder;
   }
@@ -116,7 +117,7 @@ export class DatController {
   _createReconstructionsController(gui) {
     const emitter = this._emitter;
     const eventType = this._eventTypes.reconstructionsSelected;
-    const folder = gui.addFolder('Reconstructions');
+    const folder = gui.addFolder('Clusters');
     folder.open();
     const controller = new ListController({emitter, eventType, folder});
     return controller;
@@ -130,9 +131,10 @@ export class DatController {
     this._addCameraVizualizationOption(folder);
     this._addNumericOption('cameraSize', folder);
     this._addPositionVisualizationOption(folder);
-    this._addCameraControlsOption(folder);
-    this._addBooleanOption('tilesVisible', folder);
+    this._addCameraControlOption(folder);
+    this._addBooleanOption('cellsVisible', folder);
     this._addBooleanOption('imagesVisible', folder);
+    this._addBooleanOption('axesVisible', folder);
     return folder;
   }
 
@@ -142,17 +144,18 @@ export class DatController {
     let mode = null;
     let type = null;
     switch (name) {
-      case 'camerasVisible':
+      case 'axesVisible':
+      case 'cellsVisible':
       case 'commandsVisible':
       case 'imagesVisible':
       case 'pointsVisible':
+      case 'statsVisible':
       case 'thumbnailVisible':
-      case 'tilesVisible':
         const visible = value;
         type = types[name];
         emitter.fire(type, {type, visible});
         break;
-      case 'cameraControls':
+      case 'cameraControlMode':
       case 'originalPositionMode':
       case 'cameraVisualizationMode':
         mode = value;

viewer/src/controller/KeyController.js

@@ -3,10 +3,10 @@
  */
 
 import {
-  CameraControls,
   CameraVisualizationMode,
   OriginalPositionMode,
 } from '../../node_modules/mapillary-js/dist/mapillary.module.js';
+import {CameraControlMode} from '../ui/modes.js';
 
 export class KeyController {
   constructor(options) {
@@ -18,17 +18,19 @@ export class KeyController {
     const increase = 1.1;
     this._commands = {
       // visibility
+      c: {value: 'axesVisible'},
+      d: {value: 'cellsVisible'},
       e: {value: 'commandsVisible'},
-      f: {value: 'pointsVisible'},
-      d: {value: 'tilesVisible'},
       r: {value: 'imagesVisible'},
+      f: {value: 'pointsVisible'},
+      t: {value: 'statsVisible'},
       v: {value: 'thumbnailVisible'},
       // activity
       l: {value: 'datToggle'},
       // mode
       '1': {value: 'cameraVisualizationMode'},
       '2': {value: 'originalPositionMode'},
-      '3': {value: 'cameraControls'},
+      '3': {value: 'cameraControlMode'},
       // size
       q: {value: 'pointSize', coeff: decrease},
       w: {value: 'pointSize', coeff: increase},
@@ -79,6 +81,7 @@ export class KeyController {
         case 'e':
         case 'f':
         case 'r':
+        case 't':
         case 'v':
           const visible = this._toggle(command.value);
           emitter.fire(type, {type, visible});
@@ -96,8 +99,8 @@ export class KeyController {
           emitter.fire(type, {type, mode: opm});
           break;
         case '3':
-          const cc = this._rotateCc();
-          emitter.fire(type, {type, mode: cc});
+          const ccm = this._rotateCcm();
+          emitter.fire(type, {type, mode: ccm});
           break;
         case 'a':
         case 'q':
@@ -125,19 +128,21 @@ export class KeyController {
     return key in this._commands || key in this._customCommands;
   }
 
-  _rotateCc() {
-    const cc = CameraControls;
-    const earth = cc.Earth;
-    const street = cc.Street;
+  _rotateCcm() {
+    const ccm = CameraControlMode;
+    const orbit = ccm.ORBIT;
+    const earth = ccm.EARTH;
+    const street = ccm.STREET;
 
     const modeRotation = {};
+    modeRotation[orbit] = earth;
     modeRotation[earth] = street;
-    modeRotation[street] = earth;
+    modeRotation[street] = orbit;
 
     const config = this._config;
-    const mode = cc[config.cameraControls];
-    config.cameraControls = cc[modeRotation[mode]];
-    return cc[config.cameraControls];
+    const mode = config.cameraControlMode;
+    config.cameraControlMode = modeRotation[mode];
+    return config.cameraControlMode;
   }
 
   _rotateCvm() {

viewer/src/controller/OptionController.js

@@ -3,28 +3,27 @@
  */
 
 import {
-  CameraControls,
   CameraVisualizationMode,
   OriginalPositionMode,
 } from '../../node_modules/mapillary-js/dist/mapillary.module.js';
 import {EventEmitter} from '../util/EventEmitter.js';
-import {DatController} from './DatController.js';
+import {DatController, FolderName} from './DatController.js';
 import {KeyController} from './KeyController.js';
 
 export class OptionController {
   constructor(options) {
-    const cc = CameraControls;
     const cvm = CameraVisualizationMode;
     const opm = OriginalPositionMode;
     const config = Object.assign({}, options, {
-      cameraControls: cc[options.cameraControls],
       cameraVisualizationMode: cvm[options.cameraVisualizationMode],
       originalPositionMode: opm[options.originalPositionMode],
     });
     const eventTypes = {
-      cameraControls: 'cameracontrols',
+      axesVisible: 'axesvisible',
+      cameraControlMode: 'cameracontrolmode',
       cameraSize: 'camerasize',
       cameraVisualizationMode: 'cameravisualizationmode',
+      cellsVisible: 'cellsvisible',
       commandsVisible: 'commandsvisible',
       datToggle: 'dattoggle',
       imagesVisible: 'imagesvisible',
@@ -34,14 +33,15 @@ export class OptionController {
       pointsVisible: 'pointsvisible',
       reconstructionsSelected: 'reconstructionsselected',
       thumbnailVisible: 'thumbnailvisible',
-      tilesVisible: 'tilesvisible',
+      statsVisible: 'statsvisible',
     };
     const emitter = new EventEmitter();
     const internalOptions = {config, emitter, eventTypes};
     this._datController = new DatController(internalOptions);
     this._keyController = new KeyController(internalOptions);
     this._emitter = emitter;
     this._config = config;
+    this._eventTypes = eventTypes;
   }
 
   get config() {

viewer/src/opensfm.js

@@ -22,6 +22,7 @@ export {OpensfmViewer} from './ui/OpensfmViewer.js';
 
 export {CancelledError} from './util/Error.js';
 export {EventEmitter} from './util/EventEmitter.js';
+export * from './util/coords.js';
 export * from './util/ids.js';
 export * from './util/params.js';
 export * from './util/types.js';

viewer/src/provider/DataConverter.js

@@ -13,6 +13,14 @@ export class DataConverter {
 
   cluster(cluster, clusterId, reference) {
     const points = cluster.points ?? {};
+    const normalize = 1 / 255;
+    for (const point of Object.values(points)) {
+      const color = point.color;
+      color[0] *= normalize;
+      color[1] *= normalize;
+      color[2] *= normalize;
+    }
+
     return {
       id: clusterId,
       points,
@@ -112,7 +120,7 @@ export class DataConverter {
     const cluster = {id: clusterId, url: clusterId};
     const computed_rotation = shot.rotation;
     const height = camera.height;
-    const merge_id = shot.merge_cc.toString();
+    const merge_id = shot.merge_cc == null ? null : shot.merge_cc.toString();
     const exif_orientation = shot.orientation;
     const quality_score = 1;
     const width = camera.width;