Offline Object Loader

packages/objectloader/src/index.js

@@ -107,6 +107,69 @@ class ObjectLoader {
     }
   }
 
+  static createFromJSON(json) {
+    const start = performance.now()
+    const jsonObj = JSON.parse(json)
+    console.warn('JSON Parse Time -> ', performance.now() - start)
+
+    const rootObject = jsonObj[0]
+    const loader = new (class extends ObjectLoader {
+      constructor() {
+        super({
+          serverUrl: 'dummy',
+          streamId: 'dummy',
+          undefined,
+          objectId: rootObject.id
+        })
+
+        this.objectId = rootObject.id
+      }
+
+      async getRootObject() {
+        return rootObject
+      }
+
+      async getTotalObjectCount() {
+        return Object.keys(rootObject?.__closure || {}).length
+      }
+
+      async *getObjectIterator() {
+        const t0 = Date.now()
+        let count = 0
+        for await (const { id, obj } of this.getRawObjectIterator(jsonObj)) {
+          this.buffer[id] = obj
+          count += 1
+          yield obj
+        }
+        this.logger(`Loaded ${count} objects in: ${(Date.now() - t0) / 1000}`)
+      }
+
+      async *getRawObjectIterator(data) {
+        yield { id: data[0].id, obj: data[0] }
+
+        const rootObj = data[0]
+        if (!rootObj.__closure) return
+
+        // const childrenIds = Object.keys(rootObj.__closure)
+        //   .filter((id) => !id.includes('blob'))
+        //   .sort((a, b) => rootObj.__closure[a] - rootObj.__closure[b])
+
+        // for (const id of childrenIds) {
+        //   const obj = data.find((value) => value.id === id)
+        //   // Sleep 1 ms
+        //   await new Promise((resolve) => {
+        //     setTimeout(resolve, 1)
+        //   })
+        //   yield { id, obj }
+        // }
+        for (const item of data) {
+          yield { id: item.id, obj: item }
+        }
+      }
+    })()
+    return loader
+  }
+
   async asyncPause() {
     // Don't freeze the UI
     // while ( this.existingAsyncPause ) {
@@ -140,6 +203,11 @@ class ObjectLoader {
     return totalChildrenCount
   }
 
+  async getRootObject() {
+    const rootObjJson = await this.getRawRootObject()
+    return JSON.parse(rootObjJson)
+  }
+
   /**
    * Use this method to receive and construct the object. It will return the full, de-referenced and de-chunked original object.
    * @param {*} onProgress

packages/objectloader/types/index.d.ts

@@ -41,6 +41,8 @@ class ObjectLoader {
     }>
   })
 
+  static createFromJSON(input: string): ObjectLoader
+  async getRootObject(): Promise<SpeckleObject>
   async getTotalObjectCount(): Promise<number>
   async getAndConstructObject(
     onProgress: (e: { stage: ProgressStage; current: number; total: number }) => void

packages/viewer-sandbox/src/JSONSpeckleStream.ts

@@ -0,0 +1 @@
+export const JSONSpeckleStream = `[{"id":"1d060c69832c77c34ccc08e7955c132d","units":"m","__closure":{"0a135c3f8fed261b1f540c180b52f569":3,"1facfaaf1d3682707edd9ac20ef34e62":3,"7b7d7d6668f9d548d23ae5ea6d49446a":1,"a09d3408a23af65371a50ffad57eca90":2,"a153e0873c37979646e67eb006ff1819":4,"accbfcb15d2b5745afa12fa52b86e66e":4,"b06794644292db07098f51981aa76e65":4,"dd31d125d7941a1cbffc78566e9f59ba":4},"speckle_type":"Base","applicationId":null,"totalChildrenCount":8,"Alex Test Steam [ main @ 328bd99997 ]":{"referencedId":"7b7d7d6668f9d548d23ae5ea6d49446a","speckle_type":"reference"}},{"id": "0a135c3f8fed261b1f540c180b52f569", "area": null, "bbox": {"id": "21877013d5dcd72305f7a48fd40c68a5", "area": 0, "units": "m", "xSize": {"id": "b2b2f19648b14a25a70f493e34780b94", "end": 0, "start": 0, "units": "m", "speckle_type": "Objects.Primitive.Interval", "applicationId": null, "totalChildrenCount": 0}, "ySize": {"id": "b2b2f19648b14a25a70f493e34780b94", "end": 0, "start": 0, "units": "m", "speckle_type": "Objects.Primitive.Interval", "applicationId": null, "totalChildrenCount": 0}, "zSize": {"id": "b2b2f19648b14a25a70f493e34780b94", "end": 0, "start": 0, "units": "m", "speckle_type": "Objects.Primitive.Interval", "applicationId": null, "totalChildrenCount": 0}, "volume": 0, "basePlane": {"id": "e9886620f7f7ca05d7a4fb51e98b79a2", "xdir": {"x": 0, "y": 0, "z": 0, "id": "4409121ecaa12c2c28391835ece503a9", "units": "m", "speckle_type": "Objects.Geometry.Vector", "applicationId": null, "totalChildrenCount": 0}, "ydir": {"x": 0, "y": 0, "z": 0, "id": "4409121ecaa12c2c28391835ece503a9", "units": "m", "speckle_type": "Objects.Geometry.Vector", "applicationId": null, "totalChildrenCount": 0}, "units": "m", "normal": {"x": 0, "y": 0, "z": 0, "id": "4409121ecaa12c2c28391835ece503a9", "units": "m", "speckle_type": "Objects.Geometry.Vector", "applicationId": null, "totalChildrenCount": 0}, "origin": {"x": 0, "y": 0, "z": 0, "id": "83e6d188175d85974ec7062cdfa036f9", "units": "m", "speckle_type": "Objects.Geometry.Point", "applicationId": null, "totalChildrenCount": 0}, "speckle_type": "Objects.Geometry.Plane", "applicationId": null, "totalChildrenCount": 0}, "speckle_type": "Objects.Geometry.Box", "applicationId": null, "totalChildrenCount": 0}, "name": "Cube.001", "faces": [{"referencedId": "a153e0873c37979646e67eb006ff1819", "speckle_type": "reference"}], "units": "m", "colors": [{"referencedId": "b06794644292db07098f51981aa76e65", "speckle_type": "reference"}], "volume": null, "vertices": [{"referencedId": "accbfcb15d2b5745afa12fa52b86e66e", "speckle_type": "reference"}], "__closure": {"a153e0873c37979646e67eb006ff1819": 1, "accbfcb15d2b5745afa12fa52b86e66e": 1, "b06794644292db07098f51981aa76e65": 1}, "properties": {"name": "Cube.001", "transform": {"id": "999547f8a000fc8d955a5fb2d47c72bb", "units": "m", "value": [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1], "speckle_type": "Objects.Other.Transform", "applicationId": null, "totalChildrenCount": 0}}, "speckle_type": "Objects.Geometry.Mesh", "applicationId": null, "renderMaterial": {"id": "59b104f8217bceb1d1029ba4897c5693", "name": "Material", "units": "m", "diffuse": -1, "opacity": 1, "emissive": -16777216, "metalness": 0, "roughness": 0.5, "speckle_type": "Objects.Other.RenderMaterial", "applicationId": null, "totalChildrenCount": 0}, "textureCoordinates": [{"referencedId": "b06794644292db07098f51981aa76e65", "speckle_type": "reference"}], "totalChildrenCount": 3},{"id": "1facfaaf1d3682707edd9ac20ef34e62", "area": null, "bbox": {"id": "21877013d5dcd72305f7a48fd40c68a5", "area": 0, "units": "m", "xSize": {"id": "b2b2f19648b14a25a70f493e34780b94", "end": 0, "start": 0, "units": "m", "speckle_type": "Objects.Primitive.Interval", "applicationId": null, "totalChildrenCount": 0}, "ySize": {"id": "b2b2f19648b14a25a70f493e34780b94", "end": 0, "start": 0, "units": "m", "speckle_type": "Objects.Primitive.Interval", "applicationId": null, "totalChildrenCount": 0}, "zSize": {"id": "b2b2f19648b14a25a70f493e34780b94", "end": 0, "start": 0, "units": "m", "speckle_type": "Objects.Primitive.Interval", "applicationId": null, "totalChildrenCount": 0}, "volume": 0, "basePlane": {"id": "e9886620f7f7ca05d7a4fb51e98b79a2", "xdir": {"x": 0, "y": 0, "z": 0, "id": "4409121ecaa12c2c28391835ece503a9", "units": "m", "speckle_type": "Objects.Geometry.Vector", "applicationId": null, "totalChildrenCount": 0}, "ydir": {"x": 0, "y": 0, "z": 0, "id": "4409121ecaa12c2c28391835ece503a9", "units": "m", "speckle_type": "Objects.Geometry.Vector", "applicationId": null, "totalChildrenCount": 0}, "units": "m", "normal": {"x": 0, "y": 0, "z": 0, "id": "4409121ecaa12c2c28391835ece503a9", "units": "m", "speckle_type": "Objects.Geometry.Vector", "applicationId": null, "totalChildrenCount": 0}, "origin": {"x": 0, "y": 0, "z": 0, "id": "83e6d188175d85974ec7062cdfa036f9", "units": "m", "speckle_type": "Objects.Geometry.Point", "applicationId": null, "totalChildrenCount": 0}, "speckle_type": "Objects.Geometry.Plane", "applicationId": null, "totalChildrenCount": 0}, "speckle_type": "Objects.Geometry.Box", "applicationId": null, "totalChildrenCount": 0}, "name": "Cube", "faces": [{"referencedId": "a153e0873c37979646e67eb006ff1819", "speckle_type": "reference"}], "units": "m", "colors": [{"referencedId": "b06794644292db07098f51981aa76e65", "speckle_type": "reference"}], "volume": null, "vertices": [{"referencedId": "dd31d125d7941a1cbffc78566e9f59ba", "speckle_type": "reference"}], "__closure": {"a153e0873c37979646e67eb006ff1819": 1, "b06794644292db07098f51981aa76e65": 1, "dd31d125d7941a1cbffc78566e9f59ba": 1}, "properties": {"name": "Cube", "transform": {"id": "999547f8a000fc8d955a5fb2d47c72bb", "units": "m", "value": [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1], "speckle_type": "Objects.Other.Transform", "applicationId": null, "totalChildrenCount": 0}}, "speckle_type": "Objects.Geometry.Mesh", "applicationId": null, "renderMaterial": {"id": "59b104f8217bceb1d1029ba4897c5693", "name": "Material", "units": "m", "diffuse": -1, "opacity": 1, "emissive": -16777216, "metalness": 0, "roughness": 0.5, "speckle_type": "Objects.Other.RenderMaterial", "applicationId": null, "totalChildrenCount": 0}, "textureCoordinates": [{"referencedId": "b06794644292db07098f51981aa76e65", "speckle_type": "reference"}], "totalChildrenCount": 3},{"id": "7b7d7d6668f9d548d23ae5ea6d49446a", "units": "m", "__closure": {"0a135c3f8fed261b1f540c180b52f569": 2, "1facfaaf1d3682707edd9ac20ef34e62": 2, "a09d3408a23af65371a50ffad57eca90": 1, "a153e0873c37979646e67eb006ff1819": 3, "accbfcb15d2b5745afa12fa52b86e66e": 3, "b06794644292db07098f51981aa76e65": 3, "dd31d125d7941a1cbffc78566e9f59ba": 3}, "Collection": {"referencedId": "a09d3408a23af65371a50ffad57eca90", "speckle_type": "reference"}, "speckle_type": "Base", "applicationId": null, "totalChildrenCount": 7},{"id": "a09d3408a23af65371a50ffad57eca90", "units": "m", "@objects": [{"referencedId": "1facfaaf1d3682707edd9ac20ef34e62", "speckle_type": "reference"}, {"referencedId": "0a135c3f8fed261b1f540c180b52f569", "speckle_type": "reference"}], "__closure": {"0a135c3f8fed261b1f540c180b52f569": 1, "1facfaaf1d3682707edd9ac20ef34e62": 1, "a153e0873c37979646e67eb006ff1819": 2, "accbfcb15d2b5745afa12fa52b86e66e": 2, "b06794644292db07098f51981aa76e65": 2, "dd31d125d7941a1cbffc78566e9f59ba": 2}, "speckle_type": "Base", "applicationId": null, "totalChildrenCount": 6},{"id": "a153e0873c37979646e67eb006ff1819", "data": [1, 10, 12, 14, 8, 1, 9, 15, 18, 16, 1, 17, 19, 22, 20, 1, 21, 23, 13, 11, 1, 2, 6, 4, 0, 1, 7, 3, 1, 5], "units": "m", "speckle_type": "Speckle.Core.Models.DataChunk", "applicationId": null, "totalChildrenCount": 0},{"id": "accbfcb15d2b5745afa12fa52b86e66e", "data": [-3.1788320541381836, -1, -1, -3.1788320541381836, -1, 1, -3.1788320541381836, 1, -1, -3.1788320541381836, 1, 1, -1.1788320541381836, -1, -1, -1.1788320541381836, -1, 1, -1.1788320541381836, 1, -1, -1.1788320541381836, 1, 1, -3.1788320541381836, 1, -1, -3.1788320541381836, 1, -1, -3.1788320541381836, -1, -1, -3.1788320541381836, -1, -1, -3.1788320541381836, -1, 1, -3.1788320541381836, -1, 1, -3.1788320541381836, 1, 1, -3.1788320541381836, 1, 1, -1.1788320541381836, 1, -1, -1.1788320541381836, 1, -1, -1.1788320541381836, 1, 1, -1.1788320541381836, 1, 1, -1.1788320541381836, -1, -1, -1.1788320541381836, -1, -1, -1.1788320541381836, -1, 1, -1.1788320541381836, -1, 1], "units": "m", "speckle_type": "Speckle.Core.Models.DataChunk", "applicationId": null, "totalChildrenCount": 0},{"id": "b06794644292db07098f51981aa76e65", "data": [], "units": "m", "speckle_type": "Speckle.Core.Models.DataChunk", "applicationId": null, "totalChildrenCount": 0},{"id": "dd31d125d7941a1cbffc78566e9f59ba", "data": [1.2236602306365967, -1, -1, 1.2236602306365967, -1, 1, 1.2236602306365967, 1, -1, 1.2236602306365967, 1, 1, 3.2236602306365967, -1, -1, 3.2236602306365967, -1, 1, 3.2236602306365967, 1, -1, 3.2236602306365967, 1, 1, 1.2236602306365967, 1, -1, 1.2236602306365967, 1, -1, 1.2236602306365967, -1, -1, 1.2236602306365967, -1, -1, 1.2236602306365967, -1, 1, 1.2236602306365967, -1, 1, 1.2236602306365967, 1, 1, 1.2236602306365967, 1, 1, 3.2236602306365967, 1, -1, 3.2236602306365967, 1, -1, 3.2236602306365967, 1, 1, 3.2236602306365967, 1, 1, 3.2236602306365967, -1, -1, 3.2236602306365967, -1, -1, 3.2236602306365967, -1, 1, 3.2236602306365967, -1, 1], "units": "m", "speckle_type": "Speckle.Core.Models.DataChunk", "applicationId": null, "totalChildrenCount": 0}]`

packages/viewer-sandbox/src/Sandbox.ts

@@ -16,6 +16,7 @@ import {
   OutputPass,
   Pipeline,
   SectionTool,
+  SpeckleOfflineLoader,
   SpeckleRenderer,
   SpeckleStandardMaterial,
   TAAPipeline,
@@ -1292,4 +1293,16 @@ export default class Sandbox {
     }
     localStorage.setItem('last-load-url', url)
   }
+
+  public async loadJSON(json: string) {
+    const loader = new SpeckleOfflineLoader(this.viewer.getWorldTree(), json)
+    loader.on(LoaderEvent.LoadCancelled, (resource: string) => {
+      console.warn(`Resource ${resource} loading was canceled`)
+    })
+    loader.on(LoaderEvent.LoadWarning, (arg: { message: string }) => {
+      console.error(`Loader warning: ${arg.message}`)
+    })
+
+    void this.viewer.loadObject(loader, true)
+  }
 }

packages/viewer-sandbox/src/main.ts

@@ -19,10 +19,11 @@ import {
 import { SectionTool } from '@speckle/viewer'
 import { SectionOutlines } from '@speckle/viewer'
 import { ViewModesKeys } from './Extensions/ViewModesKeys'
+import { JSONSpeckleStream } from './JSONSpeckleStream'
 import { BoxSelection } from './Extensions/BoxSelection'
 import { ExtendedSelection } from './Extensions/ExtendedSelection'
 
-const createViewer = async (containerName: string, stream: string) => {
+const createViewer = async (containerName: string, _stream: string) => {
   const container = document.querySelector<HTMLElement>(containerName)
 
   const controlsContainer = document.querySelector<HTMLElement>(
@@ -103,15 +104,16 @@ const createViewer = async (containerName: string, stream: string) => {
   sandbox.makeDiffUI()
   sandbox.makeMeasurementsUI()
 
-  await sandbox.loadUrl(stream)
+  // await sandbox.loadUrl(_stream)
+  await sandbox.loadJSON(JSONSpeckleStream)
 }
 
 const getStream = () => {
   return (
     // prettier-ignore
     // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D'
     // Revit sample house (good for bim-like stuff with many display meshes)
-    // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
+    'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
     // 'https://latest.speckle.systems/streams/c1faab5c62/commits/ab1a1ab2b6'
     // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
     // 'https://latest.speckle.systems/streams/58b5648c4d/commits/60371ecb2d'
@@ -451,13 +453,15 @@ const getStream = () => {
     // Far away house section tool
     // 'https://app.speckle.systems/projects/817c4e8daa/models/f0601ef5f9@80db5ff26a'
 
+    // 'https://app.speckle.systems/projects/00a5c443d6/models/de56edf901'
+    // 'https://latest.speckle.systems/projects/126cd4b7bb/models/49874f87a2ddd370bd2bf46b68c3660d'
     // Perfectly flat
     // 'https://app.speckle.systems/projects/344f803f81/models/5582ab673e'
 
     // 'https://speckle.xyz/streams/27e89d0ad6/commits/5ed4b74252'
 
     // DUI3 Mesh Colors
-    'https://app.speckle.systems/projects/93200a735d/models/cbacd3eaeb@344a397239'
+    // 'https://app.speckle.systems/projects/93200a735d/models/cbacd3eaeb@344a397239'
 
     // Instance toilets
     // 'https://app.speckle.systems/projects/e89b61b65c/models/2a0995f124'

packages/viewer/src/index.ts

@@ -129,6 +129,7 @@ import {
   FilterMaterialOptions,
   FilterMaterialType
 } from './modules/materials/Materials.js'
+import { SpeckleOfflineLoader } from './modules/loaders/Speckle/SpeckleOfflineLoader.js'
 import { AccelerationStructure } from './modules/objects/AccelerationStructure.js'
 import { TopLevelAccelerationStructure } from './modules/objects/TopLevelAccelerationStructure.js'
 import { ViewModeEvent, ViewModeEventPayload } from './modules/extensions/ViewModes.js'
@@ -222,6 +223,7 @@ export {
   FilterMaterial,
   FilterMaterialType,
   FilterMaterialOptions,
+  SpeckleOfflineLoader,
   NOT_INTERSECTED,
   INTERSECTED,
   CONTAINED,

packages/viewer/src/modules/loaders/Speckle/SpeckleLoader.ts

@@ -30,6 +30,29 @@ export class SpeckleLoader extends Loader {
   ) {
     super(resource, resourceData)
     this.tree = targetTree
+    try {
+      this.loader = this.initObjectLoader(
+        resource,
+        authToken,
+        enableCaching,
+        resourceData
+      )
+    } catch (e) {
+      Logger.error(e)
+      return
+    }
+
+    this.converter = new SpeckleConverter(this.loader, this.tree)
+  }
+
+  protected initObjectLoader(
+    resource: string,
+    authToken?: string,
+    enableCaching?: boolean,
+    resourceData?: string | ArrayBuffer
+  ): ObjectLoader {
+    resourceData
+
     let token = undefined
     try {
       token = authToken || (localStorage.getItem('AuthToken') as string | undefined)
@@ -58,16 +81,14 @@ export class SpeckleLoader extends Loader {
     const streamId = segments[2]
     const objectId = segments[4]
 
-    this.loader = new ObjectLoader({
+    return new ObjectLoader({
       serverUrl,
       token,
       streamId,
       objectId,
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       options: { enableCaching, customLogger: (Logger as any).log }
     })
-
-    this.converter = new SpeckleConverter(this.loader, this.tree)
   }
 
   public async load(): Promise<boolean> {
@@ -78,18 +99,18 @@ export class SpeckleLoader extends Loader {
     let viewerLoads = 0
     let firstObjectPromise = null
 
-    Logger.warn('Downloading object ', this._resource)
+    Logger.warn('Downloading object ', this.resource)
 
     const pause = new AsyncPause()
 
     for await (const obj of this.loader.getObjectIterator()) {
       if (this.isCancelled) {
-        this.emit(LoaderEvent.LoadCancelled, this._resource)
+        this.emit(LoaderEvent.LoadCancelled, this.resource)
         return Promise.resolve(false)
       }
       if (first) {
         firstObjectPromise = this.converter.traverse(
-          this._resource,
+          this.resource,
           obj as SpeckleObject,
           async () => {
             viewerLoads++
@@ -104,7 +125,7 @@ export class SpeckleLoader extends Loader {
       current++
       this.emit(LoaderEvent.LoadProgress, {
         progress: current / (total + 1),
-        id: this._resource
+        id: this.resource
       })
     }
 
@@ -113,15 +134,15 @@ export class SpeckleLoader extends Loader {
     }
 
     Logger.warn(
-      `Finished converting object ${this._resource} in ${
+      `Finished converting object ${this.resource} in ${
         (performance.now() - start) / 1000
       } seconds. Node count: ${this.tree.nodeCount}`
     )
 
     if (viewerLoads === 0) {
-      Logger.warn(`Viewer: no 3d objects found in object ${this._resource}`)
+      Logger.warn(`Viewer: no 3d objects found in object ${this.resource}`)
       this.emit(LoaderEvent.LoadWarning, {
-        message: `No displayable objects found in object ${this._resource}.`
+        message: `No displayable objects found in object ${this.resource}.`
       })
     }
     if (this.isCancelled) {
@@ -134,7 +155,7 @@ export class SpeckleLoader extends Loader {
     const t0 = performance.now()
     const geometryConverter = new SpeckleGeometryConverter()
 
-    const renderTree = this.tree.getRenderTree(this._resource)
+    const renderTree = this.tree.getRenderTree(this.resource)
     if (!renderTree) return Promise.resolve(false)
     const p = renderTree.buildRenderTree(geometryConverter)
 

packages/viewer/src/modules/loaders/Speckle/SpeckleOfflineLoader.ts

@@ -0,0 +1,30 @@
+import ObjectLoader from '@speckle/objectloader'
+import { SpeckleLoader } from './SpeckleLoader.js'
+import { WorldTree } from '../../tree/WorldTree.js'
+import Logger from '../../utils/Logger.js'
+
+export class SpeckleOfflineLoader extends SpeckleLoader {
+  constructor(targetTree: WorldTree, resourceData: string, resourceId?: string) {
+    super(targetTree, resourceId || '', undefined, undefined, resourceData)
+  }
+
+  protected initObjectLoader(
+    _resource: string,
+    _authToken?: string,
+    _enableCaching?: boolean,
+    resourceData?: string | ArrayBuffer
+  ): ObjectLoader {
+    return ObjectLoader.createFromJSON(resourceData as string)
+  }
+
+  public async load(): Promise<boolean> {
+    const rootObject = await this.loader.getRootObject()
+    if (!rootObject && this._resource) {
+      Logger.error('No root id set!')
+      return false
+    }
+    /** If not id is provided, we make one up based on the root object id */
+    this._resource = this._resource || `/json/${rootObject.id as string}`
+    return super.load()
+  }
+}