PassReader updates for View Modes (#3828)

* feat(viewer-lib): Updates to the PassReader extension along with underlying viewer library updates Implemented reading framebuffer contents for framebuffers with multiple attachements. The current version of three.js that we are using does not support this. Because we still need to drag along WebGL 1.0 support, only attachement 0 can be read for now, which does not bother us. DepthNormalPass now specifies it's MRT output target as the outputTarget PassReader's read function is now overloaded and it can take a pass name as a string or a GPass | GPass[] Had to add a small type augmentation since the current version of types-three library does a poor job when it comes to WebGLMultipleRenderTargets Updated PassReader extension in frontend and updated the call to read in order to make sure depth reading works in other view modes that write depth. The only view mode that does not draw to depth is Shaded mode * fix(viewer-lib): Fixed the classic sandbox compile error * fix(frontend-2): Updated depth reading to work with MRT depth from our view mode pipelines
specklesystems · Jan 16, 2025 · b4deade · b4deade
1 parent 35bc691
commit b4deade
Show file tree

Hide file tree

Showing 8 changed files with 549 additions and 45 deletions.
diff --git a/packages/frontend-2/components/viewer/gendo/Panel.vue b/packages/frontend-2/components/viewer/gendo/Panel.vue
@@ -158,9 +158,13 @@ const formattedResetDate = computed(() => {
 
 const enqueMagic = async () => {
   isLoading.value = true
+  const pass = [
+    ...viewerInstance.getRenderer().pipeline.getPass('DEPTH'),
+    ...viewerInstance.getRenderer().pipeline.getPass('DEPTH-NORMAL')
+  ]
   const [depthData, width, height] = await viewerInstance
     .getExtension(PassReader)
-    .read('DEPTH')
+    .read(pass)
   const screenshot = PassReader.toBase64(
     PassReader.decodeDepth(depthData),
     width,

diff --git a/packages/frontend-2/lib/viewer/extensions/PassReader.ts b/packages/frontend-2/lib/viewer/extensions/PassReader.ts
@@ -1,4 +1,4 @@
-import type { SpeckleRenderer } from '@speckle/viewer'
+import type { GPass, SpeckleRenderer } from '@speckle/viewer'
 import { Extension } from '@speckle/viewer'
 import type { WebGLRenderTarget } from 'three'
 import { Vector3, Vector4 } from 'three'
@@ -11,21 +11,34 @@ export class PassReader extends Extension {
     | ((arg: [Uint8ClampedArray, number, number]) => void)
     | null = null
 
-  public async read(passName: string): Promise<[Uint8ClampedArray, number, number]> {
+  public async read(pass: string): Promise<[Uint8ClampedArray, number, number]>
+  public async read(pass: GPass | GPass[]): Promise<[Uint8ClampedArray, number, number]>
+
+  public async read(
+    pass: string | GPass | GPass[]
+  ): Promise<[Uint8ClampedArray, number, number]> {
     return new Promise<[Uint8ClampedArray, number, number]>((resolve, reject) => {
       const renderer: SpeckleRenderer = this.viewer.getRenderer()
+      let passes: GPass[]
+      if (typeof pass === 'string') passes = renderer.pipeline.getPass(pass)
+      else if (Array.isArray(pass)) passes = pass
+      else passes = [pass]
 
-      const depthPass = renderer.pipeline.getPass(passName)[0]
+      if (!passes || !passes.length) {
+        reject(`Could not read from pass`)
+        return
+      }
+      const validPass = passes.find((pass: GPass) => this.hasFramebuffer(pass))
 
-      if (!depthPass) {
-        reject(`Pipeline does not have a ${passName} pass`)
+      if (!validPass) {
+        reject(`Requested pass does not have a valid framebuffer`)
         return
       }
 
-      this.renderTarget = depthPass.outputTarget
+      this.renderTarget = validPass.outputTarget
 
       if (!this.renderTarget) {
-        reject('Pass does not have a render target assigned')
+        reject('Requested Pass does not have a render target assigned')
         return
       }
 
@@ -38,19 +51,24 @@ export class PassReader extends Extension {
     })
   }
 
+  protected hasFramebuffer(pass: GPass) {
+    const renderer = this.viewer.getRenderer().renderer
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    return renderer.properties.get(pass.outputTarget).__webglFramebuffer !== undefined
+  }
+
   public onRender(): void {
     if (!this.needsRead || !this.renderTarget) return
 
-    this.viewer
-      .getRenderer()
-      .renderer.readRenderTargetPixels(
-        this.renderTarget,
-        0,
-        0,
-        this.renderTarget.width,
-        this.renderTarget.height,
-        this.outputBuffer
-      )
+    const renderer = this.viewer.getRenderer().renderer
+    renderer.readRenderTargetPixels(
+      this.renderTarget,
+      0,
+      0,
+      this.renderTarget.width,
+      this.renderTarget.height,
+      this.outputBuffer
+    )
 
     if (this.readbackExecutor)
       this.readbackExecutor([

diff --git a/packages/viewer-sandbox/src/Extensions/PassReader.ts b/packages/viewer-sandbox/src/Extensions/PassReader.ts
@@ -1,4 +1,4 @@
-import type { SpeckleRenderer } from '@speckle/viewer'
+import type { GPass, SpeckleRenderer } from '@speckle/viewer'
 import { Extension } from '@speckle/viewer'
 import type { WebGLRenderTarget } from 'three'
 import { Vector3, Vector4 } from 'three'
@@ -11,21 +11,34 @@ export class PassReader extends Extension {
     | ((arg: [Uint8ClampedArray, number, number]) => void)
     | null = null
 
-  public async read(passName: string): Promise<[Uint8ClampedArray, number, number]> {
+  public async read(pass: string): Promise<[Uint8ClampedArray, number, number]>
+  public async read(pass: GPass | GPass[]): Promise<[Uint8ClampedArray, number, number]>
+
+  public async read(
+    pass: string | GPass | GPass[]
+  ): Promise<[Uint8ClampedArray, number, number]> {
     return new Promise<[Uint8ClampedArray, number, number]>((resolve, reject) => {
       const renderer: SpeckleRenderer = this.viewer.getRenderer()
+      let passes: GPass[]
+      if (typeof pass === 'string') passes = renderer.pipeline.getPass(pass)
+      else if (Array.isArray(pass)) passes = pass
+      else passes = [pass]
 
-      const depthPass = renderer.pipeline.getPass(passName)[0]
+      if (!passes || !passes.length) {
+        reject(`Could not read from pass`)
+        return
+      }
+      const validPass = passes.find((pass: GPass) => this.hasFramebuffer(pass))
 
-      if (!depthPass) {
-        reject(`Pipeline does not have a ${passName} pass`)
+      if (!validPass) {
+        reject(`Requested pass does not have a valid framebuffer`)
         return
       }
 
-      this.renderTarget = depthPass.outputTarget
+      this.renderTarget = validPass.outputTarget
 
       if (!this.renderTarget) {
-        reject('Pass does not have a render target assigned')
+        reject('Requested Pass does not have a render target assigned')
         return
       }
 
@@ -38,19 +51,24 @@ export class PassReader extends Extension {
     })
   }
 
+  protected hasFramebuffer(pass: GPass) {
+    const renderer = this.viewer.getRenderer().renderer
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    return renderer.properties.get(pass.outputTarget).__webglFramebuffer !== undefined
+  }
+
   public onRender(): void {
     if (!this.needsRead || !this.renderTarget) return
 
-    this.viewer
-      .getRenderer()
-      .renderer.readRenderTargetPixels(
-        this.renderTarget,
-        0,
-        0,
-        this.renderTarget.width,
-        this.renderTarget.height,
-        this.outputBuffer
-      )
+    const renderer = this.viewer.getRenderer().renderer
+    renderer.readRenderTargetPixels(
+      this.renderTarget,
+      0,
+      0,
+      this.renderTarget.width,
+      this.renderTarget.height,
+      this.outputBuffer
+    )
 
     if (this.readbackExecutor)
       this.readbackExecutor([
@@ -89,10 +107,14 @@ export class PassReader extends Extension {
     return buffer
   }
 
-  public static toBase64(buffer: Uint8ClampedArray, width: number, height: number) {
+  public static toBase64(
+    buffer: Uint8ClampedArray,
+    width: number,
+    height: number
+  ): string {
     const canvas = document.createElement('canvas')
     const ctx = canvas.getContext('2d')
-    if (!ctx) return
+    if (!ctx) return ''
     canvas.width = width
     canvas.height = height
 

diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts
@@ -500,10 +500,18 @@ export default class Sandbox {
       title: 'Screenshot'
     })
     screenshot.on('click', async () => {
-      // console.warn(await this.viewer.screenshot())
-      this.viewer
-        .getExtension(FilteringExtension)
-        .hideObjects(['1facfaaf1d3682707edd9ac20ef34e62'])
+      console.warn(await this.viewer.screenshot())
+
+      /** Read depth */
+      // const pass = [
+      //   ...this.viewer.getRenderer().pipeline.getPass('DEPTH'),
+      //   ...this.viewer.getRenderer().pipeline.getPass('DEPTH-NORMAL')
+      // ]
+      // const [depthData, width, height] = await this.viewer
+      //   .getExtension(PassReader)
+      //   .read(pass)
+
+      // console.log(PassReader.toBase64(PassReader.decodeDepth(depthData), width, height))
     })
 
     const rotate = this.tabs.pages[0].addButton({

diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts
@@ -21,6 +21,7 @@ import { SectionTool } from '@speckle/viewer'
 import { SectionOutlines } from '@speckle/viewer'
 import { ViewModesKeys } from './Extensions/ViewModesKeys'
 import { BoxSelection } from './Extensions/BoxSelection'
+import { PassReader } from './Extensions/PassReader'
 
 const createViewer = async (containerName: string, _stream: string) => {
   const container = document.querySelector<HTMLElement>(containerName)
@@ -56,6 +57,7 @@ const createViewer = async (containerName: string, _stream: string) => {
   viewer.createExtension(ViewModesKeys)
   const boxSelect = viewer.createExtension(BoxSelection)
   boxSelect.realtimeSelection = false
+  viewer.createExtension(PassReader)
   // const rotateCamera = viewer.createExtension(RotateCamera)
   cameraController // use it
   selection // use it
@@ -111,7 +113,7 @@ const getStream = () => {
     // prettier-ignore
     // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D'
     // Revit sample house (good for bim-like stuff with many display meshes)
-    // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
+    'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
     // 'https://latest.speckle.systems/streams/c1faab5c62/commits/ab1a1ab2b6'
     // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
     // 'https://latest.speckle.systems/streams/58b5648c4d/commits/60371ecb2d'
@@ -466,7 +468,7 @@ const getStream = () => {
     // Instance toilets
     // 'https://app.speckle.systems/projects/e89b61b65c/models/2a0995f124'
 
-    'https://latest.speckle.systems/projects/3fe1880c36/models/65bb4287a8'
+    // 'https://latest.speckle.systems/projects/3fe1880c36/models/65bb4287a8'
   )
 }