feat(viewer-lib): Updates to the PassReader extension along with unde…

…rlying viewer library updates

Implemented reading framebuffer contents for framebuffers with multiple attachements. The current version of three.js that we are using does not support this. Because we still need to drag along WebGL 1.0 support, only attachement 0 can be read for now, which does not bother us.
DepthNormalPass now specifies it's MRT output target as the outputTarget
PassReader's read function is now overloaded and it can take a pass name as a string or a GPass | GPass[]
Had to add a small type augmentation since the current version of types-three library does a poor job when it comes to WebGLMultipleRenderTargets
Updated PassReader extension in frontend and updated the call to read in order to make sure depth reading works in other view modes that write depth. The only view mode that does not draw to depth is Shaded mode

packages/frontend-2/lib/viewer/extensions/PassReader.ts

@@ -1,4 +1,4 @@
-import type { SpeckleRenderer } from '@speckle/viewer'
+import type { GPass, SpeckleRenderer } from '@speckle/viewer'
 import { Extension } from '@speckle/viewer'
 import type { WebGLRenderTarget } from 'three'
 import { Vector3, Vector4 } from 'three'
@@ -11,21 +11,34 @@ export class PassReader extends Extension {
     | ((arg: [Uint8ClampedArray, number, number]) => void)
     | null = null
 
-  public async read(passName: string): Promise<[Uint8ClampedArray, number, number]> {
+  public async read(pass: string): Promise<[Uint8ClampedArray, number, number]>
+  public async read(pass: GPass | GPass[]): Promise<[Uint8ClampedArray, number, number]>
+
+  public async read(
+    pass: string | GPass | GPass[]
+  ): Promise<[Uint8ClampedArray, number, number]> {
     return new Promise<[Uint8ClampedArray, number, number]>((resolve, reject) => {
       const renderer: SpeckleRenderer = this.viewer.getRenderer()
+      let passes: GPass[]
+      if (typeof pass === 'string') passes = renderer.pipeline.getPass(pass)
+      else if (Array.isArray(pass)) passes = pass
+      else passes = [pass]
 
-      const depthPass = renderer.pipeline.getPass(passName)[0]
+      if (!passes || !passes.length) {
+        reject(`Could not read from pass`)
+        return
+      }
+      const validPass = passes.find((pass: GPass) => this.hasFramebuffer(pass))
 
-      if (!depthPass) {
-        reject(`Pipeline does not have a ${passName} pass`)
+      if (!validPass) {
+        reject(`Requested pass does not have a valid framebuffer`)
         return
       }
 
-      this.renderTarget = depthPass.outputTarget
+      this.renderTarget = validPass.outputTarget
 
       if (!this.renderTarget) {
-        reject('Pass does not have a render target assigned')
+        reject('Requested Pass does not have a render target assigned')
         return
       }
 
@@ -38,19 +51,24 @@ export class PassReader extends Extension {
     })
   }
 
+  protected hasFramebuffer(pass: GPass) {
+    const renderer = this.viewer.getRenderer().renderer
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    return renderer.properties.get(pass.outputTarget).__webglFramebuffer !== undefined
+  }
+
   public onRender(): void {
     if (!this.needsRead || !this.renderTarget) return
 
-    this.viewer
-      .getRenderer()
-      .renderer.readRenderTargetPixels(
-        this.renderTarget,
-        0,
-        0,
-        this.renderTarget.width,
-        this.renderTarget.height,
-        this.outputBuffer
-      )
+    const renderer = this.viewer.getRenderer().renderer
+    renderer.readRenderTargetPixels(
+      this.renderTarget,
+      0,
+      0,
+      this.renderTarget.width,
+      this.renderTarget.height,
+      this.outputBuffer
+    )
 
     if (this.readbackExecutor)
       this.readbackExecutor([

packages/viewer-sandbox/src/Extensions/PassReader.ts

@@ -1,4 +1,4 @@
-import type { SpeckleRenderer } from '@speckle/viewer'
+import type { GPass, SpeckleRenderer } from '@speckle/viewer'
 import { Extension } from '@speckle/viewer'
 import type { WebGLRenderTarget } from 'three'
 import { Vector3, Vector4 } from 'three'
@@ -11,21 +11,34 @@ export class PassReader extends Extension {
     | ((arg: [Uint8ClampedArray, number, number]) => void)
     | null = null
 
-  public async read(passName: string): Promise<[Uint8ClampedArray, number, number]> {
+  public async read(pass: string): Promise<[Uint8ClampedArray, number, number]>
+  public async read(pass: GPass | GPass[]): Promise<[Uint8ClampedArray, number, number]>
+
+  public async read(
+    pass: string | GPass | GPass[]
+  ): Promise<[Uint8ClampedArray, number, number]> {
     return new Promise<[Uint8ClampedArray, number, number]>((resolve, reject) => {
       const renderer: SpeckleRenderer = this.viewer.getRenderer()
+      let passes: GPass[]
+      if (typeof pass === 'string') passes = renderer.pipeline.getPass(pass)
+      else if (Array.isArray(pass)) passes = pass
+      else passes = [pass]
 
-      const depthPass = renderer.pipeline.getPass(passName)[0]
+      if (!passes || !passes.length) {
+        reject(`Could not read from pass`)
+        return
+      }
+      const validPass = passes.find((pass: GPass) => this.hasFramebuffer(pass))
 
-      if (!depthPass) {
-        reject(`Pipeline does not have a ${passName} pass`)
+      if (!validPass) {
+        reject(`Requested pass does not have a valid framebuffer`)
         return
       }
 
-      this.renderTarget = depthPass.outputTarget
+      this.renderTarget = validPass.outputTarget
 
       if (!this.renderTarget) {
-        reject('Pass does not have a render target assigned')
+        reject('Requested Pass does not have a render target assigned')
         return
       }
 
@@ -38,19 +51,24 @@ export class PassReader extends Extension {
     })
   }
 
+  protected hasFramebuffer(pass: GPass) {
+    const renderer = this.viewer.getRenderer().renderer
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    return renderer.properties.get(pass.outputTarget).__webglFramebuffer !== undefined
+  }
+
   public onRender(): void {
     if (!this.needsRead || !this.renderTarget) return
 
-    this.viewer
-      .getRenderer()
-      .renderer.readRenderTargetPixels(
-        this.renderTarget,
-        0,
-        0,
-        this.renderTarget.width,
-        this.renderTarget.height,
-        this.outputBuffer
-      )
+    const renderer = this.viewer.getRenderer().renderer
+    renderer.readRenderTargetPixels(
+      this.renderTarget,
+      0,
+      0,
+      this.renderTarget.width,
+      this.renderTarget.height,
+      this.outputBuffer
+    )
 
     if (this.readbackExecutor)
       this.readbackExecutor([
@@ -89,10 +107,14 @@ export class PassReader extends Extension {
     return buffer
   }
 
-  public static toBase64(buffer: Uint8ClampedArray, width: number, height: number) {
+  public static toBase64(
+    buffer: Uint8ClampedArray,
+    width: number,
+    height: number
+  ): string {
     const canvas = document.createElement('canvas')
     const ctx = canvas.getContext('2d')
-    if (!ctx) return
+    if (!ctx) return ''
     canvas.width = width
     canvas.height = height
 

packages/viewer-sandbox/src/Sandbox.ts

@@ -57,6 +57,7 @@ import Bright from '../assets/hdri/Bright.png'
 import { Euler, Vector3, Box3, Color, LinearFilter } from 'three'
 import { GeometryType } from '@speckle/viewer'
 import { MeshBatch } from '@speckle/viewer'
+import { PassReader } from './Extensions/PassReader'
 
 export default class Sandbox {
   private viewer: Viewer
@@ -500,10 +501,18 @@ export default class Sandbox {
       title: 'Screenshot'
     })
     screenshot.on('click', async () => {
-      // console.warn(await this.viewer.screenshot())
-      this.viewer
-        .getExtension(FilteringExtension)
-        .hideObjects(['1facfaaf1d3682707edd9ac20ef34e62'])
+      console.warn(await this.viewer.screenshot())
+
+      /** Read depth */
+      // const pass = [
+      //   ...this.viewer.getRenderer().pipeline.getPass('DEPTH'),
+      //   ...this.viewer.getRenderer().pipeline.getPass('DEPTH-NORMAL')
+      // ]
+      // const [depthData, width, height] = await this.viewer
+      //   .getExtension(PassReader)
+      //   .read(pass)
+
+      // console.log(PassReader.toBase64(PassReader.decodeDepth(depthData), width, height))
     })
 
     const rotate = this.tabs.pages[0].addButton({

packages/viewer-sandbox/src/main.ts

@@ -21,6 +21,7 @@ import { SectionTool } from '@speckle/viewer'
 import { SectionOutlines } from '@speckle/viewer'
 import { ViewModesKeys } from './Extensions/ViewModesKeys'
 import { BoxSelection } from './Extensions/BoxSelection'
+import { PassReader } from './Extensions/PassReader'
 
 const createViewer = async (containerName: string, _stream: string) => {
   const container = document.querySelector<HTMLElement>(containerName)
@@ -56,6 +57,7 @@ const createViewer = async (containerName: string, _stream: string) => {
   viewer.createExtension(ViewModesKeys)
   const boxSelect = viewer.createExtension(BoxSelection)
   boxSelect.realtimeSelection = false
+  viewer.createExtension(PassReader)
   // const rotateCamera = viewer.createExtension(RotateCamera)
   cameraController // use it
   selection // use it
@@ -111,7 +113,7 @@ const getStream = () => {
     // prettier-ignore
     // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D'
     // Revit sample house (good for bim-like stuff with many display meshes)
-    // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
+    'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
     // 'https://latest.speckle.systems/streams/c1faab5c62/commits/ab1a1ab2b6'
     // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
     // 'https://latest.speckle.systems/streams/58b5648c4d/commits/60371ecb2d'
@@ -466,7 +468,7 @@ const getStream = () => {
     // Instance toilets
     // 'https://app.speckle.systems/projects/e89b61b65c/models/2a0995f124'
 
-    'https://latest.speckle.systems/projects/3fe1880c36/models/65bb4287a8'
+    // 'https://latest.speckle.systems/projects/3fe1880c36/models/65bb4287a8'
   )
 }