PassReader updates for View Modes

packages/frontend-2/lib/viewer/extensions/PassReader.ts

@@ -1,4 +1,4 @@
-import type { SpeckleRenderer } from '@speckle/viewer'
+import type { GPass, SpeckleRenderer } from '@speckle/viewer'
 import { Extension } from '@speckle/viewer'
 import type { WebGLRenderTarget } from 'three'
 import { Vector3, Vector4 } from 'three'
@@ -11,21 +11,34 @@ export class PassReader extends Extension {
     | ((arg: [Uint8ClampedArray, number, number]) => void)
     | null = null
 
-  public async read(passName: string): Promise<[Uint8ClampedArray, number, number]> {
+  public async read(pass: string): Promise<[Uint8ClampedArray, number, number]>
+  public async read(pass: GPass | GPass[]): Promise<[Uint8ClampedArray, number, number]>
+
+  public async read(
+    pass: string | GPass | GPass[]
+  ): Promise<[Uint8ClampedArray, number, number]> {
     return new Promise<[Uint8ClampedArray, number, number]>((resolve, reject) => {
       const renderer: SpeckleRenderer = this.viewer.getRenderer()
+      let passes: GPass[]
+      if (typeof pass === 'string') passes = renderer.pipeline.getPass(pass)
+      else if (Array.isArray(pass)) passes = pass
+      else passes = [pass]
 
-      const depthPass = renderer.pipeline.getPass(passName)[0]
+      if (!passes || !passes.length) {
+        reject(`Could not read from pass`)
+        return
+      }
+      const validPass = passes.find((pass: GPass) => this.hasFramebuffer(pass))
 
-      if (!depthPass) {
-        reject(`Pipeline does not have a ${passName} pass`)
+      if (!validPass) {
+        reject(`Requested pass does not have a valid framebuffer`)
         return
       }
 
-      this.renderTarget = depthPass.outputTarget
+      this.renderTarget = validPass.outputTarget
 
       if (!this.renderTarget) {
-        reject('Pass does not have a render target assigned')
+        reject('Requested Pass does not have a render target assigned')
         return
       }
 
@@ -38,19 +51,24 @@ export class PassReader extends Extension {
     })
   }
 
+  protected hasFramebuffer(pass: GPass) {
+    const renderer = this.viewer.getRenderer().renderer
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    return renderer.properties.get(pass.outputTarget).__webglFramebuffer !== undefined
+  }
+
   public onRender(): void {
     if (!this.needsRead || !this.renderTarget) return
 
-    this.viewer
-      .getRenderer()
-      .renderer.readRenderTargetPixels(
-        this.renderTarget,
-        0,
-        0,
-        this.renderTarget.width,
-        this.renderTarget.height,
-        this.outputBuffer
-      )
+    const renderer = this.viewer.getRenderer().renderer
+    renderer.readRenderTargetPixels(
+      this.renderTarget,
+      0,
+      0,
+      this.renderTarget.width,
+      this.renderTarget.height,
+      this.outputBuffer
+    )
 
     if (this.readbackExecutor)
       this.readbackExecutor([

packages/viewer-sandbox/src/Extensions/PassReader.ts

@@ -1,4 +1,4 @@
-import type { SpeckleRenderer } from '@speckle/viewer'
+import type { GPass, SpeckleRenderer } from '@speckle/viewer'
 import { Extension } from '@speckle/viewer'
 import type { WebGLRenderTarget } from 'three'
 import { Vector3, Vector4 } from 'three'
@@ -11,21 +11,34 @@ export class PassReader extends Extension {
     | ((arg: [Uint8ClampedArray, number, number]) => void)
     | null = null
 
-  public async read(passName: string): Promise<[Uint8ClampedArray, number, number]> {
+  public async read(pass: string): Promise<[Uint8ClampedArray, number, number]>
+  public async read(pass: GPass | GPass[]): Promise<[Uint8ClampedArray, number, number]>
+
+  public async read(
+    pass: string | GPass | GPass[]
+  ): Promise<[Uint8ClampedArray, number, number]> {
     return new Promise<[Uint8ClampedArray, number, number]>((resolve, reject) => {
       const renderer: SpeckleRenderer = this.viewer.getRenderer()
+      let passes: GPass[]
+      if (typeof pass === 'string') passes = renderer.pipeline.getPass(pass)
+      else if (Array.isArray(pass)) passes = pass
+      else passes = [pass]
 
-      const depthPass = renderer.pipeline.getPass(passName)[0]
+      if (!passes || !passes.length) {
+        reject(`Could not read from pass`)
+        return
+      }
+      const validPass = passes.find((pass: GPass) => this.hasFramebuffer(pass))
 
-      if (!depthPass) {
-        reject(`Pipeline does not have a ${passName} pass`)
+      if (!validPass) {
+        reject(`Requested pass does not have a valid framebuffer`)
         return
       }
 
-      this.renderTarget = depthPass.outputTarget
+      this.renderTarget = validPass.outputTarget
 
       if (!this.renderTarget) {
-        reject('Pass does not have a render target assigned')
+        reject('Requested Pass does not have a render target assigned')
         return
       }
 
@@ -38,19 +51,24 @@ export class PassReader extends Extension {
     })
   }
 
+  protected hasFramebuffer(pass: GPass) {
+    const renderer = this.viewer.getRenderer().renderer
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    return renderer.properties.get(pass.outputTarget).__webglFramebuffer !== undefined
+  }
+
   public onRender(): void {
     if (!this.needsRead || !this.renderTarget) return
 
-    this.viewer
-      .getRenderer()
-      .renderer.readRenderTargetPixels(
-        this.renderTarget,
-        0,
-        0,
-        this.renderTarget.width,
-        this.renderTarget.height,
-        this.outputBuffer
-      )
+    const renderer = this.viewer.getRenderer().renderer
+    renderer.readRenderTargetPixels(
+      this.renderTarget,
+      0,
+      0,
+      this.renderTarget.width,
+      this.renderTarget.height,
+      this.outputBuffer
+    )
 
     if (this.readbackExecutor)
       this.readbackExecutor([
@@ -89,10 +107,14 @@ export class PassReader extends Extension {
     return buffer
   }
 
-  public static toBase64(buffer: Uint8ClampedArray, width: number, height: number) {
+  public static toBase64(
+    buffer: Uint8ClampedArray,
+    width: number,
+    height: number
+  ): string {
     const canvas = document.createElement('canvas')
     const ctx = canvas.getContext('2d')
-    if (!ctx) return
+    if (!ctx) return ''
     canvas.width = width
     canvas.height = height
 

packages/viewer-sandbox/src/Sandbox.ts

@@ -57,6 +57,7 @@ import Bright from '../assets/hdri/Bright.png'
 import { Euler, Vector3, Box3, Color, LinearFilter } from 'three'
 import { GeometryType } from '@speckle/viewer'
 import { MeshBatch } from '@speckle/viewer'
+import { PassReader } from './Extensions/PassReader'
 
 export default class Sandbox {
   private viewer: Viewer
@@ -500,10 +501,18 @@ export default class Sandbox {
       title: 'Screenshot'
     })
     screenshot.on('click', async () => {
-      // console.warn(await this.viewer.screenshot())
-      this.viewer
-        .getExtension(FilteringExtension)
-        .hideObjects(['1facfaaf1d3682707edd9ac20ef34e62'])
+      console.warn(await this.viewer.screenshot())
+
+      /** Read depth */
+      // const pass = [
+      //   ...this.viewer.getRenderer().pipeline.getPass('DEPTH'),
+      //   ...this.viewer.getRenderer().pipeline.getPass('DEPTH-NORMAL')
+      // ]
+      // const [depthData, width, height] = await this.viewer
+      //   .getExtension(PassReader)
+      //   .read(pass)
+
+      // console.log(PassReader.toBase64(PassReader.decodeDepth(depthData), width, height))
     })
 
     const rotate = this.tabs.pages[0].addButton({

packages/viewer-sandbox/src/main.ts

@@ -21,6 +21,7 @@ import { SectionTool } from '@speckle/viewer'
 import { SectionOutlines } from '@speckle/viewer'
 import { ViewModesKeys } from './Extensions/ViewModesKeys'
 import { BoxSelection } from './Extensions/BoxSelection'
+import { PassReader } from './Extensions/PassReader'
 
 const createViewer = async (containerName: string, _stream: string) => {
   const container = document.querySelector<HTMLElement>(containerName)
@@ -56,6 +57,7 @@ const createViewer = async (containerName: string, _stream: string) => {
   viewer.createExtension(ViewModesKeys)
   const boxSelect = viewer.createExtension(BoxSelection)
   boxSelect.realtimeSelection = false
+  viewer.createExtension(PassReader)
   // const rotateCamera = viewer.createExtension(RotateCamera)
   cameraController // use it
   selection // use it
@@ -111,7 +113,7 @@ const getStream = () => {
     // prettier-ignore
     // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D'
     // Revit sample house (good for bim-like stuff with many display meshes)
-    // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
+    'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
     // 'https://latest.speckle.systems/streams/c1faab5c62/commits/ab1a1ab2b6'
     // 'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
     // 'https://latest.speckle.systems/streams/58b5648c4d/commits/60371ecb2d'
@@ -466,7 +468,7 @@ const getStream = () => {
     // Instance toilets
     // 'https://app.speckle.systems/projects/e89b61b65c/models/2a0995f124'
 
-    'https://latest.speckle.systems/projects/3fe1880c36/models/65bb4287a8'
+    // 'https://latest.speckle.systems/projects/3fe1880c36/models/65bb4287a8'
   )
 }