[web-wasm] Move compositor to web worker + add camera (copied from #913…

…) (#916)

Co-authored-by: Mikołaj Radkowski <[email protected]>

ts/@live-compositor/core/src/api/input.ts

@@ -2,21 +2,38 @@ import type { Api } from '../api.js';
 import type { RegisterMp4Input, RegisterRtpInput, Inputs } from 'live-compositor';
 import { _liveCompositorInternals } from 'live-compositor';
 
-export type RegisterInputRequest = Api.RegisterInput;
+/**
+ * It represents HTTP request that can be sent to
+ * to compositor, but also additional variants that are specific to WASM like camera
+ */
+export type RegisterInputRequest =
+  | Api.RegisterInput
+  | { type: 'camera' }
+  | { type: 'screen_capture' };
 
 export type InputRef = _liveCompositorInternals.InputRef;
 export const inputRefIntoRawId = _liveCompositorInternals.inputRefIntoRawId;
 export const parseInputRef = _liveCompositorInternals.parseInputRef;
 
 export type RegisterInput =
   | ({ type: 'rtp_stream' } & RegisterRtpInput)
-  | ({ type: 'mp4' } & RegisterMp4Input);
+  | ({ type: 'mp4' } & RegisterMp4Input)
+  | { type: 'camera' }
+  | { type: 'screen_capture' };
 
+/**
+ * Converts object passed by user (or modified by platform specific interface) into
+ * HTTP request
+ */
 export function intoRegisterInput(input: RegisterInput): RegisterInputRequest {
   if (input.type === 'mp4') {
     return intoMp4RegisterInput(input);
   } else if (input.type === 'rtp_stream') {
     return intoRtpRegisterInput(input);
+  } else if (input.type === 'camera') {
+    return { type: 'camera' };
+  } else if (input.type === 'screen_capture') {
+    return { type: 'screen_capture' };
   } else {
     throw new Error(`Unknown input type ${(input as any).type}`);
   }

ts/@live-compositor/core/src/live/compositor.ts

@@ -5,14 +5,14 @@ import Output from './output.js';
 import type { CompositorManager } from '../compositorManager.js';
 import type { RegisterOutput } from '../api/output.js';
 import { intoRegisterOutput } from '../api/output.js';
-import type { RegisterInput } from '../api/input.js';
 import { intoRegisterInput } from '../api/input.js';
 import { parseEvent } from '../event.js';
 import { intoRegisterImage, intoRegisterWebRenderer } from '../api/renderer.js';
 import { handleEvent } from './event.js';
 import type { ReactElement } from 'react';
 import type { Logger } from 'pino';
 import type { ImageRef } from '../api/image.js';
+import type { RegisterInput } from '../index.js';
 
 export class LiveCompositor {
   private manager: CompositorManager;

ts/@live-compositor/core/src/offline/compositor.ts

@@ -88,6 +88,8 @@ export class OfflineCompositor {
     const inputRef = { type: 'global', id: inputId } as const;
     const result = await this.api.registerInput(inputRef, intoRegisterInput(request));
 
+    const offsetMs = 'offsetMs' in request && request.offsetMs ? request.offsetMs : 0;
+
     if (request.type === 'mp4' && request.loop) {
       this.store.addInput({
         inputId,
@@ -98,18 +100,18 @@ export class OfflineCompositor {
     } else {
       this.store.addInput({
         inputId,
-        offsetMs: request.offsetMs ?? 0,
+        offsetMs: offsetMs ?? 0,
         videoDurationMs: result.video_duration_ms,
         audioDurationMs: result.audio_duration_ms,
       });
-      if (request.offsetMs) {
-        this.inputTimestamps.push(request.offsetMs);
+      if (offsetMs) {
+        this.inputTimestamps.push(offsetMs);
       }
       if (result.video_duration_ms) {
-        this.inputTimestamps.push((request.offsetMs ?? 0) + result.video_duration_ms);
+        this.inputTimestamps.push((offsetMs ?? 0) + result.video_duration_ms);
       }
       if (result.audio_duration_ms) {
-        this.inputTimestamps.push((request.offsetMs ?? 0) + result.audio_duration_ms);
+        this.inputTimestamps.push((offsetMs ?? 0) + result.audio_duration_ms);
       }
     }
     return result;

ts/@live-compositor/node/src/utils.ts

@@ -1,8 +1,8 @@
-export async function sleep(timeout_ms: number): Promise<void> {
+export async function sleep(timeoutMs: number): Promise<void> {
   await new Promise<void>(res => {
     setTimeout(() => {
       res();
-    }, timeout_ms);
+    }, timeoutMs);
   });
 }
 

ts/@live-compositor/web-wasm/src/compositor.ts → ts/@live-compositor/web-wasm/src/compositor/compositor.ts

@@ -1,15 +1,10 @@
-import { loadWasmModule, Renderer } from '@live-compositor/browser-render';
 import { LiveCompositor as CoreLiveCompositor } from '@live-compositor/core';
-import WasmInstance from './manager/wasmInstance';
-import type { RegisterOutput } from './output/registerOutput';
-import { intoRegisterOutput } from './output/registerOutput';
-import type { RegisterInput } from './input/registerInput';
-import { intoRegisterInput } from './input/registerInput';
-import type { RegisterImage } from './renderers';
 import type { ReactElement } from 'react';
 import type { Logger } from 'pino';
 import { pino } from 'pino';
-import { assert } from './utils';
+import { assert } from '../utils';
+import type { RegisterOutput, RegisterInput, RegisterImage } from './types';
+import WasmInstance from '../wasmInstance';
 
 export type LiveCompositorOptions = {
   framerate?: Framerate;
@@ -34,7 +29,6 @@ export function setWasmBundleUrl(url: string) {
 export default class LiveCompositor {
   private coreCompositor?: CoreLiveCompositor;
   private instance?: WasmInstance;
-  private renderer?: Renderer;
   private options: LiveCompositorOptions;
   private logger: Logger = pino({ level: 'warn' });
 
@@ -47,13 +41,11 @@ export default class LiveCompositor {
    * Outputs won't produce any results until `start()` is called.
    */
   public async init(): Promise<void> {
-    await ensureWasmModuleLoaded();
-    this.renderer = await Renderer.create({
-      streamFallbackTimeoutMs: this.options.streamFallbackTimeoutMs ?? 500,
-    });
+    assert(wasmBundleUrl, 'Location of WASM bundle is not defined, call setWasmBundleUrl() first.');
     this.instance = new WasmInstance({
-      renderer: this.renderer!,
       framerate: this.options.framerate ?? { num: 30, den: 1 },
+      wasmBundleUrl,
+      logger: this.logger.child({ element: 'wasmInstance' }),
     });
     this.coreCompositor = new CoreLiveCompositor(this.instance, this.logger);
 
@@ -66,7 +58,7 @@ export default class LiveCompositor {
     request: RegisterOutput
   ): Promise<void> {
     assert(this.coreCompositor);
-    await this.coreCompositor.registerOutput(outputId, root, intoRegisterOutput(request));
+    await this.coreCompositor.registerOutput(outputId, root, request);
   }
 
   public async unregisterOutput(outputId: string): Promise<void> {
@@ -76,7 +68,7 @@ export default class LiveCompositor {
 
   public async registerInput(inputId: string, request: RegisterInput): Promise<void> {
     assert(this.coreCompositor);
-    await this.coreCompositor.registerInput(inputId, intoRegisterInput(request));
+    await this.coreCompositor.registerInput(inputId, request);
   }
 
   public async unregisterInput(inputId: string): Promise<void> {
@@ -95,8 +87,8 @@ export default class LiveCompositor {
   }
 
   public async registerFont(fontUrl: string): Promise<void> {
-    assert(this.renderer);
-    await this.renderer.registerFont(fontUrl);
+    assert(this.instance);
+    await this.instance.registerFont(fontUrl);
   }
 
   /**
@@ -114,14 +106,3 @@ export default class LiveCompositor {
     await this.instance?.terminate();
   }
 }
-
-const ensureWasmModuleLoaded = (() => {
-  let loadedState: Promise<void> | undefined = undefined;
-  return async () => {
-    assert(wasmBundleUrl, 'Location of WASM bundle is not defined, call setWasmBundleUrl() first.');
-    if (!loadedState) {
-      loadedState = loadWasmModule(wasmBundleUrl);
-    }
-    await loadedState;
-  };
-})();

ts/@live-compositor/web-wasm/src/compositor/types.ts

@@ -0,0 +1,17 @@
+import type { Resolution } from '@live-compositor/browser-render';
+import type { Renderers } from 'live-compositor';
+
+export type RegisterImage = Required<Pick<Renderers.RegisterImage, 'assetType' | 'url'>>;
+
+export type RegisterOutput = {
+  type: 'canvas';
+  video: {
+    canvas: HTMLCanvasElement;
+    resolution: Resolution;
+  };
+};
+
+export type RegisterInput =
+  | { type: 'mp4'; url: string }
+  | { type: 'camera' }
+  | { type: 'screen_capture' };

ts/@live-compositor/web-wasm/src/eventSender.ts

@@ -1,26 +1,31 @@
 import { _liveCompositorInternals } from 'live-compositor';
+import type { WorkerEvent } from './workerApi';
 
 export const CompositorEventType = _liveCompositorInternals.CompositorEventType;
 export const inputRefIntoRawId = _liveCompositorInternals.inputRefIntoRawId;
 
 export class EventSender {
-  private eventCallback?: (event: object) => void;
+  private eventCallbacks: Set<(event: object) => void> = new Set();
 
-  public setEventCallback(eventCallback: (event: object) => void) {
-    this.eventCallback = eventCallback;
+  /**
+   * Check if this is event that should be passed to core
+   */
+  public static isExternalEvent(event: WorkerEvent): event is ExternalWorkerEvent {
+    return Object.values(CompositorEventType).includes(event?.type);
   }
 
-  public sendEvent(event: WasmCompositorEvent) {
-    if (!this.eventCallback) {
-      console.warn(`Failed to send event: ${event}`);
-      return;
-    }
+  public registerEventCallback(eventCallback: (event: object) => void) {
+    this.eventCallbacks?.add(eventCallback);
+  }
 
-    this.eventCallback!(toWebSocketMessage(event));
+  public sendEvent(event: ExternalWorkerEvent) {
+    for (const cb of this.eventCallbacks) {
+      cb(toWebSocketMessage(event));
+    }
   }
 }
 
-function toWebSocketMessage(event: WasmCompositorEvent): WebSocketMessage {
+function toWebSocketMessage(event: ExternalWorkerEvent): WebSocketMessage {
   if (event.type == CompositorEventType.OUTPUT_DONE) {
     return {
       type: event.type,
@@ -34,7 +39,10 @@ function toWebSocketMessage(event: WasmCompositorEvent): WebSocketMessage {
   };
 }
 
-export type WasmCompositorEvent =
+/**
+ * Subset of WorkerEvents that should be passed outside (to the core code)
+ */
+export type ExternalWorkerEvent =
   | {
       type:
         | _liveCompositorInternals.CompositorEventType.AUDIO_INPUT_DELIVERED
@@ -49,6 +57,11 @@ export type WasmCompositorEvent =
       type: _liveCompositorInternals.CompositorEventType.OUTPUT_DONE;
       outputId: string;
     };
+
+/**
+ * Actual format that in non-WASM compositor would be sent via WebSocket. Here it's only used to match the format
+ * so the core package can handle both WASM and non-WASM instances.
+ */
 export type WebSocketMessage =
   | {
       type:

ts/@live-compositor/web-wasm/src/index.ts

@@ -1,4 +1,3 @@
-import WasmInstance from './manager/wasmInstance';
-import LiveCompositor, { setWasmBundleUrl } from './compositor';
+import LiveCompositor, { setWasmBundleUrl } from './compositor/compositor';
 
-export { WasmInstance, LiveCompositor, setWasmBundleUrl };
+export { LiveCompositor, setWasmBundleUrl };