Review suggestions

ts/@live-compositor/web/package.json → ts/@live-compositor/web-wasm/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "@live-compositor/web",
+  "name": "@live-compositor/web-wasm",
   "version": "0.1.0-rc.0",
   "description": "",
   "main": "dist/index.js",

ts/@live-compositor/web/src/compositor.ts → ts/@live-compositor/web-wasm/src/compositor.ts

@@ -25,6 +25,10 @@ export default class LiveCompositor {
     this.options = options;
   }
 
+  /*
+   * Initializes LiveCompositor instance. It needs to be called before any resource is registered.
+   * Outputs won't produce any results until `start()` is called.
+   */
   public async init(): Promise<void> {
     this.renderer = await Renderer.create({
       streamFallbackTimeoutMs: this.options.streamFallbackTimeoutMs ?? 500,
@@ -66,11 +70,17 @@ export default class LiveCompositor {
     await this.renderer!.registerFont(fontUrl);
   }
 
+  /**
+   * Starts processing pipeline. Any previously registered output will start producing video data.
+   */
   public async start(): Promise<void> {
-    await this.coreCompositor?.start();
+    await this.coreCompositor!.start();
   }
 
+  /**
+   * Stops processing pipeline.
+   */
   public stop(): void {
-    this.instance?.stop();
+    this.instance!.stop();
   }
 }

ts/@live-compositor/web/src/input/input.ts → ts/@live-compositor/web-wasm/src/input/input.ts

@@ -1,16 +1,16 @@
 import { Frame, InputId } from '@live-compositor/browser-render';
-import MP4Source from './mp4/source';
 import { CompositorEventType } from 'live-compositor';
 import { EventSender } from '../eventSender';
 import InputSource from './source';
-import { RegisterInputRequest } from '@live-compositor/core';
 
 /**
- * Represents frame produced by decoder. All `InputFrame`s have to be manually freed.
+ * Represents frame produced by decoder.
+ * `InputFrame` has to be manually freed from the memory by calling `free()` method. Once freed it no longer can be used.
+ * `Queue` on tick pulls `InputFrame` for each input and once render finishes, manually frees `InputFrame`s.
  */
 export type InputFrame = Frame & {
   /**
-   * Frees InputFrame from memory. InputFrame can not be used after `free()`.
+   * Frees `InputFrame` from memory. `InputFrame` can not be used after `free()`.
    */
   free: () => void;
 };
@@ -23,23 +23,19 @@ export class Input {
   private state: InputState;
   private eventSender: EventSender;
 
-  public constructor(id: InputId, request: RegisterInputRequest, eventSender: EventSender) {
+  public constructor(id: InputId, source: InputSource, eventSender: EventSender) {
     this.id = id;
     this.state = 'waiting_for_start';
+    this.source = source;
     this.eventSender = eventSender;
-    if (request.type === 'mp4') {
-      this.source = new MP4Source(request.url!);
-    } else {
-      throw new Error(`Unknown input type ${(request as any).type}`);
-    }
   }
 
-  public async start() {
+  public start() {
     if (this.state !== 'waiting_for_start') {
       console.warn(`Tried to start an already started input "${this.id}"`);
       return;
     }
-    await this.source.start();
+    this.source.start();
     this.state = 'buffering';
     this.eventSender.sendEvent({
       type: CompositorEventType.VIDEO_INPUT_DELIVERED,

ts/@live-compositor/web/src/input/mp4/demuxer.ts → ts/@live-compositor/web-wasm/src/input/mp4/demuxer.ts

@@ -37,7 +37,7 @@ export class MP4Demuxer {
 
   private onReady(info: MP4Info) {
     if (info.videoTracks.length == 0) {
-      throw 'No video tracks';
+      throw new Error('No video tracks');
     }
 
     const videoTrack = info.videoTracks[0];
@@ -67,12 +67,12 @@ export class MP4Demuxer {
   }
 
   private getCodecDescription(trackId: number): Uint8Array {
-    const trak = this.file.getTrackById(trackId);
-    if (!trak) {
-      throw 'Track does not exist';
+    const track = this.file.getTrackById(trackId);
+    if (!track) {
+      throw new Error('Track does not exist');
     }
 
-    for (const entry of trak.mdia.minf.stbl.stsd.entries) {
+    for (const entry of track.mdia.minf.stbl.stsd.entries) {
       const box = entry.avcC || entry.hvcC || entry.vpcC || entry.av1C;
       if (box) {
         const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
@@ -81,6 +81,6 @@ export class MP4Demuxer {
       }
     }
 
-    throw 'Codec description not found';
+    throw new Error('Codec description not found');
   }
 }

ts/@live-compositor/web/src/input/mp4/source.ts → ts/@live-compositor/web-wasm/src/input/mp4/source.ts

@@ -6,6 +6,7 @@ import InputSource from '../source';
 
 export default class MP4Source implements InputSource {
   private fileUrl: string;
+  private fileData?: ArrayBuffer;
   private demuxer: MP4Demuxer;
   private decoder: H264Decoder;
   private frameFormat: VideoPixelFormat;
@@ -24,9 +25,18 @@ export default class MP4Source implements InputSource {
     this.frameFormat = isSafari ? 'I420' : 'RGBA';
   }
 
-  public async start(): Promise<void> {
+  public async init(): Promise<void> {
     const resp = await fetch(this.fileUrl);
-    await resp.body?.pipeTo(this.sink());
+    this.fileData = await resp.arrayBuffer();
+  }
+
+  public start(): void {
+    if (!this.fileData) {
+      throw new Error('MP4Source has to be initialized first before processing can be started');
+    }
+
+    this.demuxer.demux(this.fileData);
+    this.demuxer.flush();
   }
 
   public async getFrame(): Promise<InputFrame | undefined> {
@@ -51,22 +61,4 @@ export default class MP4Source implements InputSource {
       free: () => frame.close(),
     };
   }
-
-  private sink(): WritableStream {
-    return new WritableStream(
-      {
-        write: (fileChunk: Uint8Array) => {
-          const buffer = fileChunk.buffer.slice(
-            fileChunk.byteOffset,
-            fileChunk.byteOffset + fileChunk.byteLength
-          );
-          this.demuxer.demux(buffer);
-        },
-        close: () => {
-          this.demuxer.flush();
-        },
-      },
-      { highWaterMark: 2 }
-    );
-  }
 }

ts/@live-compositor/web-wasm/src/input/source.ts

@@ -0,0 +1,20 @@
+import { RegisterInputRequest } from '@live-compositor/core';
+import { InputFrame } from './input';
+import MP4Source from './mp4/source';
+
+export default interface InputSource {
+  init(): Promise<void>;
+  /**
+   * Starts input processing. `init()` has to be called beforehand.
+   */
+  start(): void;
+  getFrame(): Promise<InputFrame | undefined>;
+}
+
+export function sourceFromRequest(request: RegisterInputRequest): InputSource {
+  if (request.type === 'mp4') {
+    return new MP4Source(request.url!);
+  } else {
+    throw new Error(`Unknown input type ${(request as any).type}`);
+  }
+}

ts/@live-compositor/web/src/manager/wasmInstance.ts → ts/@live-compositor/web-wasm/src/manager/wasmInstance.ts

@@ -12,6 +12,7 @@ import { Input } from '../input/input';
 import { EventSender } from '../eventSender';
 import { Framerate } from '../compositor';
 import { Output } from '../output/output';
+import { sourceFromRequest } from '../input/source';
 
 export type OnRegisterCallback = (event: object) => void;
 
@@ -44,13 +45,13 @@ class WasmInstance implements CompositorManager {
     if (route.type == 'input') {
       await this.handleInputRequest(route.id, route.operation, request.body);
     } else if (route.type === 'output') {
-      await this.handleOutputRequest(route.id, route.operation, request.body);
+      this.handleOutputRequest(route.id, route.operation, request.body);
     } else if (route.type === 'image') {
       await this.handleImageRequest(route.id, route.operation, request.body);
     } else if (route.type === 'shader') {
-      throw 'Shaders are not supported';
+      throw new Error('Shaders are not supported');
     } else if (route.type === 'web-renderer') {
-      throw 'Web renderers are not supported';
+      throw new Error('Web renderers are not supported');
     }
 
     return {};
@@ -62,12 +63,13 @@ class WasmInstance implements CompositorManager {
 
   private start() {
     if (this.stopQueue) {
-      throw 'Compositor is already running';
+      throw new Error('Compositor is already running');
     }
     this.stopQueue = this.queue.start();
   }
 
   public stop() {
+    // TODO(noituri): Clean all remaining `InputFrame`s
     if (this.stopQueue) {
       this.stopQueue();
       this.stopQueue = undefined;
@@ -80,51 +82,21 @@ class WasmInstance implements CompositorManager {
     body?: object
   ): Promise<void> {
     if (operation === 'register') {
-      const request = body! as RegisterInputRequest;
-      const input = new Input(inputId, request, this.eventSender);
-      this.queue.addInput(inputId, input);
-      this.renderer.registerInput(inputId);
-      await input.start();
+      await this.registerInput(inputId, body! as RegisterInputRequest);
     } else if (operation === 'unregister') {
       this.queue.removeInput(inputId);
       this.renderer.unregisterInput(inputId);
     }
   }
 
-  private async handleOutputRequest(
-    outputId: string,
-    operation: string,
-    body?: object
-  ): Promise<void> {
+  private handleOutputRequest(outputId: string, operation: string, body?: object) {
     if (operation === 'register') {
-      const request = body! as RegisterOutputRequest;
-      if (request.video) {
-        const output = new Output(request);
-        this.queue.addOutput(outputId, output);
-        try {
-          this.renderer.updateScene(
-            outputId,
-            request.video.resolution,
-            request.video.initial.root as Component
-          );
-        } catch (e) {
-          this.queue.removeOutput(outputId);
-          throw e;
-        }
-      }
+      this.registerOutput(outputId, body! as RegisterOutputRequest);
     } else if (operation === 'unregister') {
       this.queue.removeOutput(outputId);
       this.renderer.unregisterOutput(outputId);
     } else if (operation === 'update') {
-      const scene = body! as Api.UpdateOutputRequest;
-      if (!scene.video) {
-        return;
-      }
-      const output = this.queue.getOutput(outputId);
-      if (!output) {
-        throw `Unknown output "${outputId}"`;
-      }
-      this.renderer.updateScene(outputId, output.resolution, scene.video.root as Component);
+      this.updateScene(outputId, body! as Api.UpdateOutputRequest);
     }
   }
 
@@ -139,6 +111,48 @@ class WasmInstance implements CompositorManager {
       this.renderer.unregisterImage(imageId);
     }
   }
+
+  private async registerInput(inputId: string, request: RegisterInputRequest): Promise<void> {
+    const inputSource = sourceFromRequest(request);
+    await inputSource.init();
+
+    const input = new Input(inputId, inputSource, this.eventSender);
+    // `addInput` will throw an exception if input already exists
+    this.queue.addInput(inputId, input);
+    this.renderer.registerInput(inputId);
+    input.start();
+  }
+
+  private registerOutput(outputId: string, request: RegisterOutputRequest) {
+    if (request.video) {
+      const output = new Output(request);
+      this.queue.addOutput(outputId, output);
+      try {
+        // `updateScene` implicitly registers the output.
+        // In case of an error, the output has to be manually cleaned up from the renderer.
+        this.renderer.updateScene(
+          outputId,
+          request.video.resolution,
+          request.video.initial.root as Component
+        );
+      } catch (e) {
+        this.queue.removeOutput(outputId);
+        this.renderer.unregisterOutput(outputId);
+        throw e;
+      }
+    }
+  }
+
+  private updateScene(outputId: string, request: Api.UpdateOutputRequest) {
+    if (!request.video) {
+      return;
+    }
+    const output = this.queue.getOutput(outputId);
+    if (!output) {
+      throw `Unknown output "${outputId}"`;
+    }
+    this.renderer.updateScene(outputId, output.resolution, request.video.root as Component);
+  }
 }
 
 export default WasmInstance;

ts/examples/vite-browser-render/package.json

@@ -12,7 +12,7 @@
   },
   "dependencies": {
     "@live-compositor/browser-render": "0.1.0-rc.4",
-    "@live-compositor/web": "0.1.0-rc.0",
+    "@live-compositor/web-wasm": "0.1.0-rc.0",
     "react": "^18.3.1",
     "react-dom": "^18.3.1"
   },

ts/examples/vite-browser-render/src/examples/MP4Player.tsx

@@ -1,5 +1,5 @@
 import { useCallback, useEffect, useState } from 'react';
-import { LiveCompositor } from '@live-compositor/web';
+import { LiveCompositor } from '@live-compositor/web-wasm';
 import { InputStream, Text, useInputStreams, View } from 'live-compositor';
 
 const BUNNY_URL = 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4';
@@ -32,7 +32,7 @@ function Scene() {
   if (inputState !== 'playing') {
     return (
       <View backgroundColor="#000000">
-        <View width={530} height={40} bottom={300} left={500}>
+        <View width={530} height={40} bottom={340} left={500}>
           <Text fontSize={30} fontFamily="Noto Sans">
             Loading MP4 file
           </Text>