[web-wasm] Return MP4 duration on creation (#921)

ts/@live-compositor/core/src/index.ts

@@ -1,4 +1,4 @@
-export { ApiClient, ApiRequest } from './api.js';
+export { ApiClient, ApiRequest, RegisterInputResponse } from './api.js';
 export { LiveCompositor } from './live/compositor.js';
 export { OfflineCompositor } from './offline/compositor.js';
 export { CompositorManager, SetupInstanceOptions } from './compositorManager.js';

ts/@live-compositor/web-wasm/src/input/input.ts

@@ -6,6 +6,10 @@ import type InputFrameProducer from './inputFrameProducer';
 
 export type InputState = 'waiting_for_start' | 'buffering' | 'playing' | 'finished';
 
+export type InputStartInfo = {
+  videoDurationMs?: number;
+};
+
 export class Input {
   private id: InputId;
   private state: InputState;
@@ -33,18 +37,20 @@ export class Input {
     });
   }
 
-  public start() {
+  public async start(): Promise<InputStartInfo | undefined> {
     if (this.state !== 'waiting_for_start') {
       console.warn(`Tried to start an already started input "${this.id}"`);
       return;
     }
 
-    this.frameProducer.start();
+    const startInfo = await this.frameProducer.start();
     this.state = 'buffering';
     this.eventSender.sendEvent({
       type: CompositorEventType.VIDEO_INPUT_DELIVERED,
       inputId: this.id,
     });
+
+    return startInfo;
   }
 
   /**

ts/@live-compositor/web-wasm/src/input/inputFrameProducer.ts

@@ -2,6 +2,7 @@ import type { RegisterInputRequest } from '@live-compositor/core';
 import type { FrameRef } from './frame';
 import DecodingFrameProducer from './producer/decodingFrameProducer';
 import MP4Source from './mp4/source';
+import type { InputStartInfo } from './input';
 
 export type InputFrameProducerCallbacks = {
   onReady(): void;
@@ -12,7 +13,7 @@ export default interface InputFrameProducer {
   /**
    * Starts resources required for producing frames. `init()` has to be called beforehand.
    */
-  start(): void;
+  start(): Promise<InputStartInfo>;
   registerCallbacks(callbacks: InputFrameProducerCallbacks): void;
   /**
    * Produce next frame.

ts/@live-compositor/web-wasm/src/input/mp4/demuxer.ts

@@ -7,6 +7,7 @@ import type { Framerate } from '../../compositor';
 export type Mp4ReadyData = {
   decoderConfig: VideoDecoderConfig;
   framerate: Framerate;
+  videoDurationMs: number;
 };
 
 export type MP4DemuxerCallbacks = {
@@ -51,6 +52,7 @@ export class MP4Demuxer {
     }
 
     const videoTrack = info.videoTracks[0];
+    const videoDurationMs = (videoTrack.movie_duration / videoTrack.movie_timescale) * 1000;
     const codecDescription = this.getCodecDescription(videoTrack.id);
     this.samplesCount = videoTrack.nb_samples;
 
@@ -68,6 +70,7 @@ export class MP4Demuxer {
     this.callbacks.onReady({
       decoderConfig,
       framerate,
+      videoDurationMs,
     });
 
     this.file.setExtractionOptions(videoTrack.id);

ts/@live-compositor/web-wasm/src/input/mp4/mp4box.d.ts

@@ -27,6 +27,7 @@ declare module 'mp4box' {
   export interface MP4MediaTrack {
     id: number;
     movie_duration: number;
+    movie_timescale: number;
     track_width: number;
     track_height: number;
     timescale: number;

ts/@live-compositor/web-wasm/src/input/mp4/source.ts

@@ -1,25 +1,19 @@
-import type { Mp4ReadyData } from './demuxer';
 import { MP4Demuxer } from './demuxer';
 import type InputSource from '../source';
-import type { InputSourceCallbacks, SourcePayload } from '../source';
+import type { InputSourceCallbacks, SourceMetadata, SourcePayload } from '../source';
 import { Queue } from '@datastructures-js/queue';
-import type { Framerate } from '../../compositor';
 
 export default class MP4Source implements InputSource {
   private fileUrl: string;
   private fileData?: ArrayBuffer;
-  private demuxer: MP4Demuxer;
+  private demuxer?: MP4Demuxer;
   private callbacks?: InputSourceCallbacks;
   private chunks: Queue<EncodedVideoChunk>;
   private eosReceived: boolean = false;
-  private framerate?: Framerate;
+  private metadata: SourceMetadata = {};
 
   public constructor(fileUrl: string) {
     this.fileUrl = fileUrl;
-    this.demuxer = new MP4Demuxer({
-      onReady: config => this.handleOnReady(config),
-      onPayload: payload => this.handlePayload(payload),
-    });
     this.chunks = new Queue();
   }
 
@@ -28,13 +22,25 @@ export default class MP4Source implements InputSource {
     this.fileData = await resp.arrayBuffer();
   }
 
-  public start(): void {
-    if (!this.fileData) {
-      throw new Error('MP4Source has to be initialized first before processing can be started');
-    }
+  public async start(): Promise<void> {
+    await new Promise<void>(resolve => {
+      if (!this.fileData) {
+        throw new Error('MP4Source has to be initialized first before processing can be started');
+      }
+
+      this.demuxer = new MP4Demuxer({
+        onReady: data => {
+          this.callbacks?.onDecoderConfig(data.decoderConfig);
+          this.metadata.framerate = data.framerate;
+          this.metadata.videoDurationMs = data.videoDurationMs;
+          resolve();
+        },
+        onPayload: payload => this.handlePayload(payload),
+      });
 
-    this.demuxer.demux(this.fileData);
-    this.demuxer.flush();
+      this.demuxer.demux(this.fileData);
+      this.demuxer.flush();
+    });
   }
 
   public registerCallbacks(callbacks: InputSourceCallbacks): void {
@@ -45,8 +51,8 @@ export default class MP4Source implements InputSource {
     return this.eosReceived && this.chunks.isEmpty();
   }
 
-  public getFramerate(): Framerate | undefined {
-    return this.framerate;
+  public getMetadata(): SourceMetadata {
+    return this.metadata;
   }
 
   public nextChunk(): EncodedVideoChunk | undefined {
@@ -57,11 +63,6 @@ export default class MP4Source implements InputSource {
     return this.chunks.front();
   }
 
-  private handleOnReady(data: Mp4ReadyData) {
-    this.callbacks?.onDecoderConfig(data.decoderConfig);
-    this.framerate = data.framerate;
-  }
-
   private handlePayload(payload: SourcePayload) {
     if (payload.type === 'chunk') {
       this.chunks.push(payload.chunk);

ts/@live-compositor/web-wasm/src/input/producer/decodingFrameProducer.ts

@@ -5,6 +5,7 @@ import { FrameRef } from '../frame';
 import type { InputFrameProducerCallbacks } from '../inputFrameProducer';
 import type InputFrameProducer from '../inputFrameProducer';
 import type InputSource from '../source';
+import type { InputStartInfo } from '../input';
 
 const MAX_BUFFERING_SIZE = 3;
 
@@ -32,8 +33,13 @@ export default class DecodingFrameProducer implements InputFrameProducer {
     await this.source.init();
   }
 
-  public start(): void {
-    this.source.start();
+  public async start(): Promise<InputStartInfo> {
+    await this.source.start();
+
+    const metadata = this.source.getMetadata();
+    return {
+      videoDurationMs: metadata.videoDurationMs,
+    };
   }
 
   public registerCallbacks(callbacks: InputFrameProducerCallbacks): void {
@@ -133,10 +139,10 @@ export default class DecodingFrameProducer implements InputFrameProducer {
   }
 
   private enqueueChunksForPts(framePts: number) {
-    const framerate = this.source.getFramerate();
-    assert(framerate);
+    const metadata = this.source.getMetadata();
+    assert(metadata.framerate);
 
-    const frameDuration = framerateToDurationMs(framerate);
+    const frameDuration = framerateToDurationMs(metadata.framerate);
     const targetPtsUs = (framePts + frameDuration * MAX_BUFFERING_SIZE) * 1000;
 
     let chunk = this.source.peekChunk();

ts/@live-compositor/web-wasm/src/input/source.ts

@@ -2,6 +2,11 @@ import type { Framerate } from '../compositor';
 
 export type SourcePayload = { type: 'chunk'; chunk: EncodedVideoChunk } | { type: 'eos' };
 
+export type SourceMetadata = {
+  framerate?: Framerate;
+  videoDurationMs?: number;
+};
+
 export type InputSourceCallbacks = {
   onDecoderConfig: (config: VideoDecoderConfig) => void;
 };
@@ -14,13 +19,13 @@ export default interface InputSource {
   /**
    * Starts producing chunks. `init()` has to be called beforehand.
    */
-  start(): void;
+  start(): Promise<void>;
   registerCallbacks(callbacks: InputSourceCallbacks): void;
   /**
    * if `true` InputSource won't produce more chunks anymore.
    */
   isFinished(): boolean;
-  getFramerate(): Framerate | undefined;
+  getMetadata(): SourceMetadata;
   nextChunk(): EncodedVideoChunk | undefined;
   peekChunk(): EncodedVideoChunk | undefined;
 }

ts/@live-compositor/web-wasm/src/manager/wasmInstance.ts

@@ -2,6 +2,7 @@ import type {
   ApiRequest,
   CompositorManager,
   RegisterInputRequest,
+  RegisterInputResponse,
   RegisterOutputRequest,
 } from '@live-compositor/core';
 import type { Renderer, Component, ImageSpec } from '@live-compositor/browser-render';
@@ -44,18 +45,18 @@ class WasmInstance implements CompositorManager {
     }
 
     if (route.type == 'input') {
-      await this.handleInputRequest(route.id, route.operation, request.body);
+      return await this.handleInputRequest(route.id, route.operation, request.body);
     } else if (route.type === 'output') {
-      this.handleOutputRequest(route.id, route.operation, request.body);
+      return this.handleOutputRequest(route.id, route.operation, request.body);
     } else if (route.type === 'image') {
-      await this.handleImageRequest(route.id, route.operation, request.body);
+      return await this.handleImageRequest(route.id, route.operation, request.body);
     } else if (route.type === 'shader') {
       throw new Error('Shaders are not supported');
     } else if (route.type === 'web-renderer') {
       throw new Error('Web renderers are not supported');
+    } else {
+      return {};
     }
-
-    return {};
   }
 
   public registerEventListener(cb: (event: unknown) => void): void {
@@ -81,50 +82,67 @@ class WasmInstance implements CompositorManager {
     inputId: string,
     operation: string,
     body?: object
-  ): Promise<void> {
+  ): Promise<object> {
     if (operation === 'register') {
-      await this.registerInput(inputId, body! as RegisterInputRequest);
+      return await this.registerInput(inputId, body! as RegisterInputRequest);
     } else if (operation === 'unregister') {
-      this.queue.removeInput(inputId);
-      this.renderer.unregisterInput(inputId);
+      return this.unregisterInput(inputId);
+    } else {
+      return {};
     }
   }
 
-  private handleOutputRequest(outputId: string, operation: string, body?: object) {
+  private handleOutputRequest(outputId: string, operation: string, body?: object): object {
     if (operation === 'register') {
-      this.registerOutput(outputId, body! as RegisterOutputRequest);
+      return this.registerOutput(outputId, body! as RegisterOutputRequest);
     } else if (operation === 'unregister') {
-      this.queue.removeOutput(outputId);
-      this.renderer.unregisterOutput(outputId);
+      return this.unregisterOutput(outputId);
     } else if (operation === 'update') {
-      this.updateScene(outputId, body! as Api.UpdateOutputRequest);
+      return this.updateScene(outputId, body! as Api.UpdateOutputRequest);
+    } else {
+      return {};
     }
   }
 
   private async handleImageRequest(
     imageId: string,
     operation: string,
     body?: object
-  ): Promise<void> {
+  ): Promise<object> {
     if (operation === 'register') {
       await this.renderer.registerImage(imageId, body as ImageSpec);
     } else if (operation === 'unregister') {
       this.renderer.unregisterImage(imageId);
     }
+
+    return {};
   }
 
-  private async registerInput(inputId: string, request: RegisterInputRequest): Promise<void> {
+  private async registerInput(
+    inputId: string,
+    request: RegisterInputRequest
+  ): Promise<RegisterInputResponse> {
     const frameProducer = producerFromRequest(request);
     await frameProducer.init();
 
     const input = new Input(inputId, frameProducer, this.eventSender);
     // `addInput` will throw an exception if input already exists
     this.queue.addInput(inputId, input);
     this.renderer.registerInput(inputId);
-    input.start();
+
+    const startInfo = await input.start();
+    return {
+      video_duration_ms: startInfo?.videoDurationMs,
+    };
+  }
+
+  private unregisterInput(inputId: string): object {
+    this.queue.removeInput(inputId);
+    this.renderer.unregisterInput(inputId);
+    return {};
   }
 
-  private registerOutput(outputId: string, request: RegisterOutputRequest) {
+  private registerOutput(outputId: string, request: RegisterOutputRequest): object {
     if (request.video) {
       const output = new Output(request);
       this.queue.addOutput(outputId, output);
@@ -142,17 +160,27 @@ class WasmInstance implements CompositorManager {
         throw e;
       }
     }
+
+    return {};
+  }
+
+  private unregisterOutput(outputId: string): object {
+    this.queue.removeOutput(outputId);
+    this.renderer.unregisterOutput(outputId);
+    return {};
   }
 
-  private updateScene(outputId: string, request: Api.UpdateOutputRequest) {
+  private updateScene(outputId: string, request: Api.UpdateOutputRequest): object {
     if (!request.video) {
-      return;
+      return {};
     }
     const output = this.queue.getOutput(outputId);
     if (!output) {
       throw `Unknown output "${outputId}"`;
     }
     this.renderer.updateScene(outputId, output.resolution, request.video.root as Component);
+
+    return {};
   }
 }