Review suggestions

ts/@live-compositor/core/src/api/input.ts

@@ -1,15 +1,15 @@
 import { Api } from '../api.js';
 import { RegisterInput, Inputs } from 'live-compositor';
 
-export type RegisterInputRequest = Api.RegisterInput | { type: 'bytes' };
+export type RegisterInputRequest = Api.RegisterInput | { type: 'raw_frames' };
 
 export function intoRegisterInput(input: RegisterInput): RegisterInputRequest {
   if (input.type === 'mp4') {
     return intoMp4RegisterInput(input);
   } else if (input.type === 'rtp_stream') {
     return intoRtpRegisterInput(input);
-  } else if (input.type === 'bytes') {
-    return intoBytesRegisterInput();
+  } else if (input.type === 'raw_frames') {
+    return intoRawFramesRegisterInput();
   } else {
     throw new Error(`Unknown input type ${(input as any).type}`);
   }
@@ -38,8 +38,8 @@ function intoRtpRegisterInput(input: Inputs.RegisterRtpInput): RegisterInputRequ
   };
 }
 
-function intoBytesRegisterInput(): RegisterInputRequest {
-  return { type: 'bytes' };
+function intoRawFramesRegisterInput(): RegisterInputRequest {
+  return { type: 'raw_frames' };
 }
 
 function intoInputAudio(audio: Inputs.InputRtpAudioOptions): Api.InputRtpAudioOptions {

ts/@live-compositor/core/src/api/output.ts

@@ -1,14 +1,14 @@
-import { RegisterOutput, Api, Outputs, OutputByteFormat } from 'live-compositor';
+import { RegisterOutput, Api, Outputs, OutputFrameFormat } from 'live-compositor';
 
 export type RegisterOutputRequest = Api.RegisterOutput | RegisterBytesOutput;
 
 export type RegisterBytesOutput = {
-  type: 'bytes';
+  type: 'raw_frames';
   video?: OutputBytesVideoOptions;
 };
 
 export type OutputBytesVideoOptions = {
-  format: OutputByteFormat;
+  format: OutputFrameFormat;
   resolution: Api.Resolution;
   initial: Api.Video;
 };
@@ -21,8 +21,8 @@ export function intoRegisterOutput(
     return intoRegisterRtpOutput(output, initial);
   } else if (output.type === 'mp4') {
     return intoRegisterMp4Output(output, initial);
-  } else if (output.type === 'bytes') {
-    return intoRegisterBytesOutput(output, initial);
+  } else if (output.type === 'raw_frames') {
+    return intoRegisterRawFramesOutput(output, initial);
   } else {
     throw new Error(`Unknown output type ${(output as any).type}`);
   }
@@ -54,12 +54,12 @@ function intoRegisterMp4Output(
   };
 }
 
-function intoRegisterBytesOutput(
-  output: Outputs.RegisterBytesOutput,
+function intoRegisterRawFramesOutput(
+  output: Outputs.RegisterRawFramesOutput,
   initial: { video?: Api.Video; _audio?: Api.Audio }
 ): RegisterOutputRequest {
   return {
-    type: 'bytes',
+    type: 'raw_frames',
     video: {
       format: output.video.format,
       resolution: output.video.resolution,

ts/@live-compositor/node/tsconfig.json

@@ -1,9 +1,6 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "outDir": "dist",
-    "module": "ESNext",
-    "moduleResolution": "bundler",
-    "target": "ESNext"
+    "outDir": "dist"
   }
 }

ts/@live-compositor/web/package.json

@@ -17,9 +17,11 @@
     "/dist"
   ],
   "dependencies": {
+    "@datastructures-js/queue": "^4.2.3",
     "@live-compositor/browser-render": "0.1.0-rc.4",
-    "@live-compositor/core": "0.1.0-rc.5",
-    "live-compositor": "^0.1.0-rc.5",
-    "mp4box": "^0.5.2"
+    "@live-compositor/core": "0.1.0",
+    "live-compositor": "^0.1.0",
+    "mp4box": "^0.5.2",
+    "path-parser": "^6.1.0"
   }
 }

ts/@live-compositor/web/src/compositor.ts

@@ -10,8 +10,8 @@ import { Input } from './input/input';
 import { RegisterImage } from './renderers';
 
 export type LiveCompositorOptions = {
-  framerate: Framerate;
-  streamFallbackTimeoutMs: number;
+  framerate?: Framerate;
+  streamFallbackTimeoutMs?: number;
 };
 
 export type Framerate = {
@@ -20,70 +20,80 @@ export type Framerate = {
 };
 
 export default class LiveCompositor {
-  private coreCompositor: CoreLiveCompositor;
-  private queue: Queue;
-  private renderer: Renderer;
-  private eventSender?: EventSender;
+  private coreCompositor?: CoreLiveCompositor;
+  private queue?: Queue;
+  private renderer?: Renderer;
+  private eventSender: EventSender;
+  private stopQueue?: StopQueueFn;
+  private options: LiveCompositorOptions;
 
-  private constructor(renderer: Renderer, framerate: Framerate) {
+  public constructor(options: LiveCompositorOptions) {
+    this.options = options;
+    this.eventSender = new EventSender();
+  }
+
+  public async init(): Promise<void> {
+    this.renderer = await Renderer.create({
+      streamFallbackTimeoutMs: this.options.streamFallbackTimeoutMs ?? 500,
+    });
+    this.queue = new Queue(this.options.framerate ?? { num: 30, den: 1 }, this.renderer!);
     this.coreCompositor = new CoreLiveCompositor(
       new WasmInstance({
-        renderer: renderer,
-        onRegisterCallback: cb => {
-          this.eventSender = new EventSender(cb);
-        },
+        renderer: this.renderer!,
+        onRegisterCallback: cb => this.eventSender.setEventCallback(cb),
       })
     );
-    this.queue = new Queue(framerate, renderer);
-    this.renderer = renderer;
-  }
 
-  public static async create(options: LiveCompositorOptions): Promise<LiveCompositor> {
-    const renderer = await Renderer.create({
-      streamFallbackTimeoutMs: options.streamFallbackTimeoutMs,
-    });
-    const compositor = new LiveCompositor(renderer, options.framerate);
-    await compositor.coreCompositor.init();
-    return compositor;
+    await this.coreCompositor!.init();
   }
 
   public async registerOutput(outputId: string, request: RegisterOutput): Promise<void> {
-    await this.coreCompositor.registerOutput(outputId, intoRegisterOutput(request));
-    const output = Output.create(request);
-    this.queue.addOutput(outputId, output);
+    await this.coreCompositor!.registerOutput(outputId, intoRegisterOutput(request));
+    const output = new Output(request);
+    this.queue!.addOutput(outputId, output);
   }
 
   public async unregisterOutput(outputId: string): Promise<void> {
-    await this.coreCompositor.unregisterOutput(outputId);
-    this.queue.removeOutput(outputId);
+    await this.coreCompositor!.unregisterOutput(outputId);
+    this.queue!.removeOutput(outputId);
   }
 
   public async registerInput(inputId: string, request: RegisterInput): Promise<void> {
-    await this.coreCompositor.registerInput(inputId, intoRegisterInput(request));
+    await this.coreCompositor!.registerInput(inputId, intoRegisterInput(request));
 
-    const input = Input.create(inputId, request, this.eventSender!);
-    this.queue.addInput(inputId, input);
-    input.start();
+    const input = new Input(inputId, request, this.eventSender);
+    this.queue!.addInput(inputId, input);
+    await input.start();
   }
 
   public async unregisterInput(inputId: string): Promise<void> {
-    await this.coreCompositor.unregisterInput(inputId);
-    this.queue.removeInput(inputId);
+    await this.coreCompositor!.unregisterInput(inputId);
+    this.queue!.removeInput(inputId);
   }
 
   public async registerImage(imageId: string, request: RegisterImage): Promise<void> {
-    await this.coreCompositor.registerImage(imageId, request);
+    await this.coreCompositor!.registerImage(imageId, request);
   }
 
   public async unregisterImage(imageId: string): Promise<void> {
-    await this.coreCompositor.unregisterImage(imageId);
+    await this.coreCompositor!.unregisterImage(imageId);
   }
 
   public async registerFont(fontUrl: string): Promise<void> {
-    await this.renderer.registerFont(fontUrl);
+    await this.renderer!.registerFont(fontUrl);
+  }
+
+  public start(): void {
+    if (this.stopQueue) {
+      throw 'Compositor is already running';
+    }
+    this.stopQueue = this.queue!.start();
   }
 
-  public start(): StopQueueFn {
-    return this.queue.start();
+  public stop(): void {
+    if (this.stopQueue) {
+      this.stopQueue();
+      this.stopQueue = undefined;
+    }
   }
 }

ts/@live-compositor/web/src/eventSender.ts

@@ -1,24 +1,62 @@
-import { InputId } from '@live-compositor/browser-render';
-import { CompositorEventType } from 'live-compositor';
+import { CompositorEvent, CompositorEventType } from 'live-compositor';
 
 export class EventSender {
-  private eventCallback: (event: object) => void;
+  private eventCallback?: (event: object) => void;
 
-  public constructor(eventCallback: (event: object) => void) {
+  public setEventCallback(eventCallback: (event: object) => void) {
     this.eventCallback = eventCallback;
   }
 
-  public sendEvent(event: ApiEvent) {
-    this.eventCallback(event);
+  public sendEvent(event: CompositorEvent) {
+    if (!this.eventCallback) {
+      console.warn(`Failed to send event: ${event}`);
+      return;
+    }
+
+    this.eventCallback!(toWebSocketMessage(event));
   }
 }
 
-export type ApiEvent =
+function toWebSocketMessage(event: CompositorEvent): WebSocketMessage {
+  if (event.type == CompositorEventType.OUTPUT_DONE) {
+    return {
+      type: event.type,
+      output_id: event.outputId,
+    };
+  }
+
+  return {
+    type: event.type,
+    input_id: event.inputId,
+  };
+}
+
+export type WebSocketMessage =
+  | {
+      type: CompositorEventType.AUDIO_INPUT_DELIVERED;
+      input_id: string;
+    }
   | {
       type: CompositorEventType.VIDEO_INPUT_DELIVERED;
-      input_id: InputId;
+      input_id: string;
+    }
+  | {
+      type: CompositorEventType.AUDIO_INPUT_PLAYING;
+      input_id: string;
     }
   | {
       type: CompositorEventType.VIDEO_INPUT_PLAYING;
-      input_id: InputId;
+      input_id: string;
+    }
+  | {
+      type: CompositorEventType.AUDIO_INPUT_EOS;
+      input_id: string;
+    }
+  | {
+      type: CompositorEventType.VIDEO_INPUT_EOS;
+      input_id: string;
+    }
+  | {
+      type: CompositorEventType.OUTPUT_DONE;
+      output_id: string;
     };