[web-wasm] Add camera input

ts/@live-compositor/core/src/api/input.ts

@@ -2,21 +2,24 @@ import type { Api } from '../api.js';
 import type { RegisterMp4Input, RegisterRtpInput, Inputs } from 'live-compositor';
 import { _liveCompositorInternals } from 'live-compositor';
 
-export type RegisterInputRequest = Api.RegisterInput;
+export type RegisterInputRequest = Api.RegisterInput | { type: 'camera' };
 
 export type InputRef = _liveCompositorInternals.InputRef;
 export const inputRefIntoRawId = _liveCompositorInternals.inputRefIntoRawId;
 export const parseInputRef = _liveCompositorInternals.parseInputRef;
 
 export type RegisterInput =
   | ({ type: 'rtp_stream' } & RegisterRtpInput)
-  | ({ type: 'mp4' } & RegisterMp4Input);
+  | ({ type: 'mp4' } & RegisterMp4Input)
+  | { type: 'camera'; offsetMs?: number };
 
 export function intoRegisterInput(input: RegisterInput): RegisterInputRequest {
   if (input.type === 'mp4') {
     return intoMp4RegisterInput(input);
   } else if (input.type === 'rtp_stream') {
     return intoRtpRegisterInput(input);
+  } else if (input.type === 'camera') {
+    return { type: 'camera' };
   } else {
     throw new Error(`Unknown input type ${(input as any).type}`);
   }

ts/@live-compositor/web-wasm/src/input/frame.ts

@@ -58,6 +58,16 @@ export class FrameRef {
   }
 }
 
+export class NonCopyableFrameRef extends FrameRef {
+  public constructor(frame: FrameWithPts) {
+    super(frame);
+  }
+
+  public incrementRefCount(): void {
+    throw new Error('Reference count of `NonCopyableFrameRef` cannot be incremented');
+  }
+}
+
 async function downloadFrame(frameWithPts: FrameWithPts): Promise<Frame> {
   // Safari does not support conversion to RGBA
   // Chrome does not support conversion to YUV

ts/@live-compositor/web-wasm/src/input/input.ts

@@ -33,13 +33,13 @@ export class Input {
     });
   }
 
-  public start() {
+  public async start(): Promise<void> {
     if (this.state !== 'waiting_for_start') {
       console.warn(`Tried to start an already started input "${this.id}"`);
       return;
     }
 
-    this.frameProducer.start();
+    await this.frameProducer.start();
     this.state = 'buffering';
     this.eventSender.sendEvent({
       type: CompositorEventType.VIDEO_INPUT_DELIVERED,

ts/@live-compositor/web-wasm/src/input/inputFrameProducer.ts

@@ -1,7 +1,9 @@
 import type { RegisterInputRequest } from '@live-compositor/core';
 import type { FrameRef } from './frame';
 import DecodingFrameProducer from './producer/decodingFrameProducer';
+import MediaStreamFrameProducer from './producer/mediaStreamFrameProducer';
 import MP4Source from './mp4/source';
+import { initCameraMediaStream } from './producer/mediaStreamInit';
 
 export type InputFrameProducerCallbacks = {
   onReady(): void;
@@ -12,7 +14,7 @@ export default interface InputFrameProducer {
   /**
    * Starts resources required for producing frames. `init()` has to be called beforehand.
    */
-  start(): void;
+  start(): Promise<void>;
   registerCallbacks(callbacks: InputFrameProducerCallbacks): void;
   /**
    * Produce next frame.
@@ -30,6 +32,8 @@ export default interface InputFrameProducer {
 export function producerFromRequest(request: RegisterInputRequest): InputFrameProducer {
   if (request.type === 'mp4') {
     return new DecodingFrameProducer(new MP4Source(request.url!));
+  } else if (request.type === 'camera') {
+    return new MediaStreamFrameProducer(initCameraMediaStream);
   } else {
     throw new Error(`Unknown input type ${(request as any).type}`);
   }

ts/@live-compositor/web-wasm/src/input/producer/decodingFrameProducer.ts

@@ -32,7 +32,7 @@ export default class DecodingFrameProducer implements InputFrameProducer {
     await this.source.init();
   }
 
-  public start(): void {
+  public async start(): Promise<void> {
     this.source.start();
   }
 

ts/@live-compositor/web-wasm/src/input/producer/mediaStreamFrameProducer.ts

@@ -0,0 +1,104 @@
+import { assert } from '../../utils';
+import type { FrameRef } from '../frame';
+import { NonCopyableFrameRef } from '../frame';
+import type { InputFrameProducerCallbacks } from '../inputFrameProducer';
+import type InputFrameProducer from '../inputFrameProducer';
+import type { MediaStreamInitFn } from './mediaStreamInit';
+
+export default class MediaStreamFrameProducer implements InputFrameProducer {
+  private initMediaStream: MediaStreamInitFn;
+  private stream?: MediaStream;
+  private track?: MediaStreamTrack;
+  private video: HTMLVideoElement;
+  private ptsOffset?: number;
+  private callbacks?: InputFrameProducerCallbacks;
+  private prevFrame?: FrameRef;
+  private onReadySent: boolean;
+  private isVideoLoaded: boolean;
+
+  public constructor(initMediaStream: MediaStreamInitFn) {
+    this.initMediaStream = initMediaStream;
+    this.onReadySent = false;
+    this.isVideoLoaded = false;
+    this.video = document.createElement('video');
+  }
+
+  public async init(): Promise<void> {
+    this.stream = await this.initMediaStream();
+
+    const tracks = this.stream.getVideoTracks();
+    if (tracks.length === 0) {
+      throw new Error('No video track in stream');
+    }
+    this.track = tracks[0];
+
+    this.video.srcObject = this.stream;
+    await new Promise(resolve => {
+      this.video.onloadedmetadata = resolve;
+    });
+
+    this.isVideoLoaded = true;
+  }
+
+  public async start(): Promise<void> {
+    assert(this.isVideoLoaded);
+    await this.video.play();
+  }
+
+  public registerCallbacks(callbacks: InputFrameProducerCallbacks): void {
+    this.callbacks = callbacks;
+  }
+
+  public async produce(_framePts?: number): Promise<void> {
+    if (this.isFinished()) {
+      return;
+    }
+
+    this.produceFrame();
+
+    if (!this.onReadySent) {
+      this.callbacks?.onReady();
+      this.onReadySent = true;
+    }
+  }
+
+  private produceFrame() {
+    const videoFrame = new VideoFrame(this.video, { timestamp: performance.now() * 1000 });
+    if (!this.ptsOffset) {
+      this.ptsOffset = -videoFrame.timestamp;
+    }
+
+    // Only one media track video frame can be alive at the time
+    if (this.prevFrame) {
+      this.prevFrame.decrementRefCount();
+    }
+    this.prevFrame = new NonCopyableFrameRef({
+      frame: videoFrame,
+      ptsMs: (videoFrame.timestamp + this.ptsOffset) / 1000,
+    });
+  }
+
+  public getFrameRef(_framePts: number): FrameRef | undefined {
+    const frame = this.prevFrame;
+    this.prevFrame = undefined;
+    return frame;
+  }
+
+  public isFinished(): boolean {
+    if (this.track) {
+      return this.track.readyState === 'ended';
+    }
+
+    return false;
+  }
+
+  public close(): void {
+    if (!this.stream) {
+      return;
+    }
+
+    for (const track of this.stream.getTracks()) {
+      track.stop();
+    }
+  }
+}

ts/@live-compositor/web-wasm/src/input/producer/mediaStreamInit.ts

@@ -0,0 +1,5 @@
+export type MediaStreamInitFn = () => Promise<MediaStream>;
+
+export async function initCameraMediaStream(): Promise<MediaStream> {
+  return await navigator.mediaDevices.getUserMedia({ video: true });
+}

ts/@live-compositor/web-wasm/src/input/registerInput.ts

@@ -1,6 +1,6 @@
 import type { RegisterInput as InternalRegisterInput } from '@live-compositor/core';
 
-export type RegisterInput = { type: 'mp4' } & RegisterMP4Input;
+export type RegisterInput = ({ type: 'mp4' } & RegisterMP4Input) | { type: 'camera' };
 
 export type RegisterMP4Input = {
   url: string;
@@ -9,6 +9,8 @@ export type RegisterMP4Input = {
 export function intoRegisterInput(input: RegisterInput): InternalRegisterInput {
   if (input.type === 'mp4') {
     return { type: 'mp4', url: input.url };
+  } else if (input.type === 'camera') {
+    return { type: 'camera' };
   } else {
     throw new Error(`Unknown input type ${(input as any).type}`);
   }

ts/@live-compositor/web-wasm/src/manager/wasmInstance.ts

@@ -121,7 +121,7 @@ class WasmInstance implements CompositorManager {
     // `addInput` will throw an exception if input already exists
     this.queue.addInput(inputId, input);
     this.renderer.registerInput(inputId);
-    input.start();
+    await input.start();
   }
 
   private registerOutput(outputId: string, request: RegisterOutputRequest) {

ts/examples/vite-browser-render/src/App.tsx

@@ -3,6 +3,7 @@ import './App.css';
 import Counter from './examples/Counter';
 import SimpleMp4Example from './examples/SimpleMp4Example';
 import MultipleCompositors from './examples/MultipleCompositors';
+import CameraExample from './examples/CameraExample';
 import { setWasmBundleUrl } from '@live-compositor/web-wasm';
 
 setWasmBundleUrl('assets/live-compositor.wasm');
@@ -12,6 +13,7 @@ function App() {
     counter: <Counter />,
     simpleMp4: <SimpleMp4Example />,
     multipleCompositors: <MultipleCompositors />,
+    camera: <CameraExample />,
     home: <Home />,
   };
   const [currentExample, setCurrentExample] = useState<keyof typeof EXAMPLES>('home');
@@ -25,6 +27,7 @@ function App() {
         <button onClick={() => setCurrentExample('multipleCompositors')}>
           Multiple LiveCompositor instances
         </button>
+        <button onClick={() => setCurrentExample('camera')}>Camera</button>
         <button onClick={() => setCurrentExample('counter')}>Counter</button>
       </div>
       <div className="card">{EXAMPLES[currentExample]}</div>
@@ -40,12 +43,15 @@ function Home() {
         <code>@live-compositor/web-wasm</code> - LiveCompositor in the browser
       </h3>
       <li>
-        <code>Simple Mp4</code> - Take MP4 file as an input and render output on canvas
+        <code>Simple Mp4</code> - Take MP4 file as an input and render output on canvas.
       </li>
       <li>
         <code>Multiple LiveCompositor instances</code> - Runs multiple LiveCompositor instances at
         the same time.
       </li>
+      <li>
+        <code>Camera</code> - Use webcam as an input and render output on canvas.
+      </li>
       <h3>
         <code>@live-compositor/browser-render</code> - Rendering engine from LiveCompositor
       </h3>

ts/examples/vite-browser-render/src/examples/CameraExample.tsx

@@ -0,0 +1,36 @@
+import { useCallback } from 'react';
+import type { LiveCompositor } from '@live-compositor/web-wasm';
+import { InputStream, Rescaler, Text, View } from 'live-compositor';
+import CompositorCanvas from '../components/CompositorCanvas';
+
+function CameraExample() {
+  const onCanvasCreate = useCallback(async (compositor: LiveCompositor) => {
+    await compositor.registerFont(
+      'https://fonts.gstatic.com/s/notosans/v36/o-0mIpQlx3QUlC5A4PNB6Ryti20_6n1iPHjcz6L1SoM-jCpoiyD9A-9a6Vc.ttf'
+    );
+    await compositor.registerInput('camera', { type: 'camera' });
+  }, []);
+
+  return (
+    <div className="card">
+      <CompositorCanvas onCanvasCreate={onCanvasCreate} width={1280} height={720}>
+        <Scene />
+      </CompositorCanvas>
+    </div>
+  );
+}
+
+function Scene() {
+  return (
+    <View style={{ width: 1280, height: 720 }}>
+      <Rescaler>
+        <InputStream inputId="camera" />
+      </Rescaler>
+      <View style={{ width: 200, height: 40, backgroundColor: '#000000', bottom: 20, left: 520 }}>
+        <Text style={{ fontSize: 30, fontFamily: 'Noto Sans' }}>Camera input</Text>
+      </View>
+    </View>
+  );
+}
+
+export default CameraExample;

ts/package.json

@@ -30,7 +30,7 @@
     "json-schema-to-typescript": "^15.0.1",
     "prettier": "^3.3.3",
     "rimraf": "^6.0.1",
-    "typescript": "5.5.3"
+    "typescript": "5.7.2"
   },
   "overrides": {
     "rollup-plugin-copy": {