From c65572ef593042de3e60dead44dac63b627f4961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Radkowski?= Date: Wed, 15 Jan 2025 16:27:13 +0100 Subject: [PATCH] [web-wasm] Return MP4 duration on creation (#921) --- ts/@live-compositor/core/src/index.ts | 2 +- .../web-wasm/src/input/input.ts | 10 ++- .../web-wasm/src/input/inputFrameProducer.ts | 3 +- .../web-wasm/src/input/mp4/demuxer.ts | 3 + .../web-wasm/src/input/mp4/mp4box.d.ts | 1 + .../web-wasm/src/input/mp4/source.ts | 45 ++++++------ .../input/producer/decodingFrameProducer.ts | 16 +++-- .../web-wasm/src/input/source.ts | 9 ++- .../web-wasm/src/manager/wasmInstance.ts | 68 +++++++++++++------ .../src/examples/SimpleMp4Example.tsx | 6 +- 10 files changed, 107 insertions(+), 56 deletions(-) diff --git a/ts/@live-compositor/core/src/index.ts b/ts/@live-compositor/core/src/index.ts index 8d44db3c8..cde06b19b 100644 --- a/ts/@live-compositor/core/src/index.ts +++ b/ts/@live-compositor/core/src/index.ts @@ -1,4 +1,4 @@ -export { ApiClient, ApiRequest } from './api.js'; +export { ApiClient, ApiRequest, RegisterInputResponse } from './api.js'; export { LiveCompositor } from './live/compositor.js'; export { OfflineCompositor } from './offline/compositor.js'; export { CompositorManager, SetupInstanceOptions } from './compositorManager.js'; diff --git a/ts/@live-compositor/web-wasm/src/input/input.ts b/ts/@live-compositor/web-wasm/src/input/input.ts index 347233cfb..1efd4c7b5 100644 --- a/ts/@live-compositor/web-wasm/src/input/input.ts +++ b/ts/@live-compositor/web-wasm/src/input/input.ts @@ -6,6 +6,10 @@ import type InputFrameProducer from './inputFrameProducer'; export type InputState = 'waiting_for_start' | 'buffering' | 'playing' | 'finished'; +export type InputStartInfo = { + videoDurationMs?: number; +}; + export class Input { private id: InputId; private state: InputState; @@ -33,18 +37,20 @@ export class Input { }); } - public start() { + public async start(): Promise { if (this.state !== 'waiting_for_start') { console.warn(`Tried to start an already started input "${this.id}"`); return; } - this.frameProducer.start(); + const startInfo = await this.frameProducer.start(); this.state = 'buffering'; this.eventSender.sendEvent({ type: CompositorEventType.VIDEO_INPUT_DELIVERED, inputId: this.id, }); + + return startInfo; } /** diff --git a/ts/@live-compositor/web-wasm/src/input/inputFrameProducer.ts b/ts/@live-compositor/web-wasm/src/input/inputFrameProducer.ts index 66b494527..edc0cf257 100644 --- a/ts/@live-compositor/web-wasm/src/input/inputFrameProducer.ts +++ b/ts/@live-compositor/web-wasm/src/input/inputFrameProducer.ts @@ -2,6 +2,7 @@ import type { RegisterInputRequest } from '@live-compositor/core'; import type { FrameRef } from './frame'; import DecodingFrameProducer from './producer/decodingFrameProducer'; import MP4Source from './mp4/source'; +import type { InputStartInfo } from './input'; export type InputFrameProducerCallbacks = { onReady(): void; @@ -12,7 +13,7 @@ export default interface InputFrameProducer { /** * Starts resources required for producing frames. `init()` has to be called beforehand. */ - start(): void; + start(): Promise; registerCallbacks(callbacks: InputFrameProducerCallbacks): void; /** * Produce next frame. diff --git a/ts/@live-compositor/web-wasm/src/input/mp4/demuxer.ts b/ts/@live-compositor/web-wasm/src/input/mp4/demuxer.ts index 9c0810805..63727d1d3 100644 --- a/ts/@live-compositor/web-wasm/src/input/mp4/demuxer.ts +++ b/ts/@live-compositor/web-wasm/src/input/mp4/demuxer.ts @@ -7,6 +7,7 @@ import type { Framerate } from '../../compositor'; export type Mp4ReadyData = { decoderConfig: VideoDecoderConfig; framerate: Framerate; + videoDurationMs: number; }; export type MP4DemuxerCallbacks = { @@ -51,6 +52,7 @@ export class MP4Demuxer { } const videoTrack = info.videoTracks[0]; + const videoDurationMs = (videoTrack.movie_duration / videoTrack.movie_timescale) * 1000; const codecDescription = this.getCodecDescription(videoTrack.id); this.samplesCount = videoTrack.nb_samples; @@ -68,6 +70,7 @@ export class MP4Demuxer { this.callbacks.onReady({ decoderConfig, framerate, + videoDurationMs, }); this.file.setExtractionOptions(videoTrack.id); diff --git a/ts/@live-compositor/web-wasm/src/input/mp4/mp4box.d.ts b/ts/@live-compositor/web-wasm/src/input/mp4/mp4box.d.ts index df1131c38..351b00e0e 100644 --- a/ts/@live-compositor/web-wasm/src/input/mp4/mp4box.d.ts +++ b/ts/@live-compositor/web-wasm/src/input/mp4/mp4box.d.ts @@ -27,6 +27,7 @@ declare module 'mp4box' { export interface MP4MediaTrack { id: number; movie_duration: number; + movie_timescale: number; track_width: number; track_height: number; timescale: number; diff --git a/ts/@live-compositor/web-wasm/src/input/mp4/source.ts b/ts/@live-compositor/web-wasm/src/input/mp4/source.ts index 1bf433c2c..2075d2028 100644 --- a/ts/@live-compositor/web-wasm/src/input/mp4/source.ts +++ b/ts/@live-compositor/web-wasm/src/input/mp4/source.ts @@ -1,25 +1,19 @@ -import type { Mp4ReadyData } from './demuxer'; import { MP4Demuxer } from './demuxer'; import type InputSource from '../source'; -import type { InputSourceCallbacks, SourcePayload } from '../source'; +import type { InputSourceCallbacks, SourceMetadata, SourcePayload } from '../source'; import { Queue } from '@datastructures-js/queue'; -import type { Framerate } from '../../compositor'; export default class MP4Source implements InputSource { private fileUrl: string; private fileData?: ArrayBuffer; - private demuxer: MP4Demuxer; + private demuxer?: MP4Demuxer; private callbacks?: InputSourceCallbacks; private chunks: Queue; private eosReceived: boolean = false; - private framerate?: Framerate; + private metadata: SourceMetadata = {}; public constructor(fileUrl: string) { this.fileUrl = fileUrl; - this.demuxer = new MP4Demuxer({ - onReady: config => this.handleOnReady(config), - onPayload: payload => this.handlePayload(payload), - }); this.chunks = new Queue(); } @@ -28,13 +22,25 @@ export default class MP4Source implements InputSource { this.fileData = await resp.arrayBuffer(); } - public start(): void { - if (!this.fileData) { - throw new Error('MP4Source has to be initialized first before processing can be started'); - } + public async start(): Promise { + await new Promise(resolve => { + if (!this.fileData) { + throw new Error('MP4Source has to be initialized first before processing can be started'); + } + + this.demuxer = new MP4Demuxer({ + onReady: data => { + this.callbacks?.onDecoderConfig(data.decoderConfig); + this.metadata.framerate = data.framerate; + this.metadata.videoDurationMs = data.videoDurationMs; + resolve(); + }, + onPayload: payload => this.handlePayload(payload), + }); - this.demuxer.demux(this.fileData); - this.demuxer.flush(); + this.demuxer.demux(this.fileData); + this.demuxer.flush(); + }); } public registerCallbacks(callbacks: InputSourceCallbacks): void { @@ -45,8 +51,8 @@ export default class MP4Source implements InputSource { return this.eosReceived && this.chunks.isEmpty(); } - public getFramerate(): Framerate | undefined { - return this.framerate; + public getMetadata(): SourceMetadata { + return this.metadata; } public nextChunk(): EncodedVideoChunk | undefined { @@ -57,11 +63,6 @@ export default class MP4Source implements InputSource { return this.chunks.front(); } - private handleOnReady(data: Mp4ReadyData) { - this.callbacks?.onDecoderConfig(data.decoderConfig); - this.framerate = data.framerate; - } - private handlePayload(payload: SourcePayload) { if (payload.type === 'chunk') { this.chunks.push(payload.chunk); diff --git a/ts/@live-compositor/web-wasm/src/input/producer/decodingFrameProducer.ts b/ts/@live-compositor/web-wasm/src/input/producer/decodingFrameProducer.ts index fc817c4ff..6457c2b3d 100644 --- a/ts/@live-compositor/web-wasm/src/input/producer/decodingFrameProducer.ts +++ b/ts/@live-compositor/web-wasm/src/input/producer/decodingFrameProducer.ts @@ -5,6 +5,7 @@ import { FrameRef } from '../frame'; import type { InputFrameProducerCallbacks } from '../inputFrameProducer'; import type InputFrameProducer from '../inputFrameProducer'; import type InputSource from '../source'; +import type { InputStartInfo } from '../input'; const MAX_BUFFERING_SIZE = 3; @@ -32,8 +33,13 @@ export default class DecodingFrameProducer implements InputFrameProducer { await this.source.init(); } - public start(): void { - this.source.start(); + public async start(): Promise { + await this.source.start(); + + const metadata = this.source.getMetadata(); + return { + videoDurationMs: metadata.videoDurationMs, + }; } public registerCallbacks(callbacks: InputFrameProducerCallbacks): void { @@ -133,10 +139,10 @@ export default class DecodingFrameProducer implements InputFrameProducer { } private enqueueChunksForPts(framePts: number) { - const framerate = this.source.getFramerate(); - assert(framerate); + const metadata = this.source.getMetadata(); + assert(metadata.framerate); - const frameDuration = framerateToDurationMs(framerate); + const frameDuration = framerateToDurationMs(metadata.framerate); const targetPtsUs = (framePts + frameDuration * MAX_BUFFERING_SIZE) * 1000; let chunk = this.source.peekChunk(); diff --git a/ts/@live-compositor/web-wasm/src/input/source.ts b/ts/@live-compositor/web-wasm/src/input/source.ts index d80ac380a..fee0e84c1 100644 --- a/ts/@live-compositor/web-wasm/src/input/source.ts +++ b/ts/@live-compositor/web-wasm/src/input/source.ts @@ -2,6 +2,11 @@ import type { Framerate } from '../compositor'; export type SourcePayload = { type: 'chunk'; chunk: EncodedVideoChunk } | { type: 'eos' }; +export type SourceMetadata = { + framerate?: Framerate; + videoDurationMs?: number; +}; + export type InputSourceCallbacks = { onDecoderConfig: (config: VideoDecoderConfig) => void; }; @@ -14,13 +19,13 @@ export default interface InputSource { /** * Starts producing chunks. `init()` has to be called beforehand. */ - start(): void; + start(): Promise; registerCallbacks(callbacks: InputSourceCallbacks): void; /** * if `true` InputSource won't produce more chunks anymore. */ isFinished(): boolean; - getFramerate(): Framerate | undefined; + getMetadata(): SourceMetadata; nextChunk(): EncodedVideoChunk | undefined; peekChunk(): EncodedVideoChunk | undefined; } diff --git a/ts/@live-compositor/web-wasm/src/manager/wasmInstance.ts b/ts/@live-compositor/web-wasm/src/manager/wasmInstance.ts index 671ac8528..e2425d98a 100644 --- a/ts/@live-compositor/web-wasm/src/manager/wasmInstance.ts +++ b/ts/@live-compositor/web-wasm/src/manager/wasmInstance.ts @@ -2,6 +2,7 @@ import type { ApiRequest, CompositorManager, RegisterInputRequest, + RegisterInputResponse, RegisterOutputRequest, } from '@live-compositor/core'; import type { Renderer, Component, ImageSpec } from '@live-compositor/browser-render'; @@ -44,18 +45,18 @@ class WasmInstance implements CompositorManager { } if (route.type == 'input') { - await this.handleInputRequest(route.id, route.operation, request.body); + return await this.handleInputRequest(route.id, route.operation, request.body); } else if (route.type === 'output') { - this.handleOutputRequest(route.id, route.operation, request.body); + return this.handleOutputRequest(route.id, route.operation, request.body); } else if (route.type === 'image') { - await this.handleImageRequest(route.id, route.operation, request.body); + return await this.handleImageRequest(route.id, route.operation, request.body); } else if (route.type === 'shader') { throw new Error('Shaders are not supported'); } else if (route.type === 'web-renderer') { throw new Error('Web renderers are not supported'); + } else { + return {}; } - - return {}; } public registerEventListener(cb: (event: unknown) => void): void { @@ -81,23 +82,25 @@ class WasmInstance implements CompositorManager { inputId: string, operation: string, body?: object - ): Promise { + ): Promise { if (operation === 'register') { - await this.registerInput(inputId, body! as RegisterInputRequest); + return await this.registerInput(inputId, body! as RegisterInputRequest); } else if (operation === 'unregister') { - this.queue.removeInput(inputId); - this.renderer.unregisterInput(inputId); + return this.unregisterInput(inputId); + } else { + return {}; } } - private handleOutputRequest(outputId: string, operation: string, body?: object) { + private handleOutputRequest(outputId: string, operation: string, body?: object): object { if (operation === 'register') { - this.registerOutput(outputId, body! as RegisterOutputRequest); + return this.registerOutput(outputId, body! as RegisterOutputRequest); } else if (operation === 'unregister') { - this.queue.removeOutput(outputId); - this.renderer.unregisterOutput(outputId); + return this.unregisterOutput(outputId); } else if (operation === 'update') { - this.updateScene(outputId, body! as Api.UpdateOutputRequest); + return this.updateScene(outputId, body! as Api.UpdateOutputRequest); + } else { + return {}; } } @@ -105,15 +108,20 @@ class WasmInstance implements CompositorManager { imageId: string, operation: string, body?: object - ): Promise { + ): Promise { if (operation === 'register') { await this.renderer.registerImage(imageId, body as ImageSpec); } else if (operation === 'unregister') { this.renderer.unregisterImage(imageId); } + + return {}; } - private async registerInput(inputId: string, request: RegisterInputRequest): Promise { + private async registerInput( + inputId: string, + request: RegisterInputRequest + ): Promise { const frameProducer = producerFromRequest(request); await frameProducer.init(); @@ -121,10 +129,20 @@ class WasmInstance implements CompositorManager { // `addInput` will throw an exception if input already exists this.queue.addInput(inputId, input); this.renderer.registerInput(inputId); - input.start(); + + const startInfo = await input.start(); + return { + video_duration_ms: startInfo?.videoDurationMs, + }; + } + + private unregisterInput(inputId: string): object { + this.queue.removeInput(inputId); + this.renderer.unregisterInput(inputId); + return {}; } - private registerOutput(outputId: string, request: RegisterOutputRequest) { + private registerOutput(outputId: string, request: RegisterOutputRequest): object { if (request.video) { const output = new Output(request); this.queue.addOutput(outputId, output); @@ -142,17 +160,27 @@ class WasmInstance implements CompositorManager { throw e; } } + + return {}; + } + + private unregisterOutput(outputId: string): object { + this.queue.removeOutput(outputId); + this.renderer.unregisterOutput(outputId); + return {}; } - private updateScene(outputId: string, request: Api.UpdateOutputRequest) { + private updateScene(outputId: string, request: Api.UpdateOutputRequest): object { if (!request.video) { - return; + return {}; } const output = this.queue.getOutput(outputId); if (!output) { throw `Unknown output "${outputId}"`; } this.renderer.updateScene(outputId, output.resolution, request.video.root as Component); + + return {}; } } diff --git a/ts/examples/vite-browser-render/src/examples/SimpleMp4Example.tsx b/ts/examples/vite-browser-render/src/examples/SimpleMp4Example.tsx index cdc301834..44b3eb752 100644 --- a/ts/examples/vite-browser-render/src/examples/SimpleMp4Example.tsx +++ b/ts/examples/vite-browser-render/src/examples/SimpleMp4Example.tsx @@ -11,7 +11,7 @@ function SimpleMp4Example() { await compositor.registerFont( 'https://fonts.gstatic.com/s/notosans/v36/o-0mIpQlx3QUlC5A4PNB6Ryti20_6n1iPHjcz6L1SoM-jCpoiyD9A-9a6Vc.ttf' ); - await compositor.registerInput('bunny_video', { type: 'mp4', url: MP4_URL }); + await compositor.registerInput('video', { type: 'mp4', url: MP4_URL }); }, []); return ( @@ -25,12 +25,12 @@ function SimpleMp4Example() { function Scene() { const inputs = useInputStreams(); - const inputState = inputs['bunny_video']?.videoState; + const inputState = inputs['video']?.videoState; if (inputState === 'playing') { return ( - + Playing MP4 file