From 89616173a694cb7c9772e28be11f2c8c9a8971c8 Mon Sep 17 00:00:00 2001 From: Wojciech Kozyra Date: Wed, 18 Dec 2024 13:18:36 +0100 Subject: [PATCH] wip --- .../browser-render/src/wasm.ts | 2 +- .../core/src/compositorManager.ts | 1 + .../core/src/live/compositor.ts | 9 ++- ts/@live-compositor/core/src/live/output.ts | 46 +++++------ .../core/src/offline/compositor.ts | 2 +- .../core/src/offline/output.ts | 7 +- ts/@live-compositor/core/src/renderer.ts | 11 +++ ts/@live-compositor/core/src/rootComponent.ts | 42 +--------- ts/@live-compositor/core/src/utils.ts | 56 ++++++++----- ts/@live-compositor/node/src/logger.ts | 2 +- .../node/src/manager/existingInstance.ts | 6 +- .../src/manager/locallySpawnedInstance.ts | 18 ++++- ts/@live-compositor/node/src/spawn.ts | 20 +++++ ts/@live-compositor/node/src/ws.ts | 14 +++- .../web-wasm/src/compositor.ts | 53 ++++++++++--- ts/@live-compositor/web-wasm/src/index.ts | 4 +- .../web-wasm/src/manager/wasmInstance.ts | 14 ++-- ts/@live-compositor/web-wasm/src/utils.ts | 8 +- ts/examples/node-examples/src/combine_mp4.tsx | 1 - ts/examples/node-examples/src/concat_mp4.tsx | 1 - .../node-examples/src/dynamic-outputs.tsx | 1 + .../node-examples/src/dynamic-text.tsx | 2 +- ts/examples/vite-browser-render/src/App.tsx | 49 ++++++++++-- .../src/components/CompositorCanvas.tsx | 60 ++++++++++++++ ...{MP4Player.tsx => MultipleCompositors.tsx} | 78 +++++-------------- .../src/examples/SimpleMp4Example.tsx | 60 ++++++++++++++ 26 files changed, 375 insertions(+), 192 deletions(-) create mode 100644 ts/examples/vite-browser-render/src/components/CompositorCanvas.tsx rename ts/examples/vite-browser-render/src/examples/{MP4Player.tsx => MultipleCompositors.tsx} (50%) create mode 100644 ts/examples/vite-browser-render/src/examples/SimpleMp4Example.tsx diff --git a/ts/@live-compositor/browser-render/src/wasm.ts b/ts/@live-compositor/browser-render/src/wasm.ts index af037465b..481bd4b6b 100644 --- a/ts/@live-compositor/browser-render/src/wasm.ts +++ b/ts/@live-compositor/browser-render/src/wasm.ts @@ -5,7 +5,7 @@ import init, * as wasm from './generated/compositor_web'; * @param wasmModuleUrl {string} - An URL for `live-compositor.wasm` file. The file is located in `dist` folder. */ export async function loadWasmModule(wasmModuleUrl: string) { - await init(wasmModuleUrl); + await init({ module_or_path: wasmModuleUrl }); } export { wasm }; diff --git a/ts/@live-compositor/core/src/compositorManager.ts b/ts/@live-compositor/core/src/compositorManager.ts index 3292737d4..04b64df2e 100644 --- a/ts/@live-compositor/core/src/compositorManager.ts +++ b/ts/@live-compositor/core/src/compositorManager.ts @@ -14,4 +14,5 @@ export interface CompositorManager { setupInstance(opts: SetupInstanceOptions): Promise; sendRequest(request: ApiRequest): Promise; registerEventListener(cb: (event: unknown) => void): void; + terminate(): Promise; } diff --git a/ts/@live-compositor/core/src/live/compositor.ts b/ts/@live-compositor/core/src/live/compositor.ts index 9969736d3..700125f66 100644 --- a/ts/@live-compositor/core/src/live/compositor.ts +++ b/ts/@live-compositor/core/src/live/compositor.ts @@ -61,7 +61,7 @@ export class LiveCompositor { public async unregisterOutput(outputId: string): Promise { this.logger.info({ outputId }, 'Unregister output'); - this.outputs[outputId].close(); + await this.outputs[outputId].close(); delete this.outputs[outputId]; // TODO: wait for event return this.api.unregisterOutput(outputId, {}); @@ -140,6 +140,13 @@ export class LiveCompositor { this.startTime = startTime; } + public async terminate(): Promise { + for (const output of Object.values(this.outputs)) { + await output.close(); + } + await this.manager.terminate(); + } + private handleEvent(rawEvent: unknown) { const event = parseEvent(rawEvent, this.logger); if (!event) { diff --git a/ts/@live-compositor/core/src/live/output.ts b/ts/@live-compositor/core/src/live/output.ts index 72c7081b1..b863ad9ee 100644 --- a/ts/@live-compositor/core/src/live/output.ts +++ b/ts/@live-compositor/core/src/live/output.ts @@ -6,8 +6,8 @@ import type { ApiClient, Api } from '../api.js'; import Renderer from '../renderer.js'; import type { RegisterOutput } from '../api/output.js'; import { intoAudioInputsConfiguration } from '../api/output.js'; -import { throttle } from '../utils.js'; -import { OutputRootComponent, OutputShutdownStateStore } from '../rootComponent.js'; +import { ThrottledFunction } from '../utils.js'; +import { OutputRootComponent } from '../rootComponent.js'; import type { Logger } from 'pino'; type AudioContext = _liveCompositorInternals.AudioContext; @@ -21,11 +21,10 @@ class Output { audioContext: AudioContext; timeContext: LiveTimeContext; internalInputStreamStore: LiveInputStreamStore; - outputShutdownStateStore: OutputShutdownStateStore; logger: Logger; shouldUpdateWhenReady: boolean = false; - throttledUpdate: () => void; + throttledUpdate: ThrottledFunction; supportsAudio: boolean; supportsVideo: boolean; @@ -44,16 +43,21 @@ class Output { this.api = api; this.logger = logger; this.outputId = outputId; - this.outputShutdownStateStore = new OutputShutdownStateStore(); this.shouldUpdateWhenReady = false; - this.throttledUpdate = () => { - this.shouldUpdateWhenReady = true; - }; + this.throttledUpdate = new ThrottledFunction( + async () => { + this.shouldUpdateWhenReady = true; + }, + { + timeoutMs: 30, + logger: this.logger, + } + ); this.supportsAudio = 'audio' in registerRequest && !!registerRequest.audio; this.supportsVideo = 'video' in registerRequest && !!registerRequest.video; - const onUpdate = () => this.throttledUpdate(); + const onUpdate = () => this.throttledUpdate.scheduleCall(); this.audioContext = new _liveCompositorInternals.AudioContext(onUpdate); this.timeContext = new _liveCompositorInternals.LiveTimeContext(); this.internalInputStreamStore = new _liveCompositorInternals.LiveInputStreamStore(this.logger); @@ -64,7 +68,6 @@ class Output { const rootElement = createElement(OutputRootComponent, { outputContext: new OutputContext(this, this.outputId, store), outputRoot: root, - outputShutdownStateStore: this.outputShutdownStateStore, childrenLifetimeContext: new _liveCompositorInternals.ChildrenLifetimeContext(() => {}), }); @@ -87,25 +90,18 @@ class Output { }; } - public close(): void { - this.throttledUpdate = () => {}; - // close will switch a scene to just a , so we need replace `throttledUpdate` - // callback before it is called - this.outputShutdownStateStore.close(); + public async close(): Promise { + this.throttledUpdate.setFn(async () => {}); + this.renderer.stop(); + await this.throttledUpdate.waitForPendingCalls(); } public async ready() { - this.throttledUpdate = throttle( - async () => { - await this.api.updateScene(this.outputId, this.scene()); - }, - { - timeoutMs: 30, - logger: this.logger, - } - ); + this.throttledUpdate.setFn(async () => { + await this.api.updateScene(this.outputId, this.scene()); + }); if (this.shouldUpdateWhenReady) { - this.throttledUpdate(); + this.throttledUpdate.scheduleCall(); } } diff --git a/ts/@live-compositor/core/src/offline/compositor.ts b/ts/@live-compositor/core/src/offline/compositor.ts index c6385f8d3..a10b04329 100644 --- a/ts/@live-compositor/core/src/offline/compositor.ts +++ b/ts/@live-compositor/core/src/offline/compositor.ts @@ -56,7 +56,6 @@ export class OfflineCompositor { await this.api.registerOutput(OFFLINE_OUTPUT_ID, apiRequest); await output.scheduleAllUpdates(); // at this point all scene update requests should already be delivered - output.outputShutdownStateStore.close(); if (durationMs) { await this.api.unregisterOutput(OFFLINE_OUTPUT_ID, { schedule_time_ms: durationMs }); @@ -78,6 +77,7 @@ export class OfflineCompositor { await this.api.start(); await renderPromise; + await this.manager.terminate(); } public async registerInput(inputId: string, request: RegisterInput): Promise { diff --git a/ts/@live-compositor/core/src/offline/output.ts b/ts/@live-compositor/core/src/offline/output.ts index 6c7e97cc1..d0d243208 100644 --- a/ts/@live-compositor/core/src/offline/output.ts +++ b/ts/@live-compositor/core/src/offline/output.ts @@ -8,7 +8,7 @@ import type { RegisterOutput } from '../api/output.js'; import { intoAudioInputsConfiguration } from '../api/output.js'; import { sleep } from '../utils.js'; import { OFFLINE_OUTPUT_ID } from './compositor.js'; -import { OutputRootComponent, OutputShutdownStateStore } from '../rootComponent.js'; +import { OutputRootComponent } from '../rootComponent.js'; import type { Logger } from 'pino'; type AudioContext = _liveCompositorInternals.AudioContext; @@ -26,7 +26,6 @@ class OfflineOutput { timeContext: OfflineTimeContext; childrenLifetimeContext: ChildrenLifetimeContext; internalInputStreamStore: OfflineInputStreamStore; - outputShutdownStateStore: OutputShutdownStateStore; logger: Logger; durationMs?: number; @@ -48,7 +47,6 @@ class OfflineOutput { this.api = api; this.logger = logger; this.outputId = OFFLINE_OUTPUT_ID; - this.outputShutdownStateStore = new OutputShutdownStateStore(); this.durationMs = durationMs; this.supportsAudio = 'audio' in registerRequest && !!registerRequest.audio; @@ -70,7 +68,6 @@ class OfflineOutput { const rootElement = createElement(OutputRootComponent, { outputContext: new OutputContext(this, this.outputId, store), outputRoot: root, - outputShutdownStateStore: this.outputShutdownStateStore, childrenLifetimeContext: this.childrenLifetimeContext, }); @@ -117,7 +114,7 @@ class OfflineOutput { this.timeContext.setNextTimestamp(); } - this.outputShutdownStateStore.close(); + this.renderer.stop(); } } diff --git a/ts/@live-compositor/core/src/renderer.ts b/ts/@live-compositor/core/src/renderer.ts index cc3027888..41a555eb4 100644 --- a/ts/@live-compositor/core/src/renderer.ts +++ b/ts/@live-compositor/core/src/renderer.ts @@ -236,6 +236,7 @@ class Renderer { public readonly root: FiberRootNode; public readonly onUpdate: () => void; private logger: Logger; + private lastScene?: Api.Component; constructor({ rootElement, onUpdate, idPrefix, logger }: RendererOptions) { this.logger = logger; @@ -256,6 +257,11 @@ class Renderer { } public scene(): Api.Component { + if (this.lastScene) { + // Renderer was already stopped just return old scene + return this.lastScene; + } + // When resetAfterCommit is called `this.root.current` is not updated yet, so we need to rely // on `pendingChildren`. I'm not sure it is always populated, so there is a fallback to // `root.current`. @@ -264,6 +270,11 @@ class Renderer { this.root.pendingChildren[0] ?? rootHostComponent(this.root.current, this.logger); return rootComponent.scene(); } + + public stop() { + this.lastScene = this.scene(); + CompositorRenderer.updateContainer(null, this.root, null, () => {}); + } } function rootHostComponent(root: any, logger: Logger): LiveCompositorHostComponent { diff --git a/ts/@live-compositor/core/src/rootComponent.ts b/ts/@live-compositor/core/src/rootComponent.ts index 475464826..ddee58d83 100644 --- a/ts/@live-compositor/core/src/rootComponent.ts +++ b/ts/@live-compositor/core/src/rootComponent.ts @@ -1,59 +1,21 @@ -import { _liveCompositorInternals, useAfterTimestamp, View } from 'live-compositor'; -import { createElement, useEffect, useSyncExternalStore, type ReactElement } from 'react'; +import { _liveCompositorInternals, useAfterTimestamp } from 'live-compositor'; +import { createElement, useEffect, type ReactElement } from 'react'; type CompositorOutputContext = _liveCompositorInternals.CompositorOutputContext; type ChildrenLifetimeContext = _liveCompositorInternals.ChildrenLifetimeContext; -// External store to share shutdown information between React tree -// and external code that is managing it. -export class OutputShutdownStateStore { - private shutdown: boolean = false; - private onChangeCallbacks: Set<() => void> = new Set(); - - public close() { - this.shutdown = true; - this.onChangeCallbacks.forEach(cb => cb()); - } - - // callback for useSyncExternalStore - public getSnapshot = (): boolean => { - return this.shutdown; - }; - - // callback for useSyncExternalStore - public subscribe = (onStoreChange: () => void): (() => void) => { - this.onChangeCallbacks.add(onStoreChange); - return () => { - this.onChangeCallbacks.delete(onStoreChange); - }; - }; -} - const globalDelayRef = Symbol(); export function OutputRootComponent({ outputContext, outputRoot, - outputShutdownStateStore, childrenLifetimeContext, }: { outputContext: CompositorOutputContext; outputRoot: ReactElement; - outputShutdownStateStore: OutputShutdownStateStore; childrenLifetimeContext: ChildrenLifetimeContext; }) { - const shouldShutdown = useSyncExternalStore( - outputShutdownStateStore.subscribe, - outputShutdownStateStore.getSnapshot - ); - useMinimalStreamDuration(childrenLifetimeContext); - - if (shouldShutdown) { - // replace root with view to stop all the dynamic code - return createElement(View, {}); - } - return createElement( _liveCompositorInternals.LiveCompositorContext.Provider, { value: outputContext }, diff --git a/ts/@live-compositor/core/src/utils.ts b/ts/@live-compositor/core/src/utils.ts index 08916cd3b..0e400b495 100644 --- a/ts/@live-compositor/core/src/utils.ts +++ b/ts/@live-compositor/core/src/utils.ts @@ -5,37 +5,53 @@ type ThrottleOptions = { timeoutMs: number; }; -export function throttle(fn: () => Promise, opts: ThrottleOptions): () => void { - let shouldCall: boolean = false; - let running: boolean = false; +export class ThrottledFunction { + private fn: () => Promise; + private shouldCall: boolean = false; + private runningPromise?: Promise = undefined; + private opts: ThrottleOptions; - const start = async () => { - while (shouldCall) { + constructor(fn: () => Promise, opts: ThrottleOptions) { + this.opts = opts; + this.fn = fn; + } + + public scheduleCall() { + this.shouldCall = true; + if (this.runningPromise) { + return; + } + this.runningPromise = this.doCall(); + } + + public async waitForPendingCalls(): Promise { + while (this.runningPromise) { + await this.runningPromise; + } + } + + public setFn(fn: () => Promise) { + this.fn = fn; + } + + private async doCall() { + while (this.shouldCall) { const start = Date.now(); - shouldCall = false; + this.shouldCall = false; try { - await fn(); + await this.fn(); } catch (error) { - opts.logger.error(error); + this.opts.logger.error(error); } - const timeoutLeft = start + opts.timeoutMs - Date.now(); + const timeoutLeft = start + this.opts.timeoutMs - Date.now(); if (timeoutLeft > 0) { await sleep(timeoutLeft); } - running = false; - } - }; - - return () => { - shouldCall = true; - if (running) { - return; + this.runningPromise = undefined; } - running = true; - void start(); - }; + } } export async function sleep(timeoutMs: number): Promise { diff --git a/ts/@live-compositor/node/src/logger.ts b/ts/@live-compositor/node/src/logger.ts index 4a8591aef..e688e90c5 100644 --- a/ts/@live-compositor/node/src/logger.ts +++ b/ts/@live-compositor/node/src/logger.ts @@ -43,7 +43,7 @@ export function compositorInstanceLoggerOptions(): { if ([LoggerLevel.WARN, LoggerLevel.ERROR].includes(loggerLevel)) { return { - level: loggerLevel, + level: LoggerLevel.ERROR, format, }; } else if (loggerLevel === LoggerLevel.INFO) { diff --git a/ts/@live-compositor/node/src/manager/existingInstance.ts b/ts/@live-compositor/node/src/manager/existingInstance.ts index 8efd4cdbc..f94abdedb 100644 --- a/ts/@live-compositor/node/src/manager/existingInstance.ts +++ b/ts/@live-compositor/node/src/manager/existingInstance.ts @@ -45,7 +45,11 @@ class ExistingInstance implements CompositorManager { } public registerEventListener(cb: (event: object) => void): void { - this.wsConnection?.registerEventListener(cb); + this.wsConnection.registerEventListener(cb); + } + + public async terminate(): Promise { + await this.wsConnection.close(); } } diff --git a/ts/@live-compositor/node/src/manager/locallySpawnedInstance.ts b/ts/@live-compositor/node/src/manager/locallySpawnedInstance.ts index 22bf1485b..3b72618fe 100644 --- a/ts/@live-compositor/node/src/manager/locallySpawnedInstance.ts +++ b/ts/@live-compositor/node/src/manager/locallySpawnedInstance.ts @@ -8,7 +8,8 @@ import type { ApiRequest, CompositorManager, SetupInstanceOptions } from '@live- import { download, sendRequest } from '../fetch'; import { retry, sleep } from '../utils'; -import { spawn } from '../spawn'; +import type { SpawnPromise } from '../spawn'; +import { killProcess, spawn } from '../spawn'; import { WebSocketConnection } from '../ws'; import { compositorInstanceLoggerOptions } from '../logger'; @@ -30,6 +31,7 @@ class LocallySpawnedInstance implements CompositorManager { private executablePath?: string; private wsConnection: WebSocketConnection; private enableWebRenderer?: boolean; + private childSpawnPromise?: SpawnPromise; constructor(opts: ManagedInstanceOptions) { this.port = opts.port; @@ -65,8 +67,9 @@ class LocallySpawnedInstance implements CompositorManager { LIVE_COMPOSITOR_LOGGER_FORMAT: format, LIVE_COMPOSITOR_LOGGER_LEVEL: level, }; - spawn(executablePath, [], { env, stdio: 'inherit' }).catch(err => { - opts.logger.error({ err }, 'LiveCompositor instance failed'); + this.childSpawnPromise = spawn(executablePath, [], { env, stdio: 'inherit' }); + this.childSpawnPromise.catch(err => { + opts.logger.error(err, 'LiveCompositor instance failed'); // TODO: parse structured logging from compositor and send them to this logger if (err.stderr) { console.error(err.stderr); @@ -92,7 +95,14 @@ class LocallySpawnedInstance implements CompositorManager { } public registerEventListener(cb: (event: object) => void): void { - this.wsConnection?.registerEventListener(cb); + this.wsConnection.registerEventListener(cb); + } + + public async terminate(): Promise { + await this.wsConnection.close(); + if (this.childSpawnPromise) { + await killProcess(this.childSpawnPromise); + } } } diff --git a/ts/@live-compositor/node/src/spawn.ts b/ts/@live-compositor/node/src/spawn.ts index 20b2f8182..b9affa16b 100644 --- a/ts/@live-compositor/node/src/spawn.ts +++ b/ts/@live-compositor/node/src/spawn.ts @@ -1,5 +1,6 @@ import type { ChildProcess, SpawnOptions } from 'child_process'; import { spawn as nodeSpawn } from 'child_process'; +import { sleep } from './utils'; export interface SpawnPromise extends Promise { child: ChildProcess; @@ -44,3 +45,22 @@ export function spawn(command: string, args: string[], options: SpawnOptions): S promise.child = child; return promise; } + +export async function killProcess(spawnPromise: SpawnPromise): Promise { + spawnPromise.child.kill('SIGINT'); + const start = Date.now(); + while (isProcessRunning(spawnPromise)) { + if (Date.now() - start > 5000) { + spawnPromise.child.kill('SIGKILL'); + } + await sleep(100); + } +} + +function isProcessRunning(spawnPromise: SpawnPromise): boolean { + try { + return !!spawnPromise.child.kill(0); + } catch (e: any) { + return e.code === 'EPERM'; + } +} diff --git a/ts/@live-compositor/node/src/ws.ts b/ts/@live-compositor/node/src/ws.ts index 1837c54ec..5c2788b3f 100644 --- a/ts/@live-compositor/node/src/ws.ts +++ b/ts/@live-compositor/node/src/ws.ts @@ -4,8 +4,8 @@ import WebSocket from 'ws'; export class WebSocketConnection { private url: string; private listeners: Set<(event: object) => void>; - // @ts-expect-error: unused if we don't send messages via WebSocket private ws: WebSocket | null = null; + private donePromise?: Promise; constructor(url: string) { this.url = url; @@ -39,8 +39,11 @@ export class WebSocketConnection { } }); - ws.on('close', () => { - this.ws = null; + this.donePromise = new Promise(res => { + ws.on('close', () => { + this.ws = null; + res(); + }); }); }); this.ws = ws; @@ -49,6 +52,11 @@ export class WebSocketConnection { public registerEventListener(cb: (event: object) => void): void { this.listeners.add(cb); } + + public async close(): Promise { + this.ws?.close(); + await this.donePromise; + } } function parseEvent(data: WebSocket.RawData, logger: Logger): unknown { diff --git a/ts/@live-compositor/web-wasm/src/compositor.ts b/ts/@live-compositor/web-wasm/src/compositor.ts index b4faba7b2..5f54fb0b1 100644 --- a/ts/@live-compositor/web-wasm/src/compositor.ts +++ b/ts/@live-compositor/web-wasm/src/compositor.ts @@ -1,4 +1,4 @@ -import { Renderer } from '@live-compositor/browser-render'; +import { loadWasmModule, Renderer } from '@live-compositor/browser-render'; import { LiveCompositor as CoreLiveCompositor } from '@live-compositor/core'; import WasmInstance from './manager/wasmInstance'; import type { RegisterOutput } from './output/registerOutput'; @@ -9,6 +9,7 @@ import type { RegisterImage } from './renderers'; import type { ReactElement } from 'react'; import type { Logger } from 'pino'; import { pino } from 'pino'; +import { assert } from './utils'; export type LiveCompositorOptions = { framerate?: Framerate; @@ -20,6 +21,16 @@ export type Framerate = { den: number; }; +let wasmBundleUrl: string | undefined; + +/* + * Defines url where WASM bundle is hosted. This method needs to be called before + * first LiveCompositor instance is initiated. + */ +export function setWasmBundleUrl(url: string) { + wasmBundleUrl = url; +} + export default class LiveCompositor { private coreCompositor?: CoreLiveCompositor; private instance?: WasmInstance; @@ -36,6 +47,7 @@ export default class LiveCompositor { * Outputs won't produce any results until `start()` is called. */ public async init(): Promise { + await ensureWasmModuleLoaded(); this.renderer = await Renderer.create({ streamFallbackTimeoutMs: this.options.streamFallbackTimeoutMs ?? 500, }); @@ -43,7 +55,7 @@ export default class LiveCompositor { renderer: this.renderer!, framerate: this.options.framerate ?? { num: 30, den: 1 }, }); - this.coreCompositor = new CoreLiveCompositor(this.instance!, this.logger); + this.coreCompositor = new CoreLiveCompositor(this.instance, this.logger); await this.coreCompositor!.init(); } @@ -53,31 +65,38 @@ export default class LiveCompositor { root: ReactElement, request: RegisterOutput ): Promise { - await this.coreCompositor!.registerOutput(outputId, root, intoRegisterOutput(request)); + assert(this.coreCompositor); + await this.coreCompositor.registerOutput(outputId, root, intoRegisterOutput(request)); } public async unregisterOutput(outputId: string): Promise { - await this.coreCompositor!.unregisterOutput(outputId); + assert(this.coreCompositor); + await this.coreCompositor.unregisterOutput(outputId); } public async registerInput(inputId: string, request: RegisterInput): Promise { - await this.coreCompositor!.registerInput(inputId, intoRegisterInput(request)); + assert(this.coreCompositor); + await this.coreCompositor.registerInput(inputId, intoRegisterInput(request)); } public async unregisterInput(inputId: string): Promise { - await this.coreCompositor!.unregisterInput(inputId); + assert(this.coreCompositor); + await this.coreCompositor.unregisterInput(inputId); } public async registerImage(imageId: string, request: RegisterImage): Promise { - await this.coreCompositor!.registerImage(imageId, request); + assert(this.coreCompositor); + await this.coreCompositor.registerImage(imageId, request); } public async unregisterImage(imageId: string): Promise { - await this.coreCompositor!.unregisterImage(imageId); + assert(this.coreCompositor); + await this.coreCompositor.unregisterImage(imageId); } public async registerFont(fontUrl: string): Promise { - await this.renderer!.registerFont(fontUrl); + assert(this.renderer); + await this.renderer.registerFont(fontUrl); } /** @@ -90,7 +109,19 @@ export default class LiveCompositor { /** * Stops processing pipeline. */ - public stop(): void { - this.instance!.stop(); + public async terminate(): Promise { + await this.coreCompositor?.terminate(); + await this.instance?.terminate(); } } + +const ensureWasmModuleLoaded = (() => { + let loadedState: Promise | undefined = undefined; + return async () => { + assert(wasmBundleUrl, 'Location of WASM bundle is not defined, call setWasmBundleUrl() first.'); + if (!loadedState) { + loadedState = loadWasmModule(wasmBundleUrl); + } + await loadedState; + }; +})(); diff --git a/ts/@live-compositor/web-wasm/src/index.ts b/ts/@live-compositor/web-wasm/src/index.ts index 4c5a74f2c..2357b7bf7 100644 --- a/ts/@live-compositor/web-wasm/src/index.ts +++ b/ts/@live-compositor/web-wasm/src/index.ts @@ -1,4 +1,4 @@ import WasmInstance from './manager/wasmInstance'; -import LiveCompositor from './compositor'; +import LiveCompositor, { setWasmBundleUrl } from './compositor'; -export { WasmInstance, LiveCompositor }; +export { WasmInstance, LiveCompositor, setWasmBundleUrl }; diff --git a/ts/@live-compositor/web-wasm/src/manager/wasmInstance.ts b/ts/@live-compositor/web-wasm/src/manager/wasmInstance.ts index 1005d1023..671ac8528 100644 --- a/ts/@live-compositor/web-wasm/src/manager/wasmInstance.ts +++ b/ts/@live-compositor/web-wasm/src/manager/wasmInstance.ts @@ -62,19 +62,19 @@ class WasmInstance implements CompositorManager { this.eventSender.setEventCallback(cb); } - private start() { + public async terminate(): Promise { + // TODO(noituri): Clean all remaining `InputFrame`s & stop input processing if (this.stopQueue) { - throw new Error('Compositor is already running'); + this.stopQueue(); + this.stopQueue = undefined; } - this.stopQueue = this.queue.start(); } - public stop() { - // TODO(noituri): Clean all remaining `InputFrame`s & stop input processing + private start() { if (this.stopQueue) { - this.stopQueue(); - this.stopQueue = undefined; + throw new Error('Compositor is already running'); } + this.stopQueue = this.queue.start(); } private async handleInputRequest( diff --git a/ts/@live-compositor/web-wasm/src/utils.ts b/ts/@live-compositor/web-wasm/src/utils.ts index 6288225ca..5831b1472 100644 --- a/ts/@live-compositor/web-wasm/src/utils.ts +++ b/ts/@live-compositor/web-wasm/src/utils.ts @@ -1,8 +1,12 @@ import type { Framerate } from './compositor'; -export function assert(value: T): asserts value { +export function assert(value: T, msg?: string): asserts value { if (!value) { - throw new Error('Assertion failed'); + if (msg) { + throw new Error(msg); + } else { + throw new Error('Assertion failed'); + } } } diff --git a/ts/examples/node-examples/src/combine_mp4.tsx b/ts/examples/node-examples/src/combine_mp4.tsx index 12c70d7cb..e5109fddc 100644 --- a/ts/examples/node-examples/src/combine_mp4.tsx +++ b/ts/examples/node-examples/src/combine_mp4.tsx @@ -85,6 +85,5 @@ async function run() { }, 10000 ); - process.exit(0); } void run(); diff --git a/ts/examples/node-examples/src/concat_mp4.tsx b/ts/examples/node-examples/src/concat_mp4.tsx index ff63fdd08..e4e5c326b 100644 --- a/ts/examples/node-examples/src/concat_mp4.tsx +++ b/ts/examples/node-examples/src/concat_mp4.tsx @@ -127,6 +127,5 @@ async function run() { }, }, }); - process.exit(0); } void run(); diff --git a/ts/examples/node-examples/src/dynamic-outputs.tsx b/ts/examples/node-examples/src/dynamic-outputs.tsx index a0c5c46da..500620152 100644 --- a/ts/examples/node-examples/src/dynamic-outputs.tsx +++ b/ts/examples/node-examples/src/dynamic-outputs.tsx @@ -115,5 +115,6 @@ async function run() { console.log('Stop all remaining outputs.'); await compositor.unregisterOutput('output_recording_part2'); await compositor.unregisterOutput('output_stream'); + await compositor.terminate(); } void run(); diff --git a/ts/examples/node-examples/src/dynamic-text.tsx b/ts/examples/node-examples/src/dynamic-text.tsx index 4a6b3ab2e..90c3d8cec 100644 --- a/ts/examples/node-examples/src/dynamic-text.tsx +++ b/ts/examples/node-examples/src/dynamic-text.tsx @@ -55,7 +55,7 @@ async function run() { const compositor = new LiveCompositor(); await compositor.init(); - void ffplayStartPlayerAsync('127.0.0.1', 8001); + await ffplayStartPlayerAsync('127.0.0.1', 8001); await sleep(2000); await compositor.registerOutput('output_1', , { diff --git a/ts/examples/vite-browser-render/src/App.tsx b/ts/examples/vite-browser-render/src/App.tsx index 9259b9bc6..1763b79cc 100644 --- a/ts/examples/vite-browser-render/src/App.tsx +++ b/ts/examples/vite-browser-render/src/App.tsx @@ -1,26 +1,59 @@ import { useState } from 'react'; import './App.css'; import Counter from './examples/Counter'; -import MP4Player from './examples/MP4Player'; +import SimpleMp4Example from './examples/SimpleMp4Example'; +import MultipleCompositors from './examples/MultipleCompositors'; +import { setWasmBundleUrl } from '@live-compositor/web-wasm'; -const EXAMPLES = { - counter: , - mp4: , -}; +setWasmBundleUrl('assets/live-compositor.wasm'); function App() { - const [currentExample, setCurrentExample] = useState('counter'); + const EXAMPLES = { + counter: , + simpleMp4: , + multipleCompositors: , + home: , + }; + const [currentExample, setCurrentExample] = useState('home'); return ( <> -

Browser Renderer Examples

+

Examples

+ + + -
{EXAMPLES[currentExample]}
); } +function Home() { + return ( +
+

Packages:

+

+ @live-compositor/web-wasm - LiveCompositor in the browser +

+
  • + Simple Mp4 - Take MP4 file as an input and render output on canvas +
    • +
  • + Multiple LiveCompositor instances - Runs multiple LiveCompositor instances at + the same time. +
    • +

    + @live-compositor/browser-render - Rendering engine from LiveCompositor +

    +
  • + Counter - Render a GIF + counter trigged by user(with a button). +
    • +
    + ); +} + export default App; diff --git a/ts/examples/vite-browser-render/src/components/CompositorCanvas.tsx b/ts/examples/vite-browser-render/src/components/CompositorCanvas.tsx new file mode 100644 index 000000000..37234dd73 --- /dev/null +++ b/ts/examples/vite-browser-render/src/components/CompositorCanvas.tsx @@ -0,0 +1,60 @@ +import React, { useCallback, useEffect, useState } from 'react'; +import { LiveCompositor } from '@live-compositor/web-wasm'; + +type CanvasProps = React.DetailedHTMLProps< + React.CanvasHTMLAttributes, + HTMLCanvasElement +>; + +type CompositorCanvasProps = { + onCanvasCreate?: (compositor: LiveCompositor) => Promise; + onCanvasStarted?: (compositor: LiveCompositor) => Promise; + children: React.ReactElement; +} & CanvasProps; + +export default function CompositorCanvas(props: CompositorCanvasProps) { + const { onCanvasCreate, onCanvasStarted, children, ...canvasProps } = props; + + const [compositor, setCompositor] = useState(undefined); + const canvasRef = useCallback( + async (canvas: HTMLCanvasElement | null) => { + if (!canvas) { + return; + } + const compositor = new LiveCompositor({}); + + await compositor.init(); + + if (onCanvasCreate) { + await onCanvasCreate(compositor); + } + + await compositor.registerOutput('output', children, { + type: 'canvas', + canvas: canvas, + resolution: { + width: Number(canvasProps.width ?? canvas.width), + height: Number(canvasProps.height ?? canvas.height), + }, + }); + + await compositor.start(); + setCompositor(compositor); + + if (onCanvasStarted) { + await onCanvasStarted(compositor); + } + }, + [onCanvasCreate, onCanvasStarted, canvasProps.width, canvasProps.height, children] + ); + + useEffect(() => { + return () => { + if (compositor) { + void compositor.terminate(); + } + }; + }, [compositor]); + + return ; +} diff --git a/ts/examples/vite-browser-render/src/examples/MP4Player.tsx b/ts/examples/vite-browser-render/src/examples/MultipleCompositors.tsx similarity index 50% rename from ts/examples/vite-browser-render/src/examples/MP4Player.tsx rename to ts/examples/vite-browser-render/src/examples/MultipleCompositors.tsx index abc19d1c2..9800bec0c 100644 --- a/ts/examples/vite-browser-render/src/examples/MP4Player.tsx +++ b/ts/examples/vite-browser-render/src/examples/MultipleCompositors.tsx @@ -1,28 +1,28 @@ -import { useCallback, useEffect, useState } from 'react'; -import { LiveCompositor } from '@live-compositor/web-wasm'; +import { useCallback } from 'react'; +import type { LiveCompositor } from '@live-compositor/web-wasm'; import { InputStream, Text, useInputStreams, View } from 'live-compositor'; +import CompositorCanvas from '../components/CompositorCanvas'; -const BUNNY_URL = - 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4'; +const MP4_URL = + 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4'; -function MP4Player() { - const [compositor, canvasRef] = useCompositor(); - - useEffect(() => { - if (compositor == null) { - return; - } - - void compositor.start(); - return () => compositor.stop(); - }, [compositor]); +function MultipleCompositors() { + const onCanvasCreate = useCallback(async (compositor: LiveCompositor) => { + await compositor.registerFont( + 'https://fonts.gstatic.com/s/notosans/v36/o-0mIpQlx3QUlC5A4PNB6Ryti20_6n1iPHjcz6L1SoM-jCpoiyD9A-9a6Vc.ttf' + ); + await compositor.registerInput('bunny_video', { type: 'mp4', url: MP4_URL }); + }, []); return ( - <> -
    - -
    - +
    + + + + + + +
    ); } @@ -60,40 +60,4 @@ function Scene() { ); } -function useCompositor(): [LiveCompositor | undefined, (canvas: HTMLCanvasElement) => void] { - const [compositor, setCompositor] = useState(undefined); - const canvasRef = useCallback(async (canvas: HTMLCanvasElement) => { - if (!canvas) { - return; - } - - const compositor = new LiveCompositor({ - framerate: { - num: 30, - den: 1, - }, - streamFallbackTimeoutMs: 500, - }); - - await compositor.init(); - - setCompositor(compositor); - - await compositor.registerFont( - 'https://fonts.gstatic.com/s/notosans/v36/o-0mIpQlx3QUlC5A4PNB6Ryti20_6n1iPHjcz6L1SoM-jCpoiyD9A-9a6Vc.ttf' - ); - void compositor.registerInput('bunny_video', { type: 'mp4', url: BUNNY_URL }); - await compositor.registerOutput('output', , { - type: 'canvas', - canvas: canvas, - resolution: { - width: 1280, - height: 720, - }, - }); - }, []); - - return [compositor, canvasRef]; -} - -export default MP4Player; +export default MultipleCompositors; diff --git a/ts/examples/vite-browser-render/src/examples/SimpleMp4Example.tsx b/ts/examples/vite-browser-render/src/examples/SimpleMp4Example.tsx new file mode 100644 index 000000000..cdc301834 --- /dev/null +++ b/ts/examples/vite-browser-render/src/examples/SimpleMp4Example.tsx @@ -0,0 +1,60 @@ +import { useCallback } from 'react'; +import type { LiveCompositor } from '@live-compositor/web-wasm'; +import { InputStream, Text, useInputStreams, View } from 'live-compositor'; +import CompositorCanvas from '../components/CompositorCanvas'; + +const MP4_URL = + 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4'; + +function SimpleMp4Example() { + const onCanvasCreate = useCallback(async (compositor: LiveCompositor) => { + await compositor.registerFont( + 'https://fonts.gstatic.com/s/notosans/v36/o-0mIpQlx3QUlC5A4PNB6Ryti20_6n1iPHjcz6L1SoM-jCpoiyD9A-9a6Vc.ttf' + ); + await compositor.registerInput('bunny_video', { type: 'mp4', url: MP4_URL }); + }, []); + + return ( +
    + + + +
    + ); +} + +function Scene() { + const inputs = useInputStreams(); + const inputState = inputs['bunny_video']?.videoState; + + if (inputState === 'playing') { + return ( + + + + Playing MP4 file + + + ); + } + + if (inputState === 'finished') { + return ( + + + Finished playing MP4 file + + + ); + } + + return ( + + + Loading MP4 file + + + ); +} + +export default SimpleMp4Example;