From bc6a7e6b4475a3d3b9720ea34a94325d52e8f165 Mon Sep 17 00:00:00 2001 From: noituri Date: Mon, 14 Oct 2024 11:11:00 +0200 Subject: [PATCH] Review suggestions --- ts/@live-compositor/core/src/api/input.ts | 14 ++-- ts/@live-compositor/core/src/api/output.ts | 35 +++++---- ts/@live-compositor/core/src/compositor.ts | 6 +- ts/@live-compositor/core/src/index.ts | 4 +- ts/@live-compositor/core/src/output.ts | 4 +- ts/@live-compositor/web/src/compositor.ts | 41 +++------- ts/@live-compositor/web/src/input/input.ts | 6 +- .../web/src/input/registerInput.ts | 4 +- .../web/src/manager/wasmInstance.ts | 74 +++++++++++++------ ts/@live-compositor/web/src/output/output.ts | 10 ++- .../web/src/output/registerOutput.ts | 6 +- ts/@live-compositor/web/src/queue.ts | 8 ++ .../src/examples/MP4Player.tsx | 4 +- ts/live-compositor/src/index.ts | 4 +- ts/live-compositor/src/types/registerInput.ts | 5 -- .../src/types/registerOutput.ts | 19 ++--- 16 files changed, 124 insertions(+), 120 deletions(-) diff --git a/ts/@live-compositor/core/src/api/input.ts b/ts/@live-compositor/core/src/api/input.ts index 55be38326..792ba7128 100644 --- a/ts/@live-compositor/core/src/api/input.ts +++ b/ts/@live-compositor/core/src/api/input.ts @@ -1,15 +1,17 @@ import { Api } from '../api.js'; -import { RegisterInput, Inputs } from 'live-compositor'; +import { RegisterMp4Input, RegisterRtpInput, Inputs } from 'live-compositor'; -export type RegisterInputRequest = Api.RegisterInput | { type: 'raw_frames' }; +export type RegisterInputRequest = Api.RegisterInput; + +export type RegisterInput = + | ({ type: 'rtp_stream' } & RegisterRtpInput) + | ({ type: 'mp4' } & RegisterMp4Input); export function intoRegisterInput(input: RegisterInput): RegisterInputRequest { if (input.type === 'mp4') { return intoMp4RegisterInput(input); } else if (input.type === 'rtp_stream') { return intoRtpRegisterInput(input); - } else if (input.type === 'raw_frames') { - return intoRawFramesRegisterInput(); } else { throw new Error(`Unknown input type ${(input as any).type}`); } @@ -38,10 +40,6 @@ function intoRtpRegisterInput(input: Inputs.RegisterRtpInput): RegisterInputRequ }; } -function intoRawFramesRegisterInput(): RegisterInputRequest { - return { type: 'raw_frames' }; -} - function intoInputAudio(audio: Inputs.InputRtpAudioOptions): Api.InputRtpAudioOptions { if (audio.decoder === 'opus') { return { diff --git a/ts/@live-compositor/core/src/api/output.ts b/ts/@live-compositor/core/src/api/output.ts index 882da38d6..c14e10c56 100644 --- a/ts/@live-compositor/core/src/api/output.ts +++ b/ts/@live-compositor/core/src/api/output.ts @@ -1,18 +1,27 @@ -import { RegisterOutput, Api, Outputs, OutputFrameFormat } from 'live-compositor'; +import { Api, Outputs, RegisterRtpOutput, RegisterMp4Output, RegisterCanvasOutput } from 'live-compositor'; -export type RegisterOutputRequest = Api.RegisterOutput | RegisterBytesOutput; +export type RegisterOutputRequest = Api.RegisterOutput | RegisterCanvasOutputRequest; -export type RegisterBytesOutput = { - type: 'raw_frames'; - video?: OutputBytesVideoOptions; +export type RegisterCanvasOutputRequest = { + type: 'canvas'; + video: OutputCanvasVideoOptions; }; -export type OutputBytesVideoOptions = { - format: OutputFrameFormat; +export type OutputCanvasVideoOptions = { resolution: Api.Resolution; + /** + * HTMLCanvasElement + */ + canvas: any; initial: Api.Video; }; +export type RegisterOutput = + | ({ type: 'rtp_stream' } & RegisterRtpOutput) + | ({ type: 'mp4' } & RegisterMp4Output) + | ({ type: 'canvas' } & RegisterCanvasOutput); + + export function intoRegisterOutput( output: RegisterOutput, initial: { video?: Api.Video; audio?: Api.Audio } @@ -21,8 +30,8 @@ export function intoRegisterOutput( return intoRegisterRtpOutput(output, initial); } else if (output.type === 'mp4') { return intoRegisterMp4Output(output, initial); - } else if (output.type === 'raw_frames') { - return intoRegisterRawFramesOutput(output, initial); + } else if (output.type === 'canvas') { + return intoRegisterCanvasOutput(output, initial); } else { throw new Error(`Unknown output type ${(output as any).type}`); } @@ -54,15 +63,15 @@ function intoRegisterMp4Output( }; } -function intoRegisterRawFramesOutput( - output: Outputs.RegisterRawFramesOutput, +function intoRegisterCanvasOutput( + output: Outputs.RegisterCanvasOutput, initial: { video?: Api.Video; _audio?: Api.Audio } ): RegisterOutputRequest { return { - type: 'raw_frames', + type: 'canvas', video: { - format: output.video.format, resolution: output.video.resolution, + canvas: output.video.canvas, initial: initial.video!, }, }; diff --git a/ts/@live-compositor/core/src/compositor.ts b/ts/@live-compositor/core/src/compositor.ts index b4f9ee9b4..2196521bf 100644 --- a/ts/@live-compositor/core/src/compositor.ts +++ b/ts/@live-compositor/core/src/compositor.ts @@ -1,14 +1,12 @@ import { _liveCompositorInternals, - RegisterInput, - RegisterOutput, Renderers, } from 'live-compositor'; import { ApiClient } from './api.js'; import Output from './output.js'; import { CompositorManager } from './compositorManager.js'; -import { intoRegisterOutput } from './api/output.js'; -import { intoRegisterInput } from './api/input.js'; +import { intoRegisterOutput, RegisterOutput } from './api/output.js'; +import { intoRegisterInput, RegisterInput } from './api/input.js'; import { onCompositorEvent } from './event.js'; import { intoRegisterImage, intoRegisterWebRenderer } from './api/renderer.js'; diff --git a/ts/@live-compositor/core/src/index.ts b/ts/@live-compositor/core/src/index.ts index 2818e4e11..fb0b860af 100644 --- a/ts/@live-compositor/core/src/index.ts +++ b/ts/@live-compositor/core/src/index.ts @@ -1,5 +1,5 @@ export { ApiClient, ApiRequest } from './api.js'; export { LiveCompositor } from './compositor.js'; export { CompositorManager } from './compositorManager.js'; -export { RegisterInputRequest } from './api/input.js'; -export { RegisterOutputRequest, RegisterBytesOutput } from './api/output.js'; +export { RegisterInputRequest, RegisterInput } from './api/input.js'; +export { RegisterOutputRequest, RegisterOutput } from './api/output.js'; diff --git a/ts/@live-compositor/core/src/output.ts b/ts/@live-compositor/core/src/output.ts index 4ef8ddc54..b97e95717 100644 --- a/ts/@live-compositor/core/src/output.ts +++ b/ts/@live-compositor/core/src/output.ts @@ -1,8 +1,8 @@ -import { _liveCompositorInternals, RegisterOutput, View, Outputs } from 'live-compositor'; +import { _liveCompositorInternals, View, Outputs } from 'live-compositor'; import React, { useSyncExternalStore } from 'react'; import { ApiClient, Api } from './api.js'; import Renderer from './renderer.js'; -import { intoAudioInputsConfiguration } from './api/output.js'; +import { intoAudioInputsConfiguration, RegisterOutput } from './api/output.js'; import { throttle } from './utils.js'; type OutputContext = _liveCompositorInternals.OutputContext; diff --git a/ts/@live-compositor/web/src/compositor.ts b/ts/@live-compositor/web/src/compositor.ts index 41966c77d..9a59c1313 100644 --- a/ts/@live-compositor/web/src/compositor.ts +++ b/ts/@live-compositor/web/src/compositor.ts @@ -1,12 +1,8 @@ import { Renderer } from '@live-compositor/browser-render'; import { LiveCompositor as CoreLiveCompositor } from '@live-compositor/core'; import WasmInstance from './manager/wasmInstance'; -import { Queue, StopQueueFn } from './queue'; -import { EventSender } from './eventSender'; import { intoRegisterOutput, RegisterOutput } from './output/registerOutput'; -import { Output } from './output/output'; import { intoRegisterInput, RegisterInput } from './input/registerInput'; -import { Input } from './input/input'; import { RegisterImage } from './renderers'; export type LiveCompositorOptions = { @@ -21,54 +17,41 @@ export type Framerate = { export default class LiveCompositor { private coreCompositor?: CoreLiveCompositor; - private queue?: Queue; + private instance?: WasmInstance; private renderer?: Renderer; - private eventSender: EventSender; - private stopQueue?: StopQueueFn; private options: LiveCompositorOptions; public constructor(options: LiveCompositorOptions) { this.options = options; - this.eventSender = new EventSender(); } public async init(): Promise { this.renderer = await Renderer.create({ streamFallbackTimeoutMs: this.options.streamFallbackTimeoutMs ?? 500, }); - this.queue = new Queue(this.options.framerate ?? { num: 30, den: 1 }, this.renderer!); - this.coreCompositor = new CoreLiveCompositor( - new WasmInstance({ - renderer: this.renderer!, - onRegisterCallback: cb => this.eventSender.setEventCallback(cb), - }) - ); + this.instance = new WasmInstance({ + renderer: this.renderer!, + framerate: this.options.framerate ?? { num: 30, den: 1 }, + }); + this.coreCompositor = new CoreLiveCompositor(this.instance!); await this.coreCompositor!.init(); } public async registerOutput(outputId: string, request: RegisterOutput): Promise { await this.coreCompositor!.registerOutput(outputId, intoRegisterOutput(request)); - const output = new Output(request); - this.queue!.addOutput(outputId, output); } public async unregisterOutput(outputId: string): Promise { await this.coreCompositor!.unregisterOutput(outputId); - this.queue!.removeOutput(outputId); } public async registerInput(inputId: string, request: RegisterInput): Promise { await this.coreCompositor!.registerInput(inputId, intoRegisterInput(request)); - - const input = new Input(inputId, request, this.eventSender); - this.queue!.addInput(inputId, input); - await input.start(); } public async unregisterInput(inputId: string): Promise { await this.coreCompositor!.unregisterInput(inputId); - this.queue!.removeInput(inputId); } public async registerImage(imageId: string, request: RegisterImage): Promise { @@ -83,17 +66,11 @@ export default class LiveCompositor { await this.renderer!.registerFont(fontUrl); } - public start(): void { - if (this.stopQueue) { - throw 'Compositor is already running'; - } - this.stopQueue = this.queue!.start(); + public async start(): Promise { + await this.coreCompositor?.start(); } public stop(): void { - if (this.stopQueue) { - this.stopQueue(); - this.stopQueue = undefined; - } + this.instance?.stop(); } } diff --git a/ts/@live-compositor/web/src/input/input.ts b/ts/@live-compositor/web/src/input/input.ts index 16c2c2f4c..3132b1745 100644 --- a/ts/@live-compositor/web/src/input/input.ts +++ b/ts/@live-compositor/web/src/input/input.ts @@ -3,7 +3,7 @@ import MP4Source from './mp4/source'; import { CompositorEventType } from 'live-compositor'; import { EventSender } from '../eventSender'; import InputSource from './source'; -import { RegisterInput } from './registerInput'; +import { RegisterInputRequest } from '@live-compositor/core'; /** * Represents frame produced by decoder. All `InputFrame`s have to be manually freed. @@ -23,12 +23,12 @@ export class Input { private state: InputState; private eventSender: EventSender; - public constructor(id: InputId, request: RegisterInput, eventSender: EventSender) { + public constructor(id: InputId, request: RegisterInputRequest, eventSender: EventSender) { this.id = id; this.state = 'waiting_for_start'; this.eventSender = eventSender; if (request.type === 'mp4') { - this.source = new MP4Source(request.url); + this.source = new MP4Source(request.url!); } else { throw new Error(`Unknown input type ${(request as any).type}`); } diff --git a/ts/@live-compositor/web/src/input/registerInput.ts b/ts/@live-compositor/web/src/input/registerInput.ts index 04bdf3f82..1e0693593 100644 --- a/ts/@live-compositor/web/src/input/registerInput.ts +++ b/ts/@live-compositor/web/src/input/registerInput.ts @@ -1,4 +1,4 @@ -import { RegisterInput as InternalRegisterInput } from 'live-compositor'; +import { RegisterInput as InternalRegisterInput } from '@live-compositor/core'; export type RegisterInput = { type: 'mp4' } & RegisterMP4Input; @@ -8,7 +8,7 @@ export type RegisterMP4Input = { export function intoRegisterInput(input: RegisterInput): InternalRegisterInput { if (input.type === 'mp4') { - return { type: 'raw_frames' }; + return { type: 'mp4', url: input.url }; } else { throw new Error(`Unknown input type ${(input as any).type}`); } diff --git a/ts/@live-compositor/web/src/manager/wasmInstance.ts b/ts/@live-compositor/web/src/manager/wasmInstance.ts index 1cf4ed6af..eb724403a 100644 --- a/ts/@live-compositor/web/src/manager/wasmInstance.ts +++ b/ts/@live-compositor/web/src/manager/wasmInstance.ts @@ -1,42 +1,48 @@ -import { ApiRequest, CompositorManager, RegisterOutputRequest } from '@live-compositor/core'; -import { Renderer, Resolution, Component, ImageSpec } from '@live-compositor/browser-render'; +import { ApiRequest, CompositorManager, RegisterInputRequest, RegisterOutputRequest } from '@live-compositor/core'; +import { Renderer, Component, ImageSpec } from '@live-compositor/browser-render'; import { Api } from 'live-compositor'; import { Path } from 'path-parser'; - -type Output = { - resolution: Resolution; -}; +import { Queue, StopQueueFn } from '../queue'; +import { Input } from '../input/input'; +import { EventSender } from '../eventSender'; +import { Framerate } from '../compositor'; +import { Output } from '../output/output'; export type OnRegisterCallback = (event: object) => void; const apiPath = new Path('/api/:type/:id/:operation'); +const apiStartPath = new Path('/api/start'); class WasmInstance implements CompositorManager { private renderer: Renderer; - private outputs: Map; - private onRegisterCallback: (cb: OnRegisterCallback) => void; + private queue: Queue; + private eventSender: EventSender; + private stopQueue?: StopQueueFn; public constructor(props: { renderer: Renderer; - onRegisterCallback: (cb: OnRegisterCallback) => void; + framerate: Framerate; }) { this.renderer = props.renderer; - this.onRegisterCallback = props.onRegisterCallback; - this.outputs = new Map(); + this.queue = new Queue(props.framerate, props.renderer); + this.eventSender = new EventSender(); } - public async setupInstance(): Promise {} + public async setupInstance(): Promise { } public async sendRequest(request: ApiRequest): Promise { const route = apiPath.test(request.route); if (!route) { + if (apiStartPath.test(request.route)) { + this.start(); + } return {}; } if (route.type == 'input') { - this.handleInputRequest(route.id, route.operation); + await this.handleInputRequest(route.id, route.operation, request.body); } else if (route.type === 'output') { - this.handleOutputRequest(route.id, route.operation, request.body); + await this.handleOutputRequest(route.id, route.operation, request.body); } else if (route.type === 'image') { await this.handleImageRequest(route.id, route.operation, request.body); } else if (route.type === 'shader') { @@ -49,37 +55,57 @@ class WasmInstance implements CompositorManager { } public registerEventListener(cb: (event: unknown) => void): void { - this.onRegisterCallback(cb); + this.eventSender.setEventCallback(cb); + } + + private start() { + if (this.stopQueue) { + throw 'Compositor is already running'; + } + this.stopQueue = this.queue.start(); + } + + public stop() { + if (this.stopQueue) { + this.stopQueue(); + this.stopQueue = undefined; + } } - private handleInputRequest(inputId: string, operation: string): void { + private async handleInputRequest(inputId: string, operation: string, body?: object): Promise { if (operation === 'register') { + const request = body! as RegisterInputRequest; + const input = new Input(inputId, request, this.eventSender); this.renderer.registerInput(inputId); + this.queue.addInput(inputId, input); + await input.start() } else if (operation === 'unregister') { + this.queue.removeInput(inputId); this.renderer.unregisterInput(inputId); } } - private handleOutputRequest(outputId: string, operation: string, body?: object): void { + private async handleOutputRequest(outputId: string, operation: string, body?: object): Promise { if (operation === 'register') { - const outputInfo = body! as RegisterOutputRequest; - if (outputInfo.video) { - const resolution = outputInfo.video.resolution; - this.outputs.set(outputId, { resolution: resolution }); + const request = body! as RegisterOutputRequest; + if (request.video) { + const output = new Output(request); this.renderer.updateScene( outputId, - resolution, - outputInfo.video?.initial.root as Component + request.video.resolution, + request.video.initial.root as Component ); + this.queue.addOutput(outputId, output); } } else if (operation === 'unregister') { + this.queue.removeOutput(outputId); this.renderer.unregisterOutput(outputId); } else if (operation === 'update') { const scene = body! as Api.UpdateOutputRequest; if (!scene.video) { return; } - const output = this.outputs.get(outputId); + const output = this.queue.getOutput(outputId); if (!output) { throw `Unknown output "${outputId}"`; } diff --git a/ts/@live-compositor/web/src/output/output.ts b/ts/@live-compositor/web/src/output/output.ts index f2bb75dec..0a9682486 100644 --- a/ts/@live-compositor/web/src/output/output.ts +++ b/ts/@live-compositor/web/src/output/output.ts @@ -1,17 +1,19 @@ -import { Frame } from '@live-compositor/browser-render'; +import { Frame, Resolution } from '@live-compositor/browser-render'; import { OutputSink } from './sink'; import CanvasSink from './canvas'; -import { RegisterOutput } from './registerOutput'; +import { RegisterOutputRequest } from '@live-compositor/core'; export class Output { private sink: OutputSink; + public readonly resolution: Resolution; - public constructor(request: RegisterOutput) { + public constructor(request: RegisterOutputRequest) { if (request.type === 'canvas') { - this.sink = new CanvasSink(request.canvas); + this.sink = new CanvasSink(request.video.canvas); } else { throw new Error(`Unknown output type ${(request as any).type}`); } + this.resolution = request.video.resolution; } public async send(frame: Frame): Promise { diff --git a/ts/@live-compositor/web/src/output/registerOutput.ts b/ts/@live-compositor/web/src/output/registerOutput.ts index da84a3e08..603ec2293 100644 --- a/ts/@live-compositor/web/src/output/registerOutput.ts +++ b/ts/@live-compositor/web/src/output/registerOutput.ts @@ -1,5 +1,5 @@ import { Resolution } from '@live-compositor/browser-render'; -import { RegisterOutput as InternalRegisterOutput, OutputFrameFormat } from 'live-compositor'; +import { RegisterOutput as InternalRegisterOutput} from '@live-compositor/core'; export type RegisterOutput = { type: 'canvas' } & RegisterCanvasOutput; @@ -19,10 +19,10 @@ export function intoRegisterOutput(output: RegisterOutput): InternalRegisterOutp function fromRegisterCanvasOutput(output: RegisterCanvasOutput): InternalRegisterOutput { return { - type: 'raw_frames', + type: 'canvas', video: { resolution: output.resolution, - format: OutputFrameFormat.RGBA_BYTES, + canvas: output.canvas, root: output.root, }, }; diff --git a/ts/@live-compositor/web/src/queue.ts b/ts/@live-compositor/web/src/queue.ts index 96803754a..f850fd477 100644 --- a/ts/@live-compositor/web/src/queue.ts +++ b/ts/@live-compositor/web/src/queue.ts @@ -36,6 +36,10 @@ export class Queue { delete this.inputs[inputId]; } + public getInput(inputId: InputId): Input | undefined { + return this.inputs[inputId] + } + public addOutput(outputId: OutputId, output: Output) { this.outputs[outputId] = output; } @@ -44,6 +48,10 @@ export class Queue { delete this.outputs[outputId]; } + public getOutput(outputId: OutputId): Output | undefined { + return this.outputs[outputId]; + } + private async onTick() { const inputs = await this.getInputFrames(); const outputs = this.renderer.render({ diff --git a/ts/examples/vite-browser-render/src/examples/MP4Player.tsx b/ts/examples/vite-browser-render/src/examples/MP4Player.tsx index bd87eff86..592d6586b 100644 --- a/ts/examples/vite-browser-render/src/examples/MP4Player.tsx +++ b/ts/examples/vite-browser-render/src/examples/MP4Player.tsx @@ -12,7 +12,7 @@ function MP4Player() { return; } - compositor.start(); + void compositor.start(); return () => compositor.stop(); }, [compositor]) @@ -29,7 +29,7 @@ function Scene() { const inputs = useInputStreams(); const inputState = inputs['bunny_video']?.videoState; - if (!inputState || inputState == 'ready') { + if (inputState !== 'playing') { return ( diff --git a/ts/live-compositor/src/index.ts b/ts/live-compositor/src/index.ts index 8be511372..5d365a0e5 100644 --- a/ts/live-compositor/src/index.ts +++ b/ts/live-compositor/src/index.ts @@ -10,8 +10,8 @@ import { EasingFunction, Transition } from './components/common.js'; import { useAudioInput, useInputStreams } from './hooks.js'; import { CompositorEvent, CompositorEventType } from './types/events.js'; -export { RegisterInput } from './types/registerInput.js'; -export { RegisterOutput, OutputFrameFormat } from './types/registerOutput.js'; +export { RegisterRtpInput, RegisterMp4Input } from './types/registerInput.js'; +export { RegisterRtpOutput, RegisterMp4Output, RegisterCanvasOutput } from './types/registerOutput.js'; export * as Inputs from './types/registerInput.js'; export * as Outputs from './types/registerOutput.js'; diff --git a/ts/live-compositor/src/types/registerInput.ts b/ts/live-compositor/src/types/registerInput.ts index fcbd08024..faffa7d23 100644 --- a/ts/live-compositor/src/types/registerInput.ts +++ b/ts/live-compositor/src/types/registerInput.ts @@ -1,10 +1,5 @@ import * as Api from '../api.js'; -export type RegisterInput = - | ({ type: 'rtp_stream' } & RegisterRtpInput) - | ({ type: 'mp4' } & RegisterMp4Input) - | { type: 'raw_frames' }; - export type RegisterRtpInput = { /** * UDP port or port range on which the compositor should listen for the stream. diff --git a/ts/live-compositor/src/types/registerOutput.ts b/ts/live-compositor/src/types/registerOutput.ts index 39729b94e..22e653031 100644 --- a/ts/live-compositor/src/types/registerOutput.ts +++ b/ts/live-compositor/src/types/registerOutput.ts @@ -1,11 +1,6 @@ import React from 'react'; import * as Api from '../api.js'; -export type RegisterOutput = - | ({ type: 'rtp_stream' } & RegisterRtpOutput) - | ({ type: 'mp4' } & RegisterMp4Output) - | ({ type: 'raw_frames' } & RegisterRawFramesOutput); - export type RegisterRtpOutput = { /** * Depends on the value of the `transport_protocol` field: @@ -40,8 +35,8 @@ export type RegisterMp4Output = { audio?: Mp4AudioOptions; }; -export type RegisterRawFramesOutput = { - video: OutputRawFramesVideoOptions; +export type RegisterCanvasOutput = { + video: OutputCanvasVideoOptions; }; export type RtpVideoOptions = { @@ -78,15 +73,15 @@ export type Mp4VideoOptions = { root: React.ReactElement; }; -export type OutputRawFramesVideoOptions = { +export type OutputCanvasVideoOptions = { /** * Output resolution in pixels. */ resolution: Api.Resolution; /** - * Output byte format. + * HTMLCanvasElement */ - format: OutputFrameFormat; + canvas: any; root: React.ReactElement; }; @@ -192,10 +187,6 @@ export type OutputEndCondition = allInputs: boolean; }; -export enum OutputFrameFormat { - RGBA_BYTES = 'RGBA_BYTES', -} - export interface AudioInputsConfiguration { inputs: InputAudio[]; }