Skip to content

Commit

Permalink
Review suggestions
Browse files Browse the repository at this point in the history
  • Loading branch information
noituri committed Oct 14, 2024
1 parent 97de1ac commit 0ad79e0
Show file tree
Hide file tree
Showing 16 changed files with 128 additions and 120 deletions.
14 changes: 6 additions & 8 deletions ts/@live-compositor/core/src/api/input.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
import { Api } from '../api.js';
import { RegisterInput, Inputs } from 'live-compositor';
import { RegisterMp4Input, RegisterRtpInput, Inputs } from 'live-compositor';

export type RegisterInputRequest = Api.RegisterInput | { type: 'raw_frames' };
export type RegisterInputRequest = Api.RegisterInput;

export type RegisterInput =
| ({ type: 'rtp_stream' } & RegisterRtpInput)
| ({ type: 'mp4' } & RegisterMp4Input);

export function intoRegisterInput(input: RegisterInput): RegisterInputRequest {
if (input.type === 'mp4') {
return intoMp4RegisterInput(input);
} else if (input.type === 'rtp_stream') {
return intoRtpRegisterInput(input);
} else if (input.type === 'raw_frames') {
return intoRawFramesRegisterInput();
} else {
throw new Error(`Unknown input type ${(input as any).type}`);
}
Expand Down Expand Up @@ -38,10 +40,6 @@ function intoRtpRegisterInput(input: Inputs.RegisterRtpInput): RegisterInputRequ
};
}

function intoRawFramesRegisterInput(): RegisterInputRequest {
return { type: 'raw_frames' };
}

function intoInputAudio(audio: Inputs.InputRtpAudioOptions): Api.InputRtpAudioOptions {
if (audio.decoder === 'opus') {
return {
Expand Down
35 changes: 22 additions & 13 deletions ts/@live-compositor/core/src/api/output.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,27 @@
import { RegisterOutput, Api, Outputs, OutputFrameFormat } from 'live-compositor';
import { Api, Outputs, RegisterRtpOutput, RegisterMp4Output, RegisterCanvasOutput } from 'live-compositor';

export type RegisterOutputRequest = Api.RegisterOutput | RegisterBytesOutput;
export type RegisterOutputRequest = Api.RegisterOutput | RegisterCanvasOutputRequest;

export type RegisterBytesOutput = {
type: 'raw_frames';
video?: OutputBytesVideoOptions;
export type RegisterCanvasOutputRequest = {
type: 'canvas';
video: OutputCanvasVideoOptions;
};

export type OutputBytesVideoOptions = {
format: OutputFrameFormat;
export type OutputCanvasVideoOptions = {
resolution: Api.Resolution;
/**
* HTMLCanvasElement
*/
canvas: any;
initial: Api.Video;
};

export type RegisterOutput =
| ({ type: 'rtp_stream' } & RegisterRtpOutput)
| ({ type: 'mp4' } & RegisterMp4Output)
| ({ type: 'canvas' } & RegisterCanvasOutput);


export function intoRegisterOutput(
output: RegisterOutput,
initial: { video?: Api.Video; audio?: Api.Audio }
Expand All @@ -21,8 +30,8 @@ export function intoRegisterOutput(
return intoRegisterRtpOutput(output, initial);
} else if (output.type === 'mp4') {
return intoRegisterMp4Output(output, initial);
} else if (output.type === 'raw_frames') {
return intoRegisterRawFramesOutput(output, initial);
} else if (output.type === 'canvas') {
return intoRegisterCanvasOutput(output, initial);
} else {
throw new Error(`Unknown output type ${(output as any).type}`);
}
Expand Down Expand Up @@ -54,15 +63,15 @@ function intoRegisterMp4Output(
};
}

function intoRegisterRawFramesOutput(
output: Outputs.RegisterRawFramesOutput,
function intoRegisterCanvasOutput(
output: Outputs.RegisterCanvasOutput,
initial: { video?: Api.Video; _audio?: Api.Audio }
): RegisterOutputRequest {
return {
type: 'raw_frames',
type: 'canvas',
video: {
format: output.video.format,
resolution: output.video.resolution,
canvas: output.video.canvas,
initial: initial.video!,
},
};
Expand Down
6 changes: 2 additions & 4 deletions ts/@live-compositor/core/src/compositor.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
import {
_liveCompositorInternals,
RegisterInput,
RegisterOutput,
Renderers,
} from 'live-compositor';
import { ApiClient } from './api.js';
import Output from './output.js';
import { CompositorManager } from './compositorManager.js';
import { intoRegisterOutput } from './api/output.js';
import { intoRegisterInput } from './api/input.js';
import { intoRegisterOutput, RegisterOutput } from './api/output.js';
import { intoRegisterInput, RegisterInput } from './api/input.js';
import { onCompositorEvent } from './event.js';
import { intoRegisterImage, intoRegisterWebRenderer } from './api/renderer.js';

Expand Down
4 changes: 2 additions & 2 deletions ts/@live-compositor/core/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
export { ApiClient, ApiRequest } from './api.js';
export { LiveCompositor } from './compositor.js';
export { CompositorManager } from './compositorManager.js';
export { RegisterInputRequest } from './api/input.js';
export { RegisterOutputRequest, RegisterBytesOutput } from './api/output.js';
export { RegisterInputRequest, RegisterInput } from './api/input.js';
export { RegisterOutputRequest, RegisterOutput } from './api/output.js';
4 changes: 2 additions & 2 deletions ts/@live-compositor/core/src/output.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { _liveCompositorInternals, RegisterOutput, View, Outputs } from 'live-compositor';
import { _liveCompositorInternals, View, Outputs } from 'live-compositor';
import React, { useSyncExternalStore } from 'react';
import { ApiClient, Api } from './api.js';
import Renderer from './renderer.js';
import { intoAudioInputsConfiguration } from './api/output.js';
import { intoAudioInputsConfiguration, RegisterOutput } from './api/output.js';
import { throttle } from './utils.js';

type OutputContext = _liveCompositorInternals.OutputContext;
Expand Down
41 changes: 9 additions & 32 deletions ts/@live-compositor/web/src/compositor.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
import { Renderer } from '@live-compositor/browser-render';
import { LiveCompositor as CoreLiveCompositor } from '@live-compositor/core';
import WasmInstance from './manager/wasmInstance';
import { Queue, StopQueueFn } from './queue';
import { EventSender } from './eventSender';
import { intoRegisterOutput, RegisterOutput } from './output/registerOutput';
import { Output } from './output/output';
import { intoRegisterInput, RegisterInput } from './input/registerInput';
import { Input } from './input/input';
import { RegisterImage } from './renderers';

export type LiveCompositorOptions = {
Expand All @@ -21,54 +17,41 @@ export type Framerate = {

export default class LiveCompositor {
private coreCompositor?: CoreLiveCompositor;
private queue?: Queue;
private instance?: WasmInstance;
private renderer?: Renderer;
private eventSender: EventSender;
private stopQueue?: StopQueueFn;
private options: LiveCompositorOptions;

public constructor(options: LiveCompositorOptions) {
this.options = options;
this.eventSender = new EventSender();
}

public async init(): Promise<void> {
this.renderer = await Renderer.create({
streamFallbackTimeoutMs: this.options.streamFallbackTimeoutMs ?? 500,
});
this.queue = new Queue(this.options.framerate ?? { num: 30, den: 1 }, this.renderer!);
this.coreCompositor = new CoreLiveCompositor(
new WasmInstance({
renderer: this.renderer!,
onRegisterCallback: cb => this.eventSender.setEventCallback(cb),
})
);
this.instance = new WasmInstance({
renderer: this.renderer!,
framerate: this.options.framerate ?? { num: 30, den: 1 },
});
this.coreCompositor = new CoreLiveCompositor(this.instance!);

await this.coreCompositor!.init();
}

public async registerOutput(outputId: string, request: RegisterOutput): Promise<void> {
await this.coreCompositor!.registerOutput(outputId, intoRegisterOutput(request));
const output = new Output(request);
this.queue!.addOutput(outputId, output);
}

public async unregisterOutput(outputId: string): Promise<void> {
await this.coreCompositor!.unregisterOutput(outputId);
this.queue!.removeOutput(outputId);
}

public async registerInput(inputId: string, request: RegisterInput): Promise<void> {
await this.coreCompositor!.registerInput(inputId, intoRegisterInput(request));

const input = new Input(inputId, request, this.eventSender);
this.queue!.addInput(inputId, input);
await input.start();
}

public async unregisterInput(inputId: string): Promise<void> {
await this.coreCompositor!.unregisterInput(inputId);
this.queue!.removeInput(inputId);
}

public async registerImage(imageId: string, request: RegisterImage): Promise<void> {
Expand All @@ -83,17 +66,11 @@ export default class LiveCompositor {
await this.renderer!.registerFont(fontUrl);
}

public start(): void {
if (this.stopQueue) {
throw 'Compositor is already running';
}
this.stopQueue = this.queue!.start();
public async start(): Promise<void> {
await this.coreCompositor?.start();
}

public stop(): void {
if (this.stopQueue) {
this.stopQueue();
this.stopQueue = undefined;
}
this.instance?.stop();
}
}
6 changes: 3 additions & 3 deletions ts/@live-compositor/web/src/input/input.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import MP4Source from './mp4/source';
import { CompositorEventType } from 'live-compositor';
import { EventSender } from '../eventSender';
import InputSource from './source';
import { RegisterInput } from './registerInput';
import { RegisterInputRequest } from '@live-compositor/core';

/**
* Represents frame produced by decoder. All `InputFrame`s have to be manually freed.
Expand All @@ -23,12 +23,12 @@ export class Input {
private state: InputState;
private eventSender: EventSender;

public constructor(id: InputId, request: RegisterInput, eventSender: EventSender) {
public constructor(id: InputId, request: RegisterInputRequest, eventSender: EventSender) {
this.id = id;
this.state = 'waiting_for_start';
this.eventSender = eventSender;
if (request.type === 'mp4') {
this.source = new MP4Source(request.url);
this.source = new MP4Source(request.url!);
} else {
throw new Error(`Unknown input type ${(request as any).type}`);
}
Expand Down
4 changes: 2 additions & 2 deletions ts/@live-compositor/web/src/input/registerInput.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { RegisterInput as InternalRegisterInput } from 'live-compositor';
import { RegisterInput as InternalRegisterInput } from '@live-compositor/core';

export type RegisterInput = { type: 'mp4' } & RegisterMP4Input;

Expand All @@ -8,7 +8,7 @@ export type RegisterMP4Input = {

export function intoRegisterInput(input: RegisterInput): InternalRegisterInput {
if (input.type === 'mp4') {
return { type: 'raw_frames' };
return { type: 'mp4', url: input.url };
} else {
throw new Error(`Unknown input type ${(input as any).type}`);
}
Expand Down
74 changes: 50 additions & 24 deletions ts/@live-compositor/web/src/manager/wasmInstance.ts
Original file line number Diff line number Diff line change
@@ -1,42 +1,48 @@
import { ApiRequest, CompositorManager, RegisterOutputRequest } from '@live-compositor/core';
import { Renderer, Resolution, Component, ImageSpec } from '@live-compositor/browser-render';
import { ApiRequest, CompositorManager, RegisterInputRequest, RegisterOutputRequest } from '@live-compositor/core';
import { Renderer, Component, ImageSpec } from '@live-compositor/browser-render';
import { Api } from 'live-compositor';
import { Path } from 'path-parser';

type Output = {
resolution: Resolution;
};
import { Queue, StopQueueFn } from '../queue';
import { Input } from '../input/input';
import { EventSender } from '../eventSender';
import { Framerate } from '../compositor';
import { Output } from '../output/output';

export type OnRegisterCallback = (event: object) => void;

const apiPath = new Path('/api/:type/:id/:operation');
const apiStartPath = new Path('/api/start');

class WasmInstance implements CompositorManager {
private renderer: Renderer;
private outputs: Map<string, Output>;
private onRegisterCallback: (cb: OnRegisterCallback) => void;
private queue: Queue;
private eventSender: EventSender;
private stopQueue?: StopQueueFn;

public constructor(props: {
renderer: Renderer;
onRegisterCallback: (cb: OnRegisterCallback) => void;
framerate: Framerate;
}) {
this.renderer = props.renderer;
this.onRegisterCallback = props.onRegisterCallback;
this.outputs = new Map();
this.queue = new Queue(props.framerate, props.renderer);
this.eventSender = new EventSender();
}

public async setupInstance(): Promise<void> {}
public async setupInstance(): Promise<void> { }

public async sendRequest(request: ApiRequest): Promise<object> {
const route = apiPath.test(request.route);
if (!route) {
if (apiStartPath.test(request.route)) {
this.start();
}
return {};
}

if (route.type == 'input') {
this.handleInputRequest(route.id, route.operation);
await this.handleInputRequest(route.id, route.operation, request.body);
} else if (route.type === 'output') {
this.handleOutputRequest(route.id, route.operation, request.body);
await this.handleOutputRequest(route.id, route.operation, request.body);
} else if (route.type === 'image') {
await this.handleImageRequest(route.id, route.operation, request.body);
} else if (route.type === 'shader') {
Expand All @@ -49,37 +55,57 @@ class WasmInstance implements CompositorManager {
}

public registerEventListener(cb: (event: unknown) => void): void {
this.onRegisterCallback(cb);
this.eventSender.setEventCallback(cb);
}

private start() {
if (this.stopQueue) {
throw 'Compositor is already running';
}
this.stopQueue = this.queue.start();
}

public stop() {
if (this.stopQueue) {
this.stopQueue();
this.stopQueue = undefined;
}
}

private handleInputRequest(inputId: string, operation: string): void {
private async handleInputRequest(inputId: string, operation: string, body?: object): Promise<void> {
if (operation === 'register') {
const request = body! as RegisterInputRequest;
const input = new Input(inputId, request, this.eventSender);
this.renderer.registerInput(inputId);
this.queue.addInput(inputId, input);
await input.start()
} else if (operation === 'unregister') {
this.queue.removeInput(inputId);
this.renderer.unregisterInput(inputId);
}
}

private handleOutputRequest(outputId: string, operation: string, body?: object): void {
private async handleOutputRequest(outputId: string, operation: string, body?: object): Promise<void> {
if (operation === 'register') {
const outputInfo = body! as RegisterOutputRequest;
if (outputInfo.video) {
const resolution = outputInfo.video.resolution;
this.outputs.set(outputId, { resolution: resolution });
const request = body! as RegisterOutputRequest;
if (request.video) {
const output = new Output(request);
this.renderer.updateScene(
outputId,
resolution,
outputInfo.video?.initial.root as Component
request.video.resolution,
request.video.initial.root as Component
);
this.queue.addOutput(outputId, output);
}
} else if (operation === 'unregister') {
this.queue.removeOutput(outputId);
this.renderer.unregisterOutput(outputId);
} else if (operation === 'update') {
const scene = body! as Api.UpdateOutputRequest;
if (!scene.video) {
return;
}
const output = this.outputs.get(outputId);
const output = this.queue.getOutput(outputId);
if (!output) {
throw `Unknown output "${outputId}"`;
}
Expand Down
Loading

0 comments on commit 0ad79e0

Please sign in to comment.