Fix image autoregister (#953)

src/routes/unregister_request.rs

@@ -29,11 +29,10 @@ pub struct UnregisterOutput {
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(tag = "type", rename_all = "snake_case")]
-pub enum UnregisterRenderer {
-    Shader { shader_id: RendererId },
-    WebRenderer { instance_id: RendererId },
-    Image { image_id: RendererId },
+pub struct UnregisterRenderer {
+    /// Time in milliseconds when this request should be applied. Value `0` represents
+    /// time of the start request.
+    schedule_time_ms: Option<f64>,
 }
 
 pub(super) async fn handle_input(
@@ -101,26 +100,101 @@ pub(super) async fn handle_output(
 pub(super) async fn handle_shader(
     State(api): State<ApiState>,
     Path(shader_id): Path<RendererId>,
+    Json(request): Json<UnregisterRenderer>,
 ) -> Result<Response, ApiError> {
-    api.pipeline()
-        .unregister_renderer(&shader_id.into(), RegistryType::Shader)?;
+    match request.schedule_time_ms {
+        Some(schedule_time_ms) => {
+            let pipeline = api.pipeline.clone();
+            let schedule_time = Duration::from_secs_f64(schedule_time_ms / 1000.0);
+            api.pipeline().queue().schedule_event(
+                schedule_time,
+                Box::new(move || {
+                    if let Err(err) = pipeline
+                        .lock()
+                        .unwrap()
+                        .unregister_renderer(&shader_id.into(), RegistryType::Shader)
+                    {
+                        error!(
+                            "Error while running scheduled shader unregister for pts {}ms: {}",
+                            schedule_time.as_millis(),
+                            ErrorStack::new(&err).into_string()
+                        )
+                    }
+                }),
+            );
+        }
+        None => {
+            api.pipeline()
+                .unregister_renderer(&shader_id.into(), RegistryType::Shader)?;
+        }
+    }
     Ok(Response::Ok {})
 }
 
 pub(super) async fn handle_web_renderer(
     State(api): State<ApiState>,
     Path(instance_id): Path<RendererId>,
+    Json(request): Json<UnregisterRenderer>,
 ) -> Result<Response, ApiError> {
-    api.pipeline()
-        .unregister_renderer(&instance_id.into(), RegistryType::WebRenderer)?;
+    match request.schedule_time_ms {
+        Some(schedule_time_ms) => {
+            let pipeline = api.pipeline.clone();
+            let schedule_time = Duration::from_secs_f64(schedule_time_ms / 1000.0);
+            api.pipeline().queue().schedule_event(
+                schedule_time,
+                Box::new(move || {
+                    if let Err(err) = pipeline
+                        .lock()
+                        .unwrap()
+                        .unregister_renderer(&instance_id.into(), RegistryType::WebRenderer)
+                    {
+                        error!(
+                            "Error while running scheduled web renderer unregister for pts {}ms: {}",
+                            schedule_time.as_millis(),
+                            ErrorStack::new(&err).into_string()
+                        )
+                    }
+                }),
+            );
+        }
+        None => {
+            api.pipeline()
+                .unregister_renderer(&instance_id.into(), RegistryType::WebRenderer)?;
+        }
+    }
     Ok(Response::Ok {})
 }
 
 pub(super) async fn handle_image(
     State(api): State<ApiState>,
     Path(image_id): Path<RendererId>,
+    Json(request): Json<UnregisterRenderer>,
 ) -> Result<Response, ApiError> {
-    api.pipeline()
-        .unregister_renderer(&image_id.into(), RegistryType::Image)?;
+    match request.schedule_time_ms {
+        Some(schedule_time_ms) => {
+            let pipeline = api.pipeline.clone();
+            let schedule_time = Duration::from_secs_f64(schedule_time_ms / 1000.0);
+            api.pipeline().queue().schedule_event(
+                schedule_time,
+                Box::new(move || {
+                    if let Err(err) = pipeline
+                        .lock()
+                        .unwrap()
+                        .unregister_renderer(&image_id.into(), RegistryType::Image)
+                    {
+                        error!(
+                            "Error while running scheduled image unregister for pts {}ms: {}",
+                            schedule_time.as_millis(),
+                            ErrorStack::new(&err).into_string()
+                        )
+                    }
+                }),
+            );
+        }
+        None => {
+            api.pipeline()
+                .unregister_renderer(&image_id.into(), RegistryType::Image)?;
+        }
+    }
     Ok(Response::Ok {})
 }

ts/smelter-core/src/api.ts

@@ -103,11 +103,14 @@ export class ApiClient {
     });
   }
 
-  public async unregisterImage(imageRef: ImageRef): Promise<object> {
+  public async unregisterImage(
+    imageRef: ImageRef,
+    body: { schedule_time_ms?: number }
+  ): Promise<object> {
     return this.serverManager.sendRequest({
       method: 'POST',
       route: `/api/image/${encodeURIComponent(imageRefIntoRawId(imageRef))}/unregister`,
-      body: {},
+      body,
     });
   }
 

ts/smelter-core/src/live/compositor.ts

@@ -119,7 +119,7 @@ export class Smelter {
     this.logger.info({ imageId }, 'Unregister image');
     const imageRef = { type: 'global', id: imageId } as const satisfies ImageRef;
 
-    return this.api.unregisterImage(imageRef);
+    return this.api.unregisterImage(imageRef, {});
   }
 
   public async registerWebRenderer(

ts/smelter-core/src/live/output.ts

@@ -188,11 +188,14 @@ class OutputContext implements SmelterOutputContext {
     });
   }
   public async unregisterImage(imageId: number) {
-    await this.output.api.unregisterImage({
-      type: 'output-specific-image',
-      outputId: this.outputId,
-      id: imageId,
-    });
+    await this.output.api.unregisterImage(
+      {
+        type: 'output-specific-image',
+        outputId: this.outputId,
+        id: imageId,
+      },
+      {}
+    );
   }
 }
 

ts/smelter-core/src/offline/output.ts

@@ -1,4 +1,4 @@
-import type { RegisterMp4Input } from '@swmansion/smelter';
+import type { RegisterMp4Input, Renderers } from '@swmansion/smelter';
 import { _smelterInternals } from '@swmansion/smelter';
 import type { ReactElement } from 'react';
 import { createElement } from 'react';
@@ -156,7 +156,10 @@ class OutputContext implements SmelterOutputContext {
       await this.output.api.registerInput(inputRef, {
         type: 'mp4',
         offset_ms: offsetMs,
-        ...registerRequest,
+        path: registerRequest.serverPath,
+        url: registerRequest.url,
+        required: registerRequest.required,
+        video_decoder: registerRequest.videoDecoder,
       });
     this.output.internalInputStreamStore.addInput({
       inputId,
@@ -192,7 +195,7 @@ class OutputContext implements SmelterOutputContext {
       { schedule_time_ms: this.timeContext.timestampMs() }
     );
   }
-  public async registerImage(imageId: number, imageSpec: any) {
+  public async registerImage(imageId: number, imageSpec: Renderers.RegisterImage) {
     const imageRef = {
       type: 'output-specific-image',
       outputId: this.outputId,
@@ -201,17 +204,22 @@ class OutputContext implements SmelterOutputContext {
 
     await this.output.api.registerImage(imageRef, {
       url: imageSpec.url,
+      path: imageSpec.serverPath,
       asset_type: imageSpec.assetType,
     });
   }
   public async unregisterImage(imageId: number) {
-    await this.output.api.unregisterImage({
-      type: 'output-specific-image',
-      outputId: this.outputId,
-      id: imageId,
-    });
+    await this.output.api.unregisterImage(
+      {
+        type: 'output-specific-image',
+        outputId: this.outputId,
+        id: imageId,
+      },
+      { schedule_time_ms: this.timeContext.timestampMs() }
+    );
   }
 }
+
 async function waitForBlockingTasks(offlineContext: OfflineTimeContext): Promise<void> {
   while (offlineContext.isBlocked()) {
     await sleep(100);

ts/smelter/src/components/Image.ts

@@ -7,6 +7,7 @@ import { newInternalImageId } from '../context/internalImageIdManager.js';
 import { newBlockingTask } from '../hooks.js';
 import { SmelterContext } from '../context/index.js';
 import { isValidImageType } from '../types/utils.js';
+import type { RegisterImage } from '../types/registerRenderer.js';
 
 export type ImageProps = Omit<ComponentBaseProps, 'children'> &
   (
@@ -41,12 +42,10 @@ function Image(props: ImageProps) {
     setIsImageRegistered(false);
 
     const newImageId = newInternalImageId();
-    setInternalImageId(newImageId);
-    const task = newBlockingTask(ctx);
-    const pathOrUrl =
+    const pathOrUrl: Pick<RegisterImage, 'serverPath' | 'url'> =
       props.source?.startsWith('http://') || props.source?.startsWith('https://')
         ? { url: props.source }
-        : { path: props.source };
+        : { serverPath: props.source };
     const extension = props.source?.split('.').pop();
     const assetType = extension && isValidImageType(extension) ? extension : undefined;
 
@@ -56,6 +55,8 @@ function Image(props: ImageProps) {
       throw new Error('Unsupported image type');
     }
 
+    const task = newBlockingTask(ctx);
+    setInternalImageId(newImageId);
     void (async () => {
       try {
         registerPromise = ctx.registerImage(newImageId, {

ts/smelter/src/components/Mp4.ts

@@ -6,6 +6,7 @@ import { InnerInputStream } from './InputStream.js';
 import { newInternalStreamId } from '../context/internalStreamIdManager.js';
 import type { ComponentBaseProps } from '../component.js';
 import { useTimeLimitedComponent } from '../context/childrenLifetimeContext.js';
+import type { RegisterMp4Input } from '../types/registerInput.js';
 
 export type Mp4Props = Omit<ComponentBaseProps, 'children'> & {
   /**
@@ -31,13 +32,13 @@ function Mp4(props: Mp4Props) {
   useEffect(() => {
     const newInputId = newInternalStreamId();
     setInputId(newInputId);
-    const task = newBlockingTask(ctx);
-    const pathOrUrl =
+    const pathOrUrl: Pick<RegisterMp4Input, 'url' | 'serverPath'> =
       props.source.startsWith('http://') || props.source.startsWith('https://')
         ? { url: props.source }
-        : { path: props.source };
+        : { serverPath: props.source };
     let registerPromise: Promise<any>;
 
+    const task = newBlockingTask(ctx);
     void (async () => {
       try {
         registerPromise = ctx.registerMp4Input(newInputId, {