Skip to content

Commit

Permalink
feat: Storage explorer in dashboard (#28)
Browse files Browse the repository at this point in the history
  • Loading branch information
matvp91 authored Sep 26, 2024
1 parent 977088a commit 12e63cf
Show file tree
Hide file tree
Showing 54 changed files with 1,224 additions and 3,802 deletions.
14 changes: 9 additions & 5 deletions config.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@ S3_REGION=
S3_ACCESS_KEY=
S3_SECRET_KEY=
S3_BUCKET=
S3_PUBLIC_URL=

# When you run your redis elsewhere, or locally, uncomment these
# and point them to your redis.
# REDIS_HOST=
# REDIS_PORT=
# Uncomment when using docker.
# REDIS_HOST=redis
REDIS_PORT=6379

# These are public, they'll end up in client JS.
PUBLIC_API_ENDPOINT=http://localhost:52001
PUBLIC_STITCHER_ENDPOINT=http://localhost:52002
PUBLIC_S3_ENDPOINT=

3 changes: 0 additions & 3 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ services:
build:
context: .
dockerfile: ./packages/dashboard/Dockerfile
args:
- API_URL=http://127.0.0.1:52001
- STITCHER_URL=http://127.0.0.1:52002
restart: always
ports:
- 127.0.0.1:52000:8080
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "mixwave",
"scripts": {
"dev": "turbo dev",
"dev": "turbo watch dev",
"build": "turbo build",
"start": "turbo start"
},
Expand Down
1 change: 1 addition & 0 deletions packages/api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
},
"dependencies": {
"@anatine/zod-openapi": "^2.2.6",
"@aws-sdk/client-s3": "^3.623.0",
"@bull-board/api": "^5.21.3",
"@bull-board/fastify": "^5.21.3",
"@fastify/cors": "^9.0.1",
Expand Down
2 changes: 1 addition & 1 deletion packages/api/src/client/index.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
export * from "../contract";

export type { JobDto } from "../types";
export type { JobDto, FolderDto } from "../types";
32 changes: 27 additions & 5 deletions packages/api/src/contract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { initContract } from "@ts-rest/core";
import { streamSchema, inputSchema } from "@mixwave/shared/artisan";
import * as z from "zod";
import { extendZodWithOpenApi } from "@anatine/zod-openapi";
import { jobDtoSchema } from "./types.js";
import { jobDtoSchema, folderDtoSchema } from "./types.js";

extendZodWithOpenApi(z);

Expand All @@ -15,15 +15,17 @@ export const postTranscodeBodySchema = z.object({
streams: z.array(streamSchema).openapi({
description: "Streams that need to be produced.",
}),
segmentSize: z.number().default(4).openapi({
segmentSize: z.number().optional().openapi({
default: 4,
description:
"Inserts a keyframe at the start of each segment. When packaging, you can vary segmentSize as the same or a multiple of this.",
}),
assetId: z.string().uuid().optional().openapi({
description:
"Will override when specified but it is advised to leave this blank and have it auto generate a UUID.",
}),
packageAfter: z.boolean().default(false).openapi({
packageAfter: z.boolean().optional().openapi({
default: false,
description:
"When transcode is finished, package it immediately with all default settings.",
}),
Expand All @@ -34,10 +36,18 @@ export const postTranscodeBodySchema = z.object({

export const postPackageBodySchema = z.object({
assetId: z.string(),
segmentSize: z.number().default(4),
segmentSize: z.number().optional().openapi({
default: 4,
description:
"Segment size, must be equal or a multiple of the segmentSize defined in transcode.",
}),
tag: z.string().optional().openapi({
description: "An arbitrary tag, used to group jobs.",
}),
name: z.string().optional().openapi({
default: "hls",
description: "The name of the package, will be used in storage.",
}),
});

export const contract = c.router({
Expand Down Expand Up @@ -77,7 +87,19 @@ export const contract = c.router({
200: jobDtoSchema,
},
query: z.object({
fromRoot: z.coerce.boolean().default(false),
fromRoot: z.coerce.boolean().optional(),
}),
},
getStorage: {
method: "GET",
path: "/storage",
responses: {
200: folderDtoSchema,
},
query: z.object({
path: z.string(),
skip: z.string().optional(),
take: z.coerce.number().optional(),
}),
},
getJobLogs: {
Expand Down
10 changes: 8 additions & 2 deletions packages/api/src/env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,14 @@ if (configPath) {
const envSchema = z.object({
PORT: z.coerce.number().default(52001),
HOST: z.string().default("0.0.0.0"),
REDIS_HOST: z.string().default("redis"),
REDIS_PORT: z.coerce.number().default(6379),
REDIS_HOST: z.string(),
REDIS_PORT: z.coerce.number(),
PUBLIC_API_ENDPOINT: z.string(),
S3_ENDPOINT: z.string(),
S3_REGION: z.string(),
S3_ACCESS_KEY: z.string(),
S3_SECRET_KEY: z.string(),
S3_BUCKET: z.string(),
});

export const env = envSchema.parse(process.env);
7 changes: 7 additions & 0 deletions packages/api/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { initServer } from "@ts-rest/fastify";
import { addTranscodeJob, addPackageJob } from "@mixwave/artisan/producer";
import { getJobs, getJob, getJobLogs } from "./jobs.js";
import { openApiSpec } from "./openapi.js";
import { getStorage } from "./s3.js";

async function buildServer() {
const app = Fastify();
Expand Down Expand Up @@ -48,6 +49,12 @@ async function buildServer() {
body: await getJobLogs(params.id),
};
},
getStorage: async ({ query }) => {
return {
status: 200,
body: await getStorage(query.path, query.take, query.skip),
};
},
getSpec: async () => {
return {
status: 200,
Expand Down
4 changes: 2 additions & 2 deletions packages/api/src/jobs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ export async function getJobs(): Promise<JobDto[]> {
return result;
}

export async function getJob(id: string, fromRoot: boolean) {
export async function getJob(id: string, fromRoot?: boolean) {
const node = await getJobNode(id, fromRoot);
return await formatJobNode(node);
}
Expand All @@ -48,7 +48,7 @@ export async function getJobLogs(id: string) {
return logs;
}

async function getJobNode(id: string, fromRoot: boolean) {
async function getJobNode(id: string, fromRoot?: boolean) {
const [queue, jobId] = formatIdPair(id);

let job = await Job.fromId(queue, jobId);
Expand Down
7 changes: 7 additions & 0 deletions packages/api/src/openapi.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
import { generateOpenApi } from "@ts-rest/open-api";
import { contract } from "./contract.js";
import { env } from "./env.js";

export const openApiSpec = generateOpenApi(contract, {
info: {
title: "API",
version: "1.0.0",
},
servers: [
{
url: env.PUBLIC_API_ENDPOINT,
description: "Public",
},
],
});

delete openApiSpec.paths["/spec.json"];
57 changes: 57 additions & 0 deletions packages/api/src/s3.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import { S3, ListObjectsCommand } from "@aws-sdk/client-s3";
import { env } from "./env.js";
import type { FolderDto } from "./types.js";

const client = new S3({
endpoint: env.S3_ENDPOINT,
region: env.S3_REGION,
credentials: {
accessKeyId: env.S3_ACCESS_KEY,
secretAccessKey: env.S3_SECRET_KEY,
},
});

export async function getStorage(
path: string,
take: number = 10,
skip?: string,
): Promise<FolderDto> {
const response = await client.send(
new ListObjectsCommand({
Bucket: env.S3_BUCKET,
Delimiter: "/",
Prefix: path,
MaxKeys: take,
Marker: skip,
}),
);

const folder: FolderDto = {
path,
contents: [],
skip: response.IsTruncated ? response.NextMarker : undefined,
};

response.CommonPrefixes?.forEach((prefix) => {
if (!prefix.Prefix) {
return;
}
folder.contents.push({
type: "folder",
path: prefix.Prefix,
});
});

response.Contents?.forEach((content) => {
if (!content.Key) {
return;
}
folder.contents.push({
type: "file",
path: content.Key,
size: content.Size ?? 0,
});
});

return folder;
}
20 changes: 20 additions & 0 deletions packages/api/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,23 @@ export type JobDto = z.infer<typeof baseJobDtoSchema> & {
export const jobDtoSchema: z.ZodType<JobDto> = baseJobDtoSchema.extend({
children: z.lazy(() => jobDtoSchema.array()),
});

export const folderDtoSchema = z.object({
path: z.string(),
skip: z.string().optional(),
contents: z.array(
z.discriminatedUnion("type", [
z.object({
type: z.literal("file"),
path: z.string(),
size: z.number(),
}),
z.object({
type: z.literal("folder"),
path: z.string(),
}),
]),
),
});

export type FolderDto = z.infer<typeof folderDtoSchema>;
16 changes: 10 additions & 6 deletions packages/artisan/src/connection.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
let port = 6379;
if (process.env.REDIS_PORT) {
port = +process.env.REDIS_PORT;
}
import { z } from "zod";

const envSchema = z.object({
REDIS_HOST: z.string(),
REDIS_PORT: z.coerce.number(),
});

const env = envSchema.parse(process.env);

export const connection = {
host: process.env.REDIS_HOST ?? "redis",
port,
host: env.REDIS_HOST,
port: env.REDIS_PORT,
};
4 changes: 2 additions & 2 deletions packages/artisan/src/consumer/env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ const envSchema = z.object({
S3_ACCESS_KEY: z.string(),
S3_SECRET_KEY: z.string(),
S3_BUCKET: z.string(),
REDIS_HOST: z.string().default("redis"),
REDIS_PORT: z.coerce.number().default(6379),
REDIS_HOST: z.string(),
REDIS_PORT: z.coerce.number(),
});

export const env = envSchema.parse(process.env);
7 changes: 4 additions & 3 deletions packages/artisan/src/consumer/workers/package.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ export type PackageData = {
params: {
assetId: string;
segmentSize: number;
name: string;
};
metadata: {
tag?: string;
Expand Down Expand Up @@ -109,7 +110,7 @@ export default async function (job: Job<PackageData, PackageResult>) {

await once(packagerProcess, "close");

await uploadFolder(outDir.name, `package/${params.assetId}/hls`, {
await uploadFolder(outDir.name, `package/${params.assetId}/${params.name}`, {
del: true,
commandInput: (input) => ({
ContentType: lookup(input.Key) || "binary/octet-stream",
Expand All @@ -121,8 +122,8 @@ export default async function (job: Job<PackageData, PackageResult>) {
// becomes available on CDN.
// This way we ensure we have all the segments on S3 before we make the manifest available.
await copyFile(
`package/${params.assetId}/hls/master_tmp.m3u8`,
`package/${params.assetId}/hls/master.m3u8`,
`package/${params.assetId}/${params.name}/master_tmp.m3u8`,
`package/${params.assetId}/${params.name}/master.m3u8`,
"public-read",
);

Expand Down
27 changes: 17 additions & 10 deletions packages/artisan/src/producer/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,17 +33,17 @@ type AddTranscodeJobData = {
assetId?: string;
inputs: Input[];
streams: Stream[];
segmentSize: number;
packageAfter: boolean;
segmentSize?: number;
packageAfter?: boolean;
tag?: string;
};

export async function addTranscodeJob({
assetId = randomUUID(),
inputs,
streams,
segmentSize,
packageAfter,
segmentSize = 4,
packageAfter = false,
tag,
}: AddTranscodeJobData) {
const jobId = `transcode_${assetId}`;
Expand Down Expand Up @@ -130,24 +130,31 @@ export async function addTranscodeJob({

type AddPackageJobData = {
assetId: string;
segmentSize: number;
segmentSize?: number;
name?: string;
tag?: string;
};

export async function addPackageJob(data: AddPackageJobData) {
export async function addPackageJob({
assetId,
segmentSize = 4,
name = "hls",
tag,
}: AddPackageJobData) {
return await packageQueue.add(
"package",
{
params: {
assetId: data.assetId,
segmentSize: data.segmentSize,
assetId,
segmentSize,
name,
},
metadata: {
tag: data.tag,
tag,
},
} satisfies PackageData,
{
jobId: `package_${data.assetId}`,
jobId: `package_${randomUUID()}`,
},
);
}
Loading

0 comments on commit 12e63cf

Please sign in to comment.